├── .gitignore ├── LICENSE ├── README.md ├── bench.js ├── example.js ├── index.js └── package.json /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | package-lock.json 3 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2018 Mathias Buus 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # random-access-chrome-file 2 | 3 | A [random-access-storage](https://github.com/random-access-storage/random-access-storage) instance backed by the Chrome file system api 4 | 5 | ``` 6 | npm install random-access-chrome-file 7 | ``` 8 | 9 | ## Usage 10 | 11 | ``` js 12 | // Currently only works in Chrome 13 | 14 | const createFile = require('random-access-chrome-file') 15 | 16 | const file = createFile('test.txt') 17 | 18 | file.write(0, Buffer.from('hello world'), function (err) { 19 | if (err) throw err 20 | file.read(0, 11, function (err, buf) { 21 | if (err) throw err 22 | console.log(buf.toString()) 23 | }) 24 | }) 25 | ``` 26 | 27 | ## API 28 | 29 | #### `file = createFile(name, [options])` 30 | 31 | Returns a [random-access-storage](https://github.com/random-access-storage/random-access-storage) instance that supports 32 | the full API. 33 | 34 | Options include: 35 | 36 | ```js 37 | { 38 | maxSize: Number.MAX_SAFE_INTEGER 39 | } 40 | ``` 41 | 42 | `maxSize` is the storage quota it asks the browser for. If you are making an extension you can set the `unlimitedStorage` 43 | to get all the storage you want. Otherwise tweak the `maxSize` option to fit your needs. 44 | 45 | If you want to change the `maxSize` default for all instances change `createFile.DEFAULT_MAX_SIZE`. 46 | 47 | #### `createFile.requestQuota(maxSize, cb)` 48 | 49 | Manually request the `maxSize` quota without creating af file. 50 | 51 | ## License 52 | 53 | MIT 54 | -------------------------------------------------------------------------------- /bench.js: -------------------------------------------------------------------------------- 1 | const createFile = require('./') 2 | 3 | const st = createFile('benchmark.txt') 4 | st.open(tinyWrites) 5 | 6 | function tinyWrites () { 7 | let offset = 0 8 | const buf = Buffer.alloc(1) 9 | console.time('10000 tiny writes') 10 | st.write(0, buf, function onwrite (err) { 11 | if (err) throw err 12 | offset++ 13 | if (offset === 10000) { 14 | console.timeEnd('10000 tiny writes') 15 | return tinyReads() 16 | } 17 | st.write(offset, buf, onwrite) 18 | }) 19 | } 20 | 21 | function tinyReads () { 22 | let offset = 0 23 | console.time('10000 tiny reads') 24 | st.read(0, 1, function onread (err) { 25 | if (err) throw err 26 | offset++ 27 | if (offset === 10000) { 28 | console.timeEnd('10000 tiny reads') 29 | return benchWrite() 30 | } 31 | st.read(offset, 1, onread) 32 | }) 33 | } 34 | 35 | function benchRead () { 36 | let offset = 0 37 | console.time('512mb read') 38 | st.read(0, 65536, function onread (err, buf) { 39 | if (err) throw err 40 | if (offset >= 512 * 1024 * 1024) return console.timeEnd('512mb read') 41 | st.read(offset += buf.length, 65536, onread) 42 | }) 43 | } 44 | 45 | function benchWrite () { 46 | let offset = 0 47 | const buf = Buffer.alloc(65536).fill('hi') 48 | console.time('512mb write') 49 | st.write(offset, buf, function onwrite (err) { 50 | if (err) throw err 51 | if (offset >= 512 * 1024 * 1024) { 52 | console.timeEnd('512mb write') 53 | benchRead() 54 | return 55 | } 56 | st.write(offset += buf.length, buf, onwrite) 57 | }) 58 | } 59 | -------------------------------------------------------------------------------- /example.js: -------------------------------------------------------------------------------- 1 | const createFile = require('./') 2 | 3 | const st = createFile('/some/folder/hello-world.txt') 4 | let missing = 2 5 | 6 | st.write(0, Buffer.from('hello '), done) 7 | st.write(6, Buffer.from('world'), done) 8 | 9 | function done (err) { 10 | if (err) throw err 11 | if (!--missing) st.read(0, 11, (_, buf) => console.log(buf.toString())) 12 | } 13 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | const ras = require('random-access-storage') 2 | 3 | const TYPE = { type: 'octet/stream' } 4 | const requestFileSystem = window.requestFileSystem || window.webkitRequestFileSystem 5 | const persistentStorage = navigator.persistentStorage || navigator.webkitPersistentStorage 6 | const FileReader = window.FileReader 7 | const Blob = window.Blob 8 | 9 | createFile.DEFAULT_MAX_SIZE = Number.MAX_SAFE_INTEGER 10 | createFile.requestQuota = requestQuota 11 | 12 | module.exports = createFile 13 | 14 | function requestQuota (n, force, cb) { 15 | if (typeof force === 'function') return requestQuota(n, true, force) 16 | persistentStorage.queryUsageAndQuota(function (used, quota) { 17 | if (quota && !force) return cb(null, quota) 18 | persistentStorage.requestQuota(n, function (quota) { 19 | cb(null, quota) 20 | }, cb) 21 | }, cb) 22 | } 23 | 24 | function createFile (name, opts) { 25 | if (!opts) opts = {} 26 | 27 | const maxSize = opts.maxSize || createFile.DEFAULT_MAX_SIZE 28 | const mutex = new Mutex() 29 | 30 | let fs = null 31 | let entry = null 32 | let toDestroy = null 33 | let readers = [] 34 | let writers = [] 35 | let deleters = [] 36 | 37 | return ras({ read, write, del, open, stat, close, destroy }) 38 | 39 | function read (req) { 40 | const r = readers.pop() || new ReadRequest(readers, entry, mutex) 41 | r.run(req) 42 | } 43 | 44 | function write (req) { 45 | const w = writers.pop() || new WriteRequest(writers, entry, mutex) 46 | w.run(req) 47 | } 48 | 49 | function del (req) { 50 | const d = deleters.pop() || new DeleteRequest(deleters, entry, mutex) 51 | d.run(req) 52 | } 53 | 54 | function close (req) { 55 | readers = writers = deleters = entry = fs = null 56 | req.callback(null) 57 | } 58 | 59 | function stat (req) { 60 | entry.file(file => { 61 | req.callback(null, file) 62 | }, err => req.callback(err)) 63 | } 64 | 65 | function destroy (req) { 66 | toDestroy.remove(ondone, onerror) 67 | 68 | function ondone () { 69 | toDestroy = null 70 | req.callback(null, null) 71 | } 72 | 73 | function onerror (err) { 74 | toDestroy = null 75 | req.callback(err, null) 76 | } 77 | } 78 | 79 | function open (req) { 80 | requestQuota(maxSize, false, function (err, granted) { 81 | if (err) return onerror(err) 82 | requestFileSystem(window.PERSISTENT, granted, function (res) { 83 | fs = res 84 | mkdirp(parentFolder(name), function () { 85 | fs.root.getFile(name, { create: true }, function (e) { 86 | entry = toDestroy = e 87 | req.callback(null) 88 | }, onerror) 89 | }) 90 | }, onerror) 91 | }) 92 | 93 | function mkdirp (name, ondone) { 94 | if (!name) return ondone() 95 | fs.root.getDirectory(name, { create: true }, ondone, function () { 96 | mkdirp(parentFolder(name), function () { 97 | fs.root.getDirectory(name, { create: true }, ondone, ondone) 98 | }) 99 | }) 100 | } 101 | 102 | function onerror (err) { 103 | fs = entry = null 104 | req.callback(err) 105 | } 106 | } 107 | } 108 | 109 | function parentFolder (path) { 110 | const i = path.lastIndexOf('/') 111 | const j = path.lastIndexOf('\\') 112 | const p = path.slice(0, Math.max(0, i, j)) 113 | return /^\w:$/.test(p) ? '' : p 114 | } 115 | 116 | function WriteRequest (pool, entry, mutex) { 117 | this.pool = pool 118 | this.entry = entry 119 | this.mutex = mutex 120 | this.writer = null 121 | this.req = null 122 | this.locked = false 123 | this.truncating = false 124 | } 125 | 126 | WriteRequest.prototype.makeWriter = function () { 127 | const self = this 128 | this.entry.createWriter(function (writer) { 129 | self.writer = writer 130 | 131 | writer.onwriteend = function () { 132 | self.onwrite(null) 133 | } 134 | 135 | writer.onerror = function (err) { 136 | self.onwrite(err) 137 | } 138 | 139 | self.run(self.req) 140 | }) 141 | } 142 | 143 | WriteRequest.prototype.onwrite = function (err) { 144 | const req = this.req 145 | this.req = null 146 | 147 | if (this.locked) { 148 | this.locked = false 149 | this.mutex.release() 150 | } 151 | 152 | if (this.truncating) { 153 | this.truncating = false 154 | if (!err) return this.run(req) 155 | } 156 | 157 | this.pool.push(this) 158 | req.callback(err, null) 159 | } 160 | 161 | WriteRequest.prototype.truncate = function () { 162 | this.truncating = true 163 | this.writer.truncate(this.req.offset) 164 | } 165 | 166 | WriteRequest.prototype.lock = function () { 167 | if (this.locked) return true 168 | this.locked = this.mutex.lock(this) 169 | return this.locked 170 | } 171 | 172 | WriteRequest.prototype.run = function (req) { 173 | this.entry.file(file => { 174 | this.req = req 175 | 176 | if (!this.writer || this.writer.length !== file.size) return this.makeWriter() 177 | 178 | if (req.offset + req.size > file.size && !this.lock()) return 179 | 180 | if (req.offset > this.writer.length) { 181 | if (req.offset > file.size) return this.truncate() 182 | return this.makeWriter() 183 | } 184 | 185 | this.writer.seek(req.offset) 186 | this.writer.write(new Blob([req.data], TYPE)) 187 | }, err => req.callback(err)) 188 | } 189 | 190 | function Mutex () { 191 | this.queued = null 192 | } 193 | 194 | Mutex.prototype.release = function () { 195 | const queued = this.queued 196 | this.queued = null 197 | for (let i = 0; i < queued.length; i++) { 198 | queued[i].run(queued[i].req) 199 | } 200 | } 201 | 202 | Mutex.prototype.lock = function (req) { 203 | if (this.queued) { 204 | this.queued.push(req) 205 | return false 206 | } 207 | this.queued = [] 208 | return true 209 | } 210 | 211 | function ReadRequest (pool, entry, mutex) { 212 | this.pool = pool 213 | this.entry = entry 214 | this.mutex = mutex 215 | this.reader = new FileReader() 216 | this.req = null 217 | this.retry = true 218 | this.locked = false 219 | 220 | const self = this 221 | 222 | this.reader.onerror = function () { 223 | self.onread(this.error, null) 224 | } 225 | 226 | this.reader.onload = function () { 227 | const buf = Buffer.from(this.result) 228 | self.onread(null, buf) 229 | } 230 | } 231 | 232 | ReadRequest.prototype.lock = function () { 233 | if (this.locked) return true 234 | this.locked = this.mutex.lock(this) 235 | return this.locked 236 | } 237 | 238 | ReadRequest.prototype.onread = function (err, buf) { 239 | const req = this.req 240 | 241 | if (err && this.retry) { 242 | this.retry = false 243 | if (this.lock(this)) this.run(req) 244 | return 245 | } 246 | 247 | this.req = null 248 | this.pool.push(this) 249 | this.retry = true 250 | 251 | if (this.locked) { 252 | this.locked = false 253 | this.mutex.release() 254 | } 255 | 256 | req.callback(err, buf) 257 | } 258 | 259 | ReadRequest.prototype.run = function (req) { 260 | this.entry.file(file => { 261 | const end = req.offset + req.size 262 | this.req = req 263 | if (end > file.size) return this.onread(new Error('Could not satisfy length'), null) 264 | this.reader.readAsArrayBuffer(file.slice(req.offset, end)) 265 | }, err => req.callback(err)) 266 | } 267 | 268 | function DeleteRequest (pool, entry, mutex) { 269 | this.pool = pool 270 | this.entry = entry 271 | this.mutex = mutex 272 | this.writer = null 273 | this.req = null 274 | this.locked = false 275 | } 276 | 277 | DeleteRequest.prototype.makeWriter = function () { 278 | const self = this 279 | this.entry.createWriter(function (writer) { 280 | self.writer = writer 281 | 282 | writer.onwriteend = function () { 283 | self.onwrite(null) 284 | } 285 | 286 | writer.onerror = function (err) { 287 | self.onwrite(err) 288 | } 289 | 290 | self.run(self.req) 291 | }) 292 | } 293 | 294 | DeleteRequest.prototype.onwrite = function (err) { 295 | const req = this.req 296 | this.req = null 297 | 298 | if (this.locked) { 299 | this.locked = false 300 | this.mutex.release() 301 | } 302 | 303 | this.pool.push(this) 304 | req.callback(err, null) 305 | } 306 | 307 | DeleteRequest.prototype.lock = function () { 308 | if (this.locked) return true 309 | this.locked = this.mutex.lock(this) 310 | return this.locked 311 | } 312 | 313 | DeleteRequest.prototype.run = function (req) { 314 | this.entry.file(file => { 315 | this.req = req 316 | 317 | if (req.offset + req.size < file.size) return req.callback(null) 318 | 319 | if (!this.writer) return this.makeWriter() 320 | if (!this.lock()) return 321 | 322 | this.writer.truncate(req.offset) 323 | }, err => req.callback(err)) 324 | } 325 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "random-access-chrome-file", 3 | "version": "1.2.0", 4 | "description": "random-access-storage instance backed by the Chrome file system api.", 5 | "main": "index.js", 6 | "dependencies": { 7 | "random-access-storage": "^1.3.0" 8 | }, 9 | "devDependencies": { 10 | "standard": "^16.0.4" 11 | }, 12 | "scripts": { 13 | "test": "standard" 14 | }, 15 | "repository": { 16 | "type": "git", 17 | "url": "https://github.com/random-access-storage/random-access-chrome-file.git" 18 | }, 19 | "author": "Mathias Buus (@mafintosh)", 20 | "license": "MIT", 21 | "bugs": { 22 | "url": "https://github.com/random-access-storage/random-access-chrome-file/issues" 23 | }, 24 | "homepage": "https://github.com/random-access-storage/random-access-chrome-file" 25 | } 26 | --------------------------------------------------------------------------------