├── .gitignore ├── package.json ├── node.js ├── .github └── workflows │ └── mikeals-workflow.yml ├── browser.js ├── core.js ├── test └── basics.spec.js └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | .nyc_output 2 | coverage 3 | package-lock.json 4 | node_modules 5 | .DS_Store 6 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bytesish", 3 | "version": "0.0.0-dev", 4 | "main": "node.js", 5 | "browser": "browser.js", 6 | "scripts": { 7 | "test": "hundreds aegir test -t node browser", 8 | "pretest": "aegir lint" 9 | }, 10 | "keywords": [], 11 | "author": "Mikeal Rogers (https://www.mikealrogers.com/)", 12 | "license": "(Apache-2.0 AND MIT)", 13 | "devDependencies": { 14 | "aegir": "^20.4.1", 15 | "hundreds": "0.0.2", 16 | "tsame": "^2.0.1" 17 | }, 18 | "repository": { 19 | "type": "git", 20 | "url": "git+https://github.com/mikeal/bytesish.git" 21 | }, 22 | "bugs": { 23 | "url": "https://github.com/mikeal/bytesish/issues" 24 | }, 25 | "homepage": "https://github.com/mikeal/bytesish#readme", 26 | "description": "Cross-Platform Binary API" 27 | } 28 | -------------------------------------------------------------------------------- /node.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | const crypto = require('crypto') 3 | const fallback = require('./browser').from 4 | const bytes = require('./core') 5 | 6 | bytes.from = (_from, encoding) => { 7 | if (_from instanceof DataView) return _from 8 | if (_from instanceof ArrayBuffer) return new DataView(_from) 9 | if (typeof _from === 'string') { 10 | _from = Buffer.from(_from, encoding) 11 | } 12 | if (Buffer.isBuffer(_from)) { 13 | return new DataView(_from.buffer, _from.byteOffset, _from.byteLength) 14 | } 15 | return fallback(_from, encoding) 16 | } 17 | bytes.toString = (_from, encoding) => { 18 | _from = bytes(_from) 19 | return Buffer.from(_from.buffer, _from.byteOffset, _from.byteLength).toString(encoding) 20 | } 21 | 22 | bytes.native = (_from, encoding) => { 23 | if (Buffer.isBuffer(_from)) return _from 24 | _from = bytes(_from, encoding) 25 | return Buffer.from(_from.buffer, _from.byteOffset, _from.byteLength) 26 | } 27 | 28 | bytes._randomFill = crypto.randomFillSync 29 | 30 | module.exports = bytes 31 | -------------------------------------------------------------------------------- /.github/workflows/mikeals-workflow.yml: -------------------------------------------------------------------------------- 1 | on: [push, pull_request] 2 | name: Build, Test and maybe Publish 3 | jobs: 4 | test: 5 | name: Build & Test 6 | runs-on: ubuntu-latest 7 | strategy: 8 | matrix: 9 | node-version: [12.x, 14.x] 10 | steps: 11 | - uses: actions/checkout@v2 12 | - name: Use Node.js ${{ matrix.node-version }} 13 | uses: actions/setup-node@v1 14 | with: 15 | node-version: ${{ matrix.node-version }} 16 | - name: Cache node_modules 17 | id: cache-modules 18 | uses: actions/cache@v1 19 | with: 20 | path: node_modules 21 | key: ${{ matrix.node-version }}-${{ runner.OS }}-build-${{ hashFiles('package.json') }} 22 | - name: Build 23 | if: steps.cache-modules.outputs.cache-hit != 'true' 24 | run: npm install 25 | - name: Test 26 | run: npm_config_yes=true npx best-test@latest 27 | publish: 28 | name: Publish 29 | needs: test 30 | runs-on: ubuntu-latest 31 | if: github.event_name == 'push' && ( github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' ) 32 | steps: 33 | - uses: actions/checkout@v2 34 | - name: Cache node_modules 35 | id: cache-modules 36 | uses: actions/cache@v1 37 | with: 38 | path: node_modules 39 | key: 12.x-${{ runner.OS }}-build-${{ hashFiles('package.json') }} 40 | - name: Build 41 | if: steps.cache-modules.outputs.cache-hit != 'true' 42 | run: npm install 43 | - name: Test 44 | run: npm_config_yes=true npx best-test@latest 45 | 46 | - name: Publish 47 | uses: mikeal/merge-release@master 48 | env: 49 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 50 | NPM_AUTH_TOKEN: ${{ secrets.NPM_AUTH_TOKEN }} 51 | -------------------------------------------------------------------------------- /browser.js: -------------------------------------------------------------------------------- 1 | /* globals atob, btoa, crypto */ 2 | /* istanbul ignore file */ 3 | 'use strict' 4 | const bytes = require('./core') 5 | 6 | bytes.from = (_from, _encoding) => { 7 | if (_from instanceof DataView) return _from 8 | if (_from instanceof ArrayBuffer) return new DataView(_from) 9 | let buffer 10 | if (typeof _from === 'string') { 11 | if (!_encoding) { 12 | _encoding = 'utf-8' 13 | } else if (_encoding === 'base64') { 14 | buffer = Uint8Array.from(atob(_from), c => c.charCodeAt(0)).buffer 15 | return new DataView(buffer) 16 | } 17 | if (_encoding !== 'utf-8') throw new Error('Browser support for encodings other than utf-8 not implemented') 18 | return new DataView((new TextEncoder()).encode(_from).buffer) 19 | } else if (typeof _from === 'object') { 20 | if (ArrayBuffer.isView(_from)) { 21 | if (_from.byteLength === _from.buffer.byteLength) return new DataView(_from.buffer) 22 | else return new DataView(_from.buffer, _from.byteOffset, _from.byteLength) 23 | } 24 | } 25 | throw new Error('Unkown type. Cannot convert to ArrayBuffer') 26 | } 27 | 28 | bytes.toString = (_from, encoding) => { 29 | _from = bytes(_from, encoding) 30 | const uint = new Uint8Array(_from.buffer, _from.byteOffset, _from.byteLength) 31 | const str = String.fromCharCode(...uint) 32 | if (encoding === 'base64') { 33 | /* would be nice to find a way to do this directly from a buffer 34 | * instead of doing two string conversions 35 | */ 36 | return btoa(str) 37 | } else { 38 | return str 39 | } 40 | } 41 | 42 | bytes.native = (_from, encoding) => { 43 | if (_from instanceof Uint8Array) return _from 44 | _from = bytes.from(_from, encoding) 45 | return new Uint8Array(_from.buffer, _from.byteOffset, _from.byteLength) 46 | } 47 | 48 | if (process.browser) bytes._randomFill = (...args) => crypto.getRandomValues(...args) 49 | 50 | module.exports = bytes 51 | -------------------------------------------------------------------------------- /core.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const length = (a, b) => { 4 | if (a.byteLength === b.byteLength) return a.byteLength 5 | else if (a.byteLength > b.byteLength) return a.byteLength 6 | return b.byteLength 7 | } 8 | 9 | const bytes = (_from, encoding) => bytes.from(_from, encoding) 10 | 11 | bytes.sorter = (a, b) => { 12 | a = bytes(a) 13 | b = bytes(b) 14 | const len = length(a, b) 15 | let i = 0 16 | while (i < (len - 1)) { 17 | if (i >= a.byteLength) return 1 18 | else if (i >= b.byteLength) return -1 19 | 20 | if (a.getUint8(i) < b.getUint8(i)) return -1 21 | else if (a.getUint8(i) > b.getUint8(i)) return 1 22 | i++ 23 | } 24 | return 0 25 | } 26 | 27 | bytes.compare = (a, b) => !bytes.sorter(a, b) 28 | bytes.memcopy = (_from, encoding) => { 29 | const b = bytes(_from, encoding) 30 | return b.buffer.slice(b.byteOffset, b.byteOffset + b.byteLength) 31 | } 32 | bytes.arrayBuffer = (_from, encoding) => { 33 | _from = bytes(_from, encoding) 34 | if (_from.buffer.byteLength === _from.byteLength) return _from.buffer 35 | return _from.buffer.slice(_from.byteOffset, _from.byteOffset + _from.byteLength) 36 | } 37 | const sliceOptions = (_from, start = 0, end = null) => { 38 | _from = bytes(_from) 39 | end = (end === null ? _from.byteLength : end) - start 40 | return [_from.buffer, _from.byteOffset + start, end] 41 | } 42 | bytes.slice = (_from, start, end) => new DataView(...sliceOptions(_from, start, end)) 43 | 44 | bytes.memcopySlice = (_from, start, end) => { 45 | const [buffer, offset, length] = sliceOptions(_from, start, end) 46 | return buffer.slice(offset, length + offset) 47 | } 48 | bytes.typedArray = (_from, _Class = Uint8Array) => { 49 | _from = bytes(_from) 50 | return new _Class(_from.buffer, _from.byteOffset, _from.byteLength / _Class.BYTES_PER_ELEMENT) 51 | } 52 | 53 | bytes.concat = (_from) => { 54 | _from = Array.from(_from) 55 | _from = _from.map(b => bytes(b)) 56 | const length = _from.reduce((x, y) => x + y.byteLength, 0) 57 | const ret = new Uint8Array(length) 58 | let i = 0 59 | for (const part of _from) { 60 | const view = bytes.typedArray(part) 61 | ret.set(view, i) 62 | i += view.byteLength 63 | } 64 | return ret.buffer 65 | } 66 | 67 | const maxEntropy = 65536 68 | 69 | bytes.random = length => { 70 | const ab = new ArrayBuffer(length) 71 | if (length > maxEntropy) { 72 | let i = 0 73 | while (i < ab.byteLength) { 74 | let len 75 | if (i + maxEntropy > ab.byteLength) len = ab.byteLength - i 76 | else len = maxEntropy 77 | const view = new Uint8Array(ab, i, len) 78 | i += maxEntropy 79 | bytes._randomFill(view) 80 | } 81 | } else { 82 | const view = new Uint8Array(ab) 83 | bytes._randomFill(view) 84 | } 85 | return ab 86 | } 87 | 88 | module.exports = bytes 89 | -------------------------------------------------------------------------------- /test/basics.spec.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | const assert = require('assert') 3 | const tsame = require('tsame') 4 | const { it } = require('mocha') 5 | const bytes = require('../') 6 | 7 | const test = it 8 | 9 | const same = (x, y) => assert.ok(tsame(x, y)) 10 | 11 | test('string conversion', done => { 12 | const ab = bytes('hello world') 13 | assert(ab instanceof DataView) 14 | const str = bytes.toString(ab) 15 | same(str, 'hello world') 16 | done() 17 | }) 18 | 19 | test('compare', done => { 20 | const a = bytes('hello world') 21 | const b = bytes('hello world 2') 22 | assert(bytes.compare(a, a)) 23 | assert(bytes.compare(a, 'hello world')) 24 | assert(!bytes.compare(a, b)) 25 | assert(!bytes.compare(b, a)) 26 | assert(!bytes.compare(a, '123')) 27 | assert(!bytes.compare('123', a)) 28 | done() 29 | }) 30 | 31 | test('double view', done => { 32 | const a = bytes('hello world') 33 | const b = bytes(a) 34 | same(a, b) 35 | done() 36 | }) 37 | 38 | test('typed array', done => { 39 | const ab = bytes.random(4) 40 | const view = bytes(ab) 41 | const uint = bytes.typedArray(ab, Uint32Array) 42 | assert(uint instanceof Uint32Array) 43 | assert(bytes.compare(uint, view)) 44 | done() 45 | }) 46 | 47 | test('from array buffer', done => { 48 | const a = bytes('hello world') 49 | const b = bytes(bytes.memcopy(a)) 50 | same(bytes.toString(a), bytes.toString(b)) 51 | done() 52 | }) 53 | 54 | test('to array buffer', done => { 55 | const a = bytes.arrayBuffer('hello world') 56 | const b = bytes('hello world') 57 | assert(a instanceof ArrayBuffer) 58 | assert(bytes.compare(a, b)) 59 | assert(a, bytes.arrayBuffer(a)) 60 | done() 61 | }) 62 | 63 | test('Uint8Array', done => { 64 | const a = bytes('hello world') 65 | const b = bytes(new Uint8Array(bytes.memcopy(a))) 66 | same(bytes.toString(a), bytes.toString(b)) 67 | done() 68 | }) 69 | 70 | test('native', done => { 71 | let n = bytes.native('hello world') 72 | if (process.browser) { 73 | assert(n instanceof Uint8Array) 74 | n = bytes.native(n) 75 | assert(n instanceof Uint8Array) 76 | } else { 77 | assert(n instanceof Buffer) 78 | n = bytes.native(n) 79 | assert(n instanceof Buffer) 80 | } 81 | done() 82 | }) 83 | 84 | test('slice', done => { 85 | const a = bytes.slice('hello world', 2, 7) 86 | assert(a instanceof DataView) 87 | const b = bytes.arrayBuffer('hello world').slice(2, 7) 88 | same(b, bytes.arrayBuffer(a)) 89 | done() 90 | }) 91 | 92 | test('slice memcopy', done => { 93 | const a = bytes.memcopySlice('hello world', 2, 7) 94 | assert(a instanceof ArrayBuffer) 95 | const b = bytes.arrayBuffer('hello world').slice(2, 7) 96 | same(b, a) 97 | const c = bytes.memcopySlice(a) 98 | assert(a !== c) 99 | same(a, c) 100 | done() 101 | }) 102 | 103 | test('concat', done => { 104 | let values = [bytes('1'), bytes.native('2'), bytes.arrayBuffer('3')] 105 | let ab = bytes.concat(values) 106 | assert(ab instanceof ArrayBuffer) 107 | assert(bytes.compare(ab, '123')) 108 | values = [bytes('one')] 109 | ab = bytes.concat(values) 110 | assert(bytes.compare(ab, 'one')) 111 | done() 112 | }) 113 | 114 | test('random above max entropy', done => { 115 | const maxEntropy = 65536 116 | const size = (maxEntropy * 3) + 8 117 | const rand = bytes.random(size) 118 | same(rand.byteLength, size) 119 | done() 120 | }) 121 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # `bytesish` 2 | 3 | ![5002](https://img.shields.io/badge/compiled%20bundle-5k-green) ![1903](https://img.shields.io/badge/gzipped%20bundle-2k-brightgreen) 4 | 5 | If you're writing a library that needs to work in Node.js and in Browsers, 6 | it's quite difficult to figure out what "the right thing" to do with binary 7 | is. 8 | 9 | If you want to be compatible with Node.js libraries you'll need to accept 10 | and return `Buffer` instances. If you want to be compatible with Browser API's 11 | you'll need to accept and return a number of types, the browser is sort of a mess 12 | when it comes to binary with many different "views" of binary data. 13 | 14 | The moment you use the Node.js `Buffer` API in a library that is bundled for 15 | use in Browsers the bundler will inject a rather large polyfill for the entire 16 | `Buffer` API. It's quite difficult to accept and return `Buffer` instances while 17 | avoiding this penalty. 18 | 19 | However, there is some good news. No matter what the binary type there's an underlying 20 | `ArrayBuffer` associated with the instance. There's also one generic binary view object 21 | available in both Node.js and Browsers called `DataView`. This means that you can take 22 | any binary type and do a **zero memcopy** conversion to a `DataView`. 23 | 24 | But there are some problems with `DataView`. Not all APIs take it in browsers and almost 25 | none accept it in Node.js. It's a great API for reading and writing to an `ArrayBuffer` 26 | but it lacks a lot of other functionality that can be difficult to accomplish cross-platform. 27 | 28 | `bytesish` is here to help. This library helps you accept and convert different binary types 29 | into a consistent type, `DataView`, without loading any polyfills or other dependencies, then 30 | convert back into an ideal type for the platform your library is running in. 31 | 32 | What `bytesish` does: 33 | 34 | * Returns a `DataView` from any known binary type (zero copy). 35 | * Creates a `DataView` from a string with any encoding. 36 | * Converts any type to a string of any encoding. 37 | * Converts any to an ideal native object (`Buffer` or `Uint8Array`). 38 | * Provides utility functions for comparison, sorting, copying and slices 39 | any binary type or string. 40 | 41 | `bytesish` does not create a new Binary Type for accessing and manipulating 42 | binary data, because you can just use `DataView` for that. `bytesish` tries to be a 43 | small piece of code that does not contribute any more than necessary to your bundle size. 44 | It does this by containing only the binary operations you need that are difficult to 45 | do cross-platform (Node.js and Browsers). 46 | 47 | ```javascript 48 | let bytes = require('bytesish') 49 | let view = bytes('hello world') 50 | 51 | /* zero copy conversions */ 52 | view = bytes(Buffer.from('hello world')) // Buffer instance 53 | view = bytes((new TextEncoder()).encode('hello world')) // Uint8Array 54 | 55 | /* base64 conversions */ 56 | let base64String = bytes.toString(view, 'base64') 57 | base64String = bytes.toString(Buffer.from('hello world'), 'base64') 58 | base64String = bytes.toString('hello world', 'base64') 59 | 60 | /* since this is a string conversion it will create a new binary instance */ 61 | let viewCopy = bytes(base64String, 'base64') 62 | ``` 63 | 64 | # API 65 | 66 | ## Zero Copy 67 | 68 | ### `bytes(from)` 69 | 70 | ### `bytes.sort(a, b)` 71 | 72 | ### `bytes.compare(a, b)` 73 | 74 | ### `bytes.native(from[, encoding])` 75 | 76 | ### `bytes.slice(from[, start=0[, end=from.byteLength]])` 77 | 78 | ### `bytes.typedArray(from[, Class=Uint8Array])` 79 | 80 | ## Optimized (memcopy only when necessary) 81 | 82 | ### `bytes.arrayBuffer(from[, encoding])` 83 | 84 | ## Memory Copy 85 | 86 | All memcopy APIs return an `ArrayBuffer` 87 | 88 | ### `bytes.memcopy(from[, encoding])` 89 | 90 | Returns an `ArrayBuffer` copy of the given binary or string. 91 | 92 | ### `bytes.memcopySlice(from[, start=0[, end=from.byteLength]])` 93 | 94 | Returns an `ArrayBuffer` copy from a slize of the given binary or string. 95 | 96 | ### `bytes.concat(values)` 97 | 98 | `values` is an iterable of binary or string types. 99 | 100 | Returns a newly allocated `ArrayBuffer` contained the concatenated binary data. 101 | 102 | ## String Conversions 103 | 104 | ### `bytes(from[, encoding])` 105 | 106 | ### `bytes.toString(from[, outputEncoding])` 107 | --------------------------------------------------------------------------------