├── .gitignore ├── README.md ├── bin └── mm-packer.js ├── files ├── original │ ├── knight.jpg │ └── svg │ │ └── pipe.svg └── packed │ ├── pack.json │ └── pack.pack ├── index.js ├── package.json └── unpacker ├── Unpacker.js └── index.html /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | 3 | *.log 4 | 5 | node_modules -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # MM Packer 2 | 3 | [![NPM](https://nodei.co/npm/mm-packer.png)](https://www.npmjs.com/package/mm-packer) 4 | 5 | ## Goal 6 | 7 | Minimize HTTP requests. 8 | 9 | ## How it works 10 | 11 | Inspired by [Magipack.js](https://github.com/keitakun/Magipack.js/tree/master), it concatenates any kind of files into a "pack" file coming with JSON file that specifies what files are packed, where they are in the pack bitwise and what type they are. 12 | 13 | ## Requirements 14 | 15 | * node.js 16 | 17 | ## Dependencies 18 | 19 | * [cli](https://www.npmjs.com/package/cli) 20 | * [mime-types](https://www.npmjs.com/package/mime-types) 21 | 22 | ## Installation 23 | `npm install mm-packer` 24 | 25 | ## Usage 26 | 27 | ### CLI 28 | 29 | ```sh 30 | $ mm-packer -s files/original -o files/packed 31 | ``` 32 | 33 | #### Options 34 | 35 | ``` 36 | -s, --source FILE Source directory 37 | -o, --output FILE Output directory 38 | -n, --name [STRING] Pack files name (Default is pack) 39 | -k, --no-color Omit color from output 40 | --debug Show debug information 41 | -v, --version Display the current version 42 | -h, --help Display help and usage details 43 | ``` 44 | 45 | ### API 46 | 47 | ``` 48 | const packer = require("mm-packer"); 49 | const packerOptions = { 50 | source: "files/original", 51 | output: "files/packed", 52 | name: "pack" // Optional, 53 | debug: true // Optional 54 | }; 55 | packer(packerOptions); 56 | ``` 57 | 58 | ## Example 59 | 60 | If you've just downloaded the repository, run : 61 | 62 | ```sh 63 | $ ./bin/mm-packer.js -s files/original -o files/packed 64 | ``` 65 | 66 | You should see the generated files in [`files/packed`](https://github.com/MM56/mm-packer/tree/master/files/packed). 67 | 68 | You can check the pack can be correctly loaded and parsed with the demo in the folder [`unpacker`](https://github.com/MM56/mm-packer/tree/master/unpacker) that uses [`mm-unpacker`](https://www.npmjs.com/package/mm-unpacker). -------------------------------------------------------------------------------- /bin/mm-packer.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | const cli = require('cli'); 4 | const pkg = require('../package.json'); 5 | const packer = require("../index.js"); 6 | 7 | cli.setApp(pkg.name, pkg.version); 8 | cli.enable("version", "status"); 9 | cli.setUsage("mm-packer -s files/original -o files/packed"); 10 | 11 | cli.parse({ 12 | source: ["s", "Source directory", "file"], 13 | output: ["o", "Output directory", "file"], 14 | name: ["n", "Pack files name", "string", "pack"], 15 | }); 16 | 17 | cli.main((args, options) => { 18 | if(options.source && options.output && options.name) { 19 | options.debug = process.argv.indexOf("--debug") > -1; 20 | packer(options); 21 | } else { 22 | cli.getUsage(); 23 | } 24 | }); 25 | -------------------------------------------------------------------------------- /files/original/knight.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MM56/mm-packer/892431b3e537dff8d358338b2e0c6ed2c3cce36e/files/original/knight.jpg -------------------------------------------------------------------------------- /files/original/svg/pipe.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 8 | 18 | 19 | -------------------------------------------------------------------------------- /files/packed/pack.json: -------------------------------------------------------------------------------- 1 | [["knight.jpg",0,53742,"image/jpeg"],["svg/pipe.svg",53742,55372,"text/plain"]] -------------------------------------------------------------------------------- /files/packed/pack.pack: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MM56/mm-packer/892431b3e537dff8d358338b2e0c6ed2c3cce36e/files/packed/pack.pack -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | const cli = require('cli'); 2 | const fs = require('fs'); 3 | const mime = require('mime-types'); 4 | const path = require('path'); 5 | 6 | let debug = false; 7 | 8 | function packer(options) { 9 | if(!options.source) { 10 | cli.error("No source parameter specified"); 11 | return; 12 | } 13 | if(!fs.existsSync(options.source)) { 14 | cli.error("Source directory \"" + options.source + "\" doesn't exist"); 15 | return; 16 | } 17 | if(!options.output) { 18 | cli.error("No output parameter specified"); 19 | return; 20 | } 21 | if(!fs.existsSync(options.output)) { 22 | cli.error("Output directory \"" + options.output + "\" doesn't exist"); 23 | return; 24 | } 25 | 26 | const source = addTrailingSlash(options.source); 27 | const output = addTrailingSlash(options.output); 28 | const name = options.name || "pack"; 29 | debug = options.debug || false; 30 | options.mimeTypes = options.mimeTypes || []; 31 | printDebug("Source: " + source); 32 | printDebug("Output: " + output); 33 | printDebug("Name: " + name); 34 | 35 | const files = listFiles(source); 36 | // printDebug(files); 37 | 38 | pack(files, source, output, name, options.mimeTypes); 39 | } 40 | 41 | function addTrailingSlash(str) { 42 | if(str.substr(-1) == "/") { 43 | return str; 44 | } 45 | return str + "/"; 46 | } 47 | 48 | function listFiles(dir, fileList = []) { 49 | const files = fs.readdirSync(dir); 50 | for(let i = 0, l = files.length; i < l; i++) { 51 | const file = files[i]; 52 | const currentFile = dir + file; 53 | if(fs.statSync(currentFile).isDirectory()) { 54 | listFiles(currentFile + "/", fileList); 55 | } else if(file.substr(0, 1) != ".") { 56 | fileList.push(currentFile); 57 | } 58 | } 59 | return fileList; 60 | } 61 | 62 | function pack(files, source, output, name, mimeTypes) { 63 | printDebug("Packing:"); 64 | const buffers = []; 65 | const datas = []; 66 | let p = 0; 67 | for(let i = 0, l = files.length; i < l; i++) { 68 | const file = files[i]; 69 | printDebug(file); 70 | 71 | const mimetype = resolveMimetype(file, mimeTypes); 72 | printDebug("- Resolved mime-type: " + mimetype); 73 | 74 | const size = fs.statSync(file)["size"]; 75 | printDebug("- Size: " + size); 76 | 77 | const fileContent = fs.readFileSync(file); 78 | buffers.push(fileContent); 79 | 80 | datas.push([file.replace(source, ""), p, p + size, mimetype]); 81 | 82 | p += size; 83 | } 84 | fs.writeFileSync(output + name + ".pack", Buffer.concat(buffers)); 85 | fs.writeFileSync(output + name + ".json", JSON.stringify(datas)); 86 | } 87 | 88 | function resolveMimetype(file, mimeTypes) { 89 | let mimetype = mime.lookup(file); 90 | if(mimetype) { 91 | printDebug("- Detected mime-type: " + mimetype); 92 | mimetype = validateMimetype(mimetype, mimeTypes); 93 | } else { 94 | printDebug("- Detected mime-type: None"); 95 | const ext = path.extname(file); 96 | switch(ext) { 97 | case ".txt": 98 | case ".obj": 99 | mimetype = "text/plain"; 100 | break; 101 | case ".css": 102 | mimetype = "text/css"; 103 | break; 104 | case ".twig": 105 | mimetype = "text/twig"; 106 | break; 107 | case ".json": 108 | mimetype = "application/json"; 109 | break; 110 | case ".dds": 111 | case ".pvr": 112 | case ".glb": 113 | mimetype = "application/octet-stream"; 114 | break; 115 | default: 116 | mimetype = "text/plain"; 117 | break; 118 | } 119 | } 120 | return mimetype; 121 | } 122 | 123 | function validateMimetype(mimetype, mimeTypes) { 124 | const validMimetypes = mimeTypes.concat([ 125 | // Text 126 | "text/plain", 127 | // Images 128 | "image/gif", "image/jpeg", "image/png", "image/tiff", "image/webp", 129 | // JSON 130 | "application/json", 131 | // Twig 132 | "text/twig", 133 | // Others non text 134 | "application/octet-stream" 135 | ]); 136 | if(validMimetypes.indexOf(mimetype) == -1) { 137 | return "text/plain"; 138 | } else { 139 | return mimetype; 140 | } 141 | } 142 | 143 | function printDebug(msg) { 144 | if(debug) { 145 | cli.debug(msg); 146 | } 147 | } 148 | 149 | module.exports = packer; 150 | 151 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "mm-packer", 3 | "version": "4.0.5", 4 | "description": "", 5 | "main": "index.js", 6 | "bin": { 7 | "mm-packer": "bin/mm-packer.js" 8 | }, 9 | "scripts": { 10 | "test": "echo \"Error: no test specified\" && exit 1" 11 | }, 12 | "repository": { 13 | "type": "git", 14 | "url": "git@github.com:MM56/mm-packer.git" 15 | }, 16 | "author": "MM56", 17 | "files": [ 18 | "bin/mm-packer.js", 19 | "index.js" 20 | ], 21 | "dependencies": { 22 | "cli": "^1.0.1", 23 | "mime-types": "^2.1.14" 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /unpacker/Unpacker.js: -------------------------------------------------------------------------------- 1 | (function(global){ 2 | 3 | var Unpacker = (function() { 4 | var URL = window.URL || window.webkitURL || window.mozURL || window.msURL; 5 | var hasBlob; 6 | try { 7 | hasBlob = Boolean(Blob); 8 | } catch(e) { 9 | hasBlob = false; 10 | } 11 | if (!ArrayBuffer.prototype.slice) { 12 | //Returns a new ArrayBuffer whose contents are a copy of this ArrayBuffer's 13 | //bytes from `begin`, inclusive, up to `end`, exclusive 14 | ArrayBuffer.prototype.slice = function (begin, end) { 15 | //If `begin` is unspecified, Chrome assumes 0, so we do the same 16 | if (begin === void 0) { 17 | begin = 0; 18 | } 19 | 20 | //If `end` is unspecified, the new ArrayBuffer contains all 21 | //bytes from `begin` to the end of this ArrayBuffer. 22 | if (end === void 0) { 23 | end = this.byteLength; 24 | } 25 | 26 | //Chrome converts the values to integers via flooring 27 | begin = Math.floor(begin); 28 | end = Math.floor(end); 29 | 30 | //If either `begin` or `end` is negative, it refers to an 31 | //index from the end of the array, as opposed to from the beginning. 32 | if (begin < 0) { 33 | begin += this.byteLength; 34 | } 35 | if (end < 0) { 36 | end += this.byteLength; 37 | } 38 | 39 | //The range specified by the `begin` and `end` values is clamped to the 40 | //valid index range for the current array. 41 | begin = Math.min(Math.max(0, begin), this.byteLength); 42 | end = Math.min(Math.max(0, end), this.byteLength); 43 | 44 | //If the computed length of the new ArrayBuffer would be negative, it 45 | //is clamped to zero. 46 | if (end - begin <= 0) { 47 | return new ArrayBuffer(0); 48 | } 49 | 50 | var result = new ArrayBuffer(end - begin); 51 | var resultBytes = new Uint8Array(result); 52 | var sourceBytes = new Uint8Array(this, begin, end - begin); 53 | 54 | resultBytes.set(sourceBytes); 55 | 56 | return result; 57 | }; 58 | } 59 | 60 | function b64encodeString(value) { 61 | var chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'.split(""); 62 | var l = value.length; 63 | var i = cb = b = bl = v = 0; 64 | var b0, b1, b2; 65 | var c0, c1, c2, c3; 66 | var ret = ''; 67 | // String 68 | if(typeof value == "string" || value instanceof String){ 69 | while(i < l) { 70 | b0 = value.charCodeAt(i + 0) & 0xFF; 71 | b1 = value.charCodeAt(i + 1) & 0xFF; 72 | b2 = value.charCodeAt(i + 2) & 0xFF; 73 | c0 = b0 >> 2 & 0x3F; 74 | c1 = (b0 << 4 | b1 >> 4) & 0x3F; 75 | c2 = (b1 << 2 | b2 >> 6) & 0x3F; 76 | c3 = b2 & 0x3F; 77 | 78 | ret += chars[c0] + chars[c1] + chars[c2] + chars[c3]; 79 | i += 3; 80 | } 81 | } 82 | // Array like 83 | else{ 84 | while(i < l) { 85 | b0 = value[i + 0] & 0xFF; 86 | b1 = value[i + 1] & 0xFF; 87 | b2 = value[i + 2] & 0xFF; 88 | c0 = b0 >> 2 & 0x3F; 89 | c1 = (b0 << 4 | b1 >> 4) & 0x3F; 90 | c2 = (b1 << 2 | b2 >> 6) & 0x3F; 91 | c3 = b2 & 0x3F; 92 | 93 | ret += chars[c0] + chars[c1] + chars[c2] + chars[c3]; 94 | i += 3; 95 | } 96 | } 97 | 98 | i = l % 3; 99 | l = ret.length; 100 | if(i == 1) { 101 | ret = ret.substr(0, l - 2) + "=="; 102 | } else if(i == 2) { 103 | ret = ret.substr(0, l - 1) + "="; 104 | } 105 | return ret; 106 | } 107 | 108 | function Unpacker(pack, config) { 109 | if(pack) { 110 | this._init(pack, config); 111 | } 112 | } 113 | 114 | Unpacker.prototype._init = function(pack, config) { 115 | this.config = config; 116 | this.pack = pack; 117 | }; 118 | /** 119 | * Get entry media type 120 | * @param {String} id 121 | * @returns {String} 122 | */ 123 | Unpacker.prototype.getType = function(id) { 124 | return this._findFile(id).type || "text/plain"; 125 | }; 126 | /** 127 | * Get entry content as String 128 | * Support only ASCII encoding (if pack data is stored as typed array) 129 | * @param {String} id 130 | * @returns {String} 131 | */ 132 | Unpacker.prototype.getAsString = function(id){ 133 | if(this.pack == null) { 134 | return ""; 135 | } 136 | 137 | var data = this.getData(id); 138 | if(typeof data == "string" || data instanceof String){ 139 | return data; 140 | } 141 | 142 | // Read by 65KB chunks 143 | data = new Uint8Array(data); 144 | var buffer = ""; 145 | var step = 65535;// 2^16 - 1 146 | var nbSteps = Math.ceil(data.byteLength / step); 147 | var offset = 0; 148 | for(var i = 0; i < nbSteps; i++) { 149 | buffer += String.fromCharCode.apply(null, new Uint8Array(data.buffer.slice(offset, offset + step))); 150 | offset += step; 151 | } 152 | return buffer; 153 | }; 154 | /** 155 | * Get entry content as URI 156 | * @param {String} id 157 | * @returns {String} blob URI or data URI 158 | */ 159 | Unpacker.prototype.getAsURI = function(id){ 160 | var data = this.getData(id); 161 | var type = this.getType(id); 162 | if(hasBlob) { 163 | return URL.createObjectURL(new Blob([data], { type: type })); 164 | } else { 165 | return 'data:' + type + ';base64,' + b64encodeString(data); 166 | } 167 | }; 168 | 169 | /** 170 | * Get entry content as typed array 171 | * Support only ASCII decoding (if pack data is stored as string) 172 | * @param {String} id 173 | * @returns {TypedArray} bytes 174 | */ 175 | Unpacker.prototype.getAsBytes = function(id){ 176 | var data = this.getData(id); 177 | 178 | if(typeof data == "string" || data instanceof String){ 179 | if(typeof Uint8Array !== "function"){ 180 | throw new Error("TypedArray are not supported"); 181 | } 182 | return new Uint8Array(data.split("").map(function(value){return value.charCodeAt(0);})); 183 | } 184 | 185 | return data; 186 | }; 187 | 188 | /** 189 | * Get entry content as same as packer content (string or typed array) 190 | * @param {String} name 191 | * @return {String|TypedArray} 192 | */ 193 | Unpacker.prototype.getData = function(name) { 194 | var file = this._findFile(name); 195 | return this._slice(file.begin, file.end); 196 | }; 197 | 198 | 199 | /** 200 | * Get data between begin and end indexes 201 | * @param {Number} begin 202 | * @param {Number} end 203 | * @return {String|TypedArray} 204 | */ 205 | Unpacker.prototype._slice = function(begin, end) { 206 | if (this.pack == null) { 207 | return typeof Uint8Array == "function" ? new Uint8Array([]) : ""; 208 | } 209 | if (typeof this.pack.substr == "function") { 210 | return this.pack.substr(begin, end - begin); 211 | } 212 | return this.pack.slice(begin, end); 213 | }; 214 | 215 | Unpacker.prototype._findFile = function(name) { 216 | var i = this.config.length; 217 | while (i-- > 0) { 218 | if(this.config[i][0] == name) 219 | { 220 | var config = this.config[i]; 221 | return { 222 | name: config[0], 223 | begin: config[1], 224 | end: config[2], 225 | type: config[3] 226 | }; 227 | } 228 | } 229 | }; 230 | 231 | return Unpacker; 232 | })(); 233 | 234 | //exports to multiple environments 235 | if(typeof define === 'function' && define.amd){ //AMD 236 | define(function () { return Unpacker; }); 237 | } else if (typeof module !== 'undefined' && module.exports){ //node 238 | module.exports = Unpacker; 239 | } else { //browser 240 | //use string because of Google closure compiler ADVANCED_MODE 241 | /*jslint sub:true */ 242 | global['Unpacker'] = Unpacker; 243 | } 244 | }(this)); 245 | -------------------------------------------------------------------------------- /unpacker/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 23 | 34 | 35 | 36 |
37 | 38 |
39 |
40 | 41 | --------------------------------------------------------------------------------