├── .gitignore ├── README.md ├── dev └── img │ ├── 4conv.png │ ├── a2b.png │ ├── anb.png │ ├── b2a.png │ ├── efficient.png │ ├── hub.png │ ├── inefficient.png │ ├── library.png │ ├── onetool.png │ ├── pandat.svg │ ├── schema1.png │ ├── schema2.png │ ├── schemaswtypes.png │ ├── types1.png │ ├── types2.png │ └── typesnfuncs.png ├── index.js ├── module-check.js ├── package.json ├── todo.md ├── transform-cli.js └── transformer-cli.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # transformer - universal data conversion 2 | 3 | `transformer` converts all the things! 4 | 5 | - See [transform.datadex.io](http://transform.datadex.io) 6 | - Try it out: [transformer in the browser!](http://transform.datadex.io/browser) 7 | 8 | [![dat](http://img.shields.io/badge/Development%20sponsored%20by-dat-green.svg?style=flat)](http://dat-data.com) 9 | 10 | [![NPM](https://nodei.co/npm/dat-transformer.png?downloads=true&stars=true)](https://nodei.co/npm/dat-transformer/) 11 | 12 | # ALPHA WARNING 13 | 14 | transformer development is just beginning. Star/watch the project to find out when it's good to go! 15 | 16 | ## Usage 17 | 18 | ### transformer from the command line 19 | 20 | Install transformer globally: 21 | 22 | npm install -g dat-transformer 23 | 24 | 25 | Need to convert from unix-time to iso-date? 26 | 27 | ``` 28 | # install the types + conversions (globally) 29 | > transformer install -g string number integer unix-time js-date iso-date string 30 | Installed: 31 | - transformer.string 32 | - transformer.string-to-number 33 | - transformer.number 34 | - transformer.number-to-integer 35 | - transformer.integer 36 | - transformer.unix-time 37 | - transformer.unix-time-to-js-date 38 | - transformer.js-date 39 | - transformer.js-date-to-iso-date 40 | - transformer.iso-date 41 | 42 | > echo 1398810849 | transform -g number integer unix-time js-date iso-date 43 | 2014-04-29T22:34:09.000Z 44 | ``` 45 | 46 | Or an IP address to its hex value? 47 | 48 | ``` 49 | > transformer install -g string ip-address buffer hex string 50 | Installed: 51 | - transformer.string 52 | - transformer.hex 53 | - transformer.hex-to-buffer 54 | - transformer.buffer 55 | - transformer.buffer-to-ip-address 56 | - transformer.ip-address 57 | 58 | > echo "127.0.0.1" | transform -g ip-address buffer hex 59 | 7f000001 60 | ``` 61 | 62 | Show description for type: 63 | 64 | ``` 65 | # transformer src 66 | > transformer src iso-date 67 | { 68 | "@context": "http://transformer.io/context/transformer.jsonld", 69 | "id": "transformer/iso-date", 70 | "description": "ISO 8601 date format: 2006-01-02T15:04:05Z07:00" 71 | } 72 | ``` 73 | 74 | Boom. 75 | 76 | 77 | ### transformer from javascript 78 | 79 | Install transformer module: 80 | 81 | npm install transformer 82 | 83 | Converter from one format to another: 84 | 85 | ```js 86 | var transformer = require('transformer'); 87 | var a2b = transformer(formatA, formatB); 88 | var b_data = a2b(a_data); 89 | ``` 90 | 91 | Convert data from one format to another (shorthand): 92 | 93 | ```js 94 | var transformer = require('transformer'); 95 | var b_data = transformer(formatA, formatB, a_data); 96 | ``` 97 | 98 | For example: 99 | 100 | ```js 101 | // convert unixtime date to iso date 102 | var transformer = require('transformer'); 103 | var unix2iso = transformer('unix-time','iso-date') 104 | var unix = 1397788143 105 | var iso = unix2iso(unix) 106 | //'2014-04-18T02:29:03' 107 | 108 | // convert iso date to unixtime date 109 | var iso2unix = transformer('iso-date','unix-time') 110 | var unix2 = iso2unix(iso) 111 | // 1397788143 112 | ``` 113 | 114 | ### more examples 115 | 116 | I'll use command-line syntax here, because it's cleaner. But everything 117 | is available in the command-line, in js, and (soon!) in the browser. 118 | Note that these examples are longer than intended, as they are the full 119 | pipelines of the type conversions (`string | iso-date | js-date | unix-time | 120 | integer | number`). Once type inference is built, this will be a lot nicer 121 | (see second set of examples below). 122 | 123 | ``` 124 | > echo '2014-05-02T09:51:03.000Z' | transform iso-date js-date unix-time integer number 125 | 1399024263 126 | 127 | > echo 1399024263 | transform number integer unix-time js-date iso-date 128 | 2014-05-02T09:51:03.000Z 129 | 130 | > echo 1234.3123 | transform number integer number 131 | 1234 132 | 133 | > echo '127.0.0.1' | transform ip-address buffer hex 134 | 7f000001 135 | 136 | > echo '127.0.0.1' | transform ip-address buffer base32 137 | c9gq6t9k68 138 | 139 | > echo '127.0.0.1' | transform ip-address buffer base64 140 | fwAAAQ== 141 | 142 | > echo 'fwAAAQ==' | transform base64 buffer ip-address 143 | 127.0.0.1 144 | 145 | > echo "bar" | transform xml-string jsonml json 146 | ["foo","bar"] 147 | ``` 148 | 149 | Once https://github.com/jbenet/transformer/issues/8 is addressed, these should 150 | be expressible as: 151 | 152 | ``` 153 | > echo '2014-05-02T09:51:03.000Z' | transform iso-date unix-time 154 | 1399024263 155 | 156 | > echo 1399024263 | transform unix-time iso-date 157 | 2014-05-02T09:51:03.000Z 158 | 159 | > echo 1234.3123 | transform integer 160 | 1234 161 | 162 | > echo '127.0.0.1' | transform ip-address hex 163 | 7f000001 164 | 165 | > echo '127.0.0.1' | transform ip-address base32 166 | c9gq6t9k68 167 | 168 | > echo '127.0.0.1' | transform ip-address base64 169 | fwAAAQ== 170 | 171 | > echo 'fwAAAQ==' | transform base64 ip-address 172 | 127.0.0.1 173 | 174 | > echo "bar" | transform xml-string json 175 | ["foo","bar"] 176 | ``` 177 | 178 | ## Development 179 | 180 | Transformer is part of [Dat Project](http://dat-data.com). It is designed to be modular. All types and conversions are npm modules. 181 | 182 | Transformer's core is modular itself. See: 183 | 184 | - [transformer-type](https://github.com/jbenet/transformer-type) - use it to make a transformer type 185 | - [transformer-conversion](https://github.com/jbenet/transformer-conversion) - use it to make a transformer conversion 186 | - [transformer-compose](https://github.com/jbenet/transformer-compose) - composes conversions 187 | - [transformer-resolve](https://github.com/jbenet/transformer-resolve) - resolves types into the conversion pipeline between them. (dumb atm) 188 | - [transformer-loader](https://github.com/jbenet/transformer-loader) - dynamic loading of transformer modules 189 | - [transformer-compile](https://github.com/jbenet/transformer-loader) - compiles a conversion pipeline into a single module (uses requires). 190 | - Bundle compiled transformers with [browserify](https://github.com/substack/node-browserify) 191 | - [transformer](https://github.com/jbenet/transformer) (this module) - all core as one library, and two command line tools: 192 | - `transformer` - utility to install, resolve, and compile transformer modules 193 | - `transform < > ` - expose any transformer as a pipe-loving unix utility 194 | 195 | For making your own types and conversions: 196 | 197 | - [transformer-pkg](https://github.com/jbenet/transformer-pkg) - cli tool to make, test, publish types + conversions in under one minute (ACT NOW!!!) 198 | - [transformer-test](https://github.com/jbenet/transformer-test) - tests your types + conversions (used by `transformer-pkg`) 199 | 200 | Other: 201 | 202 | - [transformer-website](https://github.com/jbenet/transformer-website) - the website: http://transform.datadex.io 203 | 204 | 205 | 206 | ## transformer story 207 | 208 | Imagine you have a data file, that can be represented in two formats, `A` and `B`. 209 | 210 | ![](https://raw.githubusercontent.com/jbenet/transformer/master/dev/img/anb.png) 211 | 212 | You'd like to convert from one format to the other `A -> B`, so you look for a program to convert the file. If there isn't one, you write your own. 213 | 214 | ![](https://raw.githubusercontent.com/jbenet/transformer/master/dev/img/a2b.png) 215 | 216 | Maybe your program is smart enough to convert in both directions, but maybe that means an entirely different program 217 | 218 | ![](https://raw.githubusercontent.com/jbenet/transformer/master/dev/img/b2a.png) 219 | 220 | 221 | Say you have a _third_ format `C`. Uh oh, that could mean up to _four more_ programs. 222 | 223 | ![](https://raw.githubusercontent.com/jbenet/transformer/master/dev/img/4conv.png) 224 | 225 | So, though sometimes there are smart programs that will leverage intermediate conversions (say, `A -> C` expands to `A -> B -> C`), this is generally inefficient, potentially causing a `N to N^2` blowup in the number of programs to write. 226 | 227 | ![](https://raw.githubusercontent.com/jbenet/transformer/master/dev/img/inefficient.png) 228 | 229 | This is particularly annoying because often the conversions are simple transformations 230 | 231 | ![](https://raw.githubusercontent.com/jbenet/transformer/master/dev/img/types1.png) 232 | 233 | Or even simpler, just re-arrangements 234 | 235 | ![](https://raw.githubusercontent.com/jbenet/transformer/master/dev/img/types2.png) 236 | 237 | There's got to be a better way. What if there was some way to represent _all_ data, and build converters between _that one format_ and all others? 238 | 239 | ![](https://raw.githubusercontent.com/jbenet/transformer/master/dev/img/hub.png) 240 | 241 | This is how [Pandoc](http://johnmacfarlane.net/pandoc/) (document converter) and [LLVM](http://llvm.org) (compiler) work: 242 | 243 | ![](http://jbenet.static.s3.amazonaws.com/f44a8a0/llvm-three-phase-design.png) 244 | 245 | The problem is that picking _THE universal data format_ is hard. In a way, all data is part of the same hyperspace, and a particular dataset is just a projection into a subspace. So it's just a matter of finding to the right _dimensions_. For example, the simple transformation earlier is really just a projection from one set of dimensions to another. The _values_ of the data are the same. The dimensions are the _types_. 246 | 247 | ![](https://raw.githubusercontent.com/jbenet/transformer/master/dev/img/types1.png) 248 | 249 | So what if we already have our magical _universal data representation_, and it is just the combination of all others? What if our magical multi-format conversion tool is just a matter of defining the right _types_ and their _conversions_? Imagine if we had a massive library of types and conversion functions: 250 | 251 | ![](https://raw.githubusercontent.com/jbenet/transformer/master/dev/img/typesnfuncs.png) 252 | 253 | We should be able to build larger types that uniquely match each format -- let's call these `schemas`. The conversion between two formats would just be the aggregate conversions between the component types: 254 | 255 | ![](https://raw.githubusercontent.com/jbenet/transformer/master/dev/img/schemaswtypes.png) 256 | 257 | 258 | So, given the (a) types, (b) type conversion functions, and (c) schemas, _one_ dumb tool should be able to convert two formats. 259 | 260 | ![](https://raw.githubusercontent.com/jbenet/transformer/master/dev/img/schema1.png) 261 | 262 | If the tool needed to convert between a brand new format, it would just be a matter of defining the schema, and potentially some new types and functions. Most formats reuse types, so this gets easier over time. 263 | 264 | ![](https://raw.githubusercontent.com/jbenet/transformer/master/dev/img/schema2.png) 265 | 266 | [transformer](http://github.com/jbenet/transformer) is this _dumb_ but super-efficient multi-format conversion tool. 267 | 268 | ![](https://raw.githubusercontent.com/jbenet/transformer/master/dev/img/efficient.png) 269 | 270 | The secret sauce is in the shared library of (a) types, (b) conversion functions, and (c) schemas. 271 | 272 | ![](https://raw.githubusercontent.com/jbenet/transformer/master/dev/img/library.png) 273 | 274 | 275 | ## transformer formats -- an example 276 | 277 | How does transformer know what formats are? It has a large library of Types (formats), and Conversion functions. These compose. They are specified using JSON-LD documents and javascript code (conversions). 278 | 279 | ### the data 280 | 281 | Let's look at an example. Say you have a few data sources. You have Contacts, from your email client, in json: 282 | 283 | ```json 284 | [ 285 | { 286 | "name": "Juan Benet", 287 | "email": "juan@benet.ai", 288 | "phone": "123-456-7890" 289 | }, 290 | { 291 | "name": "Molly M", 292 | "email": "mollym@example.com", 293 | "phone": "123-456-7890" 294 | }, 295 | ... 296 | ] 297 | ``` 298 | 299 | Call logs, from your phone company, in a csv: 300 | ```csv 301 | FROM,TO,DATE,TIME,DURATION 302 | (123) 456-7890,(456) 123-7890,04/01/2013,5:28pm,0:51 303 | (456) 123-7890,(123) 456-7890,04/01/2013,10:15pm,3:27 304 | (123) 456-7890,(456) 123-7890,04/03/2013,2:03pm,9:29 305 | ... 306 | ``` 307 | 308 | GPS data, from your intelligence-agency-sponsored homing implant, in XML: 309 | ```xml 310 | 311 | 312 | 313 | 37.447462 314 | -122.160233 315 | 316 | 317 | 318 | 37.424983 319 | -122.225887 320 | 321 | 322 | 323 | 37.770498 324 | -122.445065 325 | 326 | 327 | ... 328 | 329 | ``` 330 | 331 | Wouldn't it be nice to get all the data cleaned up and reformatted automatically? And wouldn't it be great to be able to generate a call history with meaningful names and locations? Something like: 332 | 333 | ```json 334 | { 335 | "owner": "123.456.7890", 336 | "history": [ 337 | { 338 | "to": "Molly M", 339 | "number": "456.123.7890", 340 | "date": "2013-04-01 17:28:00", 341 | "location": "Palo Alto, CA" 342 | }, 343 | { 344 | "from": "Molly M", 345 | "number": "456.123.7890", 346 | "date": "2013-04-01 22:15:00", 347 | "location": "Atherton, CA" 348 | }, 349 | { 350 | "to": "Molly M", 351 | "number": "456.123.7890", 352 | "date": "2013-04-01 22:15:00", 353 | "location": "San Francisco, CA" 354 | } 355 | ] 356 | } 357 | ``` 358 | 359 | This is completely doable given format schemas and type conversions. All transformer needs is the right formats. 360 | 361 | ### the types (formats) 362 | 363 | (Common fields `[@context, id, type]` in formats omitted for simplicity). 364 | 365 | The input type for the contacts: 366 | 367 | ```json 368 | { 369 | "codec": "json", 370 | "schema": [ { 371 | "name": "person-name", 372 | "email": "email-address", 373 | "phone": "phone-number-usa-dotted" 374 | } ] 375 | } 376 | ``` 377 | 378 | 379 | The input type for the call records: 380 | 381 | ```json 382 | { 383 | "codec": "csv", 384 | "schema": { 385 | "FROM": "phone-number-usa-parens", 386 | "TO": "phone-number-usa-parens", 387 | "DATE": "date", 388 | "TIME": "time-of-day", 389 | "DURATION": "time-hms", 390 | } 391 | } 392 | ``` 393 | 394 | The input type for the location records: 395 | 396 | ```json 397 | { 398 | "codec": "xml", 399 | "schema": { 400 | "readings": [ { 401 | "reading": { 402 | "lat": "latitude", 403 | "lon": "longitude", 404 | "time": "iso-date" 405 | } 406 | } ] 407 | } 408 | } 409 | ``` 410 | 411 | The output type you want: 412 | 413 | ```json 414 | { 415 | "codec": "json", 416 | "schema": { 417 | "owner": "phone-number-usa-dotted", 418 | "history": [ { 419 | "to": "person-name", 420 | "from": "person-name", 421 | "number": "phone-number-usa-dotted", 422 | "date": "iso-date-space", 423 | "location": "us-city-name" 424 | } ] 425 | } 426 | } 427 | ``` 428 | 429 | Each of the `transformer/` `types` are transformer modules that link transformer objects and allow transformer to find the relevant functions. The `transformer/` part here shows anyone can publish new conversions or types to transformer's index. 430 | 431 | ### the conversions 432 | 433 | All the types referenced above (e.g. `transformer/person-name` and `transformer/iso-date`) have their own format description registered in transformer's index. For example 434 | 435 | ```json 436 | { 437 | "id": "https://transformer.io/transformer/iso-date", 438 | "type": "string" 439 | } 440 | ``` 441 | 442 | # work in progress 443 | 444 | 445 | 446 | ## FAQ 447 | 448 | ### Why javascript? Why not ``? 449 | 450 | 451 | transformer aims to be widely adopted, easy to use for non-programers, and extremely portable. There are many other systems which are much better tuned -- or in Haskell's case, precisely the right tool -- for solving this particular problem. However, these are unfortunately not as portable or flexible as javascript. If having to learn obscure syntax or installing obscure platforms were prerequisites, transformer would never be adoped by most users. 452 | 453 | Besides, transformer should be able to run on websites :) 454 | 455 | ### How do you write modules? 456 | 457 | A longer description will be available, but for now see: https://github.com/jbenet/transformer-pkg 458 | 459 | -------------------------------------------------------------------------------- /dev/img/4conv.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbenet/transformer/94fbc1f4e4f2a8a3fbec3b7d1435aafaddde08ba/dev/img/4conv.png -------------------------------------------------------------------------------- /dev/img/a2b.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbenet/transformer/94fbc1f4e4f2a8a3fbec3b7d1435aafaddde08ba/dev/img/a2b.png -------------------------------------------------------------------------------- /dev/img/anb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbenet/transformer/94fbc1f4e4f2a8a3fbec3b7d1435aafaddde08ba/dev/img/anb.png -------------------------------------------------------------------------------- /dev/img/b2a.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbenet/transformer/94fbc1f4e4f2a8a3fbec3b7d1435aafaddde08ba/dev/img/b2a.png -------------------------------------------------------------------------------- /dev/img/efficient.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbenet/transformer/94fbc1f4e4f2a8a3fbec3b7d1435aafaddde08ba/dev/img/efficient.png -------------------------------------------------------------------------------- /dev/img/hub.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbenet/transformer/94fbc1f4e4f2a8a3fbec3b7d1435aafaddde08ba/dev/img/hub.png -------------------------------------------------------------------------------- /dev/img/inefficient.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbenet/transformer/94fbc1f4e4f2a8a3fbec3b7d1435aafaddde08ba/dev/img/inefficient.png -------------------------------------------------------------------------------- /dev/img/library.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbenet/transformer/94fbc1f4e4f2a8a3fbec3b7d1435aafaddde08ba/dev/img/library.png -------------------------------------------------------------------------------- /dev/img/onetool.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbenet/transformer/94fbc1f4e4f2a8a3fbec3b7d1435aafaddde08ba/dev/img/onetool.png -------------------------------------------------------------------------------- /dev/img/schema1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbenet/transformer/94fbc1f4e4f2a8a3fbec3b7d1435aafaddde08ba/dev/img/schema1.png -------------------------------------------------------------------------------- /dev/img/schema2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbenet/transformer/94fbc1f4e4f2a8a3fbec3b7d1435aafaddde08ba/dev/img/schema2.png -------------------------------------------------------------------------------- /dev/img/schemaswtypes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbenet/transformer/94fbc1f4e4f2a8a3fbec3b7d1435aafaddde08ba/dev/img/schemaswtypes.png -------------------------------------------------------------------------------- /dev/img/types1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbenet/transformer/94fbc1f4e4f2a8a3fbec3b7d1435aafaddde08ba/dev/img/types1.png -------------------------------------------------------------------------------- /dev/img/types2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbenet/transformer/94fbc1f4e4f2a8a3fbec3b7d1435aafaddde08ba/dev/img/types2.png -------------------------------------------------------------------------------- /dev/img/typesnfuncs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbenet/transformer/94fbc1f4e4f2a8a3fbec3b7d1435aafaddde08ba/dev/img/typesnfuncs.png -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var Type = require('transformer-type') 2 | var Conversion = require('transformer-conversion') 3 | var loader = require('transformer-loader') 4 | var compose = require('transformer-compose') 5 | var resolve = require('transformer-resolve') 6 | 7 | var transformer = module.exports = transformerSync 8 | 9 | transformer.sync = transformerSync 10 | transformer.async = transformerAsync 11 | 12 | transformer.Type = Type 13 | transformer.Conversion = Conversion 14 | transformer.loader = loader 15 | transformer.compose = compose 16 | transformer.resolve = resolve 17 | transformer.contextUrl = Type.Object.contextUrl 18 | 19 | function transformerSync() { 20 | switch (arguments.length) { 21 | case 0: 22 | throw new Error('transformer error: no arguments.') 23 | case 1: // Load it 24 | return loader(arguments[0]) 25 | default: 26 | var conversions = resolve(_.toArray(arguments)) 27 | return compose.sync(conversions) 28 | } 29 | } 30 | 31 | function transformerAsync() { 32 | switch (arguments.length) { 33 | case 0: 34 | throw new Error('transformer error: no arguments.') 35 | case 1: // Load it 36 | return loader(arguments[0]) 37 | default: 38 | var conversions = resolve(_.toArray(arguments)) 39 | return compose.async(conversions) 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /module-check.js: -------------------------------------------------------------------------------- 1 | var log = console.log 2 | var install = require('transformer-installer') 3 | var Loader = require('transformer-loader') 4 | 5 | module.exports = { 6 | ensureModulesAreInstalled: ensureModulesAreInstalled, 7 | catchMissingModulesErr: catchInstallErr, 8 | } 9 | 10 | function stringModuleIds(str) { 11 | var re = /transformer\.([a-z0-9-.]+)/ig; 12 | var matches = []; 13 | var match; 14 | while (match = re.exec(str)) { 15 | matches.push(match[1]); 16 | } 17 | return matches; 18 | } 19 | 20 | function handleRequiresModulesError(ids) { 21 | log(install.explanation(ids)) 22 | process.exit(-1) 23 | } 24 | 25 | function ensureModulesAreInstalled(ids, global) { 26 | missing = Loader.missingModules(ids, global) 27 | if (missing.length > 0) 28 | handleRequiresModulesError(missing) 29 | } 30 | 31 | function catchInstallErr(func) { 32 | try { 33 | func() 34 | } catch (e) { 35 | if (Loader.errIsModuleNotFound(e)) { 36 | var m = stringModuleIds(e.toString()) 37 | handleRequiresModulesError(m) 38 | } else { 39 | throw e; 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dat-transformer", 3 | "version": "0.2.6", 4 | "description": "multiformat conversion", 5 | "main": "index.js", 6 | "bin": { 7 | "transform": "transform-cli.js", 8 | "transformer": "transformer-cli.js" 9 | }, 10 | "scripts": { 11 | "test": "echo \"Error: no test specified\" && exit 1" 12 | }, 13 | "repository": { 14 | "type": "git", 15 | "url": "http://github.com/jbenet/transformer.git" 16 | }, 17 | "keywords": [ 18 | "dat", 19 | "data", 20 | "conversion", 21 | "format", 22 | "schema" 23 | ], 24 | "author": "https://github.com/jbenet", 25 | "license": "MIT", 26 | "bugs": { 27 | "url": "https://github.com/jbenet/transformer/issues" 28 | }, 29 | "dependencies": { 30 | "lodash.map": "^2.4.1", 31 | "minimist": "0.0.8", 32 | "resolve": "^0.6.3", 33 | "rw": "^0.1.4", 34 | "transformer-compile": "0.0.5", 35 | "transformer-compose": "^0.1.1", 36 | "transformer-conversion": "^0.1.2", 37 | "transformer-installer": "^0.1.0", 38 | "transformer-loader": "^0.1.3", 39 | "transformer-resolve": "^0.1.4", 40 | "transformer-type": "^0.1.0" 41 | }, 42 | "devDependencies": {}, 43 | "homepage": "https://github.com/jbenet/transformer" 44 | } 45 | -------------------------------------------------------------------------------- /todo.md: -------------------------------------------------------------------------------- 1 | 2 | # transformer todo 3 | 4 | ## types 5 | 6 | (quick brainstorm) 7 | 8 | - ip addresses 9 | - ip-address-type 10 | - ipv4-address-type 11 | - ipv6-address-type 12 | - hex-ip-address-codec 13 | - string-ip-address-codec (hbo) 14 | - bytes-ip-address-codec (nbo) 15 | 16 | - phone numbers 17 | - international-phone-number-type 18 | - us-phone-number-type 19 | - parens-us-phone-number-codec 20 | - dotted-us-phone-number-codec 21 | - dashed-us-phone-number-codec 22 | 23 | - date-type 24 | - unix-time-date-codec 25 | - iso-date-codec 26 | 27 | - time-type 28 | - unix-time-codec 29 | - iso-time-codec (hrs:min:sec.subsec) 30 | - 24hr-day-time-codec 31 | - 12hr-day-time-codec (hrs:min am/pm) 32 | 33 | 34 | - number-type 35 | - int-codec 36 | - float-codec 37 | - base-n-codec 38 | - big-endian-codec 39 | - little-endian-codec 40 | - network-byte-order-codec 41 | - absolute-value-codec 42 | 43 | - buffers 44 | - hex-codec 45 | - base64-codec 46 | - base32-codec 47 | - base56-codec 48 | - base-n-codec 49 | 50 | - escaping characters 51 | - html 52 | 53 | - image-type 54 | - png-codec 55 | - jpg-codec 56 | - gif-codec 57 | 58 | - file-type 59 | 60 | - directory-type 61 | 62 | - compression codecs 63 | - zip-codec 64 | - bzip2-codec 65 | - zippy-codec 66 | - tar-codec 67 | - rar-codec 68 | 69 | - formats 70 | - json-codec 71 | - jsonml-xml-codec 72 | 73 | - strings 74 | - capitalize-string-codec 75 | - split-string-codec 76 | - lowercase-string-codec 77 | - uppercase-string-codec 78 | 79 | - isbn to book title (conversion) 80 | -------------------------------------------------------------------------------- /transform-cli.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | var rw = require('rw'); 3 | var map = require('lodash.map'); 4 | var moduleCheck = require('./module-check') 5 | var transformer = require('./'); 6 | var argv = require('minimist')(process.argv.slice(2), { 7 | boolean: ['g', 'global', 'sync', 'async'], 8 | }); 9 | 10 | var useGlobal = argv.g || argv.global 11 | var log = console.log; 12 | 13 | function usage() { 14 | var n = process.argv[1]; 15 | log('Usage: ' + n + ' < > '); 16 | }; 17 | 18 | function convert(ids, useGlobal) { 19 | if (!(ids.length > 1)) { 20 | usage(); 21 | process.exit(-1); 22 | } 23 | 24 | // wrap with string (because cli) 25 | ids.unshift('string'); 26 | ids.push('string'); 27 | 28 | var convs = []; // transformer.Conversion.pathIds(ids); 29 | moduleCheck.ensureModulesAreInstalled(ids.concat(convs), useGlobal); 30 | 31 | // for now use rw module with Sync. TODO: streams. 32 | var read = function() { 33 | return rw.readSync('/dev/stdin', 'utf8').trim(); 34 | } 35 | 36 | var write = function(output) { 37 | rw.writeSync('/dev/stdout', '' + output + '\n', 'utf8'); 38 | } 39 | 40 | // resolve ids -> conversion 41 | var conversions = transformer.resolve(ids, useGlobal) 42 | 43 | // transformer chain 44 | var async = argv["async"] 45 | map(conversions, function(c) { 46 | async = async || c.async 47 | }) 48 | 49 | if (!async || argv["sync"]) { 50 | var in2out = transformer.compose.sync(conversions); 51 | write(in2out(read())); 52 | 53 | } else { 54 | var in2out = transformer.compose.async(conversions); 55 | in2out(read(), function(err, output) { 56 | if (err) throw err; 57 | write(output); 58 | }); 59 | } 60 | } 61 | 62 | 63 | function main() { 64 | // set global resolution if wanted. 65 | convert(argv._, useGlobal); 66 | } 67 | 68 | // run main safely. 69 | moduleCheck.catchMissingModulesErr(main) 70 | -------------------------------------------------------------------------------- /transformer-cli.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | var map = require('lodash.map') 3 | var resolve = require('transformer-resolve') 4 | var install = require('transformer-installer') 5 | var compile = require('transformer-compile') 6 | var moduleCheck = require('./module-check') 7 | // var compile = require('transformer-compiler') 8 | var transformer = require('./'); 9 | var argv = require('minimist')(process.argv.slice(2), { 10 | boolean: ['g', 'global', 'async'], 11 | }); 12 | 13 | var useAsync = argv.async 14 | var useGlobal = argv.g || argv.global 15 | var log = console.log; 16 | 17 | function usage() { 18 | var n = process.argv[1]; 19 | log('Usage: transformer [] []') 20 | log('') 21 | log('Subcommands:') 22 | log('') 23 | log(' src print transformer description') 24 | log(' install [-g] install transformers') 25 | log(' resolve [-g] resolve types to conversions between them') 26 | log(' compile [-g] resolve types and compile conversion pipeline') 27 | log('') 28 | log('Flags:') 29 | log('') 30 | log(' -g, --global install and use global node_modules, instead of local') 31 | log('') 32 | }; 33 | 34 | function src(id, useGlobal) { 35 | var m = transformer.loader(id, useGlobal) 36 | return JSON.stringify(m.src, undefined, 1) 37 | } 38 | 39 | function resolveIds(types, useGlobal) { 40 | // first ensure the types are installed 41 | moduleCheck.ensureModulesAreInstalled(types, useGlobal) 42 | 43 | return map(resolve(types, useGlobal), function(c) { 44 | return c.src.id 45 | }).join('\n') 46 | } 47 | 48 | function main() { 49 | if (argv._.length < 2) { 50 | usage() 51 | process.exit(-1) 52 | } 53 | 54 | // set options. 55 | var command = argv._[0].toLowerCase() 56 | var rest = argv._.slice(1) 57 | switch (command) { 58 | case 'install': return log(install(rest, useGlobal)) 59 | case 'resolve': return log(resolveIds(rest, useGlobal)) 60 | case 'compile': return log(compile(resolve(rest, useGlobal), useAsync)) 61 | case 'src': return log(src(rest[0], useGlobal)) 62 | } 63 | 64 | usage() 65 | } 66 | 67 | 68 | // run main safely. 69 | moduleCheck.catchMissingModulesErr(main) 70 | --------------------------------------------------------------------------------