├── package.json ├── index.js ├── LICENSE ├── .gitignore ├── test.js └── README.md /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dweb-loader", 3 | "version": "2.0.0", 4 | "description": "JSON-LD document loader for IPFS and IPLD URIs", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "node test.js" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "git+https://github.com/underlay/dweb-loader.git" 12 | }, 13 | "keywords": [ 14 | "ipfs", 15 | "ipld", 16 | "dweb", 17 | "jsonld" 18 | ], 19 | "author": "Joel Gustafson", 20 | "license": "MIT", 21 | "bugs": { 22 | "url": "https://github.com/underlay/dweb-loader/issues" 23 | }, 24 | "homepage": "https://github.com/underlay/dweb-loader#readme", 25 | "dependencies": { 26 | "cids": "^0.7.3" 27 | }, 28 | "devDependencies": { 29 | "ipfs-http-client": "^40.2.1", 30 | "jsonld": "^2.0.2" 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | const CID = require("cids") 2 | 3 | async function ipldLoader(ipfs, path) { 4 | const cid = new CID(path) 5 | if (cid.codec === "dag-cbor") { 6 | return ipfs.dag.get(path).then(({ value }) => ({ document: value })) 7 | } else { 8 | throw new Error("Unsupported IPLD codec") 9 | } 10 | } 11 | 12 | function ipfsLoader(ipfs, path) { 13 | return ipfs.cat(path).then(bytes => ({ document: JSON.parse(bytes) })) 14 | } 15 | 16 | const documentLoaders = { 17 | "ipld://": ipldLoader, 18 | "dweb:/ipld/": ipldLoader, 19 | "ipfs://": ipfsLoader, 20 | "dweb:/ipfs/": ipfsLoader, 21 | } 22 | 23 | const prefixes = Object.keys(documentLoaders) 24 | 25 | module.exports = ipfs => async (url, options) => { 26 | const prefix = prefixes.find(prefix => url.indexOf(prefix) === 0) 27 | if (prefix) { 28 | return documentLoaders[prefix](ipfs, url.slice(prefix.length)) 29 | } else { 30 | throw new Error("Could not load document", url) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Joel Gustafson 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | 8 | # Runtime data 9 | pids 10 | *.pid 11 | *.seed 12 | *.pid.lock 13 | 14 | # Directory for instrumented libs generated by jscoverage/JSCover 15 | lib-cov 16 | 17 | # Coverage directory used by tools like istanbul 18 | coverage 19 | 20 | # nyc test coverage 21 | .nyc_output 22 | 23 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 24 | .grunt 25 | 26 | # Bower dependency directory (https://bower.io/) 27 | bower_components 28 | 29 | # node-waf configuration 30 | .lock-wscript 31 | 32 | # Compiled binary addons (https://nodejs.org/api/addons.html) 33 | build/Release 34 | 35 | # Dependency directories 36 | node_modules/ 37 | jspm_packages/ 38 | 39 | # TypeScript v1 declaration files 40 | typings/ 41 | 42 | # Optional npm cache directory 43 | .npm 44 | 45 | # Optional eslint cache 46 | .eslintcache 47 | 48 | # Optional REPL history 49 | .node_repl_history 50 | 51 | # Output of 'npm pack' 52 | *.tgz 53 | 54 | # Yarn Integrity file 55 | .yarn-integrity 56 | 57 | # dotenv environment variables file 58 | .env 59 | 60 | # next.js build output 61 | .next 62 | -------------------------------------------------------------------------------- /test.js: -------------------------------------------------------------------------------- 1 | const jsonld = require("jsonld") 2 | const IPFS = require("ipfs-http-client") 3 | const { Buffer } = IPFS 4 | 5 | const ipfs = IPFS() 6 | const documentLoader = require("./index.js")(ipfs) 7 | 8 | const context = { 9 | schema: "http://schema.org/", 10 | prov: "http://www.w3.org/ns/prov#", 11 | } 12 | 13 | ipfs.dag.put(context).then(cid => { 14 | const contextUri = `dweb:/ipld/${cid.toBaseEncodedString()}` 15 | console.log("context:", contextUri) 16 | const doc = { 17 | "@context": contextUri, 18 | "prov:wasAttributedTo": { 19 | "@type": "schema:Person", 20 | "schema:name": "Eve", 21 | }, 22 | "@graph": { 23 | "@type": "schema:Person", 24 | "schema:name": "Alice", 25 | "schema:knows": { 26 | "@type": "schema:Person", 27 | "schema:name": "Bob", 28 | }, 29 | }, 30 | } 31 | 32 | const bytes = Buffer.from(JSON.stringify(doc)) 33 | ipfs.add(bytes).then(([{ hash }]) => { 34 | const docUri = `ipfs://${hash}` 35 | console.log("document:", docUri) 36 | jsonld 37 | .expand(docUri, { documentLoader }) 38 | .then(expanded => { 39 | console.log("expanded", expanded) 40 | jsonld.compact(expanded, contextUri, {documentLoader}) 41 | }) 42 | .then(result => { 43 | console.log(JSON.stringify(result, null, " ")) 44 | process.exit() 45 | }) 46 | }) 47 | }) 48 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # dweb-loader 2 | 3 | JSON-LD document loader for IPFS and IPLD 4 | 5 | ```json 6 | { 7 | "@context": "dweb:/ipfs/QmUFeUYXqyKa1mXLyfiCkm1MDbwYPTFBpyKvW7Nhy98Ks1", 8 | "@type": "Digest", 9 | "digestAlgorithm": "http://www.w3.org/2000/09/xmldsig#sha1", 10 | "digestValue": "981ec496092bf6ee18d6255d96069b528633268b" 11 | } 12 | ``` 13 | 14 | ## Motivation 15 | 16 | Suppose you have some JSON-LD, like this example taken from the [W3C Security Vocabulary](https://web-payments.org/vocabs/security#Digest): 17 | 18 | ```json 19 | { 20 | "@context": "https://w3id.org/security/v1", 21 | "@type": "Digest", 22 | "digestAlgorithm": "http://www.w3.org/2000/09/xmldsig#sha1", 23 | "digestValue": "981ec496092bf6ee18d6255d96069b528633268b" 24 | } 25 | ``` 26 | 27 | Making _any_ sense of this document requires fetching the context from `https://w3id.org/security/v1` to resolve the properties and types. For example, the context is what tells us that `digestValue` is an abbrviation of `https://web-payments.org/vocabs/security#digestValue`. 28 | 29 | Contexts are wonderful and elegant and an huge feature of JSON-LD, but just to be practical in the real world, contexts need to be cached so that we don't have to hit the network every time we see someone using something from the W3C security vocabulary (and the JSON-LD folks will be the [first to tell you this](http://manu.sporny.org/2016/json-ld-context-caching/)). 30 | 31 | This is all well and good, but what if I'm not the W3C and want to publish a context that I expect to re-use (or other people to re-use)? I don't want to commit to hosting something on a permanent URL until the end of time. And I probably don't have as good security practices as they do, so if someone gets into my server they can rename `digestValue` to `http://white.house/NuclearLaunchCodes` or something nefarious without anyone noticing (you never really know if the context that you get served was the same context that the author of the document intended). _I want to cache my contexts, but I'm not in a position to host it at a URL._ 32 | 33 | _enter the decentralized web_ 34 | 35 | ## Puttting Contexts on IPFS 36 | 37 | IPFS is a decentralized filesystem that names files by their hash and lets you ask for them from the network at large, from no particular location, just whoever happens to have them, similar to BitTorrent. You can pin files you really care about to guarantee their availability, but any node that has the file will help you retrieve it. 38 | 39 | Even better, IPFS can caches files automatically: once you fetch a file from the network, you become your own closest peer, and subsequent requests for that file return without touching the network at all. The beauty of content-addressing is that we know the contents haven't changed! 40 | 41 | So what if I was worried the W3C was going to get hacked, or if I just wanted a built-in way of caching my contexts, I could use IPFS like this: 42 | 43 | First, I get the file from its current URL and add it to IPFS 44 | 45 | ``` 46 | joel$ curl -L https://w3id.org/security/v1 | ipfs add 47 | % Total % Received % Xferd Average Speed Time Time Time Current 48 | Dload Upload Total Spent Left Speed 49 | 100 310 100 310 0 0 2259 0 --:--:-- --:--:-- --:--:-- 2246 50 | 100 2019 100 2019 0 0 5790 0 --:--:-- --:--:-- --:--:-- 5790 51 | added QmUFeUYXqyKa1mXLyfiCkm1MDbwYPTFBpyKvW7Nhy98Ks1 QmUFeUYXqyKa1mXLyfiCkm1MDbwYPTFBpyKvW7Nhy98Ks1 52 | joel$ 53 | joel$ ipfs cat QmUFeUYXqyKa1mXLyfiCkm1MDbwYPTFBpyKvW7Nhy98Ks1 54 | { 55 | "@context": { 56 | "id": "@id", 57 | "type": "@type", 58 | 59 | "dc": "http://purl.org/dc/terms/", 60 | "sec": "https://w3id.org/security#", 61 | "xsd": "http://www.w3.org/2001/XMLSchema#", 62 | 63 | "EcdsaKoblitzSignature2016": "sec:EcdsaKoblitzSignature2016", 64 | ... 65 | ``` 66 | 67 | Then I rename my JSON-LD document to reference the IPFS URI: 68 | 69 | ```json 70 | { 71 | "@context": "dweb:/ipfs/QmUFeUYXqyKa1mXLyfiCkm1MDbwYPTFBpyKvW7Nhy98Ks1", 72 | "@type": "Digest", 73 | "digestAlgorithm": "http://www.w3.org/2000/09/xmldsig#sha1", 74 | "digestValue": "981ec496092bf6ee18d6255d96069b528633268b" 75 | } 76 | ``` 77 | 78 | Let's even add _that_ document to IPFS, just for fun: 79 | 80 | ``` 81 | joel$ ipfs add sample.jsonld 82 | added QmXjY3nz81qG99vbMF4Tb2NeSFmUdWBUG7ecYVtvnGxrXt sample.jsonld 83 | joel$ ipfs cat QmXjY3nz81qG99vbMF4Tb2NeSFmUdWBUG7ecYVtvnGxrXt 84 | { 85 | "@context": "dweb:/ipfs/QmUFeUYXqyKa1mXLyfiCkm1MDbwYPTFBpyKvW7Nhy98Ks1", 86 | "@type": "Digest", 87 | "digestAlgorithm": "http://www.w3.org/2000/09/xmldsig#sha1", 88 | "digestValue": "981ec496092bf6ee18d6255d96069b528633268b" 89 | } 90 | ``` 91 | 92 | Then unleash the magic! 93 | 94 | ```javascript 95 | const createDocumentLoader = require("dweb-loader") 96 | const IPFS = require("ipfs") 97 | const jsonld = require("jsonld") 98 | 99 | const ipfs = new IPFS({}) 100 | ipfs.on("ready", () => { 101 | const doc = "dweb:/ipfs/QmXjY3nz81qG99vbMF4Tb2NeSFmUdWBUG7ecYVtvnGxrXt" 102 | const documentLoader = createDocumentLoader(ipfs) 103 | jsonld.expand(doc, { documentLoader }).then(expanded => { 104 | console.log(err, JSON.stringify(expanded)) 105 | process.exit() 106 | }) 107 | }) 108 | ``` 109 | 110 | ``` 111 | joel$ node test.js 112 | context: dweb:/ipld/zdpuB2s6SPPu2TPv6RBUY7FhJkghnYfc7dDvz5Luyw4wosde1 113 | document: ipfs://QmPm5sCx6HLSmdJHFrozmsVNxC6mrE3VMHH7XuTQmtqUuA 114 | { 115 | "@context": "dweb:/ipld/zdpuB2s6SPPu2TPv6RBUY7FhJkghnYfc7dDvz5Luyw4wosde1", 116 | "@graph": [ 117 | { 118 | "@type": "schema:Person", 119 | "schema:knows": { 120 | "@type": "schema:Person", 121 | "schema:name": "Bob" 122 | }, 123 | "schema:name": "Alice" 124 | } 125 | ], 126 | "prov:wasAttributedTo": { 127 | "@type": "schema:Person", 128 | "schema:name": "Eve" 129 | } 130 | } 131 | ``` 132 | 133 | ## Support 134 | 135 | This loader supports JSON-encoded contexts and documents on IPFS under the `dweb:/ipfs/` and `ipfs://` URI schemes (the loader will atempt to parse those files as JSON and will throw an error if that fails), as well as the `dag-cbor` and `dag-json` IPLD formats, which deserialize directly to JSON. 136 | --------------------------------------------------------------------------------