├── .gitignore ├── .npmignore ├── .npmrc ├── .github └── stale.yml ├── package.json ├── index.js ├── README.md ├── emit-links.js └── test └── emit-links.test.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | test/ 2 | .github/ -------------------------------------------------------------------------------- /.npmrc: -------------------------------------------------------------------------------- 1 | package-lock=false 2 | -------------------------------------------------------------------------------- /.github/stale.yml: -------------------------------------------------------------------------------- 1 | _extends: .github 2 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ssb-backlinks", 3 | "description": "scuttlebot plugin for indexing all link mentions of messages", 4 | "version": "2.1.1", 5 | "homepage": "https://github.com/ssbc/ssb-backlinks", 6 | "repository": { 7 | "type": "git", 8 | "url": "git://github.com/ssbc/ssb-backlinks.git" 9 | }, 10 | "dependencies": { 11 | "base64-url": "^2.3.3", 12 | "flumeview-links": "^2.1.0", 13 | "pull-stream": "^3.6.14", 14 | "ssb-ref": "^2.14.0" 15 | }, 16 | "devDependencies": { 17 | "ava": "^0.25.0", 18 | "sinon": "^7.1.1" 19 | }, 20 | "scripts": { 21 | "test": "ava" 22 | }, 23 | "author": "Secure Scuttlebutt Consortium", 24 | "license": "MIT" 25 | } 26 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var FlumeLinks = require('flumeview-links') 2 | var toUrlFriendly = require('base64-url').escape 3 | var emitLinks = require('./emit-links') 4 | 5 | var indexes = [ 6 | { key: 'DTS', value: [['dest'], ['timestamp']] }, 7 | { key: 'DTA', value: [['dest'], ['rts']] }, // asserted timestamp 8 | { key: 'TDT', value: [['value', 'content', 'type'], ['dest'], ['rts']] } 9 | ] 10 | 11 | var indexVersion = 11 12 | 13 | exports.name = 'backlinks' 14 | exports.version = require('./package.json').version 15 | exports.manifest = { 16 | read: 'source' 17 | } 18 | 19 | exports.init = function (ssb, config) { 20 | var index = ssb._flumeUse( 21 | `backlinks-${toUrlFriendly(ssb.id.slice(1, 10))}`, 22 | FlumeLinks(indexes, emitLinks, indexVersion) 23 | ) 24 | 25 | return { 26 | read: function (opts) { 27 | opts.unlinkedValues = true 28 | if (opts.index) { 29 | // backwards compatibility for opts.index sorting 30 | var sort = selectValueByKey(indexes, opts.index) 31 | if (sort) { 32 | opts.query = opts.query ? [].concat(opts.query) : [] 33 | opts.query.push({ 34 | $sort: sort 35 | }) 36 | } else { 37 | throw new Error('Invalid index: ' + opts.index) 38 | } 39 | } 40 | return index.read(opts) 41 | }, 42 | close: index.close 43 | } 44 | } 45 | 46 | function selectValueByKey (indexes, key) { 47 | for (var i = 0; i < indexes.length; i++) { 48 | if (indexes[i].key === key) { 49 | return indexes[i].value 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ssb-backlinks 2 | 3 | [scuttlebot](http://scuttlebutt.nz/) plugin for indexing all link mentions of messages (including private for the current identity). 4 | 5 | Walks all values of a message searching for keys recognized as feed, channel, message or blobs. Provides an [ssb-query](https://github.com/dominictarr/ssb-query) style interface. 6 | 7 | ## Example usage 8 | 9 | ```js 10 | const pull = require('pull-stream') 11 | function createBacklinkStream (id) { 12 | var filterQuery = { 13 | $filter: { 14 | dest: id 15 | } 16 | } 17 | 18 | return sbot.backlinks.read({ 19 | query: [filterQuery], 20 | index: 'DTA', // use asserted timestamps 21 | live: true 22 | }) 23 | } 24 | 25 | const msgKey = '%+zYA9WF9cY+HqGLzqS1H7FdUdK45tUmTqiZ85p+RNOQ=.sha256' 26 | var relatedMessages = [] 27 | 28 | pull( 29 | createBacklinkStream(msgKey), 30 | pull.filter(msg => !msg.sync), 31 | // note the 'live' style streams emit { sync: true } when they're up to date! 32 | pull.drain(msg => { 33 | relatedMessages.push(msg) 34 | }) 35 | ) 36 | ``` 37 | 38 | ## Example usage as a Secret-Stack Plugin 39 | `ssb-backlinks` can also be used as a `secret-stack` 40 | [plugin](https://github.com/ssbc/secret-stack/blob/master/plugins.md) directly. 41 | 42 | ```js 43 | var SecretStack = require('secret-stack') 44 | var config = require('./some-config-file') 45 | 46 | var {pull, drain} = require('pull-stream') 47 | 48 | // you'd need many more plugins to make this useful 49 | // demo purposes only 50 | var create = SecretStack({ 51 | appKey: appKey //32 random bytes 52 | }) 53 | .use(require('ssb-db')) 54 | .use(require('ssb-backlinks')) 55 | .use(function (sbot, config) { 56 | pull( 57 | sbot.backlinks.read({ 58 | query: [{$filter: {dest: "%dfadf..."}}], // some message hash 59 | index: 'DTA', 60 | live: true 61 | }), 62 | drain(console.log) 63 | ) 64 | )} 65 | 66 | var server = create(config) // start application 67 | ``` 68 | 69 | ## Versions 70 | 71 | Please note that 0.7.0 requires scuttlebot 11.3 72 | 73 | ## License 74 | 75 | MIT 76 | -------------------------------------------------------------------------------- /emit-links.js: -------------------------------------------------------------------------------- 1 | var ref = require('ssb-ref') 2 | var matchChannel = /^#[^\s#]+$/ 3 | 4 | module.exports = emitLinks 5 | 6 | const UNKNOWN = 0 7 | const FEED = 1 8 | const MSG = 1 9 | const CLOAKED = 1 10 | const BLOB = 1 11 | const CHANNEL = 2 12 | 13 | // we don't need invite and address types, so don't use ref.type as those are quite slow 14 | function type (id) { 15 | if (typeof id !== 'string') return UNKNOWN 16 | 17 | var c = id.charAt(0) 18 | if (c === '@' && ref.isFeedId(id)) return FEED 19 | else if (c === '%' && ref.isMsgId(id)) return MSG 20 | else if (c === '%' && ref.isCloakedMsgId(id)) return CLOAKED 21 | else if (c === '&' && ref.isBlobId(id)) return BLOB 22 | else if (c === '#' && isChannel(id)) return CHANNEL 23 | else return UNKNOWN 24 | } 25 | 26 | function emitLinks (msg, emit) { 27 | var links = new Set() 28 | walk(msg.value.content, function (path, value) { 29 | // HACK: handle legacy channel mentions 30 | if (Array.isArray(path) && path[0] === 'channel') { 31 | var channel = ref.normalizeChannel(value) 32 | if (channel) { 33 | value = `#${channel}` 34 | } 35 | } 36 | 37 | var idType = type(value) 38 | 39 | if (idType === CHANNEL) 40 | links.add(`#${ref.normalizeChannel(value.slice(1))}`) 41 | else if (idType !== UNKNOWN) 42 | links.add(value) 43 | }) 44 | 45 | const rts = resolveTimestamp(msg) 46 | links.forEach(link => { 47 | emit(Object.assign({}, msg, { rts, dest: link })) 48 | }) 49 | } 50 | 51 | function isChannel (value) { 52 | return ( 53 | typeof value === 'string' && 54 | value.length < 30 && 55 | matchChannel.test(value) 56 | ) 57 | } 58 | 59 | function resolveTimestamp (msg) { 60 | if (!msg || !msg.value || !msg.value.timestamp) return 61 | if (msg.timestamp) { 62 | return Math.min(msg.timestamp, msg.value.timestamp) 63 | } else { 64 | return msg.value.timestamp 65 | } 66 | } 67 | 68 | function walk (obj, fn, prefix) { 69 | if (obj && typeof obj === 'object') { 70 | for (var k in obj) { 71 | walk(obj[k], fn, (prefix || []).concat(k)) 72 | } 73 | } else { 74 | fn(prefix, obj) 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /test/emit-links.test.js: -------------------------------------------------------------------------------- 1 | const test = require('ava') 2 | const sinon = require('sinon') 3 | const ref = require('ssb-ref') 4 | const emitLinks = require('../emit-links') 5 | 6 | let emit = sinon.stub() 7 | 8 | test.before(() => { 9 | sinon.stub(ref, 'normalizeChannel') 10 | }) 11 | 12 | test.afterEach(() => { 13 | ref.normalizeChannel.reset() 14 | emit.reset() 15 | }) 16 | 17 | test.after.always(() => { 18 | ref.normalizeChannel.restore() 19 | }) 20 | 21 | test('emitLinks does not call emit fn if there are no links', (t) => { 22 | const msg = { value: { content: {} } } 23 | emitLinks(msg, emit) 24 | t.true(emit.notCalled) 25 | }) 26 | 27 | test('emitLinks calls emit fn if links (feedId) are found in content', (t) => { 28 | const msg = { value: { content: { contact: '@6CAxOI3f+LUOVrbAl0IemqiS7ATpQvr9Mdw9LC4+Uv0=.ed25519' } } } 29 | emitLinks(msg, emit) 30 | t.true(emit.calledOnce) 31 | }) 32 | 33 | test('emitLinks calls emit fn if links are found in content (array)', (t) => { 34 | const msg = { value: { content: { contacts: ['@6CAxOI3f+LUOVrbAl0IemqiS7ATpQvr9Mdw9LC4+Uv0=.ed25519'] } } } 35 | emitLinks(msg, emit) 36 | t.true(emit.calledOnce) 37 | }) 38 | 39 | test('emitLinks calls emit fn if links (msgId) are found in content', (t) => { 40 | const msg = { value: { content: { contact: '%6CAxOI3f+LUOVrbAl0IemqiS7ATpQvr9Mdw9LC4+Uv0=.sha256' } } } 41 | emitLinks(msg, emit) 42 | t.true(emit.calledOnce) 43 | }) 44 | 45 | test('emitLinks calls emit fn if links (cloakedMsgId) are found in content', (t) => { 46 | const msg = { value: { content: { contact: '%6CAxOI3f+LUOVrbAl0IemqiS7ATpQvr9Mdw9LC4+Uv0=.cloaked' } } } 47 | emitLinks(msg, emit) 48 | t.true(emit.calledOnce) 49 | }) 50 | 51 | test('emitLinks does not call emit fn for duplicate links', (t) => { 52 | const msg = { value: { content: { contact: '@6CAxOI3f+LUOVrbAl0IemqiS7ATpQvr9Mdw9LC4+Uv0=.ed25519', contact2: '@6CAxOI3f+LUOVrbAl0IemqiS7ATpQvr9Mdw9LC4+Uv0=.ed25519' } } } 53 | emitLinks(msg, emit) 54 | t.true(emit.calledOnce) 55 | }) 56 | 57 | test('emitLinks should normalize the link if it is a channel', (t) => { 58 | const msg = { value: { content: { channel: 'channel-name' } } } 59 | ref.normalizeChannel.returnsArg(0) 60 | emitLinks(msg, emit) 61 | t.true(ref.normalizeChannel.firstCall.calledWith('channel-name')) 62 | t.true(emit.calledOnce) 63 | }) 64 | 65 | test('emitLinks should call emit fn with added rts and dest fields', (t) => { 66 | const msg = { value: { content: { contact: '@6CAxOI3f+LUOVrbAl0IemqiS7ATpQvr9Mdw9LC4+Uv0=.ed25519' }, timestamp: 1 } } 67 | emitLinks(msg, emit) 68 | t.true(emit.calledWith(Object.assign({}, msg, { 69 | rts: 1, 70 | dest: '@6CAxOI3f+LUOVrbAl0IemqiS7ATpQvr9Mdw9LC4+Uv0=.ed25519' 71 | }))) 72 | }) 73 | --------------------------------------------------------------------------------