├── .gitignore ├── .sample.env ├── .travis.yml ├── LICENSE.md ├── README.md ├── cli.js ├── hypertweet.png ├── index.js └── package.json /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | 5 | # Runtime data 6 | pids 7 | *.pid 8 | *.seed 9 | 10 | # Directory for instrumented libs generated by jscoverage/JSCover 11 | lib-cov 12 | 13 | # Coverage directory used by tools like istanbul 14 | coverage 15 | 16 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 17 | .grunt 18 | 19 | # node-waf configuration 20 | .lock-wscript 21 | 22 | # Compiled binary addons (http://nodejs.org/api/addons.html) 23 | build/Release 24 | 25 | # Dependency directory 26 | # https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git 27 | node_modules 28 | 29 | .env -------------------------------------------------------------------------------- /.sample.env: -------------------------------------------------------------------------------- 1 | TWITTER_CONSUMER_KEY=your_token 2 | TWITTER_CONSUMER_SECRET=your_token 3 | TWITTER_ACCESS_TOKEN_KEY=your_token 4 | TWITTER_ACCESS_TOKEN_SECRET=your_token 5 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - '4' 4 | - '6' 5 | cache: 6 | directories: 7 | - node_modules 8 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | # [MIT License](https://spdx.org/licenses/MIT) 2 | 3 | Copyright (c) 2017 Joe Hand 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 10 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # hypertweet 2 | 3 | Proof of concept Twitter + [Dat](https://datproject.org) feed stream thingy. 4 | 5 | Streams your twitter feed to a [hypercore](https://github.com/mafintosh/hypercore) feed. Pretty print your feed or stream it anywhere with [hyperpipe](https://github.com/mafintosh/hyperpipe). 6 | 7 | ![hypertweet](https://raw.githubusercontent.com/joehand/hypertweet/master/hypertweet.png) 8 | 9 | ## Install 10 | 11 | ```sh 12 | npm install -g hypertweet 13 | 14 | hypertweet --help # see help! 15 | ``` 16 | 17 | ### Set twitter tokens 18 | 19 | [Get Twitter developer key](https://apps.twitter.com/) and either set them as environment variables or copy `.sample.env` to `.env` file. 20 | 21 | ``` 22 | process.env.TWITTER_CONSUMER_KEY, 23 | process.env.TWITTER_CONSUMER_SECRET 24 | process.env.TWITTER_ACCESS_TOKEN_KEY, 25 | process.env.TWITTER_ACCESS_TOKEN_SECRET 26 | ``` 27 | 28 | ## Usage 29 | 30 | Use `hypertweet --help` to see all the options. 31 | 32 | ### Collect Feed Data 33 | 34 | Streams your feed into a hypercore feed: 35 | 36 | ```sh 37 | ❯ hypertweet --dir=/joe/my-twitter-data/ 38 | sharing 0c3ec59a8111fde379e7ef82e6610ec02daf6bd3b704f41554cd5fe76afd5cc4 39 | 40 | 2017-04-21T02:20:24.205Z tweet by: 'some twitter user' 41 | 2017-04-21T02:20:25.000Z tweet by: 'more chirps' 42 | 2017-04-21T02:20:30.043Z tweet by: 'asdf' 43 | ``` 44 | 45 | ### Print anywhere 46 | 47 | Pretty print it in another terminal (or another computer anywhere): 48 | 49 | ```sh 50 | ❯ hypertweet 0c3ec59a8111fde379e7ef82e6610ec02daf6bd3b704f41554cd5fe76afd5cc4 51 | 52 | # pretty tweets here 53 | ``` 54 | 55 | ### Or pipe 56 | 57 | Pipe anywhere to anything. 58 | 59 | ```sh 60 | npm install -g hyperpipe 61 | hyperpipe /db 0c3ec59a8111fde379e7ef82e6610ec02daf6bd3b704f41554cd5fe76afd5cc4 > data.json 62 | ``` 63 | 64 | ## API 65 | 66 | ### `hypertweet(dir|storage, [opts], callback(err, feed))` 67 | 68 | Create a stream from the twitter API and share via hypercore `feed`. Automatically joins network via `discovery-swarm`. 69 | 70 | * `dir|storage`: directory or random access module, e.g `random-access-memory`. 71 | * `opts.streamUrl`: the [twitter streaming api](https://dev.twitter.com/streaming/overview) endpoint you want. defaults to user. 72 | * `opts.streamOpts`: stream opts 73 | 74 | ## License 75 | 76 | [MIT](LICENSE.md) 77 | -------------------------------------------------------------------------------- /cli.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | var ram = require('random-access-memory') 4 | var hypercore = require('hypercore') 5 | var discovery = require('hyperdiscovery') 6 | var minimist = require('minimist') 7 | var chalk = require('chalk') 8 | var moment = require('moment') 9 | var dotenv = require('dotenv') 10 | var hypertweet = require('.') 11 | 12 | var argv = minimist(process.argv.slice(2), { 13 | alias: {help: 'h', dir: 'd'}, 14 | default: { 15 | dir: process.cwd() 16 | }, 17 | boolean: ['help', 'log'] 18 | }) 19 | dotenv.load() 20 | 21 | if (argv.help) { 22 | console.error('hypertweet! streaming twitter api -> hypercore') 23 | console.error('') 24 | console.error(' hypertweet create a hypertweet feed + share') 25 | console.error(' --dir /data directory to store hypercore feed') 26 | console.error(' --log display new tweets, useful for debugging') 27 | console.error(' --url specify a streaming twitter endpoint') 28 | console.error(' hypertweet pretty print a shared hypertweet feed') 29 | console.error('') 30 | console.error('Also check out https://hypertweet.glitch.me/ & remix!! =)') 31 | process.exit() 32 | } 33 | 34 | // tail hypercore stream + print 35 | if (argv._[0]) { 36 | tailStream() 37 | } else { 38 | // stream twitter data -> hypercore 39 | var opts = { 40 | streamUrl: argv.url 41 | } 42 | hypertweet(argv.dir, opts, function (err, feed) { 43 | if (err) throw err 44 | console.log('Tweets streaming at:', feed.key.toString('hex')) 45 | if (argv.log) { 46 | var stream = feed.createReadStream({tail: true, live: true}) 47 | stream.on('data', function (tweet) { 48 | console.log(`${new Date().toLocaleString()} tweet by: ${tweet.user.name}`) 49 | }) 50 | } 51 | }) 52 | } 53 | 54 | function tailStream () { 55 | var key = argv._[0] 56 | var feed = hypercore(ram, key, {sparse: true, valueEncoding: 'json'}) 57 | 58 | feed.ready(function () { 59 | console.log(chalk.magenta.bold('HyperTweet!!')) 60 | 61 | discovery(feed) 62 | feed.update(function () { 63 | var stream = feed.createReadStream({ 64 | live: true, 65 | tail: true 66 | }) 67 | stream.on('data', print) 68 | stream.on('error', function (err) { 69 | console.error(err) 70 | process.exit(1) 71 | }) 72 | }) 73 | }) 74 | 75 | function print (tweet) { 76 | console.log(`${chalk.blue.bold(tweet.user.name)} (@${tweet.user.screen_name})`) 77 | console.log(tweet.text, '\n') 78 | var date = new Date(tweet.created_at) 79 | console.log(chalk.dim(moment(date).fromNow())) 80 | console.log('\n\n') 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /hypertweet.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/joehand/hypertweet/45885febe667ddf38ebc4c90605adbd700ea1116/hypertweet.png -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var hypercore = require('hypercore') 2 | var Chirp = require('chirp-stream') 3 | var discovery = require('hyperdiscovery') 4 | var assert = require('assert') 5 | 6 | module.exports = archiverStream 7 | 8 | function archiverStream (dir, opts, cb) { 9 | if (typeof opts === 'function') cb = opts 10 | assert.ok(dir, 'hypertweet: dir/storage required') 11 | assert.ok(cb, 'hypertweet: callback required') 12 | if (!opts) opts = {} 13 | 14 | var storage = dir 15 | var streamUrl = opts.streamUrl || 'https://userstream.twitter.com/1.1/user.json' 16 | var twitter = Chirp({ 17 | consumer: { 18 | public: process.env.TWITTER_CONSUMER_KEY, 19 | secret: process.env.TWITTER_CONSUMER_SECRET 20 | }, 21 | token: { 22 | public: process.env.TWITTER_ACCESS_TOKEN_KEY, 23 | secret: process.env.TWITTER_ACCESS_TOKEN_SECRET 24 | } 25 | }) 26 | 27 | var tweetstream = twitter.stream(streamUrl, opts.streamOpts) 28 | var userFeed = hypercore(storage, {valueEncoding: 'json'}) 29 | 30 | userFeed.ready(function (err) { 31 | if (err) return cb(err) 32 | discovery(userFeed) 33 | cb(null, userFeed) 34 | }) 35 | 36 | tweetstream.on('json', function (data) { 37 | if (!data.user) return // ignore non-tweet stuff 38 | userFeed.append(data, function (err) { 39 | if (err) return console.error(err) 40 | }) 41 | }) 42 | 43 | tweetstream.on('end', function () { 44 | console.error('tweetstream end') 45 | }) 46 | 47 | tweetstream.on('error', function (err) { 48 | console.error(err) 49 | }) 50 | } 51 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "hypertweet", 3 | "description": "tweet really fast p2p ?", 4 | "version": "0.3.0", 5 | "author": "Joe Hand ", 6 | "bugs": { 7 | "url": "https://github.com/joehand/hypertweet/issues" 8 | }, 9 | "bin": { 10 | "hypertweet": "cli.js" 11 | }, 12 | "devDependencies": { 13 | "standard": "*", 14 | "tap-spec": "^4.0.2", 15 | "tape": "^4.0.0" 16 | }, 17 | "homepage": "https://github.com/joehand/hypertweet", 18 | "keywords": [ 19 | "dat", 20 | "hypercore" 21 | ], 22 | "license": "MIT", 23 | "main": "index.js", 24 | "repository": { 25 | "type": "git", 26 | "url": "https://github.com/joehand/hypertweet.git" 27 | }, 28 | "scripts": { 29 | "test": "standard" 30 | }, 31 | "dependencies": { 32 | "chalk": "^1.1.3", 33 | "chirp-stream": "^1.0.1", 34 | "dotenv": "^4.0.0", 35 | "hypercore": "^5.11.2", 36 | "hyperdiscovery": "^6.0.1", 37 | "minimist": "^1.2.0", 38 | "moment": "^2.18.1", 39 | "random-access-memory": "^2.4.0" 40 | } 41 | } 42 | --------------------------------------------------------------------------------