├── .gitignore ├── .prettierrc ├── .jshintrc ├── lib ├── client.js ├── list-members.js ├── list-remove.js ├── pager.test.js ├── decrement.js ├── decrement.test.js ├── list-add.js ├── pager.js ├── correspondents.test.js └── correspondents.js ├── serverless.yaml ├── package.json ├── index.js └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | .envrc 3 | .serverless -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "tabWidth": 2, 3 | "useTabs": false 4 | } 5 | -------------------------------------------------------------------------------- /.jshintrc: -------------------------------------------------------------------------------- 1 | { 2 | "curly": true, 3 | "eqeqeq": true, 4 | "immed": true, 5 | "latedef": false, 6 | "newcap": true, 7 | "noarg": true, 8 | "sub": true, 9 | "undef": true, 10 | "unused": true, 11 | "boss": true, 12 | "eqnull": true, 13 | "esversion": 6, 14 | "strict": false, 15 | "evil": true, 16 | "node": true, 17 | "browser": true, 18 | "globals": {}, 19 | "smarttabs": true 20 | } 21 | -------------------------------------------------------------------------------- /lib/client.js: -------------------------------------------------------------------------------- 1 | var Twitter = require("twitter"); 2 | 3 | // Setup twitter client 4 | var T = new Twitter({ 5 | consumer_key: process.env.TWITTER_OAUTH_CONSUMER_KEY, 6 | consumer_secret: process.env.TWITTER_OAUTH_CONSUMER_SECRET, 7 | access_token_key: process.env.TWITTER_OAUTH_ACCESS_TOKEN_KEY, 8 | access_token_secret: process.env.TWITTER_OAUTH_ACCESS_TOKEN_SECRET 9 | }); 10 | 11 | T.me = { 12 | screen_name: process.env.TWITTER_SCREEN_NAME, 13 | user_id: process.env.TWITTER_USER_ID 14 | }; 15 | 16 | module.exports = T; 17 | -------------------------------------------------------------------------------- /lib/list-members.js: -------------------------------------------------------------------------------- 1 | module.exports = members; 2 | 3 | const client = require("./client"); 4 | const pager = require("./pager").pager; 5 | 6 | function members(list, user) { 7 | return new Promise((resolve, reject) => { 8 | var users, members; 9 | 10 | users = []; 11 | 12 | members = pager(client, "lists/members", { 13 | count: 5000, 14 | slug: list, 15 | owner_screen_name: user || process.env.TWITTER_SCREEN_NAME 16 | }); 17 | 18 | members.then(members => { 19 | resolve(members.map(member => member.screen_name)); 20 | }); 21 | }); 22 | } 23 | -------------------------------------------------------------------------------- /lib/list-remove.js: -------------------------------------------------------------------------------- 1 | module.exports = remove; 2 | 3 | const client = require("./client"); 4 | 5 | function remove(list, accounts) { 6 | return new Promise((resolve, reject) => { 7 | client.post( 8 | "lists/members/destroy_all", 9 | { 10 | slug: list, 11 | owner_id: process.env.TWITTER_USER_ID, 12 | screen_name: accounts.join(",") 13 | }, 14 | function(err, res) { 15 | if (err) { 16 | reject(err); 17 | } 18 | resolve({ 19 | list: list, 20 | accounts: accounts 21 | }); 22 | } 23 | ); 24 | }); 25 | } 26 | -------------------------------------------------------------------------------- /lib/pager.test.js: -------------------------------------------------------------------------------- 1 | const { pager, getItems, getNext } = require("./pager"); 2 | 3 | test("getItems(array)", () => { 4 | expect(getItems(["a"])).toEqual(["a"]); 5 | }); 6 | 7 | test("getItems(object)", () => { 8 | expect(getItems({ a: false, b: ["a"] })).toEqual(["a"]); 9 | }); 10 | 11 | test("getNext", () => { 12 | expect(getNext({ next_cursor_str: "test" })).toEqual({ 13 | type: "cursor", 14 | value: "test" 15 | }); 16 | expect(getNext({ next_cursor: 0, next_cursor_str: "test" })).toEqual(false); 17 | expect(getNext([{ id_str: "2" }])).toEqual({ 18 | type: "max_id", 19 | value: "1" 20 | }); 21 | }); 22 | -------------------------------------------------------------------------------- /lib/decrement.js: -------------------------------------------------------------------------------- 1 | // Subtracts one from a number represented as a string of digits. 2 | module.exports = function strDecrement(str) { 3 | var i, 4 | numbers = str.split(""); 5 | 6 | // zero special case 7 | if (numbers.reduce((t, d) => +d + t, 0) === 0) { 8 | return "-1"; 9 | } 10 | 11 | // one special case 12 | if ( 13 | +numbers[numbers.length - 1] === 1 && 14 | numbers.reduce((t, d) => +d + t, 0) === 1 15 | ) { 16 | return "0"; 17 | } 18 | 19 | for (i = numbers.length; --i >= 0; ) { 20 | if (+numbers[i] === 0) { 21 | numbers[i] = 9; 22 | } else { 23 | numbers[i] = numbers[i] - 1; 24 | if (i === 0 && +numbers[i] === 0) { 25 | numbers.shift(); 26 | } 27 | break; 28 | } 29 | } 30 | return numbers.join(""); 31 | }; 32 | -------------------------------------------------------------------------------- /serverless.yaml: -------------------------------------------------------------------------------- 1 | service: list-cycle 2 | 3 | provider: 4 | name: google 5 | runtime: nodejs10 6 | project: sharp-cosmos-234905 7 | credentials: ~/.gcloud/the-list-cycle-744fccc90626.json 8 | environment: 9 | TWITTER_OAUTH_CONSUMER_KEY: ${env:TWITTER_OAUTH_CONSUMER_KEY} 10 | TWITTER_OAUTH_CONSUMER_SECRET: ${env:TWITTER_OAUTH_CONSUMER_SECRET} 11 | TWITTER_USER_ID: ${env:TWITTER_USER_ID} 12 | TWITTER_SCREEN_NAME: ${env:TWITTER_SCREEN_NAME} 13 | TWITTER_OAUTH_ACCESS_TOKEN_KEY: ${env:TWITTER_OAUTH_ACCESS_TOKEN_KEY} 14 | TWITTER_OAUTH_ACCESS_TOKEN_SECRET: ${env:TWITTER_OAUTH_ACCESS_TOKEN_SECRET} 15 | 16 | plugins: 17 | - serverless-google-cloudfunctions 18 | 19 | functions: 20 | first: 21 | handler: cycle 22 | events: 23 | - event: 24 | eventType: providers/cloud.pubsub/eventTypes/topic.publish 25 | resource: projects/sharp-cosmos-234905/topics/list-cycle 26 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "the-list-cycle", 3 | "version": "1.0.0", 4 | "description": "A little script which maintains a twitter list of people you've interacted with recently", 5 | "main": "index.js", 6 | "engines": { 7 | "node": ">=10.0.0", 8 | "npm": ">=6.0.0" 9 | }, 10 | "scripts": { 11 | "test": "jest" 12 | }, 13 | "repository": { 14 | "type": "git", 15 | "url": "git+https://github.com/drzax/the-list-cycle.git" 16 | }, 17 | "keywords": [ 18 | "twitter", 19 | "list", 20 | "automation" 21 | ], 22 | "author": "Simon Elvery", 23 | "license": "ISC", 24 | "bugs": { 25 | "url": "https://github.com/drzax/the-list-cycle/issues" 26 | }, 27 | "homepage": "https://github.com/drzax/the-list-cycle#readme", 28 | "dependencies": { 29 | "moment": "^2.26.0", 30 | "twitter": "^1.7.1" 31 | }, 32 | "devDependencies": { 33 | "jest": "^26.0.1", 34 | "serverless-google-cloudfunctions": "^2.4.3" 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /lib/decrement.test.js: -------------------------------------------------------------------------------- 1 | const decrement = require("./decrement"); 2 | 3 | test('subtracts one from "1"', () => { 4 | expect(decrement("1")).toBe("0"); 5 | }); 6 | 7 | test('subtracts one from "0"', () => { 8 | expect(decrement("0")).toBe("-1"); 9 | }); 10 | 11 | // TODO: netagive numbers 12 | // test('subtracts one from "-1"', () => { 13 | // expect(decrement("0")).toBe("-2"); 14 | // }); 15 | 16 | test('subtracts one from "2"', () => { 17 | expect(decrement("2")).toBe("1"); 18 | }); 19 | 20 | test('subtracts one from "10"', () => { 21 | expect(decrement("10")).toBe("9"); 22 | }); 23 | 24 | test('subtracts one from "11"', () => { 25 | expect(decrement("11")).toBe("10"); 26 | }); 27 | 28 | test('subtracts one from "29027834868298356902365129623"', () => { 29 | expect(decrement("29027834868298356902365129623")).toBe( 30 | "29027834868298356902365129622" 31 | ); 32 | }); 33 | 34 | test('subtracts one from "29027834868298356902365129621"', () => { 35 | expect(decrement("29027834868298356902365129621")).toBe( 36 | "29027834868298356902365129620" 37 | ); 38 | }); 39 | -------------------------------------------------------------------------------- /lib/list-add.js: -------------------------------------------------------------------------------- 1 | module.exports = add; 2 | 3 | const client = require("./client"); 4 | 5 | function add(list, accounts) { 6 | return new Promise((resolve, reject) => { 7 | var length = accounts.length; 8 | var delay = 500; 9 | var params = {}; 10 | 11 | // Work on a copy. 12 | accounts = accounts.slice(); 13 | 14 | params.screen_name = accounts.splice(0, 100).join(","); 15 | 16 | if (isNumeric(list)) { 17 | params.list_id = list; 18 | } else { 19 | params.slug = list; 20 | params.owner_screen_name = process.env.TWITTER_SCREEN_NAME; 21 | } 22 | 23 | client.post("lists/members/create_all", params, handleResult); 24 | 25 | function handleResult(err, res) { 26 | if (err) { 27 | reject(err); 28 | } 29 | 30 | if (accounts.length) { 31 | delay = delay * 2; 32 | params.screen_name = accounts.splice(0, 100).join(","); 33 | setTimeout( 34 | () => client.post("lists/members/create_all", params, handleResult), 35 | delay 36 | ); 37 | } else { 38 | resolve({ 39 | count: length, 40 | list: list 41 | }); 42 | } 43 | } 44 | }); 45 | } 46 | 47 | function isNumeric(n) { 48 | return !Number.isNaN(parseFloat(n)) && Number.isFinite(n); 49 | } 50 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | function cycle() { 2 | const { correspondents } = require("./lib/correspondents"); 3 | const listAdd = require("./lib/list-add"); 4 | const listRemove = require("./lib/list-remove"); 5 | const listMembers = require("./lib/list-members"); 6 | const list = process.env.TWITTER_LIST || "cycle"; 7 | 8 | var newMembers = correspondents(process.env.TWITTER_WINDOW || "1-month"); 9 | var oldMembers = listMembers(list); 10 | 11 | Promise.all([newMembers, oldMembers]).then(([newMembers, oldMembers]) => { 12 | const adding = newMembers.filter(name => oldMembers.indexOf(name) === -1); 13 | const removing = oldMembers.filter(name => newMembers.indexOf(name) === -1); 14 | // const keeping = oldMembers.filter(name => newMembers.indexOf(name) > -1); 15 | 16 | if (removing.length) { 17 | listRemove(list, removing).then( 18 | () => console.log(`Removed: ${removing.join(", ")}`), 19 | err 20 | ); 21 | } else { 22 | console.log("Nothing to remove."); 23 | } 24 | 25 | if (adding.length) { 26 | listAdd(list, adding).then( 27 | () => console.log(`Added: ${adding.join(", ")}`), 28 | err 29 | ); 30 | } else { 31 | console.log("Nothing to add."); 32 | } 33 | }, err); 34 | } 35 | 36 | function err(err) { 37 | console.error(err); 38 | } 39 | 40 | module.exports = { cycle }; 41 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # The List Cycle 2 | 3 | A little process which cycles Twitter users on and off a list based on interactions recency of interaction. 4 | 5 | The idea is that it will use one of your Twitter lists (by default, one called `cycle`) to keep a list of all the people you've interacted with recently (by default, the last month). 6 | 7 | The script considers someone you've 'interacted with' as being: 8 | 9 | - Anyone who you reply to, mention, retweet or subtweet. 10 | - Anyone who is mentioned in, or author of a tweet you favourite, retweet or subtweet. 11 | 12 | ## Deployment 13 | 14 | `npm install` 15 | 16 | ### Define the following environment variables. 17 | 18 | ``` 19 | TWITTER_OAUTH_CONSUMER_KEY= 20 | TWITTER_OAUTH_CONSUMER_SECRET= 21 | TWITTER_USER_ID= 22 | TWITTER_SCREEN_NAME= 23 | TWITTER_OAUTH_ACCESS_TOKEN_KEY= 24 | TWITTER_OAUTH_ACCESS_TOKEN_SECRET= 25 | TWITTER_WINDOW="1-month" # optional 26 | TWITTER_LIST=cycle # optional 27 | ``` 28 | 29 | One way to get these twitter API tokens is is to install [twitter-list-manager](https://github.com/drzax/twitter-list-manager) as a global NPM module and run `tw auth`. 30 | 31 | ### Setup a twitter list 32 | 33 | You'll also need to setup a list on Twitter for the use of the script. The expected default is 'cycle'. 34 | 35 | ### Setup a [Google Cloud Platform](https://cloud.google.com) project 36 | 37 | - Setup project as per the [serverless documentation](https://serverless.com/framework/docs/providers/google/guide/credentials/) 38 | - Setup a Pub/Sub topic called `list-cycle` 39 | - Setup a Cloud Scheduler job to message the `list-cycle` pub/sub topic on your chose interval (every hour recommended) 40 | 41 | ### Finally 42 | 43 | ``` 44 | serverless deploy 45 | ``` 46 | -------------------------------------------------------------------------------- /lib/pager.js: -------------------------------------------------------------------------------- 1 | const decrement = require("./decrement"); 2 | 3 | function pager(transport, endpoint, args, stop = () => false) { 4 | return new Promise((resolve, reject) => { 5 | var response = []; 6 | 7 | transport.get(endpoint, args, handleResult); 8 | 9 | function handleResult(err, res) { 10 | var next, items; 11 | 12 | // Reject on any error 13 | if (err) return reject(err); 14 | 15 | next = getNext(res); 16 | items = getItems(res); 17 | 18 | // Add new items to the response 19 | response = response.concat(items); 20 | 21 | if (next === false || stop(res)) { 22 | resolve(response); 23 | } else { 24 | args[next.type] = next.value; 25 | transport.get(endpoint, args, handleResult); 26 | } 27 | } 28 | }); 29 | } 30 | 31 | function getItems(res) { 32 | var key; 33 | 34 | if (Array.isArray(res)) { 35 | return res; 36 | } 37 | 38 | // This makes the possibly incorrect assumption that for API methods which 39 | // return a non-array result, there will be a single key on the object which 40 | // is the list of items we want returned (paged through). 41 | // This is the case for cursored methods, for example: 42 | // - lists/ownerships 43 | // - lists/members 44 | for (key in res) { 45 | if (Array.isArray(res[key])) { 46 | return res[key]; 47 | } 48 | } 49 | } 50 | 51 | function getNext(res) { 52 | if (res.next_cursor_str) { 53 | if (res.next_cursor === 0) { 54 | return false; 55 | } 56 | return { 57 | type: "cursor", 58 | value: res.next_cursor_str 59 | }; 60 | } 61 | 62 | return { 63 | type: "max_id", 64 | value: decrement(res[res.length - 1].id_str) 65 | }; 66 | } 67 | 68 | // Pages through Twitter API calls until criteria met 69 | module.exports = { pager, getItems, getNext }; 70 | -------------------------------------------------------------------------------- /lib/correspondents.test.js: -------------------------------------------------------------------------------- 1 | const { 2 | stopAt, 3 | filterByTime, 4 | filterTimeline, 5 | unique, 6 | compile 7 | } = require("./correspondents"); 8 | 9 | const testDate = new Date("Thu Apr 06 15:24:15 +0000 2017"); 10 | const before = { created_at: "Thu Apr 06 15:24:14 +0000 2017" }; 11 | const same = { created_at: "Thu Apr 06 15:24:15 +0000 2017" }; 12 | const after = { created_at: "Thu Apr 06 15:24:16 +0000 2017" }; 13 | 14 | test("stopAt", () => { 15 | const stopFn = stopAt(testDate); 16 | 17 | expect(stopFn([after])).toBe(false); 18 | expect(stopFn([same])).toBe(false); 19 | expect(stopFn([before])).toBe(true); 20 | }); 21 | 22 | test("filterByTime", () => { 23 | const filterFn = filterByTime(testDate); 24 | const filtered = filterFn([before, after, same]); 25 | expect(filtered).toEqual(expect.arrayContaining([after, same])); 26 | expect(filtered).not.toEqual(expect.arrayContaining([before])); 27 | }); 28 | 29 | test("filterTimeline", () => { 30 | const rt = { retweeted_status: {} }; 31 | const qt = { quoted_status: {} }; 32 | const r = { in_reply_to_screen_name: "name" }; 33 | const o = {}; 34 | 35 | expect(filterTimeline([rt, qt, r, o])).toEqual([rt, qt, r]); 36 | expect(filterTimeline([rt, qt, r, o])).not.toBe(expect.arrayContaining([o])); 37 | }); 38 | 39 | test("unique", () => { 40 | expect(unique(["a", "b", "c", "c"])).toEqual(["a", "b", "c"]); 41 | expect(unique(["a", "b", "c", "d"])).toEqual(["a", "b", "c", "d"]); 42 | }); 43 | 44 | test("compile", () => { 45 | const tweets = [ 46 | { 47 | user: { screen_name: "a" }, 48 | retweeted_status: { user: { screen_name: "b" } }, 49 | quoted_status: { user: { screen_name: "c" } }, 50 | in_reply_to_screen_name: "d", 51 | entities: { user_mentions: [{ screen_name: "e" }, { screen_name: "f" }] } 52 | } 53 | ]; 54 | 55 | expect(compile(tweets)).toEqual(["a", "b", "c", "d", "e", "f"]); 56 | }); 57 | -------------------------------------------------------------------------------- /lib/correspondents.js: -------------------------------------------------------------------------------- 1 | const client = require("./client"); 2 | const pager = require("./pager").pager; 3 | const moment = require("moment"); 4 | 5 | function correspondents(durationStr) { 6 | return new Promise((resolve, reject) => { 7 | // Parse duration 8 | const duration = durationStr ? durationStr.split("-") : []; 9 | const earlier = moment().subtract( 10 | +duration[0] || 1, 11 | duration[1] || "weeks" 12 | ); 13 | 14 | const stopFn = stopAt(earlier); 15 | const filterFn = filterByTime(earlier); 16 | 17 | const timeline = pager( 18 | client, 19 | "statuses/user_timeline", 20 | { count: 200 }, 21 | stopFn 22 | ) 23 | .then(filterFn) 24 | .then(filterTimeline) 25 | .then(compile); 26 | 27 | const favourites = pager(client, "favorites/list", { count: 200 }, stopFn) 28 | .then(filterFn) 29 | .then(compile); 30 | 31 | Promise.all([timeline, favourites]).then( 32 | ([timeline, favourites]) => { 33 | resolve(unique(timeline.concat(favourites))); 34 | }, 35 | err => reject(err) 36 | ); 37 | }); 38 | } 39 | 40 | // Get a list of all usernames associated with a tweet. 41 | function compile(statuses) { 42 | return statuses.reduce((t, r) => { 43 | t.push(r.user.screen_name); 44 | if (r.retweeted_status) t.push(r.retweeted_status.user.screen_name); 45 | if (r.quoted_status) t.push(r.quoted_status.user.screen_name); 46 | if (r.in_reply_to_screen_name) t.push(r.in_reply_to_screen_name); 47 | r.entities.user_mentions.forEach(mention => { 48 | t.push(mention.screen_name); 49 | }); 50 | return t; 51 | }, []); 52 | } 53 | 54 | // Make sure array of strings has only unique values 55 | function unique(arr) { 56 | return arr.reduce((t, i) => { 57 | if (t.indexOf(i) === -1) { 58 | t.push(i); 59 | } 60 | return t; 61 | }, []); 62 | } 63 | 64 | function filterTimeline(statuses) { 65 | return statuses.filter(r => { 66 | return ( 67 | !!r.retweeted_status || !!r.quoted_status || r.in_reply_to_screen_name 68 | ); 69 | }); 70 | } 71 | 72 | // Return a function which will filter statuses based on timestamp. 73 | function filterByTime(time) { 74 | return statuses => { 75 | return statuses.filter( 76 | status => moment(status.created_at, "ddd MMM DD HH:mm:ss ZZ YYYY") >= time 77 | ); 78 | }; 79 | } 80 | 81 | // Returns a function which returns a true or false when passed a twitter API response depending 82 | // on whether the response is before or after the given date/time. 83 | function stopAt(time) { 84 | return res => { 85 | return ( 86 | moment(res[res.length - 1].created_at, "ddd MMM DD HH:mm:ss ZZ YYYY") < 87 | time 88 | ); 89 | }; 90 | } 91 | 92 | module.exports = { 93 | correspondents, 94 | compile, 95 | unique, 96 | filterTimeline, 97 | filterByTime, 98 | stopAt 99 | }; 100 | --------------------------------------------------------------------------------