├── .gitignore ├── README.md └── datasources ├── basic-youtube-actions ├── manifest.json ├── plugin.js └── schema.json ├── crypto-articles ├── aggregations.json ├── manifest.json ├── plugin.js └── schema.json ├── daft-message-sent ├── manifest.json ├── plugin.js └── schema.json ├── facebook-time-per-page ├── manifest.json ├── plugin.js └── schema.json ├── generic-image-labeling ├── manifest.json ├── plugin.js └── schema.json ├── google-search-clicks ├── manifest.json ├── plugin.js └── schema.json ├── highlight-difficult-words ├── manifest.json ├── plugin.js └── schema.json ├── imgur-basic-interaction ├── manifest.json ├── plugin.js └── schema.json ├── interpals-text ├── manifest.json ├── plugin.js └── schema.json ├── irish-news-site-usage ├── manifest.json ├── plugin.js └── schema.json ├── metro-dwell-time ├── manifest.json ├── plugin.js └── schema.json ├── news-article-opened ├── manifest.json ├── plugin.js └── schema.json ├── news-dwell-and-scroll ├── aggregations.json ├── manifest.json ├── plugin.js └── schema.json ├── omny-label ├── manifest.json ├── plugin.js └── schema.json ├── quote-highlight ├── manifest.json ├── plugin.js └── schema.json ├── reddit-basic-votes ├── manifest.json ├── plugin.js └── schema.json ├── reddit-time-between-actions ├── manifest.json ├── plugin.js └── schema.json ├── stack-overflow-questions ├── aggregations.json ├── manifest.json ├── plugin.js └── schema.json ├── study-materials ├── aggregations.json ├── manifest.json ├── plugin.js └── schema.json ├── text-translation ├── manifest.json ├── plugin.js └── schema.json ├── twitter-test-again ├── manifest.json ├── plugin.js └── schema.json ├── youtube-music ├── aggregations.json ├── manifest.json ├── plugin.js └── schema.json └── youtube-soundtracks ├── aggregations.json ├── manifest.json ├── plugin.js └── schema.json /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Metro 2 | 3 | Tech companies around the world are capitilizing on your online activity data. Why shouldn't you? 4 | 5 | Metro's Chrome Extension lets you track *your own* activity in the browser. Each of these DataSources lets you capture a different kind of activity - the news articles you read, the music you listen to, etc.. 6 | 7 | The Metro [website](https://getmetro.co) lets you combine your activity with your friends or online community, to create shared Activity Feeds about whatever you want! 8 | 9 | ## Making your own DataSources 10 | Yes, you can make your own DataSources. If you'd like to learn how, please get in touch with me personally at rory@getmetro.co 11 | 12 | ## Popular Public Feeds 13 | [r/HipHopHeads](https://getmetro.co/feeds/hiphopheads/) (Spotify) 14 | [r/Techno](https://getmetro.co/feeds/rtechno/) (Spotify) 15 | -------------------------------------------------------------------------------- /datasources/basic-youtube-actions/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "basic-youtube-actions", 3 | "author": "Conor", 4 | "description": "This runs on all youtube pages and collects the time a video is played, paused or finished at, along with the page title and URL.", 5 | "version": "0.2", 6 | "sites": ["^.*youtube\\.com.*$"] 7 | } 8 | -------------------------------------------------------------------------------- /datasources/basic-youtube-actions/plugin.js: -------------------------------------------------------------------------------- 1 | const basicYoutubeActions = { 2 | mc: null, 3 | name: 'basic-youtube-actions', 4 | 5 | registerEventsHandler: function(node) { 6 | // First we need to get the video player element: 7 | let $videoPlayer = $('#movie_player'); 8 | // There is a class that is dynamically changed on this element. Either 9 | // "paused-mode" or "playing-mode" or "ended-mode". We are going to 10 | // setInterval on this and monitor it for changes to the class. When it 11 | // changes, we fire that event. 12 | let currentEvent = ""; 13 | const self = this; 14 | 15 | setInterval(function() { 16 | let playing = $videoPlayer.hasClass("playing-mode"); 17 | let paused = $videoPlayer.hasClass("paused-mode"); 18 | let ended = $videoPlayer.hasClass("ended-mode"); 19 | 20 | if(playing && currentEvent != "play") { 21 | self.sendDatapoint("play"); 22 | currentEvent = "play"; 23 | } if(paused && currentEvent != "pause") { 24 | self.sendDatapoint("pause"); 25 | currentEvent = "pause"; 26 | } if(ended && currentEvent != "finished") { 27 | self.sendDatapoint("finished"); 28 | currentEvent = "finished"; 29 | } 30 | }, 250); 31 | }, 32 | 33 | sendDatapoint: function(eventType) { 34 | // Create the datapoint: 35 | let datapoint = {}; 36 | datapoint['event'] = eventType; 37 | datapoint['time'] = Date.now(); 38 | datapoint['url'] = window.location.href; 39 | datapoint['title'] = document.title; 40 | 41 | // Log it 42 | console.log(datapoint); 43 | 44 | // And ship it off: 45 | this.mc.sendDatapoint(datapoint); 46 | }, 47 | 48 | initDataSource: function(metroClient) { 49 | // The entrypoint 50 | this.mc = metroClient; 51 | console.log("Beginning YouTube actions DataSource."); 52 | 53 | // Set up the event listeners on the page. 54 | this.registerEventsHandler(document.body); 55 | } 56 | } 57 | 58 | registerDataSource(basicYoutubeActions); 59 | -------------------------------------------------------------------------------- /datasources/basic-youtube-actions/schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "event": "[play|pause|finished]", 3 | "time": "1234567890", 4 | "url": "The current page URL", 5 | "title": "The page title" 6 | } 7 | -------------------------------------------------------------------------------- /datasources/crypto-articles/aggregations.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "Top Articles", 4 | "function": "_.chain(feed).groupBy(function(o) {return o.data._str;}).map((value, key) => {let result = value[0];result.data['primaryRank'] = value.length; result.data['primaryRankDisplay'] = value.length + ' times';return result;}).sortBy(function(item) {return -item.data.primaryRank;}).value()", 5 | "display_icon": "fas fa-bolt" 6 | }, 7 | { 8 | "name": "Top Authors", 9 | "function": "_.chain(feed).groupBy(function(o) {return o.data.author;}).filter(item => item[0].data.author.length > 0).map((value, key) => {let result = value[0];result.data['primaryRank'] = value.length; result.data['_str'] = result.data.author; result.data['primaryRankDisplay'] = result.data.primaryRank + ' articles';return result;}).sortBy(function(item) {return -item.data.primaryRank;}).value()", 10 | "display_icon": "fas fa-bolt" 11 | }, 12 | { 13 | "name": "Top Publications", 14 | "function": "_.chain(feed).groupBy(function(o) {return o.data.publication;}).filter(item => item[0].data.publication.length > 0).map((value, key) => {let result = value[0];result.data['primaryRank'] = value.length; result.data['_str'] = result.data.publication; result.data['primaryRankDisplay'] = result.data.primaryRank + ' articles'; return result;}).sortBy(function(item) {return -item.data.primaryRank;}).value()", 15 | "display_icon": "fas fa-bolt" 16 | }, 17 | { 18 | "name": "Top Keywords", 19 | "function": "_.chain(feed).reduce((memo, point) => {const result = memo.concat(point.data.keywords);return result;}, []).reduce((memo, val) => {if(memo.hasOwnProperty(val)) {memo[val] += 1;} else {memo[val] = 1;}return memo;}, {}).map((value, key) => {let result = {'data': {'primaryRank': value, '_str': key, '_key': key, 'primaryRankDisplay': value + ' appearances'}}; return result;}).sortBy(item => -item.data.primaryRank).value()", 20 | "display_icon": "fas fa-bolt" 21 | }, 22 | { 23 | "name": "Top Coins", 24 | "function": "_.chain(feed).reduce((memo, point) => {const result = memo.concat(point.data.coins);return result;}, []).reduce((memo, val) => {if(memo.hasOwnProperty(val)) {memo[val] += 1;} else {memo[val] = 1;}return memo;}, {}).map((value, key) => {let result = {'data': {'primaryRank': value, '_str': key, '_key': key, 'primaryRankDisplay': value + ' appearances', '_url': ''}}; return result;}).sortBy(item => -item.data.primaryRank).value()", 25 | "display_icon": "fas fa-bolt" 26 | }, 27 | { 28 | "name": "Completion Rate", 29 | "function": "_.chain(feed).groupBy(function(o) {return o.data._str;}).map((value, key) => {let result = value[0]; let count = value.length; let total = _.reduce(value, (memo, point) => { return memo + point.data.scrollPercentage }, 0); let avg = Math.round(total/count*100)/100; result.data['primaryRank'] = avg; result.data['primaryRankDisplay'] = avg + '% read'; return result; }).sortBy((item) => {return -item.data.primaryRank}).value()", 30 | "display_icon": "fas fa-bolt" 31 | } 32 | ] -------------------------------------------------------------------------------- /datasources/crypto-articles/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "crypto-articles", 3 | "title": "Crypto Articles", 4 | "author": "Rory", 5 | "description": "This DataSource records how long you spend on a crypto article, the % you scroll down the page, and some other misc information such as the coins mentioned in the article.", 6 | "version": "1.1.0", 7 | "sites": [ 8 | "(:\\/\\/|.*\\.)coindesk\\.com.*", 9 | "(:\\/\\/|.*\\.)cointelegraph\\.com.*", 10 | "(:\\/\\/|.*\\.)bitcoin\\.com.*", 11 | "(:\\/\\/|.*\\.)bitcoinist\\.com.*", 12 | "(:\\/\\/|.*\\.)blockonomi\\.com.*", 13 | "(:\\/\\/|.*\\.)cryptoslate\\.com.*", 14 | "(:\\/\\/|.*\\.)newsbtc\\.com.*", 15 | "(:\\/\\/|.*\\.)cryptovest\\.com.*", 16 | "(:\\/\\/|.*\\.)ethereumworldnews\\.com.*", 17 | "(:\\/\\/|.*\\.)coinhooked\\.com.*" 18 | ] 19 | } 20 | -------------------------------------------------------------------------------- /datasources/crypto-articles/plugin.js: -------------------------------------------------------------------------------- 1 | function getMetaOrBlank(metas, name) { 2 | try { 3 | const el = metas.find((val, idx) => { 4 | const propertyAttr = val.getAttribute('property') 5 | const nameAttr = val.getAttribute('name') 6 | if(propertyAttr && propertyAttr.includes(name)) { 7 | return true 8 | } else if(nameAttr && nameAttr.includes(name)) { 9 | return true 10 | } 11 | return false 12 | }) 13 | 14 | if(el !== undefined) { 15 | return el.content 16 | } 17 | } catch(e) { 18 | console.error(e) 19 | return "" 20 | } 21 | return "" 22 | } 23 | 24 | const nonCryptoAcronyms = [ 25 | 'ETF', 26 | 'ICO', 27 | 'CEO', 28 | 'CTO', 29 | 'CFO', 30 | 'CMO', 31 | 'COO', 32 | 'FTP', 33 | 'GTD', 34 | 'PDF', 35 | 'DNS', 36 | 'XML', 37 | 'GUI', 38 | 'VPN', 39 | 'USD', 40 | 'GBP', 41 | 'USA', 42 | 'API', 43 | 'FAQ', 44 | ] 45 | 46 | const cryptoWords = [ 47 | 'crypto', 48 | 'bitcoin', 49 | 'blockchain', 50 | 'dapp', 51 | 'ico', 52 | 'altcoin', 53 | 'ethereum', 54 | 'cryptocurrency', 55 | 'cryptocurrencies', 56 | 'smart contract', 57 | ] 58 | 59 | function isCryptoRelated(content) { 60 | let words; 61 | if(Array.isArray(content)) { 62 | words = content 63 | } else if(typeof(content) === 'string') { 64 | words = content.split(' '); 65 | } 66 | 67 | for(var i=0; i 0 ? keywordsStr.split(',') : [] 113 | const publication = window.location.hostname; 114 | 115 | let datapoint = { 116 | _url: URL, 117 | _image: image, 118 | _str: title, 119 | _timestamp: Date.now(), 120 | _action: "Read", 121 | author: author, 122 | title: title, 123 | keywords: keywords, 124 | coins: coins, 125 | description: description, 126 | loadTime: loadTime, 127 | leaveTime: leaveTime, 128 | scrollPercentage: Math.round(scrollPercentage), 129 | publication: publication 130 | } 131 | 132 | if(this.isValid(datapoint)) { 133 | this.mc.sendDatapoint(datapoint) 134 | 135 | // To make sure the request fires 136 | var t = Date.now() + 200; 137 | while(Date.now() < t) {}; 138 | } 139 | }.bind(this)); 140 | }, 141 | 142 | isValid: (datapoint) => { 143 | try { 144 | const urlObj = new URL(datapoint._url) 145 | if(urlObj.pathname.startsWith('/tags')) { 146 | return false 147 | } 148 | } catch(e) { 149 | return false 150 | } 151 | 152 | return datapoint._str 153 | && datapoint._url 154 | && datapoint._timestamp 155 | && datapoint._action 156 | && datapoint._image 157 | 158 | }, 159 | 160 | getCoins: async function() { 161 | let coins = []; 162 | $('p').each((idx, p) => { 163 | let newCoins = ($(p).html().match(new RegExp(/\b[A-Z]{3}\b/, 'g')) || []) 164 | 165 | coins = coins.concat(newCoins) 166 | }) 167 | const uniqueCoins = [...new Set(coins)] 168 | const coinInfo = await Promise.all( 169 | uniqueCoins.map(async (c) => { 170 | const result = await fetch(`https://min-api.cryptocompare.com/data/price?fsym=${c}&tsyms=USD&api_key=2fb48558fb7e0241a083eea864cf13f46e7e2a5970410b9856d02c8d758507f8`) 171 | .then(res => res.json()) 172 | .then(data => { 173 | return data.hasOwnProperty('USD') && !(nonCryptoAcronyms.includes(c)) 174 | }) 175 | return [c, result] 176 | }) 177 | ) 178 | const realCoins = coinInfo.filter(cp => cp[1] === true).map(cp => cp[0]) 179 | console.debug(realCoins) 180 | return realCoins 181 | }, 182 | 183 | isValidCoin: (coin) => { 184 | return fetch(`https://min-api.cryptocompare.com/data/price?fsym=${coin}&tsyms=USD&api_key=2fb48558fb7e0241a083eea864cf13f46e7e2a5970410b9856d02c8d758507f8`) 185 | .then(res => res.json()) 186 | .then(data => { 187 | return data.hasOwnProperty('USD') 188 | }) 189 | }, 190 | 191 | initDataSource: function(metroClient) { 192 | this.mc = metroClient; 193 | let contentType = $('meta[property="og:type"]').attr('content'); 194 | 195 | let pCount = $('p').length 196 | let articleCount = $('article').length 197 | 198 | if(contentType == "article" && (articleCount > 0 || pCount > 5)) { 199 | this.start(); 200 | } 201 | 202 | } 203 | } 204 | 205 | registerDataSource(CryptoArticles); 206 | -------------------------------------------------------------------------------- /datasources/crypto-articles/schema.json: -------------------------------------------------------------------------------- 1 | { 2 | 3 | "author": "author", 4 | "title": "title", 5 | "coins": ["BTC, ETH"], 6 | "keywords": ["word1", "word2"], 7 | "description": "desc.", 8 | "_str": "title", 9 | "_timestamp": "1234567890", 10 | "loadTime": "97697", 11 | "leaveTime": "869", 12 | "scrollPercentage": "66%", 13 | "_url": "https://google.ie", 14 | "publication": "theguardian.com" 15 | } 16 | -------------------------------------------------------------------------------- /datasources/daft-message-sent/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "daft-message-sent", 3 | "author": "Rory", 4 | "description": "Generates a datapoint every time you send a message on daft.ie.", 5 | "version": "0.1", 6 | "sites": [".*daft\\.ie.*"] 7 | } 8 | -------------------------------------------------------------------------------- /datasources/daft-message-sent/plugin.js: -------------------------------------------------------------------------------- 1 | const daftMessageSent = { 2 | name: 'daft-message-sent', 3 | 4 | getListingInfo: function(loadTime, leaveTime) { 5 | info = { 6 | user_data: { 7 | loadTime: loadTime, 8 | leaveTime: leaveTime, 9 | action: 'message_sent' 10 | }, 11 | daft_data: { 12 | title: this.getTitle(), 13 | price: this.getPrice(), 14 | listing_type: this.getListingType(), 15 | header_info: this.getHeaderInfo(), 16 | property_overview: this.getPropertyOverview(), 17 | available_from: this.getAvailableFrom(), 18 | available_for: this.getAvailableFor(), 19 | property_description: this.getPropertyDescription(), 20 | facilities: this.getPropertyFacilities(), 21 | shortcode: this.getPropertyShortcode(), 22 | data_entered_renewed: this.getDateEnteredRenewed(), 23 | view_count: this.getViewCount(), 24 | image_count: this.getImageCount() 25 | } 26 | } 27 | 28 | return info; 29 | }, 30 | 31 | getTitle: function() { 32 | // Get the title of the listing 33 | var title = $('#address_box .smi-object-header h1').text().trim(); 34 | 35 | return title; 36 | }, 37 | 38 | getPrice: function() { 39 | /* 40 | * Get the price of the listing 41 | * Usually of the forms: 42 | * "From €XYZ per week" 43 | * "From €XYZ per month" 44 | */ 45 | var price = $('#smi-price-string').text().trim(); 46 | 47 | return price; 48 | }, 49 | 50 | getListingType: function() { 51 | // Get the type of the listing 52 | // e.g. "House Share", "Apartment Share", etc. 53 | var type = $('#smi-summary-items .header_text').first().text().trim(); 54 | 55 | return type; 56 | }, 57 | 58 | getHeaderInfo: function() { 59 | // Get the other heading info item 60 | // e.g. "X beds available for Y months" 61 | var info = $('#smi-summary-items .header_text').last().text().trim(); 62 | 63 | return info; 64 | }, 65 | 66 | getPropertyOverview: function() { 67 | // Get the property overview bullet points 68 | var overviewItems = $('#overview ul').children().toArray(); 69 | 70 | overviewItems = $.map(overviewItems, function(x) { 71 | return x.innerText; 72 | }); 73 | 74 | return overviewItems; 75 | }, 76 | 77 | getAvailabilityBlock: function() { 78 | // Gets the block containing "Available From" and "Available To" 79 | return $('#overview').next(); 80 | }, 81 | 82 | getAvailableFrom: function() { 83 | // Get the starting move-in date 84 | var $block = this.getAvailabilityBlock(); 85 | var from = $block.clone() // clone the element 86 | .children() // select all the children 87 | .remove() // remove all the children 88 | .end() // again go back to selected element 89 | .text() // Get the text 90 | .trim(); // Trim the whitespace 91 | 92 | return from; 93 | }, 94 | 95 | getAvailableFor: function() { 96 | var $block = this.getAvailabilityBlock().find('div'); 97 | var availableFor = $block.clone() // clone the element 98 | .children() // select all the children 99 | .remove() // remove all the children 100 | .end() // again go back to selected element 101 | .text() // Get the text 102 | .trim(); // Trim the whitespace 103 | 104 | return availableFor; 105 | }, 106 | 107 | getPropertyDescription: function() { 108 | // Get the text description of the listing 109 | var $descriptionClone = $('#description').clone(); 110 | 111 | $descriptionClone.find('#dfp-smi_ad_link_unit').remove(); 112 | $descriptionClone.find('.description_extras').remove(); 113 | var description = $descriptionClone.text(); 114 | 115 | 116 | return description; 117 | }, 118 | 119 | getPropertyFacilities: function() { 120 | /* Convert the facilities table to an array and return it 121 | * 122 | * ** BEWARE ** 123 | * Fuctional Magic Below 124 | * 125 | */ 126 | var $facilitiesMain = $('#facilities tbody tr'); 127 | var facilities = $facilitiesMain.find('td').map(function(i, td) { 128 | return $(td).find('ul').children().toArray().map(function(li, j) { 129 | return li.innerText; 130 | }); 131 | }); 132 | 133 | return facilities.toArray(); 134 | }, 135 | 136 | getPropertyShortcode: function() { 137 | // Get the shortcode URL for the listing 138 | var shortcode = $('.description_extras a').attr('href'); 139 | 140 | return shortcode; 141 | }, 142 | 143 | getDateEnteredRenewed: function() { 144 | // Gets the date that the listing was created or renewed (?) 145 | var date = $("h3:contains('Date')")[0].nextSibling.textContent.trim(); 146 | 147 | return date; 148 | }, 149 | 150 | getViewCount: function() { 151 | // Gets the view count for the listing 152 | var views = $("h3:contains('Property Views')")[0].nextSibling.textContent.trim(); 153 | 154 | return views; 155 | }, 156 | 157 | getImageCount: function() { 158 | // Gets the number of images for the listing 159 | var imageCount = $('.smi-gallery-list').children().length; 160 | 161 | return imageCount; 162 | }, 163 | 164 | setUpListener: function(metroClient) { 165 | /* 166 | * When the `#ad_reply_submit` button is clicked... 167 | */ 168 | loadTime = (new Date).getTime(); 169 | URL = window.location.href; 170 | 171 | let oThis = this; 172 | $('#ad_reply_submit').click(function() { 173 | // Do this: 174 | leaveTime = (new Date).getTime(); 175 | 176 | let datapoint = oThis.getListingInfo(loadTime, leaveTime); 177 | 178 | console.log(datapoint); 179 | 180 | metroClient.sendDatapoint(datapoint); 181 | }); 182 | }, 183 | 184 | initDataSource: function(metroClient) { 185 | this.setUpListener(metroClient); 186 | } 187 | } 188 | 189 | registerDataSource(daftMessageSent); 190 | -------------------------------------------------------------------------------- /datasources/daft-message-sent/schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "user_data": { 3 | "action": "message_sent" 4 | }, 5 | "daft_data": { 6 | "title": "title", 7 | "price": "price text", 8 | "listing_type": "type", 9 | "header_info": "header info", 10 | "property_overview": ["one", "two", "three"], 11 | "available_from": "text", 12 | "available_for": "text", 13 | "property_description": "text", 14 | "facilities": ["one", "two", "three"], 15 | "shortcode": "text", 16 | "data_entered_renewed": "text", 17 | "view_count": 5, 18 | "image_count": 5 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /datasources/facebook-time-per-page/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "facebook-time-per-page", 3 | "author": "Conor", 4 | "description": "This datasource records the time you load a web page and the time you leave the page, along with the URL that you visit for Facebook and Messenger domains.", 5 | "version": "0.1", 6 | "sites": [".*\\.facebook\\.com.*", 7 | ".*\\.messenger\\.com.*"] 8 | } 9 | -------------------------------------------------------------------------------- /datasources/facebook-time-per-page/plugin.js: -------------------------------------------------------------------------------- 1 | const facebookTimePerPage = { 2 | mc: null, 3 | name: 'facebook-time-per-page', 4 | 5 | initDataSource: function(metroClient) { 6 | loadTime = (new Date).getTime(); 7 | URL = window.location.href; 8 | 9 | window.addEventListener("beforeunload", function() { 10 | leaveTime = (new Date).getTime(); 11 | 12 | let datapoint = { 13 | "loadTime": loadTime, 14 | "leaveTime": leaveTime, 15 | "URL": URL 16 | } 17 | 18 | metroClient.sendDatapoint(datapoint); 19 | }); 20 | } 21 | } 22 | 23 | registerDataSource(facebookTimePerPage); 24 | -------------------------------------------------------------------------------- /datasources/facebook-time-per-page/schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "loadTime": "12345678910", 3 | "leaveTime": "12345678910", 4 | "URL": "https://www.bbc.com" 5 | } 6 | -------------------------------------------------------------------------------- /datasources/generic-image-labeling/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "generic-image-labeling", 3 | "author": "Rory", 4 | "description": "This DataSource allows users to label an image", 5 | "version": "0.1", 6 | "sites": [".*"] 7 | } 8 | -------------------------------------------------------------------------------- /datasources/generic-image-labeling/plugin.js: -------------------------------------------------------------------------------- 1 | const imageLabel = { 2 | mc: null, // Keep a ref to the client 3 | name: 'generic-image-labeling', 4 | 5 | /* 6 | The entry point. 7 | 8 | This function is called by Metro in order to initialize the DataSource 9 | Metro passes a reference to the MetroClient to the DataSource, and 10 | the DataSource can do whatever it wants after that. 11 | */ 12 | initDataSource: function(metroClient) { 13 | this.mc = metroClient; 14 | this.createRightClickButton(); 15 | }, 16 | 17 | /* 18 | Using the DataSource API, we create a right-click menu button which appears when an image 19 | is right-clicked. 20 | The 'functionName' is used to identify the function, the title is what the user sees, the 21 | contexts are the situations in which the button appears (image right-click), and the second 22 | argument is a callback function to be executed when the user presses the button 23 | */ 24 | createRightClickButton: function() { 25 | 26 | this.mc.createContextMenuButton({ 27 | functionName: 'imageLabel', 28 | buttonTitle: 'Label', 29 | contexts: ['image'] 30 | }, this.labelRightClickCallback.bind(this)); // Bind this object's context to the function 31 | }, 32 | 33 | /* 34 | This is the callback for the right-click menu button. 35 | Does two things: 36 | 1. Gets the URL of the image that was right-clicked 37 | 2. Creates a floating (modal) input box for the user to label the image 38 | 2.1. The modal input box takes a callback which runs after the user 39 | provides a label and presses enter 40 | 2.2. That callback is where we send the image URL + label as a single 41 | datapoint 42 | */ 43 | labelRightClickCallback: function(contextInfo) { 44 | let imageUrl = contextInfo['srcUrl']; // Using the 'image' context gives us access to the srcUrl of the image 45 | 46 | // Using the DataSource API, we define an input modal which appears when the right-click menu button is clicked 47 | this.mc.createModalForm({ 48 | inputs: [ 49 | { 50 | description: 'Enter a label', 51 | type: 'input' 52 | } 53 | ], 54 | submitCallback: function(inputText) { 55 | // Callback runs when the user submits the modal form 56 | // Receives one argument: the text input from the user 57 | if(this.validate(inputText)) { 58 | this.send(imageUrl, inputText); 59 | } 60 | }.bind(this) 61 | }) 62 | 63 | return {status: 1, msg: 'success'}; // The DataSource expects a return object like this for r-click button functions 64 | }, 65 | 66 | /* 67 | Gets the dimensions of the image and then sends the datapoint 68 | */ 69 | send: function(url, label) { 70 | var mc = this.mc; 71 | $("").attr("src", url).on('load', function(){ 72 | datapoint = { 73 | 'url': url, 74 | 'label': label['inputs'][0], 75 | 'width': this.width, 76 | 'website': window.location.href, 77 | 'height': this.height 78 | } 79 | console.log(datapoint); 80 | mc.sendDatapoint(datapoint); 81 | }); 82 | }, 83 | 84 | /* 85 | Validate the label input from the user 86 | 87 | You can expand this as you please 88 | */ 89 | validate: function(text) { 90 | if(text.length == 0) { 91 | // We don't want to accept an empty label 92 | return false; 93 | } 94 | 95 | return true; 96 | } 97 | } 98 | 99 | registerDataSource(imageLabel); // Register the DataSource to start it 100 | -------------------------------------------------------------------------------- /datasources/generic-image-labeling/schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "url": "https://example.com/image.jpg", 3 | "width": 100, 4 | "height": 100, 5 | "label": "example message.,-=+?![]()" 6 | } 7 | -------------------------------------------------------------------------------- /datasources/google-search-clicks/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "google-search-clicks", 3 | "author": "Conor", 4 | "description": "This runs on all Google search pages and collects the time the page was loaded, the time a link was clicked, the URL of the link clicked, the title of the link clicked, what rank that link had on the results page and the query that was searched.", 5 | "version": "0.2", 6 | "sites": ["^.*google\\..*/search.*$"] 7 | } 8 | -------------------------------------------------------------------------------- /datasources/google-search-clicks/plugin.js: -------------------------------------------------------------------------------- 1 | const googleSearchClicks = { 2 | mc: null, 3 | name: 'google-search-clicks', 4 | 5 | registerEventsHandler: function(node) { 6 | // Get all result nodes: 7 | let resultNodes = node.getElementsByClassName("r"); 8 | 9 | for(let resultIndex=0; resultIndex {let result = value[0];result.data['primaryRank'] = value.length; result.data['_str'] = result.data._str; result.data['primaryRankDisplay'] = result.data.primaryRank + ' times'; return result;}).sortBy(function(item) {return -item.data.primaryRank;}).value()", 5 | "display_icon": "fas fa-bolt" 6 | }, 7 | { 8 | "name": "Top Authors", 9 | "function": "_.chain(feed).groupBy(function(o) {return o.data.author;}).filter(group => group[0].data.author.length > 1).map((value, key) => {let result = value[0];result.data['primaryRank'] = value.length; result.data['_str'] = result.data.author; result.data['primaryRankDisplay'] = result.data.primaryRank + ' articles';return result;}).sortBy(function(item) {return -item.data.primaryRank;}).value()", 10 | "display_icon": "fas fa-bolt" 11 | }, 12 | { 13 | "name": "Top Publications", 14 | "function": "_.chain(feed).groupBy(function(o) {return o.data.publication;}).map((value, key) => {let result = value[0];result.data['primaryRank'] = value.length; result.data['_str'] = result.data.publication; result.data['primaryRankDisplay'] = result.data.primaryRank + ' articles'; return result;}).sortBy(function(item) {return -item.data.primaryRank;}).value()", 15 | "display_icon": "fas fa-bolt" 16 | }, 17 | { 18 | "name": "Top Keywords", 19 | "function": "_.chain(feed).reduce((memo, point) => {let result = memo.concat(point.data.keywords);return result;}, []).map((keyword) => { return keyword.toLowerCase(); } ).reduce((memo, val) => {if(memo.hasOwnProperty(val)) {memo[val] += 1;} else {memo[val] = 1;}return memo;}, {}).map((value, key) => {let result = {'data': {'primaryRank': value, '_str': key, 'primaryRankDisplay': value + ' appearances', '_key': key + value, '_url': '#', '_action': 'Count'}, 'datasource': feed[0].datasource}; return result;}).sortBy(function(item) {return -item.data.primaryRank;}).value()", 20 | "display_icon": "fas fa-bolt" 21 | }, 22 | { 23 | "name": "Completion Rate", 24 | "function": "_.chain(feed).groupBy(function(o) {return o.data._str;}).map((value, key) => {let result = value[0]; let count = value.length; let total = _.reduce(value, (memo, point) => { return memo + point.data.scrollPercentage }, 0); let avg = Math.round(total/count*100)/100; result.data['primaryRank'] = avg; result.data['primaryRankDisplay'] = avg + '% read'; return result; }).sortBy((item) => {return -item.data.primaryRank}).value()", 25 | "display_icon": "fas fa-bolt" 26 | } 27 | ] -------------------------------------------------------------------------------- /datasources/news-dwell-and-scroll/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "news-dwell-and-scroll", 3 | "title": "Tech Articles", 4 | "author": "Rory", 5 | "description": "Captures tech articles you read, and the % of the article that you read.", 6 | "version": "1.1", 7 | "sites": [ 8 | "(:\\/\\/|.*\\.)arstechnica\\.com.*", 9 | "(:\\/\\/|.*\\.)bbc(\\.co\\.uk|\\.com).*technology.*", 10 | "(:\\/\\/|.*\\.)engadget\\.com.*", 11 | "(:\\/\\/|.*\\.)independent\\.co\\.uk.*/gadgets-and-tech/.*", 12 | "(:\\/\\/|.*\\.)newscientist\\.com.*", 13 | "(:\\/\\/|.*\\.)recode\\.net.*", 14 | "(:\\/\\/|.*\\.)techcrunch\\.com.*", 15 | "(:\\/\\/|.*\\.)techradar\\.com.*", 16 | "(:\\/\\/|.*\\.)economist\\.com.*/science-and-technology/.*", 17 | "(:\\/\\/|.*\\.)nytimes\\.com.*/technology/.*", 18 | "(:\\/\\/|.*\\.)thenextweb\\.com.*", 19 | "(:\\/\\/|.*\\.)theverge\\.com.*", 20 | "(:\\/\\/|.*\\.)motherboard\\.vice\\.com.*", 21 | "(:\\/\\/|.*\\.)wired\\.com.*" 22 | ] 23 | } 24 | -------------------------------------------------------------------------------- /datasources/news-dwell-and-scroll/plugin.js: -------------------------------------------------------------------------------- 1 | function isAuthorTag(metaTag) { 2 | let nameExists = metaTag.getAttribute('name') != null; 3 | let propertyExists = metaTag.getAttribute('property') != null; 4 | 5 | if(nameExists) { 6 | let nameIsAuthor = metaTag.getAttribute('name').toLowerCase().includes('author') 7 | if (nameIsAuthor) { 8 | return true; 9 | } else { 10 | return false; 11 | } 12 | } else if(propertyExists) { 13 | let propertyIsAuthor = metaTag.getAttribute('property').toLowerCase().includes('author'); 14 | if(propertyIsAuthor) { 15 | return true; 16 | } else { 17 | return false; 18 | } 19 | } else { 20 | return false; 21 | } 22 | } 23 | 24 | function getMetaOrBlank(metas, name) { 25 | try { 26 | const el = metas.find((val, idx) => { 27 | const propertyAttr = val.getAttribute('property') 28 | const nameAttr = val.getAttribute('name') 29 | if(propertyAttr && propertyAttr.includes(name)) { 30 | return true 31 | } else if(nameAttr && nameAttr.includes(name)) { 32 | return true 33 | } 34 | return false 35 | }) 36 | 37 | if(el !== undefined) { 38 | return el.content 39 | } 40 | } catch(e) { 41 | console.error(e) 42 | return "" 43 | } 44 | return "" 45 | } 46 | 47 | const dwellAndScroll = { 48 | name: 'news-dwell-and-scroll', 49 | 50 | monitorDwellTime: function() { 51 | let self = this; 52 | 53 | var scrollPercentage = 0; 54 | $(window).on('scroll', function(){ 55 | var s = $(window).scrollTop(), 56 | d = $(document).height(), 57 | c = $(window).height(); 58 | 59 | scrollPercentage = (s / (d - c)) * 100; 60 | }) 61 | 62 | const loadTime = (new Date).getTime(); 63 | 64 | 65 | window.addEventListener("beforeunload", function() { 66 | const URL = window.location.href; 67 | const leaveTime = (new Date).getTime(); 68 | const nameRegex = new RegExp("^[a-zA-Z]+(([',. -][a-zA-Z ])?[a-zA-Z]*)*$") 69 | 70 | const metas = Array.from(document.querySelectorAll('META')); 71 | metas.forEach(console.log) 72 | var author = getMetaOrBlank(metas, 'author') 73 | console.debug(author.trim()) 74 | console.debug(nameRegex.test(author.trim())) 75 | author = nameRegex.test(author.trim()) ? author : '' 76 | var title = getMetaOrBlank(metas, 'og:title') 77 | if(title === "") { 78 | title = getMetaOrBlank(metas, "title") 79 | } 80 | const description = getMetaOrBlank(metas, 'description') 81 | const image = getMetaOrBlank(metas, 'image') 82 | const keywordsStr = getMetaOrBlank(metas, 'keywords') 83 | const keywords = keywordsStr.length > 0 ? keywordsStr.split(',') : [] 84 | const publication = window.location.hostname; 85 | 86 | let datapoint = { 87 | author: author, 88 | title: title, 89 | keywords: keywords, 90 | description: description, 91 | _image: image, 92 | _str: title, 93 | _timestamp: Date.now(), 94 | _action: "Read", 95 | loadTime: loadTime, 96 | leaveTime: leaveTime, 97 | scrollPercentage: Math.round(scrollPercentage), 98 | _url: URL, 99 | publication: publication 100 | } 101 | 102 | if(this.isValid(datapoint)) { 103 | this.mc.sendDatapoint(datapoint); 104 | // To make sure the request fires 105 | var t = Date.now() + 200; 106 | while(Date.now() < t) {}; 107 | } 108 | }.bind(this)); 109 | }, 110 | 111 | isValid: (datapoint) => { 112 | if(datapoint._str.includes("Category: ")) { 113 | return false // OpenGraph tags don't update when you browse around TechCrunch :() 114 | } 115 | return datapoint._str 116 | && datapoint._url 117 | && datapoint._timestamp 118 | && datapoint._action 119 | && datapoint._image 120 | 121 | }, 122 | 123 | initDataSource: function(metroClient) { 124 | this.mc = metroClient; 125 | let contentType = $('meta[property="og:type"]').attr('content'); 126 | 127 | let pCount = $('p').length 128 | let articleCount = $('article').length 129 | 130 | if(contentType === "article" && (articleCount > 0 || pCount > 5)) { 131 | this.monitorDwellTime(); 132 | } 133 | 134 | } 135 | } 136 | 137 | registerDataSource(dwellAndScroll); 138 | -------------------------------------------------------------------------------- /datasources/news-dwell-and-scroll/schema.json: -------------------------------------------------------------------------------- 1 | { 2 | 3 | "author": "author", 4 | "title": "title", 5 | "keywords": ["word1", "word2"], 6 | "description": "desc.", 7 | "_str": "title", 8 | "_timestamp": "1234567890", 9 | "loadTime": "97697", 10 | "leaveTime": "869", 11 | "scrollPercentage": "66%", 12 | "_url": "https://google.ie", 13 | "_action": "...", 14 | "publication": "theguardian.com" 15 | } 16 | -------------------------------------------------------------------------------- /datasources/omny-label/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "omny-label", 3 | "author": "Rory", 4 | "description": "This DataSource allows users to label the brand of clothing in an image", 5 | "version": "0.2", 6 | "sites": [".*"] 7 | } 8 | -------------------------------------------------------------------------------- /datasources/omny-label/plugin.js: -------------------------------------------------------------------------------- 1 | const omnyLabel = { 2 | mc: null, // Keep a ref to the client 3 | name: 'omny-label', 4 | 5 | /* 6 | The entry point. 7 | 8 | This function is called by Metro in order to initialize the DataSource 9 | Metro passes a reference to the MetroClient to the DataSource, and 10 | the DataSource can do whatever it wants after that. 11 | */ 12 | initDataSource: function(metroClient) { 13 | this.mc = metroClient; 14 | this.createRightClickButton(); 15 | }, 16 | 17 | /* 18 | Using the DataSource API, we create a right-click menu button which appears when an image 19 | is right-clicked. 20 | The 'functionName' is used to identify the function, the title is what the user sees, the 21 | contexts are the situations in which the button appears (image right-click), and the second 22 | argument is a callback function to be executed when the user presses the button 23 | */ 24 | createRightClickButton: function() { 25 | 26 | this.mc.createContextMenuButton({ 27 | functionName: 'omnyBrandLabel', 28 | buttonTitle: 'Omny Label', 29 | contexts: ['image'] 30 | }, this.omnyRightClickCallback.bind(this)); // Bind this object's context to the function 31 | }, 32 | 33 | /* 34 | This is the callback for the right-click menu button. 35 | Does two things: 36 | 1. Gets the URL of the image that was right-clicked 37 | 2. Creates a floating (modal) input box for the user to label the image 38 | 2.1. The modal input box takes a callback which runs after the user 39 | provides a label and presses enter 40 | 2.2. That callback is where we send the image URL + label as a single 41 | datapoint 42 | */ 43 | omnyRightClickCallback: function(contextInfo) { 44 | let imageUrl = contextInfo['srcUrl']; // Using the 'image' context gives us access to the srcUrl of the image 45 | 46 | // Using the DataSource API, we define an input modal which appears when the right-click menu button is clicked 47 | this.mc.createModalForm({ 48 | description: 'Enter a brand label', 49 | submitCallback: function(inputText) { 50 | // Callback runs when the user submits the modal form 51 | // Receives one argument: the text input from the user 52 | if(this.omnyInputValid(inputText)) { 53 | this.omnySend(imageUrl, inputText); 54 | } 55 | }.bind(this) 56 | }) 57 | 58 | return {status: 1, msg: 'success'}; // The DataSource expects a return object like this for r-click button functions 59 | }, 60 | 61 | /* 62 | Gets the dimensions of the image and then sends the datapoint 63 | */ 64 | omnySend: function(url, label) { 65 | var mc = this.mc; 66 | $("").attr("src", url).on('load', function(){ 67 | mc.sendDatapoint({ 68 | 'url': url, 69 | 'label': label, 70 | 'width': this.width, 71 | 'height': this.height 72 | }); 73 | }); 74 | }, 75 | 76 | /* 77 | Validate the label input from the user 78 | 79 | You can expand this as you please 80 | */ 81 | omnyInputValid: function(text) { 82 | console.log("valid"); 83 | if(text.length == 0) { 84 | // We don't want to accept an empty label 85 | return false; 86 | } 87 | 88 | return true; 89 | } 90 | } 91 | 92 | registerDataSource(omnyLabel); // Register the DataSource to start it 93 | -------------------------------------------------------------------------------- /datasources/omny-label/schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "url": "https://example.com/image.jpg", 3 | "width": 100, 4 | "height": 100, 5 | "label": "example message.,-=+?![]()" 6 | } 7 | -------------------------------------------------------------------------------- /datasources/quote-highlight/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "quote-highlight", 3 | "author": "Rory", 4 | "description": "This DataSource allows users to highlight a piece of text as the 'top' quote of the article", 5 | "version": "0.1", 6 | "sites": [".*\\.irishtimes\\.com.*", 7 | ".*\\.independent\\..*", 8 | ".*\\.joe\\.ie.*", 9 | ".*\\.her\\.ie.*", 10 | ".*\\.theguardian\\.com.*", 11 | ".*\\.dailymail\\.co\\.uk.*", 12 | ".*\\.bbc\\.com.*", 13 | ".*\\.rte\\.ie.*", 14 | ".*\\.medium\\.com.*", 15 | ".*\\.thejournal\\.ie.*"] 16 | } 17 | -------------------------------------------------------------------------------- /datasources/quote-highlight/plugin.js: -------------------------------------------------------------------------------- 1 | function isAuthorTag(metaTag) { 2 | let nameExists = metaTag.getAttribute('name') != null; 3 | let propertyExists = metaTag.getAttribute('property') != null; 4 | 5 | if(nameExists) { 6 | let nameIsAuthor = metaTag.getAttribute('name').toLowerCase().includes('author') 7 | if (nameIsAuthor) { 8 | return true; 9 | } else { 10 | return false; 11 | } 12 | } else if(propertyExists) { 13 | let propertyIsAuthor = metaTag.getAttribute('property').toLowerCase().includes('author'); 14 | if(propertyIsAuthor) { 15 | return true; 16 | } else { 17 | return false; 18 | } 19 | } else { 20 | return false; 21 | } 22 | } 23 | 24 | const quoteHighlight = { 25 | mc: null, 26 | name: "quote-highlight", 27 | 28 | initDataSource: function(metroClient) { 29 | // Initialize the context-menu buttons 30 | this.mc = metroClient; 31 | this.createHighlightButton(); 32 | }, 33 | 34 | getAuthor: function() { 35 | // Tries to find the author of the article from the webpage 36 | var info = document.getElementsByTagName('META'); 37 | 38 | for (var i=0;i 100) { 86 | var resp = { 87 | 'status': 0, 88 | 'msg': 'Quote is too long' 89 | } 90 | } else { 91 | var resp = { 92 | 'status': 1, 93 | 'msg': 'Success!' 94 | }; 95 | } 96 | 97 | return resp; 98 | }, 99 | 100 | createHighlightButton: function() { 101 | let contentType = $('meta[property="og:type"]').attr('content'); 102 | 103 | if(contentType == "article") { 104 | // Context-menu button for highlighted text 105 | this.mc.createContextMenuButton({ 106 | functionName: 'highlightFunction', 107 | buttonTitle: 'Highlight', 108 | contexts: ['selection'] 109 | }, this.sendHighlight.bind(this)); 110 | } 111 | }, 112 | } 113 | 114 | registerDataSource(quoteHighlight); 115 | -------------------------------------------------------------------------------- /datasources/quote-highlight/schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "text": "example message.,-=+?![]()" 3 | } 4 | -------------------------------------------------------------------------------- /datasources/reddit-basic-votes/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "reddit-basic-votes", 3 | "author": "Conor", 4 | "description": "This is a basic Reddit source that collects up/down votes and the post title.", 5 | "version": "0.2", 6 | "sites": [".*reddit\\.com.*"] 7 | } 8 | -------------------------------------------------------------------------------- /datasources/reddit-basic-votes/plugin.js: -------------------------------------------------------------------------------- 1 | const redditBasicVotes = { 2 | mc: null, 3 | name: 'reddit-basic-votes', 4 | 5 | registerVoteActions: function (node) { 6 | let gThis = this; 7 | // First we get all the "rows" denoting posts on the reddit front page: 8 | var postDivs = node.getElementsByClassName("thing"); 9 | 10 | // Then for each of these posts, we find the divs with the "arrow" class 11 | // denoting vote buttons: 12 | for(var i=0; i 0) { 33 | // Find the time difference: 34 | let duration = eventTime - prevTime; 35 | console.log(duration); 36 | 37 | // And create the datapoint: 38 | let datapoint = {}; 39 | datapoint['event'] = currentEvent; 40 | datapoint['time'] = duration; 41 | 42 | // Don't send more than 1 datapoint per 5 seconds: 43 | if(duration > 5000) { 44 | gThis.mc.sendDatapoint(datapoint); 45 | } 46 | } 47 | 48 | // Finally, store the updated timepoint: 49 | gThis.mc.storeData("timepoint", eventTime); 50 | }); 51 | }); 52 | } 53 | }, 54 | 55 | initDataSource: function(metroClient) { 56 | this.mc = metroClient; 57 | console.log("Beginning reddit-time-between-actions data source."); 58 | 59 | // We want to set the initial load time here: 60 | let currentTime = Date.now(); 61 | this.mc.storeData("timepoint", currentTime); 62 | 63 | // Then start our plugin. 64 | this.registerEventsHandler(document.body); 65 | } 66 | } 67 | 68 | registerDataSource(redditTimeBetweenActions); 69 | -------------------------------------------------------------------------------- /datasources/reddit-time-between-actions/schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "event": "String", 3 | "time": "String" 4 | } 5 | -------------------------------------------------------------------------------- /datasources/stack-overflow-questions/aggregations.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "Views", 4 | "function": "_.chain(feed).filter(item => item.data._action.toLowerCase() == 'viewed').groupBy(function(o) {return o.data._str;}).map((value, key) => {let result = value[0]; result.data['primaryRank'] = value.length;result.data['primaryRankDisplay'] = result.data.primaryRank.toLocaleString() + ' views';return result;}).sortBy(function(item) {return -item.data.primaryRank;}).value()", 5 | "display_icon": "fas fa-bolt" 6 | }, 7 | { 8 | "name": "Upvotes", 9 | "function": "_.chain(feed).filter(item => item.data._action.toLowerCase() == 'upvoted').groupBy(function(o) {return o.data._str;}).map((value, key) => {let result = value[0]; result.data['primaryRank'] = value.length;result.data['primaryRankDisplay'] = result.data.primaryRank.toLocaleString() + ' votes';return result;}).sortBy(function(item) {return -item.data.primaryRank;}).value()", 10 | "display_icon": "fas fa-bolt" 11 | }, 12 | { 13 | "name": "Copy/Pastes", 14 | "function": "_.chain(feed).filter(item => item.data._action.toLowerCase() == 'copied').groupBy(function(o) {return o.data._str;}).map((value, key) => {let result = value[0]; result.data['primaryRank'] = value.length;result.data['primaryRankDisplay'] = result.data.primaryRank.toLocaleString() + ' copies';return result;}).sortBy(function(item) {return -item.data.primaryRank;}).value()", 15 | "display_icon": "fas fa-bolt" 16 | }, 17 | { 18 | "name": "Favorites", 19 | "function": "_.chain(feed).filter(item => item.data._action.toLowerCase().startsWith('fav')).groupBy(function(o) {return o.data._str;}).map((value, key) => {let result = value[0]; result.data['primaryRank'] = value.length;result.data['primaryRankDisplay'] = result.data.primaryRank.toLocaleString() + ' favs';return result;}).sortBy(function(item) {return -item.data.primaryRank;}).value()", 20 | "display_icon": "fas fa-bolt" 21 | }, 22 | { 23 | "name": "Top Tags", 24 | "function": "_.chain(feed).reduce((memo, point) => {let result = memo.concat(point.data.tags);return result;}, []).map((tag) => { return tag.toLowerCase(); } ).reduce((memo, val) => {if(memo.hasOwnProperty(val)) {memo[val] += 1;} else {memo[val] = 1;}return memo;}, {}).map((value, key) => {let result = {'data': {'primaryRank': value, '_str': key, 'primaryRankDisplay': value + ' appearances', '_key': key + value}, 'datasource': feed[0].datasource}; return result;}).sortBy(function(item) {return -item.data.primaryRank;}).value()", 25 | "display_icon": "fas fa-bolt" 26 | }, 27 | { 28 | "name": "Top Languages", 29 | "function": "_.chain(feed).reduce((memo, point) => {let result = memo.concat(point.data.languages);return result;}, []).map((tag) => { return tag.toLowerCase(); } ).reduce((memo, val) => {if(memo.hasOwnProperty(val)) {memo[val] += 1;} else {memo[val] = 1;}return memo;}, {}).map((value, key) => {let result = {'data': {'primaryRank': value, '_str': key, 'primaryRankDisplay': value + ' appearances', '_key': key + value}, 'datasource': feed[0].datasource}; return result;}).sortBy(function(item) {return -item.data.primaryRank;}).value()", 30 | "display_icon": "fas fa-bolt" 31 | }, 32 | { 33 | "name": "Recently Asked", 34 | "function": "_.chain(feed).uniq(item => item.data._str).sortBy(item => -item.data.askedTimestamp).map((result, key) => { result.data['primaryRank'] = result.data.askedTimestamp - feed[feed.length - 1].data.askedTimestamp; var a = new Date(result.data.askedTimestamp*1000);var months = ['Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sep','Oct','Nov','Dec'];var month = months[a.getMonth()];var date = a.getDate(); var year = a.getFullYear(); var time = date + ' ' + month + ', ' + year ; result.data['primaryRankDisplay'] = time;return result;}).value()", 35 | "display_icon": "fas fa-bolt" 36 | }, 37 | { 38 | "name": "Time To Answer", 39 | "function": "_.chain(feed).uniq((item) => item.data._str).filter(item => item.data.hasAcceptedAnswer).map((value, key) => { value.data['primaryRank'] = value.data.acceptedAnswerTimestamp - value.data.askedTimestamp; let temp = value.data.primaryRank; let days = Math.floor( ( temp %= 31536000 ) / 86400 ); let hours = Math.floor( ( temp %= 86400 ) / 3600 ); let minutes = Math.floor( ( temp %= 3600 ) / 60 ); let timeTaken = Math.floor(value.data.primaryRank / (60)); value.data['primaryRankDisplay'] = ( days ? days + ' days' : hours ? hours + ' hours' : minutes ? minutes + ' minutes' : ''); return value;}).sortBy(item => -item.data.primaryRank).value()", 40 | "display_icon": "fas fa-bolt" 41 | } 42 | ] -------------------------------------------------------------------------------- /datasources/stack-overflow-questions/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "stack-overflow-questions", 3 | "title": "Stack Overflow Questions", 4 | "author": "Rory", 5 | "description": "Captures the questions that you read on Stack Overflow.", 6 | "version": "0.0.1", 7 | "sites": [".*stackoverflow\\.com/questions/.*"] 8 | } 9 | -------------------------------------------------------------------------------- /datasources/stack-overflow-questions/plugin.js: -------------------------------------------------------------------------------- 1 | const LANGUAGES = [ 2 | 'python', 3 | 'java', 4 | 'javascript', 5 | 'haskell', 6 | 'c++', 7 | 'c', 8 | 'rust', 9 | 'html', 10 | 'css', 11 | 'sql', 12 | 'bash', 13 | 'shell', 14 | 'c#', 15 | 'php', 16 | 'ruby', 17 | 'swift', 18 | 'assembly', 19 | 'go', 20 | 'objective-c', 21 | 'r', 22 | 'scala', 23 | 'groovy', 24 | 'perl', 25 | 'kotlin', 26 | 'matlab', 27 | 'typescript', 28 | 'vb.net', 29 | 'vba', 30 | 'elixir', 31 | ] 32 | 33 | const StackOverflowQuestions = { 34 | name: 'stack-overflow-questions', 35 | 36 | getQuestionTitle: function() { 37 | return $('#question-header h1').text(); 38 | }, 39 | 40 | getTags: function() { 41 | let tags = $('.post-taglist .grid a').toArray(); 42 | return tags.map(tagElement => tagElement.innerText ) 43 | }, 44 | 45 | getQInfo: function() { 46 | let qInfo = $('#qinfo tbody') 47 | return qInfo; 48 | }, 49 | 50 | getAskedTimestamp: function() { 51 | // The 0th in the table 52 | let qInfo = this.getQInfo(); 53 | let $tr = qInfo.find('tr:eq(0)') 54 | let value = $tr.find('td:eq(1) p').attr('title'); 55 | let timestamp = new Date(value).getTime()/1000; 56 | 57 | return timestamp 58 | }, 59 | 60 | getActiveTimestamp: function() { 61 | // The 2nd in the table 62 | let qInfo = this.getQInfo(); 63 | let $tr = qInfo.find('tr:eq(2)') 64 | let value = $tr.find('td:eq(1) p b a').attr('title'); 65 | let timestamp = new Date(value).getTime()/1000; 66 | 67 | return timestamp 68 | }, 69 | 70 | getViews: function() { 71 | // The 1st in the table 72 | let qInfo = this.getQInfo(); 73 | let $tr = qInfo.find('tr:eq(1)') 74 | let value = $tr.find('td:eq(1) p b').text() 75 | 76 | return value.split(' ')[0] 77 | }, 78 | 79 | getQuestionUpvotes: function() { 80 | return $('#question').find('div [itemprop="upvoteCount"]').text(); 81 | }, 82 | 83 | getAnswerCount: function() { 84 | return $('#answers-header .subheader h2').data('answercount'); 85 | }, 86 | 87 | hasAcceptedAnswer: function() { 88 | return $('.accepted-answer').length !== 0; 89 | }, 90 | 91 | getAcceptedAnswerTimestamp: function() { 92 | let time = $('.accepted-answer').find('time').attr('datetime') + "Z"; 93 | console.log(time) 94 | let timestamp = new Date(time).getTime()/1000; 95 | 96 | return timestamp 97 | }, 98 | 99 | getFavoriteCount: function() { 100 | let count = $('.js-favorite-count').text(); 101 | return (count.length === 0 ? 0 : count) 102 | }, 103 | 104 | filterProgrammingLanguages: function(tags) { 105 | // tags -> [ 'buttons', 'python', 'dianarocks', 'java', 'c++' ] 106 | let languages = tags.filter(tag => LANGUAGES.includes(tag)) // True or False 107 | 108 | return languages // [ 'python', 'java', 'c++' ] 109 | }, 110 | 111 | run: function() { 112 | let self = this; 113 | 114 | let questionTitle = this.getQuestionTitle(); 115 | let tags = this.getTags(); 116 | let askedTimestamp = parseInt(this.getAskedTimestamp()); 117 | let activeTimestamp = this.getActiveTimestamp(); 118 | let questionUpvotes = this.getQuestionUpvotes(); 119 | let answerCount = this.getAnswerCount(); 120 | let hasAcceptedAnswer = this.hasAcceptedAnswer(); 121 | let acceptedAnswerTimestamp = parseInt(this.getAcceptedAnswerTimestamp()); 122 | let languages = this.filterProgrammingLanguages(tags); 123 | let views = this.getViews(); 124 | let favCount = this.getFavoriteCount(); 125 | let action = "Viewed"; 126 | 127 | let datapoint = { 128 | _str: questionTitle, 129 | _url: window.location.href, 130 | _timestamp: Date.now(), 131 | _action: action, 132 | title: questionTitle, 133 | tags: tags, 134 | languages: languages, 135 | hasAcceptedAnswer: hasAcceptedAnswer, 136 | acceptedAnswerTimestamp: acceptedAnswerTimestamp, 137 | questionUpvotes: questionUpvotes, 138 | questionFavs: favCount, 139 | answerCount: answerCount, 140 | askedTimestamp: askedTimestamp, 141 | activeTimestamp: activeTimestamp, 142 | views: views, 143 | } 144 | 145 | this.mc.sendDatapoint(datapoint) 146 | 147 | $('#content').on('copy', function() { 148 | // Set the action before we send it 149 | datapoint['_action'] = "Copied" 150 | datapoint['_timestamp'] = Date.now() 151 | self.mc.sendDatapoint(datapoint); 152 | }) 153 | 154 | $('.question').find('.js-vote-up-btn').click(function() { 155 | // Set the action before we send it 156 | datapoint['_action'] = "Upvoted" 157 | datapoint['_timestamp'] = Date.now() 158 | self.mc.sendDatapoint(datapoint); 159 | }) 160 | 161 | $('.js-favorite-btn').click(function() { 162 | // Set the action before we send it 163 | datapoint['_action'] = "Fav'd" 164 | datapoint['_timestamp'] = Date.now() 165 | self.mc.sendDatapoint(datapoint); 166 | }) 167 | }, 168 | 169 | initDataSource: function(metroClient) { 170 | this.mc = metroClient; 171 | 172 | this.run() 173 | } 174 | } 175 | 176 | registerDataSource(StackOverflowQuestions); 177 | -------------------------------------------------------------------------------- /datasources/stack-overflow-questions/schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "_str": "questionTitle", 3 | "_url": "window.location.href", 4 | "_timestamp": "Date.now()", 5 | "_action": "...", 6 | "title": "questionTitle", 7 | "tags": "tags", 8 | "languages": [], 9 | "hasAcceptedAnswer": "hasAcceptedAnswer", 10 | "acceptedAnswerTimestamp": "hasAcceptedAnswer", 11 | "questionUpvotes": "questionUpvotes", 12 | "questionFavs": "favCount", 13 | "answerCount": "answerCount", 14 | "askedTimestamp": "askedTimestamp", 15 | "activeTimestamp": "activeTimestamp", 16 | "views": "views" 17 | } 18 | -------------------------------------------------------------------------------- /datasources/study-materials/aggregations.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "Most Popular", 4 | "function": "_.chain(feed).groupBy(function(o) {return o.data._str;}).map((value, key) => {let result = value[0];result.data['primaryRank'] = value.length; result.data['primaryRankDisplay'] = result.data.primaryRank + ' times'; result.data['maxPrimaryRank'] = _.max(feed, function(item) { return item.data.primaryRank }).data.primaryRank; return result;}).sortBy(function(item) {return -item.data.primaryRank;}).value()", 5 | "display_icon": "fas fa-bolt" 6 | } 7 | ] -------------------------------------------------------------------------------- /datasources/study-materials/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "study-materials", 3 | "title": "Study Materials", 4 | "author": "Rory", 5 | "description": "Captures study materials that you open on YouTube, Loop, and Google Scholar. On YouTube, it automatically captures any videos in the 'Education' category, and on Loop/Scholar it captures any links you open.", 6 | "version": "0.3", 7 | "sites": ["^.*youtube\\.com.*$", "^.*scholar\\.google\\.com.*$", "^.*loop\\..*\\.ie\/course\/view\\.php\\?id=.*$"] 8 | } 9 | -------------------------------------------------------------------------------- /datasources/study-materials/plugin.js: -------------------------------------------------------------------------------- 1 | /* 2 | YouTube dynamically loads data, which means that we can't just scrape normally, we have to 3 | wait for the DOM to be updated. 4 | */ 5 | 6 | function isVideo(url) { 7 | let r = new RegExp("http(?:s?):\\/\\/(?:www\\.)?youtu(?:be\\.com\\/watch\\?v=|\\.be\\/)([\\w\\-\\_]*)(&(amp;)?‌​[\\w\\?‌​=]*)?") 8 | return r.test(url); 9 | } 10 | 11 | const Study = { 12 | mc: null, 13 | name: 'study-materials', 14 | 15 | getSearchQuery: () => { 16 | return $('#gs_hdr_tsi').value 17 | }, 18 | 19 | getCitationCount: (node) => { 20 | const text = node.find('.gs_ri').find('.gs_fl a:eq(2)').text() 21 | 22 | const last = a => a[a.length - 1]; 23 | return parseInt(last(text.split(' '))) 24 | }, 25 | 26 | runScholar: function() { 27 | const searchQuery = this.getSearchQuery(); 28 | $('.gs_r.gs_or.gs_scl').each((idx, val) => { 29 | const sideLinks = $(val).find('.gs_ggs.gs_fl').find('a') 30 | const mainLink = $(val).find('.gs_rt').find('a') 31 | const contentLink = $(sideLinks[0]) 32 | const citationCount = this.getCitationCount($(val)) 33 | 34 | const handleClick = (e) => { 35 | if(e.which === 1 || e.which === 2) { 36 | this.sendDatapointScholar(contentLink.attr('href'), mainLink.text(), searchQuery, citationCount) 37 | } 38 | } 39 | 40 | sideLinks.each((idx, link) => { 41 | $(link).on('mouseup', handleClick) 42 | }) 43 | 44 | mainLink.on('mouseup', handleClick) 45 | }) 46 | 47 | }, 48 | 49 | createDatapoint: function(self, url, $videoPlayer) { 50 | const category = this.getCategory(); 51 | const title = $('#container h1.title yt-formatted-string').text(); 52 | const views = $('#container div#info').find('span.view-count').text().replace(/\D/g, ''); 53 | const lengthText = $videoPlayer.find('.ytp-time-duration').text() 54 | // Convert hh:mm:ss to seconds 55 | const lengthSeconds = +(lengthText.split(':').reduce((acc,time) => (60 * acc) + +time)); 56 | 57 | self.sendDatapointYt(url, title, views, category, lengthSeconds); 58 | }, 59 | 60 | getCategory: function () { 61 | // Get the container div which has the #title and #content of the Category field 62 | let categoryContainer = $('ytd-expander.ytd-video-secondary-info-renderer ytd-metadata-row-renderer:has(#title yt-formatted-string:contains("Category"))'); 63 | // Extract the text from the #content div 64 | let category = categoryContainer.find('#content').text().trim(); 65 | // Click the "Show Less" button 66 | $('ytd-expander.ytd-video-secondary-info-renderer #less').click(); 67 | 68 | return category; 69 | }, 70 | 71 | showMore: function () { 72 | let clickBtn = $('ytd-expander.ytd-video-secondary-info-renderer #more'); 73 | clickBtn.click(); // Click the "show more button" 74 | }, 75 | 76 | validateDatapoint: function(datapoint) { 77 | switch(datapoint.type) { 78 | case "youtube": 79 | if (datapoint.youtubeData.category == "Education") { 80 | return true; 81 | } else { 82 | console.log(`Video category is not 'Education' so not sending it`) 83 | return false; 84 | } 85 | case "googleScholar": 86 | return true; 87 | case "loop": 88 | return true; 89 | default: 90 | console.error("Invalid datapoint type: " + datapoint.type) 91 | return false; 92 | } 93 | }, 94 | 95 | getFavicon: function() { 96 | try { 97 | return document.querySelector('link[rel="shortcut icon"]').href; 98 | } catch(err) { 99 | return document.querySelector('link[rel="icon"]').href; 100 | } 101 | }, 102 | 103 | sendDatapointLoop: function (url, title) { 104 | const img = this.getFavicon(); 105 | 106 | // Create the datapoint: 107 | let datapoint = { 108 | _url: url, 109 | _str: title, 110 | _action: "Viewed", 111 | _timestamp: Date.now(), 112 | _image: img, 113 | type: 'loop', 114 | scholarData: {}, 115 | youtubeData: {}, 116 | loopData: {} 117 | }; 118 | 119 | if(this.validateDatapoint(datapoint)) { 120 | console.log(datapoint) 121 | // Validate and send the datapoint 122 | this.mc.sendDatapoint(datapoint); 123 | } 124 | }, 125 | 126 | sendDatapointScholar: function (url, title, searchQuery, citationCount) { 127 | const img = this.getFavicon(); 128 | 129 | // Create the datapoint: 130 | let datapoint = { 131 | _url: url, 132 | _str: title, 133 | _action: "Viewed", 134 | _timestamp: Date.now(), 135 | _image: img, 136 | type: 'googleScholar', 137 | scholarData: { 138 | searchQuery: searchQuery, 139 | citationsCount: citationCount 140 | }, 141 | youtubeData: {}, 142 | loopData: {} 143 | }; 144 | 145 | if(this.validateDatapoint(datapoint)) { 146 | // Validate and send the datapoint 147 | this.mc.sendDatapoint(datapoint); 148 | } 149 | }, 150 | 151 | getYouTubeID: (url) => { 152 | url = url.split(/(vi\/|v=|\/v\/|youtu\.be\/|\/embed\/)/); 153 | return (url[2] !== undefined) ? url[2].split(/[^0-9a-z_\-]/i)[0] : url[0]; 154 | }, 155 | 156 | sendDatapointYt: function (url, title, views, category, lengthSeconds) { 157 | const id = this.getYouTubeID(url) 158 | const img = `https://i1.ytimg.com/vi/${id}/default.jpg` 159 | // Create the datapoint: 160 | let datapoint = { 161 | _url: url, 162 | _str: title, 163 | _action: "Watched", 164 | _timestamp: Date.now(), 165 | _image: img, 166 | type: 'youtube', 167 | scholarData: {}, 168 | youtubeData: { 169 | category: category, 170 | views: views, 171 | lengthSeconds: lengthSeconds 172 | }, 173 | loopData: {} 174 | }; 175 | 176 | // Log it 177 | console.log(datapoint); 178 | 179 | if(this.validateDatapoint(datapoint)) { 180 | // Validate and send the datapoint 181 | this.mc.sendDatapoint(datapoint); 182 | } 183 | }, 184 | 185 | runYoutube: function(previousUrl) { 186 | let currentUrl = window.location.href; 187 | if(previousUrl == currentUrl) { 188 | return currentUrl; 189 | } else if(isVideo(currentUrl)) { // New page 190 | const self = this; 191 | setTimeout(function() { 192 | // Get the video player 193 | let $videoPlayer = $('#movie_player'); 194 | 195 | // YouTube loads data dynamically, so we must click the "show more" button 196 | // extract the category, and then click the "show less" button. 197 | self.showMore(); 198 | setTimeout(function() { 199 | self.createDatapoint(self, currentUrl, $videoPlayer); 200 | }, 10000); 201 | }, 2000); 202 | 203 | return currentUrl; 204 | } else { 205 | return currentUrl; 206 | } 207 | }, 208 | 209 | runLoop: function() { 210 | const $main = $('#page-content').find('.course-content') 211 | const $sections = $main.find('li.section') 212 | $sections.each((idx, val) => { 213 | const $val = $(val) 214 | const $links = $val.find('li.activity').find('a').not('.showdescription') 215 | 216 | $links.each((idx, link) => { 217 | $(link).on('mouseup', (e) => { 218 | if(e.which === 1 || e.which === 2) { 219 | const url = $(link).attr('href') 220 | const text = $(link).find('.instancename').text() 221 | this.sendDatapointLoop(url, text) 222 | } 223 | }) 224 | }) 225 | }) 226 | }, 227 | 228 | initDataSource: function (metroClient) { 229 | // The entrypoint 230 | this.mc = metroClient; 231 | const self = this; 232 | 233 | // setTimeout(this.addRecommendation, 5000) 234 | 235 | const hostname = window.location.hostname.replace('www.', '') 236 | if (hostname === "youtube.com") { 237 | let url = ""; 238 | setInterval(function () { 239 | url = self.runYoutube(url); 240 | }, 500); 241 | } else if (hostname === "scholar.google.com") { 242 | this.runScholar() 243 | } else if(hostname.includes("loop")) { 244 | this.runLoop() 245 | } 246 | } 247 | } 248 | 249 | registerDataSource(Study); -------------------------------------------------------------------------------- /datasources/study-materials/schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "_url": "...", 3 | "_timestamp": "...", 4 | "_str": "...", 5 | "_action": "...", 6 | "_image": "...", 7 | "type": "...", 8 | "scholarData": {}, 9 | "youtubeData": {}, 10 | "loopData": {} 11 | } 12 | -------------------------------------------------------------------------------- /datasources/text-translation/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "text-translation", 3 | "author": "Rory", 4 | "description": "This DataSource allows you to highlight a piece of text, right-click, and enter a translation", 5 | "version": "0.1", 6 | "sites": [".*"] 7 | } 8 | -------------------------------------------------------------------------------- /datasources/text-translation/plugin.js: -------------------------------------------------------------------------------- 1 | const translateText = { 2 | mc: null, 3 | name: "text-translation", 4 | 5 | initDataSource: function(metroClient) { 6 | this.mc = metroClient; 7 | this.createMenuButton(metroClient); 8 | }, 9 | 10 | send: function(original, translation, source, destination) { 11 | let datapoint = {}; 12 | datapoint['original'] = original; 13 | datapoint['translation'] = translation; 14 | datapoint['source'] = source; 15 | datapoint['destination'] = destination; 16 | console.log(datapoint); 17 | 18 | this.mc.sendDatapoint(datapoint); 19 | }, 20 | 21 | // Validate the answer 22 | translationValid: function(translation) { 23 | if(translation['inputs'][0] === '') { 24 | return { 25 | 'status': 0, 26 | 'msg': 'Translation cannot be blank' 27 | } 28 | } else if(translation['inputs'][1] === translation['inputs'][2]) { 29 | return { 30 | 'status': 0, 31 | 'msg': 'Source and destination language cannot be the same' 32 | } 33 | } else { 34 | return { 35 | 'status': 1, 36 | 'msg': 'Success!' 37 | }; 38 | } 39 | }, 40 | 41 | createMenuButton: function() { 42 | this.mc.createContextMenuButton({ 43 | functionName: 'translateFunction', 44 | buttonTitle: 'Translate', 45 | contexts: ['selection'] 46 | }, this.translateWordRightClickCallback.bind(this)); 47 | }, 48 | 49 | translateWordRightClickCallback: function(contextInfo) { 50 | let original = contextInfo['selectionText']; 51 | 52 | this.mc.createModalForm({ 53 | inputs: [ 54 | { 55 | description: 'Translation: ', 56 | type: 'input' 57 | }, 58 | { 59 | description: 'Source Language: ', 60 | type: 'select', 61 | options: [ 62 | {val : 'fr', text: 'fr'}, 63 | {val : 'en', text: 'en'}, 64 | {val : 'de', text: 'de'}, 65 | {val : 'ru', text: 'ru'}, 66 | {val : 'it', text: 'it'}, 67 | {val : 'por', text: 'por'}, 68 | {val : 'esp', text: 'esp'}, 69 | ] 70 | }, 71 | { 72 | description: 'Dest. Language: ', 73 | type: 'select', 74 | options: [ 75 | {val : 'fr', text: 'fr'}, 76 | {val : 'en', text: 'en'}, 77 | {val : 'de', text: 'de'}, 78 | {val : 'ru', text: 'ru'}, 79 | {val : 'it', text: 'it'}, 80 | {val : 'por', text: 'por'}, 81 | {val : 'esp', text: 'esp'}, 82 | ] 83 | } 84 | ], 85 | submitCallback: function(translation) { 86 | // Callback runs when the user submits the modal form 87 | // Receives one argument: the text input from the user 88 | 89 | var resp = this.translationValid(translation); 90 | if(resp['status'] == 1) { 91 | this.send(original, translation['inputs'][0], translation['inputs'][1], translation['inputs'][2]); 92 | } else { 93 | alert(resp['msg']); 94 | } 95 | }.bind(this) 96 | }) 97 | 98 | return {status: 1, msg: 'success'}; 99 | } 100 | } 101 | 102 | registerDataSource(translateText); 103 | -------------------------------------------------------------------------------- /datasources/text-translation/schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "original": "example message.,-=+?![]()", 3 | "translation": "example message.,-=+?![]()", 4 | "source": "example message.,-=+?![]()", 5 | "destination": "example message.,-=+?![]()" 6 | } 7 | -------------------------------------------------------------------------------- /datasources/twitter-test-again/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "twitter-test-again", 3 | "author": "Conor", 4 | "description": "This is the general Twitter datasource improved with a schema.", 5 | "version": "0.2", 6 | "sites": [".*twitter\\.com.*"] 7 | } 8 | -------------------------------------------------------------------------------- /datasources/twitter-test-again/plugin.js: -------------------------------------------------------------------------------- 1 | const twitterTestAgain = { 2 | mc: null, 3 | name: 'twitter-test-again', 4 | 5 | registerTweetActions: function (node) { 6 | let gThis = this; 7 | var tweets = node.getElementsByClassName("tweet"); 8 | 9 | for(var i=0; i {let result = value[0];result.data['primaryRank'] = value.length; result.data['primaryRankDisplay'] = 'Plays: ' + result.data.primaryRank; result.data['maxPrimaryRank'] = _.max(feed, function(item) { return item.data.primaryRank }).data.primaryRank; return result;}).sortBy(function(item) {return -item.data.primaryRank;}).value()", 5 | "display_icon": "fas fa-bolt" 6 | }, 7 | { 8 | "name": "Most Obscure", 9 | "function": "_.chain(feed).groupBy(function(o) {return o.data._str;}).map((value, key) => {let result = value[0];let obscurity = (value.length / parseInt(result.data['views']))*1000000;result.data['primaryRank'] = Math.round(obscurity * 100) / 100;result.data['primaryRankDisplay'] = result.data.primaryRank; result.data['maxPrimaryRank'] = _.max(feed, function(item) { return item.data.primaryRank }).data.primaryRank;return result;}).sortBy(function(item) {return item.data.primaryRank;}).value().reverse()", 10 | "display_icon": "fas fa-bolt" 11 | } 12 | ] -------------------------------------------------------------------------------- /datasources/youtube-music/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "youtube-music", 3 | "title": "YouTube Music", 4 | "author": "Rory", 5 | "description": "This runs on all youtube pages and collects the time a video is played, paused or finished at, along with the page title and URL. It only works on videos", 6 | "version": "0.5", 7 | "sites": ["^.*youtube\\.com.*$"] 8 | } 9 | -------------------------------------------------------------------------------- /datasources/youtube-music/plugin.js: -------------------------------------------------------------------------------- 1 | /* 2 | YouTube dynamically loads data, which means that we can't just scrape normally, we have to 3 | wait for the DOM to be updated. 4 | */ 5 | 6 | function isVideo(url) { 7 | let r = new RegExp("http(?:s?):\\/\\/(?:www\\.)?youtu(?:be\\.com\\/watch\\?v=|\\.be\\/)([\\w\\-\\_]*)(&(amp;)?‌​[\\w\\?‌​=]*)?") 8 | return r.test(url); 9 | } 10 | 11 | const youtubeMusic = { 12 | mc: null, 13 | name: 'youtube-music', 14 | 15 | operate: function(previousUrl) { 16 | let currentUrl = window.location.href; 17 | if(previousUrl == currentUrl) { 18 | return currentUrl; 19 | } else if(isVideo(currentUrl)) { // New page 20 | const self = this; 21 | setTimeout(function() { 22 | // Get the video player 23 | let $videoPlayer = $('#movie_player'); 24 | 25 | // YouTube loads data dynamically, so we must click the "show more" button 26 | // extract the category, and then click the "show less" button. 27 | self.showMore(); 28 | setTimeout(function() { 29 | self.createDatapoint(self, currentUrl, $videoPlayer); 30 | }, 10000); 31 | }, 2000); 32 | 33 | return currentUrl; 34 | } else { 35 | return currentUrl; 36 | } 37 | }, 38 | 39 | createDatapoint: function(self, url, $videoPlayer) { 40 | const category = this.getCategory(); 41 | let title = $('#container h1.title yt-formatted-string').text(); 42 | let views = $('#container div#info').find('span.view-count').text().replace(/\D/g,''); 43 | 44 | if($videoPlayer.hasClass('playing-mode')){ 45 | self.sendDatapoint("opened", category, url, title, views); 46 | } 47 | }, 48 | 49 | showMore: function() { 50 | let clickBtn = $('ytd-expander.ytd-video-secondary-info-renderer #more'); 51 | clickBtn.click(); // Click the "show more button" 52 | }, 53 | 54 | getCategory: function() { 55 | // Get the container div which has the #title and #content of the Category field 56 | let categoryContainer = $('ytd-expander.ytd-video-secondary-info-renderer ytd-metadata-row-renderer:has(#title yt-formatted-string:contains("Category"))'); 57 | // Extract the text from the #content div 58 | let category = categoryContainer.find('#content').text().trim(); 59 | // Click the "Show Less" button 60 | $('ytd-expander.ytd-video-secondary-info-renderer #less').click(); 61 | 62 | return category; 63 | }, 64 | 65 | validateDatapoint: function(datapoint) { 66 | // Only accept the datapoint if it's in the 'music' category 67 | if(datapoint['category'] == "Music") { 68 | return true; 69 | } 70 | 71 | console.log(`Video category is ${datapoint['category']} so not sending it`) 72 | return false; 73 | }, 74 | 75 | sendDatapoint: function(eventType, category, url, title, views) { 76 | // Create the datapoint: 77 | let datapoint = { 78 | _url: url, 79 | url: url, 80 | _str: title, 81 | _action: "Viewed", 82 | _timestamp: Date.now(), 83 | time: Date.now(), 84 | event: eventType, 85 | category: category, 86 | title: title, 87 | views: views 88 | }; 89 | 90 | // Log it 91 | console.log(datapoint); 92 | 93 | if(this.validateDatapoint(datapoint)) { 94 | // Validate and send the datapoint 95 | this.mc.sendDatapoint(datapoint); 96 | } 97 | }, 98 | 99 | initDataSource: function(metroClient) { 100 | // The entrypoint 101 | this.mc = metroClient; 102 | const self = this; 103 | 104 | let url = ""; 105 | setInterval(function() { 106 | url = self.operate(url); 107 | }, 500); 108 | } 109 | } 110 | 111 | registerDataSource(youtubeMusic); -------------------------------------------------------------------------------- /datasources/youtube-music/schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "_url": "...", 3 | "_timestamp": "...", 4 | "_str": "...", 5 | "_action": "...", 6 | "event": "[play|pause|finished]", 7 | "time": "1234567890", 8 | "url": "The current page URL", 9 | "title": "The page title", 10 | "category": "Category of the video", 11 | "views": "0123456789" 12 | } 13 | -------------------------------------------------------------------------------- /datasources/youtube-soundtracks/aggregations.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "Most Popular", 4 | "function": "_.chain(feed).groupBy(function(o) {return o.data._str;}).map((value, key) => {let result = value[0];result.data['primaryRank'] = value.length; result.data['primaryRankDisplay'] = 'Plays: ' + result.data.primaryRank; result.data['maxPrimaryRank'] = _.max(feed, function(item) { return item.data.primaryRank }).data.primaryRank; return result;}).sortBy(function(item) {return -item.data.primaryRank;}).value()", 5 | "display_icon": "fas fa-bolt" 6 | }, 7 | { 8 | "name": "Best Obscure", 9 | "function": "_.chain(feed).groupBy(function(o) {return o.data._str;}).map((value, key) => {let result = value[0];let obscurity = (value.length / parseInt(result.data['views']))*1000000;result.data['primaryRank'] = Math.round(obscurity * 100) / 100;result.data['primaryRankDisplay'] = result.data.primaryRank; result.data['maxPrimaryRank'] = _.max(feed, function(item) { return item.data.primaryRank }).data.primaryRank; console.log(result);return result;}).sortBy(function(item) {return item.data.primaryRank;}).value().reverse()", 10 | "display_icon": "fas fa-bolt" 11 | }, 12 | { 13 | "name": "Longest", 14 | "function": "_.chain(feed).groupBy(function(o) {return o.data._str;}).map((value, key) => {let result = value[0];result.data['primaryRank'] = result.data.lengthSeconds; var d = result.data.lengthSeconds;var h = Math.floor(d / 3600); var m = Math.floor(d % 3600 / 60);var s = Math.floor(d % 3600 % 60);result.data['primaryRankDisplay'] = ('0' + h).slice(-2) + ':' + ('0' + m).slice(-2) + ':' + ('0' + s).slice(-2); result.data['maxPrimaryRank'] = _.max(feed, function(item) { return item.data.primaryRank }).data.primaryRank; console.log(result);return result;}).sortBy(function(item) {return -item.data.primaryRank;}).value()", 15 | "display_icon": "fas fa-bolt" 16 | }, 17 | { 18 | "name": "Recently Released", 19 | "function": "_.chain(feed).sortBy(item => -item.data.uploadTimestamp).groupBy(function(o) {return o.data._str;}).map((value, key) => {let result = value[0]; console.log(result); result.data['primaryRank'] = result.data.uploadTimestamp - feed[feed.length - 1].data.uploadTimestamp; var a = new Date(result.data.uploadTimestamp*1000);var months = ['Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sep','Oct','Nov','Dec'];var month = months[a.getMonth()];var date = a.getDate(); var year = a.getFullYear(); var time = date + ' ' + month + ', ' + year ; result.data['primaryRankDisplay'] = time; console.log(result);return result;}).value()", 20 | "display_icon": "fas fa-bolt" 21 | } 22 | ] -------------------------------------------------------------------------------- /datasources/youtube-soundtracks/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "youtube-soundtracks", 3 | "title": "YouTube Soundtracks/OST", 4 | "author": "Rory", 5 | "description": "This runs on all youtube pages and collects the time a soundtrack video is played, paused or finished at, along with the page title and URL. It only works on videos", 6 | "version": "0.0.2", 7 | "sites": ["^.*youtube\\.com.*$"] 8 | } 9 | -------------------------------------------------------------------------------- /datasources/youtube-soundtracks/plugin.js: -------------------------------------------------------------------------------- 1 | const soundtrackWords = [ 2 | 'vgm', 3 | 'ost', 4 | 'soundtrack', 5 | 'videogame', 6 | 'video game', 7 | 'motion picture' 8 | ] 9 | 10 | function isSoundtrack(title) { 11 | for(i = 0; i < soundtrackWords.length; i++) { 12 | let w = soundtrackWords[i]; 13 | if(title.toLowerCase().indexOf(w) !== -1) { 14 | return true; 15 | } 16 | } 17 | 18 | return false; 19 | } 20 | 21 | /* 22 | YouTube Soundtrack DataSource 23 | 24 | Captures the details of movie/game soundtracks on YouTube 25 | 26 | YouTube dynamically loads data, which means that we can't just scrape normally, we have to 27 | wait for the DOM to be updated. 28 | */ 29 | 30 | function isVideo(url) { 31 | let r = new RegExp("http(?:s?):\\/\\/(?:www\\.)?youtu(?:be\\.com\\/watch\\?v=|\\.be\\/)([\\w\\-\\_]*)(&(amp;)?‌​[\\w\\?‌​=]*)?") 32 | return r.test(url); 33 | } 34 | 35 | const youtubeSoundtracks = { 36 | mc: null, 37 | name: 'youtube-soundtracks', 38 | 39 | operate: function(previousUrl) { 40 | let currentUrl = window.location.href; 41 | if(previousUrl == currentUrl) { 42 | return currentUrl; 43 | } else if(isVideo(currentUrl)) { // New page 44 | const self = this; 45 | setTimeout(function() { 46 | // Get the video player 47 | let $videoPlayer = $('#movie_player'); 48 | 49 | // YouTube loads data dynamically, so we must click the "show more" button 50 | // extract the category, and then click the "show less" button. 51 | self.showMore(); 52 | setTimeout(function() { 53 | self.createDatapoint(self, currentUrl, $videoPlayer); 54 | }, 10000); 55 | }, 2000); 56 | 57 | return currentUrl; 58 | } else { 59 | return currentUrl; 60 | } 61 | }, 62 | 63 | createDatapoint: function(self, url, $videoPlayer) { 64 | const category = this.getCategory(); 65 | let title = $('#container h1.title yt-formatted-string').text(); 66 | let views = $('#container div#info').find('span.view-count').text().replace(/\D/g,''); 67 | let lengthStr = $videoPlayer.find('.ytp-bound-time-right').text(); 68 | let seconds = lengthStr.split(':').reduce((acc,time) => (60 * acc) + +time) 69 | 70 | let date = $('span.date').text() 71 | let uploadTimestamp = new Date(date).getTime()/1000 72 | 73 | if($videoPlayer.hasClass('playing-mode')){ 74 | self.sendDatapoint("Listened", category, url, title, views, seconds, uploadTimestamp); 75 | } 76 | }, 77 | 78 | showMore: function() { 79 | let clickBtn = $('ytd-expander.ytd-video-secondary-info-renderer #more'); 80 | clickBtn.click(); // Click the "show more button" 81 | }, 82 | 83 | getCategory: function() { 84 | // Get the container div which has the #title and #content of the Category field 85 | let categoryContainer = $('ytd-expander.ytd-video-secondary-info-renderer ytd-metadata-row-renderer:has(#title yt-formatted-string:contains("Category"))'); 86 | // Extract the text from the #content div 87 | let category = categoryContainer.find('#content').text().trim(); 88 | // Click the "Show Less" button 89 | $('ytd-expander.ytd-video-secondary-info-renderer #less').click(); 90 | 91 | return category; 92 | }, 93 | 94 | validateDatapoint: function(datapoint) { 95 | // Only accept the datapoint if it's in the 'music' category 96 | if(!(datapoint['category'] == "Music")) { 97 | console.log(`[Metro - YouTube Soundtrack] Video category is ${datapoint['category']} so not sending it`) 98 | return false; 99 | } else if(!isSoundtrack(datapoint['title'])) { 100 | console.log(`[Metro - YouTube Soundtrack] Video title doesn't look like a soundtrack, so not sending it`) 101 | return false; 102 | } 103 | 104 | return true; 105 | }, 106 | 107 | sendDatapoint: function(action, category, url, title, views, seconds, uploadTimestamp) { 108 | // Create the datapoint: 109 | let datapoint = { 110 | _url: url, 111 | _str: title, 112 | _timestamp: Date.now(), 113 | _action: action, 114 | time: Date.now(), 115 | category: category, 116 | title: title, 117 | views: views, 118 | lengthSeconds: seconds, 119 | uploadTimestamp: uploadTimestamp 120 | }; 121 | 122 | // Log it 123 | console.log(datapoint); 124 | 125 | if(this.validateDatapoint(datapoint)) { 126 | // Validate and send the datapoint 127 | this.mc.sendDatapoint(datapoint); 128 | } 129 | }, 130 | 131 | initDataSource: function(metroClient) { 132 | // The entrypoint 133 | this.mc = metroClient; 134 | const self = this; 135 | 136 | let url = ""; 137 | setInterval(function() { 138 | url = self.operate(url); 139 | }, 500); 140 | } 141 | } 142 | 143 | registerDataSource(youtubeSoundtracks); -------------------------------------------------------------------------------- /datasources/youtube-soundtracks/schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "_str": "String representation of the datapoint", 3 | "_url": "The current page URL", 4 | "_timestamp": "Timestamp of the event", 5 | "_action": "Action taken", 6 | "time": "1234567890", 7 | "title": "The page title", 8 | "category": "Category of the video", 9 | "views": "0123456789", 10 | "lengthSeconds": "0123456789", 11 | "uploadTimestamp": "0123456789" 12 | } 13 | --------------------------------------------------------------------------------