├── .gitignore ├── README.md ├── lexrank.js └── package.json /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | gs 3 | logs 4 | *.log 5 | 6 | # Runtime data 7 | pids 8 | *.pid 9 | *.seed 10 | 11 | # Directory for instrumented libs generated by jscoverage/JSCover 12 | lib-cov 13 | 14 | # Coverage directory used by tools like istanbul 15 | coverage 16 | 17 | # Grunt intermediate storage 18 | # (http://gruntjs.com/creating-plugins#storing-task-files) 19 | .grunt 20 | 21 | # node-waf configuration 22 | .lock-wscript 23 | 24 | # Compiled binary addons (http://nodejs.org/api/addons.html) 25 | build/Release 26 | 27 | # Dependency directory 28 | # https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git 29 | node_modules 30 | 31 | test.js 32 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # lexrank.js 2 | 3 | Implements Radev's Lexrank algorithm [http://www.jair.org/papers/paper1523.html](http://www.jair.org/papers/paper1523.html) for unsupervised text summarization in node. Basically applying PageRank to each sentence in a document, finding the most important sentences, and reranking them. 4 | 5 | ## Quick Start 6 | 7 | ```js 8 | var lexrank = require('lexrank'); 9 | ``` 10 | 11 | The `lexrank` provides two functions: 12 | 13 | ```js 14 | lexrank.summarize(text, lineCount, callback) 15 | ``` 16 | 17 | where 18 | 19 | - `text` is the original text. There's no need to tokenize the text. Raw text is fine. `lexrank.js` does a sentence tokenization and then word tokenization. 20 | - `lineCount` is the number of sentences in the output desired. 21 | 22 | ```js 23 | lexrank.summarizePage(url, lineCount, callback) 24 | ``` 25 | 26 | where 27 | 28 | - `url` is the URL of the page to summarize. `lexrank` then uses [`node-unfluff`](https://github.com/ageitgey/node-unfluff) to extract the text content from the page, then passes it to `lexrank.summarize` 29 | 30 | The other parameters are the same. 31 | 32 | ## Examples 33 | 34 | ```js 35 | var lexrank = require('lexrank'); 36 | var originalText = 'some...text...here...'; 37 | var topLines = lexrank.summarize(originalText, 5, function (err, toplines, text) { 38 | if (err) { 39 | console.log(err); 40 | } 41 | console.log(toplines); 42 | // [{ 43 | // weight: 0.16398118821187135, 44 | // text: 'The intention of all these rules is to prevent a repeat of the bankruptcies and bail-outs of 2008.', 45 | // index: 3 46 | // }, { 47 | // weight: 0.1603637724075845, 48 | // text: 'By throttling the bits of banks that “make markets” in bonds, shares, currencies and commodities, the theory goes, watchdogs have made such assets less liquid.', 49 | // index: 5 50 | // }, { 51 | // weight: 0.1670898954042791, 52 | // text: 'Meanwhile, the value of outstanding bonds has swollen to record levels, most of them in the hands of asset managers (see chart).', 53 | // index: 12 54 | // }, { 55 | // weight: 0.17266004699219448, 56 | // text: 'That is in part a corollary of banks trimming lending, and so pushing borrowers to the bond market instead, and in part a natural response to low interest rates.', 57 | // index: 13 58 | // }, { 59 | // weight: 0.16298329363189878, 60 | // text: 'They also suspect, however, that the high level of liquidity before the crisis was an anomaly that bankers are harping on about in an effort to roll back regulation.', 61 | // index: 32 62 | // }] 63 | 64 | console.log(text); 65 | // The intention of all these rules is to prevent a repeat of the bankruptcies 66 | // and bail-outs of 2008. By throttling the bits of banks that “make markets” in 67 | // bonds, shares, currencies and commodities, the theory goes, watchdogs have 68 | // made such assets less liquid. Meanwhile, the value of outstanding bonds has 69 | // swollen to record levels, most of them in the hands of asset managers (see 70 | // chart). That is in part a corollary of banks trimming lending, and so pushing 71 | // borrowers to the bond market instead, and in part a natural response to low 72 | // interest rates. They also suspect, however, that the high level of liquidity 73 | // before the crisis was an anomaly that bankers are harping on about in an 74 | // effort to roll back regulation. 75 | }); 76 | ``` 77 | 78 | lexrank returns both a toplines object which contains an array of the top sentences objects together with their text, eigenvalues associated with them ,and their sentence index. 79 | 80 | Since most of us will be using this to grab text from sites, lexrank can take URLs as well. 81 | 82 | ```js 83 | var lexrank = require('lexrank'); 84 | var url = 'http://www.economist.com/news/finance-and-economics/21648673-regulators-have-made-banking-safer-has-made-markets-riskier-frozen'; 85 | var topLines = lexrank.summarizePage(url, 5, function (err, topLines, text) { 86 | if (err) { 87 | console.log(err); 88 | } 89 | console.log(toplines); 90 | 91 | console.log(text); 92 | }); 93 | ``` 94 | 95 | ## Evaluation 96 | 97 | This is **not** a great summarization algorithm. However, it is an improvement on picking the first few sentences at little to no cost in time. 98 | 99 | For example using this [article from The Economist on liquidity in financial markets](http://www.economist.com/news/finance-and-economics/21648673-regulators-have-made-banking-safer-has-made-markets-riskier-frozen): 100 | 101 | The first paragraph shows 102 | 103 | > TO ENSURE that it meets the 750 new rules on capital imposed in the aftermath of the financial crisis, JPMorgan Chase employs over 950 people. A further 400 or so try to follow around 500 regulations on the liquidity of its assets, designed to stop the bank toppling over if markets seize up. A team of 300 is needed to monitor compliance with the Volcker rule, which in almost 1,000 pages restricts banks from trading on their own account. 104 | 105 | lexrank produces 106 | 107 | > The intention of all these rules is to prevent a repeat of the bankruptcies and bail-outs of 2008. By throttling the bits of banks that “make markets” in bonds, shares, currencies and commodities, the theory goes, watchdogs have made such assets less liquid. Meanwhile, the value of outstanding bonds has swollen to record levels, most of them in the hands of asset managers (see chart). That is in part a corollary of banks trimming lending, and so pushing borrowers to the bond market instead, and in part a natural response to low interest rates. They also suspect, however, that the high level of liquidity before the crisis was an anomaly that bankers are harping on about in an effort to roll back regulation. 108 | 109 | Which is way more relevant than the story on JPMorgan. I'm sorry JPMorgan. 110 | 111 | ## License 112 | 113 | The MIT License (MIT) 114 | 115 | Copyright (c) 2015 [Linan Qiu](github.com/linanqiu) 116 | 117 | Permission is hereby granted, free of charge, to any person obtaining a copy 118 | of this software and associated documentation files (the "Software"), to deal 119 | in the Software without restriction, including without limitation the rights 120 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 121 | copies of the Software, and to permit persons to whom the Software is 122 | furnished to do so, subject to the following conditions: 123 | 124 | The above copyright notice and this permission notice shall be included in 125 | all copies or substantial portions of the Software. 126 | 127 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 128 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 129 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 130 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 131 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 132 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 133 | THE SOFTWARE. 134 | -------------------------------------------------------------------------------- /lexrank.js: -------------------------------------------------------------------------------- 1 | (function () { 2 | 'use strict'; 3 | var unfluff = require('unfluff'); 4 | var request = require('request'); 5 | var Tokenizer = require('sentence-tokenizer'); 6 | var natural = require('natural'); 7 | var wuzzy = require('wuzzy'); 8 | 9 | function summarizePage(url, lines, callback) { 10 | request(url, function (err, resp, body) { 11 | if (err || resp.statusCode != 200) { 12 | return callback(err); 13 | } 14 | var text = unfluff(body).text; 15 | 16 | if (!text) { 17 | return callback(new Error('no text to summarize')); 18 | } 19 | 20 | summarize(text, lines, callback); 21 | }); 22 | } 23 | 24 | function summarize(text, lines, callback) { 25 | var sentenceTokenizer = new Tokenizer('utterer'); 26 | sentenceTokenizer.setEntry(text); 27 | var sentences = sentenceTokenizer.getSentences(); 28 | var sentencesOriginal = sentences.slice(); 29 | 30 | var wordTokenizer = new natural.TreebankWordTokenizer(); 31 | sentences.forEach(function (sentence, index, array) { 32 | array[index] = wordTokenizer.tokenize(sentence.toLowerCase()); 33 | }); 34 | 35 | var matrix = constructMatrix(sentences); 36 | var sortedSentences = pageRank(matrix, sentencesOriginal); 37 | 38 | var topLines = []; 39 | 40 | for (var i = 0; i < Math.min(lines, sortedSentences.length); i++) { 41 | topLines.push(sortedSentences[i]); 42 | } 43 | 44 | topLines.sort(function (a, b) { 45 | return a.index - b.index; 46 | }); 47 | 48 | var concatenated = ''; 49 | 50 | for(var i = 0 ; i < topLines.length; i++) { 51 | concatenated += topLines[i].text + ' '; 52 | } 53 | 54 | callback(false, topLines, concatenated); 55 | } 56 | 57 | function fillArray(length, value) { 58 | var array = []; 59 | for (var i = 0; i < length; i++) { 60 | array[i] = value; 61 | } 62 | return array; 63 | } 64 | 65 | function pageRank(matrix, sentencesOriginal) { 66 | var eigen = fillArray(sentencesOriginal.length, 1); 67 | 68 | for (var h = 0; h < 10; h++) { 69 | var w = fillArray(sentencesOriginal.length, 0); 70 | 71 | for (var i = 0; i < sentencesOriginal.length; i++) { 72 | for (var j = 0; j < sentencesOriginal.length; j++) { 73 | w[i] = w[i] + (matrix[i][j] * eigen[j]); 74 | } 75 | } 76 | eigen = normalize(w); 77 | } 78 | 79 | // sort bags according to eigen value 80 | var eigenCounts = []; 81 | 82 | for (var i = 0; i < sentencesOriginal.length; i++) { 83 | var eigenObject = { 84 | weight: eigen[i], 85 | text: sentencesOriginal[i], 86 | index: i 87 | } 88 | eigenCounts.push(eigenObject); 89 | } 90 | eigenCounts.sort(function (a, b) { 91 | return b.weight - a.weight; 92 | }); 93 | 94 | return eigenCounts; 95 | } 96 | 97 | function constructMatrix(sentences, threshold) { 98 | var matrix = []; 99 | for (var i = 0; i < sentences.length; i++) { 100 | matrix[i] = []; 101 | 102 | var sentenceA = sentences[i]; 103 | 104 | for (var j = 0; j < sentences.length; j++) { 105 | var sentenceB = sentences[j]; 106 | var value = wuzzy.tanimoto(sentenceB, sentenceA); 107 | if(!!threshold && value < threshold) { 108 | value = 0; 109 | } 110 | matrix[i][j] = value; 111 | } 112 | 113 | matrix[i] = normalize(matrix[i]); 114 | } 115 | return matrix; 116 | } 117 | 118 | function normalize(array) { 119 | var distance = 0; 120 | 121 | for (var i = 0; i < array.length; i++) { 122 | distance += array[i] * array[i]; 123 | } 124 | 125 | distance = Math.sqrt(distance); 126 | 127 | for (var i = 0; i < array.length; i++) { 128 | array[i] = array[i] / distance; 129 | } 130 | 131 | return array; 132 | } 133 | 134 | module.exports = { 135 | summarize: summarize, 136 | summarizePage: summarizePage 137 | } 138 | })(); 139 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "lexrank", 3 | "version": "1.0.3", 4 | "description": "Text summarization using Lexrank", 5 | "main": "lexrank.js", 6 | "dependencies": { 7 | "natural": "^0.2.1", 8 | "request": "^2.55.0", 9 | "sentence-tokenizer": "^0.0.7", 10 | "unfluff": "^0.10.0", 11 | "wuzzy": "^0.1.2" 12 | }, 13 | "devDependencies": {}, 14 | "scripts": { 15 | "test": "echo \"Error: no test specified\" && exit 1" 16 | }, 17 | "repository": { 18 | "type": "git", 19 | "url": "https://github.com/linanqiu/lexrank" 20 | }, 21 | "keywords": [ 22 | "lexrank" 23 | ], 24 | "author": "linanqiu", 25 | "license": "MIT", 26 | "bugs": { 27 | "url": "https://github.com/linanqiu/lexrank/issues" 28 | }, 29 | "homepage": "https://github.com/linanqiu/lexrank" 30 | } 31 | --------------------------------------------------------------------------------