s
616 | .replace(/(?:(?:\s| ?)*)+(?=<\/?p)/g, "")
617 | // Remove spaces in front of s
618 | .replace(/(?:\s| ?)+(?=)/g, "")
619 | // Turn all double+ s into
s
620 | .replace(/(?:){2,}/g, "
")
621 | // Trim the result
622 | .trim()
623 | );
624 | }
625 |
626 | getText(node = this._getCandidateNode()) {
627 | return node
628 | .getFormattedText()
629 | .trim()
630 | .replace(/\n+(?=\n{2})/g, "");
631 | }
632 |
633 | getEvents(cbs) {
634 | (function process(node) {
635 | cbs.onopentag(node.name, node.attributes);
636 | node.children.forEach((child) =>
637 | typeof child === "string" ? cbs.ontext(child) : process(child)
638 | );
639 | cbs.onclosetag(node.name);
640 | })(this._getCandidateNode());
641 | }
642 |
643 | getArticle(type) {
644 | const elem = this._getCandidateNode();
645 |
646 | const ret = {
647 | title: this._headerTitle || this.getTitle(),
648 | nextPage: this.getNextPage(),
649 | textLength: elem.info.textLength,
650 | score: this._topCandidate ? this._topCandidate.totalScore : 0,
651 | };
652 |
653 | if (!type && this._settings.type) ({ type } = this._settings);
654 |
655 | if (type === "text") ret.text = this.getText(elem);
656 | else ret.html = this.getHTML(elem);
657 |
658 | return ret;
659 | }
660 | }
661 |
662 | module.exports = Readability;
663 |
--------------------------------------------------------------------------------
/readme.md:
--------------------------------------------------------------------------------
1 | # readabilitySAX
2 |
3 | a fast and platform independent readability port
4 |
5 | ## About
6 |
7 | This is a port of the algorithm used by the
8 | [Readability](http://code.google.com/p/arc90labs-readability/) bookmarklet to
9 | extract relevant pieces of information from websites, using a SAX parser.
10 |
11 | The advantage over other ports, e.g.
12 | [arrix/node-readability](https://github.com/arrix/node-readability), is a
13 | smaller memory footprint and a much faster execution. In my tests, most pages,
14 | even large ones, were finished within 15ms (on node, see below for more
15 | information). It works with Rhino, so it runs on
16 | [YQL](http://developer.yahoo.com/yql "Yahoo! Query Language"), which may have
17 | interesting uses. And it works within a browser.
18 |
19 | The Readability extraction algorithm was completely ported, but some adjustments
20 | were made:
21 |
22 | - `` and `` tags are recognized and gain a higher value
23 |
24 | - If a heading is part of the pages ``, it is removed (Readability
25 | removed any single `<h2>`, and ignored other tags)
26 |
27 | - `henry` and `instapaper-body` are classes to show an algorithm like this
28 | where the content is. readabilitySAX recognizes them and adds additional
29 | points
30 |
31 | - Every bit of code that was taken from the original algorithm was optimized,
32 | eg. RegExps should now perform faster (they were optimized & use
33 | `RegExp#test` instead of `String#match`, which doesn't force the interpreter
34 | to build an array)
35 |
36 | - Some improvements made by
37 | [GGReadability](https://github.com/curthard89/COCOA-Stuff/tree/master/GGReadability)
38 | (an Obj-C port of Readability) were adopted
39 | - Images get additional scores when their `height` or `width` attributes
40 | are high - icon sized images (<= 32px) get skipped
41 | - Additional classes & ids are checked
42 |
43 | ## How To
44 |
45 | ### Install readabilitySAX
46 |
47 | npm install readabilitySAX
48 |
49 | ##### CLI
50 |
51 | A command line interface (CLI) may be installed via
52 |
53 | npm install -g readabilitySAX
54 |
55 | It's then available via
56 |
57 | readability <domain> [<format>]
58 |
59 | To get this readme, just run
60 |
61 | readability https://github.com/FB55/readabilitySAX
62 |
63 | The format is optional (it's either `text` or `html`, the default value is
64 | `text`).
65 |
66 | ### Usage
67 |
68 | ##### Node
69 |
70 | Just run `require("readabilitySAX")`. You'll get an object containing three
71 | methods:
72 |
73 | - `Readability(settings)`: The readability constructor. It works as a handler
74 | for `htmlparser2`. Read more about it
75 | [in the wiki](https://github.com/FB55/readabilitySAX/wiki/The-Readability-constructor)!
76 |
77 | - `WritableStream(settings, cb)`: A constructor that unites `htmlparser2` and
78 | the `Readability` constructor. It's a writable stream, so simply `.write`
79 | all your data to it. Your callback will be called once `.end` was called.
80 | Bonus: You can also `.pipe` data into it!
81 |
82 | - `createWritableStream(settings, cb)`: Returns a new instance of the
83 | `WritableStream`. (It's a simple factory method.)
84 |
85 | There are two methods available that are deprecated and **will be removed** in a
86 | future version:
87 |
88 | - `get(link, [settings], callback)`: Gets a webpage and process it.
89 |
90 | - `process(data)`: Takes a string, runs readabilitySAX and returns the page.
91 |
92 | **Please don't use those two methods anymore**. Streams are the way you should
93 | build interfaces in node, and that's what I want encourage people to use.
94 |
95 | ##### Browsers
96 |
97 | I started to implement simplified SAX-"parsers" for Rhino/YQL (using E4X) and
98 | the browser (using the DOM) to increase the overall performance on those
99 | platforms. The DOM version is inside the `/browsers` dir.
100 |
101 | A demo of how to use readabilitySAX inside a browser may be found at
102 | [jsFiddle](http://jsfiddle.net/pXqYR/embedded/). Some basic example files are
103 | inside the `/browsers` directory.
104 |
105 | ##### YQL
106 |
107 | A table using E4X-based events is available as the community table
108 | `redabilitySAX`, as well as
109 | [here](https://github.com/FB55/yql-tables/tree/master/readabilitySAX).
110 |
111 | ## Parsers (on node)
112 |
113 | Most SAX parsers (as sax.js) fail when a document is malformed XML, even if it's
114 | correct HTML. readabilitySAX should be used with
115 | [htmlparser2](http://npm.im/htmlparser2), my fork of the `htmlparser`-module
116 | (used by eg. `jsdom`), which corrects most faults. It's listed as a dependency,
117 | so npm should install it with readabilitySAX.
118 |
119 | ## Performance
120 |
121 | ##### Speed
122 |
123 | Using a package of 724 pages from [CleanEval](http://cleaneval.sigwac.org.uk)
124 | (their website seems to be down, try to google it), readabilitySAX processed all
125 | of them in 5768 ms, that's an average of 7.97 ms per page.
126 |
127 | The benchmark was done using `tests/benchmark.js` on a MacBook (late 2010) and
128 | is probably far from perfect.
129 |
130 | Performance is the main goal of this project. The current speed should be good
131 | enough to run readabilitySAX on a singe-threaded web server with an average
132 | number of requests. That's an accomplishment!
133 |
134 | ##### Accuracy
135 |
136 | The main goal of CleanEval is to evaluate the accuracy of an algorithm.
137 |
138 | **_// TODO_**
139 |
140 | ## Todo
141 |
142 | - Add documentation & examples
143 | - Add support for URLs containing hash-bangs (`#!`)
144 | - Allow fetching articles with more than one page
145 | - Don't remove all images inside `<a>` tags
146 |
--------------------------------------------------------------------------------
/tests/benchmark.js:
--------------------------------------------------------------------------------
1 | const { createWritableStream } = require("../");
2 | const fs = require("fs");
3 | const dir = "/Users/felix/Downloads/CleanEval/finalrun-input/";
4 | const files = fs.readdirSync(dir);
5 | let time = 0;
6 | const total = files.length;
7 | let skipped = 0;
8 | let min = 1 / 0;
9 | let max = -1 / 0;
10 |
11 | function run(name) {
12 | if (!name || name.charAt(0) === ".") return proc();
13 |
14 | const file = fs.readFileSync(dir + name).toString();
15 | const start = Date.now();
16 |
17 | createWritableStream((ret) => {
18 | if (!ret.score) skipped++;
19 | else {
20 | const took = Date.now() - start;
21 | time += took;
22 | if (took < min) min = took;
23 | if (took > max) max = took;
24 | }
25 | }).end(file);
26 | }
27 |
28 | function proc() {
29 | if (!files.length) return;
30 | run(files.pop());
31 | process.nextTick(proc);
32 | if (files.length % 10 === total % 10) {
33 | console.log("did", total - files.length);
34 | }
35 | }
36 |
37 | proc();
38 |
39 | process.on("exit", () => {
40 | const did = total - skipped;
41 | console.log("took", time);
42 | console.log("runs", did);
43 | console.log("average", Math.round((time / did) * 1e4) / 1e4);
44 | console.log("min", min);
45 | console.log("max", max);
46 | console.log("skipped", skipped);
47 | });
48 |
--------------------------------------------------------------------------------
/tests/cleaneval.js:
--------------------------------------------------------------------------------
1 | const getReadableContent = require("../").process;
2 | const fs = require("fs");
3 | const dir = "/Users/felix/Downloads/CleanEval/";
4 | const input = `${dir}finalrun-input/`;
5 | const output = `${dir}finalrun-output/`;
6 | const ents = require("entities");
7 |
8 | fs.readdirSync(input).forEach((name) => {
9 | if (!name || name.charAt(0) === ".") return;
10 |
11 | const ret = getReadableContent(fs.readFileSync(input + name).toString(), {
12 | type: "text",
13 | });
14 |
15 | // If(ret.score < 100) return;
16 |
17 | fs.writeFileSync(
18 | output + name.replace(".html", ".txt"),
19 | (ret.title ? `${ret.title}\n\n` : "") + ents.decodeHTML5(ret.text)
20 | );
21 | });
22 |
23 | console.log("Finished all files!");
24 |
25 | const check = require("child_process").spawn("python", [
26 | `${dir}cleaneval.py`,
27 | "-t",
28 | output,
29 | `${dir}GoldStandard/`,
30 | ]);
31 |
32 | check.stdout.pipe(process.stdout);
33 | check.stderr.pipe(process.stderr);
34 |
--------------------------------------------------------------------------------
/tests/test_output.js:
--------------------------------------------------------------------------------
1 | const assert = require("assert");
2 | const { Parser } = require("htmlparser2");
3 | const Readability = require("../readabilitySAX");
4 | const file = require("fs").readFileSync(`${__dirname}/testpage.html`, "utf8");
5 |
6 | const expectedLinks = 2;
7 |
8 | const expectedUrl = {
9 | protocol: "http:",
10 | domain: "foo.bar",
11 | path: ["this.2", "is", "a", "long", "path"],
12 | full: "http://foo.bar/this.2/is/a/long/path/index?isnt=it",
13 | };
14 |
15 | const expectedData = {
16 | title: "Realtime Performance Visualizations using Node.js",
17 | nextPage: "http://howtonode.org/heat-tracer/dummy/page/2",
18 | textLength: 11668,
19 | score: 83,
20 | html: "<a href=\"http://howtonode.org/243bfe84f43affd3244e1828d90a8dca7fcc34c4/heat-tracer\">Static Version</a><p>This article outlines how to create a realtime heatmap of your syscall latency using HTML5, some great node modules, and DTrace. It was inspired by talk that Bryan Cantrill and Brendan Greg gave on Joyent's cool cloud analytics tools. While specific, the code provided could easily be adapted to provide a heatmap of any type of aggregation Dtrace is capable of providing. </p>\n\n<h2>System Requirements</h2>\n\n<p>First thing's first, you're going to need a system with DTrace. This likely means Solaris (or one of its decedents), OS X, or a BSD variant. There doesn't appear to be Dtrace available for Linux. </p>\n\n<h2>Security</h2>\n\n<p>Secondly, please be aware that at the time of writing the demo code contains a fairly substantial secruity vulnerabilty. Namely the d script is sent from the client with no authentication what so ever. If you bind to localhost this shouldn't be a big deal for a demo. Time permitting I intend to clean up the code. </p>\n\n<h2>Dependencies</h2>\n\n<p>For this tutorial you'll also need:</p>\n\n<pre><code>node - http://nodejs.org/#download (duh)<br/>npm - https://github.com/isaacs/npm (makes installing modules a breeze)<br/>node-libdtrace - https://github.com/bcantrill/node-libdtrace (provides dtrace functionality)<br/>Socket.IO - 'npm install socket.io' (web sockets made easy)<br/></code></pre>\n\n<h2>Server</h2>\n\n<p>Now we're ready to start writing our web server: </p>\n\n<p><embed src=\"http://youtube.com/\"></embed> This is just a test embed! </p>\n\n<div><a href=\"http://howtonode.org/heat-tracer/heat-tracer/heat_tracer.js\">heat_tracer.js</a><pre><code>var http = require('http');<br/>var libdtrace = require('libdtrace');<br/>var io = require('socket.io');<br/>var express = require('express');</p><p>/* create our express server and prepare to serve javascript files in ./public<br/>*/<br/>var app = express.createServer();<br/>app.configure(function(){<br/> app.use(express.staticProvider(__dirname + '/public'));<br/> });</p><p>/* Before we go any further we must realize that each time a user connects we're going to want to<br/> them send them dtrace aggregation every second. We can do so using 'setInterval', but we must<br/> keep track of both the intervals we set and the dtrace consumers that are created as we'll need<br/> them later when the client disconnects.<br/>*/<br/>var interval_id_by_session_id = {};<br/>var dtp_by_session_id = {};</p><p>/* In order to effecienctly send packets we're going to use the Socket.IO library which seemlessly<br/> integrates with express.<br/>*/<br/>var websocket_server = io.listen(app);</p><p>/* Now that we have a web socket server, we need to create a handler for connection events. These<br/> events represet a client connecting to our server */<br/>websocket_server.on('connection', function(socket) {</p><p> /* Like the web server object, we must also define handlers for various socket events that<br/> will happen during the lifetime of the connection. These will define how we interact with<br/> the client. The first is a message event which occurs when the client sends something to<br/> the server. */<br/> socket.on( 'message', function(message) {<br/> /* The only message the client ever sends will be sent right after connecting.<br/> So it will happen only once during the lifetime of a socket. This message also<br/> contains a d script which defines an agregation to walk.<br/> */<br/> var dtp = new libdtrace.Consumer();<br/> var dscript = message['dscript'];<br/> console.log( dscript );<br/> dtp.strcompile(dscript);<br/> dtp.go();<br/> dtp_by_session_id[socket.sessionId] = dtp;</p><p> /* All that's left to do is send the aggration data from the dscript. */<br/> interval_id_by_session_id[socket.sessionId] = setInterval(function () {<br/> var aggdata = {};<br/> try {<br/> dtp.aggwalk(function (id, key, val) {<br/> for( index in val ) {<br/> /* console.log( 'key: ' + key + ', interval: ' +<br/> val[index][0][0] + '-' + val[index][0][1], ', count ' + val[index][1] ); */</p><p> aggdata[key] = val;<br/> }<br/> } );<br/> socket.send( aggdata );<br/> } catch( err ) {<br/> console.log(err);<br/> }</p><p> }, 1001 );<br/> } );</p><p> /* Not so fast. If a client disconnects we don't want their respective dtrace consumer to<br/> keep collecting data any more. We also don't want to try to keep sending anything to them<br/> period. So clean up. */<br/> socket.on('disconnect', function(){<br/> clearInterval(clearInterval(interval_id_by_session_id[socket.sessionId]));<br/> var dtp = dtp_by_session_id[socket.sessionId];<br/> delete dtp_by_session_id[socket.sessionId];<br/> dtp.stop();<br/> console.log('disconnected');<br/> });</p><p> } );</p><p>app.listen(80);</code></pre></div>\n\n<h2>Client</h2>\n\n<p>In order to display our heatmap, we're going to need some basic HTML with a canvas element:</p>\n\n<div><a href=\"http://howtonode.org/heat-tracer/heat-tracer/public/heat_tracer.html\">public/heat_tracer.html</a><pre><code><html><br/><head><br/><script src=\"http://localhost/socket.io/socket.io.js\"></script><br/><script src=\"http://localhost/heat_tracer_client.js\"></script><br/></head><br/><body onLoad='heat_tracer()'><br/><canvas id='canvas' width='1024' height='512'></canvas><br/></body><br/></html></code></pre></div>\n\n<p>Finally the JavaScript client which translates the raw streaming data into pretty picture:</p>\n\n<div><a href=\"http://howtonode.org/heat-tracer/heat-tracer/public/heat_tracer_client.js\">public/heat_tracer_client.js</a><pre><code>/* On load we create our web socket (or flash socket if your browser doesn't support it ) and<br/> send the d script we wish to be tracing. This extremely powerful and *insecure*. */<br/>function heat_tracer() {</p><p> //Global vars<br/> setup();</p><p> var socket = new io.Socket('localhost'); //connect to localhost presently<br/> socket.connect();</p><p> socket.on('connect', function(){<br/> console.log('on connection');<br/> var dscript = \"syscall:::entry\\n{\\nself->syscall_entry_ts[probefunc] = vtimestamp;\\n}\\nsyscall:::return\\n/self->syscall_entry_ts[probefunc]/\\n{\\n\\n@time[probefunc] = lquantize((vtimestamp - self->syscall_entry_ts[probefunc] ) / 1000, 0, 63, 2);\\nself->syscall_entry_ts[probefunc] = 0;\\n}\";<br/> socket.send( { 'dscript' : dscript } );<br/> });</p><p> /* The only messages we recieve should contain contain the dtrace aggregation data we requested<br/> on connection. */<br/> socket.on('message', function(message){<br/> //console.log( message );<br/> draw(message);</p><p> /* for ( key in message ) {<br/> val = message[key];<br/> console.log( 'key: ' + key + ', interval: ' + val[0][0] + '-' + val[0][1], ', count ' + val[1] );<br/> }<br/> */<br/> });</p><p> socket.on('disconnect', function(){<br/> });</p><p>}</p><p>/* Take the aggregation data and update the heatmap */<br/>function draw(message) {</p><p> /* Latest data goes in the right most column, initialize it */<br/> var syscalls_by_latency = [];<br/> for ( var index = 0; index < 32; index++ ) {<br/> syscalls_by_latency[index] = 0;<br/> }</p><p> /* Presently we have the latency for each system call quantized in our message. Merge the data<br/> such that we have all the system call latency quantized together. This gives us the number<br/> of syscalls made with latencies in each particular band. */<br/> for ( var syscall in message ) {<br/> var val = message[syscall];<br/> for ( result_index in val ) {<br/> var latency_start = val[result_index][0][0];<br/> var count = val[result_index][1];<br/> /* The d script we're using lquantizes from 0 to 63 in steps of two. So dividing by 2<br/> tells us which row this result belongs in */<br/> syscalls_by_latency[Math.floor(latency_start/2)] += count;<br/> }<br/> }</p><p> /* We just created a new column, shift the console to the left and add it. */<br/> console_columns.shift();<br/> console_columns.push(syscalls_by_latency);<br/> drawArray(console_columns);<br/>}</p><p>/* Draw the columns and rows that map up the heatmap on to the canvas element */<br/>function drawArray(console_columns) {<br/> var canvas = document.getElementById('canvas');<br/> if (canvas.getContext) {<br/> var ctx = canvas.getContext('2d');<br/> for ( var column_index in console_columns ) {<br/> var column = console_columns[column_index];<br/> for ( var entry_index in column ) {<br/> entry = column[entry_index];</p><p> /* We're using a logarithmic scale for the brightness. This was all arrived at by<br/> trial and error and found to work well on my Mac. In the future this<br/> could all be adjustable with controls */<br/> var red_value = 0;<br/> if ( entry != 0 ) {<br/> red_value = Math.floor(Math.log(entry)/Math.log(2));<br/> }<br/> //console.log(red_value);<br/> ctx.fillStyle = 'rgb(' + (red_value * 25) + ',0,0)';<br/> ctx.fillRect(column_index*16, 496-(entry_index*16), 16, 16);<br/> }<br/> }<br/> }<br/>}</p><p>/* The heatmap is is really a 64x32 grid. Initialize the array which contains the grid data. */<br/>function setup() {<br/> console_columns = [];</p><p> for ( var column_index = 0; column_index < 64; column_index++ ) {<br/> var column = [];<br/> for ( var entry_index = 0; entry_index < 32; entry_index++ ) {<br/> column[entry_index] = 0;<br/> }<br/> console_columns.push(column);<br/> }</p><p>}</code></pre></div>\n\n<h2>Run It!</h2>\n\n<p>Run Heat Tacer with the following. Note, sudo is required by dtrace as it does kernal magic.</p>\n\n<pre><code>sudo node heat_tracer.js<br/></code></pre>\n\n<p>If all goes well you should see something a moving version of something like the image below.</p>\n\n<blockquote>\n <p><img src=\"http://howtonode.org/heat-tracer/heat_tracer.png\" alt=\"Alt value of image\"></img> </p>\n</blockquote>\n\n<h2>Contribute</h2>\n\n<p>You can find the latest version of Heat Tracer <a href=\"https://github.com/gflarity/Heat-Tracer\">here</a>. It is my hope that this article will provide the ground work for a much more abitious performance analytics project. If you're interested in contributing please let me know.</p>\n\n<h2>Further Research</h2>\n\n<p>More information about Bryan and Brendan's demo can be found <a href=\"http://dtrace.org/blogs/brendan/2011/01/24/cloud-analytics-first-video/\">here</a>.</p>\n\n<p>Socket.IO can be found <a href=\"http://socket.io/\">here</a>.</p><hr/>\n\n<a href=\"http://disqus.com/forums/howtonodeorg/?url=ref\">View the discussion thread.</a>",
21 | };
22 |
23 | const readable = new Readability({
24 | pageURL: "http://howtonode.org/heat-tracer/",
25 | resolvePaths: true,
26 | });
27 | const parser = new Parser(readable);
28 |
29 | parser.parseComplete(file);
30 |
31 | const article = readable.getArticle();
32 |
33 | assert.strictEqual(
34 | article.title,
35 | expectedData.title,
36 | "didn't get expected title!"
37 | );
38 | assert.strictEqual(
39 | article.nextPage,
40 | expectedData.nextPage,
41 | "didn't get expected nextPage!"
42 | );
43 | assert.strictEqual(
44 | article.textLength,
45 | expectedData.textLength,
46 | "didn't get expected textLength!"
47 | );
48 | assert.strictEqual(
49 | article.score,
50 | expectedData.score,
51 | "didn't get expected score!"
52 | );
53 | assert.strictEqual(
54 | article.html,
55 | expectedData.html,
56 | "didn't get expected html!"
57 | );
58 |
59 | assert.strictEqual(
60 | require("util").inspect(readable._currentElement, false, 1 / 0).length,
61 | 245663,
62 | "tree had false size!"
63 | );
64 | assert.strictEqual(
65 | Object.keys(readable._scannedLinks).length,
66 | expectedLinks,
67 | "wrong number of links!"
68 | );
69 |
70 | testURL();
71 |
72 | console.log("Passed!");
73 |
74 | function testURL() {
75 | const readable = new Readability({
76 | pageURL: "http://foo.bar/this.2/is/a/long/path/index?isnt=it",
77 | resolvePaths: true,
78 | });
79 |
80 | assert.strictEqual(
81 | JSON.stringify(readable._url),
82 | JSON.stringify(expectedUrl),
83 | "wrong url"
84 | );
85 | assert.strictEqual(
86 | readable._baseURL,
87 | "http://foo.bar/this.2/is/a/long/path",
88 | "wrong base"
89 | );
90 | assert.strictEqual(
91 | readable._convertLinks("../asdf/foo/"),
92 | "http://foo.bar/this.2/is/a/long/asdf/foo/",
93 | "link1 wasn't resolved!"
94 | );
95 | assert.strictEqual(
96 | readable._convertLinks("/asdf/foo/"),
97 | "http://foo.bar/asdf/foo/",
98 | "link2 wasn't resolved!"
99 | );
100 | assert.strictEqual(
101 | readable._convertLinks("foo/"),
102 | "http://foo.bar/this.2/is/a/long/path/foo/",
103 | "link3 wasn't resolved!"
104 | );
105 | }
106 |
--------------------------------------------------------------------------------
/tests/test_performance.js:
--------------------------------------------------------------------------------
1 | const getReadableContent = require("../");
2 | const { Parser } = require("htmlparser2");
3 | const Readability = require("../readabilitySAX");
4 | const request = require("request");
5 | const url = require("url");
6 |
7 | function ben(times, func) {
8 | const start = Date.now();
9 | while (times-- > 0) func();
10 | return Date.now() - start;
11 | }
12 |
13 | const processContent = function (data, settings) {
14 | const readable = new Readability(settings);
15 | const parser = new Parser(readable);
16 |
17 | console.log(
18 | "parsing took (ms):",
19 | ben(1e3, () => {
20 | parser.parseComplete(data);
21 | })
22 | );
23 | console.log(
24 | "getHTML took (ms):",
25 | ben(1e3, () => {
26 | readable.getHTML();
27 | })
28 | );
29 | console.log(
30 | "getText took (ms):",
31 | ben(1e3, () => {
32 | readable.getText();
33 | })
34 | );
35 | console.log(
36 | "getArticle took (ms):",
37 | ben(1e3, () => {
38 | readable.getArticle();
39 | })
40 | );
41 | console.log(
42 | "Whole parsing took (ms):",
43 | ben(500, () => {
44 | getReadableContent.process(data, settings);
45 | })
46 | );
47 | };
48 |
49 | if (process.argv.length > 2) {
50 | console.log("connecting to:", process.argv[2]);
51 |
52 | request(process.argv[2], (err, resp, body) => {
53 | processContent(body, {
54 | pageURL: url.format(resp.request.uri),
55 | log: false,
56 | });
57 | });
58 | } else {
59 | const file = require("fs").readFileSync(
60 | `${__dirname}/testpage.html`,
61 | "utf8"
62 | );
63 | processContent(file, {
64 | pageURL: "http://howtonode.org/heat-tracer",
65 | log: false,
66 | });
67 | }
68 |
--------------------------------------------------------------------------------
/tests/testpage.html:
--------------------------------------------------------------------------------
1 |
2 | <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
3 | <html lang="en" xmlns="http://www.w3.org/1999/xhtml"><head><meta http-equiv="content-type" content="text/html; charset=UTF-8" /><title>Realtime Performance Visualizations using Node.js - How To Node - NodeJS
This article outlines how to create a realtime heatmap of your syscall latency using HTML5, some great node modules, and DTrace. It was inspired by talk that Bryan Cantrill and Brendan Greg gave on Joyent's cool cloud analytics tools. While specific, the code provided could easily be adapted to provide a heatmap of any type of aggregation Dtrace is capable of providing.
4 |
5 |
System Requirements
6 |
7 |
First thing's first, you're going to need a system with DTrace. This likely means Solaris (or one of its decedents), OS X, or a BSD variant. There doesn't appear to be Dtrace available for Linux.
8 |
9 |
Security
10 |
11 |
Secondly, please be aware that at the time of writing the demo code contains a fairly substantial secruity vulnerabilty. Namely the d script is sent from the client with no authentication what so ever. If you bind to localhost this shouldn't be a big deal for a demo. Time permitting I intend to clean up the code.
var http =require('http'); var libdtrace =require('libdtrace'); var io =require('socket.io'); var express =require('express');
/* create our express server and prepare to serve javascript files in ./public */ var app = express.createServer(); app.configure(function(){ app.use(express.staticProvider(__dirname +'/public')); });
/* Before we go any further we must realize that each time a user connects we're going to want to them send them dtrace aggregation every second. We can do so using 'setInterval', but we must keep track of both the intervals we set and the dtrace consumers that are created as we'll need them later when the client disconnects. */ var interval_id_by_session_id ={}; var dtp_by_session_id ={};
/* In order to effecienctly send packets we're going to use the Socket.IO library which seemlessly integrates with express. */ var websocket_server = io.listen(app);
/* Now that we have a web socket server, we need to create a handler for connection events. These events represet a client connecting to our server */ websocket_server.on('connection',function(socket){
/* Like the web server object, we must also define handlers for various socket events that will happen during the lifetime of the connection. These will define how we interact with the client. The first is a message event which occurs when the client sends something to the server. */ socket.on('message',function(message){ /* The only message the client ever sends will be sent right after connecting. So it will happen only once during the lifetime of a socket. This message also contains a d script which defines an agregation to walk. */ var dtp =new libdtrace.Consumer(); var dscript = message['dscript']; console.log( dscript ); dtp.strcompile(dscript); dtp.go(); dtp_by_session_id[socket.sessionId]= dtp;
/* All that's left to do is send the aggration data from the dscript. */ interval_id_by_session_id[socket.sessionId]= setInterval(function(){ var aggdata ={}; try{ dtp.aggwalk(function(id, key, val){ for( index in val ){ /* console.log( 'key: ' + key + ', interval: ' + val[index][0][0] + '-' + val[index][0][1], ', count ' + val[index][1] ); */
/* Not so fast. If a client disconnects we don't want their respective dtrace consumer to keep collecting data any more. We also don't want to try to keep sending anything to them period. So clean up. */ socket.on('disconnect',function(){ clearInterval(clearInterval(interval_id_by_session_id[socket.sessionId])); var dtp = dtp_by_session_id[socket.sessionId]; delete dtp_by_session_id[socket.sessionId]; dtp.stop(); console.log('disconnected'); });
});
app.listen(80);
26 |
27 |
Client
28 |
29 |
In order to display our heatmap, we're going to need some basic HTML with a canvas element:
/* On load we create our web socket (or flash socket if your browser doesn't support it ) and send the d script we wish to be tracing. This extremely powerful and *insecure*. */ function heat_tracer(){
//Global vars setup();
var socket =new io.Socket('localhost');//connect to localhost presently socket.connect();
/* The only messages we recieve should contain contain the dtrace aggregation data we requested on connection. */ socket.on('message',function(message){ //console.log( message ); draw(message);
/* Take the aggregation data and update the heatmap */ function draw(message){
/* Latest data goes in the right most column, initialize it */ var syscalls_by_latency =[]; for(var index =0; index <32; index++){ syscalls_by_latency[index]=0; }
/* Presently we have the latency for each system call quantized in our message. Merge the data such that we have all the system call latency quantized together. This gives us the number of syscalls made with latencies in each particular band. */ for(var syscall in message ){ var val = message[syscall]; for( result_index in val ){ var latency_start = val[result_index][0][0]; var count = val[result_index][1]; /* The d script we're using lquantizes from 0 to 63 in steps of two. So dividing by 2 tells us which row this result belongs in */ syscalls_by_latency[Math.floor(latency_start/2)]+= count; } }
/* We just created a new column, shift the console to the left and add it. */ console_columns.shift(); console_columns.push(syscalls_by_latency); drawArray(console_columns); }
/* Draw the columns and rows that map up the heatmap on to the canvas element */ function drawArray(console_columns){ var canvas = document.getElementById('canvas'); if(canvas.getContext){ var ctx = canvas.getContext('2d'); for(var column_index in console_columns ){ var column = console_columns[column_index]; for(var entry_index in column ){ entry = column[entry_index];
/* We're using a logarithmic scale for the brightness. This was all arrived at by trial and error and found to work well on my Mac. In the future this could all be adjustable with controls */ var red_value =0; if( entry !=0){ red_value =Math.floor(Math.log(entry)/Math.log(2)); } //console.log(red_value); ctx.fillStyle ='rgb('+(red_value *25)+',0,0)'; ctx.fillRect(column_index*16,496-(entry_index*16),16,16); } } } }
/* The heatmap is is really a 64x32 grid. Initialize the array which contains the grid data. */ function setup(){ console_columns =[];
Run Heat Tacer with the following. Note, sudo is required by dtrace as it does kernal magic.
40 |
41 |
sudo node heat_tracer.js
42 |
43 |
If all goes well you should see something a moving version of something like the image below.
44 |
45 |
46 |
47 |
48 |
49 |
Contribute
50 |
51 |
You can find the latest version of Heat Tracer here. It is my hope that this article will provide the ground work for a much more abitious performance analytics project. If you're interested in contributing please let me know.
52 |
53 |
Further Research
54 |
55 |
More information about Bryan and Brendan's demo can be found here.
HowToNode.org is a community supported blog created by Tim Caswell. The purpose of the blog is to teach how to do various tasks in node.js as well as teach fundamental concepts that are needed to write effective code.
85 |
86 |
This site is powered by Wheat, a git based blogging engine written in node.JS.
87 |
88 |
The content for this site is stored in a git repository that anyone can fork, write an article, and send a pull request. If your article passes the quality standards it will be published and help support the greater node community.
