├── .gitignore ├── .jscsrc ├── .jshintignore ├── .jshintrc ├── .travis.yml ├── LICENSE ├── README.md ├── config ├── default.yml ├── pac-no-tunnel.js ├── pac-tunnel.js ├── production.yml ├── statsd.js ├── test │ ├── default.yml │ └── test.yml └── urlshorteners.txt ├── keys ├── crt.pem └── key.pem ├── lib ├── cache.js ├── index.js ├── log.js ├── metrics.js ├── pac.js ├── plugins │ ├── adblock.js │ ├── cache.js │ ├── dom │ │ ├── gif2video.js │ │ ├── index.js │ │ └── plugins.yaml │ ├── egress.js │ ├── fork.js │ ├── gunzip.js │ ├── gzip.js │ ├── imgcompression.js │ ├── index.js │ ├── ingress.js │ ├── placeholderImage.js │ ├── plugins.yaml │ ├── urlexpander.js │ ├── util.js │ ├── video.js │ └── xz.js ├── proxy.js ├── storage │ ├── basic.js │ ├── index.js │ └── redis.js └── util.js ├── package.json ├── proxy └── test ├── helper ├── content │ ├── basic.html │ ├── block-list │ ├── dummy.html │ ├── gif2video │ │ ├── simple.gif │ │ └── simple.html │ ├── imgs │ │ ├── fennec.jpg │ │ ├── panda.jpg │ │ └── speedy.png │ ├── iso8859.html │ └── videos │ │ └── test.mp4 ├── dummyRequest.js ├── dummyResponse.js ├── networksimulator.js ├── profile │ ├── extensions.json │ └── extensions │ │ └── certvalidator@mozilla.org │ │ ├── bootstrap.js │ │ ├── chrome.manifest │ │ └── install.rdf └── testHelper.js ├── perfomance └── marionette.js └── tests ├── adblock.js ├── cache.js ├── fork.js ├── gif2video.js ├── global.js ├── imgcompression.js ├── metrics.js ├── urlexpander.js └── video.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | config/*runtime.js* 3 | config/test/runtime.json 4 | config/local.yml 5 | coverage/ 6 | *.log 7 | -------------------------------------------------------------------------------- /.jscsrc: -------------------------------------------------------------------------------- 1 | { 2 | "requireSpaceAfterKeywords": [ 3 | "if", "else", "for", "while", "do", "switch", "return", "try", "catch" 4 | ], 5 | "requireSpaceBeforeBlockStatements": true, 6 | "requireParenthesesAroundIIFE": true, 7 | "requireSpacesInConditionalExpression": true, 8 | "disallowSpacesInFunctionExpression": { 9 | "beforeOpeningRoundBrace": true 10 | }, 11 | "disallowMultipleVarDecl": true, 12 | "requireBlocksOnNewline": 1, 13 | "disallowSpacesInsideArrayBrackets": true, 14 | "disallowSpacesInsideParentheses": true, 15 | "requireSpacesInsideObjectBrackets": "allButNested", 16 | "disallowSpaceAfterObjectKeys": true, 17 | "requireCommaBeforeLineBreak": true, 18 | "requireOperatorBeforeLineBreak": [ 19 | "?", "+", "-", "/", "*", "=", "==", "===", "!=", "!==", ">", ">=", "<", "<=", "||", "&&" 20 | ], 21 | "disallowLeftStickedOperators": [ 22 | "?", "+", "/", "*", "=", "==", "===", "!=", "!==", ">", ">=", "<", "<=", "||", "&&" 23 | ], 24 | "requireRightStickedOperators": [ 25 | "!", "~" 26 | ], 27 | "disallowRightStickedOperators": [ 28 | "?", "+", "/", "*", ":", "=", "==", "===", "!=", "!==", ">", ">=", "<", "<=", "||", "&&" 29 | ], 30 | "requireLeftStickedOperators": [ 31 | "," 32 | ], 33 | "disallowSpaceAfterPrefixUnaryOperators": ["++", "--", "+", "-", "~", "!"], 34 | "disallowSpaceBeforePostfixUnaryOperators": ["++", "--"], 35 | "requireSpaceBeforeBinaryOperators": [ 36 | "=", "+", "-", "/", "*", "=", "==", "===", "!=", "!==" 37 | ], 38 | "requireSpaceAfterBinaryOperators": [ 39 | "=", ",", "+", "-", "/", "*", "=", "==", "===", "!=", "!==" 40 | ], 41 | "disallowImplicitTypeConversion": [ 42 | "numeric", "boolean", "binary", "string" 43 | ], 44 | "requireCamelCaseOrUpperCaseIdentifiers": true, 45 | "disallowKeywords": [ 46 | "with" 47 | ], 48 | "disallowMultipleLineBreaks": true, 49 | "validateLineBreaks": "LF", 50 | "validateQuoteMarks": { 51 | "mark": "'", 52 | "escape": true 53 | }, 54 | "validateIndentation": 2, 55 | "disallowMixedSpacesAndTabs": true, 56 | "disallowTrailingWhitespace": true, 57 | "disallowKeywordsOnNewLine": ["else"], 58 | "requireLineFeedAtFileEnd": true, 59 | "maximumLineLength": 80, 60 | "requireCapitalizedConstructors": true, 61 | "disallowYodaConditions": true, 62 | "excludeFiles": [ 63 | ] 64 | } 65 | -------------------------------------------------------------------------------- /.jshintignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mozilla/node-janus/27e0fc7a58b9ffd92016a6797d69078fcea249d0/.jshintignore -------------------------------------------------------------------------------- /.jshintrc: -------------------------------------------------------------------------------- 1 | { 2 | "camelcase": true, 3 | "curly": true, 4 | "eqeqeq": true, 5 | "es3": false, 6 | "forin": true, 7 | "freeze": true, 8 | "immed": false, 9 | "indent": 2, 10 | "latedef": true, 11 | "newcap": true, 12 | "noarg": true, 13 | "noempty": false, 14 | "nonbsp": true, 15 | "nonew": true, 16 | "plusplus": false, 17 | "quotmark": "single", 18 | "undef": true, 19 | "unused": true, 20 | "globalstrict": true, 21 | "trailing": true, 22 | "maxparams": 10, 23 | "maxdepth": 4, 24 | "maxstatements": 30, 25 | "maxcomplexity": 12, 26 | "maxlen": 80, 27 | "node": true, 28 | "browser": false, 29 | "esnext": true, 30 | "sub": true 31 | } 32 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - "0.10" 4 | 5 | before_install: 6 | - sudo apt-get update -qq 7 | - sudo apt-get install -qq ffmpeg 8 | - sudo apt-get install -qq nasm 9 | 10 | notifications: 11 | irc: 12 | channels: 13 | - "irc.mozilla.org#janus" 14 | on_success: change 15 | on_failure: always 16 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | Copyright (c) 2014 Mozilla Foundation 3 | 4 | Permission is hereby granted, free of charge, to any person obtaining a copy of 5 | this software and associated documentation files (the "Software"), to deal in the 6 | Software without restriction, including without limitation the rights to use, 7 | copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the 8 | Software, and to permit persons to whom the Software is furnished to do so, 9 | subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be included in all 12 | copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO 17 | EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 | OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 20 | THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Janus - Privacy & Compression Proxy 2 | [![Build Status](https://travis-ci.org/mozilla/node-janus.svg?branch=develop)](https://travis-ci.org/mozilla/node-janus) 3 | 4 | ## Requirements 5 | * [Node.js v10.0](http://nodejs.org) 6 | * [FFmpeg](http://ffmpeg.org/) 7 | 8 | ## Installation 9 | Get the code first. 10 | 11 | git clone https://github.com/mozilla/node-janus 12 | 13 | Next, use NPM to install all the dependencies. 14 | 15 | cd node-janus 16 | npm install 17 | 18 | ## Configuration and Usage 19 | ### Proxy 20 | You can find the default proxy configuration in `config/default.yml`. 21 | All settings are exposed and documented there. 22 | 23 | You may edit the settings directly in the default configuration file or 24 | preferably override some of settings using a custom configuration file, 25 | see the [node-config documentation](https://lorenwest.github.io/node-config/latest/) 26 | for more details about the configuration system. 27 | 28 | To start the proxy, just run 29 | 30 | ./proxy 31 | 32 | The only command-line arguments supported are `-h` for help and `-v` for 33 | showing the version. 34 | 35 | ### Firefox 36 | #### Minimal Version 37 | You need at least Firefox 33 for SPDY proxy support. 38 | 39 | #### Self-Signed Certificates 40 | When using a self-signed certificate, you need to add it to Firefox first. To do 41 | this, use Firefox to open the proxy via its host-port combination. 42 | 43 | https://:/ 44 | 45 | This should prompt you to add an exception for the self-signed certificate. 46 | 47 | ### Automatic Client Configuration Using the Add-On 48 | The prefered way for using the proxy is by installing the [Janus 49 | add-on](https://addons.mozilla.org/en-US/firefox/addon/janus-proxy-configurator/). 50 | When using the add-on, you can conveniently configure the optional features of 51 | the proxy and view some statistics on bandwidth savings. 52 | 53 | Should you have reasons to set up the proxy without the add-on, please follow 54 | the manual instructions next. 55 | 56 | ### Manual Client Configuration 57 | #### Desktop 58 | You can configure the secure proxy in `Preferences/Advanced/Network/Settings`. 59 | Select `Automatic proxy configuration URL` and set it to your custom PAC file or 60 | use the default configuration served by the integrated PAC server. 61 | 62 | http://: 63 | 64 | This will serve a suitable PAC file with the proper host and ports set. 65 | Check `config/default.yml` for the default PAC server connection details. 66 | 67 | #### Android 68 | For Fennec the steps are similar. Open `about:config` and set 69 | `network.proxy.autoconfig_url` to the location of your PAC file or the Janus 70 | PAC server. 71 | To load the PAC file and activate the proxy, set `network.proxy.type` to `2`. 72 | 73 | ## Production Deployment 74 | ### Additional Requirements 75 | * `optional` [Redis](http://redis.io) 76 | * `optional` [StatsD](https://github.com/etsy/statsd) 77 | 78 | By default, the proxy uses a basic in-memory cache and does only log basic 79 | metric stats. Additionally, the proxy supports a Redis-based caching solution 80 | and StatsD metrics reporting. 81 | 82 | To enable the Redis cache, you need to have a running Redis server instance. 83 | The proxy-side configuration is straight-forward using `config/default.yml`, 84 | where you set the host and port accordingly and switch caching modes by setting 85 | `cache.type`. 86 | 87 | To view and process the full metrics, you need a receiver compatible to StatsD 88 | metrics. To establish a connection, simply set the `metrics.statsd` settings 89 | accordingly in `config/default.yml` or your local overriding config files. 90 | 91 | ### Self-Signed Certificate 92 | You will also need to use your own certificate for your server FQDN. You can 93 | generate a new key and a new certificate simply by executing this command from 94 | `node-janus` root directory: 95 | 96 | openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout keys/key.pem -out keys/crt.pem 97 | 98 | Be careful to correctly set the `Common Name` with your server FQDN, e.g., for 99 | `example.com`: 100 | 101 | Common Name (e.g. server FQDN or YOUR name) []:example.com 102 | 103 | Because a self-signed certificate is not delivered by a trusted CA, you will 104 | have to manually add it to your browser. Please have a look to the [Firefox](#firefox) 105 | section for more details. 106 | 107 | ## Development 108 | We would be happy to accept pull requests for known issues or useful new 109 | features, so we encourage you to contribute! 110 | 111 | Please make sure all tests pass locally before putting the request up for 112 | review, additional tests for new features would be great, too. 113 | 114 | ### Tests 115 | To run all tests use 116 | 117 | npm test 118 | 119 | To get coverage statistics use 120 | 121 | npm run-script coverage 122 | 123 | To run performance tests using Marionette you need to point the configuration 124 | to your Firefox binary in file `config/test/test.yml`, setting 125 | `test.firefoxPath`. Then launch the tests using 126 | 127 | npm run-script marionette 128 | 129 | To simulate different mobile network environments, use 130 | 131 | npm run-script networksimulation 2G|3G|4G 132 | 133 | and stop the system-wide simulation by reverting to the defaults using 134 | 135 | npm run-script networksimulation default 136 | 137 | -------------------------------------------------------------------------------- /config/default.yml: -------------------------------------------------------------------------------- 1 | # Logging settings. 2 | logging: 3 | # The minimum logging level for console messages 4 | # (via Winston, https://github.com/flatiron/winston). 5 | level: debug 6 | # Whether or not to colorize console logging. 7 | colorize: true 8 | console: 9 | enabled: true 10 | level: debug 11 | timestamp: false 12 | # Whether or not to colorize console logging. 13 | colorize: true 14 | 15 | file: 16 | enabled: true 17 | level: debug 18 | timestamp: true 19 | # Maximum size of the log file before it is rolled over in MB. 20 | maxSize: 20 21 | # Whether messages should be logged as json or not. 22 | json: false 23 | # The filename for the log file. 24 | filename: proxy.log 25 | # Overwrite log on start. 26 | clobber: false 27 | 28 | # Node Cluster options. 29 | cluster: 30 | enabled: true 31 | workers: 32 | # When true (the default), the number of workers 33 | # is determined automatically based on number 34 | # of CPUs on the machine. 35 | auto: true 36 | 37 | # When 'auto' is false, this number is used to 38 | # determine the number of workers. 39 | count: 8 40 | 41 | # Proxy server settings. 42 | proxy: 43 | # Proxy listen port. 44 | port: 55055 45 | # SSL key path. 46 | sslKeyPath: ./keys/key.pem 47 | # SSL certificate path. 48 | sslCertPath: ./keys/crt.pem 49 | # Whether or not to tunnel SSL traffic. 50 | tunnelSsl: false 51 | 52 | # Plugin settings below. 53 | # 54 | # If a plugin has no config, the default looks like this: 55 | # 56 | # pluginName: 57 | # enabled: false 58 | # optional: false 59 | # 60 | # If 'enabled' is true, the plugin is loaded and used 61 | # by default. If false, it is not. If 'optional' is 62 | # true, it means the plugin can be enabled and disabled 63 | # by request-specific headers. The initial state is 64 | # still determined by the 'enabled' value. 65 | 66 | # DOM plugin, provides its own plugin system for 67 | # operating on a parsed HTML DOM. 68 | dom: 69 | enabled: true 70 | 71 | # fork plugin, streams the source stream directly to the destination stream and 72 | # to the response chain for further processing. 73 | fork: 74 | enabled: false 75 | optional: false 76 | 77 | # ingress plugin, reads data from the forwarded response. 78 | ingress: 79 | enabled: true 80 | # Whether or not it should accumulate the entire response 81 | # in order to write out a 'x-original-content-length' 82 | # header when there is no 'content-length' header. 83 | accumulateUnknownLengths: false 84 | 85 | # egress plugin delivers the end result to the client. 86 | egress: 87 | enabled: true 88 | 89 | # gunzip plugin unzips gzip-formatted content, making it 90 | # easier to process or recompress the content 91 | gunzip: 92 | enabled: false 93 | 94 | # Cache settings. 95 | cache: 96 | enabled: false 97 | 98 | # Supported cache types are: 99 | # + basic (in-memory JS) 100 | # + redis (requires Redis database) 101 | type: basic 102 | 103 | # Settings for non-basic database server. 104 | database: 105 | host: localhost 106 | port: 55255 107 | 108 | # Expiration time settings for cache items in seconds. 109 | expire: 110 | default: 30 111 | max: 86400 112 | 113 | # Cache items management. 114 | items: 115 | # 0 means no limit on number of items in cache. 116 | limit: 0 117 | # Min and max item sizes stored in KB. 118 | size: 119 | min: 0 120 | max: 51200 121 | 122 | # Cache memory usage settings in MB. 123 | memory: 124 | # 0 means no limit on memory size. 125 | limit: 2000 126 | 127 | # Image compression settings. 128 | imgcompression: 129 | enabled: true 130 | # Use libjpeg-turbo instead of mozjpeg. 131 | turbo: true 132 | # Minimum image size to compress in KB. 133 | minSize: 20 134 | 135 | # Adblock settings. 136 | adblock: 137 | enabled: false 138 | optional: true 139 | listUrl: 'http://pgl.yoyo.org/adservers/serverlist.php?&mimetype=plaintext' 140 | 141 | # URL expander settings (resolves shortened URLs). 142 | urlexpander: 143 | enabled: false 144 | optional: true 145 | # Number of redirections after which the request will be directly forwarded. 146 | maxRedirect: 5 147 | hostsFile: 'config/urlshorteners.txt' 148 | 149 | # XZ compression settings. 150 | xz: 151 | enabled: true 152 | # Anything higher requires a lot of memory to decode (> 32MB). 153 | level: 8 154 | 155 | # Gzip compression settings. 156 | gzip: 157 | enabled: true 158 | # Compression level. 159 | level: 9 160 | 161 | # GIF to video transcoding settings. 162 | gif2video: 163 | enabled: false 164 | optional: true 165 | 166 | # Transcode videos to make them smaller 167 | video: 168 | enabled: false 169 | optional: true 170 | 171 | # The quality level to use. 0-50, lower is better 172 | quality: 30 173 | 174 | # Metrics reporting settings. 175 | metrics: 176 | enabled: true 177 | # Metrics selection allows to restrict metrics reports to a whitelisted set. 178 | selection: 179 | enabled: false 180 | set: ['proxy.request', 'proxy.response'] 181 | # All update interval settings are in seconds. 182 | # General system metrics settings. 183 | system: 184 | enabled: true 185 | # Update interval for general system stats reporting. 186 | interval: 10 187 | # Settings for the StatsD backend (see config/statsd.js). 188 | # Only the host and prefix settings are special, as they are used for the 189 | # client-only configuration. 190 | statsd: 191 | enabled: true 192 | host: localhost 193 | port: 55355 194 | prefix: janus 195 | # Spawn an instance of StatsD to connect to if enabled, otherwise just 196 | # use the settings to connect to an existing service. 197 | spawn: true 198 | # The following settings only have effect if spawning is enabled. 199 | flushInterval: 10000 200 | # Native backends include: 201 | # + ./backends/graphite 202 | # + ./backends/console 203 | # + ./backends/repeater 204 | # Otherwise all compatible backends installed via npm may be used, e.g.: 205 | # + statsd-console-backend (cleaner console output) 206 | backends: [statsd-console-backend] 207 | graphiteHost: localhost 208 | graphitePort: 55455 209 | console: 210 | prettyprint: true 211 | # Profiling settings. 212 | profile: 213 | # Heap profiling, can degrade performance. 214 | heap: 215 | enabled: true 216 | # Lightweight leak detection. 217 | leaks: 218 | enabled: true 219 | 220 | # Request-specific metrics. 221 | request: 222 | # Enable locale metrics. 223 | locale: true 224 | # Enable IP hash metrics. 225 | ipHash: true 226 | -------------------------------------------------------------------------------- /config/pac-no-tunnel.js: -------------------------------------------------------------------------------- 1 | function FindProxyForURL(url, host) { 2 | if (url.substring(0, 5) != "http:" || 3 | isPlainHostName(host) || 4 | shExpMatch(host, "*.local") || 5 | isInNet(dnsResolve(host), "10.0.0.0", "255.0.0.0") || 6 | isInNet(dnsResolve(host), "172.16.0.0", "255.240.0.0") || 7 | isInNet(dnsResolve(host), "192.168.0.0", "255.255.0.0") || 8 | isInNet(dnsResolve(host), "127.0.0.0", "255.255.255.0")) { 9 | return "DIRECT"; 10 | } 11 | return "HTTPS "; 12 | } 13 | -------------------------------------------------------------------------------- /config/pac-tunnel.js: -------------------------------------------------------------------------------- 1 | function FindProxyForURL(url, host) { 2 | if ((url.substring(0, 5) != "http:" && 3 | url.substring(0, 6) != "https:") || 4 | isPlainHostName(host) || 5 | shExpMatch(host, "*.local") || 6 | isInNet(dnsResolve(host), "10.0.0.0", "255.0.0.0") || 7 | isInNet(dnsResolve(host), "172.16.0.0", "255.240.0.0") || 8 | isInNet(dnsResolve(host), "192.168.0.0", "255.255.0.0") || 9 | isInNet(dnsResolve(host), "127.0.0.0", "255.255.255.0")) { 10 | return "DIRECT"; 11 | } 12 | return "HTTPS "; 13 | } 14 | -------------------------------------------------------------------------------- /config/production.yml: -------------------------------------------------------------------------------- 1 | logging: 2 | console: 3 | enabled: false 4 | 5 | file: 6 | level: error 7 | 8 | metrics: 9 | statsd: 10 | spawn: false 11 | profile: 12 | heap: 13 | enabled: false 14 | -------------------------------------------------------------------------------- /config/statsd.js: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | Required Variables: 4 | 5 | port: StatsD listening port [default: 8125] 6 | 7 | Graphite Required Variables: 8 | 9 | (Leave these unset to avoid sending stats to Graphite. 10 | Set debug flag and leave these unset to run in 'dry' debug mode - 11 | useful for testing statsd clients without a Graphite server.) 12 | 13 | graphiteHost: hostname or IP of Graphite server 14 | graphitePort: port of Graphite server 15 | 16 | Optional Variables: 17 | 18 | backends: an array of backends to load. Each backend must exist 19 | by name in the directory backends/. If not specified, 20 | the default graphite backend will be loaded. 21 | debug: debug flag [default: false] 22 | address: address to listen on over UDP [default: 0.0.0.0] 23 | address_ipv6: defines if the address is an IPv4 or IPv6 address [true or false, default: false] 24 | port: port to listen for messages on over UDP [default: 8125] 25 | mgmt_address: address to run the management TCP interface on 26 | [default: 0.0.0.0] 27 | mgmt_port: port to run the management TCP interface on [default: 8126] 28 | title : Allows for overriding the process title. [default: statsd] 29 | if set to false, will not override the process title and let the OS set it. 30 | The length of the title has to be less than or equal to the binary name + cli arguments 31 | NOTE: This does not work on Mac's with node versions prior to v0.10 32 | 33 | healthStatus: default health status to be returned and statsd process starts ['up' or 'down', default: 'up'] 34 | dumpMessages: log all incoming messages 35 | flushInterval: interval (in ms) to flush to Graphite 36 | percentThreshold: for time information, calculate the Nth percentile(s) 37 | (can be a single value or list of floating-point values) 38 | negative values mean to use "top" Nth percentile(s) values 39 | [%, default: 90] 40 | flush_counts: send stats_counts metrics [default: true] 41 | 42 | keyFlush: log the most frequently sent keys [object, default: undefined] 43 | interval: how often to log frequent keys [ms, default: 0] 44 | percent: percentage of frequent keys to log [%, default: 100] 45 | log: location of log file for frequent keys [default: STDOUT] 46 | deleteIdleStats: don't send values to graphite for inactive counters, sets, gauges, or timeers 47 | as opposed to sending 0. For gauges, this unsets the gauge (instead of sending 48 | the previous value). Can be individually overriden. [default: false] 49 | deleteGauges : don't send values to graphite for inactive gauges, as opposed to sending the previous value [default: false] 50 | deleteTimers: don't send values to graphite for inactive timers, as opposed to sending 0 [default: false] 51 | deleteSets: don't send values to graphite for inactive sets, as opposed to sending 0 [default: false] 52 | deleteCounters: don't send values to graphite for inactive counters, as opposed to sending 0 [default: false] 53 | prefixStats: prefix to use for the statsd statistics data for this running instance of statsd [default: statsd] 54 | applies to both legacy and new namespacing 55 | 56 | console: 57 | prettyprint: whether to prettyprint the console backend 58 | output [true or false, default: true] 59 | 60 | log: log settings [object, default: undefined] 61 | backend: where to log: stdout or syslog [string, default: stdout] 62 | application: name of the application for syslog [string, default: statsd] 63 | level: log level for [node-]syslog [string, default: LOG_INFO] 64 | 65 | graphite: 66 | legacyNamespace: use the legacy namespace [default: true] 67 | globalPrefix: global prefix to use for sending stats to graphite [default: "stats"] 68 | prefixCounter: graphite prefix for counter metrics [default: "counters"] 69 | prefixTimer: graphite prefix for timer metrics [default: "timers"] 70 | prefixGauge: graphite prefix for gauge metrics [default: "gauges"] 71 | prefixSet: graphite prefix for set metrics [default: "sets"] 72 | globalSuffix: global suffix to use for sending stats to graphite [default: ""] 73 | This is particularly useful for sending per host stats by 74 | settings this value to: require('os').hostname().split('.')[0] 75 | 76 | repeater: an array of hashes of the for host: and port: 77 | that details other statsd servers to which the received 78 | packets should be "repeated" (duplicated to). 79 | e.g. [ { host: '10.10.10.10', port: 8125 }, 80 | { host: 'observer', port: 88125 } ] 81 | 82 | repeaterProtocol: whether to use udp4 or udp6 for repeaters. 83 | ["udp4" or "udp6", default: "udp4"] 84 | 85 | histogram: for timers, an array of mappings of strings (to match metrics) and 86 | corresponding ordered non-inclusive upper limits of bins. 87 | For all matching metrics, histograms are maintained over 88 | time by writing the frequencies for all bins. 89 | 'inf' means infinity. A lower limit of 0 is assumed. 90 | default: [], meaning no histograms for any timer. 91 | First match wins. examples: 92 | * histogram to only track render durations, with unequal 93 | class intervals and catchall for outliers: 94 | [ { metric: 'render', bins: [ 0.01, 0.1, 1, 10, 'inf'] } ] 95 | * histogram for all timers except 'foo' related, 96 | equal class interval and catchall for outliers: 97 | [ { metric: 'foo', bins: [] }, 98 | { metric: '', bins: [ 50, 100, 150, 200, 'inf'] } ] 99 | 100 | */ 101 | { 102 | port: 55355, 103 | backends: [ '../../lib/statsdConsoleBackend' ], 104 | flush_counts: false, 105 | flushInterval: 60000 106 | } 107 | -------------------------------------------------------------------------------- /config/test/default.yml: -------------------------------------------------------------------------------- 1 | ../default.yml -------------------------------------------------------------------------------- /config/test/test.yml: -------------------------------------------------------------------------------- 1 | 2 | cache: 3 | # make cache optional so we can disable it for testing 4 | enabled: false 5 | optional: true 6 | 7 | gzip: 8 | optional: true 9 | 10 | logging: 11 | level: error 12 | 13 | logging: 14 | http2: true 15 | console: 16 | enabled: false 17 | 18 | file: 19 | enabled: true 20 | level: debug 21 | filename: test.log 22 | clobber: true 23 | 24 | metrics: 25 | enabled: true 26 | system: 27 | enabled: false 28 | statsd: 29 | enabled: false 30 | spawn: false 31 | 32 | proxy: 33 | port: 44044 34 | 35 | pac: 36 | port: 44144 37 | 38 | fork: 39 | optional: true 40 | 41 | # Test settings 42 | test: 43 | # test/tests/performance.js settings 44 | performance: 45 | enabled: false 46 | 47 | # path to the Firefox binary to run for marionette tests. 48 | firefoxPath: null 49 | 50 | localServer: 51 | port: 8080 52 | -------------------------------------------------------------------------------- /config/urlshorteners.txt: -------------------------------------------------------------------------------- 1 | bit.ly 2 | chkit.in 3 | digs.by 4 | fb.me 5 | feedproxy.google.com 6 | ff.im 7 | goo.gl 8 | is.gd 9 | is.gd 10 | ow.ly 11 | p.ly 12 | ping.fm 13 | snipurl.com 14 | snurl.com 15 | t.co 16 | tcrn.ch 17 | tiny.cc 18 | tinyurl.com 19 | trib.al 20 | twurl.nl 21 | u.nu 22 | ur1.ca 23 | wp.me 24 | wrd.cm 25 | -------------------------------------------------------------------------------- /keys/crt.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIICyTCCAjICCQDCmC99WJujsDANBgkqhkiG9w0BAQUFADCBqDELMAkGA1UEBhMC 3 | REUxDzANBgNVBAgMBkJlcmxpbjEPMA0GA1UEBwwGQmVybGluMSEwHwYDVQQKDBhJ 4 | bnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQxFzAVBgNVBAsMDlBsYXRmb3JtIEhhY2tz 5 | MRowGAYDVQQDDBFnb256YWxlcy5tZTczLmNvbTEfMB0GCSqGSIb3DQEJARYQZXNh 6 | d2luQGdtYWlsLmNvbTAeFw0xNDAyMjgwMDU2MzFaFw0xNTAyMjgwMDU2MzFaMIGo 7 | MQswCQYDVQQGEwJERTEPMA0GA1UECAwGQmVybGluMQ8wDQYDVQQHDAZCZXJsaW4x 8 | ITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEXMBUGA1UECwwOUGxh 9 | dGZvcm0gSGFja3MxGjAYBgNVBAMMEWdvbnphbGVzLm1lNzMuY29tMR8wHQYJKoZI 10 | hvcNAQkBFhBlc2F3aW5AZ21haWwuY29tMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCB 11 | iQKBgQDIUDIRlhcuBBNU/fNiT8Kg9V9YXtrN+GkRXqjUhD/5dWR5A2vuIM0sCw6b 12 | LYlHO/JR7wG+2CszRcAtHCI+M/uEfTCQrMNB0QHCaurtP4VZ+fMxuA81XaeNIQr5 13 | ZeWxmZSSfHpdw9rG3KdIwrMKJC1IMiBp0fr4Tn2dGeSm8feKAwIDAQABMA0GCSqG 14 | SIb3DQEBBQUAA4GBABhW0EMEtNT6dJcJne+dGKuKmY/6HXz57rMcY+mDaNAHOYR4 15 | qOu9eszKn0ylTqNetmoMNrxqM/5DX1lFGIc+sHMoKnHEeqHF/l9xsvuegNx5+fr0 16 | Yj/1xP0Xis+llxmf4miOIhLoR8fNnJ7HX/fyXPJE0G48T7usdUYhwxQg5pyQ 17 | -----END CERTIFICATE----- 18 | -------------------------------------------------------------------------------- /keys/key.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIICXgIBAAKBgQDIUDIRlhcuBBNU/fNiT8Kg9V9YXtrN+GkRXqjUhD/5dWR5A2vu 3 | IM0sCw6bLYlHO/JR7wG+2CszRcAtHCI+M/uEfTCQrMNB0QHCaurtP4VZ+fMxuA81 4 | XaeNIQr5ZeWxmZSSfHpdw9rG3KdIwrMKJC1IMiBp0fr4Tn2dGeSm8feKAwIDAQAB 5 | AoGBAIwdnIg0po4nh6ZB5Mb157xTJphX8VF123is8AeHYoCKHveO7yYoh5uxOExP 6 | c+ECC6RIPL5T0xQQTDbBoSpFjj6HzH35X9Z9oHLhQc3rTGMhQ4/fcq70yIXtamc8 7 | t25dfTU1zRy+U/qU94ly+iBTh+JRziKJC9nq9pm47FASlMLxAkEA47+70eU40wBt 8 | fI0dc3Uj9WenCrDPz9xqhUw27whcE2tkyZxCaSCsw3Cz8AwToE013f8E2ZJoaNre 9 | 4PAdgeTt1QJBAOEpOsVeUhhQStYPMB8w43W/hq8vpyxtRMYpkwkMmwW42Zpq1KXa 10 | /S1cbhkHdxp5YvJeD7k2lmdkU/cXtohLDHcCQQCxzRH4f5epQwA26IRBiwYTpGRI 11 | eFkE0fNnNWT9n+0iTAlXTGKcaCH4Qph3ozX/Q8f2FA3ZPe+9TIIL4elnay4xAkEA 12 | xwa0xoWLN6axn+mo9ck3Jnv3x57tvJ2Rr0BMkjEsTrCI2LAZ68lZBeGwCDvLEgG+ 13 | btKqP2N7K0VJ2x6A4JTGHQJABlRgxZmrUANszmwBHJxhnNjm7/rsJ7w8OVGH94V2 14 | m8ypKrwV/nfkTkcXLT3hK+03BFqMDju+W0mkl9bQZRKBHw== 15 | -----END RSA PRIVATE KEY----- 16 | -------------------------------------------------------------------------------- /lib/cache.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var storage = require('./storage'); 4 | var ut = require('./util'); 5 | var log = require('./log'); 6 | 7 | // Resource caching interface, creates a new storage. 8 | var Cache = function(options) { 9 | this.maxSize = options.cache.items.limit; 10 | this.maxMemSize = options.cache.memory.limit && 11 | ut.mbToByte(options.cache.memory.limit); 12 | this.storage = storage.create({ 13 | type: options.cache.type, 14 | maxSize: this.maxSize, 15 | maxMemSize: this.maxMemSize, 16 | host: options.cache.database.host, 17 | port: options.cache.database.port 18 | }); 19 | }; 20 | 21 | // Save the key-value item with given expire time in ms. 22 | Cache.prototype.save = function(key, value, expire) { 23 | this.storage.save(key, value, expire); 24 | }; 25 | 26 | // Load the item for given key, return result via callback call. 27 | Cache.prototype.load = function(key, callback) { 28 | this.storage.load(key, callback); 29 | }; 30 | 31 | // Default cache instance. 32 | var instance = null; 33 | 34 | // Initializes a new cache for given storage type. 35 | exports.init = function(options) { 36 | instance = new Cache(options); 37 | }; 38 | 39 | // Saves new cache entry. 40 | exports.save = function(key, value, expire) { 41 | if (instance) { 42 | instance.save(key, value, expire); 43 | } else { 44 | log.error('No cache instance!'); 45 | } 46 | }; 47 | 48 | // Returns cache entry for given key if available. 49 | exports.load = function(key, callback) { 50 | if (instance) { 51 | instance.load(key, callback); 52 | } else { 53 | log.error('No cache instance!'); 54 | callback(); 55 | } 56 | }; 57 | -------------------------------------------------------------------------------- /lib/index.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var cluster = require('cluster'); 4 | var spawn = require('child_process').spawn; 5 | var fs = require('fs'); 6 | var crypto = require('crypto'); 7 | 8 | var log = require('./log'); 9 | var pkg = require('../package.json'); 10 | var SpdyProxy = require('./proxy'); 11 | 12 | var CONFIG = require('config'); 13 | var NAME = pkg.name; 14 | var TITLE = NAME + '/' + pkg.version; 15 | var ROOT_DIR = __dirname + '/..'; 16 | var STATSD_CONFIG_PATH = ROOT_DIR + '/config/statsd-runtime.js'; 17 | 18 | function showHelp() { 19 | console.log('%s\n' + 20 | '-h [--help] shows help\n' + 21 | '-v [--version] shows the version', TITLE); 22 | } 23 | 24 | function showVersion() { 25 | console.log(TITLE); 26 | } 27 | 28 | function addVersionConfig() { 29 | CONFIG.version = pkg.version; 30 | CONFIG.title = TITLE; 31 | CONFIG.name = NAME; 32 | } 33 | 34 | // Parse YAML configuration file and set options. 35 | CONFIG.getConfigSources().forEach(function(config) { 36 | addVersionConfig(); 37 | log.info('Using configuration %s', config.name); 38 | }); 39 | 40 | var cryptoSettings = {}; 41 | 42 | function initCryptoSettings() { 43 | cryptoSettings = { 44 | salt: crypto.createHash('sha256') 45 | .update(crypto.randomBytes(256)) 46 | .digest('hex') 47 | }; 48 | } 49 | 50 | function loadCryptoSettings() { 51 | cryptoSettings = JSON.parse(process.env.crypto); 52 | } 53 | 54 | // Handle command-line arguments. 55 | function handleArgs() { 56 | process.argv.forEach(function(value) { 57 | if (value === '-h' || value === '--help') { 58 | showHelp(); 59 | process.exit(1); 60 | } 61 | if (value === '-v' || value === '--version') { 62 | showVersion(); 63 | process.exit(1); 64 | } 65 | }); 66 | } 67 | 68 | function getWorkerCount() { 69 | if (!CONFIG.cluster.enabled) { 70 | // We start at least one worker, the master only serves PACs. 71 | return 1; 72 | } 73 | 74 | if (CONFIG.cluster.workers.auto) { 75 | // Using the general heuristic of 2x physical cores. 76 | return 2 * require('os').cpus().length; 77 | } else { 78 | return CONFIG.cluster.workers.count || 1; 79 | } 80 | } 81 | 82 | function startProxy() { 83 | var proxy = new SpdyProxy(CONFIG, cryptoSettings); 84 | proxy.listen(CONFIG.proxy.port); 85 | } 86 | 87 | function spawnWorkers(n) { 88 | log.debug('starting %d workers', n); 89 | var env = { crypto: JSON.stringify(cryptoSettings) }; 90 | for (var i = 0; i < n; ++i) { 91 | cluster.fork(env); 92 | } 93 | 94 | cluster.on('exit', function(worker, code, signal) { 95 | log.debug('worker %d died (%s)', worker.process.pid, signal || code); 96 | cluster.fork(env); 97 | }); 98 | } 99 | 100 | // Spawns a StatsD backend with the provided configuration. 101 | function spawnMetricsServer() { 102 | // Write the StatsD configuration to file. 103 | fs.writeFileSync(STATSD_CONFIG_PATH, JSON.stringify(CONFIG.metrics.statsd)); 104 | 105 | // Spawn the daemon. 106 | var metrics = spawn('node', [ 107 | ROOT_DIR + '/node_modules/statsd/stats.js', 108 | STATSD_CONFIG_PATH 109 | ]); 110 | 111 | metrics.stdout.on('data', function(data) { 112 | log.debug(data.toString()); 113 | }); 114 | 115 | metrics.stderr.on('data', function(data) { 116 | log.error(data.toString()); 117 | }); 118 | 119 | metrics.on('close', function(code) { 120 | log.debug(code); 121 | }); 122 | 123 | metrics.on('error', function(err) { 124 | log.error(err); 125 | }); 126 | } 127 | 128 | if (cluster.isMaster) { 129 | handleArgs(); 130 | initCryptoSettings(); 131 | 132 | if (CONFIG.metrics.enabled && CONFIG.metrics.statsd.spawn) { 133 | spawnMetricsServer(); 134 | } 135 | 136 | spawnWorkers(getWorkerCount()); 137 | } else { 138 | loadCryptoSettings(); 139 | startProxy(); 140 | } 141 | -------------------------------------------------------------------------------- /lib/log.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var winston = require('winston'); 4 | var cluster = require('cluster'); 5 | var fs = require('fs'); 6 | 7 | var util = require('./util'); 8 | 9 | var label = cluster.isMaster ? 'master ' + process.pid : 10 | 'worker ' + cluster.worker.process.pid; 11 | 12 | var LOGGING_CONFIG = require('config').logging; 13 | 14 | // Remove the default transports 15 | winston.clear(); 16 | 17 | if (LOGGING_CONFIG.console && LOGGING_CONFIG.console.enabled) { 18 | winston.add(winston.transports.Console, { 19 | level: LOGGING_CONFIG.console.level, 20 | timestamp: LOGGING_CONFIG.console.timestamp, 21 | colorize: LOGGING_CONFIG.console.colorize, 22 | label: label || null 23 | }); 24 | } 25 | 26 | if (LOGGING_CONFIG.file && LOGGING_CONFIG.file.enabled) { 27 | // If the 'clobber' option was specified, remove the existing log file. 28 | if (LOGGING_CONFIG.file.clobber && LOGGING_CONFIG.file.filename) { 29 | try { 30 | fs.unlinkSync(LOGGING_CONFIG.file.filename); 31 | } catch (err) { 32 | // Ignore errors 33 | } 34 | } 35 | 36 | winston.add(winston.transports.File, { 37 | level: LOGGING_CONFIG.file.level, 38 | timestamp: LOGGING_CONFIG.file.timestamp, 39 | filename: LOGGING_CONFIG.file.filename, 40 | maxsize: util.mbToByte(LOGGING_CONFIG.file.maxSize), 41 | json: LOGGING_CONFIG.file.json || false 42 | }); 43 | } 44 | 45 | exports.logger = winston; 46 | exports.log = winston.log; 47 | exports.debug = winston.debug; 48 | exports.info = winston.info; 49 | exports.warn = winston.warn; 50 | exports.error = winston.error; 51 | 52 | exports.logify = function(obj, objectLabel) { 53 | ['log', 'debug', 'info', 'warn', 'error'].forEach(function(level) { 54 | obj[level] = function() { 55 | var args = Array.prototype.slice.call(arguments); 56 | if (objectLabel && args.length > 0) { 57 | args[0] = '[' + objectLabel + '] ' + args[0]; 58 | } 59 | 60 | return winston[level].apply(winston, args); 61 | }; 62 | }); 63 | }; 64 | -------------------------------------------------------------------------------- /lib/metrics.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var StatsdClient = require('node-statsd').StatsD; 4 | var CONFIG = require('config'); 5 | var os = require('os'); 6 | var cluster = require('cluster'); 7 | var http = require('http'); 8 | var memwatch = require('memwatch'); 9 | 10 | var forEach = require('./util').forEach; 11 | var log = require('./log'); 12 | 13 | // Default sytem update interval in ms. 14 | var UPDATE_INTERVAL = 10000; 15 | 16 | // Return the current time. 17 | function now() { 18 | return new Date(); 19 | } 20 | 21 | // Return time difference in ms. 22 | function durationSince(startTime) { 23 | return now().getTime() - startTime.getTime(); 24 | } 25 | 26 | // Timer used to report time metrics. 27 | var Timer = function(metrics, key, timeout) { 28 | this.metrics = metrics; 29 | this.key = key; 30 | this.timeout = Math.max(0, timeout || 0); 31 | this.duration = 0; 32 | this.clear(); 33 | }; 34 | 35 | // Reset and start the timer. 36 | Timer.prototype.start = function() { 37 | var metrics = this.metrics; 38 | var timeout = this.timeout; 39 | var key = this.key; 40 | 41 | this.clear(); 42 | this.startTime = now(); 43 | 44 | if (timeout) { 45 | this._timeout = setTimeout(function() { 46 | metrics.count(key + '.timeout'); 47 | log.warn('metrics timeout (%d) for %s', timeout, key); 48 | }, timeout); 49 | } 50 | 51 | return this; 52 | }; 53 | 54 | // Resume a paused timer, do nothing if timer is active. 55 | Timer.prototype.resume = function() { 56 | if (this.startTime === null) { 57 | this.startTime = now(); 58 | } 59 | return this; 60 | }; 61 | 62 | // Return the duration in ms. 63 | Timer.prototype.getDuration = function() { 64 | var duration = this.duration; 65 | if (this.startTime) { 66 | duration += durationSince(this.startTime); 67 | } 68 | return duration; 69 | }; 70 | 71 | // Pause timer, return the accumulated duration up to this point. 72 | Timer.prototype.pause = function() { 73 | var accDuration = this.getDuration(); 74 | this.clear(); 75 | this.duration = accDuration; 76 | 77 | return accDuration; 78 | }; 79 | 80 | // Stop and clear timer, report metric, return the duration in ms. 81 | Timer.prototype.stop = function() { 82 | var duration = this.getDuration(); 83 | this.metrics.timing(this.key, duration); 84 | this.clear(); 85 | 86 | return duration; 87 | }; 88 | 89 | // Stop tracking time and timeouts, don't report anything. 90 | Timer.prototype.clear = function() { 91 | clearTimeout(this._timeout); 92 | this._timeout = null; 93 | this.startTime = null; 94 | this.duration = 0; 95 | }; 96 | 97 | // Named metrics session used to auto-prepend name prefix to all keys. 98 | var Session = function(metrics, name) { 99 | this.metrics = metrics; 100 | this.prefix = name + '.'; 101 | }; 102 | 103 | Session.prototype.key = function(key) { 104 | return this.prefix + key; 105 | }; 106 | 107 | Session.prototype.count = function(key, delta) { 108 | return this.metrics.count(this.key(key), delta); 109 | }; 110 | 111 | Session.prototype.gauge = function(key, value) { 112 | return this.metrics.gauge(this.key(key), value); 113 | }; 114 | 115 | Session.prototype.set = function(key, value) { 116 | return this.metrics.set(this.key(key), value); 117 | }; 118 | 119 | Session.prototype.timing = function(key, duration) { 120 | return this.metrics.timing(this.key(key), duration); 121 | }; 122 | 123 | // Create metrics timer, autostarts by default. 124 | Session.prototype.timer = function(key, timeout) { 125 | var timer = new Timer(this.metrics, this.key(key), timeout); 126 | timer.start(); 127 | return timer; 128 | }; 129 | 130 | // Create metrics timer for streams, autostarts by default. 131 | Session.prototype.streamTimer = function(stream, key, timeout) { 132 | var session = this; 133 | var timer = session.timer(key, timeout); 134 | 135 | // Inject into read to measure delays. 136 | var readableTimer = session.timer(key + '.read.delay'); 137 | readableTimer.pause(); 138 | var streamRead = stream.read; 139 | var readSize = 0; 140 | 141 | if (streamRead) { 142 | stream.read = function() { 143 | var ret = streamRead.apply(this, arguments); 144 | 145 | if (ret === null) { 146 | // Track empty read buffer to "readable" delay. 147 | readableTimer.resume(); 148 | } else { 149 | readSize += ret.length || 0; 150 | } 151 | return ret; 152 | }; 153 | } 154 | 155 | // Inject into write to measure delays. 156 | var drainTimer = session.timer(key + '.write.delay'); 157 | drainTimer.pause(); 158 | var streamWrite = stream.write; 159 | var writeSize = 0; 160 | 161 | if (streamWrite) { 162 | stream.write = function(data) { 163 | // Track empty write buffer to write dleay. 164 | drainTimer.pause(); 165 | 166 | writeSize += data.length || 0; 167 | 168 | return streamWrite.apply(this, arguments); 169 | }; 170 | } 171 | 172 | stream.once('finish', function() { 173 | timer.stop(); 174 | session.count(key + '.read.transfer', readSize); 175 | session.count(key + '.write.transfer', writeSize); 176 | session.count(key + '.finish'); 177 | 178 | readableTimer.stop(); 179 | drainTimer.stop(); 180 | }); 181 | 182 | stream.once('close', function() { 183 | timer.clear(); 184 | session.count(key + '.close'); 185 | 186 | readableTimer.clear(); 187 | drainTimer.clear(); 188 | }); 189 | 190 | stream.once('error', function() { 191 | timer.clear(); 192 | session.count(key + '.error'); 193 | 194 | readableTimer.clear(); 195 | drainTimer.clear(); 196 | }); 197 | 198 | stream.on('readable', function() { 199 | readableTimer.pause(); 200 | }); 201 | 202 | stream.on('drain', function() { 203 | drainTimer.resume(); 204 | }); 205 | 206 | return timer; 207 | }; 208 | 209 | // Metrics engine, used to track times and counters. 210 | var Metrics = function(options) { 211 | var metrics = this; 212 | metrics.options = options.metrics; 213 | metrics.clients = []; 214 | metrics.systemSession = metrics.session('system'); 215 | 216 | if (!options.metrics.enabled) { 217 | return; 218 | } 219 | 220 | if (options.metrics.system && options.metrics.system.enabled) { 221 | // Activate general system metrics. 222 | var updateInterval = options.metrics.system.interval * 1000 || 223 | UPDATE_INTERVAL; 224 | setInterval(metrics.update.bind(metrics), updateInterval); 225 | } 226 | 227 | if (options.metrics.statsd && options.metrics.statsd.enabled) { 228 | // Create a StatsD client and connect to the service. 229 | var statsdClient = new StatsdClient({ 230 | host: options.metrics.statsd.host, 231 | port: options.metrics.statsd.port, 232 | prefix: options.metrics.statsd.prefix + '.' 233 | }); 234 | 235 | metrics.clients.push(statsdClient); 236 | } 237 | 238 | if (options.metrics.profile.leaks.enabled) { 239 | // Heuristics-based leak detection. 240 | memwatch.on('leak', function(info) { 241 | metrics.systemSession.count('heap.growth', info.growth); 242 | log.warn('memory leak detected: ' + JSON.stringify(info)); 243 | }); 244 | 245 | // Post-GC stats reporting. 246 | memwatch.on('stats', function(stats) { 247 | forEach(stats, function(value, key) { 248 | metrics.systemSession.count('heap.' + key, value); 249 | }); 250 | }); 251 | } 252 | 253 | // Extensive (and expensive) heap profiling stats. 254 | var heapDiff = null; 255 | if (options.metrics.profile.heap.enabled) { 256 | memwatch.on('stats', function() { 257 | if (heapDiff) { 258 | // Report heap diff stats. 259 | var stats = heapDiff.end(); 260 | heapDiff = null; 261 | 262 | metrics.systemSession.count('heap.size', stats['size_bytes']); 263 | metrics.systemSession.count('heap.nodes.current', stats.nodes); 264 | metrics.systemSession.count('heap.nodes.freed', 265 | stats.change['freed_nodes']); 266 | metrics.systemSession.count('heap.nodes.allocated', 267 | stats.change.allocated); 268 | 269 | stats.change.details.forEach(function(elem) { 270 | metrics.systemSession.count('heap.details.' + elem.what + '.size', 271 | elem['size_bytes']); 272 | metrics.systemSession.count('heap.details.' + elem.what + 273 | '.nodes.allocated', elem['+']); 274 | metrics.systemSession.count('heap.details.' + elem.what + 275 | '.nodes.freed', elem['-']); 276 | }); 277 | } else { 278 | // Initial heap snapshot. 279 | heapDiff = new memwatch.HeapDiff(); 280 | } 281 | }); 282 | } 283 | }; 284 | 285 | // Return a new metrics session for given prefix name. 286 | Metrics.prototype.session = function(name) { 287 | return new Session(this, name); 288 | }; 289 | 290 | // Add middleware to handle metrics with following (optional) functions: 291 | // { 292 | // counter: function(key, delta), 293 | // gauge: function(key, value), 294 | // set: function(key, value), 295 | // timing: function(key, duration) 296 | // } 297 | Metrics.prototype.use = function(client) { 298 | if (client) { 299 | // Attach user-provided middleware. 300 | this.clients.push(client); 301 | } 302 | 303 | return this; 304 | }; 305 | 306 | // Detach the given middleware. 307 | Metrics.prototype.detach = function(client) { 308 | var index = this.clients.indexOf(client); 309 | if (index >= 0) { 310 | this.clients.splice(index, 1); 311 | } 312 | 313 | return this; 314 | }; 315 | 316 | // Update the system metrics. 317 | Metrics.prototype.update = function() { 318 | var metrics = this; 319 | 320 | if (cluster.isMaster) { 321 | var numCpus = os.cpus().length; 322 | 323 | this.systemSession.gauge('cpu.num', numCpus); 324 | this.systemSession.gauge('cpu.load.avg', os.loadavg()[0]); 325 | this.systemSession.gauge('memory.total', os.totalmem()); 326 | this.systemSession.gauge('memory.free', os.freemem()); 327 | 328 | // Collect average CPU usage stats. 329 | var cpuStats = { speed: 0, user: 0, nice: 0, sys: 0, idle: 0, irq: 0 }; 330 | os.cpus().forEach(function(cpu) { 331 | cpuStats.speed += cpu.speed; 332 | cpuStats.user += cpu.times.user; 333 | cpuStats.nice += cpu.times.nice; 334 | cpuStats.sys += cpu.times.sys; 335 | cpuStats.idle += cpu.times.idle; 336 | cpuStats.irq += cpu.times.irq; 337 | }); 338 | 339 | forEach(cpuStats, function(stat, s) { 340 | stat /= numCpus; 341 | metrics.systemSession.gauge('cpu.' + s + '.avg', stat); 342 | }); 343 | } 344 | 345 | function lengthSum(obj) { 346 | var s = 0; 347 | forEach(obj, function(e) { 348 | s += e.length; 349 | }); 350 | return s; 351 | } 352 | 353 | // Report per-process HTTP stats. 354 | this.systemSession.count('sockets', lengthSum(http.globalAgent.sockets)); 355 | this.systemSession.count('requests', lengthSum(http.globalAgent.requests)); 356 | 357 | log.debug('reporting system metrics'); 358 | }; 359 | 360 | // Check whether we should log the specific metric. 361 | Metrics.prototype.shouldLog = function(key) { 362 | if (!this.options.selection.enabled) { 363 | return true; 364 | } 365 | 366 | for (var i = 0; i < this.options.selection.set.length; ++i) { 367 | if (this.options.selection.set[i] === key) { 368 | return true; 369 | } 370 | } 371 | 372 | return false; 373 | }; 374 | 375 | // Change the count value for given key by the given delta. 376 | Metrics.prototype.count = function(key, delta) { 377 | if (!this.shouldLog(key)) { 378 | return; 379 | } 380 | 381 | if (delta === undefined) { 382 | delta = 1; 383 | } 384 | 385 | if (delta === 0) { 386 | return; 387 | } 388 | 389 | this.clients.forEach(function(client) { 390 | var counter = client.counter; 391 | if (!counter) { 392 | // Some clients separate increment and decrement counts. 393 | if (delta > 0) { 394 | counter = client.increment; 395 | } else { 396 | counter = client.decrement; 397 | } 398 | } 399 | 400 | if (counter) { 401 | counter.call(client, key, delta); 402 | } 403 | }); 404 | }; 405 | 406 | // Update the 'gauge' value for given key. 407 | Metrics.prototype.gauge = function(key, value) { 408 | if (!this.shouldLog(key)) { 409 | return; 410 | } 411 | 412 | this.clients.forEach(function(client) { 413 | if (client.gauge) { 414 | client.gauge(key, value); 415 | } 416 | }); 417 | }; 418 | 419 | Metrics.prototype.set = function(key, value) { 420 | if (!this.shouldLog(key)) { 421 | return; 422 | } 423 | 424 | this.clients.forEach(function(client) { 425 | if (client.set) { 426 | client.set(key, value); 427 | } 428 | }); 429 | }; 430 | 431 | // Add a timing entry for given key with given duration. 432 | Metrics.prototype.timing = function(key, duration) { 433 | if (!this.shouldLog(key)) { 434 | return; 435 | } 436 | 437 | this.clients.forEach(function(client) { 438 | if (client.timing) { 439 | client.timing(key, duration); 440 | } 441 | }); 442 | }; 443 | 444 | module.exports = new Metrics(CONFIG); 445 | -------------------------------------------------------------------------------- /lib/pac.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var fs = require('fs'); 4 | 5 | var TEMPLATE_TUNNEL_HTTPS = fs.readFileSync('config/pac-tunnel.js').toString(); 6 | var TEMPLATE_NO_TUNNEL = fs.readFileSync('config/pac-no-tunnel.js').toString(); 7 | 8 | exports.generate = function generate(hostAndPort, tunnelSsl) { 9 | var template = tunnelSsl ? TEMPLATE_TUNNEL_HTTPS : TEMPLATE_NO_TUNNEL; 10 | 11 | return template.replace('', hostAndPort); 12 | }; 13 | -------------------------------------------------------------------------------- /lib/plugins/adblock.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var url = require('url'); 4 | var http = require('http'); 5 | var log = require('../log'); 6 | var punycode = require('punycode'); 7 | 8 | var NAME = exports.name = 'adblock'; 9 | 10 | var metrics = require('../metrics').session(NAME); 11 | 12 | function fetchBlockList(url, list, callback) { 13 | log.debug('fetching blocklist'); 14 | // Fetch the list of hostnames to block 15 | http.get(url, function(res) { 16 | var bufs = []; 17 | 18 | res.on('data', function(chunk) { 19 | bufs.push(chunk); 20 | }); 21 | 22 | res.on('end', function() { 23 | var lines = Buffer.concat(bufs).toString().split('\n'); 24 | for (var i = 0; i < lines.length; i++) { 25 | var hostname = punycode.toASCII(lines[i].trim()); 26 | if (hostname === '') { 27 | continue; 28 | } 29 | 30 | // Use the hashtable as a set 31 | list[hostname] = true; 32 | } 33 | 34 | log.debug('fetched adblock list'); 35 | if (callback) { 36 | callback(); 37 | } 38 | }); 39 | }).on('error', function(e) { 40 | log.error('failed to fetch block list', e); 41 | }); 42 | } 43 | 44 | exports.blockList = {}; 45 | 46 | exports.init = function(config, callback) { 47 | var url = config.adblock.listUrl; 48 | 49 | fetchBlockList(url, exports.blockList, callback); 50 | }; 51 | 52 | exports.handleRequest = function(request, response, options, callback) { 53 | var requestedUrl = url.parse(request.url); 54 | var hosts = requestedUrl.host.split('.'); 55 | 56 | // We try to match all the subdomains 57 | // e.g.: a.b.example.com => a.b.example.com, b.example.com, example.com 58 | for (var i = hosts.length; i >= 2; i--) { 59 | var h = hosts.slice(-i).join('.'); 60 | 61 | if (h in exports.blockList) { 62 | metrics.count('hit'); 63 | request.debug('blocked ad ' + requestedUrl); 64 | 65 | response.writeHead(403, '', { 'content-type': 'text/plain' }); 66 | response.write('Blocked by adblock'); 67 | response.end(); 68 | 69 | callback(null, true); 70 | return; 71 | } 72 | } 73 | 74 | metrics.count('miss'); 75 | callback(null, false); 76 | }; 77 | -------------------------------------------------------------------------------- /lib/plugins/cache.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var CONFIG = require('config').cache; 4 | 5 | var cache = require('../cache'); 6 | var util = require('../util'); 7 | 8 | var MAX_EXPIRE = CONFIG.expire.max; 9 | var DEF_EXPIRE = CONFIG.expire.default; 10 | var MIN_ITEM_SIZE = CONFIG.items.size.min || 0; 11 | var MAX_ITEM_SIZE = CONFIG.items.size.max && 12 | util.kbToByte(CONFIG.items.size.max) || 13 | Infinity; 14 | 15 | var NAME = exports.name = 'cache'; 16 | 17 | var metrics = require('../metrics').session(NAME); 18 | 19 | // Return the key used to cache resources for given request. 20 | function createKey(request) { 21 | var key = [request.headers.host, request.path, 22 | request.headers['accept-language'] || '']; 23 | return key; 24 | } 25 | 26 | // Parses cache control header and last-modified. 27 | function parseCacheControl(headers) { 28 | var cacheHeaders = {}; 29 | 30 | var lastMod = headers['last-modified']; 31 | if (lastMod) { 32 | cacheHeaders['last-modified'] = new Date(lastMod); 33 | } 34 | 35 | cacheHeaders.date = headers.date ? new Date(headers.date) : new Date(); 36 | cacheHeaders.age = headers.age; 37 | 38 | if (headers.expires) { 39 | cacheHeaders.expires = new Date(headers.expires); 40 | } 41 | 42 | var cacheControl = headers['cache-control']; 43 | if (cacheControl) { 44 | cacheControl.split(',').forEach(function(elem) { 45 | elem = elem.trim(); 46 | var i = elem.indexOf('='); 47 | if (i === -1) { 48 | cacheHeaders[elem] = true; 49 | } else { 50 | cacheHeaders[elem.substr(0, i)] = elem.substr(i + 1); 51 | } 52 | }); 53 | } 54 | 55 | return cacheHeaders; 56 | } 57 | 58 | // Returns the maximum freshness age for the resource. 59 | function maxAge(cacheHeaders) { 60 | // Max age can be set to 0 to disable caching. 61 | var expire = cacheHeaders['s-maxage']; 62 | if (typeof expire === 'undefined') { 63 | expire = cacheHeaders['max-age']; 64 | } 65 | 66 | if (typeof expire !== 'undefined') { 67 | expire = parseInt(expire); 68 | } else if (cacheHeaders.expires) { 69 | expire = (cacheHeaders.expires.getTime() - 70 | cacheHeaders.date.getTime()) / 1000; 71 | } else { 72 | expire = DEF_EXPIRE; 73 | } 74 | 75 | return Math.max(0, expire); 76 | } 77 | 78 | // Returns the conservative age of the resource. 79 | function currentAge(cacheHeaders) { 80 | var age = ((new Date()).getTime() - cacheHeaders.date.getTime()) / 1000; 81 | 82 | // TODO(esawin): foreward request date needs to be considered here, too. 83 | age = Math.max(age, cacheHeaders.age || 0); 84 | 85 | return Math.max(0, age); 86 | } 87 | 88 | // Returns the expire time in seconds. 89 | function expirationTime(cacheHeaders) { 90 | var expire = maxAge(cacheHeaders) - currentAge(cacheHeaders); 91 | 92 | return Math.max(0, Math.min(MAX_EXPIRE, expire)); 93 | } 94 | 95 | function unserialize(entry) { 96 | var pack = {}; 97 | pack.statusCode = parseInt(entry.statusCode); 98 | pack.size = parseInt(entry.size); 99 | pack.headers = JSON.parse(entry.headers); 100 | pack.cacheControl = JSON.parse(entry.cacheControl); 101 | pack.data = entry.data; 102 | return pack; 103 | } 104 | 105 | function serialize(entry) { 106 | var pack = {}; 107 | pack.statusCode = entry.statusCode; 108 | pack.size = entry.size; 109 | pack.headers = JSON.stringify(entry.headers); 110 | pack.cacheControl = JSON.stringify(entry.cacheControl); 111 | pack.data = Buffer.concat(entry.data, entry.size); 112 | return pack; 113 | } 114 | 115 | exports.handleRequest = function(request, response, options, callback) { 116 | var cacheControl = parseCacheControl(request.headers); 117 | 118 | if (request.method !== 'GET' || 119 | request.headers.authorization || 120 | request.headers.range || 121 | cacheControl['no-cache'] || 122 | cacheControl['max-age'] === '0') { 123 | // Do nothing for POST requests or when caching is disabled by 124 | // the client. 125 | return callback(null, false); 126 | } 127 | 128 | var key = createKey(request); 129 | cache.load(key, function(error, cached) { 130 | var handled = false; 131 | 132 | if (!error && cached) { 133 | handled = true; 134 | 135 | var entry = unserialize(cached); 136 | request.debug('delivering %d bytes from cache', entry.size); 137 | response.writeHead(entry.statusCode, '', entry.headers); 138 | response.end(entry.data); 139 | 140 | metrics.count('hit'); 141 | metrics.count('transfer', entry.size); 142 | } else { 143 | if (cacheControl['only-if-cached']) { 144 | handled = true; 145 | response.writeHead(504, 'Failed fetching only-if-cached resource'); 146 | response.end(); 147 | } 148 | 149 | metrics.count('miss'); 150 | } 151 | 152 | callback(error, handled); 153 | }); 154 | }; 155 | 156 | // Return true if the given resource entry is cacheable, false otherwise. 157 | function cacheable(request, entry) { 158 | var requestCacheControl = parseCacheControl(request.headers); 159 | 160 | if (request.method !== 'GET' || 161 | request.headers.authorization || 162 | request.headers.range || 163 | requestCacheControl['no-store'] || 164 | entry.headers['accept-ranges'] || 165 | entry.headers['content-range'] || 166 | entry.cacheControl['private'] || 167 | entry.cacheControl['no-store']) { 168 | // Non-cacheable resource. 169 | return false; 170 | } 171 | if (entry.headers.vary) { 172 | var vary = entry.headers.vary.trim().split(','); 173 | if (vary.length > 0 && 174 | (vary.length > 1 || vary[0].trim() !== 'accept-encoding')) { 175 | // We only support accept-encoding vary directives. 176 | return false; 177 | } 178 | } 179 | return true; 180 | } 181 | 182 | // Aggregates data and caches it when appropriate. 183 | exports.handleResponse = function(request, source, dest) { 184 | var entry = { 185 | statusCode: source.statusCode, 186 | headers: source.headers, 187 | cacheControl: parseCacheControl(source.headers), 188 | data: [], 189 | size: 0 190 | }; 191 | 192 | // Expire time in seconds. 193 | var expire = expirationTime(entry.cacheControl); 194 | 195 | if (!cacheable(request, entry) || expire <= 0) { 196 | metrics.count('no-save'); 197 | 198 | // Do nothing for POST requests or when caching is disabled. 199 | source.forward(dest); 200 | source.resume(); 201 | return; 202 | } 203 | 204 | dest.writeHead(entry.statusCode, entry.headers); 205 | 206 | source.on('data', function(chunk) { 207 | dest.write(chunk); 208 | entry.size += chunk.length; 209 | 210 | if (entry.size <= MAX_ITEM_SIZE) { 211 | // Only aggregate data if we are going to cache it. 212 | entry.data.push(chunk); 213 | } 214 | }); 215 | 216 | source.on('end', function() { 217 | dest.end(); 218 | 219 | if (entry.size < MIN_ITEM_SIZE || entry.size > MAX_ITEM_SIZE) { 220 | // Do not cache item outside of configured size constraints. 221 | return; 222 | } 223 | // Cache data. 224 | var key = createKey(request); 225 | cache.save(key, serialize(entry), expire); 226 | 227 | request.debug('cached %d bytes for %d s', entry.size, expire); 228 | }); 229 | 230 | source.resume(); 231 | }; 232 | -------------------------------------------------------------------------------- /lib/plugins/dom/gif2video.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var url = require('url'); 4 | var fs = require('fs'); 5 | var temp = require('temp'); 6 | var util = require('util'); 7 | var spawn = require('child_process').spawn; 8 | 9 | temp.track(); 10 | 11 | var pendingIntercepts = {}; 12 | 13 | var NAME = exports.name = 'gif2video'; 14 | 15 | var metrics = require('../../metrics').session(NAME); 16 | 17 | function transcodeGif(source, callback) { 18 | var gifPath = temp.path({ suffix: '.gif' }); 19 | 20 | var gifStream = fs.createWriteStream(gifPath); 21 | source.pipe(gifStream).on('finish', function() { 22 | var destPath = temp.path({ suffix: '.webm' }); 23 | var ffmpeg = spawn('ffmpeg', 24 | ['-i', gifPath, '-vcodec', 'libvpx', destPath]); 25 | 26 | metrics.streamTimer(ffmpeg, 'transcode'); 27 | 28 | ffmpeg.on('close', function(code) { 29 | fs.unlink(gifPath); 30 | 31 | if (code !== 0) { 32 | callback('Failed, exit code ' + code); 33 | return; 34 | } 35 | 36 | fs.stat(destPath, function(err, stats) { 37 | if (err) { 38 | callback(err); 39 | return; 40 | } 41 | 42 | var videoSource = fs.createReadStream(destPath); 43 | videoSource.pause(); 44 | 45 | videoSource.on('end', function() { 46 | fs.unlink(destPath); 47 | }); 48 | 49 | callback(null, videoSource, stats.size); 50 | }); 51 | }); 52 | }); 53 | } 54 | 55 | // Intercept GIF requests and serve transcoded to webm 56 | exports.handleRequest = function(request, response, options, callback) { 57 | var gifUrl = pendingIntercepts[request.originalUrl]; 58 | if (gifUrl) { 59 | // Replace the url with the intercepted version 60 | request._webmUrl = request.originalUrl; 61 | request.url = request.originalUrl = gifUrl; 62 | 63 | request.path = url.parse(gifUrl).path; 64 | } 65 | 66 | callback(null, false); 67 | }; 68 | 69 | // Replace with