├── .gitignore ├── .ssl ├── www.example.com.cert └── www.example.com.key ├── Makefile ├── README.md ├── package.json ├── server.js ├── t.gif └── test └── index.html /.gitignore: -------------------------------------------------------------------------------- 1 | *.log 2 | pid.txt 3 | node_modules 4 | -------------------------------------------------------------------------------- /.ssl/www.example.com.cert: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDBzCCAe+gAwIBAgIJAKk5o8Ap+PiJMA0GCSqGSIb3DQEBBQUAMBoxGDAWBgNV 3 | BAMMD3d3dy5leGFtcGxlLmNvbTAeFw0xMzEwMTYxODM2MjhaFw0yMzEwMTQxODM2 4 | MjhaMBoxGDAWBgNVBAMMD3d3dy5leGFtcGxlLmNvbTCCASIwDQYJKoZIhvcNAQEB 5 | BQADggEPADCCAQoCggEBAMeEGp9IM/uM4o48mm3erLf/ptGybKwGhrHQ0G+sJp9L 6 | ij+l6Q6yq6x0XbrUe1fz4tYWgXuR6FvXGM7eI1STLBLJSXbHkKef7FLr64WUq513 7 | BTE0nInTPgmHyo+vSlgUPZXpP7wbAzznZKP7XcA2chhD6iWKG824reid4ZCFCugq 8 | JpqZEikOLX3rloj6+kvMlNNR8MVejwHRBA/PR5kBMiaAQEWWV1YXa8qTu0qxjDQn 9 | xnoR2mu/kZhmgTE6i8hSrbGlX0j5q5KFoqawxF4IdkJgCwtG7JndznaVV4dMgach 10 | OjRDiCFm6MNOOQl7+qc1azfEQG3G+fxI8N2TaANp/sMCAwEAAaNQME4wHQYDVR0O 11 | BBYEFIZnu6jnGfRREgQfAtzbIkSngDcLMB8GA1UdIwQYMBaAFIZnu6jnGfRREgQf 12 | AtzbIkSngDcLMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6Lfpn8 13 | ep4SAAgkxWA1csBZBQeDgJHLjthXgdiF+LNIXeXQtqEB4Lr2x6N7ZGqXAvDDXl/D 14 | nEFAbCeioMGOY2lGs0ONWmioIc5DKxAXTkGDdGKGPJJbPERpg+khWl2IOeHpjflj 15 | 1fHlIUe0GBuu1EFqqKX9SXFJx9Si9FwgA7AR3esDS+M6soHxAHnhe2Yncfv5My9+ 16 | oChIh923VhVFGaNE3T2LCTjTSyOgYvtINgw2W8JtuapwvK0c/hxZAhANOLbxmClI 17 | ZuSKJrVbN+BnxBP/G6FFPOS/e/IuDpzd2BwsboALy+bW7q1Isy7gfAA03kMoe8HI 18 | hDSJ9DZU/oW13hE= 19 | -----END CERTIFICATE----- 20 | -------------------------------------------------------------------------------- /.ssl/www.example.com.key: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEpQIBAAKCAQEAx4Qan0gz+4zijjyabd6st/+m0bJsrAaGsdDQb6wmn0uKP6Xp 3 | DrKrrHRdutR7V/Pi1haBe5HoW9cYzt4jVJMsEslJdseQp5/sUuvrhZSrnXcFMTSc 4 | idM+CYfKj69KWBQ9lek/vBsDPOdko/tdwDZyGEPqJYobzbit6J3hkIUK6CommpkS 5 | KQ4tfeuWiPr6S8yU01HwxV6PAdEED89HmQEyJoBARZZXVhdrypO7SrGMNCfGehHa 6 | a7+RmGaBMTqLyFKtsaVfSPmrkoWiprDEXgh2QmALC0bsmd3OdpVXh0yBpyE6NEOI 7 | IWbow045CXv6pzVrN8RAbcb5/Ejw3ZNoA2n+wwIDAQABAoIBAQCpQisLocdnyjeR 8 | D1y7hMLmPd4Z29JRhh/SziFl+5ewX/di06+Jpo35eabijsws61wu31ztpfSHpU0I 9 | gm9ampgzP8wxFlBjxEpKdpaR9nQ42/XtroJ2cl0Y3Lg9eSoK2vD8Mqq9O/VdP7ij 10 | XOZF4Gqep08GlcnMlrYCt53aauO73DaLURObUPtJBl6hsAsY5Jf4Ty3hoo+/eHMB 11 | pb+4EkKxEg89DkXEQFOCsOtkaRC/mVb99ZduUXFof8Xm2ABjB+vcm3/8gaZ6V+AA 12 | ZNd6Jb8AE+02SR/RPYfDdBaKxzhOJvMAS3ytYoDSqcrh+zKS8bqL7YJwee/M9wJG 13 | 3lXtPI/hAoGBAO0DWgfERaSgY/HKd+MmBjiTnq3Ju9/40b73IOksOd/n6bFZQmL3 14 | gNwJCYz1UJY0H8/ocVBLhjW+4NN28Vn0ZAaZMNYJDFSOrNp2ZiVxbJaKLtYHX127 15 | UU1k3S4YmdudkkBSxpW9nB+nT2uUa2O5nvzyzKDHbZ/ESyNEXjIok6sLAoGBANd/ 16 | w+ArNMiCK1f6XgLyr1faHGGP6otLhZSY0gUIyaqBm4U/pMl9eX1s7QLeyvXB8/x3 17 | P/yCxj0g/M3PEiyz8rWy0Z4bpT0HAPya1devCpQrgwXWNUWlrWKACMSXG53YINKG 18 | ci1VFtUOphEr/cXpW/vZf5eg/NjbMrsD7X8a2w4pAoGBANjCxPbvcQYDzgQXOJfc 19 | cboSf//OzO0kYac12rqFwRRexCJ3ULi0RPx3o21v+di1KRb7LY7S05aZ0IJ1eHvd 20 | gBFszvYg5k77AWj2+apq1nXDQNxrd7OAmfWfNo1u4F+y90uuqIHQHFXyrTblUWWu 21 | IJKT98NfQInqexFw+HkFFTBLAoGAJG/E5b1IcnKX84swpBz2isslK1XTGXROhL6G 22 | HDXNK1g4vIHzUeI2TX/CX07eUElYAKMFHaPa8vEF7aKKdyaB7jjq+mnAOZ5ai1t+ 23 | trYw+raUs8LxRPJra5EsalkGYVzux8nVulZ9ws50Q8kFYpY/aEjxKukcd2ownLBg 24 | UrJuwWkCgYEA2J1fzsXQml+NvhWpYnG9qWO9OQYY03Sm6ktLJ++bNSFfTqPFOAKa 25 | 5/MuDc2Q8CNhJ/Ki82iuCjD4KoI2TlfQUuplW16apzanDm8j62iuSAPZvOMp5vbH 26 | S5HYCU694phDcjSwFE6p5VONGOGFemclRDhriMcb8ZG6bGMCpWHa6QY= 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Starts collector server 2 | server: 3 | node server.js & 4 | 5 | # Kill collector server 6 | kill: 7 | kill -9 `cat pid.txt` 8 | rm pid.txt 9 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Backend Collector for Client-Side Data 2 | This is a Node.js backend collector for client-side data that is tracked by [sp.js](#appendix-how-to-use-spjs-analytics-javascript-library) Analytics JavaScript library. 3 | All tracked events are collected in events.log following [logging best practices](http://dev.splunk.com/view/logging-best-practices/SP-CAAADP6) for Splunk log ingestion. 4 | 5 | Refer to [appendix](#appendix-how-to-use-spjs-analytics-javascript-library) below on how to use sp.js simple API for tracking. 6 | 7 | ### Getting Started 8 | * Install with Node.js package manager [npm](http://npmjs.org/): 9 | 10 | $ npm install 11 | 12 | * Configure your collector server HTTP/HTTPS ports and SSL certs by changing HTTP_PORT, HTTPS_PORT, and SSL_OPTS variables at the top of server.js file. 13 | 14 | This configuration step can be skipped for **test & dev purposes**. By default, the server binds to ports 3000 and 4443 for HTTP/HTTPS traffic. It also uses self-signed certificates for SSL under `.ssl/` directory, so we recommend you replace them with real certificates for a secure production solution. 15 | 16 | * Start the collector server by typing: 17 | 18 | $ node server.js 19 | 20 | If you have configured server ports to standard ports 80 and 443, you'll need to `sudo node server.js` to start the server as root unless you have rights to bind to privileged ports < 1024 21 | 22 | You should see something similar to: 23 | 24 | Listening to HTTP on port 3000 25 | Listening to HTTPS on port 4443 26 | 27 | That's it! 28 | After pointing sp.js library to your collector server address using `sp.load()`, watch the tracked events being collected in newly created local file **events.log** 29 | 30 | ### Additional Resources 31 | 32 | * Still using 3rd party web analytics providers? Build your own using Splunk! 33 | 34 | ## Appendix: How to use sp.js Analytics JavaScript Library 35 | ### Setup 36 | To use sp.js, simply paste the following snippet of code before the closing `` tag on your page: 37 | ```html 38 | 42 | ``` 43 | Make sure to replace `https://www.example.com` with your **own collector server URL** to send the data to. 44 | 45 | ### API 46 | sp.js provides a common set of tracking methods similar to leading web analytics providers & exemplified by the clean API provided by [segment.io](https://segment.io/libraries/analytics.js/). 47 | 48 | Here’s the list of tracking methods provided by sp.js: 49 | 50 | * [sp.track(event, properties, fn)](#sptrackevent-properties-fn) 51 | * [sp.trackLink(links, event, properties)](#sptracklinklinks-event-properties) 52 | * [sp.pageview(url)](#sppageviewurl) 53 | * [sp.identify(userId, userTraits)](#spidentifyuserid-usertraits) 54 | 55 | Full Definition: 56 | 57 | #### sp.track(event, properties, fn) 58 | Track a custom event (i.e. user action) along with a set of associated event properties. 59 | ```js 60 | sp.track('Preview Movie', { 61 | title: 'World War Z', 62 | category: 'Action', 63 | loggedIn: false 64 | }); 65 | ``` 66 | Parameters: 67 | * `event`: name string of the event to track 68 | * `properties` (optional): properties object of key-value pairs associated with the event 69 | * `fn` (optional): callback function to be called after short timeout 70 | 71 | #### sp.trackLink(links, event, properties) 72 | Track link clicks, including outbound links, with a custom event and custom properties. Tracking occurs before page changes. This automatically records properties such as the anchor (a) tag's href and text. 73 | ```js 74 | sp.trackLink($('a.free-download'), 'Click Free Download Link', { 75 | linkColor: 'Green' 76 | }); 77 | ``` 78 | Parameters: 79 | * `links`: link DOM or jQuery element to track clicks on. This can also be an array of such elements. 80 | * `event`: name string of the event to track. This can also be a function which returns event string name using the clicked link element as argument. 81 | * `properties` (optional): properties object of key-value pairs associated with the event. This can also be a function which returns properties object using the clicked link element as argument. 82 | 83 | #### sp.pageview(url) 84 | Tracks a 'pageview' event including document title and referrer. This is automatically called by default. 85 | ```js 86 | sp.pageview(); 87 | ``` 88 | Parameters: 89 | * `url` (optional): url string. Defaults to page url. 90 | 91 | #### sp.identify(userId, userTraits) 92 | Associate a user with an ID, and record user-specific traits or persistent properties. These persistent properties will be automatically added as properties to any subsequent tracked event. 93 | ```js 94 | sp.identify("power-user-3961", { 95 | email: "abc@example.com", 96 | age: 30, 97 | gender: "male" 98 | }); 99 | ``` 100 | Parameters: 101 | * `userId` (optional): unique ID string to associate with the user. sp.js automatically assigns a universally unique id to each visitor, so you can skip this. 102 | * `userTraits` (optional): properties object of key-value pairs associated with the user. userTraits are automatically included with all events by this user. 103 | 104 | #### sp.init(settings) 105 | **Advanced Usage**: method to configure library parameters. Typically `sp.load()` is all you need, and it's already called in the JavaScript snippet that you included in your page header. See [Setup](#setup) section above. 106 | ```js 107 | sp.init({ 108 | api_host: // typically set via sp.load(YOUR_COLLECTOR_URL) 109 | tracking_pageview: true, // default to tracking all page views 110 | track_links_timeout: 300, // default to 300ms 111 | cookie_name: "_sp", // defaults to "_sp" 112 | cookie_expiration: 365, // defaults to 365 days 113 | cookie_domain: "example.com" // defaults to your website domain 114 | }); 115 | ``` 116 | Parameters: 117 | * `settings`: settings object to apply one more custom configurations to sp.js. For most purposes, default values are applicable. 118 | 119 | 120 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "collector", 3 | "description": "Sample backend collection engine for splunk javascript client tag", 4 | "version": "0.0.1", 5 | "private": true, 6 | "dependencies": { 7 | "express": "3.2.5" 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /server.js: -------------------------------------------------------------------------------- 1 | var express = require('express'); 2 | var fs = require('fs'); 3 | var path = require('path'); 4 | var http = require('http'); 5 | var https = require('https'); 6 | var app = express(); 7 | 8 | var HTTP_PORT = 3000, 9 | HTTPS_PORT = 4443, 10 | SSL_OPTS = { 11 | key: fs.readFileSync(path.resolve(__dirname,'.ssl/www.example.com.key')), 12 | cert: fs.readFileSync(path.resolve(__dirname,'.ssl/www.example.com.cert')) 13 | }; 14 | 15 | /* 16 | * Define Middleware & Utilties 17 | ********************************** 18 | */ 19 | var allowCrossDomain = function(req, res, next) { 20 | if (req.headers.origin) { 21 | res.header('Access-Control-Allow-Origin', req.headers.origin); 22 | } 23 | res.header('Access-Control-Allow-Credentials', true); 24 | // send extra CORS headers when needed 25 | if ( req.headers['access-control-request-method'] || 26 | req.headers['access-control-request-headers']) { 27 | res.header('Access-Control-Allow-Headers', 'X-Requested-With'); 28 | res.header('Access-Control-Allow-Methods', 'GET,POST,OPTIONS'); 29 | res.header('Access-Control-Max-Age', 1728000); // 20 days 30 | // intercept OPTIONS method 31 | if (req.method == 'OPTIONS') { 32 | res.send(200); 33 | } 34 | } 35 | else { 36 | next(); 37 | } 38 | }; 39 | 40 | // trim string value and enclose it with double quotes if needed 41 | var parseValue = function(value) { 42 | if (typeof value === "string") { 43 | // trim 44 | value = value.replace(/^\s+|\s+$/g, ''); 45 | if (value == "") { 46 | value = '""'; 47 | } else if (value.split(' ').length > 1) { 48 | // enclose with "" if needed 49 | value = '"' + value + '"'; 50 | } 51 | } 52 | return value; 53 | } 54 | 55 | // decode and parse query param param 56 | var parseDataQuery = function(req, debug) { 57 | if (!req.query.data) { 58 | if (debug) { console.error('No \'data\' query param defined!') }; 59 | return false; 60 | } 61 | var data = {}; 62 | try { 63 | data = JSON.parse(decodeURIComponent(req.query.data)); 64 | } catch (e) { 65 | if (debug) { console.error('Failed to JSON parse \'data\' query param') }; 66 | return false; 67 | } 68 | return data; 69 | } 70 | 71 | // create single event based on data which includes time, event & properties 72 | var createAndLogEvent = function(data, req) { 73 | var time = (data && data.t) || new Date().toISOString(), 74 | event = (data && data.e) || "unknown", 75 | properties = (data && data.kv) || {}; 76 | 77 | // append some request headers (ip, referrer, user-agent) to list of properties 78 | properties.ip = req.ip; 79 | properties.origin = (req.get("Origin")) ? req.get("Origin").replace(/^https?:\/\//, '') : ""; 80 | properties.page = req.get("Referer"); 81 | properties.useragent = req.get("User-Agent"); 82 | 83 | // log event data in splunk friendly timestamp + key/value(s) format 84 | var entry = time + " event=" + parseValue(event); 85 | for (var key in properties) { 86 | var value = parseValue(properties[key]); 87 | entry += " " + key + "=" + value; 88 | } 89 | entry += "\n"; 90 | fs.appendFile(path.resolve(__dirname, './events.log'), entry, function(err) { 91 | if (err) { 92 | console.log(err); 93 | } else { 94 | //console.log("Logged tracked data"); 95 | } 96 | }); 97 | } 98 | 99 | /* 100 | * Use Middlewares 101 | ********************************** 102 | */ 103 | app.use(express.logger()); 104 | //app.use(express.compress()); 105 | app.use(allowCrossDomain); 106 | app.use(function(err, req, res, next) { 107 | console.error(err.stack); 108 | res.send(500, 'Something broke!'); 109 | }); 110 | 111 | /* 112 | * Create Tracking Endpoints 113 | ********************************** 114 | */ 115 | 116 | // API endpoint tracking 117 | app.get('/track', function(req, res) { 118 | res.setHeader('Content-Type', 'application/json'); 119 | var data; 120 | // data query param required here 121 | if ((data = parseDataQuery(req, true)) === false) { 122 | res.send('0'); 123 | } 124 | createAndLogEvent(data, req); 125 | res.send('1'); 126 | }); 127 | 128 | // IMG beacon tracking - data query optional 129 | app.get('/t.gif', function(req, res) { 130 | res.setHeader('Content-Type', 'image/gif'); 131 | res.setHeader('Cache-Control', 'private, no-cache, no-cache=Set-Cookie, proxy-revalidate'); 132 | res.setHeader('Expires', 'Sat, 01 Jan 2000 12:00:00 GMT'); 133 | res.setHeader('Pragma', 'no-cache'); 134 | // data query param optional here 135 | var data = parseDataQuery(req) || {}; 136 | // fill in default success event if none specified 137 | if (!data.e) { data.e = "success";} 138 | createAndLogEvent(data, req); 139 | res.sendfile(path.resolve(__dirname, './t.gif')); 140 | }); 141 | 142 | // root 143 | app.get('/', function(req, res) { 144 | res.send(""); 145 | }); 146 | 147 | var pidFile = path.resolve(__dirname, './pid.txt'); 148 | fs.writeFileSync(pidFile, process.pid, 'utf-8'); 149 | 150 | // Create an HTTP service. 151 | http.createServer(app).listen(HTTP_PORT,function() { 152 | console.log('Listening to HTTP on port ' + HTTP_PORT); 153 | }); 154 | 155 | // Create an HTTPS service identical to the HTTP service. 156 | https.createServer(SSL_OPTS, app).listen(HTTPS_PORT,function() { 157 | console.log('Listening to HTTPS on port ' + HTTPS_PORT); 158 | }); 159 | -------------------------------------------------------------------------------- /t.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/splunk/splunk-demo-collector-for-analyticsjs/41015a74e3f230efd58afac5f8a08fee92c58492/t.gif -------------------------------------------------------------------------------- /test/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 12 | 13 | 14 | 15 | 16 | 17 | 23 |
24 | Outgoing link A 25 | Outgoing link B 26 | 27 | 48 | 49 | 50 | 51 | --------------------------------------------------------------------------------