├── img ├── thoth.png ├── thoth-api-schema.png └── powered-trulia-black.png ├── assets └── img │ └── thoth.png ├── .gitignore ├── endpoints ├── list │ ├── list_request.js │ └── list_response.js ├── pools │ ├── pools_request.js │ └── pools_response.js ├── servers │ ├── servers_request.js │ ├── list_request.js │ ├── list_response.js │ └── servers_response.js └── realtime │ ├── realtime_request.js │ └── realtime_response.js ├── environment.js ├── package.json ├── fields_mapping.js ├── helpers └── http_request.js ├── dispatchers ├── list_dispatcher.js ├── pools_dispatcher.js ├── realtime_dispatcher.js └── servers_dispatcher.js ├── LICENSE ├── README.mkd └── index.js /img/thoth.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trulia/thoth-api/master/img/thoth.png -------------------------------------------------------------------------------- /assets/img/thoth.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trulia/thoth-api/master/assets/img/thoth.png -------------------------------------------------------------------------------- /img/thoth-api-schema.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trulia/thoth-api/master/img/thoth-api-schema.png -------------------------------------------------------------------------------- /img/powered-trulia-black.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trulia/thoth-api/master/img/powered-trulia-black.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | lib-cov 2 | *.seed 3 | *.log 4 | *.csv 5 | *.dat 6 | *.out 7 | *.pid 8 | *.gz 9 | 10 | pids 11 | logs 12 | results 13 | .DS_Store 14 | npm-debug.log 15 | node_modules 16 | -------------------------------------------------------------------------------- /endpoints/list/list_request.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | /** 3 | * createListRequest 4 | * @param req 5 | * @param filter 6 | * @returns Object 7 | */ 8 | createListRequest: function(req, filter) { 9 | return { 10 | 'q': '*:*', 11 | 'rows': 0, 12 | 'facet': true, 13 | 'facet.field': filter, 14 | 'facet.limit': 1000 15 | }; 16 | } 17 | }; -------------------------------------------------------------------------------- /environment.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Wrapper for process.env 3 | * 4 | * @author Damiano Braga 5 | */ 6 | 7 | module.exports = function () { 8 | /** 9 | * Environment 10 | */ 11 | process.env.PORT = process.env.PORT || 3001; 12 | process.env.THOTH_PORT = process.env.THOTH_PORT || '8983'; 13 | process.env.THOTH_HOST = process.env.THOTH_HOST || 'localhost'; 14 | 15 | }; 16 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "thoth-api", 3 | "version": "0.0.1", 4 | "description": "Rest API for thoth", 5 | "main": "index.js", 6 | "repository": { 7 | "type": "git", 8 | "url": "git@github.com:dbraga/thoth-api.git" 9 | }, 10 | "keywords": [ 11 | "thoth", 12 | "api", 13 | "node" 14 | ], 15 | "scripts": { 16 | "start": "node index.js" 17 | }, 18 | "dependencies": { 19 | "express": ">=3.0.6", 20 | "socket.io": "~1.1.0", 21 | "nodemon": "=1.0.17", 22 | "chai": "=1.9.1" 23 | }, 24 | "author": "Damiano Braga", 25 | "license": "BSD", 26 | "devDependencies": { 27 | "socket.io": "~1.1.0" 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /fields_mapping.js: -------------------------------------------------------------------------------- 1 | module.exports = function () { 2 | return { 3 | avg : { 4 | qtime: 'avg_qtime_d', 5 | nqueries: 'tot-count_i', 6 | queriesOnDeck: 'avg_requestsInProgress_d' 7 | }, 8 | count: { 9 | exception: 'exceptionCount_i', 10 | zeroHits: 'zeroHits-count_i', 11 | nqueries: 'tot-count_i' 12 | }, 13 | integral: { 14 | exception: 'exceptionCount_i', 15 | nqueries: 'tot-count_i', 16 | zeroHits: 'zeroHits-count_i' 17 | }, 18 | distribution: { 19 | qtime: 'range-0-10_i,range-10-100_i,range-100-1000_i,range-1000-OVER_i' 20 | }, 21 | slowqueries: 'params_s,qtime_i', 22 | exception: 'params_s,stackTrace_s' 23 | 24 | }; 25 | }; 26 | -------------------------------------------------------------------------------- /endpoints/pools/pools_request.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | 3 | /** 4 | * createSolrPoolRequest 5 | * @param req 6 | * @param filter 7 | * @returns {{q: string, rows: number, sort: string}} 8 | */ 9 | createSolrPoolRequest: function(req, filter) { 10 | var pool = req.params.pool; 11 | var core = req.params.core; 12 | var port = req.params.port; 13 | var start = req.params.start; 14 | var end = req.params.end; 15 | var solrQueryInformation = { 16 | 'q': 'masterDocumentMin_b:true AND pool_s:' + pool + ' AND coreName_s:' + core + ' AND port_i:' + port + ' AND masterTime_dt:[' + start +' TO ' + end +'] ', 17 | 'rows': 10000, 18 | 'sort': 'masterTime_dt asc' 19 | }; 20 | solrQueryInformation.fl = filter +',masterTime_dt,hostname_s' ; 21 | return solrQueryInformation; 22 | } 23 | }; -------------------------------------------------------------------------------- /endpoints/servers/servers_request.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | /** 3 | * createSolrServerRequest 4 | * @param req 5 | * @param filter 6 | * @returns {{q: string, rows: number, sort: string}} 7 | */ 8 | createSolrServerRequest: function(req, filter) { 9 | var server = req.params.server; 10 | var core = req.params.core; 11 | var port = req.params.port; 12 | var start = req.params.start; 13 | var end = req.params.end; 14 | var solrQueryInformation = { 15 | 'q': 'masterDocumentMin_b:true AND hostname_s:' + server + ' AND coreName_s:' + core + ' AND port_i:' + port + ' AND masterTime_dt:[' + start +' TO ' + end +'] ', 16 | 'rows': 10000, 17 | 'sort': 'masterTime_dt asc' 18 | }; 19 | solrQueryInformation.fl = filter +',masterTime_dt'; 20 | return solrQueryInformation; 21 | } 22 | }; -------------------------------------------------------------------------------- /helpers/http_request.js: -------------------------------------------------------------------------------- 1 | var querystring = require('querystring'); 2 | 3 | // Thoth configuration 4 | var thoth_hostname = process.env.THOTH_HOST; 5 | var thoth_port = process.env.THOTH_PORT; 6 | 7 | module.exports = { 8 | 9 | /** 10 | * prepareHttpRequest 11 | * @param solrOptions 12 | * @param requestPath 13 | * @returns {{}} 14 | */ 15 | prepareHttpRequest: function (solrOptions, requestPath) { 16 | 17 | var requestOptions = {}; 18 | 19 | requestOptions.host = thoth_hostname; 20 | requestOptions.port = thoth_port; 21 | requestOptions.path = requestPath; 22 | 23 | var solrQueryOptions = {}; 24 | solrQueryOptions.wt = 'json'; 25 | solrQueryOptions.omitHeader = true; 26 | 27 | for (var key in solrOptions) { 28 | if (solrOptions.hasOwnProperty(key)) { 29 | var value = solrOptions[key]; 30 | if (value != null ) { 31 | solrQueryOptions[key] = value ; 32 | } 33 | } 34 | } 35 | 36 | requestOptions.path += querystring.stringify(solrQueryOptions); 37 | return requestOptions; 38 | 39 | } 40 | }; -------------------------------------------------------------------------------- /dispatchers/list_dispatcher.js: -------------------------------------------------------------------------------- 1 | var thothFieldsMappings = require('./../fields_mapping')(); 2 | 3 | var http = require('http'); 4 | var httpRequestHelper = require('./../helpers/http_request'); 5 | var listRequest = require('./../endpoints/list/list_request'); 6 | var listResponse = require('./../endpoints/list/list_response'); 7 | 8 | module.exports = { 9 | 10 | /** 11 | * dispatchList 12 | * @param req 13 | * @param res 14 | */ 15 | dispatchList: function (req, res) { 16 | var attribute = req.params.attribute; 17 | var filter = ''; 18 | 19 | if (attribute === 'servers') filter = 'hostname_s'; 20 | if (attribute === 'pools') filter = 'pool_s'; 21 | if (attribute === 'cores') filter = 'coreName_s'; 22 | if (attribute === 'ports') filter = 'port_i'; 23 | 24 | http.get(httpRequestHelper.prepareHttpRequest(listRequest.createListRequest(req, filter), '/solr/shrank/select?'), function(resp) { 25 | listResponse.listJsonResponse(res, resp, filter); 26 | }).on('error', function(e) { 27 | console.log(e); 28 | }); 29 | } 30 | }; 31 | -------------------------------------------------------------------------------- /endpoints/servers/list_request.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | 3 | /** 4 | * createListInfoServerRequest 5 | * @param req 6 | * @param filter 7 | * @returns {{q: string, rows: number, sort: *, start: number}} 8 | */ 9 | createListInfoServerRequest: function(req, filter) { 10 | var server = req.params.server; 11 | var core = req.params.core; 12 | var port = req.params.port; 13 | var start = req.params.start; 14 | var end = req.params.end; 15 | var page = req.params.page; 16 | var attribute = req.params.attribute; 17 | 18 | var listType; 19 | var sortParam; 20 | if (attribute === 'slowqueries'){ 21 | listType = 'slowQueryDocument_b'; 22 | sortParam = 'qtime_i desc'; 23 | } else{ 24 | listType = 'exception_b'; 25 | sortParam = 'timestamp_dt desc'; 26 | } 27 | 28 | var solrQueryInformation = { 29 | 30 | 'q': listType + ':true AND hostname_s:' + server + ' AND coreName_s:' + core + ' AND port_i:' + port + ' AND date_dt:[' + start +' TO ' + end +']', 31 | 'rows': 12, 32 | 'sort': sortParam, 33 | 'start' : page*12 34 | }; 35 | solrQueryInformation.fl = filter +',date_dt' ; 36 | return solrQueryInformation; 37 | } 38 | }; -------------------------------------------------------------------------------- /endpoints/list/list_response.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | 3 | /** 4 | * listJsonResponse 5 | * @param backendResp 6 | * @param resp 7 | * @param attribute 8 | */ 9 | listJsonResponse: function(backendResp, resp, attribute) { 10 | try{ 11 | var data =''; 12 | var blob = ''; 13 | // Fetch data 14 | resp.on('data', function(chunk){ 15 | data += chunk; 16 | }); 17 | // Parse and fix data 18 | resp.on('end', function(){ 19 | try { 20 | // Get only the docs 21 | var json = JSON.parse(data).facet_counts; 22 | var facet_fields = json.facet_fields[attribute]; 23 | var list = []; 24 | for (var i = 0; i < facet_fields.length; i++){ 25 | if (i % 2 === 0) list.push(facet_fields[i]); 26 | } 27 | // Avoid CORS http://en.wikipedia.org/wiki/Cross-origin_resource_sharing 28 | backendResp.header('Access-Control-Allow-Origin', '*'); 29 | // backendResp.header('Access-Control-Allow-Headers', 'X-Requested-With'); 30 | 31 | backendResp.json( 32 | {'numFound' : list.length, 'list' : list } 33 | ); 34 | } catch(err){ 35 | backendResp.status(404).send('Data not found. Most probably wrong query was sent to the thoth index' + err); 36 | } 37 | }); 38 | } 39 | catch(err){ 40 | backendResp.status(503).send('Thoth index not available' + err); 41 | } 42 | } 43 | }; -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014 Trulia, Inc. 2 | 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without modification, are permitted (subject to the limitations in the disclaimer below) provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 8 | 9 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 10 | 11 | * Neither the name of Trulia, Inc. nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 12 | 13 | NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE GRANTED BY THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- /dispatchers/pools_dispatcher.js: -------------------------------------------------------------------------------- 1 | var thothFieldsMappings = require('./../fields_mapping')(); 2 | 3 | var http = require('http'); 4 | var httpRequestHelper = require('./../helpers/http_request'); 5 | var poolsRequest = require('./../endpoints/pools/pools_request'); 6 | var poolsResponse = require('./../endpoints/pools/pools_response'); 7 | 8 | module.exports = { 9 | 10 | /** 11 | * dispatchPool 12 | * @param req 13 | * @param res 14 | */ 15 | dispatchPool: function (req, res) { 16 | var self = this; 17 | var information = req.params.information; 18 | var attribute = req.params.attribute; 19 | var filter, jsonFieldWithValue; 20 | var integral = false; 21 | 22 | if (information === 'avg') { 23 | filter = thothFieldsMappings.avg[attribute]; 24 | jsonFieldWithValue = [thothFieldsMappings.avg[attribute]]; 25 | } 26 | 27 | if (information === 'count') { 28 | filter = thothFieldsMappings.count[attribute] + ',tot-count_i'; 29 | jsonFieldWithValue = [thothFieldsMappings.count[attribute],'tot-count_i']; 30 | } 31 | 32 | if (information === 'integral') { 33 | filter = thothFieldsMappings.integral[attribute]; 34 | jsonFieldWithValue = [thothFieldsMappings.integral[attribute]]; 35 | integral = true; 36 | } 37 | 38 | http.get(httpRequestHelper.prepareHttpRequest(poolsRequest.createSolrPoolRequest(req, filter), '/solr/shrank/select?'), function (resp) { 39 | poolsResponse.poolJsonResponse(res, resp, jsonFieldWithValue, integral); 40 | }).on('error', function(e) { 41 | console.log('error while trying to dispatch pool request', e); 42 | }); 43 | } 44 | }; 45 | -------------------------------------------------------------------------------- /README.mkd: -------------------------------------------------------------------------------- 1 |    Thoth API 2 | ====================== 3 |

4 | 5 | What is Thoth ? 6 | ===================== 7 | **Thoth** is a real-time solr monitor and search analysis engine. It's a set of tools that can help you collect, visualize and leverage data coming from your solr search infrastructure. 8 | Want to learn more? [See the Wiki](https://github.com/trulia/thoth/wiki) 9 | 10 | Thoth API 11 | ====================== 12 | The **Thoth API** module provides an abstraction for data contained in the Thoth Index. 13 |
14 | For instructions on how to use this module, or a list of the other modules [See the Wiki](https://github.com/trulia/thoth-api/wiki) 15 | 16 | Installation 17 | ====================== 18 | 19 | Make sure you have [node.js](http://nodejs.org) installed. 20 | 21 | ``` 22 | git clone git@github.com:trulia/thoth-api.git 23 | cd thoth-api 24 | npm install 25 | npm start // starts api web server 26 | ``` 27 | 28 | 29 | 30 | Contributing 31 | ======================= 32 | 1. Fork it 33 | 2. Create your feature/bug fix branch (git checkout -b my-new-feature) 34 | 3. Commit your changes (git commit -am 'Add some feature') 35 | 4. Push to the branch (git push origin my-new-feature) 36 | 5. Create new pull request 37 | 38 | Thoth Monitor - Contributors 39 | ======================= 40 | - [Damiano Braga](https://github.com/dbraga) 41 | - [Giulio Grillanda](https://github.com/ingiulio) 42 | - [JD Cantrell](https://github.com/jdcantrell) 43 | - [Derek Reynolds](https://github.com/derekr) 44 | 45 | License 46 | ============= 47 | Copyright (c) 2014 Trulia, Inc. See the LICENSE file for license rights and limitations. 48 | 49 | Powered by 50 | ============= 51 | 52 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Rest api for thoth 3 | * 4 | * @author Damiano Braga 5 | */ 6 | 7 | require('./environment')(); 8 | 9 | var app = require('express')(); 10 | var http = require('http').Server(app); 11 | var io = require('socket.io')(http); 12 | var util = require('util'); 13 | 14 | // Dispatchers 15 | var serversDispatcher = require('./dispatchers/servers_dispatcher'); 16 | var poolsDispatcher = require('./dispatchers/pools_dispatcher'); 17 | var listDispatcher = require('./dispatchers/list_dispatcher'); 18 | 19 | // Setup websocket for real time data flow 20 | io.on('connection', function(socket){ 21 | socket.on('queryParams', function(queryParams) { 22 | 23 | module.exports.io = io; 24 | var realtimeDispatcher = require('./dispatchers/realtime_dispatcher'); 25 | realtimeDispatcher.pollRealTimeData(queryParams); 26 | }); 27 | }); 28 | 29 | // Start node server 30 | http.listen(process.env.PORT); 31 | console.info('Server started at http://localhost:' + process.env.PORT); 32 | 33 | /** 34 | * Routes available 35 | */ 36 | var routeBase = [ 37 | '/api/%s/%s', 38 | '/core/:core', 39 | '/port/:port', 40 | '/start/:start', 41 | '/end/:end', 42 | '/:information', 43 | '/:attribute' 44 | ].join(''); 45 | 46 | /** 47 | * Servers 48 | */ 49 | var serverRoute = util.format(routeBase, 'server', ':server'); 50 | app.get(serverRoute, function (req, res) { 51 | serversDispatcher.dispatchServer(req, res) 52 | }); 53 | 54 | app.get(serverRoute + '/:page', function (req, res) { 55 | serversDispatcher.dispatchServer(req, res) 56 | }); 57 | 58 | /** 59 | * Pools 60 | */ 61 | app.get(util.format(routeBase, 'pool', ':pool'), function (req, res) { 62 | poolsDispatcher.dispatchPool(req, res) 63 | }); 64 | 65 | /** 66 | * Info 67 | */ 68 | app.get('/api/list/:attribute', function (req, res) { 69 | listDispatcher.dispatchList(req, res); 70 | }); 71 | -------------------------------------------------------------------------------- /endpoints/servers/list_response.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | 3 | /** 4 | * prepareListInfoJsonResponse 5 | * @param backendResp 6 | * @param resp 7 | * @param attribute 8 | * @param integral 9 | */ 10 | prepareListInfoJsonResponse: function(backendResp, resp, attribute, integral){ 11 | try{ 12 | var data ='', 13 | blob = ''; 14 | // Fetch data 15 | resp.on('data', function(chunk){ 16 | data += chunk; 17 | }); 18 | // Parse and fix data 19 | resp.on('end', function(){ 20 | try { 21 | // // Get only the docs 22 | var json = JSON.parse(data).response; 23 | var docs = json.docs; 24 | var numFound = json.numFound; 25 | blob = docs; 26 | // console.log(blob); 27 | for (var i=0; i