├── .eslintignore
├── public
    ├── font
    │   ├── icons.eot
    │   ├── icons.ttf
    │   ├── icons.woff
    │   ├── icons.woff2
    │   └── icons.svg
    ├── search.js
    ├── icons.css
    └── styles.css
├── config
    ├── index.js
    └── config.development.js
├── .editorconfig
├── CHANGELOG.md
├── views
    ├── error.pug
    ├── profile.pug
    ├── navigation.pug
    ├── result.pug
    ├── layout.pug
    ├── login.pug
    ├── help.pug
    └── search.pug
├── .eslintrc
├── .gitignore
├── lib
    ├── checkLogin.js
    ├── checkToken.js
    ├── queryElastic.js
    └── findUser.js
├── package.json
├── LICENSE
├── bin
    └── www
├── routes
    └── index.js
├── app.js
└── README.md


/.eslintignore:
--------------------------------------------------------------------------------
1 | dist/
2 | 


--------------------------------------------------------------------------------
/public/font/icons.eot:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/br-data/elasticsearch-frontend/HEAD/public/font/icons.eot


--------------------------------------------------------------------------------
/public/font/icons.ttf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/br-data/elasticsearch-frontend/HEAD/public/font/icons.ttf


--------------------------------------------------------------------------------
/public/font/icons.woff:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/br-data/elasticsearch-frontend/HEAD/public/font/icons.woff


--------------------------------------------------------------------------------
/public/font/icons.woff2:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/br-data/elasticsearch-frontend/HEAD/public/font/icons.woff2


--------------------------------------------------------------------------------
/config/index.js:
--------------------------------------------------------------------------------
1 | const env = process.env.NODE_ENV || 'development';
2 | const config = require(`./config.${env}`);
3 | 
4 | module.exports = config;
5 | 


--------------------------------------------------------------------------------
/.editorconfig:
--------------------------------------------------------------------------------
 1 | [*]
 2 | indent_style = space
 3 | indent_size = 2
 4 | end_of_line = lf
 5 | charset = utf-8
 6 | trim_trailing_whitespace = true
 7 | insert_final_newline = true
 8 | 
 9 | [*.md]
10 | trim_trailing_whitespace = false
11 | 


--------------------------------------------------------------------------------
/CHANGELOG.md:
--------------------------------------------------------------------------------
 1 | # Changelog
 2 | 
 3 | ### 1.1
 4 | - Add simple REST API for search
 5 | - Add authentication for API endpoints
 6 | - Add user profile page
 7 | 
 8 | ### 1.0
 9 | - Add basic search interface
10 | - Add login and session-based authentication
11 | - Add help page and setup instructions
12 | - Add simple user management
13 | - Add configuration
14 | 


--------------------------------------------------------------------------------
/views/error.pug:
--------------------------------------------------------------------------------
 1 | extends layout
 2 | 
 3 | block content
 4 | 
 5 |   include navigation
 6 | 
 7 |   section.error.white
 8 | 
 9 |     .center
10 | 
11 |       h1=title
12 | 
13 |       p Sorry for the inconvenience. Return to the <a href="/">homepage.</a>
14 | 
15 |       if error
16 |         p.fail
17 |           strong=error.message
18 | 
19 |         pre=message.stack
20 | 
21 | 


--------------------------------------------------------------------------------
/views/profile.pug:
--------------------------------------------------------------------------------
 1 | extends layout
 2 | 
 3 | block content
 4 | 
 5 |   include navigation
 6 | 
 7 |   section.profile.white
 8 | 
 9 |     .center
10 | 
11 |       h1 User profile
12 | 
13 |       p Hello #{user.screenname}
14 | 
15 |       p Your API token:
16 |       pre=user.apiToken
17 | 
18 |       if error
19 |         p.fail
20 |           strong=error.message
21 | 
22 |         pre=message.stack
23 | 


--------------------------------------------------------------------------------
/public/search.js:
--------------------------------------------------------------------------------
 1 | document.addEventListener('DOMContentLoaded', changeIcon, false);
 2 | 
 3 | function changeIcon() {
 4 |   var $loadingIcon = document.querySelector('.loading');
 5 |   var $submitButton = document.querySelector('.submit');
 6 | 
 7 |   if ($loadingIcon) {
 8 |     $submitButton.addEventListener('click', function () {
 9 |       $loadingIcon.className = 'icon-spinner';
10 |     }, false);
11 |   }
12 | }
13 | 


--------------------------------------------------------------------------------
/.eslintrc:
--------------------------------------------------------------------------------
 1 | {
 2 |   extends: 'eslint:recommended',
 3 |   parser: 'babel-eslint',
 4 |   'rules': {
 5 |     'indent': [2, 2, { 'SwitchCase': 1 }],
 6 |     'quotes': [2, 'single'],
 7 |     'linebreak-style': [2, 'unix'],
 8 |     'semi': [2, 'always'],
 9 |     'comma-dangle': [1, 'never'],
10 |     'no-trailing-spaces': [2, { 'skipBlankLines': true }],
11 |     'no-console': 0
12 |   },
13 |   'env': {
14 |     'node': true
15 |   }
16 | }
17 | 


--------------------------------------------------------------------------------
/views/navigation.pug:
--------------------------------------------------------------------------------
 1 | nav.navigation
 2 | 
 3 |   span
 4 |     a(href='/')
 5 |       i(class='icon-search')
 6 |       | Search
 7 |   span
 8 |     a(href='#')
 9 |       i(class='icon-upload-cloud')
10 |       | Import
11 |   span
12 |     a(href='/help')
13 |       i(class='icon-help')
14 |       | Help
15 | 
16 |   if user
17 |     span.logout
18 |       a(href='/logout')
19 |         i(class='icon-logout')
20 |         | Logout
21 |     span.profile
22 |       a(href='/profile')
23 |         i(class='icon-user')
24 |         | Profile
25 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
 1 | ### Project ###
 2 | pdf/
 3 | text/
 4 | contratos-text/
 5 | contratos-pdf/
 6 | 
 7 | ### Node ###
 8 | node_modules
 9 | logs
10 | *.log
11 | pids
12 | *.pid
13 | *.seed
14 | .grunt
15 | .tmp
16 | build/Release
17 | .lock-wscript
18 | 
19 | ### Windows ###
20 | Thumbs.db
21 | ehthumbs.db
22 | Desktop.ini
23 | $RECYCLE.BIN/
24 | 
25 | ### OSX ###
26 | .DS_Store
27 | .AppleDouble
28 | .LSOverride
29 | Icon
30 | ._*
31 | .Spotlight-V100
32 | .Trashes
33 | 
34 | ### Project Files ###
35 | .idea/
36 | .idea_modules/
37 | atlassian-ide-plugin.xml
38 | com_crashlytics_export_strings.xml
39 | *.sublime-workspace
40 | 


--------------------------------------------------------------------------------
/lib/checkLogin.js:
--------------------------------------------------------------------------------
 1 | // Checks if a user is authenticated
 2 | // @TODO Preserve query string on redirect
 3 | function checkLogin(options) {
 4 |   options = options || {};
 5 | 
 6 |   const url = options.redirectTo || '/login';
 7 |   const setReturnTo = (options.setReturnTo === undefined) ? true : options.setReturnTo;
 8 | 
 9 |   return (req, res, next) => {
10 |     if (!req.isAuthenticated || !req.isAuthenticated()) {
11 |       if (setReturnTo && req.session) {
12 |         req.session.returnTo = req.originalUrl || req.url;
13 |       }
14 |       res.redirect(url);
15 |     }
16 |     next();
17 |   };
18 | }
19 | 
20 | module.exports = checkLogin;
21 | 


--------------------------------------------------------------------------------
/views/result.pug:
--------------------------------------------------------------------------------
 1 | extends search
 2 | 
 3 | block result
 4 | 
 5 |   if error
 6 |     p.fail
 7 |       strong=error.message
 8 | 
 9 |   else
10 |     .count
11 |       p Found #{result.hits.total} documents (#{result.took} ms)
12 | 
13 |     each doc in result.hits.hits
14 |       .document
15 |         .download
16 |           a.pdf(href=doc._source.file target='blank')
17 |             i(class='icon-file-pdf')
18 |             span PDF
19 |           a.text(href=doc._source.file target='blank')
20 |             i(class='icon-doc-text')
21 |             span Text
22 |         .header.doc
23 |           span.title #{doc._source.name}
24 |         each hit in doc.highlight.body
25 |           p.hit!=hit
26 | 


--------------------------------------------------------------------------------
/lib/checkToken.js:
--------------------------------------------------------------------------------
 1 | // Checks if a token is valid
 2 | const passport = require('passport');
 3 | 
 4 | function checkToken() {
 5 |   return (req, res, next) => {
 6 |     return passport.authenticate('bearer', {
 7 |       session: false
 8 |     }, (error, user) => {
 9 |       if (!user) {
10 |         if (error) {
11 |           res.status(401);
12 |           res.json({ error: error.message });
13 |           res.end();
14 |         } else {
15 |           res.status(401);
16 |           res.json({ error: 'Please provide an API token' });
17 |           res.end();
18 |         }
19 |       } else {
20 |         next();
21 |       }
22 |     })(req, res, next);
23 |   };
24 | }
25 | 
26 | module.exports = checkToken;
27 | 


--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "name": "elasticsearch-frontend",
 3 |   "description": "Search and explore document leaks with Elasticsearch",
 4 |   "author": "Steffen Kühne",
 5 |   "version": "1.1.0",
 6 |   "license": "MIT",
 7 |   "private": true,
 8 |   "repository": {
 9 |     "type": "git",
10 |     "url": "git@github.com:br-data/elasticsearch-frontend.git"
11 |   },
12 |   "scripts": {
13 |     "start": "node ./bin/www"
14 |   },
15 |   "dependencies": {
16 |     "bcrypt": "^5.0.0",
17 |     "body-parser": "^1.19.0",
18 |     "connect-ensure-login": "^0.1.1",
19 |     "cookie-parser": "^1.4.4",
20 |     "debug": "^3.2.6",
21 |     "elasticsearch": "^15.5.0",
22 |     "express": "^4.17.1",
23 |     "express-session": "^1.16.2",
24 |     "node-dir": "^0.1.17",
25 |     "passport": "^0.4.0",
26 |     "passport-http-bearer": "^1.0.1",
27 |     "passport-local": "^1.0.0",
28 |     "pug": "^3.0.1",
29 |     "req-flash": "0.0.3"
30 |   },
31 |   "devDependencies": {
32 |     "eslint-config-esnext": "^4.0.0"
33 |   }
34 | }
35 | 


--------------------------------------------------------------------------------
/views/layout.pug:
--------------------------------------------------------------------------------
 1 | doctype html
 2 | 
 3 | html
 4 | 
 5 |   head
 6 |     meta(charset='utf-8')
 7 |     title=page.title
 8 |     link(href='data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABgAAAAYCAYAAADgdz34AAAB8UlEQVRIS7XVS6hNcRTH8c/1KGSCARIDUkrERMljYiAUwkwSKRMyQAYIAyLMZSAZSEkIEzOvMJA8rmSglJQJ8oyQlta97bvPPvvswbmrdvuc/r//+q7Hf/13j0G2nob+RyCez/jbcM9/WR1gJrZhOaak0594hPM4gx+dYFWA4TiGHRhS4+ANNuBOHaQMGIZLWFnY9A738QVTMR8RRNgvrMGNdpAy4Cj2pPgDtuNCqe4TcRzrUxfgOXhdBSkCpuElIouPWIgXNemfwM5cj6zXdQJEVLtStAlnOzRwKB5jVmY4GVHOAVbM4AlmI0ozHr87nRBsxanUbcS5OsB3jMRNLG3gPCQRUAQWdhj76gARcaR9FasbAqbjVWpPFkrcv71YoreYlI2NIWtiK3A9hbsRjW/bg4uFkxCpP2tAiJrHsIXFqbtXB1iFKym4hSX4UwNZgNs57TEDUa6We6pYovj9APPSaWS0GV8rIItwGeNyLZobTW6x8iRHFAEZm8o416cz0pjYGMYYqLWlizKCWIa7dSXqW5uLa9nwBm3ol1RC2l3XY3AQWzCqgtKLAznF8e6zFkinD85oLMaMBL3HQzwtOI1A2kI6AZqWqAoS1/rzbgEikDJkL450ExCQ/TiET4ij3NttQEAm5NfvW/wZDMCAvv0DZEFgGQvWNqMAAAAASUVORK5CYII=' rel='icon' type='image/x-icon')
 9 | 
10 |     link(rel='stylesheet' href='/icons.css')
11 |     link(rel='stylesheet' href='/styles.css')
12 | 
13 |   body
14 |     block content
15 | 


--------------------------------------------------------------------------------
/views/login.pug:
--------------------------------------------------------------------------------
 1 | extends layout
 2 | 
 3 | block content
 4 | 
 5 |   include navigation
 6 | 
 7 |   section.login.white
 8 | 
 9 |     .center
10 | 
11 |       h1.title=page.title
12 |       p.description=page.description
13 | 
14 |       if error.message
15 |         p.fail
16 |           strong=error.message
17 |       else
18 |         p
19 |           strong Please log in to continue:
20 | 
21 |       form(action='/login' method='post')
22 | 
23 |         fieldset
24 |           span.right
25 |             i(class='icon-user')
26 |           span.left
27 |             input(
28 |               type='text'
29 |               name='username'
30 |               placeholder='Username'
31 |             )
32 | 
33 |         fieldset
34 |           span.right
35 |             i(class='icon-lock-open-alt')
36 |           span.left
37 |             input(
38 |               type='password'
39 |               name='password'
40 |               placeholder='Password'
41 |             )
42 | 
43 |         fieldset
44 |           button.submit(type='submit' value='Login') Login
45 |             i.icon-login
46 | 


--------------------------------------------------------------------------------
/lib/queryElastic.js:
--------------------------------------------------------------------------------
 1 | // Queries Elasticsearch for data using different methods
 2 | const elastic = require('elasticsearch');
 3 | const config = require('../config');
 4 | 
 5 | const elasticClient = new elastic.Client({ host: config.database.host });
 6 | 
 7 | function queryElastic() {
 8 |   return (req, res, next) => {
 9 |     if (req.query.query && req.query.type) {
10 |       elasticClient.search(buildQuery(req), (error, data) => {
11 |         req.error = error;
12 |         req.result = data;
13 |         next();
14 |       });
15 |     } else {
16 |       req.error = new Error('Query string and query type are undefined');
17 |       req.result = null;
18 |       next();
19 |     }
20 |   };
21 | }
22 | 
23 | function buildQuery(req) {
24 |   let query = config.queries[req.query.type];
25 | 
26 |   query.setQuery(req.query.query);
27 |   query = query.query;
28 | 
29 |   return {
30 |     index: config.database.index,
31 |     size: 500,
32 |     body: {
33 |       query: query,
34 |       _source: config._source,
35 |       highlight: config.highlight
36 |     }
37 |   };
38 | }
39 | 
40 | module.exports = queryElastic;
41 | 


--------------------------------------------------------------------------------
/lib/findUser.js:
--------------------------------------------------------------------------------
 1 | function findByUsername(username, database, callback) {
 2 |   process.nextTick(() => {
 3 |     const user = database.filter(user => user.username === username);
 4 | 
 5 |     if (user && user.length > 0) {
 6 |       callback(null, user[0]);
 7 |     } else {
 8 |       callback(new Error('Invalid username'));
 9 |     }
10 |   });
11 | }
12 | 
13 | function findById (id, database, callback) {
14 |   process.nextTick(() => {
15 |     const user = database.filter(user => user.id === id);
16 | 
17 |     if (user && user.length > 0) {
18 |       callback(null, user[0]);
19 |     } else {
20 |       callback(new Error('Invalid user ID'));
21 |     }
22 |   });
23 | }
24 | 
25 | function findByToken(token, database, callback) {
26 |   process.nextTick(() => {
27 |     const user = database.filter(user => user.apiToken === token);
28 | 
29 |     if (user && user.length > 0) {
30 |       callback(null, user[0]);
31 |     } else {
32 |       callback(new Error('Invalid API token'));
33 |     }
34 |   });
35 | }
36 | 
37 | module.exports =  {
38 |   byUsername: findByUsername,
39 |   byId: findById,
40 |   byToken: findByToken
41 | };
42 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2017 Bayerischer Rundfunk
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/bin/www:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env node
 2 | 
 3 | const debug = require('debug')('app');
 4 | const app = require('../app.js');
 5 | const port = process.env.PORT || 3000;
 6 | 
 7 | const server = app.listen(process.env.PORT || 3000, () => {
 8 |   debug(`Express server listening on port ${server.address().port}`);
 9 | });
10 | 
11 | function onError(err) {
12 |   if (err.syscall !== 'listen') {
13 |     throw err;
14 |   }
15 | 
16 |   const bind = typeof port === 'string' ? `Pipe ${port}` : `Port ${port}`;
17 | 
18 |   // Handle specific listen errors with friendly messages
19 |   switch (err.code) {
20 |     case 'EACCES':
21 |       console.error(`${bind} requires elevated privileges`);
22 |       process.exit(1);
23 |       break;
24 |     case 'EADDRINUSE':
25 |       console.error(`${bind} is already in use`);
26 |       process.exit(1);
27 |       break;
28 |     default:
29 |       throw err;
30 |   }
31 | }
32 | 
33 | function onListening() {
34 |   const addr = server.address();
35 |   const bind = typeof addr === 'string' ? `Pipe ${addr}` : `Port ${addr.port}`;
36 |   debug(`Listening on ${bind}`);
37 | }
38 | 
39 | server.on('error', onError);
40 | server.on('listening', onListening);
41 | 


--------------------------------------------------------------------------------
/views/help.pug:
--------------------------------------------------------------------------------
 1 | extends layout
 2 | 
 3 | block content
 4 | 
 5 |   include navigation
 6 | 
 7 |   section.help.white
 8 | 
 9 |     .center
10 | 
11 |       h1 About the search
12 |       p There are four different ways to search for whole sentences (full-text) or a single words (term):
13 | 
14 |       h3 Standard search
15 |       p Full-text search. Finds exact word combinations like <code>John Doe</code>. Diacritcs are ignored and a search for <code>John Doe</code> will also find <code>Jóhñ Döé</code>.
16 | 
17 |       h3 Custom search
18 |       p Full-text search. By default, the custom search finds all documents that contain <code>John</code> AND <code>Doe</code>. Supports wildcards and simple search operators:
19 |       ul
20 |         li <code>+</code> signifies AND operation
21 |         li <code>|</code> signifies OR operation
22 |         li <code>-</code> negates a single token
23 |         li <code>"</code> wraps a number of tokens to signify a phrase for searching
24 |         li <code>*</code> at the end of a term signifies a prefix query
25 |         li <code>~N</code> after a word signifies edit distance (fuzziness)
26 |         li <code>~N</code> after a phrase signifies slop amount
27 | 
28 |       h3 Fuzzy search
29 |       p Term-based search. Finds words, even if they contain a typo or OCR mistake. A search for <code>Jhon</code> or <code>J°hn</code> will also find <code>John</code>.
30 | 
31 |       h3 Regex search
32 |       p Term-based search for experts. Uses Regex patterns like <code>J.h*</code> for searching. This Regex will find words such as <code>John</code>, <code>Jahn</code> and <code>Johnson</code>.
33 | 


--------------------------------------------------------------------------------
/config/config.development.js:
--------------------------------------------------------------------------------
 1 | const config = {};
 2 | 
 3 | config.page = {
 4 |   title: 'Document Search',
 5 |   description: 'Search Elasticsearch documents for persons, companies and addresses.'
 6 | };
 7 | 
 8 | config.users = [
 9 |   {
10 |     id: 1,
11 |     username: 'user',
12 |     screenname: 'Demo User',
13 |     password: '$2a$10$vP0qJyEd0hvvpG5MAaHg9ObUJJpJj9HxINZ/Yqz5nPo5Ms2nhR4r.',
14 |     apiToken: '0b414d8433124406be6500833f1672e5'
15 |   }
16 | ];
17 | 
18 | config.database = {
19 |   type: 'elasticsearch',
20 |   host: 'localhost:9200',
21 |   index: 'joram'
22 | };
23 | 
24 | config.queries = {
25 |   match: {
26 |     name: 'Standard Search',
27 |     query: {
28 |       multi_match: {
29 |         query: undefined,
30 |         fields: ['body', 'body.folded'],
31 |         type: 'phrase'
32 |       }
33 |     },
34 |     setQuery: function (query) {
35 |       this.query.multi_match.query = query;
36 |     }
37 |   },
38 |   custom: {
39 |     name: 'Custom Search',
40 |     query: {
41 |       simple_query_string: {
42 |         query: undefined,
43 |         fields: ['body','body.folded'],
44 |         default_operator: 'and',
45 |         analyze_wildcard: true
46 |       }
47 |     },
48 |     setQuery: function (query) {
49 |       this.query.simple_query_string.query = query;
50 |     }
51 |   },
52 |   fuzzy: {
53 |     name: 'Fuzzy Search',
54 |     query: {
55 |       fuzzy: {
56 |         body: undefined
57 |       }
58 |     },
59 |     setQuery: function (query) {
60 |       this.query.fuzzy.body = query;
61 |     }
62 |   },
63 |   regexp: {
64 |     name: 'Regex Search',
65 |     query: {
66 |       regexp: {
67 |         body: undefined
68 |       }
69 |     },
70 |     setQuery: function (query) {
71 |       this.query.regexp.body = query;
72 |     }
73 |   }
74 | };
75 | 
76 | config._source = {
77 |   excludes: ['body*']
78 | };
79 | 
80 | config.highlight = {
81 |   fields: {
82 |     body: {}
83 |     //'body.folded': {}
84 |   }
85 | };
86 | 
87 | module.exports = config;
88 | 


--------------------------------------------------------------------------------
/views/search.pug:
--------------------------------------------------------------------------------
 1 | extends layout
 2 | 
 3 | block content
 4 | 
 5 |   include navigation
 6 | 
 7 |   script(src='/search.js')
 8 | 
 9 |   header.search.white
10 | 
11 |     .center
12 | 
13 |       h1.title=page.title
14 |       p.description=page.description
15 | 
16 |       form(action='search')
17 | 
18 |         fieldset.main
19 |           span.right
20 |             button.submit(type='submit') Search
21 |               i.icon-search.loading
22 |           span.left
23 |             input(
24 |               type='search'
25 |               name='query'
26 |               placeholder='Start searching'
27 |               value=(query.query ? query.query : '')
28 |             )
29 | 
30 |         fieldset.options.type
31 |           strong Type:
32 |           div
33 |             input(
34 |               type='radio'
35 |               name='type'
36 |               value='match'
37 |               checked=(query.type === 'match' ? 'checked' : '')
38 |             )
39 |             label Standard
40 |           div
41 |             input(
42 |               type='radio'
43 |               name='type'
44 |               value='custom'
45 |               checked=(query.type === 'custom' ? 'checked' : false)
46 |             )
47 |             label Custom
48 |           div
49 |             input(
50 |               type='radio'
51 |               name='type'
52 |               value='fuzzy'
53 |               checked=(query.type === 'fuzzy' ? 'checked' : false)
54 |             )
55 |             label Fuzzy
56 |           div
57 |             input(
58 |               type='radio'
59 |               name='type'
60 |               value='regexp'
61 |               checked=(query.type === 'regexp' ? 'checked' : false)
62 |             )
63 |             label Regex
64 | 
65 |         fieldset.options.sort
66 |           strong Sort:
67 |           div
68 |             input(
69 |               type='radio'
70 |               name='sorting'
71 |               value='date'
72 |               checked=(query.sorting === 'date' ? 'checked' : '')
73 |             )
74 |             label Date
75 |           div
76 |             input(
77 |               type='radio'
78 |               name='sorting'
79 |               value='relevance'
80 |               checked=(query.sorting === 'relevance' ? 'checked' : false)
81 |             )
82 |             label Relevance
83 | 
84 |   section.result.center
85 |     block result
86 | 


--------------------------------------------------------------------------------
/public/icons.css:
--------------------------------------------------------------------------------
 1 | @font-face {
 2 |   font-family: 'icons';
 3 |   src: url('./font/icons.eot?43890422');
 4 |   src: url('./font/icons.eot?43890422#iefix') format('embedded-opentype'),
 5 |        url('./font/icons.woff2?43890422') format('woff2'),
 6 |        url('./font/icons.woff?43890422') format('woff'),
 7 |        url('./font/icons.ttf?43890422') format('truetype'),
 8 |        url('./font/icons.svg?43890422#icons') format('svg');
 9 |   font-weight: normal;
10 |   font-style: normal;
11 | }
12 | 
13 | [class^="icon-"]:before,
14 | [class*=" icon-"]:before {
15 |   font-family: "icons";
16 |   font-style: normal;
17 |   font-weight: normal;
18 |   speak: none;
19 |   display: inline-block;
20 |   text-decoration: inherit;
21 |   width: 1em;
22 |   margin-right: .2em;
23 |   text-align: center;
24 |   font-variant: normal;
25 |   text-transform: none;
26 |   line-height: 1em;
27 |   margin-left: .2em;
28 |   -webkit-font-smoothing: antialiased;
29 |   -moz-osx-font-smoothing: grayscale;
30 | }
31 | 
32 | .icon-logout:before { content: '\e800'; } /* '' */
33 | .icon-login:before { content: '\e801'; } /* '' */
34 | .icon-search:before { content: '\e802'; } /* '' */
35 | .icon-user:before { content: '\e804'; } /* '' */
36 | .icon-lock:before { content: '\e803'; } /* '' */
37 | .icon-github-circled:before { content: '\f09b'; } /* '' */
38 | .icon-upload-cloud:before { content: '\f0ee'; } /* '' */
39 | .icon-doc-text:before { content: '\f0f6'; } /* '' */
40 | .icon-spinner:before { content: '\f110'; } /* '' */
41 | .icon-help:before { content: '\f128'; } /* '' */
42 | .icon-lock-open-alt:before { content: '\f13e'; } /* '' */
43 | .icon-doc-inv:before { content: '\f15b'; } /* '' */
44 | .icon-doc-text-inv:before { content: '\f15c'; } /* '' */
45 | .icon-sort-alt-down:before { content: '\f161'; } /* '' */
46 | .icon-file-pdf:before { content: '\f1c1'; } /* '' */
47 | 
48 | .icon-spinner {
49 |   display: inline-block;
50 |   webkit-animation: spin 1s infinite steps(8);
51 |   animation: spin 1s infinite steps(8);
52 | }
53 | 
54 | @-webkit-keyframes spin {
55 |   0% {
56 |     -webkit-transform: rotate(0deg);
57 |     transform: rotate(0deg);
58 |   }
59 |   100% {
60 |     -webkit-transform: rotate(359deg);
61 |     transform: rotate(359deg);
62 |   }
63 | }
64 | 
65 | @keyframes spin {
66 |   0% {
67 |     -webkit-transform: rotate(0deg);
68 |     transform: rotate(0deg);
69 |   }
70 |   100% {
71 |     -webkit-transform: rotate(359deg);
72 |     transform: rotate(359deg);
73 |   }
74 | }
75 | 


--------------------------------------------------------------------------------
/routes/index.js:
--------------------------------------------------------------------------------
  1 | const express = require('express');
  2 | const router = express.Router();
  3 | const passport = require('passport');
  4 | 
  5 | const checkLogin = require('../lib/checkLogin');
  6 | const checkToken = require('../lib/checkToken');
  7 | const queryElastic = require('../lib/queryElastic');
  8 | 
  9 | // Define routes
 10 | router.get('/',
 11 |   checkLogin({ redirectTo: 'login' }),
 12 |   (req, res) => {
 13 |     res.render('search', {
 14 |       error: req.error,
 15 |       result: req.result,
 16 |       query: req.query,
 17 |       user: req.user
 18 |     });
 19 |   }
 20 | );
 21 | 
 22 | router.get('/login',
 23 |   (req, res) => {
 24 |     res.render('login', {
 25 |       error: { message: req.flash('error') }
 26 |     });
 27 |   }
 28 | );
 29 | 
 30 | router.post('/login',
 31 |   passport.authenticate('local', {
 32 |     failureRedirect: '/login',
 33 |     failureFlash: true
 34 |   }),
 35 |   (req, res) => {
 36 |     res.redirect('/');
 37 |   }
 38 | );
 39 | 
 40 | router.get('/logout',
 41 |   checkLogin({ redirectTo: 'login' }),
 42 |   (req, res) => {
 43 |     req.logout();
 44 |     res.redirect('/');
 45 |   }
 46 | );
 47 | 
 48 | router.get('/profile',
 49 |   checkLogin({ redirectTo: 'login' }),
 50 |   (req, res) => {
 51 |     res.render('profile', {
 52 |       error: req.error,
 53 |       result: req.result,
 54 |       query: req.query,
 55 |       user: req.user
 56 |     });
 57 |   }
 58 | );
 59 | 
 60 | router.get('/search',
 61 |   checkLogin({ redirectTo: 'login' }),
 62 |   queryElastic(),
 63 |   (req, res) => {
 64 |     res.render('result', {
 65 |       error: req.error,
 66 |       result: req.result,
 67 |       query: req.query,
 68 |       user: req.user
 69 |     });
 70 |   }
 71 | );
 72 | 
 73 | router.get('/help',
 74 |   checkLogin({ redirectTo: 'login' }),
 75 |   (req, res) => {
 76 |     res.render('help', {
 77 |       error: req.error,
 78 |       result: req.result,
 79 |       query: req.query,
 80 |       user: req.user
 81 |     });
 82 |   }
 83 | );
 84 | 
 85 | router.get('/api',
 86 |   passport.authenticate('bearer', {
 87 |     session: false
 88 |   }),
 89 |   (req, res) => {
 90 |     res.json({
 91 |       'message': 'API is up and running',
 92 |       'username': req.user.username
 93 |     });
 94 |   }
 95 | );
 96 | 
 97 | router.get('/api/search',
 98 |   checkToken(),
 99 |   queryElastic(),
100 |   (req, res) => {
101 |     res.json({
102 |       error: req.error,
103 |       result: req.result
104 |     });
105 |   }
106 | );
107 | 
108 | module.exports = router;
109 | 


--------------------------------------------------------------------------------
/app.js:
--------------------------------------------------------------------------------
  1 | const path = require('path');
  2 | 
  3 | const express = require('express');
  4 | const session = require('express-session');
  5 | const app = express();
  6 | 
  7 | const bcrypt = require('bcrypt');
  8 | 
  9 | const passport = require('passport');
 10 | const LocalStrategy = require('passport-local').Strategy;
 11 | const BearerStrategy = require('passport-http-bearer').Strategy;
 12 | 
 13 | const flash = require('req-flash');
 14 | const cookieParser = require('cookie-parser');
 15 | const bodyParser = require('body-parser');
 16 | 
 17 | const routes = require('./routes');
 18 | const findUser = require('./lib/findUser');
 19 | 
 20 | const config = require('./config');
 21 | 
 22 | // Copy page config to global
 23 | app.locals.page = config.page;
 24 | 
 25 | // Set the authentication strategy for the web interface
 26 | passport.use(new LocalStrategy(
 27 |   (username, password, callback) => {
 28 |     findUser.byUsername(username, config.users, (error, user) => {
 29 |       if (error) { return callback(null, false, error); }
 30 |       if (user) {
 31 |         // Check if the passwort matches the salted hash
 32 |         bcrypt.compare(password, user.password, (passwordError, isValid) => {
 33 |           if (passwordError) { return callback(passwordError); }
 34 |           if (isValid) { return callback(null, user); }
 35 |           return callback(null, false, new Error('Wrong password'));
 36 |         });
 37 |       }
 38 |     });
 39 |   }
 40 | ));
 41 | 
 42 | // Set the authentication strategy for API endpoints
 43 | passport.use(new BearerStrategy(
 44 |   (token, callback) => {
 45 |     findUser.byToken(token, config.users, (error, user) => {
 46 |       if (error) { return callback(error, false, error); }
 47 |       if (user) { return callback(null, user, { scope: 'all' }); }
 48 |     });
 49 |   }
 50 | ));
 51 | 
 52 | passport.serializeUser((user, callback) => {
 53 |   callback(null, user.id);
 54 | });
 55 | 
 56 | passport.deserializeUser((id, callback) => {
 57 |   findUser.byId(id, config.users, (error, user) => {
 58 |     if (error) { return callback(error); }
 59 |     callback(null, user);
 60 |   });
 61 | });
 62 | 
 63 | // Configure view engine to render pug templates.
 64 | app.set('views', path.join(__dirname, 'views'));
 65 | app.set('view engine', 'pug');
 66 | 
 67 | // Use application-level middleware for common functionality, including
 68 | // logging, parsing, and session handling.
 69 | app.use(cookieParser());
 70 | app.use(bodyParser.urlencoded({ extended: true }));
 71 | app.use(session({
 72 |   secret: 'keyboard cat',
 73 |   resave: false,
 74 |   saveUninitialized: false
 75 | }));
 76 | app.use(flash());
 77 | app.use(express.static(path.join(__dirname, 'public')));
 78 | 
 79 | // Initialize Passport and restore authentication state, if any, from the session.
 80 | app.use(passport.initialize());
 81 | app.use(passport.session());
 82 | 
 83 | // Connect routes
 84 | app.use('/', routes);
 85 | 
 86 | // Handle 404 errors
 87 | app.use((req, res) => {
 88 |   res.status(404);
 89 |   res.render('error', {
 90 |     title: 'Page Not Found (404)',
 91 |     url: req.url
 92 |   });
 93 | });
 94 | 
 95 | // Handle 500 internal server errors
 96 | app.use((error, req, res) => {
 97 |   res.status(500);
 98 |   res.render('error', {
 99 |     title: 'Internal Server Error (500)',
100 |     error: error,
101 |     url: req.url
102 |   });
103 | });
104 | 
105 | module.exports = app;
106 | 


--------------------------------------------------------------------------------
/public/styles.css:
--------------------------------------------------------------------------------
  1 | html {
  2 |   box-sizing: border-box;
  3 | }
  4 | 
  5 | *,
  6 | *:before,
  7 | *:after {
  8 |   box-sizing: inherit;
  9 | }
 10 | 
 11 | body {
 12 |   font-family: Helvetica, Arial, sans-serif;
 13 |   margin: 0;
 14 |   background: #f0f0f4;
 15 | }
 16 | 
 17 | /*
 18 | Global font style
 19 | */
 20 | 
 21 | h1 {
 22 |   margin-top: 0;
 23 |   font-size: 1.6em;
 24 | }
 25 | 
 26 | h2 {
 27 |   margin-top: 0;
 28 |   font-size: 1.3em;
 29 | }
 30 | 
 31 | h3 {
 32 |   margin: 1.5em 0 0.5em;
 33 |   font-size: 1em;
 34 | }
 35 | 
 36 | a {
 37 |   color: inherit;
 38 | }
 39 | 
 40 | code, pre {
 41 |   color: #b22222;
 42 |   font-size: 1.2em;
 43 | }
 44 | 
 45 | .fail {
 46 |   color: #b22222;
 47 | }
 48 | 
 49 | .warning {
 50 |   color: #ffa500;
 51 | }
 52 | 
 53 | .description {
 54 |   margin-top: -.5em;
 55 | }
 56 | 
 57 | /*
 58 | Global input element style
 59 | */
 60 | 
 61 | fieldset {
 62 |   border: 0;
 63 |   padding: 0;
 64 |   margin: .5em 0;
 65 |   min-width: 0;
 66 | }
 67 | 
 68 | input,
 69 | button,
 70 | label {
 71 |   border: 0;
 72 |   font-size: 1em;
 73 | }
 74 | 
 75 | input {
 76 |   transition: border .2s ease;
 77 | }
 78 | 
 79 | input:focus,
 80 | button:focus {
 81 |   outline: 0;
 82 | }
 83 | 
 84 | input:-webkit-autofill {
 85 |   -webkit-box-shadow: 0 0 0px 1000px #fff inset;
 86 | }
 87 | 
 88 | ::-webkit-input-placeholder {
 89 |   color: #889;
 90 | }
 91 | 
 92 | :-moz-placeholder {
 93 |   color: #889;
 94 |   opacity: 1;
 95 | }
 96 | 
 97 | ::-moz-placeholder {
 98 |   color: #889;
 99 |   opacity: 1;
100 | }
101 | 
102 | :-ms-input-placeholder {
103 |   color: #889;
104 | }
105 | 
106 | /*
107 | Specific element style
108 | */
109 | 
110 | .left {
111 |   overflow: hidden;
112 |   display: block;
113 | }
114 | 
115 | .left input {
116 |   width: 100%;
117 |   height: 2em;
118 |   line-height: 2;
119 |   padding: 0 .4em;
120 |   border: 2px solid #ccd;
121 |   border-top-left-radius: 4px;
122 |   border-bottom-left-radius: 4px;
123 |   border-right: 0;
124 | }
125 | 
126 | .right {
127 |   border-top-right-radius: 4px;
128 |   border-bottom-right-radius: 4px;
129 |   float: right;
130 | }
131 | 
132 | /*
133 | Global page layout
134 | */
135 | 
136 | .center {
137 |   width: 100%;
138 |   max-width: 840px;
139 |   margin: 0 auto;
140 |   padding: 0 2em;
141 | }
142 | 
143 | .white {
144 |   background: #fff;
145 |   padding: 2.5em 0 3em;
146 |   margin-bottom: 3em;
147 |   box-shadow: 0 1px 4px rgba(0, 0, 0, .3)
148 | }
149 | 
150 | /*
151 | Login
152 | */
153 | 
154 | .login .center {
155 |   max-width: 460px;
156 | }
157 | 
158 | .login input:focus {
159 |   border-color: #334;
160 | }
161 | 
162 | .login .right {
163 |   height: 2em;
164 |   line-height: 2em;
165 |   color: #fff;
166 |   background: #334;
167 |   text-align: center;
168 |   width: 40px;
169 | }
170 | 
171 | .login .submit {
172 |   margin-top: 1em;
173 |   width: 100%;
174 |   height: 2em;
175 |   text-align: center;
176 |   color: #fff;
177 |   background: #2e8b57;
178 |   border-radius: 4px;
179 |   cursor: pointer;
180 |   transition: background .2s ease;
181 | }
182 | 
183 | .login .submit:hover,
184 | .login .submit:focus {
185 |   background: #21653f;
186 | }
187 | 
188 | /*
189 | Navigation bar
190 | */
191 | 
192 | .navigation {
193 |   font-size: .8em;
194 |   color: #ccd;
195 |   background: #334;
196 |   padding: .5em;
197 | }
198 | 
199 | .navigation a {
200 |   text-decoration: none;
201 | }
202 | 
203 | .navigation a:hover {
204 |   color: #fff;
205 | }
206 | 
207 | .navigation .logout,
208 | .navigation .profile {
209 |   float: right;
210 | }
211 | 
212 | .navigation span {
213 |   margin: 0 .5em;
214 | }
215 | 
216 | /*
217 | Search
218 | */
219 | 
220 | .search .main {
221 |   margin: 1.5em 0 2em;
222 | }
223 | 
224 | .search .left input:focus {
225 |   border-color: #2e8b57;
226 | }
227 | 
228 | .search .submit {
229 |   height: 2em;
230 |   width: 110px;
231 |   float: right;
232 |   color: #fff;
233 |   background: #2e8b57;
234 |   border-top-right-radius: 4px;
235 |   border-bottom-right-radius: 4px;
236 |   cursor: pointer;
237 |   text-align: left;
238 |   padding: 2px 6px 3px 14px;
239 |   transition: background .2s ease;
240 | }
241 | 
242 | .search .submit:hover,
243 | .search .submit:focus {
244 |   background: #21653f;
245 | }
246 | 
247 | .search .submit i {
248 |   margin-left: 5px;
249 | }
250 | 
251 | .search .options.sort {
252 |   margin-top: -.25em;
253 | }
254 | 
255 | .search .options > div {
256 |   display: inline-block;
257 |   white-space: nowrap;
258 | }
259 | 
260 | .search .options strong {
261 |   width: 3.5em;
262 |   display: inline-block;
263 | }
264 | 
265 | .search .options label {
266 |   margin-right: .5em;
267 | }
268 | 
269 | /*
270 | Results
271 | */
272 | 
273 | .result .document {
274 |   margin: 20px 0 40px;
275 |   padding: 10px;
276 |   position: relative;
277 |   background: #fff;
278 |   box-shadow: 0 1px 4px rgba(0, 0, 0, .3);
279 | }
280 | 
281 | .result .document: before,
282 | .result .document: after {
283 |   content: "";
284 |   position: absolute;
285 |   z-index: -1;
286 |   box-shadow: 0 0 20px rgba(0, 0, 0, .8);
287 |   top: 50%;
288 |   bottom: 0;
289 |   left: 10px;
290 |   right: 10px;
291 |   border-radius: 100px / 10px;
292 | }
293 | 
294 | .result .header {
295 |   margin-top: 5px;
296 |   color: #889;
297 | }
298 | 
299 | .result .download {
300 |   float: right;
301 |   cursor: pointer;
302 | }
303 | 
304 | .result .download a {
305 |   display: inline-block;
306 |   font-size: .8em;
307 |   padding: 4px 5px;
308 |   background: #889;
309 |   color: #fff;
310 |   border-radius: 3px;
311 |   margin-right: 3px;
312 |   text-decoration: none;
313 |   transition: background .2s ease;
314 | }
315 | 
316 | .result .download a:hover {
317 |   background: #667;
318 | }
319 | 
320 | .result .hit {
321 |   font-size: .9em;
322 | }
323 | 
324 | .result .hit em {
325 |   font-style: normal;
326 |   font-weight: bold;
327 |   background: #ffff00;
328 | }
329 | 
330 | .result .count p:first-child {
331 |   font-weight: bold;
332 | }
333 | 
334 | /*
335 | Help
336 | */
337 | 
338 | .help ul {
339 |   list-style: none;
340 |   padding: 0;
341 | }
342 | 


--------------------------------------------------------------------------------
/public/font/icons.svg:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" standalone="no"?>
 2 | <!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
 3 | <svg xmlns="http://www.w3.org/2000/svg">
 4 | <metadata>Copyright (C) 2017 by original authors @ fontello.com</metadata>
 5 | <defs>
 6 | <font id="icons" horiz-adv-x="1000" >
 7 | <font-face font-family="icons" font-weight="400" font-stretch="normal" units-per-em="1000" ascent="850" descent="-150" />
 8 | <missing-glyph horiz-adv-x="1000" />
 9 | <glyph glyph-name="logout" unicode="&#xe800;" d="M357 46q0-2 1-11t0-14-2-14-5-11-12-3h-178q-67 0-114 47t-47 114v392q0 67 47 114t114 47h178q8 0 13-5t5-13q0-2 1-11t0-15-2-13-5-11-12-3h-178q-37 0-63-26t-27-64v-392q0-37 27-63t63-27h174t6 0 7-2 4-3 4-5 1-8z m518 304q0-14-11-25l-303-304q-11-10-25-10t-25 10-11 25v161h-250q-14 0-25 11t-11 25v214q0 15 11 25t25 11h250v161q0 14 11 25t25 10 25-10l303-304q11-10 11-25z" horiz-adv-x="928.6" />
10 | 
11 | <glyph glyph-name="login" unicode="&#xe801;" d="M661 350q0-14-11-25l-303-304q-11-10-26-10t-25 10-10 25v161h-250q-15 0-25 11t-11 25v214q0 15 11 25t25 11h250v161q0 14 10 25t25 10 26-10l303-304q11-10 11-25z m196 196v-392q0-67-47-114t-114-47h-178q-7 0-13 5t-5 13q0 2-1 11t0 15 2 13 5 11 12 3h178q37 0 64 27t26 63v392q0 37-26 64t-64 26h-174t-6 0-6 2-5 3-4 5-1 8q0 2-1 11t0 15 2 13 5 11 12 3h178q67 0 114-47t47-114z" horiz-adv-x="857.1" />
12 | 
13 | <glyph glyph-name="search" unicode="&#xe802;" d="M643 386q0 103-73 176t-177 74-177-74-73-176 73-177 177-73 177 73 73 177z m286-465q0-29-22-50t-50-21q-30 0-50 21l-191 191q-100-69-223-69-80 0-153 31t-125 84-84 125-31 153 31 152 84 126 125 84 153 31 153-31 125-84 84-126 31-152q0-123-69-223l191-191q21-21 21-51z" horiz-adv-x="928.6" />
14 | 
15 | <glyph glyph-name="lock" unicode="&#xe803;" d="M179 421h285v108q0 59-42 101t-101 41-101-41-41-101v-108z m464-53v-322q0-22-16-37t-38-16h-535q-23 0-38 16t-16 37v322q0 22 16 38t38 15h17v108q0 102 74 176t176 74 177-74 73-176v-108h18q23 0 38-15t16-38z" horiz-adv-x="642.9" />
16 | 
17 | <glyph glyph-name="user" unicode="&#xe804;" d="M714 69q0-60-35-104t-84-44h-476q-49 0-84 44t-35 104q0 48 5 90t17 85 33 73 52 50 76 19q73-72 174-72t175 72q42 0 75-19t52-50 33-73 18-85 4-90z m-143 495q0-88-62-151t-152-63-151 63-63 151 63 152 151 63 152-63 62-152z" horiz-adv-x="714.3" />
18 | 
19 | <glyph glyph-name="github-circled" unicode="&#xf09b;" d="M429 779q116 0 215-58t156-156 57-215q0-140-82-252t-211-155q-15-3-22 4t-7 17q0 1 0 43t0 75q0 54-29 79 32 3 57 10t53 22 45 37 30 58 11 84q0 67-44 115 21 51-4 114-16 5-46-6t-51-25l-21-13q-52 15-107 15t-108-15q-8 6-23 15t-47 22-47 7q-25-63-5-114-44-48-44-115 0-47 12-83t29-59 45-37 52-22 57-10q-21-20-27-58-12-5-25-8t-32-3-36 12-31 35q-11 18-27 29t-28 14l-11 1q-12 0-16-2t-3-7 5-8 7-6l4-3q12-6 24-21t18-29l6-13q7-21 24-34t37-17 39-3 31 1l13 3q0-22 0-50t1-30q0-10-8-17t-22-4q-129 43-211 155t-82 252q0 117 58 215t155 156 216 58z m-267-616q2 4-3 7-6 1-8-1-1-4 4-7 5-3 7 1z m18-19q4 3-1 9-6 5-9 2-4-3 1-9 5-6 9-2z m16-25q6 4 0 11-4 7-9 3-5-3 0-10t9-4z m24-23q4 4-2 10-7 7-11 2-5-5 2-11 6-6 11-1z m32-14q1 6-8 9-8 2-10-4t7-9q8-3 11 4z m35-3q0 7-10 6-9 0-9-6 0-7 10-6 9 0 9 6z m32 5q-1 7-10 5-9-1-8-8t10-4 8 7z" horiz-adv-x="857.1" />
20 | 
21 | <glyph glyph-name="upload-cloud" unicode="&#xf0ee;" d="M714 368q0 8-5 13l-196 196q-5 5-13 5t-13-5l-196-196q-5-6-5-13 0-8 5-13t13-5h125v-196q0-8 5-13t12-5h108q7 0 12 5t5 13v196h125q8 0 13 5t5 13z m357-161q0-89-62-151t-152-63h-607q-103 0-177 73t-73 177q0 72 39 134t105 92q-1 17-1 24 0 118 84 202t202 84q87 0 159-49t105-129q40 35 93 35 59 0 101-42t42-101q0-43-23-77 72-17 119-76t46-133z" horiz-adv-x="1071.4" />
22 | 
23 | <glyph glyph-name="doc-text" unicode="&#xf0f6;" d="M819 638q16-16 27-42t11-50v-642q0-23-15-38t-38-16h-750q-23 0-38 16t-16 38v892q0 23 16 38t38 16h500q22 0 49-11t42-27z m-248 136v-210h210q-5 17-12 23l-175 175q-6 7-23 12z m215-853v572h-232q-23 0-38 16t-16 37v233h-429v-858h715z m-572 483q0 7 5 12t13 5h393q8 0 13-5t5-12v-36q0-8-5-13t-13-5h-393q-8 0-13 5t-5 13v36z m411-125q8 0 13-5t5-13v-36q0-8-5-13t-13-5h-393q-8 0-13 5t-5 13v36q0 8 5 13t13 5h393z m0-143q8 0 13-5t5-13v-36q0-8-5-13t-13-5h-393q-8 0-13 5t-5 13v36q0 8 5 13t13 5h393z" horiz-adv-x="857.1" />
24 | 
25 | <glyph glyph-name="spinner" unicode="&#xf110;" d="M294 72q0-29-21-50t-51-21q-29 0-50 21t-21 50q0 30 21 51t50 21 51-21 21-51z m277-115q0-29-20-50t-51-21-50 21-21 50 21 51 50 21 51-21 20-51z m-392 393q0-30-21-50t-51-21-50 21-21 50 21 51 50 20 51-20 21-51z m670-278q0-29-21-50t-50-21q-30 0-51 21t-20 50 20 51 51 21 50-21 21-51z m-538 556q0-37-26-63t-63-26-63 26-26 63 26 63 63 26 63-26 26-63z m653-278q0-30-21-50t-50-21-51 21-21 50 21 51 51 20 50-20 21-51z m-357 393q0-45-31-76t-76-31-76 31-31 76 31 76 76 31 76-31 31-76z m296-115q0-52-37-88t-88-37q-52 0-88 37t-37 88q0 51 37 88t88 37q51 0 88-37t37-88z" horiz-adv-x="1000" />
26 | 
27 | <glyph glyph-name="help" unicode="&#xf128;" d="M393 149v-134q0-9-7-15t-15-7h-134q-9 0-16 7t-7 15v134q0 9 7 16t16 6h134q9 0 15-6t7-16z m176 335q0-30-8-56t-20-43-31-33-32-25-34-19q-23-13-38-37t-15-37q0-10-7-18t-16-9h-134q-8 0-14 11t-6 20v26q0 46 37 87t79 60q33 16 47 32t14 42q0 24-26 41t-60 18q-36 0-60-16-20-14-60-64-7-9-17-9-7 0-14 4l-91 70q-8 6-9 14t3 16q89 148 259 148 45 0 90-17t81-46 59-72 23-88z" horiz-adv-x="571.4" />
28 | 
29 | <glyph glyph-name="lock-open-alt" unicode="&#xf13e;" d="M589 421q23 0 38-15t16-38v-322q0-22-16-37t-38-16h-535q-23 0-38 16t-16 37v322q0 22 16 38t38 15h17v179q0 103 74 177t176 73 177-73 73-177q0-14-10-25t-25-11h-36q-14 0-25 11t-11 25q0 59-42 101t-101 42-101-42-41-101v-179h410z" horiz-adv-x="642.9" />
30 | 
31 | <glyph glyph-name="doc-inv" unicode="&#xf15b;" d="M571 564v264q13-8 21-16l227-228q8-7 16-20h-264z m-71-18q0-22 16-37t38-16h303v-589q0-23-15-38t-38-16h-750q-23 0-38 16t-16 38v892q0 23 16 38t38 16h446v-304z" horiz-adv-x="857.1" />
32 | 
33 | <glyph glyph-name="doc-text-inv" unicode="&#xf15c;" d="M819 584q8-7 16-20h-264v264q13-8 21-16z m-265-91h303v-589q0-23-15-38t-38-16h-750q-23 0-38 16t-16 38v892q0 23 16 38t38 16h446v-304q0-22 16-37t38-16z m89-411v36q0 8-5 13t-13 5h-393q-8 0-13-5t-5-13v-36q0-8 5-13t13-5h393q8 0 13 5t5 13z m0 143v36q0 8-5 13t-13 5h-393q-8 0-13-5t-5-13v-36q0-8 5-13t13-5h393q8 0 13 5t5 13z m0 143v36q0 7-5 12t-13 5h-393q-8 0-13-5t-5-12v-36q0-8 5-13t13-5h393q8 0 13 5t5 13z" horiz-adv-x="857.1" />
34 | 
35 | <glyph glyph-name="sort-alt-down" unicode="&#xf161;" d="M679-25v-107q0-8-5-13t-13-5h-143q-8 0-13 5t-5 13v107q0 8 5 13t13 5h143q8 0 13-5t5-13z m-268 71q0-6-6-13l-178-178q-5-5-13-5-6 0-12 5l-179 179q-8 9-4 19 4 11 17 11h107v768q0 8 5 13t13 5h107q8 0 13-5t5-13v-768h107q8 0 13-5t5-13z m375 215v-107q0-8-5-13t-13-5h-250q-8 0-13 5t-5 13v107q0 8 5 13t13 5h250q8 0 13-5t5-13z m107 285v-107q0-7-5-12t-13-6h-357q-8 0-13 6t-5 12v107q0 8 5 13t13 5h357q8 0 13-5t5-13z m107 286v-107q0-8-5-13t-13-5h-464q-8 0-13 5t-5 13v107q0 8 5 13t13 5h464q8 0 13-5t5-13z" horiz-adv-x="1000" />
36 | 
37 | <glyph glyph-name="file-pdf" unicode="&#xf1c1;" d="M819 638q16-16 27-42t11-50v-642q0-23-15-38t-38-16h-750q-23 0-38 16t-16 38v892q0 23 16 38t38 16h500q22 0 49-11t42-27z m-248 136v-210h210q-5 17-12 23l-175 175q-6 7-23 12z m215-853v572h-232q-23 0-38 16t-16 37v233h-429v-858h715z m-287 331q18-14 47-31 33 4 65 4 82 0 99-27 9-13 1-29 0-1-1-1l-1-2v0q-3-21-39-21-27 0-64 11t-73 29q-123-13-219-46-85-146-135-146-8 0-15 4l-14 7q0 0-3 2-6 6-4 20 5 23 32 51t73 54q8 5 13-3 1-1 1-2 29 47 60 110 38 76 58 146-13 46-17 89t4 71q6 22 23 22h12q13 0 20-8 10-12 5-38-1-3-2-4 0-2 0-5v-17q-1-68-8-107 31-91 82-133z m-321-229q29 13 76 88-29-22-49-47t-27-41z m222 513q-9-23-2-73 1 4 4 24 0 2 4 24 1 3 3 5-1 0-1 1-1 1-1 2 0 12-7 20 0-1 0-1v-2z m-70-368q76 30 159 45-1 0-7 5t-9 8q-43 37-71 98-15-48-46-110-17-31-26-46z m361 9q-13 13-78 13 42-16 69-16 8 0 10 1 0 0-1 2z" horiz-adv-x="857.1" />
38 | </font>
39 | </defs>
40 | </svg>


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # Elasticsearch Frontend
  2 | 
  3 | [![License](https://img.shields.io/github/license/br-data/elasticsearch-frontend.svg?style=flat-square)]() [![GitHub release](https://img.shields.io/github/release/br-data/elasticsearch-frontend.svg?style=flat-square)]() [![GitHub issues](https://img.shields.io/github/issues/br-data/elasticsearch-frontend.svg?style=flat-square)]()
  4 | 
  5 | Simple search interface for large document collections in [Elasticsearch](https://www.elastic.co/de/products/elasticsearch). Made for the exploration and analysis of big document leaks. The application is build with [Express](https://expressjs.com/) and [Pug](https://pugjs.org/). User authentication and protected routes are provided by [Passport](http://passportjs.org/).
  6 | 
  7 | ## History
  8 | The initial prototype was build to uncover the tax haven in the free trade zone of Madeira. We used Elasticsearch to build a document search for the [Madeira Gazette](www.gov-madeira.pt/joram/). Many of those big PDF files are simple document scans which we wanted to search for persons and company names. Read the whole story: [Madeira – A Tax Haven Approved by the European Commission](http://web.br.de/madeira/english/)
  9 | 
 10 | Why build another document search engine? – Because it super lightweight and customizable. Until we add more features.
 11 | 
 12 | ## Requirements
 13 | The application is written in JavaScript. You'll need **Node.js v6** at least, to run the application. Check out the [Node.js installation guide](https://nodejs.org/en/download/package-manager/). We use **Elasticsearch 2.4** for document storage and search. For further details, please refer to the [Elasticsearch installation guide](https://www.elastic.co/guide/en/elasticsearch/reference/2.4/_installation.html).
 14 | 
 15 | To check if your Elasticsearch is up and running, call the REST-Interface from the command line:
 16 | 
 17 | ```
 18 | $ curl -XGET http://localhost:9200/_cluster/health\?pretty\=1
 19 | ```
 20 | 
 21 | If you are seeing a _Unassigned shards_ warning, you might consider setting the numbers of replicas to 0. This works fine in a development environment:
 22 | 
 23 | ```
 24 | $ curl -XPUT 'localhost:9200/_settings' -d '         
 25 | {                  
 26 |   index: {
 27 |     number_of_replicas : 0
 28 |   }
 29 | }'
 30 | ```
 31 | 
 32 | To check if your document are all in place, run a simple search query on your index:
 33 | 
 34 | ```
 35 | $ curl -XGET 'localhost:9200/my-index/_search?q=body:my-query&pretty'
 36 | ```
 37 | 
 38 | ## Installation
 39 | Installation and configuration is straight forward, once Elasticsearch is set up. 
 40 | 
 41 | 1. Import documents to Elasticsearch: If you have never done that before, there is another repo dedicated to extracting text from PDF files and importing them to Elasticsearch: [elasticsearch-import-tools](https://github.com/br-data/elasticsearch-import-tools)
 42 | 2. Edit the `config/config.development.js` file. 
 43 | 3. Start the server: `npm start`.
 44 | 4. Go to http://localhost:3000. The default username is `user` and the password is `password`.
 45 | 
 46 | ## Searching
 47 | There are four different ways to search for whole sentences (full-text) or a single word (term):
 48 | 
 49 | **Standard search** (full-text search): Finds exact word combinations like `John Doe`. Diacritcs are ignored and a search for `John Doe` will also find `Jóhñ Döé`.
 50 | 
 51 | **Custom search** (full-text search): By default, the custom search finds all documents that contain `John` AND `Doe`. Supports wildcards and simple search operators:
 52 | 
 53 | - `+` signifies AND operation
 54 | - `|` signifies OR operation
 55 | - `-` negates a single token
 56 | - `"` wraps a number of tokens to signify a phrase for searching
 57 | - `*` at the end of a term signifies a prefix query
 58 | - `~N` after a word signifies edit distance (fuzziness)
 59 | - `~N` after a phrase signifies slop amount
 60 | 
 61 | **Fuzzy search** (term-based search): Finds words, even if they contain a typo or OCR mistake. A search for `Jhon` or `J°hn` will also find `John`.
 62 | 
 63 | **Regex search** (term-based search): Uses Regex patterns like `J.h*` for searching. This Regex will find words such as `John`, `Jahn` and `Johnson`.
 64 | 
 65 | ## Customization
 66 | If you want to change the page title and description, simply update the configuration `config/config.development.js`.
 67 | 
 68 | ```
 69 | config.page = {
 70 |   title: 'Document Search',
 71 |   description: 'Search Elasticsearch documents for persons, companies and addresses.'
 72 | };
 73 | ```
 74 | 
 75 | ## Authentication
 76 | The current authentication strategy is username and password, using [passport-local](https://github.com/jaredhanson/passport-local). Passport provides many different authentication strategies as Express middleware. If you want to change the authentication method, go ahead, check out the [Passport docs](http://passportjs.org/).
 77 | 
 78 | For the ease of development, valid users are stored in the configuration `config/config.development.js`:
 79 | 
 80 | ```javascript
 81 | config.users = [
 82 |   {
 83 |     id: 1,
 84 |     username: 'user',
 85 |     password: '$2a$10$vP0qJyEd0hvvpG5MAaHg9ObUJJpJj9HxINZ/Yqz5nPo5Ms2nhR4r.',
 86 |     displayName: 'Demo User',
 87 |     apiToken: '0b414d8433124406be6500833f1672e5'
 88 |   }
 89 | ];
 90 | ```
 91 | 
 92 | New password hashes are created using [bcrypt](https://github.com/kelektiv/node.bcrypt.js):
 93 | 
 94 | ```javacript
 95 | const bcrypt = require('bcrypt')
 96 | const saltRounds = 10
 97 | const myPlaintextPassword = 'password'
 98 | const salt = bcrypt.genSaltSync(saltRounds)
 99 | const passwordHash = bcrypt.hashSync(myPlaintextPassword, salt)
100 | 
101 | ``` 
102 | 
103 | Note that the list of user could easily be stored in a database like MongoDB.
104 | 
105 | ## API
106 | 
107 | ```
108 | curl -H "Authorization: Bearer 0b414d8433124406be6500833f1672e5" http://127.0.0.1:3000/api
109 | curl "http://127.0.0.1:3000/api?access_token=0b414d8433124406be6500833f1672e5"
110 | 
111 | curl -H "Authorization: Bearer 0b414d8433124406be6500833f1672e5" "http://localhost:3000/api/search?query=ciboule&type=match&sorting=date"
112 | ```
113 | 
114 | ## Deployment
115 | To deploy the application in a live environment, create a new configuration `config/config.production.js`. Update it with all your server information, Elasticsearch host, credentials etc.
116 | 
117 | Use the new configuration by starting node with the environment variable set to `production`:
118 | 
119 | ```
120 | $ NODE_ENV=production node bin/www
121 | ```
122 | 
123 | To keep it running, use a process manager like [forever](https://github.com/foreverjs/forever) or [PM2](https://github.com/Unitech/pm2):
124 | 
125 | ```
126 | $ NODE_ENV=production forever start bin/www
127 | ```
128 | 
129 | It's advisable to use SSL/TLS encryption for all connections to the server. One way to do this, is routing your Node.js application through an Apache or Nginx proxy with HTTPS enabled.
130 | 
131 | ## Debugging
132 | The app uses [debug](https://github.com/visionmedia/debug) as it's core debugging utility. To set the app into debug mode set the environment variable `debug`.
133 | 
134 | ```
135 | export DEBUG=*
136 | ```
137 | 
138 | If you are on a Winodws machine use:
139 | 
140 | ```
141 | set DEBUG=*
142 | ```
143 | 
144 | ## Planned features
145 | - Add (inline) document viewer
146 | - Add document import and ingestion
147 | - Add direct API access
148 | - Split data retrieval and rendering
149 | 
150 | ## Similar projects:
151 | If you are looking for alternatives, check out:
152 | - OCCRP: [Aleph](https://github.com/alephdata/aleph), powering the [Investigative Dashboard](http://data.occrp.org)
153 | - ICIJ: [Datashare](https://datashare.icij.org/)
154 | - EIC: [Hoover](https://hoover.github.io/)
155 | - New York Times: [Stevedore](https://github.com/newsdev/stevedore)
156 | - [DocumentCloud](https://github.com/documentcloud) 
157 | - [Open Semantic Search](https://www.opensemanticsearch.org)
158 | - [Overview](https://www.overviewdocs.com/)
159 | 


--------------------------------------------------------------------------------