2 | {{#each Sites}}
3 |
--------------------------------------------------------------------------------
/webserver/controllers/components/StopwordComponent.js:
--------------------------------------------------------------------------------
1 | var _ = require('lodash')
2 | , async = require('async')
3 | , fs = require('fs')
4 | , path = require('path')
5 | ;
6 |
7 | module.exports = function(){
8 | var options = {
9 | path: path.join(__dirname,'/../..', '/data/stopwords/german.txt')
10 | }
11 | , stopwords = {}
12 | ;
13 |
14 | if(arguments[0])
15 | options = _.defaults(arguments[0], options);
16 |
17 | /*
18 | Read stopword file
19 |
20 | Comments begin with vertical bar. Each stop | word is at the start of a line.
21 | Source: Snowball (http://snowball.tartarus.org/algorithms/german/stop.txt)
22 |
23 | stopword | comment
24 | stopword2
25 |
26 | |comment
27 | stopword3 | comment3
28 | */
29 | function readStopwords(){
30 | fs.readFileSync(options.path).toString().split("\n").forEach((row) => {
31 | row = row.split("|"); //Remove comments
32 | stopword = row[0].replace(/^\s+/g,'').replace(/\s+$/g,'');
33 | if(stopword.length > 0)
34 | stopwords[stopword] = true;
35 | });
36 |
37 | console.log('Read ' + _.keys(stopwords).length + ' stopwords from ' + options.path);
38 | }
39 |
40 | if(!_.isEmpty(options.stopwords))
41 | stopwords = options.stopwords;
42 | else
43 | readStopwords();
44 |
45 |
46 | function is(token){
47 | return !_.isUndefined(stopwords[token.toLowerCase()]);
48 | }
49 |
50 | this.is = is;
51 | }
--------------------------------------------------------------------------------
/webserver/libs/tokenizer.js:
--------------------------------------------------------------------------------
1 | var async = require('async')
2 | , _ = require('lodash')
3 | ;
4 |
5 | //Treebank word tokenizer
6 | module.exports = new (function(){
7 | var contractions2 = [
8 | /(.)('ll|'re|'ve|n't|'s|'m|'d)\b/ig,
9 | /\b(can)(not)\b/ig,
10 | /\b(D)('ye)\b/ig,
11 | /\b(Gim)(me)\b/ig,
12 | /\b(Gon)(na)\b/ig,
13 | /\b(Got)(ta)\b/ig,
14 | /\b(Lem)(me)\b/ig,
15 | /\b(Mor)('n)\b/ig,
16 | /\b(T)(is)\b/ig,
17 | /\b(T)(was)\b/ig,
18 | /\b(Wan)(na)\b/ig];
19 |
20 | var contractions3 = [
21 | /\b(Whad)(dd)(ya)\b/ig,
22 | /\b(Wha)(t)(cha)\b/ig
23 | ];
24 |
25 | this.tokenize = function(text) {
26 | contractions2.forEach(function(regexp) {
27 | text = text.replace(regexp,"$1 $2");
28 | });
29 |
30 | contractions3.forEach(function(regexp) {
31 | text = text.replace(regexp,"$1 $2 $3");
32 | });
33 |
34 | // most punctuation
35 | //@mod Umlaute hinzugefügt
36 | text = text.replace(/([^\w\.\'\-\/\+\<\>,&äöüßÄÖÜ])/g, " $1 ");
37 |
38 | // commas if followed by space
39 | text = text.replace(/(,)/g, " $1");
40 |
41 | // single quotes if followed by a space
42 | text = text.replace(/('\s)/g, " $1");
43 |
44 | // periods before newline or end of string
45 | text = text.replace(/\. *(\n|$)/g, " . ");
46 |
47 | return _.without(text.split(/\s+/), '');
48 | };
49 | })();
--------------------------------------------------------------------------------
/webserver/views/Articles/get.handlebars:
--------------------------------------------------------------------------------
1 |
4 |
32 | {{/each}}
33 |
5 |
6 |
31 | {{last_visited}}
7 | {{#if title}} 8 | {{title}} 9 | {{url}} 10 | {{else}} 11 | {{shortHost}} 12 | {{shortUrl}} 13 | {{/if}} 14 |
15 |
16 | {{#each Article.sentences}}
17 |
26 | »{{{html}}}
18 | {{/each}}
19 | {{#if sentencesMore}}
20 |
21 |
24 | {{/if}}
25 | View {{sentencesMore}} more sentences
22 | Hide {{sentencesMore}} sentences
23 | -
27 |
- Open site 28 |
- Open archived article 29 |
2 |
--------------------------------------------------------------------------------
/webserver/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "storyfinder-server",
3 | "version": "1.0.0",
4 | "description": "",
5 | "main": "index.js",
6 | "scripts": {
7 | "test": "echo \"Error: no test specified\" && exit 1",
8 | "build": "npm install -g browserify && npm install && cd public/js && npm install && browserify index.js -t babelify -t [hbsfy -t] -o storyfinder.js && browserify register/register.js -t babelify -t [hbsfy -t] -o register.js",
9 | "publish": "docker login && docker build -t uhhlt/storyfinder:latest . && docker push uhhlt/storyfinder:latest"
10 | },
11 | "author": "",
12 | "license": "ISC",
13 | "dependencies": {
14 | "async": "^1.5.2",
15 | "bcrypt": "^0.8.7",
16 | "body-parser": "^1.15.1",
17 | "color-thief": "^2.2.2",
18 | "compute-cosine-distance": "^1.0.0",
19 | "connect-ensure-login": "^0.1.1",
20 | "cookie-session": "^2.0.0-alpha.1",
21 | "cookies": "^0.6.1",
22 | "couchbase": "^2.1.6",
23 | "escape-string-regexp": "^1.0.5",
24 | "express": "^4.13.4",
25 | "express-handlebars": "^3.0.0",
26 | "hiredis": "^0.4.1",
27 | "lodash": "^4.12.0",
28 | "method-override": "^2.3.5",
29 | "mysql": "^2.10.2",
30 | "node-kmeans": "^1.1.0",
31 | "node-stanford-postagger": "0.0.1",
32 | "pagerank-js": "^0.3.0",
33 | "passport": "^0.3.2",
34 | "passport-http": "^0.3.0",
35 | "passport-local": "^1.0.0",
36 | "redis": "^2.5.3",
37 | "request": "^2.72.0",
38 | "serve-static": "^1.10.2",
39 | "socket.io": "^1.4.6"
40 | }
41 | }
42 |
--------------------------------------------------------------------------------
/webserver/public/js/constants/ActionTypes.js:
--------------------------------------------------------------------------------
1 | /*export const SHOW_SITE = 'SHOW_SITE';
2 | export const SHOW_GLOBAL = 'SHOW_GLOBAL';
3 | export const SELECT_NODE = 'SELECT_NODE';
4 | export const OPEN_NODE = 'OPEN_NODE';
5 | export const EXPAND_NODE = 'EXPAND_NODE';
6 | export const DELETE_NODE = 'DELETE_NODE';
7 | export const ADD_NODE = 'ADD_NODE';*/
8 | export const SHOW_RELATION = 'SHOW_RELATION';
9 | export const TO_RELATION = 'TO_RELATION';
10 | export const REQUEST_RELATION = 'REQUEST_RELATION';
11 | export const RECEIVE_RELATION = 'RECEIVE_RELATION';
12 | export const TO_GRAPH = 'TO_GRAPH';
13 | export const SHOW_GRAPH = 'SHOW_GRAPH';
14 | export const TO_LOCALGRAPH = 'TO_LOCALGRAPH';
15 | export const TO_GROUPGRAPH = 'TO_GROUPGRAPH';
16 | export const SHOW_LOCALGRAPH = 'SHOW_LOCALGRAPH';
17 | export const SHOW_GROUPGRAPH = 'SHOW_GROUPGRAPH';
18 | export const TO_GLOBAL = 'TO_GLOBAL';
19 | export const SHOW_GLOBAL = 'SHOW_GLOBAL';
20 | export const REQUEST_GLOBAL = 'REQUEST_GLOBAL';
21 | export const RECEIVE_GLOBAL = 'RECEIVE_GLOBAL';
22 | export const REQUEST_DELETE_NODE = 'REQUEST_DELETE_NODE';
23 | export const RECEIVE_DELETE_NODE = 'RECEIVE_DELETE_NODE';
24 | export const REQUEST_NEIGHBOURS = 'REQUEST_NEIGHBOURS';
25 | export const RECEIVE_NEIGHBOURS = 'RECEIVE_NEIGHBOURS';
26 | export const CREATE_NODE = 'CREATE_NODE';
27 | export const REQUEST_SAVE_NODE = 'REQUEST_SAVE_NODE';
28 | export const RECEIVE_SAVE_NODE = 'RECEIVE_SAVE_NODE';
29 | export const REQUEST_CREATE_RELATION = 'REQUEST_CREATE_RELATION';
30 | export const RECEIVE_CREATE_RELATION = 'RECEIVE_CREATE_RELATION';
--------------------------------------------------------------------------------
/plugin/src/menu.js:
--------------------------------------------------------------------------------
1 | var buttonParse = document.getElementById("button-parse");
2 | var buttonSidebar = document.getElementById("button-sidebar");
3 | var buttonReadability = document.getElementById("button-readability");
4 | var checkboxHighlight = document.getElementById("checkbox-highlight");
5 |
6 | // display the correct status
7 | chrome.storage.sync.get({
8 | highlightEntities: false,
9 | }, function (items) {
10 | checkboxHighlight.checked = items.highlightEntities;
11 | });
12 |
13 | buttonParse.addEventListener("click", function() {
14 | chrome.storage.sync.get({
15 | userInitialized: false,
16 | serverInitialized: false
17 | }, function (items) {
18 | if (!items.serverInitialized && !items.userInitialized) {
19 | alert("To use this function, please provide a Server URL in Extensions > Storyfinder > Options and log into your account.");
20 | } else {
21 | chrome.runtime.sendMessage({type:"force-parse-site"});
22 | window.close();
23 | }
24 | });
25 | });
26 |
27 | buttonSidebar.addEventListener("click", function() {
28 | chrome.runtime.sendMessage({type:"toggle-sidebar"});
29 | window.close();
30 | });
31 |
32 | buttonReadability.addEventListener("click", function() {
33 | chrome.runtime.sendMessage({type:"do-readability"});
34 | window.close();
35 | });
36 |
37 | checkboxHighlight.addEventListener("change", function() {
38 | chrome.storage.sync.set({
39 | highlightEntities: checkboxHighlight.checked
40 | }, function() {
41 | chrome.runtime.sendMessage({type:"highlight-changed", checked: checkboxHighlight.checked});
42 | });
43 | });
--------------------------------------------------------------------------------
/webserver/public/js/templates/graph/title.hbs:
--------------------------------------------------------------------------------
1 | {{#if loading}}
2 |
3 | This page contains the cached version of the article »{{Site.title}}« on »{{Site.host}}«, created on {{Article.created}}.
4 |
5 |
28 |
29 | -
6 |
- Host 7 |
- {{#if Site.favicon}}
{{/if}}{{Site.host}}
8 |
9 | - Source 10 |
- {{Site.url}} 11 | 12 |
- First visit 13 |
- {{Site.created}} 14 | 15 |
- Last visit 16 |
- {{Site.last_visited}} 17 | 18 |
- Abstract 19 |
- {{Article.excerpt}} 20 | 21 |
- Screenshot 22 |
23 |
24 | - Article screenshot 25 |
26 |
{{#if Article.title}}{{Article.title}}{{else}}{{Site.title}}{{/if}}
30 | {{{Article.raw}}} 31 | Parsing...
3 | {{else}}
4 | {{#if site_ids}}
5 | {{#if groupsize}}
6 | arrow_back
7 | Group of {{groupsize}} sites
8 | {{log sites}}
9 | {{#each sites}}
10 | {{#if @index}}
11 | ,
12 | {{/if}}
13 |
20 | {{/each}}
21 |
22 |
23 | {{else}}
24 |
25 | {{/if}}
26 | {{else}}
27 | {{#if site_id}}
28 | {{#if site}}
29 | arrow_back
30 | {{#if site.title}}
31 |
{{site.title}}{{site.url}}
32 | {{else}}
33 | {{site.host}}{{site.url}}
34 | {{/if}}
35 | {{else}}
36 |
37 | {{/if}}
38 | {{else}}
39 | menu
40 |
Global network
41 | {{/if}}
42 | {{/if}}
43 |
44 | searchclose
45 |
46 | {{/if}}
--------------------------------------------------------------------------------
/plugin/src/popup.js:
--------------------------------------------------------------------------------
1 | var baseUrl = "";
2 |
3 | chrome.runtime.onMessage.addListener(function(msg, sender){
4 | switch (msg.type) {
5 | case "msg":
6 | document.querySelector('iframe.active').contentWindow.postMessage(msg.data, '*');
7 | break;
8 | case "settings-changed":
9 | window.location.reload();
10 | break;
11 | }
12 | });
13 |
14 | chrome.storage.sync.get({
15 | server: "",
16 | serverInitialized: false
17 | }, function(items) {
18 | if(!items.serverInitialized || (items.serverInitialized && items.server === "")) {
19 | var message = document.getElementById("message");
20 | message.innerHTML = "The Storyfinder Server has not been set! Please go to Extensions > Storyfinder > Options and set the Server URL.";
21 | return;
22 | }
23 |
24 | baseUrl = items.server;
25 | var iframe = document.createElement('iframe');
26 | iframe.setAttribute('src', baseUrl);
27 | iframe.classList.add("active");
28 | iframe.frameBorder = "none";
29 |
30 |
31 | document.body.appendChild(iframe);
32 |
33 | iframe.onload = function() {
34 | chrome.runtime.sendMessage({type: 'onAttach'});
35 | }
36 |
37 | window.addEventListener("message", function(event){
38 | var origin = event.origin || event.originalEvent.origin;
39 | if(baseUrl !== '' && baseUrl.substr(0, origin.length) !== origin){
40 | // alert('Wrong origin: ' + baseUrl.substr(0, origin.length) + ' !== ' + origin);
41 | return;
42 | }
43 |
44 | if(event.data[0] === 'msg'){
45 | chrome.runtime.sendMessage({type: 'msg', data: event.data[1]});
46 | }
47 | }, false);
48 | });
49 |
50 |
51 |
--------------------------------------------------------------------------------
/webserver/public/js/templates/search/results.hbs:
--------------------------------------------------------------------------------
1 |
2 | {{#if Entities}}
3 |
47 | {{/if}}
48 | {{#unless Entities}}
49 | {{#unless Sites}}
50 | No results found!
51 | {{/unless}}
52 | {{/unless}}
--------------------------------------------------------------------------------
/webserver/docker/docker-compose-build.yml:
--------------------------------------------------------------------------------
1 | ##
2 | #
3 | ##
4 | version: '2.1'
5 |
6 | networks:
7 | sf-net:
8 |
9 | services:
10 | mysqlserver:
11 | image: "mysql:5.5"
12 | environment:
13 | - MYSQL_RANDOM_ROOT_PASSWORD=yes
14 | - MYSQL_DATABASE=storyfinder
15 | - MYSQL_USER=storyfinder
16 | - MYSQL_PORT=3306
17 | - MYSQL_PASSWORD=storyfinder
18 | volumes:
19 | - ${PWD}/mysql-data:/var/lib/mysql
20 | command: ["--character-set-server=utf8", "--collation-server=utf8_bin"]
21 | healthcheck:
22 | test: ["CMD", "mysqladmin" ,"ping", "-h", "localhost", "-pstoryfinder", "-ustoryfinder"]
23 | interval: 20s
24 | timeout: 10s
25 | retries: 3
26 | networks:
27 | sf-net:
28 | aliases:
29 | - mysql
30 |
31 | corenlpserver:
32 | image: "kisad/corenlp-german:latest"
33 | healthcheck:
34 | test: wget --post-data 'The quick brown fox jumped over the lazy dog.' 'localhost:9000/?properties={"annotators":"tokenize,ssplit,pos,ner","outputFormat":"json"}' -O -
35 | interval: 90s
36 | timeout: 10s
37 | retries: 3
38 | mem_limit: 4g
39 | memswap_limit: 5g
40 | networks:
41 | sf-net:
42 | aliases:
43 | - corenlp
44 |
45 | webserver:
46 | depends_on:
47 | mysqlserver:
48 | condition: service_healthy
49 | corenlpserver:
50 | condition: service_healthy
51 | build:
52 | context: ../
53 | dockerfile: Dockerfile
54 | ports:
55 | - "3055:3055"
56 | environment:
57 | - NODE_ENV=production
58 | - COOKIE_SECRET=92bls37gfc
59 | - NER=corenlp
60 | - NO_KEYWORDS=T
61 | restart: unless-stopped
62 | mem_limit: 4g
63 | memswap_limit: 5g
64 | networks:
65 | sf-net:
66 |
--------------------------------------------------------------------------------
/webserver/public/js/templates/nodes/create.hbs:
--------------------------------------------------------------------------------
1 |
6 | Entities
4 |
5 | {{#each Entities}}
6 |
9 |
10 | {{/if}}
11 | {{#if Sites}}
12 | {{caption}}
7 | {{/each}}
8 | Sites
13 |
14 | {{#each Sites}}
15 |
46 |
16 |
44 | {{/each}}
45 |
17 |
18 |
43 | {{last_visited}}
19 | {{#if title}} 20 | {{title}} 21 | {{url}} 22 | {{else}} 23 | {{shortHost}} 24 | {{shortUrl}} 25 | {{/if}} 26 |
27 |
28 | {{#each Article.Sentences}}
29 |
37 | »{{{this}}}
30 | {{/each}}
31 | {{#if sentencesMore}}
32 |
33 |
35 | {{/if}}
36 | + {{sentencesMore}} more sentences
34 | -
38 |
- Open network 39 |
- Open site 40 |
- Open archived article 41 |
7 |
55 |
--------------------------------------------------------------------------------
/plugin/README.md:
--------------------------------------------------------------------------------
1 | # Recommended workflow
2 |
3 | 0. provide the private key under /storyfinder/release/storyfinder-plugin.pem
4 | 1. make changes to the code
5 | 2. increase version number in /storyfinder/src/manifest.json
6 | 3. automatically build the plugin with npm run build
7 | 4. automatically publish the plugin with npm run publish
8 | 5. upload the generated zip file /storyfinder/release/storyfinder.zip to the google developer console
9 |
10 | # Build the plugin
11 |
12 | ## Automatic build
13 | this will automatically build the files: backgroundscript.js, contentscript.js, contentstyle.css
14 |
15 | ```
16 | $ cd /storyfinder/plugin
17 | $ npm run build
18 | ```
19 |
20 | ## Manual build
21 |
22 | ### Build the contentscript
23 |
24 | ```
25 | $ cd /storyfinder/plugin/src/js-contentscript
26 | $ npm install
27 | $ browserify index.js -t babelify -t [hbsfy -t] -o ../contentscript.js
28 | ```
29 |
30 | ### Build the backgroundscript
31 |
32 | ```
33 | $ cd /storyfinder/plugin/src/js-backgroundscript
34 | $ npm install
35 | $ browserify index.js -t babelify -t [hbsfy -t] -o ../backgroundscript.js
36 | ```
37 |
38 | ### Build the contentstyle
39 |
40 | ```
41 | $ cd /storyfinder/plugin/src/js-contentstyle
42 | $ npm install
43 | $ grunt less
44 | ```
45 |
46 | # Publish the plugin
47 |
48 | ## Automatic publish
49 | this will automatically create storyfinder.crx and storyfinder.zip in /storyfinder/release
50 |
51 | ```
52 | $ cd /storyfinder/plugin
53 | $ npm run publish
54 | ```
55 |
56 | ## Manual publish
57 |
58 | ### via command line with chrome
59 |
60 | ```
61 | $ chrome.exe --pack-extension=/storyfinder/plugin/src --pack-extension-key=path/to/storyfinder-plugin.pem
62 | ```
63 |
64 | ### via chrome
65 | 1. navigate to chrome://extensions/
66 | 2. click pack extension
67 | 3. provide /storyfinder/plugin/src as the path to the extension
68 | provide /path/to/storyfinder-plugin.pem as the path to the private key
--------------------------------------------------------------------------------
/webserver/models/Collection.js:
--------------------------------------------------------------------------------
1 | var DatasourceMysql = require('../datasources/mysql.js')
2 | , async = require('async')
3 | , _ = require('lodash')
4 | ;
5 |
6 | module.exports = function(db){
7 | //console.log('Collection');
8 | var name = 'Collection'
9 | , table = 'collections'
10 | , datasource = new DatasourceMysql(db, name, table)
11 | ;
12 |
13 | function findById(id, callback){
14 | datasource.find(_.isArray(id)?'all':'first', {
15 | fields: ['id', 'user_id', 'is_default', 'name', 'created', 'modified'],
16 | conditions: {
17 | id: id,
18 | is_deleted: 0
19 | },
20 | order: 'name ASC'
21 | }, (err, result) => {
22 | if(err)
23 | return setImmediate(() => callback(err));
24 |
25 | setImmediate(() => callback(null, result));
26 | });
27 | }
28 |
29 | this.findById = findById;
30 |
31 | function getDefault(userId, callback){
32 | datasource.find('first', {
33 | fields: ['id'],
34 | conditions: {
35 | user_id: userId,
36 | is_default: 1,
37 | is_deleted: 0
38 | }
39 | }, (err, result) => {
40 | if(err)
41 | return setImmediate(() => callback(err));
42 |
43 | if(result != null)
44 | return setImmediate(() => callback(null, result));
45 |
46 | createDefault(userId, callback);
47 | });
48 | }
49 |
50 | this.getDefault = getDefault;
51 |
52 | function createDefault(userId, callback){
53 | var date = (new Date()).toISOString().slice(0,19).replace('T',' ');
54 |
55 | db.query('INSERT INTO `collections` (user_id, is_default, name, created, modified, is_deleted) VALUES (?, 1, ?, ?, ?, 0)', [
56 | userId,
57 | 'default',
58 | date,
59 | date
60 | ], function(err, res){
61 | if(err)
62 | return setImmediate(() => {
63 | callback(err);
64 | });
65 |
66 | setImmediate(() => {
67 | callback(null, {
68 | id: res.insertId
69 | });
70 | });
71 | });
72 | }
73 | }
74 |
75 |
--------------------------------------------------------------------------------
/webserver/public/login.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
14 |
51 |
52 |
53 |
--------------------------------------------------------------------------------
/webserver/libs/corenlp.js:
--------------------------------------------------------------------------------
1 | var request = require('request')
2 | , _ = require('lodash')
3 | , async = require('async')
4 | ;
5 |
6 | module.exports = function(){
7 | var options = _.defaults(arguments[0], {
8 | host: 'localhost',
9 | port: '9000'
10 | });
11 |
12 | var defaultProperties = {};
13 | if(typeof arguments[1] != 'undefined' && arguments[1] != null)
14 | defaultProperties = _.defaults(arguments[1], defaultProperties);
15 |
16 | function tag(data){
17 | var properties = {};
18 |
19 | if(arguments.length == 3)
20 | properties = arguments[1];
21 |
22 | properties = _.defaults(properties, defaultProperties);
23 |
24 | var callback = arguments[arguments.length - 1];
25 |
26 | request({
27 | uri: 'http://' + options.host + ':' + options.port + '/?properties=' + encodeURIComponent(JSON.stringify(properties)),
28 | method: 'POST',
29 | body: data,
30 | contentType: 'text/plain'
31 | }, function (error, response, body) {
32 |
33 | body = body.replace(/\\n/g, "\\n")
34 | .replace(/\\'/g, "\\'")
35 | .replace(/\\"/g, '\\"')
36 | .replace(/\\&/g, "\\&")
37 | .replace(/\\r/g, "\\r")
38 | .replace(/\\t/g, "\\t")
39 | .replace(/\\b/g, "\\b")
40 | .replace(/\\f/g, "\\f")
41 | .replace(/[\u0000-\u0019]+/g,"")
42 | ;
43 |
44 | if (!error && response.statusCode == 200) {
45 | setImmediate(function(){
46 | callback(null, JSON.parse(body));
47 | });
48 | }else{
49 | callback(error);
50 | }
51 | });
52 | }
53 |
54 | this.tag = tag;
55 |
56 | function pos(data){
57 | var properties = {
58 | "annotators": "pos"
59 | };
60 |
61 | if(arguments.length == 3)
62 | properties = arguments[1];
63 |
64 | properties = _.defaults(properties, defaultProperties);
65 |
66 | var callback = arguments[arguments.length - 1];
67 |
68 | tag(data, properties, callback);
69 | }
70 |
71 | this.pos = pos;
72 | };
--------------------------------------------------------------------------------
/webserver/models/Ngram.js:
--------------------------------------------------------------------------------
1 | var DatasourceMysql = require('../datasources/mysql.js')
2 | , async = require('async')
3 | , _ = require('lodash')
4 | ;
5 |
6 | module.exports = function(db){
7 | //console.log('NGram');
8 | var name = 'Ngram'
9 | , models = _.defaults({Ngram: this}, arguments[1] || {})
10 | , table = 'ngrams'
11 | , datasource = new DatasourceMysql(db, name, table)
12 | ;
13 |
14 | function add(memo, next){
15 | async.eachOfSeries(memo.data.ngrams, (ngrams, n, nextN) => {
16 | if(parseInt(n) > 2)return setImmediate(nextN);
17 |
18 | var normalized = {};
19 | _.forOwn(ngrams, (v, k) => {
20 | var l = k.toLowerCase();
21 | normalized[l] = v;
22 | });
23 |
24 | async.eachOfSeries(normalized, (occurances, ngram, nextNgram) => {
25 | if(ngram.length > 64)return setImmediate(nextNgram);
26 | if(ngram.indexOf('"') != -1 || ngram.indexOf("'") == -1)return setImmediate(nextNgram);
27 | if(ngram.indexOf('\\') != -1)return setImmediate(nextNgram);
28 |
29 | datasource.insertOrUpdate(memo.changelog_id, {
30 | values: {
31 | value: ngram,
32 | collection_id: memo.Collection.id,
33 | docs: 1
34 | },
35 | conditions: {
36 | value: ngram,
37 | collection_id: memo.Collection.id
38 | },
39 | update: {
40 | docs: (prev, next) => {
41 | return parseInt(prev.docs) + parseInt(next.docs);
42 | }
43 | }
44 | }, (err, insertId) => {
45 | if(err)return setImmediate(() => nextNgram(err));
46 | setImmediate(nextNgram);
47 | });
48 | }, (err) => {
49 | setImmediate(() => nextN(err));
50 | });
51 | }, (err) => {
52 | console.log('Done ngrams');
53 | if(err)return setImmediate(() => next(err));
54 |
55 | setImmediate(() => next(null, memo));
56 | });
57 | }
58 |
59 | this.add = add;
60 |
61 | function docsByValue(ngrams, collectionId, callback){
62 | datasource.find('list', {
63 | fields: ['value', 'docs'],
64 | conditions: {
65 | collection_id: collectionId,
66 | value: ngrams
67 | }
68 | }, callback);
69 | }
70 |
71 | this.docsByValue = docsByValue;
72 | }
73 |
--------------------------------------------------------------------------------
/plugin/src/icon.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
53 |
54 |
55 |
56 |
15 |
50 |
16 |
49 |
17 |
30 |
48 |
18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 |
28 | StoryFinder
29 |57 | 58 |
59 |
66 |
67 |
60 |
61 |
62 |
63 |
64 |
65 | 68 | 69 |
70 |
77 |
78 |
71 |
72 |
73 |
74 |
75 |
76 | 79 | 80 |
81 |
88 |
89 |
90 |
91 |
--------------------------------------------------------------------------------
/webserver/docker/docker-compose.yml:
--------------------------------------------------------------------------------
1 | ##
2 | #
3 | ##
4 | version: '2.1'
5 |
6 | networks:
7 | sf-net:
8 |
9 | services:
10 | mysqlserver:
11 | image: "mysql:5.5"
12 | restart: unless-stopped
13 | environment:
14 | - MYSQL_RANDOM_ROOT_PASSWORD=yes
15 | - MYSQL_DATABASE=storyfinder
16 | - MYSQL_USER=storyfinder
17 | - MYSQL_PORT=3306
18 | - MYSQL_PASSWORD=storyfinder
19 | volumes:
20 | - ${PWD}/mysql-data:/var/lib/mysql
21 | command: ["--character-set-server=utf8", "--collation-server=utf8_bin"]
22 | healthcheck:
23 | test: ["CMD", "mysqladmin" ,"ping", "-h", "localhost", "-pstoryfinder", "-ustoryfinder"]
24 | interval: 20s
25 | timeout: 10s
26 | retries: 3
27 | networks:
28 | sf-net:
29 | aliases:
30 | - mysql
31 |
32 | corenlpserver:
33 | image: "kisad/corenlp-german:latest"
34 | restart: unless-stopped
35 | healthcheck:
36 | test: wget --post-data 'The quick brown fox jumped over the lazy dog.' 'localhost:9000/?properties={"annotators":"tokenize,ssplit,pos,ner","outputFormat":"json"}' -O -
37 | interval: 90s
38 | timeout: 10s
39 | retries: 3
40 | mem_limit: 4g
41 | memswap_limit: 5g
42 | networks:
43 | sf-net:
44 | aliases:
45 | - corenlp
46 |
47 | germanerserver:
48 | image: "kisad/storyfinder-germaner-git:latest"
49 | restart: unless-stopped
50 | stdin_open: true
51 | tty: true
52 | networks:
53 | sf-net:
54 | aliases:
55 | - germaner
56 |
57 | webserver:
58 | image: "uhhlt/storyfinder:latest"
59 | volumes:
60 | - ${PWD}/image-data:/usr/src/app/public/images
61 | ports:
62 | - "3055:3055"
63 | environment:
64 | - NODE_ENV=production
65 | - COOKIE_SECRET=92bls37gfc
66 | - NER=germaner
67 | #- NER=corenlp
68 | - NO_KEYWORDS=T
69 | - CHROME_ID=hnndfanecdfnonofigcceaahflgfpgbd
70 | depends_on:
71 | mysqlserver:
72 | condition: service_healthy
73 | corenlpserver:
74 | condition: service_healthy
75 | restart: unless-stopped
76 | mem_limit: 4g
77 | memswap_limit: 4g
78 | networks:
79 | sf-net:
80 |
--------------------------------------------------------------------------------
/webserver/controllers/components/GermanerComponent.js:
--------------------------------------------------------------------------------
1 | var _ = require('lodash')
2 | , async = require('async')
3 | , request = require('request')
4 | ;
5 |
6 | module.exports = function(){
7 | var options = {
8 | host: '127.0.0.1',
9 | port: '8080'
10 | };
11 |
12 | if(arguments[0])
13 | options = _.defaults(arguments[0], options);
14 |
15 | function parse(sentences, callback){
16 | /*
17 | Build GermaNer input format:
18 | sentence1_token1
19 | sentence1_token2
20 | sentence1_token3
21 | .
22 | sentence2_token1
23 | sentence2_token2
24 | ...
25 | */
26 | var data = [];
27 |
28 | _.each(sentences, function(tokens){
29 | var sentence = tokens.join("\n");
30 |
31 | if(sentence.replace(/\s+/g,'').length > 0){
32 | sentence = sentence.replace(/\n+/g, '\n');
33 | data.push(sentence);
34 | }
35 | });
36 |
37 | data = data.join("\n\n") + "\n.\n";
38 | //console.log(data);
39 | request({
40 | uri: 'http://' + options.host + ':' + options.port + '/germaner', //ToDo: read from config
41 | method: 'POST',
42 | body: data,
43 | contentType: 'text/plain'
44 | }, (err, response) => {
45 | //console.log('http://' + options.host + ':' + options.port + '/germaner', err, response, body);
46 | if(err)return setImmediate(() => callback(err));
47 | if(response.statusCode != 200)return setImmediate(() => callback(new Error('Invalid status code ' + response.statusCode + ' of GermaNer (expected: 200)')));
48 |
49 | /*
50 | The server responses with one entity per line prefixed by the entity type:
51 | TYPE entity1
52 | TYPE entity2
53 | TYPE entity3
54 | */
55 |
56 | var entities = [];
57 | response.body.split('\n').forEach(entity => {
58 | entity = entity.replace(/^\s+/g,'').replace(/\s+$/g,''); //Remove surrounding spaces
59 |
60 | if(entity.length > 0){
61 | var tokens = entity.split(/\s+/g)
62 | , type = tokens.shift()
63 | , value = tokens.join(' ')
64 | ;
65 |
66 | entities.push({
67 | type: type,
68 | value: value
69 | });
70 | }
71 | });
72 |
73 | setImmediate(() => callback(null, entities));
74 | });
75 | }
76 |
77 | this.parse = parse;
78 | }
--------------------------------------------------------------------------------
/webserver/public/register.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
82 |
83 |
84 |
85 |
86 |
87 |
14 |
56 |
57 |
58 |
--------------------------------------------------------------------------------
/webserver/public/js/templates/relations/relation.hbs:
--------------------------------------------------------------------------------
1 |
15 |
55 |
16 |
54 |
17 |
30 |
53 |
18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 |
28 | StoryFinder
29 |
2 | arrow_back
3 |
5 | {{Entities.[0].caption}}
4 |
6 | {{#if Relation}}
7 |
55 |
--------------------------------------------------------------------------------
/webserver/controllers/components/KeywordComponent.js:
--------------------------------------------------------------------------------
1 | var _ = require('lodash')
2 | , async = require('async')
3 | , fs = require('fs')
4 | , path = require('path')
5 | ;
6 |
7 | module.exports = function(Ngram, Article){
8 | var t = {
9 | 0: 3,
10 | 1: 3,
11 | 2: 3
12 | };
13 |
14 | function getKeywords(candidates, nsize, collectionId, callback){
15 | Article.countDocumentsInCollection(collectionId, (err, d) => {
16 | if(err)return setImmediate(() => callback(err));
17 | d++; //Add +1 for the current document
18 |
19 | if(d < 4){
20 | console.log('Keyword extraction expects at least 3 documents in the corpus');
21 | return setImmediate(() => callback(null, []));
22 | }
23 |
24 | var weights = [];
25 |
26 | Ngram.docsByValue(_.keys(candidates), collectionId, (err, docs) => {
27 | if(err)return setImmediate(() => callback(err));
28 |
29 | //Get the maximum term frequency for normalization
30 | var tfMax = _.reduce(candidates, (n, o) => {
31 | return Math.max(n, o.length);
32 | }, 1);
33 |
34 | //Calculate weights
35 | _.forOwn(candidates, (o, candidate) => {
36 | var n = 0;
37 | if(docs[candidate])
38 | n = docs[candidate].docs;
39 | n++;
40 |
41 | var tf = o.length / tfMax
42 | ;
43 |
44 | /*
45 | d = #documents
46 | n = #documents containg candidate
47 | tf = Normalized termfrequency of candidate
48 | */
49 |
50 | var idf = Math.log(d/n);
51 | if(o.caption.match(/^[A-Za-z0-9\s]+$/))
52 | weights.push({w: tf * idf, ngram: o.caption, idf: idf, n: n, tf: tf});
53 | });
54 |
55 | //Sort weights
56 | weights.sort((a, b) => {
57 | return b.w - a.w;
58 | });
59 |
60 | /*
61 | Only extract elements with a value > t[n]
62 | */
63 |
64 | console.log(weights.slice(0, 10));
65 |
66 | var keywords = [];
67 |
68 | //Use the top 3 elements as keywords
69 | for(var c of weights){
70 | if(_.isUndefined(t[nsize]))break;
71 | if(c.w < t[nsize])break;
72 |
73 | keywords.push(c.ngram);
74 | }
75 |
76 | /*weights.slice(0, 3).forEach((c) => {
77 | keywords.push(c.ngram);
78 | }); */
79 |
80 | /*var c = weights.shift()
81 | keywords = []
82 | ;
83 |
84 | while(c.w > t){
85 | keywords.push(c.ngram);
86 |
87 | if(weights.length == 0)break;
88 | c = weights.shift();
89 | }*/
90 |
91 | console.log('Keywords', keywords);
92 |
93 | setImmediate(() => callback(null, keywords));
94 | });
95 | });
96 | }
97 |
98 | this.getKeywords = getKeywords;
99 | }
--------------------------------------------------------------------------------
/webserver/README.md:
--------------------------------------------------------------------------------
1 | # Instructions to start the server
2 |
3 | 1. Create dockers
4 |
5 | only for dev-server:
6 | 2. Attach to docker
7 | 3. Build the server with npm run build
8 | 4. npm start
9 |
10 | 5. Server is now accessible at http://localhost:3055/
11 |
12 | # Docker
13 |
14 | ### Choose docker-compose file:
15 | docker-compose-dev: Development Version, uses CHROME_ID = pebdjeaapfkjiceloeecpoedbliefnap
16 |
17 | docker-compose: Live Version, uses CHROME_ID = hnndfanecdfnonofigcceaahflgfpgbd
18 |
19 | ### Create dockers
20 |
21 | ```
22 | $ cd storyfinder/webserver/docker
23 | $ docker-compose -f docker-compose-dev.yml -p storyfinder up -d
24 | ```
25 |
26 | ### Attach docker
27 |
28 | ```
29 | $ docker attach storyfinder_webserver-dev_1
30 | ```
31 |
32 | ### Delete dockers
33 |
34 | ```
35 | $ cd storyfinder/webserver/docker
36 | $ docker-compose -f docker-compose-dev.yml -p storyfinder down
37 | ```
38 |
39 | # Recommended workflow
40 |
41 | 1. make changes to the code
42 | 2. automatically build the server with npm run build
43 | 3. automatically publish the server to dockerhub with npm run publish
44 |
45 | # Build the server
46 |
47 | ## Automatic build
48 | this installs the npm modules and builds storyfinder.js and register.js
49 |
50 | ```
51 | $ cd storyfinder/webserver
52 | $ npm run build
53 | ```
54 |
55 | ## Manual build
56 |
57 | ### Manually install npm modules
58 |
59 | ```
60 | $ npm install -g browserify
61 |
62 | $ cd /storyfinder/webserver
63 | $ npm install
64 |
65 | $ cd /storyfinder/webserver/public/js
66 | $ npm install
67 | ```
68 |
69 | ### Manually build storyfinder.js
70 |
71 | ```
72 | $ cd /storyfinder/webserver/public/js
73 | $ browserify index.js -t babelify -t [hbsfy -t] -o storyfinder.js
74 | ```
75 |
76 | ### Manually build register.js
77 |
78 | ```
79 | $ cd /storyfinder/webserver/public/js
80 | $ browserify register/register.js -t babelify -t [hbsfy -t] -o register.js
81 | ```
82 |
83 | # Publish the server
84 |
85 | ## Automatic publish
86 | this automatically builds a the storyfinder-docker and pushes it to uhhlt/storyfinder:latest
87 |
88 | ```
89 | $ cd storyfinder/webserver
90 | $ npm run publish
91 | ```
92 |
93 | ## Manual publish
94 |
95 | ### Set the version
96 |
97 | ```
98 | $ v=0.0.10
99 | ```
100 |
101 | ### Build the storyfinder-docker
102 |
103 | ```
104 | $ cd storyfinder/webserver
105 | $ docker login --username=
8 |
12 | keyboard_arrow_up
9 |
10 | keyboard_arrow_down
11 | -
13 |
- 14 | 20 | 21 | {{#if Sentences}} 22 | {{#each Sentences}} 23 |
-
24 | »{{{html}}} «25 |– Source {{Article.Site.title}} visited on {{Article.Site.last_visited}} –26 | 32 |
33 | {{/each}}
34 | {{else}}
35 |
-
36 | There is currently no source for this relation.37 |
38 | {{/if}}
39 |
42 | There is currently no relation between »{{Entities.[0].caption}}« and »{{Entities.[1].caption}}«.
Do you want to create a new relation? 43 | 44 |
53 | {{/if}}
54 | Do you want to create a new relation? 43 | 44 |
45 |
46 |
47 |
48 |
49 | Yes, create relation
50 | No
51 |
52 |
24 |
28 |
115 |
116 |
123 |
124 |
125 |
126 |
--------------------------------------------------------------------------------
/webserver/libs/changelog.js:
--------------------------------------------------------------------------------
1 | var _ = require('lodash')
2 | , async = require('async')
3 | ;
4 |
5 | function buildConditions(conditions){
6 | var args = [];
7 |
8 | for(var key in conditions){
9 | if(key.match(/^\d+$/)){
10 | args.push(conditions[key]);
11 | }else if(key == 'or'){
12 | args.push(buildConditions(conditions[key]).join(' or '));
13 | }else if(key == 'and'){
14 | args.push(buildConditions(conditions[key]).join(' and '));
15 | }else{
16 | var rel = '=';
17 | var val = conditions[key];
18 | if(key.indexOf(' ') != -1){
19 | key = key.split(' ');
20 | rel = key[1];
21 | key = key[0];
22 | }
23 |
24 | if(conditions[key] == null && rel == '='){
25 | args.push('`' + key + '` IS NULL');
26 | }else{
27 | //console.log(typeof val);
28 |
29 | if(_.isArray(val)){
30 | val = '("' + _.map(val, function(v){
31 | if(isNaN(v))
32 | return v.replace(/\"/g,"\\\"")
33 | else
34 | return v;
35 | }).join('","') + '")';
36 |
37 | if(rel == '<>' || rel == '!='){
38 | rel = ' NOT IN ';
39 | }else{
40 | rel = ' IN ';
41 | }
42 | }else if(isNaN(val))
43 | val = '"' + val.replace(/\"/g,"\\\"") + '"';
44 |
45 | args.push('`' + key + '` ' + rel + ' ' + val);
46 | }
47 | }
48 | }
49 |
50 | return args;
51 | }
52 |
53 | module.exports.update = function update(connection, userId){
54 | var date = (new Date()).toISOString().slice(0,19).replace('T',' ')
55 | ;
56 |
57 | var options = _.defaults(arguments[2], {
58 | values: null,
59 | table: null,
60 | conditions: null,
61 | limit: null
62 | });
63 |
64 | var callback = arguments[3]
65 | , fields = _.keys(options.values)
66 | , where = buildConditions(options.conditions).join(' and ')
67 | , limit = ''
68 | ;
69 |
70 | if(!_.isNull(options.limit)){
71 | if(_.isArray(options.limit)){
72 | options.limit = _.map(options.limit, function(l){
73 | if(isNaN(l))
74 | return 0;
75 | return l;
76 | });
77 | limit = ' LIMIT ' + options.limit.join(',');
78 | }else if(!isNaN(options.limit))
79 | limit = ' LIMIT ' + options.limit;
80 | }
81 |
82 | var q = 'SELECT `id`, `' + fields.join('`,`') + '` FROM `' + options.table + '` WHERE ' + where + limit;
83 | //console.log(q);
84 | connection.query(q, function(err, results, f){
85 | if(err){
86 | setImmediate(function(){callback(err);});
87 | return;
88 | }
89 |
90 | if(!_.isNull(results)){
91 | var updates = [];
92 |
93 | async.eachSeries(results, function(row, nextRow){
94 | var changes = {};
95 | for(var i = 0;i < fields.length; i++){
96 | if(row[fields[i]] != options.values[fields[i]]){
97 | changes[fields[i]] = row[fields[i]];
98 | }
99 | }
100 |
101 | if(!_.isEmpty(changes)){
102 | updates.push([row.id, JSON.stringify(changes), options.table]);
103 | }
104 |
105 | setImmediate(nextRow);
106 | }, function(err){
107 | if(err){
108 | callback(err);
109 | return;
110 | }
111 |
112 | q = 'INSERT INTO changelogs (`created`, `user_id`) VALUES (?, ?)';
113 | //console.log(q);
114 | connection.query(q, [date, userId], function(err, result){
115 | if(err){
116 | callback(err);
117 | return;
118 | }
119 |
120 | var changelog_id = result.insertId;
121 |
122 | async.each(updates, function(update, nextUpdate){
123 | //console.log('INSERT INTO changelogs_updates (`changelog_id`, `foreign_id`, `vals`, `tbl`) VALUES (' + changelog_id + ', ?, ?, ?)');
124 | connection.query('INSERT INTO changelogs_updates (`changelog_id`, `foreign_id`, `vals`, `tbl`) VALUES (' + changelog_id + ', ?, ?, ?)', update, nextUpdate);
125 | }, function(err){
126 | if(err){
127 | callback(err);
128 | return;
129 | }
130 |
131 | //console.log('UPDATE `' + options.table + '` SET ' + _.keys(options.values).join('=?, ') + '=? WHERE ' + where + limit);
132 |
133 | connection.query('UPDATE `' + options.table + '` SET ' + _.keys(options.values).join('=?, ') + '=? WHERE ' + where + limit, _.values(options.values), function(err){
134 | if(err){
135 | callback(err);
136 | return;
137 | }
138 |
139 |
140 | setImmediate(function(){
141 | callback(null, changelog_id);
142 | });
143 | });
144 | });
145 | });
146 | });
147 | }else{
148 | console.log('RESULT IS NULL!!!', q);
149 | }
150 | });
151 | }
--------------------------------------------------------------------------------
/webserver/controllers/ArticlesController.js:
--------------------------------------------------------------------------------
1 | var _ = require('lodash')
2 | , async = require('async')
3 | , fs = require('fs')
4 | , path = require('path')
5 | , ensureLoggedIn = require('connect-ensure-login').ensureLoggedIn
6 | , evallog = new (require('../libs/evallog.js'))()
7 | ;
8 |
9 | module.exports = function(connection, app, passport){
10 | var User = new (require('../models/User.js'))(connection)
11 | , Site = new (require('../models/Site.js'))(connection)
12 | , Article = new (require('../models/Article.js'))(connection)
13 | , Collection = new (require('../models/Collection.js'))(connection)
14 | ;
15 |
16 | app.get('/Articles/:articleId', ensureLoggedIn((process.env.PATH_PREFIX || '/') + 'login'), function (req, res) {
17 | var userId = req.user.id
18 | , articleId = req.params.articleId
19 | ;
20 |
21 | evallog.log('Open article ' + articleId);
22 |
23 | async.waterfall([
24 | (next) => setImmediate(() => next(null, {article_id: articleId})),
25 | /*
26 | Get the article
27 | */
28 | (memo, next) => {
29 | Article.findById(memo.article_id,
30 | (err, article) => {
31 | if(err)return setImmediate(() => next(err));
32 | memo.Article = article;
33 | setImmediate(() => next(null, memo));
34 | }
35 | );
36 | },
37 | /*
38 | Get the site containing the article
39 | */
40 | (memo, next) => {
41 | Site.findById(memo.Article.site_id,
42 | (err, site) => {
43 | if(err)return setImmediate(() => next(err));
44 | memo.Site = site;
45 | setImmediate(() => next(null, memo));
46 | }
47 | );
48 | },
49 | /*
50 | Verify that the site belongs to one of the user's collection
51 | */
52 | (memo, next) => {
53 | Collection.findById(memo.Site.collection_id,
54 | (err, collection) => {
55 | if(err)return setImmediate(() => next(err));
56 |
57 | if(collection.user_id != userId)
58 | memo = null;
59 |
60 | setImmediate(() => next(null, memo));
61 | }
62 | );
63 | }
64 | ],
65 | (err, result) => {
66 | if(err){
67 | console.log(err);
68 | return setImmediate(() => res.sendStatus(500));
69 | }
70 |
71 | if(_.isNull(result))
72 | res.sendStatus(404);
73 | else
74 | res.render('Articles/get', result);
75 | });
76 | });
77 |
78 | app.put('/Articles/:articleId/image', passport.authenticate('basic', {session: false}), function(req, res){
79 | var userId = req.user.id
80 | , articleId = req.params.articleId
81 | , img = req.body.image.replace(/^data:image\/png;base64,/, "")
82 | ;
83 |
84 | async.waterfall([
85 | (next) => setImmediate(() => next(null, {
86 | user_id: userId
87 | })),
88 | _mGetCollection, //Get the id of user's default collection
89 | (memo, next) => {
90 | async.reduce(['..', 'public', 'images', '/' + memo.Collection.id, 'articles'], __dirname, (p, folder, nextFolder) => {
91 | p = path.join(p, folder);
92 |
93 | fs.stat(p, (err, stats) => {
94 | if(err && err.code == 'ENOENT') {
95 | fs.mkdir(p, (err) => {
96 | if(err)return setImmediate(() => nextFolder(err));
97 |
98 | setImmediate(() => nextFolder(null, p));
99 | });
100 | }else if(err){
101 | return setImmediate(() => nextFolder(err));
102 | }else{
103 | return setImmediate(() => nextFolder(null, p));
104 | }
105 | });
106 | }, (err, p) => {
107 | if(err)return setImmediate(() => next(err));
108 |
109 | memo.path = p;
110 | setImmediate(() => next(null, memo));
111 | });
112 | },
113 | (memo, next) => {
114 | fs.writeFile(path.join(memo.path, articleId + '.png'), img, 'base64', function(err){
115 | if(err)return setImmediate(() => next(err));
116 |
117 | setImmediate(() => next(null, memo));
118 | });
119 | }
120 | ],
121 | (err, result) => {
122 | if(err){
123 | console.log(err);
124 | res.sendStatus(500);
125 | return false;
126 | }
127 |
128 | res.send({
129 | success: true
130 | });
131 | });
132 | });
133 |
134 | /*
135 | Memo functions
136 | */
137 | /*
138 | Get the id of user's default collection
139 | */
140 | function _mGetCollection(memo, next){
141 | Collection.getDefault(memo.user_id,
142 | (err, collection) => {
143 | if(err)return setImmediate(() => next(err));
144 |
145 | memo.Collection = collection;
146 |
147 | setImmediate(() => next(null, memo))
148 | }
149 | );
150 | }
151 | }
--------------------------------------------------------------------------------
/webserver/public/js/search.js:
--------------------------------------------------------------------------------
1 | var _ = require('lodash')
2 | , async = require('async')
3 | , Delegate = require('dom-delegate')
4 | , tplResults = require('./templates/search/results.hbs')
5 | , actions = require('./actions/StoryfinderActions.js')
6 | ;
7 |
8 | module.exports = function(store, vis){
9 | var inputDelegate = null
10 | , searchDelegate = null
11 | , searchQ = async.queue(search, 1)
12 | , elResults = document.getElementById('search-results')
13 | ;
14 |
15 | function initialize(){
16 | inputDelegate = new Delegate(document.body.querySelector('#graph-title'));
17 |
18 | inputDelegate.on('keyup', '.btn-search > input', function(event){
19 | var value = event.target.value;
20 |
21 | if(value.replace(/\s+/g, '').length == 0){
22 | event.target.classList.remove('active');
23 | event.target.parentNode.classList.remove('active');
24 | elResults.classList.remove('active');
25 | }else{
26 | elResults.classList.add('active');
27 | event.target.classList.add('active');
28 | event.target.parentNode.classList.add('active');
29 | searchQ.push(value);
30 | }
31 | });
32 |
33 | inputDelegate.on('click', '.btn-search .icon-close', function(event){
34 | clear();
35 | return false;
36 | });
37 |
38 | searchDelegate = new Delegate(elResults);
39 |
40 | searchDelegate.on('click', '.site .show-graph', function(event){
41 | clear();
42 | store.dispatch(actions.toLocalgraph(event.target.getAttribute('data-id')));
43 | return false;
44 | });
45 |
46 | searchDelegate.on('click', '.entity', function(event){
47 | clear();
48 | vis.highlight(parseInt(event.target.getAttribute('data-id')), () => {
49 | vis.selectNode('.label[data-id="' + parseInt(event.target.getAttribute('data-id')) + '"]');
50 | });
51 | return false;
52 | });
53 | }
54 |
55 | function clear(){
56 | var input = document.body.querySelector('#graph-title .btn-search > input');
57 | if(input != null){
58 | input.value = '';
59 | input.blur();
60 | input.classList.remove('active');
61 | input.parentNode.classList.remove('active');
62 | }
63 | if(elResults != null)
64 | elResults.classList.remove('active');
65 | }
66 |
67 | function search(searchValue, callback){
68 | if(searchQ.length() > 0)
69 | return setTimeout(callback, 0); //Process only the latest query
70 |
71 | fetch('/Entities/search?q=' + encodeURIComponent(searchValue), {
72 | credentials: 'same-origin'
73 | })
74 | .then(response => response.json())
75 | .then(json => {
76 | showResults(json);
77 | setTimeout(callback, 0);
78 | }).catch(err => {
79 | console.log(err);
80 | setTimeout(callback, 0);
81 | });
82 | }
83 |
84 | function showResults(results){
85 | results.Sites.forEach(function(site){
86 | if(_.isUndefined(site.Article.Sentences))
87 | site.Article.Sentences = [];
88 |
89 | site.Article.Sentences = _.map(site.Article.Sentences, (sentence) => {
90 | return sentence.text.split(results.search).join('' + results.search + '');
91 | });
92 |
93 | site.sentencesMore = (site.Article.Sentences.length > 3)?(site.Article.Sentences.length - 3):false;
94 | });
95 |
96 | elResults.innerHTML = tplResults(results);
97 |
98 | //Scale elements equaly
99 | var prevY = null
100 | , sitesByY = {}
101 | ;
102 |
103 | function rescale(y){
104 | var row = sitesByY[y];
105 |
106 | var max = 0;
107 | row.forEach(function(c){
108 | if(c.height > max)
109 | max = c.height;
110 | });
111 |
112 | row.forEach(function(c){
113 | var s = c.el.querySelector('.sentences');
114 | if(s == null)return;
115 | var h = s.getBoundingClientRect().height;
116 | s.style.maxHeight = h + ((max - c.height)) + 'px';
117 | s.style.minHeight = h + ((max - c.height)) + 'px';
118 | });
119 | }
120 |
121 | var sites = document.body.querySelectorAll('.site');
122 |
123 | for(var site of sites){
124 | var rect = site.getBoundingClientRect();
125 | var y = rect.top;
126 |
127 | if(prevY != y && prevY != null){
128 | rescale(prevY);
129 |
130 | rect = site.getBoundingClientRect();
131 | y = rect.top;
132 | }
133 |
134 | if(_.isUndefined(sitesByY[y]))
135 | sitesByY[y] = [];
136 |
137 | sitesByY[y].push({
138 | el: site,
139 | height: rect.height
140 | });
141 |
142 | prevY = y;
143 | }
144 |
145 | if(!_.isNull(prevY))
146 | rescale(prevY);
147 | }
148 |
149 | this.clear = clear;
150 |
151 | initialize();
152 | }
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | ####
2 | # created by steffen
3 | webserver/public/images/
4 | webserver/**/*-data
5 | release
6 |
7 | plugin/dist/
8 |
9 | ####
10 | # Created by https://www.gitignore.io/api/node,bower,eclipse,webstorm+all,intellij+all
11 |
12 | ### Bower ###
13 | bower_components
14 | .bower-cache
15 | .bower-registry
16 | .bower-tmp
17 |
18 | ### Eclipse ###
19 |
20 | .metadata
21 | bin/
22 | tmp/
23 | *.tmp
24 | *.bak
25 | *.swp
26 | *~.nib
27 | local.properties
28 | .settings/
29 | .loadpath
30 | .recommenders
31 |
32 | # External tool builders
33 | .externalToolBuilders/
34 |
35 | # Locally stored "Eclipse launch configurations"
36 | *.launch
37 |
38 | # PyDev specific (Python IDE for Eclipse)
39 | *.pydevproject
40 |
41 | # CDT-specific (C/C++ Development Tooling)
42 | .cproject
43 |
44 | # Java annotation processor (APT)
45 | .factorypath
46 |
47 | # PDT-specific (PHP Development Tools)
48 | .buildpath
49 |
50 | # sbteclipse plugin
51 | .target
52 |
53 | # Tern plugin
54 | .tern-project
55 |
56 | # TeXlipse plugin
57 | .texlipse
58 |
59 | # STS (Spring Tool Suite)
60 | .springBeans
61 |
62 | # Code Recommenders
63 | .recommenders/
64 |
65 | # Scala IDE specific (Scala & Java development for Eclipse)
66 | .cache-main
67 | .scala_dependencies
68 | .worksheet
69 |
70 | ### Eclipse Patch ###
71 | # Eclipse Core
72 | .project
73 |
74 | # JDT-specific (Eclipse Java Development Tools)
75 | .classpath
76 |
77 | ### Intellij+all ###
78 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm
79 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
80 |
81 | # User-specific stuff:
82 | .idea/**/workspace.xml
83 | .idea/**/tasks.xml
84 | .idea/dictionaries
85 |
86 | # Sensitive or high-churn files:
87 | .idea/**/dataSources/
88 | .idea/**/dataSources.ids
89 | .idea/**/dataSources.xml
90 | .idea/**/dataSources.local.xml
91 | .idea/**/sqlDataSources.xml
92 | .idea/**/dynamic.xml
93 | .idea/**/uiDesigner.xml
94 |
95 | # Gradle:
96 | .idea/**/gradle.xml
97 | .idea/**/libraries
98 |
99 | # CMake
100 | cmake-build-debug/
101 |
102 | # Mongo Explorer plugin:
103 | .idea/**/mongoSettings.xml
104 |
105 | ## File-based project format:
106 | *.iws
107 |
108 | ## Plugin-specific files:
109 |
110 | # IntelliJ
111 | /out/
112 |
113 | # mpeltonen/sbt-idea plugin
114 | .idea_modules/
115 |
116 | # JIRA plugin
117 | atlassian-ide-plugin.xml
118 |
119 | # Cursive Clojure plugin
120 | .idea/replstate.xml
121 |
122 | # Crashlytics plugin (for Android Studio and IntelliJ)
123 | com_crashlytics_export_strings.xml
124 | crashlytics.properties
125 | crashlytics-build.properties
126 | fabric.properties
127 |
128 | ### Intellij+all Patch ###
129 | # Ignores the whole idea folder
130 | # See https://github.com/joeblau/gitignore.io/issues/186 and https://github.com/joeblau/gitignore.io/issues/360
131 |
132 | .idea/
133 |
134 | ### Node ###
135 | # Logs
136 | logs
137 | *.log
138 | npm-debug.log*
139 | yarn-debug.log*
140 | yarn-error.log*
141 |
142 | # Runtime data
143 | pids
144 | *.pid
145 | *.seed
146 | *.pid.lock
147 |
148 | # Directory for instrumented libs generated by jscoverage/JSCover
149 | lib-cov
150 |
151 | # Coverage directory used by tools like istanbul
152 | coverage
153 |
154 | # nyc test coverage
155 | .nyc_output
156 |
157 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
158 | .grunt
159 |
160 | # Bower dependency directory (https://bower.io/)
161 |
162 | # node-waf configuration
163 | .lock-wscript
164 |
165 | # Compiled binary addons (http://nodejs.org/api/addons.html)
166 | build/Release
167 |
168 | # Dependency directories
169 | node_modules/
170 | jspm_packages/
171 |
172 | # Typescript v1 declaration files
173 | typings/
174 |
175 | # Optional npm cache directory
176 | .npm
177 |
178 | # Optional eslint cache
179 | .eslintcache
180 |
181 | # Optional REPL history
182 | .node_repl_history
183 |
184 | # Output of 'npm pack'
185 | *.tgz
186 |
187 | # Yarn Integrity file
188 | .yarn-integrity
189 |
190 | # dotenv environment variables file
191 | .env
192 |
193 |
194 | ### WebStorm+all ###
195 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm
196 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
197 |
198 | # User-specific stuff:
199 |
200 | # Sensitive or high-churn files:
201 |
202 | # Gradle:
203 |
204 | # CMake
205 |
206 | # Mongo Explorer plugin:
207 |
208 | ## File-based project format:
209 |
210 | ## Plugin-specific files:
211 |
212 | # IntelliJ
213 |
214 | # mpeltonen/sbt-idea plugin
215 |
216 | # JIRA plugin
217 |
218 | # Cursive Clojure plugin
219 |
220 | # Crashlytics plugin (for Android Studio and IntelliJ)
221 |
222 | ### WebStorm+all Patch ###
223 | # Ignores the whole idea folder
224 | # See https://github.com/joeblau/gitignore.io/issues/186 and https://github.com/joeblau/gitignore.io/issues/360
225 |
226 |
227 | # End of https://www.gitignore.io/api/node,bower,eclipse,webstorm+all,intellij+all
228 |
--------------------------------------------------------------------------------
/plugin/src/contentstyle.css:
--------------------------------------------------------------------------------
1 | .storyfinder-main .de-tu-darmstadt-lt-storyfinder-root{transition:color .3s ease}.storyfinder-main .de-tu-darmstadt-lt-storyfinder-root.storyfinder-highlighted{color:#AAAAAA}.storyfinder-main .de-tu-darmstadt-lt-storyfinder-root.storyfinder-highlighted sf-entity.type-ORG{text-shadow:none !important;color:#C2185B;display:inline-block}.storyfinder-main .de-tu-darmstadt-lt-storyfinder-root.storyfinder-highlighted sf-entity.type-ORG:hover{transform:scale(1.1)}.storyfinder-main .de-tu-darmstadt-lt-storyfinder-root.storyfinder-highlighted sf-entity.type-LOC{text-shadow:none !important;color:#388E3C;display:inline-block}.storyfinder-main .de-tu-darmstadt-lt-storyfinder-root.storyfinder-highlighted sf-entity.type-LOC:hover{transform:scale(1.1)}.storyfinder-main .de-tu-darmstadt-lt-storyfinder-root.storyfinder-highlighted sf-entity.type-PER{text-shadow:none !important;color:#0288D1;display:inline-block}.storyfinder-main .de-tu-darmstadt-lt-storyfinder-root.storyfinder-highlighted sf-entity.type-PER:hover{transform:scale(1.1)}.storyfinder-main .de-tu-darmstadt-lt-storyfinder-root.storyfinder-highlighted sf-entity.type-OTH{text-shadow:none !important;color:#512DA8;display:inline-block}.storyfinder-main .de-tu-darmstadt-lt-storyfinder-root.storyfinder-highlighted sf-entity.type-OTH:hover{transform:scale(1.1)}.storyfinder-main .de-tu-darmstadt-lt-storyfinder-root.storyfinder-highlighted sf-entity.type-KEY{text-shadow:none !important;color:#E64A19;display:inline-block}.storyfinder-main .de-tu-darmstadt-lt-storyfinder-root.storyfinder-highlighted sf-entity.type-KEY:hover{transform:scale(1.1)}.storyfinder-main .de-tu-darmstadt-lt-storyfinder-root .entity{cursor:context-menu;white-space:nowrap}.storyfinder-main .de-tu-darmstadt-lt-storyfinder-root .entity sf-marker{display:inline-block;font-size:70%;text-rendering:auto;-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale;transform:translate(0, 0);padding-left:5px}.storyfinder-main .de-tu-darmstadt-lt-storyfinder-root sf-entity{text-shadow:0 0 5px #FFFF00}.storyfinder-main .de-tu-darmstadt-lt-storyfinder-root sf-entity.type-ORG{text-shadow:0 0 5px #E91E63;transition:all .3s ease}.storyfinder-main .de-tu-darmstadt-lt-storyfinder-root sf-entity.type-ORG.storyfinder-highlighted{text-shadow:0 0 2px #C2185B;color:#F8BBD0}.storyfinder-main .de-tu-darmstadt-lt-storyfinder-root sf-entity.type-LOC{text-shadow:0 0 5px #4CAF50;transition:all .3s ease}.storyfinder-main .de-tu-darmstadt-lt-storyfinder-root sf-entity.type-LOC.storyfinder-highlighted{text-shadow:0 0 2px #388E3C;color:#C8E6C9}.storyfinder-main .de-tu-darmstadt-lt-storyfinder-root sf-entity.type-PER{text-shadow:0 0 5px #03A9F4;transition:all .3s ease}.storyfinder-main .de-tu-darmstadt-lt-storyfinder-root sf-entity.type-PER.storyfinder-highlighted{text-shadow:0 0 2px #0288D1;color:#B3E5FC}.storyfinder-main .de-tu-darmstadt-lt-storyfinder-root sf-entity.type-OTH{text-shadow:0 0 5px #673AB7;transition:all .3s ease}.storyfinder-main .de-tu-darmstadt-lt-storyfinder-root sf-entity.type-OTH.storyfinder-highlighted{text-shadow:0 0 2px #512DA8;color:#D1C4E9}.storyfinder-main .de-tu-darmstadt-lt-storyfinder-root sf-entity.type-KEY{text-shadow:0 0 5px #FF5722;transition:all .3s ease}.storyfinder-main .de-tu-darmstadt-lt-storyfinder-root sf-entity.type-KEY.storyfinder-highlighted{text-shadow:0 0 2px #E64A19;color:#FFCCBC}.storyfinder-main .de-tu-darmstadt-lt-storyfinder-root.storyfinder{position:absolute;right:0;width:40px;height:auto;top:0;bottom:0;background-color:#FFFFFF;box-shadow:-2px 2px 2px rgba(0,0,0,0.3);z-index:99900;display:block}.storyfinder-main .de-tu-darmstadt-lt-storyfinder-root.storyfinder.fixed{top:0 !important;position:fixed}.storyfinder-main .de-tu-darmstadt-lt-storyfinder-root.storyfinder sf-toolbar .loading{width:40px;height:40px;font-size:30px;vertical-align:middle;line-height:40px;text-align:center}.storyfinder-main .de-tu-darmstadt-lt-storyfinder-root.storyfinder sf-toolbar .btn{height:40px;border-left:none;border-right:none;line-height:40px;vertical-align:middle}.storyfinder-main .de-tu-darmstadt-lt-storyfinder-root.storyfinder .articlemap{position:fixed;right:40px;bottom:0;box-shadow:-2px 2px 2px rgba(0,0,0,0.3)}.storyfinder-main .de-tu-darmstadt-lt-storyfinder-root.storyfinder .articlemap img{max-width:150px;height:350px}.storyfinder-main .storyfinder-overlay{position:absolute;top:0;left:0;pointer-events:none;z-index:999999}.storyfinder-main .storyfinder-overlay node{pointer-events:visiblePainted}.storyfinder-main .storyfinder-overlay path{stroke:#999999;fill:none}.storyfinder-main sf-source{position:absolute;right:130px;background-color:#FFFFFF;border:1px solid #CCCCCC;padding:3px;z-index:999998;max-width:120px;font-size:80%}.storyfinder-main sf-source img{width:120px}.storyfinder-main sf-marker-container{z-index:999999}.storyfinder-main sf-marker-container sf-marker{display:none;opacity:0;transition:opacity .1s}.storyfinder-main sf-marker-container sf-marker .marker-above,.storyfinder-main sf-marker-container sf-marker .marker-below{display:none}.storyfinder-main sf-marker-container sf-marker.above,.storyfinder-main sf-marker-container sf-marker.below{position:fixed;display:inline-block;opacity:1;padding:10px;background-color:rgba(0,0,0,0.8);color:#FFFFFF;transition:opacity .1s}.storyfinder-main sf-marker-container sf-marker.above{bottom:auto;top:0}.storyfinder-main sf-marker-container sf-marker.above .marker-above{display:inline-block}.storyfinder-main sf-marker-container sf-marker.below{top:auto;bottom:0}.storyfinder-main sf-marker-container sf-marker.below .marker-below{display:inline-block}
--------------------------------------------------------------------------------
/webserver/public/js/register/register.js:
--------------------------------------------------------------------------------
1 | var Delegate = require('dom-delegate')
2 | ;
3 |
4 | require('es6-promise').polyfill();
5 | require('isomorphic-fetch');
6 |
7 |
8 | function disableForm(){
9 | var formElements = document.querySelectorAll('form input, form button');
10 |
11 | for(var element of formElements)
12 | element.setAttribute('disabled', 'disabled');
13 | }
14 |
15 | function showLoading(){
16 | document.querySelector('.avatar').classList.add('loading');
17 | }
18 |
19 | function hideLoading(){
20 | document.querySelector('.avatar').classList.remove('loading');
21 | }
22 |
23 | function enableForm(){
24 | var formElements = document.querySelectorAll('form input, form button');
25 |
26 | for(var element of formElements)
27 | element.removeAttribute('disabled');
28 | }
29 |
30 | function setError(element, msg){
31 | var el = document.querySelector(element);
32 | el.classList.add('has-error');
33 |
34 | if(typeof msg != 'undefined' && msg != null && msg.length > 0){
35 | el.querySelector('.help-block').innerHTML = msg;
36 | }
37 | }
38 |
39 | function resetValidation(){
40 | var formGroups = document.querySelectorAll('#frm-register .form-group');
41 |
42 | for(var element of formGroups){
43 | element.classList.remove('has-error');
44 | element.querySelector('.help-block').innerHTML = '';
45 | }
46 | }
47 |
48 | function register(){
49 | resetValidation();
50 | disableForm();
51 | showLoading();
52 |
53 | var formElements = document.querySelectorAll('#frm-register input, #frm-register button');
54 |
55 | var data = {};
56 |
57 | for(var element of formElements)
58 | data[element.getAttribute('name')] = element.value;
59 |
60 | if(data['password'] != data['password-verify']){
61 | setError('.form-group-verify', 'Passwords don\'t match!');
62 | setError('.form-group-password');
63 | enableForm();
64 | hideLoading();
65 | return;
66 | }
67 |
68 | if(!data['username'].match(/^[-a-z0-9~!$%^&*_=+}{\'?]+(\.[-a-z0-9~!$%^&*_=+}{\'?]+)*@([a-z0-9_][-a-z0-9_]*(\.[-a-z0-9_]+)*\.(aero|arpa|biz|com|coop|edu|gov|info|int|mil|museum|name|net|org|pro|travel|mobi|[a-z][a-z])|([0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}))(:[0-9]{1,5})?$/i)){
69 | setError('.form-group-username', 'Please enter a valid Email address!');
70 | enableForm();
71 | hideLoading();
72 | return;
73 | }
74 |
75 | if(data['password'].length < 8){
76 | setError('.form-group-password', 'Password is too short! Please select a password with at least 8 characters.');
77 | setError('.form-group-verify');
78 | enableForm();
79 | hideLoading();
80 | return;
81 | }
82 |
83 | fetch('/Users', {
84 | method: 'PUT',
85 | headers: {
86 | "Content-Type": "application/json"
87 | },
88 | body: JSON.stringify(data)
89 | })
90 | .then(function(response) {
91 | if (response.status >= 400) {
92 | console.log(response.status);
93 | setError('.form-group-username', 'Error connection to the storyfinder server! Please try again later.');
94 | enableForm();
95 | hideLoading();
96 | return null;
97 | }
98 |
99 | return response.json();
100 | })
101 | .then(function(json) {
102 | if(!json)return;
103 |
104 | if(!json.success){
105 | setError('.form-group-username', json.message);
106 | enableForm();
107 | hideLoading();
108 | return null;
109 | }
110 |
111 | if(typeof parent != null)
112 | parent.postMessage(["msg", {
113 | action: 'userRegistered',
114 | username: data['username'],
115 | password: data['password']
116 | }], "*");
117 |
118 | document.location = '/';
119 | });
120 | }
121 |
122 | function login(){
123 | resetValidation();
124 | disableForm();
125 | showLoading();
126 |
127 | var formElements = document.querySelectorAll('form input');
128 |
129 | var data = {};
130 | var formData = [];
131 |
132 | for(var element of formElements){
133 | formData.push(element.getAttribute('name') + '=' + encodeURIComponent(element.value));
134 | data[element.getAttribute('name')] = element.value;
135 | }
136 |
137 | fetch('/login', {
138 | method: 'POST',
139 | credentials: 'same-origin',
140 | headers: {
141 | "Content-type": "application/x-www-form-urlencoded; charset=UTF-8"
142 | },
143 | body: formData.join('&')
144 | })
145 | .then(function(response) {
146 | if (response.status >= 400) {
147 | console.log(response.status);
148 | setError('.form-group-username', 'Login failed!');
149 | enableForm();
150 | hideLoading();
151 | return null;
152 | }
153 |
154 | return response.json();
155 | })
156 | .then(function(json) {
157 | if(!json)return;
158 |
159 | if(!json.success){
160 | setError('.form-group-username', json.message);
161 | enableForm();
162 | hideLoading();
163 | return null;
164 | }
165 |
166 | if(typeof parent != null)
167 | parent.postMessage(["msg", {
168 | action: 'userRegistered',
169 | username: data['username'],
170 | password: data['password']
171 | }], "*");
172 |
173 | document.location = '/';
174 | });
175 | }
176 |
177 | var frmRegister = document.querySelector('#frm-register');
178 | if(frmRegister)
179 | frmRegister.addEventListener('submit', function(e){
180 | e.preventDefault();
181 | e.stopPropagation();
182 |
183 | register();
184 | });
185 |
186 | var frmLogin = document.querySelector('#frm-login');
187 | if(frmLogin)
188 | frmLogin.addEventListener('submit', function(e){
189 | e.preventDefault();
190 | e.stopPropagation();
191 |
192 | login();
193 | });
--------------------------------------------------------------------------------
/webserver/data/stopwords/german.txt:
--------------------------------------------------------------------------------
1 |
2 | | A German stop word list. Comments begin with vertical bar. Each stop
3 | | word is at the start of a line.
4 |
5 | | The number of forms in this list is reduced significantly by passing it
6 | | through the German stemmer.
7 |
8 |
9 | aber | but
10 |
11 | alle | all
12 | allem
13 | allen
14 | aller
15 | alles
16 |
17 | als | than, as
18 | also | so
19 | am | an + dem
20 | an | at
21 |
22 | ander | other
23 | andere
24 | anderem
25 | anderen
26 | anderer
27 | anderes
28 | anderm
29 | andern
30 | anderr
31 | anders
32 |
33 | auch | also
34 | auf | on
35 | aus | out of
36 | bei | by
37 | bin | am
38 | bis | until
39 | bist | art
40 | da | there
41 | damit | with it
42 | dann | then
43 |
44 | der | the
45 | den
46 | des
47 | dem
48 | die
49 | das
50 |
51 | daß | that
52 |
53 | derselbe | the same
54 | derselben
55 | denselben
56 | desselben
57 | demselben
58 | dieselbe
59 | dieselben
60 | dasselbe
61 |
62 | dazu | to that
63 |
64 | dein | thy
65 | deine
66 | deinem
67 | deinen
68 | deiner
69 | deines
70 |
71 | denn | because
72 |
73 | derer | of those
74 | dessen | of him
75 |
76 | dich | thee
77 | dir | to thee
78 | du | thou
79 |
80 | dies | this
81 | diese
82 | diesem
83 | diesen
84 | dieser
85 | dieses
86 |
87 |
88 | doch | (several meanings)
89 | dort | (over) there
90 |
91 |
92 | durch | through
93 |
94 | ein | a
95 | eine
96 | einem
97 | einen
98 | einer
99 | eines
100 |
101 | einig | some
102 | einige
103 | einigem
104 | einigen
105 | einiger
106 | einiges
107 |
108 | einmal | once
109 |
110 | er | he
111 | ihn | him
112 | ihm | to him
113 |
114 | es | it
115 | etwas | something
116 |
117 | euer | your
118 | eure
119 | eurem
120 | euren
121 | eurer
122 | eures
123 |
124 | für | for
125 | gegen | towards
126 | gewesen | p.p. of sein
127 | hab | have
128 | habe | have
129 | haben | have
130 | hat | has
131 | hatte | had
132 | hatten | had
133 | hier | here
134 | hin | there
135 | hinter | behind
136 |
137 | ich | I
138 | mich | me
139 | mir | to me
140 |
141 |
142 | ihr | you, to her
143 | ihre
144 | ihrem
145 | ihren
146 | ihrer
147 | ihres
148 | euch | to you
149 |
150 | im | in + dem
151 | in | in
152 | indem | while
153 | ins | in + das
154 | ist | is
155 |
156 | jede | each, every
157 | jedem
158 | jeden
159 | jeder
160 | jedes
161 |
162 | jene | that
163 | jenem
164 | jenen
165 | jener
166 | jenes
167 |
168 | jetzt | now
169 | kann | can
170 |
171 | kein | no
172 | keine
173 | keinem
174 | keinen
175 | keiner
176 | keines
177 |
178 | können | can
179 | könnte | could
180 | machen | do
181 | man | one
182 |
183 | manche | some, many a
184 | manchem
185 | manchen
186 | mancher
187 | manches
188 |
189 | mein | my
190 | meine
191 | meinem
192 | meinen
193 | meiner
194 | meines
195 |
196 | mit | with
197 | muss | must
198 | musste | had to
199 | nach | to(wards)
200 | nicht | not
201 | nichts | nothing
202 | noch | still, yet
203 | nun | now
204 | nur | only
205 | ob | whether
206 | oder | or
207 | ohne | without
208 | sehr | very
209 |
210 | sein | his
211 | seine
212 | seinem
213 | seinen
214 | seiner
215 | seines
216 |
217 | selbst | self
218 | sich | herself
219 |
220 | sie | they, she
221 | ihnen | to them
222 |
223 | sind | are
224 | so | so
225 |
226 | solche | such
227 | solchem
228 | solchen
229 | solcher
230 | solches
231 |
232 | soll | shall
233 | sollte | should
234 | sondern | but
235 | sonst | else
236 | über | over
237 | um | about, around
238 | und | and
239 |
240 | uns | us
241 | unse
242 | unsem
243 | unsen
244 | unser
245 | unses
246 |
247 | unter | under
248 | viel | much
249 | vom | von + dem
250 | von | from
251 | vor | before
252 | während | while
253 | war | was
254 | waren | were
255 | warst | wast
256 | was | what
257 | weg | away, off
258 | weil | because
259 | weiter | further
260 |
261 | welche | which
262 | welchem
263 | welchen
264 | welcher
265 | welches
266 |
267 | wenn | when
268 | werde | will
269 | werden | will
270 | wie | how
271 | wieder | again
272 | will | want
273 | wir | we
274 | wird | will
275 | wirst | willst
276 | wo | where
277 | wollen | want
278 | wollte | wanted
279 | würde | would
280 | würden | would
281 | zu | to
282 | zum | zu + dem
283 | zur | zu + der
284 | zwar | indeed
285 | zwischen | between
--------------------------------------------------------------------------------
/webserver/controllers/GraphsController.js:
--------------------------------------------------------------------------------
1 | var _ = require('lodash')
2 | , async = require('async')
3 | , fs = require('fs')
4 | , ensureLoggedIn = require('connect-ensure-login').ensureLoggedIn
5 | , evallog = new (require('../libs/evallog.js'))()
6 | ;
7 |
8 | module.exports = function(connection, app, passport){
9 | var User = new (require('../models/User.js'))(connection)
10 | , Site = new (require('../models/Site.js'))(connection)
11 | , Article = new (require('../models/Article.js'))(connection)
12 | , Collection = new (require('../models/Collection.js'))(connection)
13 | , Entity = new (require('../models/Entity.js'))(connection)
14 | , Relation = new (require('../models/Relation.js'))(connection)
15 | ;
16 |
17 | app.get('/Graphs', ensureLoggedIn((process.env.PATH_PREFIX || '/') + 'login'), function (req, res) {
18 | var userId = req.user.id
19 |
20 | async.waterfall([
21 | (next) => {
22 | setImmediate(() => next(null, {user_id: userId}))
23 | },
24 | _mGetCollection, //Get the id of user's default collection
25 | _mGetEntities, //Get the entities in the collection
26 | _mGetRelations //Get the relations of the entities
27 | ],
28 | (err, result) => {
29 | if(err){
30 | console.log(err);
31 | return setImmediate(() => res.sendStatus(500));
32 | }
33 |
34 | res.send(result);
35 | });
36 | });
37 |
38 | app.get('/Graphs/Site/:siteId', ensureLoggedIn((process.env.PATH_PREFIX || '/') + 'login'), function (req, res) {
39 | var userId = req.user.id
40 | , siteId = parseInt(req.params.siteId)
41 | ;
42 |
43 | evallog.log('Open graph ' + req.params.siteId);
44 |
45 | async.waterfall([
46 | (next) => {
47 | setImmediate(() => next(null, {user_id: userId, site_id: siteId}))
48 | },
49 | _mGetCollection, //Get the id of user's default collection
50 | _mGetSites, //Get Site(s)
51 | _mGetArticles, //Get newest Article
52 | _mGetEntities, //Get all entities for the site(s)
53 | _mGetRelations //Get the relations of the entities
54 | ],
55 | (err, result) => {
56 | if(err){
57 | console.log(err);
58 | return setImmediate(() => res.sendStatus(500));
59 | }
60 |
61 | res.send(result);
62 | });
63 | });
64 |
65 | app.get('/Graphs/Group/:siteIds', ensureLoggedIn((process.env.PATH_PREFIX || '/') + 'login'), function (req, res) {
66 | var userId = req.user.id
67 | , siteIds = _.map(req.params.siteIds.split(';'), siteId => parseInt(siteId))
68 | ;
69 |
70 | evallog.log('Open group ' + siteIds.join(';'));
71 |
72 | async.waterfall([
73 | (next) => {
74 | setImmediate(() => next(null, {user_id: userId, site_ids: siteIds}))
75 | },
76 | _mGetCollection, //Get the id of user's default collection
77 | _mGetSites, //Get Site(s)
78 | _mGetArticles, //Get newest Article
79 | _mGetEntities, //Get all entities for the site(s)
80 | _mGetRelations //Get the relations of the entities
81 | ],
82 | (err, result) => {
83 | if(err){
84 | console.log(err);
85 | return setImmediate(() => res.sendStatus(500));
86 | }
87 |
88 | res.send(result);
89 | });
90 | });
91 |
92 | /*
93 | Memo functions:
94 | */
95 |
96 | /*
97 | Get the id of user's default collection
98 | */
99 | function _mGetCollection(memo, next){
100 | Collection.getDefault(memo.user_id,
101 | (err, collection) => {
102 | if(err)return setImmediate(() => next(err));
103 |
104 | memo.Collection = collection;
105 |
106 | setImmediate(() => next(null, memo))
107 | }
108 | );
109 | }
110 |
111 | /*
112 | Get the entities in the collection
113 | */
114 | function _mGetEntities(memo, next){
115 | var method = 'getAll'
116 | , param = memo.Collection.id
117 | ;
118 |
119 | if(!_.isEmpty(memo.Articles)){
120 | method = 'getInArticle';
121 | param = _.map(memo.Articles, 'id');
122 | }else if(!_.isEmpty(memo.Article)){
123 | method = 'getInArticle';
124 | param = memo.Article.id;
125 | }
126 |
127 | Entity[method](param,
128 | (err, entities) => {
129 | if(err)return setImmediate(() => next(err));
130 |
131 | memo.Entities = entities;
132 |
133 | setImmediate(() => next(null, memo));
134 | }
135 | );
136 | }
137 |
138 | /*
139 | Get the relations of the entities
140 | */
141 | function _mGetRelations(memo, next){
142 | Relation.getByEntities(_.map(memo.Entities, 'id'),
143 | {withTypes: true},
144 | (err, relations) => {
145 | if(err)return setImmediate(() => next(err));
146 |
147 | memo.Relations = relations;
148 |
149 | setImmediate(() => next(null, memo));
150 | }
151 | );
152 | }
153 |
154 | /*
155 | Get Site(s)
156 | */
157 | function _mGetSites(memo, next){
158 | var multi = false;
159 | if(!_.isUndefined(memo.site_ids))
160 | multi = true;
161 |
162 | Site.getById(memo[multi?'site_ids':'site_id'], memo.Collection.id,
163 | (err, site) => {
164 | if(err)return setImmediate(() => next(err));
165 |
166 | memo[multi?'Sites':'Site'] = site;
167 |
168 | setImmediate(() => next(null, memo));
169 | }
170 | );
171 | }
172 |
173 | /*
174 | Get newest articles of the sites
175 | */
176 | function _mGetArticles(memo, next){
177 | if(_.isEmpty(memo.Site) && _.isEmpty(memo.Sites))return setImmediate(() => next(null, memo));
178 |
179 | var multi = false;
180 | if(!_.isUndefined(memo.Sites))
181 | multi = true;
182 |
183 | Article.getLatest(multi?_.map(memo.Sites, 'id'):memo.Site.id,
184 | (err, article) => {
185 | if(err)return setImmediate(() => next(err));
186 |
187 | memo[multi?'Articles':'Article'] = article;
188 |
189 | setImmediate(() => next(null, memo));
190 | }
191 | );
192 | }
193 | }
--------------------------------------------------------------------------------
/webserver/controllers/RelationsController.js:
--------------------------------------------------------------------------------
1 | var _ = require('lodash')
2 | , async = require('async')
3 | , fs = require('fs')
4 | , ensureLoggedIn = require('connect-ensure-login').ensureLoggedIn
5 | , evallog = new (require('../libs/evallog.js'))()
6 | ;
7 |
8 | module.exports = function(connection, app, passport){
9 | var User = new (require('../models/User.js'))(connection)
10 | , Relation = new (require('../models/Relation.js'))(connection)
11 | , Collection = new (require('../models/Collection.js'))(connection)
12 | , Entity = new (require('../models/Entity.js'))(connection)
13 | , RelationSentence = new (require('../models/RelationSentence.js'))(connection)
14 | ;
15 |
16 | app.get('/Relations/Entity/:entityId', ensureLoggedIn((process.env.PATH_PREFIX || '/') + 'login'), function (req, res) {
17 | var entityId = parseInt(req.params.entityId)
18 | , userId = req.user.id
19 | ;
20 |
21 | evallog.log('Get relations of entity ' + entityId);
22 |
23 | async.waterfall([
24 | (next) => {setImmediate(() => next(null, {user_id: userId, entity_id: entityId}))},
25 | _mGetCollection, //Get the id of user's default collection
26 | _mGetEntity, //Get the entity
27 | _mGetRelationsOfEntity, //Get relations of the entity
28 | ],
29 | (err, result) => {
30 | if(err){
31 | console.log(err);
32 | return setImmediate(() => res.sendStatus(500));
33 | }
34 |
35 | res.send(result);
36 | });
37 | });
38 |
39 | app.get('/Relations/:entity1Id/:entity2Id', ensureLoggedIn((process.env.PATH_PREFIX || '/') + 'login'), function (req, res) {
40 | var entity1Id = parseInt(req.params.entity1Id)
41 | , entity2Id = parseInt(req.params.entity2Id)
42 | , userId = req.user.id
43 | ;
44 |
45 |
46 | evallog.log('Get relations of entities ' + entity1Id + ' / ' + entity2Id);
47 |
48 | async.waterfall([
49 | (next) => {setImmediate(() => next(null, {user_id: userId, entity_ids: [entity1Id, entity2Id]}))},
50 | _mGetCollection, //Get the id of user's default collection
51 | _mGetEntity, //Get the entities
52 | _mGetRelationBetweenEntities, //Get relation
53 | _mGetSentences //Get sentences containing the relation
54 | ],
55 | (err, result) => {
56 | if(err){
57 | console.log(err);
58 | return setImmediate(() => res.sendStatus(500));
59 | }
60 |
61 | res.send(result);
62 | });
63 | });
64 |
65 | app.put('/Relations/:entity1Id/:entity2Id', ensureLoggedIn((process.env.PATH_PREFIX || '/') + 'login'), function (req, res) {
66 | var entity1Id = parseInt(req.params.entity1Id)
67 | , entity2Id = parseInt(req.params.entity2Id)
68 | , data = req.body
69 | , userId = req.user.id
70 | , sentenceId = req.body.sentence_id
71 | , label = req.body.label
72 | ;
73 |
74 | evallog.log('Set relation of entities ' + entity1Id + ' / ' + entity2Id + ' with label ' + label);
75 |
76 | async.waterfall([
77 | (next) => {
78 | setImmediate(() => next(null, {
79 | user_id: userId,
80 | entity1_id: entity1Id,
81 | entity2_id: entity2Id,
82 | label: label,
83 | sentenceId: sentenceId
84 | }));
85 | },
86 | _mGetCollection,
87 | Relation.create
88 | ],
89 | (err, result) => {
90 | if(err){
91 | console.log(err);
92 | return setImmediate(() => res.sendStatus(500));
93 | }
94 |
95 | result.Relation.Relationtype = result.Relationtype;
96 |
97 | res.send(result);
98 | });
99 | });
100 |
101 | /*
102 | Memo functions
103 | */
104 | /*
105 | Get the id of user's default collection
106 | */
107 | function _mGetCollection(memo, next){
108 | Collection.getDefault(memo.user_id,
109 | (err, collection) => {
110 | if(err)return setImmediate(() => next(err));
111 |
112 | memo.Collection = collection;
113 |
114 | setImmediate(() => next(null, memo))
115 | }
116 | );
117 | }
118 |
119 | /*
120 | Get the entity
121 | */
122 | function _mGetEntity(memo, next){
123 | Entity.getById(memo.entity_ids || memo.entity_id, {
124 | collection: memo.Collection.id,
125 | withMerged: true
126 | },
127 | (err, entity) => {
128 | if(err)return setImmediate(() => next(err));
129 |
130 | if(!_.isUndefined(memo.entity_ids))
131 | memo.Entities = entity;
132 | else
133 | memo.Entity = entity;
134 |
135 | setImmediate(() => next(null, memo));
136 | }
137 | );
138 | }
139 |
140 | /*
141 | Get relations of the entity with neighbours
142 | */
143 | function _mGetRelationsOfEntity(memo, next){
144 | Relation.getByEntities(memo.Entity.id, {
145 | withCounts: true,
146 | withTypes: true,
147 | withEntities: true
148 | },
149 | (err, relations) => {
150 | if(err)return setImmediate(() => next(err));
151 |
152 | memo.Relations = relations;
153 |
154 | setImmediate(() => next(null, memo));
155 | }
156 | )
157 | }
158 |
159 | /*
160 | Get the relation between two entities
161 | */
162 | function _mGetRelationBetweenEntities(memo, next){
163 | Relation.getBetweenEntities(memo.entity_ids[0], memo.entity_ids[1],
164 | {
165 | withTypes: true
166 | },
167 | (err, relation) => {
168 | if(err)return setImmediate(() => next(err));
169 |
170 | memo.Relation = relation;
171 |
172 | setImmediate(() => next(null, memo));
173 | }
174 | )
175 | }
176 |
177 | /*
178 | Get sentences containing the relation
179 | */
180 | function _mGetSentences(memo, next){
181 | if(_.isEmpty(memo.Relation))return setImmediate(() => next(null, memo));
182 |
183 | RelationSentence.sentencesWithRelation(memo.Relation.id, {
184 | withArticles: true,
185 | withTypes: true,
186 | highlight: memo.Entities
187 | }, (err, sentences) => {
188 | if(err)return setImmediate(() => next(err));
189 |
190 | memo.Sentences = sentences;
191 | setImmediate(() => next(null, memo));
192 | });
193 | }
194 | }
--------------------------------------------------------------------------------
/plugin/src/js-contentstyle/less/storyfinder.less:
--------------------------------------------------------------------------------
1 | @colors: #F44366, #E91E63, #9C27B0, #673AB7, #3F51B5, #2196F3, #03A9F4, #00BCD4, #009688, #4CAF50, #8BC34A, #FFC107, #FF9800, #FF5722, #795548;
2 | @colors2: #D32F2F, #C2185B, #7B1FA2, #512DA8, #303F9F, #1976D2, #0288D1, #0097A7, #00796B, #388E3C, #689F38, #FFA000, #F57C00, #E64A19, #5D4037;
3 | @colors3: #FFCDD2, #F8BBD0, #E1BEE7, #D1C4E9, #C5CAE9, #BBDEFB, #B3E5FC, #B2EBF2, #B2DFDB, #C8E6C9, #DCEDC8, #FFECB3, #FFE0B2, #FFCCBC, #D7CCC8;
4 |
5 | @ORG: 2;
6 | @OTH: 4;
7 | @PER: 7;
8 | @LOC: 10;
9 | @KEY: 14;
10 |
11 | .storyfinder-main {
12 | .typed-entity(@type, @idx){
13 | @n: ~'.type-@{type}';
14 |
15 | &@{n} {
16 | text-shadow: 0px 0px 5px extract(@colors, @idx);
17 |
18 | transition: all 0.3s ease;
19 |
20 | &.storyfinder-highlighted {
21 | text-shadow: 0px 0px 2px extract(@colors2, @idx);
22 | color: extract(@colors3, @idx);
23 | }
24 | }
25 | }
26 |
27 | .typed-entity-highlighted(@type, @idx){
28 | @n: ~'.type-@{type}';
29 |
30 | &@{n} {
31 | text-shadow: none !important;
32 | color: extract(@colors2, @idx);
33 | display: inline-block;
34 |
35 | &:hover {
36 | transform: scale(1.1);
37 | }
38 | }
39 | }
40 |
41 | .de-tu-darmstadt-lt-storyfinder-root {
42 |
43 | transition: color 0.3s ease;
44 |
45 | &.storyfinder-highlighted {
46 | color: #AAAAAA;
47 |
48 | sf-entity {
49 | .typed-entity-highlighted(ORG, @ORG);
50 | .typed-entity-highlighted(LOC, @LOC);
51 | .typed-entity-highlighted(PER, @PER);
52 | .typed-entity-highlighted(OTH, @OTH);
53 | .typed-entity-highlighted(KEY, @KEY);
54 | }
55 | }
56 |
57 | /*border: 1px solid #0000FF;
58 | background-color: #EEEEEE;*/
59 |
60 | .entity {
61 | //text-decoration: underline;
62 | //border-bottom: 1px dotted #0000FF;
63 | cursor: context-menu;
64 | white-space: nowrap;
65 | //color: #FF0000 !important;
66 |
67 | sf-marker {
68 | display: inline-block;
69 | //font: normal normal normal @fa-font-size-base/1 StoryFinderFontAwesome; // shortening font declaration
70 | font-size: 70%; // can't have font-size inherit on line above, so need to override
71 | text-rendering: auto; // optimizelegibility throws things off #1094
72 | -webkit-font-smoothing: antialiased;
73 | -moz-osx-font-smoothing: grayscale;
74 | transform: translate(0, 0);
75 | //content: @fa-var-info-circle;
76 | padding-left: 5px;
77 |
78 | &.internal-ref {
79 | &:before {
80 | //content: @fa-var-circle;
81 | }
82 | }
83 |
84 | &.external-ref {
85 | &:before {
86 | //content: @fa-var-external-link;
87 | }
88 | }
89 | }
90 | }
91 |
92 | sf-entity {
93 | text-shadow: 0px 0px 5px #FFFF00;
94 | .typed-entity(ORG, @ORG);
95 | .typed-entity(LOC, @LOC);
96 | .typed-entity(PER, @PER);
97 | .typed-entity(OTH, @OTH);
98 | .typed-entity(KEY, @KEY);
99 | }
100 |
101 | sf-entity.storyfinder-highlighted {
102 | //text-shadow: 0px 0px 5px #0074ff;
103 | }
104 |
105 | /*.entity:after {
106 | display: inline-block;
107 | font: normal normal normal @fa-font-size-base/1 StoryFinderFontAwesome; // shortening font declaration
108 | font-size: inherit; // can't have font-size inherit on line above, so need to override
109 | text-rendering: auto; // optimizelegibility throws things off #1094
110 | -webkit-font-smoothing: antialiased;
111 | -moz-osx-font-smoothing: grayscale;
112 | transform: translate(0, 0);
113 | content: @fa-var-info-circle;
114 | padding-left: 5px;
115 | }*/
116 |
117 |
118 | &.storyfinder {
119 | position: absolute;
120 | right: 0;
121 | width: 40px;
122 | height: auto;
123 | top: 0px;
124 | bottom: 0px;
125 | background-color: #FFFFFF;
126 | box-shadow: -2px 2px 2px rgba(0,0,0,0.3);
127 | z-index: 99900;
128 | display: block;
129 | //@import "svg.less";
130 |
131 | &.fixed {
132 | top: 0 !important;
133 | position: fixed;
134 | }
135 |
136 | sf-toolbar {
137 | .loading {
138 | width: 40px;
139 | height: 40px;
140 | font-size: 30px;
141 | vertical-align: middle;
142 | line-height: 40px;
143 | text-align: center;
144 | }
145 |
146 | .btn {
147 | height: 40px;
148 | border-left: none;
149 | border-right: none;
150 | line-height: 40px;
151 | vertical-align: middle;
152 | }
153 | }
154 |
155 | .articlemap {
156 | position: fixed;
157 | right: 40px;
158 | bottom: 0px;
159 | box-shadow: -2px 2px 2px rgba(0,0,0,0.3);
160 |
161 | img {
162 | max-width: 150px;
163 | height: 350px;
164 | }
165 | }
166 | }
167 | }
168 |
169 | .storyfinder-overlay {
170 | position: absolute;
171 | top: 0;
172 | left: 0;
173 | pointer-events: none;
174 | z-index: 999999;
175 |
176 | node {
177 | pointer-events: visiblePainted;
178 | }
179 |
180 | path {
181 | stroke: #999999;
182 | fill: none;
183 | }
184 | }
185 |
186 | sf-source {
187 | position: absolute;
188 | right: 130px;
189 | background-color: #FFFFFF;
190 | border: 1px solid #CCCCCC;
191 | padding: 3px;
192 | z-index: 999998;
193 | max-width: 120px;
194 | font-size: 80%;
195 |
196 | img {
197 | width: 120px;
198 | }
199 | }
200 |
201 | sf-marker-container {
202 | z-index: 999999;
203 |
204 | sf-marker {
205 | display: none;
206 | opacity: 0;
207 | transition: opacity 0.1s;
208 |
209 | .marker-above, .marker-below {
210 | display:none;
211 | }
212 |
213 | &.above, &.below {
214 | position: fixed;
215 | display: inline-block;
216 | opacity: 1;
217 |
218 | padding: 10px;
219 | background-color: rgba(0,0,0,0.8);
220 | color: #FFFFFF;
221 | transition: opacity 0.1s;
222 | }
223 |
224 | &.above {
225 | bottom: auto;
226 | top: 0;
227 | .marker-above {
228 | display: inline-block;
229 | }
230 | }
231 |
232 | &.below {
233 | top: auto;
234 | bottom: 0;
235 | .marker-below {
236 | display: inline-block;
237 | }
238 | }
239 | }
240 | }
241 | }
--------------------------------------------------------------------------------
/webserver/models/ArticleEntity.js:
--------------------------------------------------------------------------------
1 | var DatasourceMysql = require('../datasources/mysql.js')
2 | , async = require('async')
3 | , _ = require('lodash')
4 | ;
5 |
6 | module.exports = function(db){
7 | //console.log('ArticleEntity');
8 | var name = 'ArticleEntity'
9 | , table = 'articles_entities'
10 | , datasource = new DatasourceMysql(db, name, table)
11 | ;
12 |
13 | function getCounts(entities, callback){
14 | datasource.find('list', {
15 | fields: ['entity_id', 'sum(count) `total`'],
16 | conditions: {
17 | entity_id: entities,
18 | is_deleted: 0
19 | },
20 | group: 'entity_id'
21 | }, callback);
22 | }
23 |
24 | this.getCounts = getCounts;
25 |
26 | function getInArticle(articleId, callback){
27 | datasource.find('list', {
28 | fields: ['entity_id', 'sum(count) `total`'],
29 | conditions: {
30 | article_id: articleId,
31 | is_deleted: 0
32 | },
33 | group: 'entity_id'
34 | }, callback);
35 | }
36 |
37 | this.getInArticle = getInArticle;
38 |
39 | function getDocumentfrequency(entityId, callback){
40 | datasource.find('list', {
41 | fields: ['entity_id', 'count(*) `df`'],
42 | conditions: {
43 | entity_id: entityId,
44 | is_deleted: 0
45 | },
46 | group: 'entity_id'
47 | }, callback);
48 | }
49 |
50 | this.getDocumentfrequency = getDocumentfrequency;
51 |
52 | function findArticlesWithEntity(entityId, callback){
53 | datasource.find('list', {
54 | fields: ['article_id', 'id', 'entity_id', 'count'],
55 | conditions: {
56 | entity_id: entityId,
57 | is_deleted: 0
58 | }
59 | }, callback);
60 | }
61 |
62 | this.findArticlesWithEntity = findArticlesWithEntity;
63 |
64 | function reassign(/*target, source, changelogId, callback*/){
65 | console.log('Reassign articleentity');
66 | var targetId = arguments[0]
67 | , sourceId = arguments[1]
68 | , changelogId = arguments[2]
69 | , callback = arguments[arguments.length - 1]
70 | ;
71 |
72 | datasource.find('all', {
73 | fields: ['article_id', 'id', 'entity_id', 'count'],
74 | conditions: {
75 | entity_id: [targetId, sourceId],
76 | is_deleted: 0
77 | }
78 | }, (err, articles) => {
79 | if(err)return setImmediate(() => callback(err));
80 | if(_.isEmpty(articles))return setImmediate(() => callback(null, changelogId));
81 |
82 | var byId = {};
83 |
84 | byId[targetId] = {};
85 | byId[sourceId] = {};
86 |
87 | articles.forEach(article => {
88 | if(article.entity_id == sourceId)
89 | byId[sourceId][article.article_id] = article;
90 | else
91 | byId[targetId][article.article_id] = article;
92 | });
93 |
94 | async.forEachOf(byId[sourceId], (article, article_id, nextArticle) => {
95 | console.log(article_id);
96 |
97 | if(_.isUndefined(byId[targetId][article_id])){
98 | console.log('reassign');
99 | //Target is not connected with the article => move
100 | datasource.update(changelogId, {
101 | values: {
102 | entity_id: targetId
103 | },
104 | conditions: {
105 | id: article.id
106 | },
107 | limit: 1
108 | }, nextArticle);
109 | }else{
110 | console.log('update');
111 | //console.log(byId[targetId][article_id].id)
112 | //Target is already connected with the article => sum
113 | datasource.update(changelogId, { //Sum the count of the article relation of the target node
114 | values: {
115 | count: byId[targetId][article_id].count + article.count
116 | },
117 | conditions: {
118 | id: byId[targetId][article_id].id
119 | },
120 | limit: 1
121 | }, (err) => {
122 | if(err)return setImmediate(() => nextArticle(err));
123 |
124 | console.log('Done Article');
125 |
126 | datasource.update(changelogId, { //Delete the article relation of the source node
127 | values: {
128 | is_deleted: 1
129 | },
130 | conditions: {
131 | id: article.id
132 | },
133 | limit: 1
134 | }, nextArticle);
135 | });
136 | }
137 | }, (err) => {
138 | if(err)
139 | return setImmediate(() => callback(err));
140 |
141 | setImmediate(() => callback(null, changelogId));
142 | });
143 | });
144 | }
145 |
146 | this.reassign = reassign;
147 |
148 | function add(memo, next){
149 | async.eachOfSeries(memo.data.Entities, (entity, key, nextEntity) => {
150 | datasource.insertOrUpdate(memo.changelog_id, {
151 | values: {
152 | article_id: memo.data.Article.id,
153 | entity_id: entity.id,
154 | count: (memo.data.ngrams[entity.tokens - 1][entity.caption] || []).length,
155 | is_deleted: 0
156 | },
157 | conditions: {
158 | article_id: memo.data.Article.id,
159 | entity_id: entity.id
160 | },
161 | update: {
162 | count: (prevEntry, nextEntry) => {
163 | return parseInt(prevEntry) + parseInt(nextEntry)
164 | }
165 | }
166 | }, (err, insertId) => {
167 | if(err)return setImmediate(() => nextEntity(err));
168 |
169 | //entity.id = insertId;
170 | setImmediate(nextEntity);
171 | });
172 | }, (err) => {
173 | if(err)return setImmediate(() => next(err));
174 |
175 | setImmediate(() => next(null, memo));
176 | });
177 | }
178 |
179 | this.add = add;
180 |
181 | function addByEntity(memo, next){
182 | async.eachOfSeries(memo.Articles, (sentences, articleId, nextArticle) => {
183 | datasource.insertOrUpdate(memo.changelog_id, {
184 | values: {
185 | article_id: articleId,
186 | entity_id: memo.Entity.id,
187 | count: sentences.length,
188 | is_deleted: 0
189 | },
190 | conditions: {
191 | article_id: articleId,
192 | entity_id: memo.Entity.id
193 | },
194 | update: {
195 | count: (prevEntry, nextEntry) => {
196 | return parseInt(prevEntry.count) + parseInt(nextEntry.count)
197 | }
198 | }
199 | }, (err, insertId) => {
200 | if(err)return setImmediate(() => nextArticle(err));
201 |
202 | setImmediate(nextArticle);
203 | });
204 | }, (err) => {
205 | if(err)return setImmediate(() => next(err));
206 |
207 | setImmediate(() => next(null, memo));
208 | });
209 | }
210 |
211 | this.addByEntity = addByEntity;
212 | }
213 |
--------------------------------------------------------------------------------
/webserver/public/js/actions/StoryfinderActions.js:
--------------------------------------------------------------------------------
1 | import * as types from '../constants/ActionTypes';
2 |
3 | export function toRelation(entity1_id, entity2_id) {
4 | return {
5 | type: types.TO_RELATION,
6 | entity1_id: entity1_id,
7 | entity2_id: entity2_id
8 | };
9 | }
10 |
11 | export function showRelation(entity1_id, entity2_id) {
12 | return function(dispatch){
13 | dispatch(requestRelation(entity1_id, entity2_id));
14 |
15 | return fetch('/Relations/' + entity1_id + '/' + entity2_id, {
16 | credentials: 'same-origin'
17 | })
18 | .then(response => response.json())
19 | .then(json => {
20 | dispatch(receiveRelation(json));
21 | }).catch(err => {
22 | dispatch(receiveRelation({}));
23 | })
24 | }
25 | }
26 |
27 | export function requestRelation(entity1_id, entity2_id){
28 | return {
29 | type: types.REQUEST_RELATION,
30 | entity1_id: entity1_id,
31 | entity2_id: entity2_id
32 | };
33 | }
34 |
35 | export function receiveRelation(data){
36 | return {
37 | type: types.RECEIVE_RELATION,
38 | data: data
39 | };
40 | }
41 |
42 | export function toGraph(){
43 | return {
44 | type: types.TO_GRAPH
45 | };
46 | }
47 |
48 | export function showGraph(){
49 | return {
50 | type: types.SHOW_GRAPH
51 | };
52 | }
53 |
54 | export function toLocalgraph(site_id){
55 | return {
56 | type: types.TO_LOCALGRAPH,
57 | site_id: site_id
58 | };
59 | }
60 |
61 | export function toGroupgraph(sites){
62 | return {
63 | type: types.TO_GROUPGRAPH,
64 | sites: sites
65 | };
66 | }
67 |
68 | export function showGroupgraph(sites, articles){
69 | return {
70 | type: types.SHOW_GROUPGRAPH,
71 | sites: sites,
72 | articles: articles
73 | }
74 | }
75 |
76 | export function showLocalgraph(site, article){
77 | return {
78 | type: types.SHOW_LOCALGRAPH,
79 | site: site,
80 | article: article
81 | }
82 | }
83 |
84 | export function initializeGlobal() {
85 | return function(dispatch){
86 | dispatch(requestGlobal());
87 |
88 | return fetch('/Globalgraphs')
89 | .then(response => response.json())
90 | .then(json => {
91 | dispatch(receiveGlobal(json));
92 | }).catch(err => {
93 | dispatch(receiveGlobal({}));
94 | })
95 | }
96 | }
97 |
98 | export function requestGlobal(){
99 | return {
100 | type: types.REQUEST_GLOBAL
101 | };
102 | }
103 |
104 | export function receiveGlobal(data){
105 | return {
106 | type: types.RECEIVE_GLOBAL,
107 | data: data
108 | };
109 | }
110 |
111 | export function deleteNode(userId, id) {
112 | return function(dispatch){
113 | dispatch(requestDeleteNode());
114 |
115 | return fetch('/Entities/' + id, {
116 | method: 'DELETE',
117 | credentials: 'same-origin'
118 | })
119 | .then(response => response.json())
120 | .then(json => {
121 | dispatch(receiveDeleteNode(json));
122 | }).catch(err => {
123 | dispatch(receiveDeleteNode({}));
124 | })
125 | }
126 | }
127 |
128 | export function requestDeleteNode(){
129 | return {
130 | type: types.REQUEST_DELETE_NODE
131 | };
132 | }
133 |
134 | export function receiveDeleteNode(data){
135 | return {
136 | type: types.RECEIVE_DELETE_NODE,
137 | data: data
138 | };
139 | }
140 |
141 | export function toGlobal(){
142 | return {
143 | type: types.TO_GLOBAL
144 | }
145 | }
146 |
147 | export function showGlobal(){
148 | return {
149 | type: types.SHOW_GLOBAL
150 | }
151 | }
152 |
153 | export function expandNode(nodeId) {
154 | return function(dispatch){
155 | dispatch(requestNeighbours(nodeId));
156 |
157 | return fetch('http://127.0.0.1:3055/1/nodes/' + nodeId + '/neighbours')
158 | .then(response => response.json())
159 | .then(json => {
160 | dispatch(receiveNeighbours(nodeId, json));
161 | }).catch(err => {
162 | dispatch(receiveNeighbours(nodeId, {}));
163 | })
164 | }
165 | }
166 |
167 | export function requestNeighbours(nodeId){
168 | return {
169 | type: types.REQUEST_NEIGHBOURS,
170 | node: nodeId
171 | }
172 | }
173 |
174 | export function receiveNeighbours(nodeId, data){
175 | return {
176 | type: types.RECEIVE_NEIGHBOURS,
177 | node: nodeId,
178 | data: data
179 | }
180 | }
181 |
182 | export function createNode(data){
183 | return {
184 | type: types.CREATE_NODE,
185 | data: data
186 | }
187 | }
188 |
189 | export function saveNode(nodeData){
190 | return function(dispatch){
191 | dispatch(requestSaveNode());
192 |
193 | return fetch('/Entities', {
194 | method: 'PUT',
195 | headers: {
196 | 'Accept': 'application/json',
197 | 'Content-Type': 'application/json'
198 | },
199 | body: JSON.stringify(nodeData),
200 | credentials: 'same-origin'
201 | })
202 | .then(response => response.json())
203 | .then(json => {
204 | dispatch(receiveSaveNode(json));
205 | }).catch(err => {
206 | dispatch(receiveSaveNode({}));
207 | })
208 | }
209 | }
210 |
211 | export function requestSaveNode(){
212 | return {
213 | type: types.REQUEST_SAVE_NODE
214 | }
215 | }
216 |
217 | export function receiveSaveNode(data){
218 | return {
219 | type: types.RECEIVE_SAVE_NODE,
220 | data: data
221 | }
222 | }
223 |
224 | export function createRelation(entity1, entity2, label){
225 | return function(dispatch){
226 | dispatch(requestCreateRelation());
227 |
228 | return fetch('/Relations/' + entity1 + '/' + entity2, {
229 | method: 'PUT',
230 | headers: {
231 | 'Accept': 'application/json',
232 | 'Content-Type': 'application/json'
233 | },
234 | body: JSON.stringify({
235 | label: label
236 | }),
237 | credentials: 'same-origin'
238 | })
239 | .then(response => response.json())
240 | .then(json => {
241 | dispatch(receiveCreateRelation(json));
242 | }).catch(err => {
243 | dispatch(receiveCreateRelation({}));
244 | })
245 | }
246 | }
247 |
248 | export function requestCreateRelation(){
249 | return {
250 | type: types.REQUEST_CREATE_RELATION
251 | }
252 | }
253 |
254 | export function receiveCreateRelation(data){
255 | return {
256 | type: types.RECEIVE_CREATE_RELATION,
257 | data: data
258 | }
259 | }
260 |
261 | /*export function selectLayer(path, layerId) {
262 | return function(dispatch){
263 | path = path.push(layerId);
264 |
265 | dispatch(requestLayers(path));
266 |
267 | return fetch('http://127.0.0.1:3000/Layers.json?path=' + path.toJS().join('/') + '&fields=data')
268 | .then(response => response.json())
269 | .then(json => {
270 | dispatch(receiveLayers(path, json.data));
271 | })
272 | }
273 | }*/
--------------------------------------------------------------------------------
/webserver/server.js:
--------------------------------------------------------------------------------
1 | var express = require('express')
2 | , app = express()
3 | , http = require('http').Server(app)
4 | , io = require('socket.io')(http)
5 | , bodyParser = require('body-parser')
6 | , cookieSession = require('cookie-session')
7 | , methodOverride = require('method-override')
8 | , serveStatic = require('serve-static')
9 | , port = 3055
10 | , mysql = require('mysql')
11 | , connection = mysql.createConnection({
12 | host : process.env.MYSQL_HOST || "mysql",
13 | port : process.env.MYSQL_PORT || 3306,
14 | user : process.env.MYSQL_USER || 'storyfinder',
15 | password : process.env.MYSQL_PASSWORD || 'storyfinder',
16 | database : process.env.MYSQL_DATABASE || 'storyfinder',
17 | charset: "utf8_general_ci"
18 | })
19 | , fs = require('fs')
20 | , handleVisit = require('./libs/handle_visit.js')
21 | , async = require('async')
22 | , _ = require('lodash')
23 | , passport = require('passport')
24 | , BasicStrategy = require('passport-http').BasicStrategy
25 | , LocalStrategy = require('passport-local').Strategy
26 | , User = new (require('./models/User.js'))(connection)
27 | , ensureLoggedIn = require('connect-ensure-login').ensureLoggedIn
28 | , exphbs = require('express-handlebars')
29 | , tables = {'articles': true,'articles_entities': true,'articles_tokens': true,'changelogs': true,'changelogs_updates': true,'collections': true,'entities': true,'entities_sentences': true,'log_entities': true,'ngrams': true,'relations': true,'relations_sentences': true,'relationtypes': true,'sentences': true,'sites': true,'tokens': true,'users': true,'visits': true}
30 | ;
31 |
32 | function startServer(){
33 | http.listen(port, function () {
34 | console.log('Storyfinder listening on port ' + port);
35 | });
36 | }
37 |
38 | /* Write CHROME_ID to file */
39 | var fs = require('fs')
40 | fs.readFile("public/js/storyfinder.js", 'utf8', function (err,data) {
41 | if (err) {
42 | return console.log(err);
43 | }
44 | var result = data.replace(/INSERT_CHROME_PLUGIN_ID_HERE/g, process.env.CHROME_ID);
45 |
46 | fs.writeFile("public/js/storyfinder.js", result, 'utf8', function (err) {
47 | if (err) return console.log(err);
48 | });
49 | });
50 |
51 | /*
52 | Initialise database.
53 | */
54 | connection.query('SHOW TABLES', (err, result, fields) => {
55 | if(err){
56 | throw err;
57 | return;
58 | }
59 |
60 | var fieldName = fields[0].name;
61 |
62 | for(var row of result) {
63 | var tbl = row[fieldName];
64 |
65 | if(typeof tables[tbl] !== 'undefined')
66 | delete tables[tbl];
67 | }
68 |
69 | if(!_.isEmpty(tables)){
70 | console.log('Some database tables are missing. Creating tables ' + _.keys(tables).join("\n"));
71 | fs.readFile('./data/sql/schema.sql', (err, content) => {
72 | if(err){
73 | throw err;
74 | return;
75 | }
76 |
77 | var queries = content.toString().split("\n\n");
78 |
79 | async.each(queries, (query, nextQuery) => {
80 | connection.query(query, nextQuery);
81 | }, startServer);
82 | });
83 | }else{
84 | setImmediate(startServer);
85 | }
86 | });
87 |
88 | /*
89 | Authentication
90 | */
91 | passport.use(new BasicStrategy(
92 | function(username, password, done){
93 | User.verify(username, password, function(err, isValid, user){
94 | if (err) { return done(err); }
95 | if (!isValid) {
96 | return done(null, false, { message: 'Incorrect username or password.' });
97 | }
98 | return done(null, user);
99 | });
100 | }
101 | ));
102 |
103 | passport.use(new LocalStrategy(
104 | function(username, password, done){
105 | User.verify(username, password, function(err, isValid, user){
106 | if (err) {
107 | console.log('Passport ERROR');
108 | return done(err);
109 | }
110 | if (!isValid) {
111 | console.log('Passport Incorrect username or password');
112 | return done(null, false, { message: 'Incorrect username or password.' });
113 | }
114 | console.log('Passport success!');
115 | return done(null, user);
116 | });
117 | }
118 | ));
119 |
120 | passport.serializeUser(function(user, done) {
121 | if(user == null)
122 | done(null, null);
123 | else
124 | done(null, user.id);
125 | });
126 |
127 | passport.deserializeUser(function(id, done) {
128 | User.findById(id, function(err, user) {
129 | console.log('User deserialized');
130 | done(err, user);
131 | });
132 | });
133 |
134 | function loggedInHTTP(req, res, next){
135 | if (!req.isAuthenticated || !req.isAuthenticated()) {
136 | passport.authenticate('basic', {session: true})(req, res, next);
137 | }else{
138 | next();
139 | }
140 | }
141 |
142 | // parse application/json
143 | app.use(bodyParser.json({limit: '50mb'}))
144 | app.use(bodyParser.urlencoded({ extended: false }))
145 |
146 | app.use(cookieSession({secret: process.env.COOKIE_SECRET, maxAge: 24 * 60 * 60 * 1000 * 365}));
147 |
148 | //MethodOverride
149 | app.use(methodOverride('X-HTTP-Method-Override'))
150 | app.use(methodOverride('_method'))
151 |
152 | //Serve static files
153 | app.use(serveStatic(__dirname + '/public'))
154 | app.use(passport.initialize());
155 | app.use(passport.session());
156 |
157 | var hbs = exphbs.create({
158 | // Specify helpers which are only registered on this instance.
159 | helpers: {
160 | style: function () { return fs.readFileSync(__dirname + '/public/css/bootstrap.css').toString(); }
161 | },
162 | defaultLayout: 'main'
163 | });
164 |
165 | app.engine('handlebars', hbs.engine);
166 |
167 | app.set('view engine', 'handlebars');
168 |
169 | var UsersController = new (require('./controllers/UsersController.js'))(connection, app, passport)
170 | , GraphsController = new (require('./controllers/GraphsController.js'))(connection, app, passport)
171 | , SitesController = new (require('./controllers/SitesController.js'))(connection, app, passport, io)
172 | , EntitiesController = new (require('./controllers/EntitiesController.js'))(connection, app, passport, io)
173 | , ArticlesController = new (require('./controllers/ArticlesController.js'))(connection, app, passport)
174 | , RelationsController = new (require('./controllers/RelationsController.js'))(connection, app, passport)
175 | ;
176 |
177 | app.get('/', ensureLoggedIn((process.env.PATH_PREFIX || '/') + 'login'), function (req, res) {
178 | fs.readFile(__dirname + '/public/css/bootstrap.css', function(err, style){
179 | if(err){
180 | console.log(err);
181 | return;
182 | }
183 | fs.readFile(__dirname + '/public/_index.html', function(err, html){
184 | if(err){
185 | console.log(err);
186 | return;
187 | }
188 | res.send(html.toString().replace(/\{\{style\}\}/, style.toString()));
189 | });
190 | });
191 | });
192 |
193 | app.get('/reseteval', function (req, res) {
194 | var t = ['articles',
195 | 'articles_entities',
196 | 'articles_tokens',
197 | 'changelogs',
198 | 'changelogs_updates',
199 | 'entities',
200 | 'entities_sentences',
201 | 'log_entities',
202 | 'ngrams',
203 | 'relations',
204 | 'relations_sentences',
205 | 'relationtypes',
206 | 'sentences',
207 | 'sites',
208 | 'tokens'];
209 |
210 | for(var i = 0;i < t.length; i++)
211 | connection.query('TRUNCATE ' + t[i]);
212 |
213 | res.send('ok');
214 | });
215 |
--------------------------------------------------------------------------------
/webserver/public/js/libs/shortestpaths.js:
--------------------------------------------------------------------------------
1 | ///
29 |
101 |
30 |
40 |
48 |
56 |
49 |
55 | -
50 |
- 51 | 52 | 53 |
57 |
63 |
73 |
74 |
100 |
75 |
90 |
76 | arrow_back
77 |
88 |
89 | Global network
78 | Subtitle
79 | -
80 |
- 81 | Sources 82 | 83 |
- 84 | Relations 85 | 86 |
91 |
92 |
93 |
94 |
95 |
96 |
97 |
98 |
99 |
102 |
113 |
114 |