├── storyweb ├── analysis │ ├── __init__.py │ ├── html.py │ ├── text.py │ ├── calais.py │ └── extract.py ├── migrate │ ├── alembic.ini │ ├── versions │ │ ├── 353dff346d3_initial_migration.py │ │ └── 176bef6a90a3_basic_schema.py │ ├── script.py.mako │ └── env.py ├── static │ ├── img │ │ ├── edit.png │ │ └── grano.png │ ├── templates │ │ ├── card_icon.html │ │ ├── card_item.html │ │ ├── reference.html │ │ ├── pager.html │ │ ├── card_new.html │ │ ├── article_list.html │ │ ├── search.html │ │ ├── link_new.html │ │ ├── card.html │ │ └── link.html │ ├── js │ │ ├── directives │ │ │ ├── card_icon.js │ │ │ ├── reference.js │ │ │ ├── card_item.js │ │ │ ├── pager.js │ │ │ ├── new_link.js │ │ │ ├── editor.js │ │ │ └── link.js │ │ ├── controllers │ │ │ ├── article_list.js │ │ │ ├── search.js │ │ │ ├── card_new.js │ │ │ ├── app.js │ │ │ └── card.js │ │ └── app.js │ └── style │ │ ├── medium.less │ │ ├── loader.less │ │ └── app.less ├── __init__.py ├── authz.py ├── model │ ├── __init__.py │ ├── forms.py │ ├── spider_tag.py │ ├── util.py │ ├── user.py │ ├── reference.py │ ├── link.py │ └── card.py ├── upgrade.py ├── spiders │ ├── wiki.py │ ├── __init__.py │ ├── util.py │ ├── openduka.py │ └── opencorp.py ├── manage.py ├── search │ ├── queries.py │ ├── __init__.py │ ├── result_proxy.py │ └── mapping.py ├── views │ ├── ui.py │ ├── auth.py │ ├── __init__.py │ ├── references_api.py │ ├── cards_api.py │ ├── links_api.py │ └── admin.py ├── default_settings.py ├── assets.py ├── queue.py ├── templates │ ├── login.html │ ├── app.html │ └── layout.html ├── core.py └── util.py ├── .bowerrc ├── Procfile ├── MANIFEST.in ├── requirements.txt ├── settings.py.tmpl ├── fabric_deploy ├── supervisor.template ├── nginx.template └── fabfile.py ├── setup.py ├── bower.json ├── LICENSE ├── .gitignore ├── app.json └── README.md /storyweb/analysis/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.bowerrc: -------------------------------------------------------------------------------- 1 | { 2 | "directory": "storyweb/static/vendor" 3 | } 4 | -------------------------------------------------------------------------------- /storyweb/migrate/alembic.ini: -------------------------------------------------------------------------------- 1 | [alembic] 2 | script_location=. 3 | -------------------------------------------------------------------------------- /Procfile: -------------------------------------------------------------------------------- 1 | web: gunicorn --preload --log-file - storyweb.manage:app 2 | worker: celery -A storyweb.queue worker 3 | -------------------------------------------------------------------------------- /storyweb/static/img/edit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pudo-attic/storyweb/HEAD/storyweb/static/img/edit.png -------------------------------------------------------------------------------- /storyweb/static/img/grano.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pudo-attic/storyweb/HEAD/storyweb/static/img/grano.png -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE requirements.txt Procfile README.md 2 | recursive-include storyweb/migrate * 3 | global-exclude *.pyc 4 | -------------------------------------------------------------------------------- /storyweb/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | # shut up useless SA warning: 3 | import warnings 4 | warnings.filterwarnings('ignore', 5 | 'Unicode type received non-unicode bind param value.') 6 | from sqlalchemy.exc import SAWarning 7 | warnings.filterwarnings('ignore', category=SAWarning) 8 | -------------------------------------------------------------------------------- /storyweb/authz.py: -------------------------------------------------------------------------------- 1 | from werkzeug.exceptions import Forbidden 2 | from flask.ext.login import current_user 3 | 4 | 5 | def logged_in(): 6 | return current_user.is_authenticated() 7 | 8 | 9 | def require(pred): 10 | if not pred: 11 | raise Forbidden("Sorry, you're not permitted to do this!") 12 | -------------------------------------------------------------------------------- /storyweb/static/templates/card_icon.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /storyweb/static/templates/card_item.html: -------------------------------------------------------------------------------- 1 |
2 |

3 | 4 | {{card.title}} 5 |

6 | 7 |

{{card.summary}}

8 |

9 | {{relDate()}} by 10 | {{card.author.display_name}} 11 |

12 |
13 | -------------------------------------------------------------------------------- /storyweb/static/templates/reference.html: -------------------------------------------------------------------------------- 1 |
2 | {{reference.citation}} 3 | 4 | {{reference.score}}% match from 5 | {{reference.source}} 6 | 7 |
8 | -------------------------------------------------------------------------------- /storyweb/migrate/versions/353dff346d3_initial_migration.py: -------------------------------------------------------------------------------- 1 | """initial migration 2 | 3 | Revision ID: 353dff346d3 4 | Revises: None 5 | Create Date: 2014-12-12 22:09:09.696427 6 | 7 | """ 8 | 9 | # revision identifiers, used by Alembic. 10 | revision = '353dff346d3' 11 | down_revision = None 12 | 13 | 14 | def upgrade(): 15 | pass 16 | 17 | 18 | def downgrade(): 19 | pass 20 | -------------------------------------------------------------------------------- /storyweb/static/js/directives/card_icon.js: -------------------------------------------------------------------------------- 1 | storyweb.directive('storywebCardIcon', ['$http', 'cfpLoadingBar', function($http, cfpLoadingBar) { 2 | return { 3 | restrict: 'E', 4 | transclude: true, 5 | scope: { 6 | 'category': '=' 7 | }, 8 | templateUrl: 'card_icon.html', 9 | link: function (scope, element, attrs, model) { 10 | } 11 | }; 12 | }]); 13 | -------------------------------------------------------------------------------- /storyweb/static/js/directives/reference.js: -------------------------------------------------------------------------------- 1 | storyweb.directive('storywebReference', ['$http', function($http) { 2 | return { 3 | restrict: 'E', 4 | transclude: true, 5 | scope: { 6 | 'story': '=', 7 | 'card': '=', 8 | 'reference': '=' 9 | }, 10 | templateUrl: 'reference.html', 11 | link: function (scope, element, attrs, model) { 12 | 13 | } 14 | }; 15 | }]); 16 | -------------------------------------------------------------------------------- /storyweb/model/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from storyweb.core import db 3 | from storyweb.model.user import User 4 | from storyweb.model.card import Card, Alias # noqa 5 | from storyweb.model.link import Link # noqa 6 | from storyweb.model.reference import Reference # noqa 7 | from storyweb.model.spider_tag import SpiderTag # noqa 8 | 9 | log = logging.getLogger(__name__) 10 | 11 | 12 | def make_fixtures(): 13 | User.default_user() 14 | db.session.commit() 15 | -------------------------------------------------------------------------------- /storyweb/static/js/controllers/article_list.js: -------------------------------------------------------------------------------- 1 | storyweb.controller('ArticleListCtrl', ['$scope', '$location', '$http', 'cfpLoadingBar', 2 | function($scope, $location, $http, cfpLoadingBar) { 3 | 4 | $scope.articles = []; 5 | 6 | cfpLoadingBar.start(); 7 | var params = {'category': 'Article'}; 8 | $http.get('/api/1/cards', {params: params}).then(function(res) { 9 | $scope.articles = res.data.results; 10 | cfpLoadingBar.complete(); 11 | }); 12 | 13 | }]); 14 | 15 | -------------------------------------------------------------------------------- /storyweb/static/templates/pager.html: -------------------------------------------------------------------------------- 1 | 2 | 9 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | Flask==0.10.1 2 | Flask-Admin==1.0.8 3 | Flask-Assets==0.10 4 | Flask-Login==0.2.11 5 | Flask-SQLAlchemy==2.0 6 | Flask-Migrate==1.3.0 7 | Flask-Script==2.0.5 8 | PyYAML==3.11 9 | colander==1.0 10 | lxml==3.4.0 11 | mistune==0.4.1 12 | pyelasticsearch==0.7.1 13 | requests==2.4.3 14 | simplejson==3.6.5 15 | timestring==1.6.1 16 | restpager==0.1 17 | python-slugify==0.1.0 18 | python-dateutil==2.3 19 | gunicorn 20 | psycopg2 21 | fabric 22 | celery==3.1.17 23 | wikipedia==1.4.0 24 | python-Levenshtein==0.11.2 25 | -------------------------------------------------------------------------------- /settings.py.tmpl: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | APP_NAME = 'storyweb' 4 | APP_TITLE = 'Grano StoryWeb' 5 | APP_DESCRIPTION = 'You know, an IDE for news!' 6 | 7 | # MOTD = 'A message shown at login.' 8 | 9 | DEBUG = False 10 | ASSETS_DEBUG = DEBUG 11 | SECRET_KEY = 'banana pancakes' 12 | SQLALCHEMY_DATABASE_URI = 'sqlite:///storyweb.sqlite3' 13 | ELASTICSEARCH_URL = 'http://localhost:9200' 14 | 15 | OPENCORPORATES_KEY = '' 16 | CALAIS_KEY = '' 17 | 18 | CELERY_ALWAYS_EAGER = False 19 | CELERY_BROKER_URL = 'amqp://guest:guest@localhost:5672//' 20 | -------------------------------------------------------------------------------- /storyweb/migrate/script.py.mako: -------------------------------------------------------------------------------- 1 | """${message} 2 | 3 | Revision ID: ${up_revision} 4 | Revises: ${down_revision} 5 | Create Date: ${create_date} 6 | 7 | """ 8 | 9 | # revision identifiers, used by Alembic. 10 | revision = ${repr(up_revision)} 11 | down_revision = ${repr(down_revision)} 12 | 13 | from alembic import op 14 | import sqlalchemy as sa 15 | ${imports if imports else ""} 16 | 17 | def upgrade(): 18 | ${upgrades if upgrades else "pass"} 19 | 20 | 21 | def downgrade(): 22 | ${downgrades if downgrades else "pass"} 23 | -------------------------------------------------------------------------------- /storyweb/upgrade.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from flask.ext import migrate 3 | 4 | from storyweb.model import make_fixtures 5 | from storyweb.search import init_search, reindex 6 | 7 | 8 | log = logging.getLogger(__name__) 9 | 10 | 11 | def upgrade(): 12 | log.info("Beginning database migration...") 13 | migrate.upgrade() 14 | log.info("Ensuring database fixtures exist...") 15 | make_fixtures() 16 | log.info("Reconfiguring the search index...") 17 | init_search() 18 | log.info("Re-indexing any existing database objects...") 19 | reindex() 20 | -------------------------------------------------------------------------------- /storyweb/analysis/html.py: -------------------------------------------------------------------------------- 1 | from lxml.html.clean import Cleaner 2 | 3 | TAGS = ['br', 'p', 'hr', 'b', 'strong', 'em', 'i', 'a', 4 | 'blockquote', 'ul', 'li'] 5 | ATTRS = ['href', 'target'] 6 | 7 | cleaner = Cleaner(style=False, links=True, add_nofollow=False, 8 | page_structure=False, safe_attrs_only=True, 9 | remove_unknown_tags=False, comments=False, 10 | safe_attrs=ATTRS, allow_tags=TAGS) 11 | 12 | 13 | def clean_html(html): 14 | if html is None or not len(html): 15 | return '

' 16 | return cleaner.clean_html(html) 17 | -------------------------------------------------------------------------------- /storyweb/static/templates/card_new.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |
4 |

Create a new {{card.category}}

5 |
6 |
7 | 8 |
9 |
10 | 11 |
12 |
13 |
14 | -------------------------------------------------------------------------------- /storyweb/static/js/directives/card_item.js: -------------------------------------------------------------------------------- 1 | storyweb.directive('storywebCardItem', ['$http', '$location', function($http, $location) { 2 | return { 3 | restrict: 'E', 4 | transclude: true, 5 | scope: { 6 | 'card': '=' 7 | }, 8 | templateUrl: 'card_item.html', 9 | link: function (scope, element, attrs, model) { 10 | scope.visit = function() { 11 | $location.search({}); 12 | $location.path('/cards/' + scope.card.id); 13 | } 14 | 15 | scope.relDate = function() { 16 | return moment(scope.card.updated_at).fromNow(); 17 | } 18 | } 19 | }; 20 | }]); 21 | -------------------------------------------------------------------------------- /storyweb/spiders/wiki.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import wikipedia 3 | 4 | from storyweb.spiders.util import Spider 5 | 6 | 7 | log = logging.getLogger(__name__) 8 | 9 | 10 | class Wikipedia(Spider): 11 | 12 | def search_all(self, card): 13 | try: 14 | if card.text is None or not len(card.text.strip()): 15 | text = wikipedia.summary(card.title) 16 | if text is not None and len(text): 17 | text = text.split('\n', 1)[0] 18 | card.text = '

%s

' % text 19 | except wikipedia.WikipediaException, pe: 20 | log.exception(pe) 21 | -------------------------------------------------------------------------------- /storyweb/manage.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from flask.ext.script import Manager 3 | from flask.ext.assets import ManageAssets 4 | from flask.ext.migrate import MigrateCommand 5 | 6 | from storyweb.views import app, assets 7 | from storyweb.upgrade import upgrade as upgrade_ 8 | 9 | 10 | log = logging.getLogger(__name__) 11 | manager = Manager(app) 12 | manager.add_command("assets", ManageAssets(assets)) 13 | manager.add_command('db', MigrateCommand) 14 | 15 | 16 | @manager.command 17 | def upgrade(): 18 | """ Upgrade the database and re-index search. """ 19 | upgrade_() 20 | 21 | 22 | if __name__ == "__main__": 23 | manager.run() 24 | -------------------------------------------------------------------------------- /storyweb/static/js/controllers/search.js: -------------------------------------------------------------------------------- 1 | storyweb.controller('SearchCtrl', ['$scope', '$location', '$http', 'cfpLoadingBar', 2 | function($scope, $location, $http, cfpLoadingBar) { 3 | 4 | $scope.search = $location.search(); 5 | $scope.result = {'results': []}; 6 | 7 | $scope.loadPage = function(page) { 8 | $location.search('offset', $scope.result.limit * (page-1)); 9 | }; 10 | 11 | $http.get('/api/1/cards/_search', {'params': $scope.search}).then(function(res) { 12 | $scope.result = res.data; 13 | $scope.result.end = Math.min($scope.result.total, $scope.result.offset + $scope.result.limit); 14 | }); 15 | }]); 16 | -------------------------------------------------------------------------------- /fabric_deploy/supervisor.template: -------------------------------------------------------------------------------- 1 | [program:%(server-name)s-web] 2 | environment=STORYWEB_SETTINGS='%(deploy-dir)ssettings.py' 3 | directory=%(project-dir)s 4 | command=%(ve-dir)s/bin/gunicorn --log-file - -b %(host)s:%(port)s storyweb.manage:app -w 5 5 | user=%(user)s 6 | stdout_logfile=%(gunicorn-log)s 7 | stderr_logfile=%(gunicorn-err-log)s 8 | stopsignal=QUIT 9 | 10 | [program:%(server-name)s-worker] 11 | environment=STORYWEB_SETTINGS='%(deploy-dir)ssettings.py' 12 | directory=%(project-dir)s 13 | command=%(ve-dir)s/bin/celery -A storyweb.queue worker 14 | user=%(user)s 15 | stdout_logfile=%(celery-log)s 16 | stderr_logfile=%(celery-err-log)s 17 | stopsignal=QUIT 18 | -------------------------------------------------------------------------------- /storyweb/analysis/text.py: -------------------------------------------------------------------------------- 1 | import re 2 | from unicodedata import normalize as ucnorm, category 3 | 4 | REMOVE_SPACES = re.compile(r'\s+') 5 | 6 | 7 | def normalize(text): 8 | if not isinstance(text, unicode): 9 | text = unicode(text) 10 | chars = [] 11 | for char in ucnorm('NFKD', text): 12 | cat = category(char)[0] 13 | if cat in ['C', 'Z', 'S']: 14 | chars.append(u' ') 15 | elif cat in ['M', 'P']: 16 | continue 17 | else: 18 | chars.append(char) 19 | text = u''.join(chars) 20 | text = REMOVE_SPACES.sub(' ', text) 21 | text = text.strip().lower() 22 | #return ucnorm('NFKC', text) 23 | return text 24 | -------------------------------------------------------------------------------- /storyweb/search/queries.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | def cards_query(req): 4 | qstr = req.get('q', '').strip() 5 | if len(qstr): 6 | q = {'query_string': {'query': qstr}} 7 | bq = [ 8 | {"term": {"title": {"value": qstr, "boost": 10.0}}}, 9 | {"term": {"aliases": {"value": qstr, "boost": 6.0}}}, 10 | {"term": {"text": {"value": qstr, "boost": 3.0}}} 11 | ] 12 | q = { 13 | "bool": { 14 | "must": q, 15 | "should": bq 16 | } 17 | } 18 | else: 19 | q = {'match_all': {}} 20 | return { 21 | 'query': q, 22 | '_source': ['title', 'category', 'summary', 'id', 'updated_at', 'author'] 23 | } 24 | -------------------------------------------------------------------------------- /storyweb/static/templates/article_list.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |

4 |   5 |

6 |
7 |
8 |
9 | 10 | New article 11 | 12 |
13 |
14 |
15 | 16 |
17 |
18 |
19 | 20 |
21 |
22 |
23 |
24 |
25 | -------------------------------------------------------------------------------- /storyweb/static/js/controllers/card_new.js: -------------------------------------------------------------------------------- 1 | storyweb.controller('CardNewCtrl', ['$scope', '$routeParams', '$location', '$interval', '$http', 'cfpLoadingBar', 2 | function($scope, $routeParams, $location, $interval, $http, cfpLoadingBar) { 3 | $scope.card = { 4 | 'title': '', 5 | 'category': $location.search().category, 6 | }; 7 | 8 | $scope.canSubmit = function() { 9 | return $scope.card.title && $scope.card.title.length > 1 && $scope.card.category; 10 | }; 11 | 12 | $scope.saveCard = function () { 13 | cfpLoadingBar.start(); 14 | $http.post('/api/1/cards', $scope.card).then(function(res) { 15 | cfpLoadingBar.complete(); 16 | $location.path('/cards/' + res.data.id); 17 | }); 18 | }; 19 | 20 | }]); 21 | -------------------------------------------------------------------------------- /storyweb/views/ui.py: -------------------------------------------------------------------------------- 1 | import os 2 | from flask import render_template, redirect, url_for 3 | 4 | from storyweb.core import app 5 | from storyweb import authz 6 | 7 | 8 | def angular_templates(): 9 | partials_dir = os.path.join(app.static_folder, 'templates') 10 | for (root, dirs, files) in os.walk(partials_dir): 11 | for file_name in files: 12 | file_path = os.path.join(root, file_name) 13 | with open(file_path, 'rb') as fh: 14 | file_name = file_path[len(partials_dir) + 1:] 15 | yield (file_name, fh.read().decode('utf-8')) 16 | 17 | 18 | @app.route('/app/') 19 | def ui(): 20 | if not authz.logged_in(): 21 | return redirect(url_for('login')) 22 | return render_template("app.html", templates=angular_templates()) 23 | -------------------------------------------------------------------------------- /storyweb/model/forms.py: -------------------------------------------------------------------------------- 1 | import colander 2 | from colander import Invalid # noqa 3 | 4 | 5 | class LoginForm(colander.MappingSchema): 6 | email = colander.SchemaNode(colander.String(), 7 | validator=colander.Email()) 8 | password = colander.SchemaNode(colander.String()) 9 | 10 | 11 | class Ref(object): 12 | def serialize(self, node, appstruct): 13 | if appstruct is colander.null: 14 | return colander.null 15 | return None 16 | 17 | def deserialize(self, node, cstruct): 18 | if cstruct is colander.null: 19 | return colander.null 20 | value = self.decode(cstruct) 21 | if value is None: 22 | raise colander.Invalid(node, 'Missing') 23 | return value 24 | 25 | def cstruct_children(self, node, cstruct): 26 | return [] 27 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | setup( 4 | name='storyweb', 5 | version='0.1', 6 | description="Create story networks out of text snippets.", 7 | long_description="", 8 | classifiers=[ 9 | "Development Status :: 3 - Alpha", 10 | "Intended Audience :: Developers", 11 | "Operating System :: OS Independent", 12 | "Programming Language :: Python", 13 | ], 14 | keywords='sql influence networks journalism ddj entities', 15 | author='Annabel Church, Friedrich Lindenberg', 16 | author_email='friedrich@pudo.org', 17 | url='http://granoproject.org', 18 | license='MIT', 19 | packages=find_packages(exclude=['ez_setup', 'examples', 'tests']), 20 | namespace_packages=[], 21 | include_package_data=True, 22 | zip_safe=False, 23 | install_requires=[], 24 | entry_points={}, 25 | tests_require=[] 26 | ) 27 | -------------------------------------------------------------------------------- /fabric_deploy/nginx.template: -------------------------------------------------------------------------------- 1 | server { 2 | listen %(server-port)s; 3 | server_name %(server-name)s; 4 | client_max_body_size 128m; 5 | gzip_types application/x-javascript text/css; 6 | 7 | location ^~ /static/ { 8 | alias %(static-path)s; 9 | expires 31d; 10 | } 11 | 12 | location / { 13 | proxy_pass_header Server; 14 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 15 | proxy_set_header Host $http_host; 16 | proxy_redirect off; 17 | proxy_set_header X-Real-IP $remote_addr; 18 | proxy_set_header X-Scheme $scheme; 19 | proxy_set_header X-Site-Type web; 20 | proxy_connect_timeout 120; 21 | proxy_read_timeout 120; 22 | proxy_pass http://%(proxy-host)s:%(proxy-port)s/; 23 | proxy_intercept_errors on; 24 | keepalive_timeout 0; 25 | } 26 | 27 | access_log %(log)s; 28 | error_log %(err-log)s; 29 | } 30 | -------------------------------------------------------------------------------- /bower.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "storyweb", 3 | "version": "0.1.0", 4 | "homepage": "https://github.com/granoproject/storyweb", 5 | "authors": [ 6 | "Annabel Church", 7 | "Friedrich Lindenberg" 8 | ], 9 | "description": "This is what you do when you have too much information", 10 | "license": "MIT", 11 | "private": true, 12 | "ignore": [ 13 | "**/.*", 14 | "node_modules", 15 | "bower_components", 16 | "storyweb/static/vendor", 17 | "test", 18 | "tests" 19 | ], 20 | "dependencies": { 21 | "bootstrap": "~3.3.1", 22 | "angular": "~1.2.27", 23 | "angular-route": "~1.2.27", 24 | "angular-animate": "~1.2.27", 25 | "angular-bootstrap": "~0.12.0", 26 | "angular-loading-bar": "~0.6.0", 27 | "angular-contenteditable": "~0.3.7", 28 | "angular-truncate": "*", 29 | "moment": "~2.8.4", 30 | "medium-editor": "~1.9.13" 31 | }, 32 | "resolutions": { 33 | "medium-editor": "~1.9.13" 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /storyweb/static/templates/search.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |

4 | Found {{result.total}} cards 5 | for {{search.q}} 6 | in total. 7 |

8 |

9 | No matches. 10 |

11 |
12 |
13 |
14 | 15 | New article 16 | 17 |
18 |
19 |
20 | 21 |
22 |
23 |
24 | 25 |
26 |
27 |
28 |
29 |
30 | 31 | 32 | -------------------------------------------------------------------------------- /storyweb/spiders/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from storyweb.model import SpiderTag, Card 4 | 5 | from storyweb.spiders.openduka import OpenDuka 6 | from storyweb.spiders.wiki import Wikipedia 7 | from storyweb.spiders.opencorp import OpenCorporates 8 | 9 | log = logging.getLogger(__name__) 10 | 11 | SPIDERS = { 12 | 'OpenDuka': OpenDuka, 13 | 'OpenCorporates': OpenCorporates, 14 | 'Wikipedia': Wikipedia 15 | } 16 | 17 | 18 | def lookup(card, spider_name): 19 | if SpiderTag.find(card, spider_name): 20 | return 21 | 22 | cls = SPIDERS.get(spider_name) 23 | try: 24 | spider = cls() 25 | if card.category == Card.PERSON: 26 | spider.search_person(card) 27 | elif card.category == Card.COMPANY: 28 | spider.search_company(card) 29 | elif card.category == Card.ORGANIZATION: 30 | spider.search_organization(card) 31 | else: 32 | spider.search_generic(card) 33 | except Exception, e: 34 | log.exception(e) 35 | 36 | SpiderTag.done(card, spider_name) 37 | -------------------------------------------------------------------------------- /storyweb/static/js/app.js: -------------------------------------------------------------------------------- 1 | var storyweb = angular.module('storyweb', ['ngRoute', 'ngAnimate', 'ui.bootstrap', 'angular-loading-bar', 'truncate']); 2 | 3 | storyweb.config(['cfpLoadingBarProvider', function(cfpLoadingBarProvider) { 4 | cfpLoadingBarProvider.includeSpinner = false; 5 | cfpLoadingBarProvider.latencyThreshold = 100; 6 | }]); 7 | 8 | storyweb.config(['$routeProvider', '$locationProvider', 9 | function($routeProvider, $locationProvider) { 10 | 11 | $routeProvider.when('/', { 12 | templateUrl: 'article_list.html', 13 | controller: 'ArticleListCtrl' 14 | }); 15 | 16 | $routeProvider.when('/cards/:id', { 17 | templateUrl: 'card.html', 18 | controller: 'CardCtrl' 19 | }); 20 | 21 | $routeProvider.when('/search', { 22 | templateUrl: 'search.html', 23 | controller: 'SearchCtrl' 24 | }); 25 | 26 | $routeProvider.when('/new', { 27 | templateUrl: 'card_new.html', 28 | controller: 'CardNewCtrl' 29 | }); 30 | 31 | $routeProvider.otherwise({ 32 | redirectTo: '/' 33 | }); 34 | 35 | $locationProvider.html5Mode(false); 36 | }]); 37 | -------------------------------------------------------------------------------- /storyweb/static/templates/link_new.html: -------------------------------------------------------------------------------- 1 |
2 |

3 |
4 |
5 |

Add a card for a {{card.category}}

6 |
7 |
8 |
12 | 13 |
14 |
15 |
16 | 19 |
20 |
21 | 22 |
23 |
24 |
25 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Friedrich Lindenberg 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .env 2 | *.DS_Store 3 | *.sqlite3 4 | *.sqlite3-journal 5 | settings.py 6 | storyweb/static/vendor/* 7 | storyweb/static/.webassets-cache/* 8 | legacy/* 9 | demo/data/* 10 | demo/calais/* 11 | 12 | # Byte-compiled / optimized / DLL files 13 | __pycache__/ 14 | *.py[cod] 15 | 16 | # C extensions 17 | *.so 18 | 19 | # Distribution / packaging 20 | .Python 21 | env/ 22 | build/ 23 | develop-eggs/ 24 | dist/ 25 | downloads/ 26 | eggs/ 27 | lib/ 28 | lib64/ 29 | parts/ 30 | sdist/ 31 | var/ 32 | *.egg-info/ 33 | .installed.cfg 34 | *.egg 35 | 36 | # PyInstaller 37 | # Usually these files are written by a python script from a template 38 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 39 | *.manifest 40 | *.spec 41 | 42 | # Installer logs 43 | pip-log.txt 44 | pip-delete-this-directory.txt 45 | 46 | # Unit test / coverage reports 47 | htmlcov/ 48 | .tox/ 49 | .coverage 50 | .cache 51 | nosetests.xml 52 | coverage.xml 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | 61 | # Sphinx documentation 62 | docs/_build/ 63 | 64 | # PyBuilder 65 | target/ 66 | 67 | # Node 68 | node_modules 69 | -------------------------------------------------------------------------------- /storyweb/static/js/controllers/app.js: -------------------------------------------------------------------------------- 1 | 2 | storyweb.controller('AppCtrl', ['$scope', '$location', '$http', 'cfpLoadingBar', 3 | function($scope, $location, $http, cfpLoadingBar) { 4 | 5 | $scope.searchQuery = ''; 6 | $scope.session = {'logged_in': false, 'is_admin': false}; 7 | 8 | $scope.search = function() { 9 | $location.search('q', $scope.searchQuery); 10 | $location.search('offset', null); 11 | $location.path('/search'); 12 | }; 13 | 14 | $scope.$on('$locationChangeSuccess', function(event) { 15 | $scope.searchQuery = $location.search().q || ''; 16 | }); 17 | 18 | $scope.searchGo = function($item) { 19 | $location.search('q', null); 20 | $location.path('/cards/' + $item.id); 21 | }; 22 | 23 | $scope.suggestCard = function(q) { 24 | var params = {'prefix': q}; 25 | return $http.get('/api/1/cards/_suggest', {'params': params}).then(function(res) { 26 | return res.data.options; 27 | }); 28 | }; 29 | 30 | $http.get('/api/1/session').then(function(res) { 31 | $scope.session = res.data; 32 | if (!$scope.session.logged_in) { 33 | document.location.href = '/'; 34 | } 35 | }); 36 | 37 | }]); 38 | -------------------------------------------------------------------------------- /storyweb/spiders/util.py: -------------------------------------------------------------------------------- 1 | from Levenshtein import distance 2 | 3 | SCORE_CUTOFF = 50 4 | 5 | 6 | def light_normalize(text): 7 | text = unicode(text) 8 | text = text.strip().lower() 9 | return text 10 | 11 | 12 | def text_score(match, candidates): 13 | if isinstance(candidates, basestring): 14 | candidates = [candidates] 15 | match_n = light_normalize(match) 16 | best_score = 0 17 | for candidate in candidates: 18 | cand_n = light_normalize(candidate) 19 | dist = float(distance(match_n, cand_n)) 20 | l = float(max(len(match_n), len(cand_n))) 21 | score = ((l - dist) / l) * 100 22 | best_score = max(int(score), best_score) 23 | return best_score 24 | 25 | 26 | class Spider(object): 27 | 28 | def search_generic(self, card): 29 | return self.search_all(card) 30 | 31 | def search_person(self, card): 32 | return self.search_all(card) 33 | 34 | def search_company(self, card): 35 | return self.search_all(card) 36 | 37 | def search_organization(self, card): 38 | return self.search_all(card) 39 | 40 | def search_all(self, card): 41 | return card 42 | -------------------------------------------------------------------------------- /storyweb/model/spider_tag.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | 3 | from storyweb.core import db 4 | 5 | 6 | class SpiderTag(db.Model): 7 | id = db.Column(db.Integer, primary_key=True) 8 | spider = db.Column(db.Unicode) 9 | 10 | card_id = db.Column(db.Integer(), db.ForeignKey('card.id')) 11 | card = db.relationship('Card') 12 | 13 | created_at = db.Column(db.DateTime, default=datetime.utcnow) 14 | updated_at = db.Column(db.DateTime, default=datetime.utcnow, 15 | onupdate=datetime.utcnow) 16 | 17 | def __repr__(self): 18 | return '' % (self.id, self.card, 19 | self.spider, self.status) 20 | 21 | @classmethod 22 | def find(cls, card, spider): 23 | q = db.session.query(cls) 24 | q = q.filter(cls.card == card) 25 | q = q.filter(cls.spider == spider) 26 | return q.first() 27 | 28 | @classmethod 29 | def done(cls, card, spider): 30 | obj = cls.find(card, spider) 31 | if obj is None: 32 | obj = cls() 33 | obj.card = card 34 | obj.spider = spider 35 | db.session.add(obj) 36 | -------------------------------------------------------------------------------- /storyweb/default_settings.py: -------------------------------------------------------------------------------- 1 | from os import environ as env, path, getcwd 2 | 3 | APP_NAME = 'storyweb' 4 | APP_TITLE = env.get('APP_TITLE', 'StoryWeb') 5 | APP_DESCRIPTION = env.get('APP_DESCRIPTION', 'networked data for news in context') 6 | 7 | MOTD = ''' 8 | This is a demo instance. You can log in using the username 9 | admin@grano.cc and the password admin. 10 | ''' 11 | 12 | DEBUG = env.get('DEBUG', '').lower().strip() 13 | DEBUG = DEBUG not in ['no', 'false', '0'] 14 | ASSETS_DEBUG = DEBUG 15 | SECRET_KEY = env.get('SECRET_KEY', 'banana pancakes') 16 | 17 | ALEMBIC_DIR = path.join(path.dirname(__file__), 'migrate') 18 | ALEMBIC_DIR = path.abspath(ALEMBIC_DIR) 19 | 20 | db_uri = 'sqlite:///%s.sqlite3' % path.join(getcwd(), APP_NAME) 21 | SQLALCHEMY_DATABASE_URI = env.get('DATABASE_URL', db_uri) 22 | ELASTICSEARCH_URL = env.get('BONSAI_URL', 'http://localhost:9200') 23 | 24 | OPENCORPORATES_KEY = env.get('OPENCORPORATES_KEY') 25 | CALAIS_KEY = env.get('CALAIS_KEY') 26 | 27 | CELERY_ALWAYS_EAGER = False 28 | CELERY_TASK_SERIALIZER = 'json' 29 | CELERY_ACCEPT_CONTENT = ['json'] 30 | CELERY_TIMEZONE = 'UTC' 31 | CELERY_BROKER_URL = env.get('RABBITMQ_BIGWIG_URL', 32 | 'amqp://guest:guest@localhost:5672//') 33 | -------------------------------------------------------------------------------- /storyweb/migrate/env.py: -------------------------------------------------------------------------------- 1 | from __future__ import with_statement 2 | from alembic import context 3 | from sqlalchemy import engine_from_config, pool 4 | 5 | from storyweb.core import app 6 | from storyweb.model import db 7 | 8 | config = context.config 9 | config.set_main_option('sqlalchemy.url', app.config['SQLALCHEMY_DATABASE_URI']) 10 | target_metadata = db.metadata 11 | 12 | 13 | def run_migrations_offline(): 14 | url = config.get_main_option("sqlalchemy.url") 15 | context.configure(url=url) 16 | 17 | with context.begin_transaction(): 18 | context.run_migrations() 19 | 20 | 21 | def run_migrations_online(): 22 | engine = engine_from_config( 23 | config.get_section(config.config_ini_section), 24 | prefix='sqlalchemy.', 25 | poolclass=pool.NullPool) 26 | 27 | connection = engine.connect() 28 | context.configure( 29 | connection=connection, 30 | target_metadata=target_metadata 31 | ) 32 | 33 | try: 34 | with context.begin_transaction(): 35 | context.run_migrations() 36 | finally: 37 | connection.close() 38 | 39 | if context.is_offline_mode(): 40 | run_migrations_offline() 41 | else: 42 | run_migrations_online() 43 | 44 | -------------------------------------------------------------------------------- /storyweb/search/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from pyelasticsearch.exceptions import ElasticHttpNotFoundError 3 | 4 | from storyweb.core import es, es_index, db 5 | from storyweb.util import AppEncoder 6 | from storyweb.model.card import Card 7 | from storyweb.search.mapping import CARD_MAPPING 8 | from storyweb.search.result_proxy import ESResultProxy 9 | from storyweb.search.queries import cards_query # noqa 10 | 11 | log = logging.getLogger(__name__) 12 | 13 | 14 | def init_search(): 15 | try: 16 | es.delete_index(es_index) 17 | except ElasticHttpNotFoundError: 18 | pass 19 | es.create_index(es_index) 20 | log.info("Creating ElasticSearch index and uploading mapping...") 21 | es.put_mapping(es_index, Card.doc_type, {Card.doc_type: CARD_MAPPING}) 22 | 23 | 24 | def index_card(card): 25 | es.json_encoder = AppEncoder 26 | body = card.to_index() 27 | es.index(es_index, Card.doc_type, body, card.id) 28 | 29 | 30 | def reindex(): 31 | cards = db.session.query(Card) 32 | cards = cards.order_by(Card.created_at.desc()) 33 | for card in cards.yield_per(500): 34 | log.info("Indexing %r", card) 35 | index_card(card) 36 | es.refresh(index=es_index) 37 | 38 | 39 | def search_cards(query): 40 | return ESResultProxy(Card.doc_type, query) 41 | -------------------------------------------------------------------------------- /app.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Grano StoryWeb", 3 | "description": "An investigative research manager.", 4 | "keywords": [ 5 | "journalism", 6 | "investigative", 7 | "research" 8 | ], 9 | "website": "http://granoproject.org/", 10 | "repository": "https://github.com/granoproject/storyweb", 11 | "logo": "http://granoproject.org/static/images/logo.png", 12 | "success_url": "/", 13 | "scripts": { 14 | }, 15 | "env": { 16 | "APP_TITLE": { 17 | "description": "A title for this instance, e.g. the name of the news org.", 18 | "value": "Grano StoryWeb" 19 | }, 20 | "PROJECT_DESCRIPTION": { 21 | "description": "A description for this instance." 22 | }, 23 | "DEBUG": { 24 | "description": "Whether to run in development mode.", 25 | "value": "false" 26 | }, 27 | "SECRET_KEY": { 28 | "description": "A secret key for verifying the integrity of signed cookies.", 29 | "generator": "secret" 30 | }, 31 | "CALAIS_KEY": { 32 | "description": "An API key for Reuter's OpenCalais API." 33 | }, 34 | "OPENCORPORATES_KEY": { 35 | "description": "An API key for the OpenCorporates API." 36 | } 37 | }, 38 | "addons": [ 39 | "heroku-postgresql:hobby-basic", 40 | "rabbitmq-bigwig:pipkin", 41 | "bonsai" 42 | ] 43 | } 44 | -------------------------------------------------------------------------------- /storyweb/assets.py: -------------------------------------------------------------------------------- 1 | from flask.ext.assets import Bundle 2 | 3 | from storyweb.core import assets 4 | 5 | js_assets = Bundle( 6 | 'vendor/moment/moment.js', 7 | 'vendor/medium-editor/dist/js/medium-editor.js', 8 | 'vendor/angular/angular.js', 9 | 'vendor/angular-route/angular-route.js', 10 | 'vendor/angular-animate/angular-animate.js', 11 | 'vendor/angular-contenteditable/angular-contenteditable.js', 12 | 'vendor/angular-truncate/src/truncate.js', 13 | 'vendor/angular-bootstrap/ui-bootstrap.js', 14 | 'vendor/angular-bootstrap/ui-bootstrap-tpls.js', 15 | 'vendor/angular-loading-bar/src/loading-bar.js', 16 | 'js/app.js', 17 | 'js/controllers/app.js', 18 | 'js/controllers/article_list.js', 19 | 'js/controllers/card.js', 20 | 'js/controllers/card_new.js', 21 | 'js/controllers/search.js', 22 | 'js/directives/card_icon.js', 23 | 'js/directives/card_item.js', 24 | 'js/directives/editor.js', 25 | 'js/directives/link.js', 26 | 'js/directives/pager.js', 27 | 'js/directives/new_link.js', 28 | 'js/directives/reference.js', 29 | filters='uglifyjs', 30 | output='assets/app.js' 31 | ) 32 | 33 | css_assets = Bundle( 34 | 'style/app.less', 35 | filters='less', 36 | output='assets/style.css' 37 | ) 38 | 39 | assets.register('js', js_assets) 40 | assets.register('css', css_assets) 41 | -------------------------------------------------------------------------------- /storyweb/analysis/calais.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import requests 3 | 4 | from storyweb.core import app 5 | 6 | log = logging.getLogger(__name__) 7 | 8 | 9 | def extract_calais(text): 10 | calais_key = app.config.get('CALAIS_KEY') 11 | if calais_key is None: 12 | log.warning('No CALAIS_KEY is set, skipping entity extraction') 13 | return 14 | if text is None or len(text.strip()) < 10: 15 | return 16 | URL = 'http://api.opencalais.com/tag/rs/enrich' 17 | headers = { 18 | 'x-calais-licenseID': calais_key, 19 | 'content-type': 'text/raw', 20 | 'accept': 'application/json', 21 | 'enableMetadataType': 'SocialTags' 22 | } 23 | res = requests.post(URL, headers=headers, 24 | data=text.encode('utf-8')) 25 | data = res.json() 26 | for k, v in data.items(): 27 | _type = v.get('_type') 28 | if _type in ['Person', 'Organization', 'Company']: 29 | aliases = set([v.get('name')]) 30 | for instance in v.get('instances', [{}]): 31 | alias = instance.get('exact') 32 | if alias is not None and len(alias) > 3: 33 | aliases.add(alias) 34 | 35 | offset = v.get('instances', [{}])[0].get('offset') 36 | data = { 37 | 'title': v.get('name'), 38 | 'aliases': list(aliases), 39 | 'category': _type, 40 | 'text': '' 41 | } 42 | yield offset, data 43 | -------------------------------------------------------------------------------- /storyweb/views/auth.py: -------------------------------------------------------------------------------- 1 | from flask import render_template, redirect, request, url_for, g 2 | from flask.ext.login import login_required, login_user 3 | from flask.ext.login import logout_user, current_user 4 | 5 | from storyweb.core import app 6 | from storyweb.util import jsonify 7 | from storyweb.model import User 8 | from storyweb.model.forms import LoginForm, Invalid 9 | 10 | 11 | @app.route("/", methods=["POST", "GET"]) 12 | def login(): 13 | if current_user.is_authenticated(): 14 | return redirect(url_for('ui')) 15 | error, data = None, dict(request.form.items()) 16 | try: 17 | data = LoginForm().deserialize(data) 18 | user = User.query.filter_by(email=data.get('email')).first() 19 | if user is None or not user.check_password(data.get('password')): 20 | error = "Invalid email or password" 21 | else: 22 | login_user(user, remember=True) 23 | return redirect(request.args.get("next") or url_for('ui')) 24 | except Invalid: 25 | pass 26 | return render_template("login.html", data=data, error=error) 27 | 28 | 29 | @app.route("/logout", methods=["GET"]) 30 | @login_required 31 | def logout(): 32 | logout_user() 33 | return redirect(url_for('login')) 34 | 35 | 36 | @app.route("/api/1/session", methods=["GET"]) 37 | def get_session(): 38 | return jsonify({ 39 | 'user': g.user if g.user.is_authenticated() else None, 40 | 'logged_in': g.user.is_authenticated(), 41 | 'is_admin': g.user.is_admin if g.user.is_authenticated() else False 42 | }) 43 | -------------------------------------------------------------------------------- /storyweb/queue.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from storyweb.core import celery as app 4 | from storyweb.analysis.extract import extract_entities 5 | from storyweb.model import Card, db 6 | from storyweb.search import index_card 7 | from storyweb import spiders 8 | 9 | log = logging.getLogger(__name__) 10 | 11 | 12 | @app.task(bind=True) 13 | def extract(self, card_id): 14 | parent = Card.by_id(card_id) 15 | if parent is None: 16 | raise self.retry(countdown=1) 17 | log.info('Extracting entities from "%s"...', parent.title) 18 | try: 19 | extract_entities(parent) 20 | db.session.commit() 21 | except Exception, e: 22 | log.exception(e) 23 | finally: 24 | db.session.remove() 25 | 26 | 27 | def lookup_all(card_id): 28 | for spider_name in spiders.SPIDERS: 29 | lookup.apply_async((card_id, spider_name), {}, countdown=1) 30 | 31 | 32 | @app.task(bind=True) 33 | def lookup(self, card_id, spider_name): 34 | try: 35 | card = Card.by_id(card_id) 36 | if card is None: 37 | raise self.retry(countdown=1) 38 | spiders.lookup(card, spider_name) 39 | db.session.commit() 40 | except Exception, e: 41 | log.exception(e) 42 | finally: 43 | db.session.remove() 44 | 45 | 46 | @app.task(bind=True) 47 | def index(self, card_id): 48 | try: 49 | card = Card.by_id(card_id) 50 | if card is None: 51 | raise self.retry(countdown=2) 52 | index_card(card) 53 | except Exception, e: 54 | log.exception(e) 55 | finally: 56 | db.session.remove() 57 | -------------------------------------------------------------------------------- /storyweb/templates/login.html: -------------------------------------------------------------------------------- 1 | {% extends "layout.html" %} 2 | 3 | {% block title %} 4 | Sign in to the story editor - 5 | {% endblock %} 6 | 7 | {% block description %} 8 | 9 | {% endblock %} 10 | 11 | {% block content %} 12 |
13 |

{{config.APP_TITLE}}

14 |

{{config.APP_DESCRIPTION}}

15 | {% if config.MOTD %} 16 |
17 | {{config.MOTD | safe}} 18 |
19 | {% endif %} 20 |
21 |
22 | 23 |
24 | 26 | {% if error %} 27 |

{{ error }}

28 | {% endif %} 29 |
30 |
31 |
32 | 33 |
34 | 35 |
36 |
37 |
38 |
39 | 40 |
41 |
42 |
43 |
44 | {% endblock %} 45 | -------------------------------------------------------------------------------- /storyweb/templates/app.html: -------------------------------------------------------------------------------- 1 | {% extends "layout.html" %} 2 | {% block title %}{% endblock %} 3 | 4 | {% block content %} 5 |
6 | 32 | 33 |
34 |
35 |
36 |
37 | 38 | {% for path, tmpl in templates %} 39 | 40 | {% endfor %} 41 | {% endblock %} 42 | -------------------------------------------------------------------------------- /storyweb/search/result_proxy.py: -------------------------------------------------------------------------------- 1 | from storyweb.core import es, es_index 2 | 3 | 4 | class ESResultProxy(object): 5 | """ This is required for the pager to work. """ 6 | 7 | def __init__(self, doc_type, query): 8 | self.doc_type = doc_type 9 | self.query = query 10 | self._result = None 11 | self._count = None 12 | self._limit = 10 13 | self._offset = 0 14 | 15 | def limit(self, num): 16 | self._result = None 17 | self._limit = num 18 | return self 19 | 20 | def offset(self, num): 21 | self._result = None 22 | self._offset = num 23 | return self 24 | 25 | @property 26 | def result(self): 27 | if self._result is None: 28 | q = self.query.copy() 29 | q['from'] = self._offset 30 | q['size'] = self._limit 31 | self._result = es.search(index=es_index, 32 | doc_type=self.doc_type, 33 | query=q) 34 | return self._result 35 | 36 | def __len__(self): 37 | if self._count is None: 38 | if self._result is None: 39 | self.limit(0) 40 | res = self.result 41 | self._result = None 42 | else: 43 | res = self.result 44 | self._count = res.get('hits', {}).get('total') 45 | return self._count 46 | 47 | def __iter__(self): 48 | for hit in self.result.get('hits', {}).get('hits', []): 49 | res = hit.get('_source') 50 | res['score'] = hit.get('_score') 51 | yield res 52 | -------------------------------------------------------------------------------- /storyweb/core.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from flask import Flask 3 | from flask import url_for as _url_for 4 | from flask.ext.sqlalchemy import SQLAlchemy 5 | from flask.ext.login import LoginManager 6 | from flask.ext.assets import Environment 7 | from flask.ext.migrate import Migrate 8 | from kombu import Exchange, Queue 9 | from celery import Celery 10 | from pyelasticsearch import ElasticSearch 11 | 12 | from storyweb import default_settings 13 | 14 | logging.basicConfig(level=logging.DEBUG) 15 | 16 | # specific loggers 17 | logging.getLogger('requests').setLevel(logging.WARNING) 18 | logging.getLogger('urllib3').setLevel(logging.WARNING) 19 | logging.getLogger('pyelasticsearch').setLevel(logging.WARNING) 20 | 21 | 22 | app = Flask(__name__) 23 | app.config.from_object(default_settings) 24 | app.config.from_envvar('STORYWEB_SETTINGS', silent=True) 25 | app_name = app.config.get('APP_NAME') 26 | 27 | db = SQLAlchemy(app) 28 | migrate = Migrate(app, db, directory=app.config.get('ALEMBIC_DIR')) 29 | 30 | es = ElasticSearch(app.config.get('ELASTICSEARCH_URL')) 31 | es_index = app.config.get('ELASTICSEARCH_INDEX', app_name) 32 | 33 | login_manager = LoginManager() 34 | login_manager.init_app(app) 35 | login_manager.login_view = 'login' 36 | 37 | queue_name = app_name + '_q' 38 | app.config['CELERY_DEFAULT_QUEUE'] = queue_name 39 | app.config['CELERY_QUEUES'] = ( 40 | Queue(queue_name, Exchange(queue_name), routing_key=queue_name), 41 | ) 42 | 43 | celery = Celery(app_name, broker=app.config['CELERY_BROKER_URL']) 44 | celery.config_from_object(app.config) 45 | 46 | assets = Environment(app) 47 | 48 | 49 | def url_for(*a, **kw): 50 | try: 51 | kw['_external'] = True 52 | return _url_for(*a, **kw) 53 | except RuntimeError: 54 | return None 55 | -------------------------------------------------------------------------------- /storyweb/static/js/directives/pager.js: -------------------------------------------------------------------------------- 1 | storyweb.directive('storywebPager', ['$timeout', function ($timeout) { 2 | return { 3 | restrict: 'E', 4 | scope: { 5 | 'response': '=', 6 | 'load': '&load' 7 | }, 8 | templateUrl: 'pager.html', 9 | link: function (scope, element, attrs, model) { 10 | scope.$watch('response', function(e) { 11 | scope.showPager = false; 12 | scope.pages = []; 13 | if (scope.response.pages <= 1) { 14 | return; 15 | } 16 | var pages = [], 17 | current = (scope.response.offset / scope.response.limit) + 1, 18 | num = Math.ceil(scope.response.total / scope.response.limit), 19 | range = 2, 20 | low = current - range, 21 | high = current + range; 22 | 23 | if (low < 1) { 24 | low = 1; 25 | high = Math.min((2*range)+1, num); 26 | } 27 | if (high > num) { 28 | high = num; 29 | low = Math.max(1, num - (2*range)+1); 30 | } 31 | 32 | for (var page = low; page <= high; page++) { 33 | var offset = (page-1) * scope.response.limit, 34 | url = scope.response.format.replace('LIMIT', scope.response.limit).replace('OFFSET', offset); 35 | pages.push({ 36 | page: page, 37 | current: page==current, 38 | url: url 39 | }); 40 | } 41 | scope.showPager = true; 42 | scope.pages = pages; 43 | }); 44 | } 45 | }; 46 | }]); 47 | -------------------------------------------------------------------------------- /storyweb/templates/layout.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | {% block title %}{% endblock %} {{config.APP_TITLE}} 7 | 8 | 9 | 10 | 11 | {% assets "css" %} 12 | 13 | {% endassets %} 14 | 15 | 16 | 17 | 18 | 19 | {% block content %}{% endblock %} 20 | 41 | {% assets "js" %} 42 | 43 | {% endassets %} 44 | 45 | 46 | -------------------------------------------------------------------------------- /storyweb/model/util.py: -------------------------------------------------------------------------------- 1 | import json 2 | from jinja2.filters import do_truncate as truncate 3 | from jinja2.filters import do_striptags as striptags 4 | 5 | from sqlalchemy import func 6 | from sqlalchemy.types import TypeDecorator, VARCHAR 7 | from sqlalchemy.ext.mutable import Mutable 8 | 9 | 10 | def db_norm(col): 11 | return func.trim(func.lower(col)) 12 | 13 | 14 | def db_compare(col, text): 15 | text_ = text.lower().strip() 16 | return db_norm(col) == text_ 17 | 18 | 19 | def html_summary(html): 20 | return truncate(striptags(html), length=160) 21 | 22 | 23 | class JSONEncodedDict(TypeDecorator): 24 | "Represents an immutable structure as a json-encoded string." 25 | 26 | impl = VARCHAR 27 | 28 | def process_bind_param(self, value, dialect): 29 | if value is not None: 30 | value = json.dumps(value) 31 | return value 32 | 33 | def process_result_value(self, value, dialect): 34 | if value is not None: 35 | value = json.loads(value) 36 | return value 37 | 38 | 39 | class MutableDict(Mutable, dict): 40 | @classmethod 41 | def coerce(cls, key, value): 42 | "Convert plain dictionaries to MutableDict." 43 | 44 | if not isinstance(value, MutableDict): 45 | if isinstance(value, dict): 46 | return MutableDict(value) 47 | 48 | # this call will raise ValueError 49 | return Mutable.coerce(key, value) 50 | else: 51 | return value 52 | 53 | def __setitem__(self, key, value): 54 | "Detect dictionary set events and emit change events." 55 | 56 | dict.__setitem__(self, key, value) 57 | self.changed() 58 | 59 | def __delitem__(self, key): 60 | "Detect dictionary del events and emit change events." 61 | 62 | dict.__delitem__(self, key) 63 | self.changed() 64 | -------------------------------------------------------------------------------- /storyweb/static/js/directives/new_link.js: -------------------------------------------------------------------------------- 1 | storyweb.directive('storywebNewLink', ['$http', 'cfpLoadingBar', function($http, cfpLoadingBar) { 2 | return { 3 | restrict: 'E', 4 | transclude: true, 5 | scope: { 6 | 'parent': '=' 7 | }, 8 | templateUrl: 'link_new.html', 9 | link: function (scope, element, attrs, model) { 10 | scope.card = {'category': 'Company'}; 11 | scope.categories = ["Company", "Person", "Organization"]; 12 | 13 | scope.selectCategory = function(category) { 14 | scope.card.category = category; 15 | }; 16 | 17 | scope.canSubmit = function() { 18 | return scope.card.title && scope.card.title.length > 1; 19 | }; 20 | 21 | scope.suggestCard = function(q) { 22 | var params = {'prefix': q}; 23 | return $http.get('/api/1/cards/_suggest', {'params': params}).then(function(res) { 24 | return res.data.options; 25 | }); 26 | }; 27 | 28 | scope.linkCard = function(card) { 29 | saveLink(card); 30 | }; 31 | 32 | scope.saveCard = function() { 33 | if (!scope.canSubmit()) return; 34 | cfpLoadingBar.start(); 35 | var card = angular.copy(scope.card); 36 | // create new card 37 | $http.post('/api/1/cards', card).then(function(res) { 38 | saveLink(res.data); 39 | }); 40 | scope.card = {'category': 'Company'}; 41 | }; 42 | 43 | var saveLink = function(child) { 44 | var link = { 45 | 'child': child, 46 | 'status': 'approved', 47 | 'offset': 0 48 | }; 49 | var url = '/api/1/cards/' + scope.parent.id + '/links'; 50 | $http.post(url, link).then(function(res) { 51 | cfpLoadingBar.complete(); 52 | scope.$emit('linkChange'); 53 | scope.$emit('pendingTab'); 54 | }); 55 | }; 56 | } 57 | }; 58 | }]); 59 | -------------------------------------------------------------------------------- /storyweb/static/templates/card.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |

4 | 5 | 6 |

7 |
8 |
9 |
10 | 13 |
14 |
15 |
16 | 17 |
18 |
19 |
20 |

21 |

24 |
25 |

26 |
27 |
28 |
29 | 30 | 31 |
32 | There are {{updatesPending.updated.length}} new associated cards available. 33 | Click here to reload. 34 |
35 |
36 |
37 | 38 |
39 |
40 | 41 | 42 |
43 |
44 | 45 |
46 |
47 | 48 | 49 | 50 | 51 | 52 |
53 |
54 | -------------------------------------------------------------------------------- /storyweb/views/__init__.py: -------------------------------------------------------------------------------- 1 | from flask import g, request 2 | from flask.ext.login import current_user 3 | from werkzeug.exceptions import HTTPException 4 | 5 | from storyweb.core import app 6 | from storyweb.model.forms import Invalid 7 | from storyweb.util import jsonify 8 | from storyweb.assets import assets # noqa 9 | from storyweb.views.ui import ui # noqa 10 | from storyweb.views.auth import login, logout # noqa 11 | from storyweb.views.admin import admin # noqa 12 | from storyweb.views.cards_api import blueprint as cards_api 13 | from storyweb.views.links_api import blueprint as links_api 14 | from storyweb.views.references_api import blueprint as references_api 15 | 16 | 17 | @app.before_request 18 | def before_request(): 19 | g.user = current_user 20 | 21 | app.register_blueprint(cards_api) 22 | app.register_blueprint(links_api) 23 | app.register_blueprint(references_api) 24 | 25 | 26 | @app.errorhandler(401) 27 | @app.errorhandler(403) 28 | @app.errorhandler(404) 29 | @app.errorhandler(410) 30 | @app.errorhandler(500) 31 | def handle_exceptions(exc): 32 | if isinstance(exc, HTTPException): 33 | message = exc.get_description(request.environ) 34 | message = message.replace('

', '').replace('

', '') 35 | body = { 36 | 'status': exc.code, 37 | 'name': exc.name, 38 | 'message': message 39 | } 40 | headers = exc.get_headers(request.environ) 41 | else: 42 | body = { 43 | 'status': 500, 44 | 'name': exc.__class__.__name__, 45 | 'message': unicode(exc) 46 | } 47 | headers = {} 48 | return jsonify(body, status=body.get('status'), 49 | headers=headers) 50 | 51 | 52 | @app.errorhandler(Invalid) 53 | def handle_invalid(exc): 54 | body = { 55 | 'status': 400, 56 | 'name': 'Invalid Data', 57 | 'message': unicode(exc), 58 | 'errors': exc.asdict() 59 | } 60 | return jsonify(body, status=400) 61 | -------------------------------------------------------------------------------- /storyweb/static/js/directives/editor.js: -------------------------------------------------------------------------------- 1 | // this is derived from: 2 | // https://github.com/thijsw/angular-medium-editor 3 | 4 | storyweb.directive('mediumEditor', function() { 5 | return { 6 | require: 'ngModel', 7 | restrict: 'AE', 8 | scope: { 9 | }, 10 | link: function(scope, iElement, iAttrs, ctrl) { 11 | 12 | angular.element(iElement).addClass('angular-medium-editor'); 13 | 14 | // Parse options 15 | var placeholder = '', 16 | opts = { 17 | 'buttons': ["bold", "italic", "anchor", "header1", "header2", "quote", "orderedlist"], 18 | 'cleanPastedHTML': true 19 | }; 20 | 21 | var onChange = function() { 22 | scope.$apply(function() { 23 | 24 | // If user cleared the whole text, we have to reset the editor because MediumEditor 25 | // lacks an API method to alter placeholder after initialization 26 | if (iElement.html() === '


' || iElement.html() === '') { 27 | opts.placeholder = placeholder; 28 | var editor = new MediumEditor(iElement, opts); 29 | } 30 | 31 | ctrl.$setViewValue(iElement.html()); 32 | }); 33 | }; 34 | 35 | // view -> model 36 | iElement.on('blur', onChange); 37 | iElement.on('input', onChange); 38 | 39 | // model -> view 40 | ctrl.$render = function() { 41 | 42 | if (!this.editor) { 43 | // Hide placeholder when the model is not empty 44 | if (!ctrl.$isEmpty(ctrl.$viewValue)) { 45 | opts.placeholder = ''; 46 | } 47 | 48 | this.editor = new MediumEditor(iElement, opts); 49 | } 50 | 51 | iElement.html(ctrl.$isEmpty(ctrl.$viewValue) ? '' : ctrl.$viewValue); 52 | 53 | // hide placeholder when view is not empty 54 | if(!ctrl.$isEmpty(ctrl.$viewValue)) { 55 | angular.element(iElement).removeClass('medium-editor-placeholder'); 56 | } 57 | }; 58 | 59 | } 60 | }; 61 | 62 | }); 63 | -------------------------------------------------------------------------------- /storyweb/views/references_api.py: -------------------------------------------------------------------------------- 1 | from werkzeug.exceptions import Gone 2 | from flask import Blueprint, g 3 | from restpager import Pager 4 | 5 | from storyweb import authz 6 | from storyweb.model import db, Card, Reference 7 | from storyweb.util import jsonify, obj_or_404, request_data 8 | 9 | 10 | blueprint = Blueprint('references_api', __name__) 11 | 12 | 13 | @blueprint.route('/api/1/cards//references', methods=['GET']) 14 | def index(card_id): 15 | authz.require(authz.logged_in()) 16 | card = obj_or_404(Card.by_id(card_id)) 17 | references = db.session.query(Reference) 18 | references = references.filter(Reference.card == card) 19 | pager = Pager(references, card_id=card_id) 20 | return jsonify(pager, index=True) 21 | 22 | 23 | @blueprint.route('/api/1/cards//references', methods=['POST', 'PUT']) 24 | def create(card_id): 25 | authz.require(authz.logged_in()) 26 | card = obj_or_404(Card.by_id(card_id)) 27 | reference = Reference().save(request_data(), card, g.user) 28 | db.session.commit() 29 | return jsonify(reference, status=201) 30 | 31 | 32 | @blueprint.route('/api/1/cards//references/', methods=['GET']) 33 | def view(card_id, id): 34 | authz.require(authz.logged_in()) 35 | reference = obj_or_404(Reference.by_id(id, card_id=card_id)) 36 | return jsonify(reference) 37 | 38 | 39 | @blueprint.route('/api/1/cards//references/', methods=['POST', 'PUT']) 40 | def update(card_id, id): 41 | authz.require(authz.logged_in()) 42 | reference = obj_or_404(Reference.by_id(id, card_id=card_id)) 43 | reference.save(request_data(), reference.card, g.user) 44 | db.session.commit() 45 | return jsonify(reference) 46 | 47 | 48 | @blueprint.route('/api/1/cards//references/', methods=['DELETE']) 49 | def delete(card_id, id): 50 | authz.require(authz.logged_in()) 51 | reference = obj_or_404(Reference.by_id(id, card_id=card_id)) 52 | db.session.delete(reference) 53 | db.session.commit() 54 | raise Gone() 55 | -------------------------------------------------------------------------------- /storyweb/spiders/openduka.py: -------------------------------------------------------------------------------- 1 | import requests 2 | #from pprint import pprint 3 | 4 | from storyweb.model import Reference, User 5 | from storyweb.spiders.util import Spider, text_score 6 | 7 | URL = "http://www.openduka.org/index.php/" 8 | API_KEY = '86a6b32f398fe7b3e0a7e13c96b4f032' 9 | 10 | 11 | class OpenDuka(Spider): 12 | 13 | def make_ref(self, card, id, score, type_, record, idx): 14 | label = record.get('Citation', 15 | record.get('title', 16 | record.get('Name'))) 17 | if len(label) > 80: 18 | label = label[:80] + '...' 19 | data = { 20 | 'citation': '%s: %s' % (type_, label), 21 | 'url': URL + 'homes/tree/%s#match%s' % (id, idx), 22 | 'source': 'OpenDuka', 23 | 'score': score, 24 | 'source_url': 'http://openduka.org' 25 | } 26 | ref = Reference.find(card, data.get('url')) 27 | if ref is None: 28 | ref = Reference() 29 | ref.save(data, card, User.default_user()) 30 | 31 | def search_all(self, card): 32 | args = {'key': API_KEY, 'term': card.title} 33 | r = requests.get(URL + "api/search", params=args) 34 | idx = 1 35 | for match in r.json(): 36 | score = text_score(match.get('Name'), list(card.aliases)) 37 | if score < 50: 38 | continue 39 | args = {'key': API_KEY, 'id': match.get('ID')} 40 | r = requests.get(URL + "api/entity", params=args) 41 | for type_set in r.json().get('data'): 42 | for data in type_set['dataset_type']: 43 | for type_, ds in data.items(): 44 | for item in ds: 45 | for record in item.get('dataset'): 46 | self.make_ref(card, match.get('ID'), 47 | score, type_, record, idx) 48 | idx = idx + 1 49 | return card 50 | -------------------------------------------------------------------------------- /storyweb/util.py: -------------------------------------------------------------------------------- 1 | import simplejson 2 | from datetime import datetime, date 3 | from inspect import isgenerator 4 | from sqlalchemy.orm import Query 5 | from sqlalchemy.ext.associationproxy import _AssociationList 6 | from werkzeug.exceptions import NotFound 7 | from flask import Response, request 8 | 9 | 10 | def obj_or_404(obj): 11 | if obj is None: 12 | raise NotFound() 13 | return obj 14 | 15 | 16 | def request_data(overlay={}): 17 | """ Decode a JSON-formatted POST body. """ 18 | data = request.json 19 | if data is None: 20 | try: 21 | data = simplejson.loads(request.form.get('data')) 22 | except (ValueError, TypeError): 23 | data = dict(request.form.items()) 24 | data.update(overlay) 25 | return data 26 | 27 | 28 | class AppEncoder(simplejson.JSONEncoder): 29 | """ This encoder will serialize all entities that have a to_dict 30 | method by calling that method and serializing the result. """ 31 | 32 | def default(self, obj): 33 | if hasattr(obj, 'to_dict'): 34 | return obj.to_dict() 35 | elif isinstance(obj, datetime): 36 | return obj.isoformat() + 'Z' 37 | elif isinstance(obj, date): 38 | return obj.isoformat() 39 | elif isinstance(obj, Query) or isinstance(obj, _AssociationList): 40 | return [r for r in obj] 41 | elif isgenerator(obj): 42 | return [o for o in obj] 43 | elif isinstance(obj, set): 44 | return [o for o in obj] 45 | return super(AppEncoder, self).default(obj) 46 | 47 | 48 | def jsonify(obj, status=200, headers=None, index=False, encoder=AppEncoder): 49 | """ Custom JSONificaton to support obj.to_dict protocol. """ 50 | data = encoder().encode(obj) 51 | if 'callback' in request.args: 52 | cb = request.args.get('callback') 53 | data = '%s && %s(%s)' % (cb, cb, data) 54 | return Response(data, headers=headers, 55 | status=status, 56 | mimetype='application/json') 57 | -------------------------------------------------------------------------------- /storyweb/static/templates/link.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |
4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 |
16 |
17 | 18 |
19 | {{card.title}} 20 | 21 | {{card.references.length}} leads 22 | 23 | 24 | one lead 25 | 26 | 27 | no leads 28 | 29 |
30 | 31 |
32 |
33 |
34 | 35 |
36 |

37 | Notes 38 |

39 |
40 |

41 |

No notes have been made

42 |
43 |
44 | 45 |
46 | 47 |
48 |
49 |
50 |

References

51 |
    52 |
  • 53 | 54 |
    • 55 |
56 |
57 |
58 |
59 | -------------------------------------------------------------------------------- /storyweb/search/mapping.py: -------------------------------------------------------------------------------- 1 | 2 | CARD_MAPPING = { 3 | "_id": { 4 | "path": "id" 5 | }, 6 | "_all": { 7 | "enabled": True 8 | }, 9 | "dynamic": "strict", 10 | "properties": { 11 | "id": {"type": "string", "index": "not_analyzed"}, 12 | "title": {"type": "string", "index": "analyzed"}, 13 | "summary": {"type": "string", "index": "analyzed"}, 14 | "category": {"type": "string", "index": "not_analyzed"}, 15 | "text": {"type": "string", "index": "analyzed"}, 16 | "aliases": {"type": "string", "index": "analyzed"}, 17 | "created_at": {"type": "date", "index": "not_analyzed"}, 18 | "updated_at": {"type": "date", "index": "not_analyzed"}, 19 | "author": { 20 | "_id": { 21 | "path": "id" 22 | }, 23 | "type": "nested", 24 | "include_in_parent": True, 25 | "properties": { 26 | "id": {"type": "string", "index": "not_analyzed"}, 27 | "display_name": {"type": "string", "index": "not_analyzed"} 28 | } 29 | }, 30 | "references": { 31 | "_id": { 32 | "path": "id" 33 | }, 34 | "type": "nested", 35 | "include_in_parent": True, 36 | "properties": { 37 | "id": {"type": "string", "index": "not_analyzed"}, 38 | "citation": {"type": "string", "index": "analyzed"}, 39 | "score": {"type": "integer", "index": "not_analyzed"}, 40 | "url": {"type": "string", "index": "not_analyzed"}, 41 | "source": {"type": "string", "index": "not_analyzed"}, 42 | "source_url": {"type": "string", "index": "not_analyzed"} 43 | } 44 | }, 45 | "links": { 46 | "_id": { 47 | "path": "id" 48 | }, 49 | "type": "nested", 50 | "include_in_parent": True, 51 | "properties": { 52 | "id": {"type": "string", "index": "not_analyzed"}, 53 | "title": {"type": "string", "index": "analyzed"}, 54 | "category": {"type": "string", "index": "not_analyzed"}, 55 | "rejected": {"type": "boolean", "index": "not_analyzed"}, 56 | "status": {"type": "string", "index": "not_analyzed"}, 57 | "offset": {"type": "integer", "index": "not_analyzed"} 58 | } 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /storyweb/model/user.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from datetime import datetime 3 | from werkzeug.security import generate_password_hash 4 | from werkzeug.security import check_password_hash 5 | 6 | from storyweb.core import db, login_manager 7 | 8 | log = logging.getLogger(__name__) 9 | 10 | 11 | @login_manager.user_loader 12 | def load_user(id): 13 | return User.query.get(int(id)) 14 | 15 | 16 | class User(db.Model): 17 | id = db.Column(db.Integer, primary_key=True) 18 | email = db.Column(db.Unicode, nullable=False) 19 | display_name = db.Column(db.Unicode) 20 | password_hash = db.Column(db.Unicode, nullable=False) 21 | is_admin = db.Column(db.Boolean, nullable=False, default=False) 22 | is_editor = db.Column(db.Boolean, nullable=False, default=False) 23 | active = db.Column(db.Boolean, nullable=False, default=True) 24 | 25 | created_at = db.Column(db.DateTime, default=datetime.utcnow) 26 | updated_at = db.Column(db.DateTime, default=datetime.utcnow, 27 | onupdate=datetime.utcnow) 28 | 29 | @property 30 | def password(self): 31 | return self.password_hash 32 | 33 | @password.setter 34 | def password(self, password): 35 | self.password_hash = generate_password_hash(password) 36 | 37 | def check_password(self, password): 38 | return check_password_hash(self.password, password) 39 | 40 | def is_active(self): 41 | return self.active 42 | 43 | def is_authenticated(self): 44 | return True 45 | 46 | def is_anonymous(self): 47 | return False 48 | 49 | def get_id(self): 50 | return unicode(self.id) 51 | 52 | def __repr__(self): 53 | return '' % (self.id, self.email) 54 | 55 | def __unicode__(self): 56 | return self.display_name 57 | 58 | def to_dict(self): 59 | return { 60 | 'id': self.id, 61 | 'display_name': self.display_name 62 | } 63 | 64 | @classmethod 65 | def default_user(cls): 66 | q = db.session.query(cls) 67 | q = q.filter(cls.is_admin == True) # noqa 68 | q = q.filter(cls.active == True) # noqa 69 | q = q.order_by(cls.created_at.asc()) 70 | user = q.first() 71 | if user is None: 72 | user = cls() 73 | user.email = 'admin@grano.cc' 74 | user.password = 'admin' 75 | user.display_name = 'Administrator' 76 | user.is_editor = True 77 | user.is_admin = True 78 | log.info("Created user: admin@grano.cc, password: admin") 79 | db.session.add(user) 80 | return user 81 | -------------------------------------------------------------------------------- /storyweb/analysis/extract.py: -------------------------------------------------------------------------------- 1 | import re 2 | import logging 3 | from sqlalchemy.orm import aliased 4 | 5 | from storyweb.model import Card, User, Link, Alias, db 6 | from storyweb.analysis.text import normalize 7 | from storyweb.analysis.calais import extract_calais 8 | 9 | log = logging.getLogger(__name__) 10 | 11 | 12 | def card_titles(): 13 | alias = aliased(Alias) 14 | card = aliased(Card) 15 | q = db.session.query(alias.name) 16 | q = q.join(card, alias.card) 17 | q = q.add_column(card.id) 18 | to_d = lambda r: { 19 | 'id': r.id, 20 | 'text': r.name, 21 | 'normalized': normalize(r.name) 22 | } 23 | return [to_d(r) for r in q.all()] 24 | 25 | 26 | def extract_known(parent): 27 | if parent.text is None or not len(parent.text): 28 | return 29 | cards = card_titles() 30 | names = [c['normalized'] for c in cards] 31 | names = re.compile('(%s)' % '|'.join(names)) 32 | matches = [] 33 | for match in names.finditer(normalize(parent.text)): 34 | for card in cards: 35 | if card['normalized'] == match.group(1): 36 | matches.append((card['id'], match.start(1))) 37 | 38 | ids = set([m[0] for m in matches]) 39 | q = db.session.query(Card) 40 | q = q.filter(Card.id.in_(ids)) 41 | cards = {c.id: c for c in q.all()} 42 | 43 | for id, offset in matches: 44 | card = cards.pop(id, None) 45 | if card is not None: 46 | yield offset, card 47 | 48 | 49 | def extract_multi(parent): 50 | seen = set([parent.id]) 51 | for offset, card in extract_known(parent): 52 | if card.id not in seen: 53 | seen.add(card.id) 54 | yield offset, card 55 | 56 | for offset, data in extract_calais(parent.text): 57 | author = User.default_user() 58 | card = Card.find(data.get('title')) 59 | if card is None: 60 | card = Card() 61 | elif card.id in seen: 62 | continue 63 | else: 64 | data['text'] = card.text 65 | author = card.author 66 | card.save(data, author) 67 | seen.add(card.id) 68 | yield offset, card 69 | 70 | 71 | def extract_entities(parent): 72 | from storyweb.queue import index 73 | for offset, child in extract_multi(parent): 74 | log.info("Extraced: %r in %r", child, parent) 75 | data = { 76 | 'offset': offset, 77 | 'child': child 78 | } 79 | link = Link.find(parent, child) 80 | if link is None: 81 | link = Link() 82 | else: 83 | data['status'] = link.status 84 | link.save(data, parent, child.author) 85 | 86 | index.delay(parent.id) 87 | -------------------------------------------------------------------------------- /storyweb/views/cards_api.py: -------------------------------------------------------------------------------- 1 | from werkzeug.exceptions import Gone 2 | from flask import Blueprint, g, request 3 | from restpager import Pager 4 | 5 | from storyweb import authz 6 | from storyweb.model import db, Card 7 | from storyweb.search import search_cards, cards_query 8 | from storyweb.util import jsonify, obj_or_404, request_data 9 | from storyweb.queue import extract 10 | 11 | 12 | blueprint = Blueprint('cards_api', __name__) 13 | 14 | 15 | @blueprint.route('/api/1/categories', methods=['GET']) 16 | def categories(): 17 | authz.require(authz.logged_in()) 18 | return jsonify({'categories': Card.CATEGORIES}, index=True) 19 | 20 | 21 | @blueprint.route('/api/1/cards', methods=['GET']) 22 | def index(): 23 | authz.require(authz.logged_in()) 24 | cards = db.session.query(Card) 25 | if 'category' in request.args: 26 | cards = cards.filter(Card.category == request.args.get('category')) 27 | 28 | # TODO: find a better solution 29 | cards = cards.filter(Card.title != '') 30 | cards = cards.order_by(Card.created_at.desc()) 31 | 32 | pager = Pager(cards) 33 | return jsonify(pager, index=True) 34 | 35 | 36 | @blueprint.route('/api/1/cards/_suggest', methods=['GET']) 37 | def suggest(): 38 | authz.require(authz.logged_in()) 39 | options = Card.suggest(request.args.get('prefix'), 40 | categories=request.args.getlist('category')) 41 | return jsonify({'options': options}, index=True) 42 | 43 | 44 | @blueprint.route('/api/1/cards/_search', methods=['GET']) 45 | def search(): 46 | authz.require(authz.logged_in()) 47 | query = cards_query(request.args) 48 | pager = Pager(search_cards(query)) 49 | return jsonify(pager, index=True) 50 | 51 | 52 | @blueprint.route('/api/1/cards', methods=['POST', 'PUT']) 53 | def create(): 54 | authz.require(authz.logged_in()) 55 | card = Card().save(request_data(), g.user) 56 | db.session.commit() 57 | extract.delay(card.id) 58 | return jsonify(card, status=201) 59 | 60 | 61 | @blueprint.route('/api/1/cards/', methods=['GET']) 62 | def view(id): 63 | authz.require(authz.logged_in()) 64 | card = obj_or_404(Card.by_id(id)) 65 | return jsonify(card) 66 | 67 | 68 | @blueprint.route('/api/1/cards/', methods=['POST', 'PUT']) 69 | def update(id): 70 | authz.require(authz.logged_in()) 71 | card = obj_or_404(Card.by_id(id)) 72 | card.save(request_data(), g.user) 73 | db.session.commit() 74 | extract.delay(card.id) 75 | return jsonify(card) 76 | 77 | 78 | @blueprint.route('/api/1/cards/', methods=['DELETE']) 79 | def delete(id): 80 | authz.require(authz.logged_in()) 81 | card = obj_or_404(Card.by_id(id)) 82 | db.session.delete(card) 83 | db.session.commit() 84 | raise Gone() 85 | -------------------------------------------------------------------------------- /storyweb/static/js/controllers/card.js: -------------------------------------------------------------------------------- 1 | storyweb.controller('CardCtrl', ['$scope', '$routeParams', '$location', '$interval', '$http', 'cfpLoadingBar', 2 | function($scope, $routeParams, $location, $interval, $http, cfpLoadingBar) { 3 | var refreshSince = null, 4 | realText = null; 5 | 6 | $scope.cardId = $routeParams.id; 7 | $scope.updatesPending = false; 8 | $scope.card = {}; 9 | $scope.links = []; 10 | $scope.activeLinks = 0; 11 | $scope.rejectedLinks = 0; 12 | $scope.tabs = { 13 | 'pending': true 14 | }; 15 | 16 | $scope.$on('pendingTab', function() { 17 | $scope.tabs.pending = true; 18 | }); 19 | 20 | $http.get('/api/1/cards/' + $scope.cardId).then(function(res) { 21 | $scope.card = res.data; 22 | }); 23 | 24 | $scope.$on('highlight', function(e, words) { 25 | realText = $scope.card.text; 26 | var regex = '(' + words.join('|') + ')'; 27 | $scope.card.text = realText.replace(new RegExp(regex, 'gi'), function(t) { 28 | return '' + t + ''; 29 | }); 30 | }); 31 | 32 | $scope.$on('clearHighlight', function(e, words) { 33 | $scope.card.text = realText; 34 | }); 35 | 36 | $scope.$on('linkChange', function() { 37 | $scope.updateLinks(); 38 | }); 39 | 40 | $scope.updateLinks = function() { 41 | $scope.updatesPending = false; 42 | refreshSince = new Date(); 43 | cfpLoadingBar.start(); 44 | $http.get('/api/1/cards/' + $scope.cardId + '/links').then(function(res) { 45 | $scope.rejectedLinks = 0; 46 | $scope.activeLinks = 0; 47 | 48 | angular.forEach(res.data.results, function(link) { 49 | if (link.rejected) { 50 | $scope.rejectedLinks++; 51 | } else { 52 | $scope.activeLinks++; 53 | } 54 | }); 55 | 56 | $scope.links = res.data.results; 57 | cfpLoadingBar.complete(); 58 | }); 59 | }; 60 | 61 | var checkRefresh = function() { 62 | if (!refreshSince) return; 63 | var params = {'since': refreshSince.toISOString()}, 64 | url = $scope.card.api_url + '/links/_refresh'; 65 | $http.get(url, {'params': params}).then(function(res) { 66 | if (res.data.updated.length == 0) { 67 | $scope.updatesPending = false; 68 | } else { 69 | $scope.updatesPending = res.data.updated; 70 | } 71 | }); 72 | } 73 | 74 | $scope.saveCard = function () { 75 | cfpLoadingBar.start(); 76 | $http.post('/api/1/cards/' + $scope.cardId, $scope.card).then(function(res) { 77 | $scope.card = res.data; 78 | cfpLoadingBar.complete(); 79 | }); 80 | }; 81 | 82 | $scope.updateLinks(); 83 | var refreshInterval = $interval(checkRefresh, 2000); 84 | $scope.$on('$destroy', function() { 85 | $interval.cancel(refreshInterval); 86 | }); 87 | 88 | }]); 89 | -------------------------------------------------------------------------------- /storyweb/static/js/directives/link.js: -------------------------------------------------------------------------------- 1 | storyweb.directive('storywebLink', ['$http', '$sce', 'cfpLoadingBar', function($http, $sce, cfpLoadingBar) { 2 | return { 3 | restrict: 'E', 4 | transclude: true, 5 | scope: { 6 | 'parent': '=', 7 | 'link': '=' 8 | }, 9 | templateUrl: 'link.html', 10 | link: function (scope, element, attrs, model) { 11 | scope.mode = 'view'; 12 | scope.card = {}; 13 | scope.expanded = false; 14 | 15 | scope.$watch('link', function(l) { 16 | if (l) { 17 | scope.card = l.child; 18 | scope.html = $sce.trustAsHtml(scope.card.text); 19 | } 20 | }); 21 | 22 | var saveCard = function() { 23 | cfpLoadingBar.start(); 24 | var url = '/api/1/cards/' + scope.card.id; 25 | $http.post(url, scope.card).then(function(res) { 26 | scope.card = res.data; 27 | cfpLoadingBar.complete(); 28 | }); 29 | }; 30 | 31 | var saveLink = function() { 32 | cfpLoadingBar.start(); 33 | var url = '/api/1/cards/' + scope.parent.id + '/links/' + scope.link.id; 34 | scope.link.rejected = scope.link.status == 'rejected'; 35 | $http.post(url, scope.link).then(function(res) { 36 | scope.link = res.data; 37 | cfpLoadingBar.complete(); 38 | scope.$emit('linkChange'); 39 | }); 40 | }; 41 | 42 | scope.mouseIn = function() { 43 | scope.$emit('highlight', scope.card.aliases); 44 | }; 45 | 46 | scope.mouseOut = function() { 47 | scope.$emit('clearHighlight'); 48 | }; 49 | 50 | scope.toggleMode = function() { 51 | if (scope.editMode()) { 52 | saveCard(); 53 | } 54 | scope.mode = scope.mode == 'view' ? 'edit' : 'view'; 55 | }; 56 | 57 | scope.toggleRejected = function() { 58 | if (scope.link.status == 'rejected') { 59 | scope.link.status = 'approved'; 60 | } else { 61 | scope.link.status = 'rejected'; 62 | } 63 | saveLink(); 64 | }; 65 | 66 | scope.expandCard = function() { 67 | scope.expanded = !scope.expanded; 68 | }; 69 | 70 | scope.editMode = function() { 71 | return scope.mode == 'edit'; 72 | }; 73 | 74 | scope.viewMode = function() { 75 | return scope.mode == 'view'; 76 | }; 77 | 78 | scope.hasReferences = function() { 79 | return scope.card.references && scope.card.references.length > 0; 80 | }; 81 | 82 | scope.hasText = function() { 83 | return scope.card.text && scope.card.text.length > 0; 84 | }; 85 | 86 | scope.hasAliases = function() { 87 | return scope.card.aliases && scope.card.aliases.length > 1; 88 | }; 89 | } 90 | }; 91 | }]); 92 | -------------------------------------------------------------------------------- /storyweb/views/links_api.py: -------------------------------------------------------------------------------- 1 | import dateutil.parser 2 | from werkzeug.exceptions import Gone 3 | from flask import Blueprint, g, request 4 | from sqlalchemy import or_ 5 | from sqlalchemy.orm import aliased 6 | from restpager import Pager 7 | 8 | from storyweb import authz 9 | from storyweb.model import db, Card, Link 10 | from storyweb.util import jsonify, obj_or_404, request_data 11 | 12 | 13 | blueprint = Blueprint('links_api', __name__) 14 | 15 | 16 | @blueprint.route('/api/1/cards//links', methods=['GET']) 17 | def index(parent_id): 18 | authz.require(authz.logged_in()) 19 | card = obj_or_404(Card.by_id(parent_id)) 20 | links = db.session.query(Link) 21 | links = links.filter(Link.parent == card) 22 | links = links.order_by(Link.offset.asc()) 23 | pager = Pager(links, parent_id=parent_id) 24 | return jsonify(pager, index=True) 25 | 26 | 27 | @blueprint.route('/api/1/cards//links', methods=['POST', 'PUT']) 28 | def create(parent_id): 29 | authz.require(authz.logged_in()) 30 | card = obj_or_404(Card.by_id(parent_id)) 31 | reference = Link().save(request_data(), card, g.user) 32 | db.session.commit() 33 | return jsonify(reference, status=201) 34 | 35 | 36 | @blueprint.route('/api/1/cards//links/_refresh') 37 | def refresh(parent_id): 38 | authz.require(authz.logged_in()) 39 | card = obj_or_404(Card.by_id(parent_id)) 40 | link = aliased(Link) 41 | links = db.session.query(link.id) 42 | links = links.filter(link.parent == card) 43 | response = {'status': 'ok'} 44 | 45 | try: 46 | since = request.args.get('since') 47 | if since is not None: 48 | since = dateutil.parser.parse(since) 49 | 50 | child = aliased(Card) 51 | links = links.join(child, link.child) 52 | links = links.filter(or_(link.updated_at > since, 53 | child.updated_at > since)) 54 | except (AttributeError, ValueError), e: 55 | response['status'] = 'ok' 56 | response['error'] = unicode(e) 57 | 58 | response['updated'] = [l.id for l in links] 59 | return jsonify(response) 60 | 61 | 62 | @blueprint.route('/api/1/cards//links/', methods=['GET']) 63 | def view(parent_id, id): 64 | authz.require(authz.logged_in()) 65 | link = obj_or_404(Link.by_id(id, parent_id=parent_id)) 66 | return jsonify(link) 67 | 68 | 69 | @blueprint.route('/api/1/cards//links/', methods=['POST', 'PUT']) 70 | def update(parent_id, id): 71 | authz.require(authz.logged_in()) 72 | link = obj_or_404(Link.by_id(id, parent_id=parent_id)) 73 | link.save(request_data(), link.parent, g.user) 74 | db.session.commit() 75 | return jsonify(link) 76 | 77 | 78 | @blueprint.route('/api/1/cards//links/', methods=['DELETE']) 79 | def delete(parent_id, id): 80 | authz.require(authz.logged_in()) 81 | link = obj_or_404(Link.by_id(id, parent_id=parent_id)) 82 | db.session.delete(link) 83 | db.session.commit() 84 | raise Gone() 85 | -------------------------------------------------------------------------------- /storyweb/model/reference.py: -------------------------------------------------------------------------------- 1 | import colander 2 | from datetime import datetime 3 | 4 | from storyweb.core import db, url_for 5 | from storyweb.model.user import User 6 | from storyweb.model.card import Card 7 | 8 | 9 | class Reference(db.Model): 10 | id = db.Column(db.Integer, primary_key=True) 11 | score = db.Column(db.Integer, nullable=False, default=0) 12 | citation = db.Column(db.Unicode, nullable=False) 13 | url = db.Column(db.Unicode, nullable=False) 14 | source = db.Column(db.Unicode) 15 | source_url = db.Column(db.Unicode) 16 | 17 | author_id = db.Column(db.Integer(), db.ForeignKey('user.id')) 18 | author = db.relationship(User, backref=db.backref('references', 19 | lazy='dynamic')) 20 | 21 | card_id = db.Column(db.Integer(), db.ForeignKey('card.id')) 22 | card = db.relationship(Card, backref=db.backref('references', 23 | lazy='dynamic', order_by='Reference.score.desc()')) 24 | 25 | created_at = db.Column(db.DateTime, default=datetime.utcnow) 26 | updated_at = db.Column(db.DateTime, default=datetime.utcnow, 27 | onupdate=datetime.utcnow) 28 | 29 | def save(self, raw, card, author): 30 | data = ReferenceForm().deserialize(raw) 31 | self.score = data.get('score') 32 | self.citation = data.get('citation') 33 | self.url = data.get('url') 34 | self.source = data.get('source') 35 | self.source_url = data.get('source_url') 36 | self.card = card 37 | self.author = author 38 | db.session.add(self) 39 | return self 40 | 41 | def to_dict(self): 42 | return { 43 | 'id': self.id, 44 | 'api_url': url_for('references_api.view', 45 | card_id=self.card_id, id=self.id), 46 | 'citation': self.citation, 47 | 'url': self.url, 48 | 'score': self.score, 49 | 'source': self.source, 50 | 'source_url': self.source_url, 51 | 'author': self.author, 52 | 'created_at': self.created_at, 53 | 'updated_at': self.updated_at, 54 | } 55 | 56 | @classmethod 57 | def by_id(cls, id, card_id=None): 58 | q = db.session.query(cls) 59 | q = q.filter_by(id=id) 60 | if card_id is not None: 61 | q = q.filter_by(card_id=card_id) 62 | return q.first() 63 | 64 | @classmethod 65 | def find(cls, card, url): 66 | q = db.session.query(cls) 67 | q = q.filter_by(card=card) 68 | q = q.filter_by(url=url) 69 | return q.first() 70 | 71 | def __repr__(self): 72 | return '' % (self.id, self.citation, self.url) 73 | 74 | def __unicode__(self): 75 | return self.citation 76 | 77 | 78 | class ReferenceForm(colander.MappingSchema): 79 | score = colander.SchemaNode(colander.Int()) 80 | citation = colander.SchemaNode(colander.String()) 81 | url = colander.SchemaNode(colander.String(), validator=colander.url) 82 | source = colander.SchemaNode(colander.String()) 83 | source_url = colander.SchemaNode(colander.String(), validator=colander.url) 84 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Grano StoryWeb 2 | 3 | [![Deploy](https://www.herokucdn.com/deploy/button.png)](https://heroku.com/deploy?template=https://github.com/granoproject/storyweb) 4 | 5 | We treat journalism as the act of writing a piece of text, when it is much better understood as weaving a series of related facts into a narrative. Storyweb aims to reflect this, and help to structure the thought behind a story. 6 | 7 | While the idea of structured journalism has been around for a while, Storyweb is an experiment in how tightly we can join the process of collecting and selecting rich, networked research data with the process of actually writing a story. 8 | 9 | Storyweb creates a journalistic memory that can be shared inside a research team and automatically enriched both with material from a news organizations archives and with public data from governments and companies. 10 | 11 | --- 12 | 13 | This is an implementation of text-snippet based SNA; the idea that data 14 | in a social network isn't necessarily well-structured but can also be 15 | little pieces of text that link to each other (and to entities). 16 | 17 | Original mockup: [here](http://opendatalabs.org/misc/demo/grano/_mockup). 18 | 19 | ## Installation 20 | 21 | Before you can install ``storyweb``, the following dependencies are required: 22 | 23 | * A SQL database. While we recommend Postgres, the app can also run with other databases, such as SQLite. 24 | * ElasticSearch for full-text indexing. 25 | * ``less``, installed via ``npm``. 26 | * Python, and Python ``virtualenv``. 27 | * RabbitMQ 28 | 29 | Once these dependencies are satisfied, run the following command to install the application: 30 | 31 | git clone https://github.com/granoproject/storyweb.git storyweb 32 | cd storyweb 33 | virtualenv env 34 | source env/bin/activate 35 | pip install -r requirements.txt 36 | python setup.py develop 37 | npm install -g bower uglify-js 38 | 39 | Next, you need to customize the configuration file. Copy the template configuration file, ``settings.py.tmpl`` to a new file, e.g. ``settings.py`` in the project root and set the required settings. Then export the environment variable ``TMI_SETTINGS`` to point at this file: 40 | 41 | cp settings.py.tmpl settings.py 42 | export STORYWEB_SETTINGS=`pwd`/settings.py 43 | 44 | Use bower to install javascript dependencies: 45 | 46 | bower install 47 | 48 | To create a new database and search index, run the following command: 49 | 50 | python storyweb/manage.py upgrade 51 | 52 | This will also create an admin user with the email address ``admin@grano.cc`` and the password ``admin`` which you can use to log in and create more users. 53 | 54 | Congratulations, you've installed ``storyweb``. You can run the application using: 55 | 56 | python storyweb/manage.py runserver 57 | 58 | 59 | ## Credits 60 | 61 | This tool is heavily inspired by [Newsclip.se](http://canvas.challengepost.com/submissions/30703-newsclip-se), a hack from the Al Jazeera "[Media in Context](http://canvas.aljazeera.com/)" hackathon in December 2014. Thanks to the team: Eva Constantaras, Kasia Dybek, Bruno Faviero, Heinze Havinga, Friedrich Lindenberg, Phillip Smith. 62 | 63 | It is licensed under an open source MIT license. We welcome any contributions to the code base. 64 | -------------------------------------------------------------------------------- /storyweb/model/link.py: -------------------------------------------------------------------------------- 1 | import colander 2 | from datetime import datetime 3 | 4 | from storyweb.core import db, url_for 5 | from storyweb.model.user import User 6 | from storyweb.model.card import Card, CardRef 7 | 8 | 9 | class Link(db.Model): 10 | PENDING = 'pending' 11 | APPROVED = 'approved' 12 | REJECTED = 'rejected' 13 | STATUSES = [PENDING, APPROVED, REJECTED] 14 | 15 | id = db.Column(db.Integer, primary_key=True) 16 | offset = db.Column(db.Integer, nullable=False, default=0) 17 | status = db.Column(db.Enum(*STATUSES, name='link_statuses'), 18 | nullable=False, default=PENDING) 19 | 20 | author_id = db.Column(db.Integer(), db.ForeignKey('user.id')) 21 | author = db.relationship(User, backref=db.backref('links', 22 | lazy='dynamic')) 23 | 24 | parent_id = db.Column(db.Integer(), db.ForeignKey('card.id')) 25 | parent = db.relationship(Card, backref=db.backref('links', lazy='dynamic'), 26 | primaryjoin='Card.id==Link.parent_id', 27 | ) 28 | 29 | child_id = db.Column(db.Integer(), db.ForeignKey('card.id')) 30 | child = db.relationship(Card, backref=db.backref('linked', lazy='dynamic'), 31 | primaryjoin='Card.id==Link.child_id' 32 | ) 33 | 34 | created_at = db.Column(db.DateTime, default=datetime.utcnow) 35 | updated_at = db.Column(db.DateTime, default=datetime.utcnow, 36 | onupdate=datetime.utcnow) 37 | 38 | def __repr__(self): 39 | return '' % (self.id, self.parent, self.child) 40 | 41 | def save(self, raw, parent, author): 42 | data = LinkForm().deserialize(raw) 43 | self.status = data.get('status') 44 | self.offset = data.get('offset') 45 | self.child = data.get('child') 46 | self.parent = parent 47 | self.author = author 48 | db.session.add(self) 49 | return self 50 | 51 | def to_dict(self): 52 | return { 53 | 'id': self.id, 54 | 'api_url': url_for('links_api.view', 55 | parent_id=self.parent_id, id=self.id), 56 | 'status': self.status, 57 | 'rejected': self.rejected, 58 | 'child': self.child, 59 | 'created_at': self.created_at, 60 | 'updated_at': self.updated_at, 61 | } 62 | 63 | @property 64 | def rejected(self): 65 | return self.status == self.REJECTED 66 | 67 | @classmethod 68 | def by_id(cls, id, parent_id=None): 69 | q = db.session.query(cls) 70 | q = q.filter_by(id=id) 71 | if parent_id is not None: 72 | q = q.filter_by(parent_id=parent_id) 73 | return q.first() 74 | 75 | @classmethod 76 | def find(cls, parent, child): 77 | q = db.session.query(cls) 78 | q = q.filter_by(parent=parent) 79 | q = q.filter_by(child=child) 80 | return q.first() 81 | 82 | 83 | class LinkForm(colander.MappingSchema): 84 | status = colander.SchemaNode(colander.String(), missing=Link.PENDING) 85 | offset = colander.SchemaNode(colander.Int(), missing=0) 86 | child = colander.SchemaNode(CardRef()) 87 | 88 | -------------------------------------------------------------------------------- /storyweb/static/style/medium.less: -------------------------------------------------------------------------------- 1 | @medium-editor-bgcolor: @theme-darker; 2 | @medium-editor-border-color: #fff; //darken(@theme-darker, 15%); 3 | @medium-editor-button-size: 60px; 4 | @medium-editor-hover-color: darken(@theme-darker, 15%); 5 | @medium-editor-link-color: #fff; 6 | @medium-editor-border-radius: @default-radius; 7 | 8 | .medium-toolbar-arrow-under:after { 9 | top: @medium-editor-button-size; 10 | border-color: @medium-editor-bgcolor transparent transparent transparent; 11 | } 12 | 13 | .medium-toolbar-arrow-over:before { 14 | border-color: transparent transparent @medium-editor-bgcolor transparent; 15 | } 16 | 17 | .medium-editor-toolbar { 18 | border: 1px solid @medium-editor-border-color; 19 | background-color: @medium-editor-bgcolor; 20 | border-radius: @medium-editor-border-radius; 21 | transition: top .075s ease-out,left .075s ease-out; 22 | 23 | li { 24 | button { 25 | min-width: @medium-editor-button-size; 26 | height: @medium-editor-button-size; 27 | border: none; 28 | border-right: 1px solid @medium-editor-border-color; 29 | background-color: transparent; 30 | color: @medium-editor-link-color; 31 | box-sizing: border-box; 32 | transition: background-color .2s ease-in, color .2s ease-in; 33 | &:hover { 34 | background-color: @medium-editor-hover-color; 35 | color: #fff; 36 | } 37 | } 38 | 39 | .medium-editor-button-first { 40 | border-top-left-radius: @medium-editor-border-radius; 41 | border-bottom-left-radius: @medium-editor-border-radius; 42 | } 43 | 44 | .medium-editor-button-last { 45 | border-right: none; 46 | border-top-right-radius: @medium-editor-border-radius; 47 | border-bottom-right-radius: @medium-editor-border-radius; 48 | } 49 | 50 | .medium-editor-button-active { 51 | background-color: @medium-editor-hover-color; 52 | color: #fff; 53 | } 54 | } 55 | } 56 | 57 | .medium-editor-toolbar-form-anchor { 58 | background: @medium-editor-bgcolor; 59 | color: #fff; 60 | border-radius: @medium-editor-border-radius; 61 | 62 | input { 63 | height: @medium-editor-button-size; 64 | background: @medium-editor-bgcolor; 65 | color: @medium-editor-link-color; 66 | &::-webkit-input-placeholder { 67 | color: #fff; 68 | color: rgba(255, 255, 255, .8); 69 | } 70 | &:-moz-placeholder { /* Firefox 18- */ 71 | color: #fff; 72 | color: rgba(255, 255, 255, .8); 73 | } 74 | &::-moz-placeholder { /* Firefox 19+ */ 75 | color: #fff; 76 | color: rgba(255, 255, 255, .8); 77 | } 78 | &:-ms-input-placeholder { 79 | color: #fff; 80 | color: rgba(255, 255, 255, .8); 81 | } 82 | } 83 | 84 | a { 85 | color: @medium-editor-link-color; 86 | } 87 | } 88 | 89 | .medium-editor-toolbar-anchor-preview { 90 | background: @medium-editor-bgcolor; 91 | color: @medium-editor-link-color; 92 | border-radius: @medium-editor-border-radius; 93 | } 94 | 95 | .medium-editor-placeholder:after { 96 | color: @medium-editor-border-color; 97 | } 98 | -------------------------------------------------------------------------------- /storyweb/views/admin.py: -------------------------------------------------------------------------------- 1 | from flask import g, redirect, url_for, request 2 | from wtforms import PasswordField, TextAreaField 3 | from flask.ext.admin import Admin 4 | from flask.ext.admin.contrib.sqla import ModelView 5 | 6 | from storyweb.core import app, db 7 | from storyweb.model import User, Card, Reference, Link 8 | 9 | 10 | class AppModelView(ModelView): 11 | 12 | def is_accessible(self): 13 | if g.user is None: 14 | return False 15 | if not g.user.is_active(): 16 | return False 17 | if not g.user.is_admin: 18 | return False 19 | return True 20 | 21 | def _handle_view(self, name, **kwargs): 22 | if not self.is_accessible(): 23 | return redirect(url_for('login', next=request.url)) 24 | 25 | 26 | class UserAdmin(AppModelView): 27 | column_list = [ 28 | 'email', 29 | 'display_name', 30 | 'active', 31 | 'is_admin', 32 | 'is_editor' 33 | ] 34 | column_exclude_list = ['password_hash'] 35 | 36 | form_excluded_columns = [ 37 | 'password_hash', 38 | 'blocks', 39 | 'created_at', 40 | 'updated_at', 41 | 'active' 42 | ] 43 | 44 | column_labels = { 45 | 'email': 'E-Mail', 46 | 'is_admin': 'Administrator', 47 | 'is_editor': 'Editor', 48 | 'display_name': 'Name' 49 | } 50 | 51 | def scaffold_form(self): 52 | form_class = super(UserAdmin, self).scaffold_form() 53 | form_class.password = PasswordField('Password') 54 | return form_class 55 | 56 | def on_model_change(self, form, model, is_created): 57 | model.password = form.password.data 58 | 59 | 60 | class CardAdmin(AppModelView): 61 | column_list = [ 62 | 'title', 63 | 'category' 64 | ] 65 | 66 | form_overrides = { 67 | 'text': TextAreaField 68 | } 69 | 70 | form_excluded_columns = [ 71 | 'created_at', 72 | 'updated_at', 73 | 'links', 74 | 'references', 75 | 'linked', 76 | 'alias_objects' 77 | ] 78 | 79 | column_labels = { 80 | 'text': 'Text' 81 | } 82 | 83 | def on_model_change(self, form, model, is_created): 84 | pass 85 | 86 | 87 | class ReferenceAdmin(AppModelView): 88 | column_list = [ 89 | 'citation', 90 | 'source' 91 | ] 92 | 93 | #form_overrides = { 94 | # 'text': TextAreaField 95 | #} 96 | 97 | form_excluded_columns = [ 98 | 'created_at', 99 | 'updated_at' 100 | ] 101 | 102 | column_labels = { 103 | 'source_url': 'Source Link', 104 | 'url': 'Web Link' 105 | } 106 | 107 | def on_model_change(self, form, model, is_created): 108 | pass 109 | 110 | 111 | class LinkAdmin(AppModelView): 112 | column_list = [ 113 | 'parent', 114 | 'status', 115 | 'child' 116 | ] 117 | 118 | form_excluded_columns = [ 119 | 'created_at', 120 | 'updated_at' 121 | ] 122 | 123 | column_labels = { 124 | } 125 | 126 | def on_model_change(self, form, model, is_created): 127 | pass 128 | 129 | 130 | admin = Admin(app, name=app.config.get('APP_TITLE')) 131 | admin.add_view(UserAdmin(User, db.session)) 132 | admin.add_view(CardAdmin(Card, db.session)) 133 | admin.add_view(LinkAdmin(Link, db.session)) 134 | admin.add_view(ReferenceAdmin(Reference, db.session)) 135 | -------------------------------------------------------------------------------- /storyweb/static/style/loader.less: -------------------------------------------------------------------------------- 1 | /*! 2 | * angular-loading-bar v0.6.0 3 | * https://chieffancypants.github.io/angular-loading-bar 4 | * Copyright (c) 2014 Wes Cruver 5 | * License: MIT 6 | */ 7 | 8 | /* Make clicks pass-through */ 9 | #loading-bar, 10 | #loading-bar-spinner { 11 | pointer-events: none; 12 | -webkit-pointer-events: none; 13 | -webkit-transition: 350ms linear all; 14 | -moz-transition: 350ms linear all; 15 | -o-transition: 350ms linear all; 16 | transition: 350ms linear all; 17 | } 18 | 19 | #loading-bar.ng-enter, 20 | #loading-bar.ng-leave.ng-leave-active, 21 | #loading-bar-spinner.ng-enter, 22 | #loading-bar-spinner.ng-leave.ng-leave-active { 23 | opacity: 0; 24 | } 25 | 26 | #loading-bar.ng-enter.ng-enter-active, 27 | #loading-bar.ng-leave, 28 | #loading-bar-spinner.ng-enter.ng-enter-active, 29 | #loading-bar-spinner.ng-leave { 30 | opacity: 1; 31 | } 32 | 33 | #loading-bar .bar { 34 | -webkit-transition: width 350ms; 35 | -moz-transition: width 350ms; 36 | -o-transition: width 350ms; 37 | transition: width 350ms; 38 | 39 | background: @loader-color; 40 | position: fixed; 41 | z-index: 10002; 42 | top: 0; 43 | left: 0; 44 | width: 100%; 45 | height: 2px; 46 | border-bottom-right-radius: 1px; 47 | border-top-right-radius: 1px; 48 | } 49 | 50 | /* Fancy blur effect */ 51 | #loading-bar .peg { 52 | position: absolute; 53 | width: 70px; 54 | right: 0; 55 | top: 0; 56 | height: 2px; 57 | opacity: .45; 58 | -moz-box-shadow: @loader-color 1px 0 6px 1px; 59 | -ms-box-shadow: @loader-color 1px 0 6px 1px; 60 | -webkit-box-shadow: @loader-color 1px 0 6px 1px; 61 | box-shadow: @loader-color 1px 0 6px 1px; 62 | -moz-border-radius: 100%; 63 | -webkit-border-radius: 100%; 64 | border-radius: 100%; 65 | } 66 | 67 | #loading-bar-spinner { 68 | display: block; 69 | position: fixed; 70 | z-index: 10002; 71 | top: 10px; 72 | left: 10px; 73 | } 74 | 75 | #loading-bar-spinner .spinner-icon { 76 | width: 14px; 77 | height: 14px; 78 | 79 | border: solid 2px transparent; 80 | border-top-color: @loader-color; 81 | border-left-color: @loader-color; 82 | border-radius: 10px; 83 | 84 | -webkit-animation: loading-bar-spinner 400ms linear infinite; 85 | -moz-animation: loading-bar-spinner 400ms linear infinite; 86 | -ms-animation: loading-bar-spinner 400ms linear infinite; 87 | -o-animation: loading-bar-spinner 400ms linear infinite; 88 | animation: loading-bar-spinner 400ms linear infinite; 89 | } 90 | 91 | @-webkit-keyframes loading-bar-spinner { 92 | 0% { -webkit-transform: rotate(0deg); transform: rotate(0deg); } 93 | 100% { -webkit-transform: rotate(360deg); transform: rotate(360deg); } 94 | } 95 | @-moz-keyframes loading-bar-spinner { 96 | 0% { -moz-transform: rotate(0deg); transform: rotate(0deg); } 97 | 100% { -moz-transform: rotate(360deg); transform: rotate(360deg); } 98 | } 99 | @-o-keyframes loading-bar-spinner { 100 | 0% { -o-transform: rotate(0deg); transform: rotate(0deg); } 101 | 100% { -o-transform: rotate(360deg); transform: rotate(360deg); } 102 | } 103 | @-ms-keyframes loading-bar-spinner { 104 | 0% { -ms-transform: rotate(0deg); transform: rotate(0deg); } 105 | 100% { -ms-transform: rotate(360deg); transform: rotate(360deg); } 106 | } 107 | @keyframes loading-bar-spinner { 108 | 0% { transform: rotate(0deg); transform: rotate(0deg); } 109 | 100% { transform: rotate(360deg); transform: rotate(360deg); } 110 | } 111 | -------------------------------------------------------------------------------- /storyweb/spiders/opencorp.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from urlparse import urljoin 3 | from itertools import count 4 | import requests 5 | #from pprint import pprint 6 | 7 | from storyweb.core import app 8 | from storyweb.model import Reference, User 9 | from storyweb.spiders.util import Spider, text_score 10 | 11 | log = logging.getLogger(__name__) 12 | API_HOST = 'https://api.opencorporates.com/' 13 | CORP_ID = 'https://opencorporates.com/companies/' 14 | 15 | 16 | def opencorporates_get(path, query): 17 | abs_url = path.startswith('http:') or path.startswith('https:') 18 | url = path if abs_url else urljoin(API_HOST, path) 19 | params = {'per_page': 200} 20 | params.update(query) 21 | res = requests.get(url, params=params) 22 | return res.json() 23 | 24 | 25 | def opencorporates_paginate(path, collection_name, item_name, query): 26 | res = {} 27 | for i in count(1): 28 | if i > res.get('total_pages', 10000): 29 | return 30 | res = opencorporates_get(path, query) 31 | if 'error' in res: 32 | return 33 | res = res.get('results', {}) 34 | for data in res.get(collection_name, []): 35 | data = data.get(item_name) 36 | yield data 37 | 38 | 39 | class OpenCorporates(Spider): 40 | 41 | def make_api_url(self, url): 42 | if '://api.' not in url: 43 | url = url.replace('://', '://api.') 44 | return url 45 | 46 | def make_ref(self, card, score, url, citation): 47 | data = { 48 | 'citation': citation, 49 | 'url': url, 50 | 'source': 'OpenCorporates', 51 | 'score': score, 52 | 'source_url': 'https://opencorporates.com' 53 | } 54 | ref = Reference.find(card, data.get('url')) 55 | if ref is None: 56 | ref = Reference() 57 | ref.save(data, card, User.default_user()) 58 | 59 | def search_organization(self, card): 60 | return self.search_company(card) 61 | 62 | def make_query(self, query): 63 | query = {'q': query} 64 | api_token = app.config.get('OPENCORPORATES_KEY') 65 | if api_token is not None and len(api_token): 66 | query['api_token'] = api_token 67 | else: 68 | log.warning('No OPENCORPORATES_KEY is set') 69 | return query 70 | 71 | def search_company(self, card): 72 | query = self.make_query(card.title) 73 | failures = 0 74 | for company in opencorporates_paginate('companies/search', 'companies', 75 | 'company', query): 76 | url = company.get('opencorporates_url') 77 | score = text_score(company.get('name'), list(card.aliases)) 78 | if score < 70: 79 | failures += 1 80 | continue 81 | else: 82 | failures = 0 83 | 84 | if failures > 3: 85 | break 86 | 87 | citation = 'Company record: %s' % company.get('name') 88 | self.make_ref(card, score, url, citation) 89 | return self.search_person(card) 90 | 91 | def search_person(self, card): 92 | query = self.make_query(card.title) 93 | failures = 0 94 | for officer in opencorporates_paginate('officers/search', 'officers', 95 | 'officer', query): 96 | url = officer.get('opencorporates_url') 97 | score = text_score(officer.get('name'), list(card.aliases)) 98 | if score < 70: 99 | failures += 1 100 | continue 101 | else: 102 | failures = 0 103 | 104 | if failures > 3: 105 | break 106 | 107 | corp_data = officer.get('company') 108 | position = officer.get('position') 109 | if not position: 110 | position = 'an officer' 111 | citation = '%s is %s of %s' % (officer.get('name'), 112 | position, 113 | corp_data.get('name')) 114 | self.make_ref(card, score, url, citation) 115 | return card 116 | -------------------------------------------------------------------------------- /storyweb/migrate/versions/176bef6a90a3_basic_schema.py: -------------------------------------------------------------------------------- 1 | """basic schema 2 | 3 | Revision ID: 176bef6a90a3 4 | Revises: 353dff346d3 5 | Create Date: 2014-12-14 14:49:32.149387 6 | 7 | """ 8 | 9 | # revision identifiers, used by Alembic. 10 | revision = '176bef6a90a3' 11 | down_revision = '353dff346d3' 12 | 13 | from alembic import op 14 | import sqlalchemy as sa 15 | 16 | 17 | def upgrade(): 18 | op.create_table('user', 19 | sa.Column('id', sa.Integer(), nullable=False), 20 | sa.Column('email', sa.Unicode(), nullable=False), 21 | sa.Column('display_name', sa.Unicode(), nullable=True), 22 | sa.Column('password_hash', sa.Unicode(), nullable=False), 23 | sa.Column('is_admin', sa.Boolean(), nullable=False), 24 | sa.Column('is_editor', sa.Boolean(), nullable=False), 25 | sa.Column('active', sa.Boolean(), nullable=False), 26 | sa.Column('created_at', sa.DateTime(), nullable=True), 27 | sa.Column('updated_at', sa.DateTime(), nullable=True), 28 | sa.PrimaryKeyConstraint('id') 29 | ) 30 | op.create_table('card', 31 | sa.Column('id', sa.Integer(), nullable=False), 32 | sa.Column('title', sa.Unicode(), nullable=False), 33 | sa.Column('category', sa.Enum('Person', 'Company', 'Organization', 'Article', name='card_categories'), nullable=False), 34 | sa.Column('text', sa.Unicode(), nullable=True), 35 | sa.Column('author_id', sa.Integer(), nullable=True), 36 | sa.Column('created_at', sa.DateTime(), nullable=True), 37 | sa.Column('updated_at', sa.DateTime(), nullable=True), 38 | sa.ForeignKeyConstraint(['author_id'], ['user.id'], ), 39 | sa.PrimaryKeyConstraint('id') 40 | ) 41 | op.create_table('spider_tag', 42 | sa.Column('id', sa.Integer(), nullable=False), 43 | sa.Column('spider', sa.Unicode(), nullable=True), 44 | sa.Column('card_id', sa.Integer(), nullable=True), 45 | sa.Column('created_at', sa.DateTime(), nullable=True), 46 | sa.Column('updated_at', sa.DateTime(), nullable=True), 47 | sa.ForeignKeyConstraint(['card_id'], ['card.id'], ), 48 | sa.PrimaryKeyConstraint('id') 49 | ) 50 | op.create_table('reference', 51 | sa.Column('id', sa.Integer(), nullable=False), 52 | sa.Column('score', sa.Integer(), nullable=False), 53 | sa.Column('citation', sa.Unicode(), nullable=False), 54 | sa.Column('url', sa.Unicode(), nullable=False), 55 | sa.Column('source', sa.Unicode(), nullable=True), 56 | sa.Column('source_url', sa.Unicode(), nullable=True), 57 | sa.Column('author_id', sa.Integer(), nullable=True), 58 | sa.Column('card_id', sa.Integer(), nullable=True), 59 | sa.Column('created_at', sa.DateTime(), nullable=True), 60 | sa.Column('updated_at', sa.DateTime(), nullable=True), 61 | sa.ForeignKeyConstraint(['author_id'], ['user.id'], ), 62 | sa.ForeignKeyConstraint(['card_id'], ['card.id'], ), 63 | sa.PrimaryKeyConstraint('id') 64 | ) 65 | op.create_table('alias', 66 | sa.Column('id', sa.Integer(), nullable=False), 67 | sa.Column('name', sa.Unicode(), nullable=True), 68 | sa.Column('card_id', sa.Integer(), nullable=True), 69 | sa.ForeignKeyConstraint(['card_id'], ['card.id'], ), 70 | sa.PrimaryKeyConstraint('id') 71 | ) 72 | op.create_table('link', 73 | sa.Column('id', sa.Integer(), nullable=False), 74 | sa.Column('offset', sa.Integer(), nullable=False), 75 | sa.Column('status', sa.Enum('pending', 'approved', 'rejected', name='link_statuses'), nullable=False), 76 | sa.Column('author_id', sa.Integer(), nullable=True), 77 | sa.Column('parent_id', sa.Integer(), nullable=True), 78 | sa.Column('child_id', sa.Integer(), nullable=True), 79 | sa.Column('created_at', sa.DateTime(), nullable=True), 80 | sa.Column('updated_at', sa.DateTime(), nullable=True), 81 | sa.ForeignKeyConstraint(['author_id'], ['user.id'], ), 82 | sa.ForeignKeyConstraint(['child_id'], ['card.id'], ), 83 | sa.ForeignKeyConstraint(['parent_id'], ['card.id'], ), 84 | sa.PrimaryKeyConstraint('id') 85 | ) 86 | 87 | 88 | def downgrade(): 89 | op.drop_table('link') 90 | op.drop_table('alias') 91 | op.drop_table('reference') 92 | op.drop_table('spider_tag') 93 | op.drop_table('card') 94 | op.drop_table('user') 95 | -------------------------------------------------------------------------------- /fabric_deploy/fabfile.py: -------------------------------------------------------------------------------- 1 | import os 2 | from fabric.api import cd, env, task, require, sudo, prefix 3 | from fabric.contrib.files import exists, upload_template 4 | 5 | 6 | VIRTUALENV_DIR = 'env' 7 | CODE_DIR = 'app' 8 | PACKAGES = ( 9 | 'python-dev', 10 | 'python-virtualenv', 11 | 'supervisor', 12 | ) 13 | LOG_DIR = 'logs' 14 | 15 | 16 | @task 17 | def demo(): 18 | env.server_name = 'demo.storyweb.grano.cc' 19 | env.deploy_user = 'fl' 20 | env.deploy_dir = '/var/www/%s/' % env.server_name 21 | env.repo_dir = os.path.join(env.deploy_dir, CODE_DIR) 22 | env.branch = 'master' 23 | env.nginx_bind = '127.0.0.1:80' 24 | env.proxy_host = '127.0.0.1' 25 | env.proxy_port = 11020 26 | env.hosts = ['norton.pudo.org'] 27 | 28 | 29 | @task 30 | def provision(): 31 | require('deploy_user', 'deploy_dir', provided_by=[demo]) 32 | 33 | commands = ( 34 | 'apt-get install -y %s --no-upgrade' % ' '.join(PACKAGES), 35 | 'apt-get build-dep -y lxml --no-upgrade', 36 | 'mkdir -p %s' % env.deploy_dir, 37 | 'chown -R %s:%s %s' % (env.deploy_user, env.deploy_user, env.deploy_dir), 38 | ) 39 | sudo('; '.join(commands)) 40 | 41 | 42 | @task 43 | def deploy(): 44 | require('deploy_user', 'deploy_dir', 'branch', 45 | provided_by=[demo]) 46 | 47 | ve_dir = os.path.join(env.deploy_dir, VIRTUALENV_DIR) 48 | 49 | if not exists(ve_dir): 50 | sudo('virtualenv -p python2.7 %s' % ve_dir, user=env.deploy_user) 51 | 52 | if not exists(env.repo_dir): 53 | with cd(env.deploy_dir): 54 | sudo('git clone -b %s https://github.com/granoproject/storyweb.git %s' 55 | % (env.branch, env.repo_dir), user=env.deploy_user) 56 | else: 57 | with cd(env.repo_dir): 58 | sudo('git reset --hard HEAD', user=env.deploy_user) 59 | sudo('git checkout -B %s' % env.branch, user=env.deploy_user) 60 | sudo('git pull origin %s' % env.branch, user=env.deploy_user) 61 | 62 | with cd(env.repo_dir), prefix('. ../%s/bin/activate' % VIRTUALENV_DIR): 63 | sudo('pip install -e ./', user=env.deploy_user) 64 | sudo('pip install -r requirements.txt', user=env.deploy_user) 65 | sudo('bower install', user=env.deploy_user) 66 | 67 | # render and upload templates 68 | upload_template(os.path.join(os.path.dirname(__file__), 'nginx.template'), 69 | '/etc/nginx/sites-enabled/%s' % env.server_name, 70 | get_nginx_template_context(), use_sudo=True, backup=False) 71 | upload_template(os.path.join(os.path.dirname(__file__), 'supervisor.template'), 72 | '/etc/supervisor/conf.d/%s.conf' % env.server_name, 73 | get_supervisor_template_context(), use_sudo=True, backup=False) 74 | # make sure logging dir exists and update processes 75 | log_dir = os.path.join(env.deploy_dir, LOG_DIR) 76 | sudo('mkdir -p %s' % log_dir, user=env.deploy_user) 77 | sudo('supervisorctl update') 78 | sudo('supervisorctl restart %s-web' % env.server_name) 79 | sudo('supervisorctl restart %s-worker' % env.server_name) 80 | sudo('/etc/init.d/nginx reload') 81 | 82 | 83 | def get_nginx_template_context(): 84 | static_dir = '%s/storyweb/static/' % env.repo_dir 85 | return { 86 | 'server-name': env.server_name, 87 | 'server-port': env.nginx_bind, 88 | 'static-path': os.path.join(env.deploy_dir, static_dir), 89 | 'log': os.path.join(env.deploy_dir, LOG_DIR, 'nginx.log'), 90 | 'err-log': os.path.join(env.deploy_dir, LOG_DIR, 'nginx.err'), 91 | 'proxy-host': env.proxy_host, 92 | 'proxy-port': env.proxy_port 93 | } 94 | 95 | 96 | def get_supervisor_template_context(): 97 | return { 98 | 'server-name': env.server_name, 99 | 'user': env.deploy_user, 100 | 'deploy-dir': env.deploy_dir, 101 | 'project-dir': os.path.join(env.deploy_dir, CODE_DIR), 102 | 've-dir': os.path.join(env.deploy_dir, VIRTUALENV_DIR), 103 | 'gunicorn-log': os.path.join(env.deploy_dir, LOG_DIR, 'gunicorn.log'), 104 | 'gunicorn-err-log': os.path.join(env.deploy_dir, LOG_DIR, 'gunicorn.err'), 105 | 'celery-log': os.path.join(env.deploy_dir, LOG_DIR, 'celery.log'), 106 | 'celery-err-log': os.path.join(env.deploy_dir, LOG_DIR, 'celery.err'), 107 | 'host': env.proxy_host, 108 | 'port': env.proxy_port 109 | } 110 | -------------------------------------------------------------------------------- /storyweb/model/card.py: -------------------------------------------------------------------------------- 1 | import colander 2 | from datetime import datetime 3 | from hashlib import sha1 4 | from sqlalchemy import or_ 5 | from sqlalchemy.orm import aliased 6 | from sqlalchemy.ext.associationproxy import association_proxy 7 | 8 | from storyweb.core import db, url_for 9 | from storyweb.model.user import User 10 | from storyweb.analysis.html import clean_html 11 | from storyweb.model.util import db_compare, db_norm 12 | from storyweb.model.util import html_summary 13 | from storyweb.model.forms import Ref 14 | 15 | 16 | class Alias(db.Model): 17 | id = db.Column(db.Integer, primary_key=True) 18 | name = db.Column(db.Unicode()) 19 | card_id = db.Column(db.Integer(), db.ForeignKey('card.id')) 20 | 21 | card = db.relationship('Card', 22 | backref=db.backref("alias_objects", 23 | cascade="all, delete-orphan")) 24 | 25 | def __init__(self, name): 26 | self.name = name 27 | 28 | 29 | class Card(db.Model): 30 | doc_type = 'card' 31 | 32 | PERSON = 'Person' 33 | COMPANY = 'Company' 34 | ORGANIZATION = 'Organization' 35 | ARTICLE = 'Article' 36 | CATEGORIES = [PERSON, COMPANY, ORGANIZATION, ARTICLE] 37 | 38 | id = db.Column(db.Integer, primary_key=True) 39 | title = db.Column(db.Unicode, nullable=False) 40 | category = db.Column(db.Enum(*CATEGORIES, name='card_categories'), 41 | nullable=False) 42 | text = db.Column(db.Unicode) 43 | 44 | author_id = db.Column(db.Integer(), db.ForeignKey('user.id')) 45 | author = db.relationship(User, backref=db.backref('cards', 46 | lazy='dynamic')) 47 | 48 | created_at = db.Column(db.DateTime, default=datetime.utcnow) 49 | updated_at = db.Column(db.DateTime, default=datetime.utcnow, 50 | onupdate=datetime.utcnow) 51 | 52 | aliases = association_proxy('alias_objects', 'name') 53 | 54 | def sign(self): 55 | sig = sha1(self.text.encode('utf-8')) 56 | sig.update(unicode(self.date or '')) 57 | sig.update(self.title.encode('utf-8')) 58 | return sig.hexdigest() 59 | 60 | def __repr__(self): 61 | return '' % (self.id, self.title, self.category) 62 | 63 | def save(self, raw, author): 64 | from storyweb import queue 65 | raw['id'] = self.id 66 | form = CardForm(validator=unique_title) 67 | data = form.deserialize(raw) 68 | self.title = data.get('title', '').strip() 69 | self.category = data.get('category') 70 | self.text = clean_html(data.get('text', '').strip()) 71 | self.date = data.get('date') 72 | self.aliases = set(data.get('aliases', []) + [data.get('title')]) 73 | self.author = author 74 | db.session.add(self) 75 | db.session.flush() 76 | queue.lookup_all(self.id) 77 | queue.index.apply_async((self.id,), {}, countdown=1) 78 | return self 79 | 80 | def to_dict(self): 81 | return { 82 | 'id': self.id, 83 | 'api_url': url_for('cards_api.view', id=self.id), 84 | 'title': self.title, 85 | 'summary': html_summary(self.text), 86 | 'category': self.category, 87 | 'text': self.text, 88 | 'author': self.author, 89 | 'aliases': self.aliases, 90 | 'references': self.references, 91 | 'created_at': self.created_at, 92 | 'updated_at': self.updated_at, 93 | } 94 | 95 | def to_index(self): 96 | data = self.to_dict() 97 | data.pop('api_url', None) 98 | data['links'] = [] 99 | for link in self.links: 100 | ldata = link.to_dict() 101 | ldata.update(link.child.to_dict()) 102 | ldata.pop('api_url', None) 103 | ldata.pop('links', None) 104 | ldata.pop('aliases', None) 105 | ldata.pop('references', None) 106 | ldata.pop('author', None) 107 | ldata.pop('child', None) 108 | ldata.pop('text', None) 109 | ldata.pop('summary', None) 110 | ldata.pop('created_at', None) 111 | ldata.pop('updated_at', None) 112 | data['links'].append(ldata) 113 | data['references'] = [] 114 | for ref in self.references: 115 | rdata = ref.to_dict() 116 | rdata.pop('api_url', None) 117 | rdata.pop('author', None) 118 | rdata.pop('created_at', None) 119 | rdata.pop('updated_at', None) 120 | data['references'].append(rdata) 121 | return data 122 | 123 | def __unicode__(self): 124 | return self.title 125 | 126 | @classmethod 127 | def suggest(cls, prefix, categories=[]): 128 | if prefix is None or len(prefix) < 0: 129 | return [] 130 | c = aliased(cls) 131 | q = db.session.query(c.id, c.title, c.category) 132 | prefix = prefix.strip().lower() + '%' 133 | q = q.filter(db_norm(c.title).like(prefix)) 134 | if len(categories): 135 | q = q.filter(c.category.in_(categories)) 136 | q = q.limit(10) 137 | options = [] 138 | for row in q.all(): 139 | options.append({ 140 | 'id': row.id, 141 | 'title': row.title, 142 | 'category': row.category 143 | }) 144 | return options 145 | 146 | @classmethod 147 | def by_id(cls, id): 148 | q = db.session.query(cls) 149 | q = q.filter_by(id=id) 150 | return q.first() 151 | 152 | @classmethod 153 | def find(cls, title, category=None): 154 | title = title.lower().strip() 155 | q = db.session.query(cls) 156 | q = q.outerjoin(Alias) 157 | q = q.filter(or_(db_compare(cls.title, title), 158 | db_compare(Alias.name, title))) 159 | if category is not None: 160 | q = q.filter(cls.category == category) 161 | return q.first() 162 | 163 | 164 | class CardRef(Ref): 165 | 166 | def decode(self, data): 167 | if isinstance(data, Card): 168 | return data 169 | if isinstance(data, dict): 170 | data = data.get('id') 171 | return Card.by_id(data) 172 | 173 | 174 | class AliasList(colander.SequenceSchema): 175 | alias = colander.SchemaNode(colander.String()) 176 | 177 | 178 | def unique_title(node, data): 179 | card = Card.find(data.get('title', '')) 180 | if card is not None and card.id != data.get('id'): 181 | raise colander.Invalid(node.get('title'), msg="Already exists") 182 | 183 | 184 | class CardForm(colander.MappingSchema): 185 | id = colander.SchemaNode(colander.Integer(), default=None, missing=None) 186 | title = colander.SchemaNode(colander.String(), default='', missing='') 187 | category = colander.SchemaNode(colander.String(), 188 | validator=colander.OneOf(Card.CATEGORIES)) 189 | text = colander.SchemaNode(colander.String(), default='', missing='') 190 | date = colander.SchemaNode(colander.Date(), default=None, missing=None) 191 | aliases = AliasList(missing=[], default=[]) 192 | -------------------------------------------------------------------------------- /storyweb/static/style/app.less: -------------------------------------------------------------------------------- 1 | // Core variables and mixins 2 | @import "../vendor/bootstrap/less/variables.less"; 3 | @import "../vendor/bootstrap/less/mixins.less"; 4 | 5 | // Local variables 6 | @theme-color: #559dc0; 7 | @theme-darker: darken(@theme-color, 10%); 8 | 9 | @brand-primary: @theme-color; 10 | @loader-color: #d62323; 11 | @default-radius: 3px; 12 | 13 | // Style tiles, Colours 14 | @white: #ffffff; 15 | @border-color: #d7d7d7; 16 | @boxed-color: #fafafa; 17 | 18 | // Style tiles, Fonts 19 | @font-default: 'Open Sans', sans-serif; 20 | @font-fancy: 'Georgia', serif; 21 | 22 | // Reset 23 | @import "../vendor/bootstrap/less/normalize.less"; 24 | @import "../vendor/bootstrap/less/print.less"; 25 | 26 | // Core CSS 27 | @import "../vendor/bootstrap/less/scaffolding.less"; 28 | @import "../vendor/bootstrap/less/type.less"; 29 | @import "../vendor/bootstrap/less/code.less"; 30 | @import "../vendor/bootstrap/less/grid.less"; 31 | @import "../vendor/bootstrap/less/tables.less"; 32 | @import "../vendor/bootstrap/less/forms.less"; 33 | @import "../vendor/bootstrap/less/buttons.less"; 34 | 35 | // Components 36 | @import "../vendor/bootstrap/less/component-animations.less"; 37 | @import "../vendor/bootstrap/less/glyphicons.less"; 38 | @import "../vendor/bootstrap/less/dropdowns.less"; 39 | @import "../vendor/bootstrap/less/button-groups.less"; 40 | @import "../vendor/bootstrap/less/input-groups.less"; 41 | @import "../vendor/bootstrap/less/navs.less"; 42 | @import "../vendor/bootstrap/less/navbar.less"; 43 | @import "../vendor/bootstrap/less/breadcrumbs.less"; 44 | @import "../vendor/bootstrap/less/pagination.less"; 45 | @import "../vendor/bootstrap/less/pager.less"; 46 | @import "../vendor/bootstrap/less/labels.less"; 47 | @import "../vendor/bootstrap/less/badges.less"; 48 | @import "../vendor/bootstrap/less/jumbotron.less"; 49 | @import "../vendor/bootstrap/less/thumbnails.less"; 50 | @import "../vendor/bootstrap/less/alerts.less"; 51 | @import "../vendor/bootstrap/less/progress-bars.less"; 52 | @import "../vendor/bootstrap/less/media.less"; 53 | @import "../vendor/bootstrap/less/list-group.less"; 54 | @import "../vendor/bootstrap/less/panels.less"; 55 | @import "../vendor/bootstrap/less/wells.less"; 56 | @import "../vendor/bootstrap/less/close.less"; 57 | 58 | // Components w/ JavaScript 59 | @import "../vendor/bootstrap/less/modals.less"; 60 | @import "../vendor/bootstrap/less/tooltip.less"; 61 | @import "../vendor/bootstrap/less/popovers.less"; 62 | @import "../vendor/bootstrap/less/carousel.less"; 63 | 64 | // Utility classes 65 | @import "../vendor/bootstrap/less/utilities.less"; 66 | @import "../vendor/bootstrap/less/responsive-utilities.less"; 67 | 68 | // External tools 69 | @import "loader.less"; 70 | @import "../vendor/medium-editor/dist/css/medium-editor.css"; 71 | @import "medium.less"; 72 | 73 | 74 | // Globals & Mixins 75 | 76 | .rounded-corners (@radius: @default-radius) { 77 | -webkit-border-radius: @radius; 78 | -moz-border-radius: @radius; 79 | -ms-border-radius: @radius; 80 | -o-border-radius: @radius; 81 | border-radius: @radius; 82 | } 83 | 84 | .placeholder-color (@color) { 85 | &::-webkit-input-placeholder { 86 | color: @color; 87 | } 88 | 89 | :-moz-placeholder { /* Firefox 18- */ 90 | color: @color; 91 | } 92 | 93 | ::-moz-placeholder { /* Firefox 19+ */ 94 | color: @color; 95 | } 96 | 97 | :-ms-input-placeholder { 98 | color: @color; 99 | } 100 | } 101 | 102 | .box-shadow (@x: 0; @y: 0; @blur: 1px; @color: rgba(0,0,0,0.3)) { 103 | -moz-box-shadow: @arguments; 104 | -webkit-box-shadow: @arguments; 105 | box-shadow: @arguments; 106 | } 107 | 108 | .heading-style { 109 | font-size: 1em; 110 | font-family: @font-default; 111 | font-weight: 400; 112 | letter-spacing: 0.05em; 113 | text-transform: uppercase; 114 | } 115 | 116 | .pseudo-link { 117 | cursor: pointer; 118 | margin-top: -5px; 119 | margin-left: 5px; 120 | } 121 | 122 | .boxed (@color: @boxed-color) { 123 | .rounded-corners(3px); 124 | background-color: @color; 125 | border-right: 1px solid darken(@color, 15%); 126 | border-bottom: 1px solid darken(@color, 15%); 127 | border-left: 1px solid lighten(@color, 15%); 128 | border-top: 1px solid lighten(@color, 15%); 129 | //padding-top: 1em; 130 | //padding-bottom: 1em; 131 | } 132 | 133 | 134 | // Base styles 135 | 136 | body { 137 | font: 1.0em; 138 | background-color: @brand-primary; 139 | font-family: @font-default; 140 | } 141 | 142 | #page { 143 | //background-color: @white; 144 | padding-bottom: 2em; 145 | min-height: 40em; 146 | } 147 | 148 | #login-page { 149 | width: 40em; 150 | margin: 10em auto; 151 | color: white; 152 | 153 | .btn { 154 | .boxed(@theme-darker); 155 | color: white; 156 | 157 | &:hover { 158 | .boxed(darken(@theme-darker, 10%)); 159 | } 160 | } 161 | } 162 | 163 | h1, h2, h3, h4 { 164 | font-family: @font-default; 165 | } 166 | 167 | .hint { 168 | color: @border-color; 169 | } 170 | 171 | .storyweb-card-icon { 172 | display: inline; 173 | } 174 | 175 | .pagination a { 176 | cursor: pointer; 177 | } 178 | 179 | .alert { 180 | .rounded-corners; 181 | &.alert-info { 182 | .boxed(); 183 | //color: white; 184 | } 185 | } 186 | 187 | 188 | // Module: Navbar 189 | 190 | .navbar-inverse { 191 | border-radius: 0; 192 | background-color: @theme-darker; 193 | border: 0; 194 | border-bottom: 1px solid darken(@theme-darker, 3%); 195 | color: @white; 196 | margin-bottom: 1em; 197 | 198 | .navbar-nav li a, .navbar-brand { 199 | color: @white; 200 | .heading-style; 201 | cursor: pointer; 202 | } 203 | 204 | .navbar-nav li a:hover { 205 | color: lighten(@theme-color, 30%); 206 | } 207 | 208 | input { 209 | background-color: @theme-color; 210 | border: 1px solid darken(@theme-darker, 3%); 211 | color: white; 212 | .placeholder-color(lighten(@theme-color, 40%)) 213 | } 214 | 215 | .dropdown-menu { 216 | &, a, a:hover { 217 | color: @theme-darker; 218 | } 219 | } 220 | 221 | .navbar-brand { 222 | position: relative; 223 | 224 | .text { 225 | display: inline-block; 226 | letter-spacing: 0.2em; 227 | 228 | .boldish { 229 | font-weight: 500; 230 | } 231 | } 232 | img { 233 | position: absolute; 234 | width: 3.5em; 235 | z-index: 1000; 236 | } 237 | } 238 | } 239 | 240 | // Module: Footer 241 | 242 | footer { 243 | min-height: 4em; 244 | color: @white; 245 | //.heading-style; 246 | padding-top: 2em; 247 | padding-bottom: 2em; 248 | font-size: 1em !important; 249 | 250 | ul { 251 | list-style-type: none; 252 | margin: 0; 253 | padding: 0; 254 | } 255 | 256 | 257 | a { 258 | color: white; 259 | text-decoration: underline; 260 | } 261 | 262 | a:hover { 263 | color: lighten(@theme-color, 40%); 264 | } 265 | 266 | img { 267 | float: right; 268 | width: 150px; 269 | } 270 | } 271 | 272 | // Module page header 273 | 274 | .page-header { 275 | border-bottom: 0; 276 | color: white; 277 | margin: 0; 278 | 279 | h1 { 280 | font-weight: 800; 281 | margin: 0.25em 0 0.5em -15px; 282 | font-size: 1.7em; 283 | 284 | .card-icon { 285 | display: inline-block; 286 | width: 30px; 287 | } 288 | } 289 | 290 | .input-title { 291 | width: 80%; 292 | background-color: inherit; 293 | letter-spacing: 0.05em; 294 | border: 0px; 295 | padding: 9px 7px 9px 7px; 296 | margin: -9px 0px -9px 0; 297 | cursor: pointer; 298 | 299 | &:focus { 300 | outline: none; 301 | background-color: lighten(@theme-color, 10%); 302 | border: 1px solid lighten(@theme-color, 3%); 303 | cursor: text; 304 | } 305 | } 306 | 307 | .btn { 308 | .boxed(@theme-darker); 309 | color: white; 310 | margin-top: 0.25em; 311 | margin-right: -15px; 312 | 313 | &:hover { 314 | .boxed(darken(@theme-darker, 10%)); 315 | } 316 | } 317 | 318 | } 319 | 320 | .border-right { 321 | border-right: 1px solid @border-color; 322 | } 323 | 324 | // Module: Text editor 325 | 326 | 327 | .input-text { 328 | font-size: 1.2em; 329 | font-weight: 400; 330 | font-family: @font-fancy; 331 | outline: none; 332 | padding: 0.2em 0.5em 0.2em 0.5em; 333 | 334 | &:focus { 335 | outline: 1px solid @border-color; 336 | background-color: #fff; 337 | } 338 | 339 | min-height: 10em; 340 | 341 | .highlight { 342 | background-color: @brand-primary; 343 | color: @white; 344 | } 345 | } 346 | 347 | /* Module: Cards */ 348 | 349 | .card-frame { 350 | .boxed; 351 | padding-top: 1em; 352 | padding-bottom: 1em; 353 | } 354 | 355 | .card { 356 | border-bottom:1px solid @border-color; 357 | padding: 0.6em 0 0.4em 0; 358 | 359 | .card-icon { 360 | color: @brand-primary; 361 | display: inline-block; 362 | font-size: 2em; 363 | width: 50px; 364 | float: left; 365 | text-align: center; 366 | } 367 | 368 | .card-header { 369 | .actions a { 370 | display: inline-block; 371 | font-size: 1.1em; 372 | margin-left: 0.5em; 373 | margin-top: 0.5em; 374 | cursor: pointer; 375 | } 376 | } 377 | 378 | .card-title { 379 | .heading-style; 380 | padding-top: 0.7em; 381 | cursor: pointer; 382 | 383 | .count { 384 | color: darken(@border-color, 10%); 385 | } 386 | } 387 | 388 | .card-title:hover { 389 | color: @brand-primary; 390 | } 391 | 392 | .card-title.card-title-edit{ 393 | width: 70%; 394 | margin-top: 0.6em; 395 | padding-top: 0.1em; 396 | } 397 | 398 | .edit-field { 399 | width: 78%; 400 | height: 300px; 401 | border: 1px solid @border-color; 402 | } 403 | 404 | ul { 405 | padding-left: 1.3em; 406 | list-style-type: square; 407 | } 408 | 409 | 410 | button.btn { 411 | border-radius: 0; 412 | .heading-style; 413 | border: 1px solid @brand-primary; 414 | background-color: white; 415 | } 416 | 417 | button.btn:hover{ 418 | background: @brand-primary; 419 | color:#fff; 420 | } 421 | } 422 | 423 | /* Nav Tabs aka a mess right now. */ 424 | 425 | .link-tabs { 426 | .nav-tabs { 427 | padding: 0 0 1em 0; 428 | } 429 | 430 | .nav-tabs.nav-justified > .active > a, .nav-tabs.nav-justified > .active > a:hover, .nav-tabs.nav-justified > .active > a:focus{ 431 | border: 0px; 432 | border-radius: 0px; 433 | background-color: @brand-primary; 434 | color: @white; 435 | } 436 | 437 | .nav-tabs.nav-justified > .active > a:hover{ 438 | background: #e1e1e1; 439 | } 440 | 441 | .nav-tabs.nav-justified > li > a{ 442 | border: 0px; 443 | border-radius:0px; 444 | .heading-style; 445 | } 446 | 447 | .nav > li > a{ 448 | position: relative; 449 | display: block; 450 | padding: 5px 15px; 451 | } 452 | } 453 | 454 | .link-list { 455 | overflow-y: scroll; 456 | } 457 | 458 | .reference { 459 | margin: 2px 0px 7px; 460 | 461 | .citation { 462 | .heading-style; 463 | text-overflow: ellipsis; 464 | } 465 | 466 | .source { 467 | display: block; 468 | font-size: 0.8em; 469 | color: #999; 470 | a { 471 | color: #999; 472 | text-decoration: underline; 473 | } 474 | 475 | .score { 476 | //color: @brand-primary; 477 | } 478 | } 479 | } 480 | 481 | .sub_header { 482 | .heading-style; 483 | font-weight:400; 484 | font-size:1.2em; 485 | } 486 | 487 | 488 | // Module: Front page 489 | 490 | .card-item { 491 | .boxed; 492 | padding: 0em 1em 0em 1em; 493 | margin-bottom: 2em; 494 | min-height: 5em; 495 | cursor: pointer; 496 | 497 | h2 { 498 | .heading-style; 499 | font-size: 1.3em; 500 | line-height: 1.3em; 501 | } 502 | 503 | p { 504 | line-height: 1.5em; 505 | color: darken(@border-color, 30%); 506 | } 507 | 508 | p.credit { 509 | .heading-style; 510 | color: #333; 511 | font-size: 0.8em; 512 | } 513 | } 514 | 515 | 516 | // Module: new card 517 | 518 | .new-card { 519 | .cur { 520 | color: @brand-primary; 521 | } 522 | 523 | .title-field { 524 | padding-bottom: 2em; 525 | 526 | input { 527 | font-size: 1.3em; 528 | width: 100%; 529 | padding: 0.2em; 530 | } 531 | } 532 | 533 | .cat-select { 534 | display: inline-block; 535 | 536 | .card-icon { 537 | margin-top: 10px; 538 | display: inline-block; 539 | font-size: 1.5em; 540 | width: 40px; 541 | text-align: center; 542 | cursor: pointer; 543 | } 544 | 545 | &.selected .card-icon { 546 | color: @brand-primary; 547 | } 548 | 549 | } 550 | 551 | input.btn { 552 | border-radius: 0; 553 | float: right; 554 | .heading-style; 555 | border: 1px solid @brand-primary; 556 | background-color: white; 557 | } 558 | 559 | input.btn:hover { 560 | background: @brand-primary; 561 | color:#fff; 562 | } 563 | } 564 | --------------------------------------------------------------------------------