To install the bookmarklet, drag this link -- SciBot %s -- to your bookmarks bar.
98 |If you need to copy/paste the bookmarklet's code into a bookmarklet, it's here:
99 |%s
100 |
101 |
102 | """
103 |
104 |
105 | def bookmarklet_wrapper(request, endpoint):
106 | """ Return text of the SciBot bookmarklet """
107 | normalized = 'https://' + request.host
108 | code = bookmarklet_base % (normalized, endpoint)
109 | bookmarklet = code.replace('"', '"').replace('\n','')
110 | html = html_base % (bookmarklet, request.host.split('.', 1)[-1], code)
111 | return html
112 |
113 |
114 | # rrid endpoint
115 |
116 | from scibot.extract import process_POST_request, find_rrids as finder
117 | from scibot.check import check_already_submitted
118 | from scibot.services import existing_tags, get_pmid, rrid_resolver_xml
119 | from scibot.submit import annotate_doi_pmid, submit_to_h
120 |
121 |
122 | def make_find_check_resolve_submit(finder: Finder, notSubmittedCheck: Checker,
123 | resolver: Resolver, submitter: Submitter) -> Processor:
124 | def inner(text: str) -> Generator:
125 | for found in finder(text):
126 | log.info(found)
127 | if notSubmittedCheck(found):
128 | resolved = resolver(found)
129 | yield submitter(found, resolved)
130 |
131 | return inner
132 |
133 |
134 | def pmid_logic(doi, pmid_from_source, target_uri=None, document=None, h=None, tags=None):
135 | # TODO move the annotation of errors out of this
136 | if doi:
137 | pmid_from_doi = get_pmid(doi)
138 | else:
139 | pmid_from_doi = None
140 |
141 | if pmid_from_source and pmid_from_doi:
142 | if pmid_from_source == pmid_from_doi:
143 | pmid = pmid_from_source
144 | else:
145 | # TODO responses -> db
146 | # TODO tag for marking errors explicitly without the dashboard?
147 | r1 = annotate_doi_pmid(target_uri, document, None, pmid_from_doi, h, tags, 'ERROR\nPMID from DOI')
148 | r2 = annotate_doi_pmid(target_uri, document, None, pmid_from_source, h, tags, 'ERROR\nPMID from source')
149 | pmid = None
150 | elif pmid_from_source:
151 | pmid = pmid_from_source
152 | elif pmid_from_doi:
153 | pmid = pmid_from_doi
154 | else:
155 | pmid = None
156 |
157 | return pmid
158 |
159 |
160 | def rrid_POST(request, h, logloc, URL_LOCK):
161 | (target_uri, document, doi, pmid_from_source,
162 | head, body, text, cleaned_text) = process_POST_request(request)
163 | running = URL_LOCK.start_uri(target_uri)
164 | log.info(target_uri)
165 | if running:
166 | log.info('################# EARLY EXIT')
167 | return 'URI Already running ' + target_uri
168 |
169 | try:
170 | tags, unresolved_exacts = existing_tags(target_uri, h)
171 | pmid = pmid_logic(doi, pmid_from_source, target_uri, document, h, tags)
172 | r = annotate_doi_pmid(target_uri, document, doi, pmid, h, tags) # todo r -> db with responses
173 |
174 | # these values are defined up here as shared state that will be
175 | # mutated across multiple calls to checker, resolver, and submitter
176 | # this is a really bad design because it is not clear that processText
177 | # actually does this ... once again, python is best if you just use the
178 | # objects and give up any hope for an alternative approach, the way it
179 | # is done here also makes the scope where these values could be used
180 | # completely ambiguous and hard to understand/reason about
181 |
182 | found_rrids = {}
183 | existing = []
184 | existing_with_suffixes = []
185 |
186 | def checker(found):
187 | prefix, exact, exact_for_hypothesis, suffix = found
188 | return not check_already_submitted(exact, exact_for_hypothesis,
189 | found_rrids, tags, unresolved_exacts)
190 |
191 | def resolver(found):
192 | prefix, exact, exact_for_hypothesis, suffix = found
193 | return rrid_resolver_xml(exact, found_rrids)
194 |
195 | def submitter(found, resolved):
196 | return submit_to_h(target_uri, document, found, resolved, h, found_rrids,
197 | existing, existing_with_suffixes)
198 |
199 | processText = make_find_check_resolve_submit(finder, checker, resolver, submitter)
200 |
201 | responses = list(processText(cleaned_text)) # this call runs everything
202 |
203 | results = ', '.join(found_rrids.keys())
204 | write_stdout(target_uri, document, doi, pmid, found_rrids, head, body, text, h)
205 | write_log(target_uri, document, doi, pmid, found_rrids, head, body, text, h)
206 |
207 | except BaseException as e:
208 | # there are some other linger issues that are what was causing
209 | # uris to get stuck as always running in sync
210 | log.exception(e)
211 | raise e
212 |
213 | finally:
214 | URL_LOCK.stop_uri(target_uri)
215 |
216 | return results, 200, {'Content-Type': 'text/plain',
217 | 'Access-Control-Allow-Origin':'*'}
218 |
219 |
220 | def rrid_OPTIONS(request):
221 | try:
222 | request_headers = request.headers['Access-Control-Request-Headers'].lower()
223 | request_headers = re.findall('\w(?:[-\w]*\w)', request_headers)
224 | except KeyError:
225 | request_headers = []
226 | response_headers = ['access-control-allow-origin']
227 | for req_acoa_header in request_headers:
228 | if req_acoa_header not in response_headers:
229 | response_headers.append(req_acoa_header)
230 | response_headers = ','.join(response_headers)
231 | return '', 204, {'Access-Control-Allow-Origin': '*',
232 | 'Access-Control-Allow-Headers': response_headers}
233 |
234 |
235 | def rrid_wrapper(request, h, logloc, URL_LOCK):
236 | """ Receive an article, parse RRIDs, resolve them, create annotations, log results """
237 | if request.method == 'OPTIONS':
238 | return rrid_OPTIONS(request)
239 | elif request.method == 'POST':
240 | return rrid_POST(request, h, logloc, URL_LOCK)
241 | else:
242 | return abort(405)
243 |
244 |
245 | def main(local=False):
246 | from scibot.config import api_token, username, group, group2
247 | print(username, group, group2) # sanity check
248 | from scibot.sync import __doc__ as sync__doc__, Locker, client
249 | from scibot.config import syncword
250 | if syncword is None:
251 | raise KeyError('Please set the SCIBOT_SYNC environment variable')
252 |
253 | from docopt import docopt, parse_defaults
254 | _sdefaults = {o.name:o.value if o.argcount else None for o in parse_defaults(sync__doc__)}
255 | _backup_sync_port = int(_sdefaults['--port'])
256 |
257 | app = Flask('scibot bookmarklet server')
258 |
259 | h = HypothesisUtils(username=username, token=api_token, group=group)
260 | h2 = HypothesisUtils(username=username, token=api_token, group=group2)
261 |
262 | if __name__ == '__main__':
263 | args = docopt(__doc__)
264 | _sync_port = args['--sync-port']
265 |
266 | if _sync_port:
267 | sync_port = int(_sync_port)
268 | else:
269 | sync_port = _backup_sync_port
270 | else:
271 | sync_port = _backup_sync_port
272 |
273 | chan = 'localhost', sync_port
274 |
275 | # TODO
276 | #try:
277 | #except AuthenticationError as e:
278 | #raise e
279 | send = run(client, chan, syncword)
280 | URL_LOCK = Locker(send)
281 | app.URL_LOCK = URL_LOCK
282 |
283 | #@app.route('/synctest', methods=['GET'])
284 | def synctest():
285 | URL_LOCK.start_uri('a-test-uri')
286 | URL_LOCK.stop_uri('a-test-uri')
287 | return 'test-passed?'
288 |
289 | synctest()
290 |
291 | @app.route('/controlled-tags', methods=['GET'])
292 | def route_controlled_tags():
293 | curator_tags = curatorTags() # TODO need client support for workflow:RRID -> * here
294 | return '\n'.join(curator_tags), 200, {'Content-Type':'text/plain; charset=utf-8'}
295 |
296 | @app.route('/rrid', methods=['POST', 'OPTIONS'])
297 | def rrid():
298 | return rrid_wrapper(request, h, 'logs/rrid/', URL_LOCK)
299 |
300 | @app.route('/validaterrid', methods=['POST', 'OPTIONS'])
301 | def validaterrid(request):
302 | return rrid_wrapper(request, h2, 'logs/validaterrid/', URL_LOCK)
303 |
304 | @app.route('/bookmarklet', methods=['GET'])
305 | def bookmarklet():
306 | return bookmarklet_wrapper(request, 'rrid')
307 |
308 | @app.route('/validatebookmarklet', methods=['GET'])
309 | def validatebookmarklet():
310 | return bookmarklet_wrapper(request, 'validaterrid')
311 |
312 | @app.route('/export', methods=['GET'])
313 | def export():
314 | print('starting csv export')
315 | output_rows, DATE = export_impl()
316 | data = StringIO()
317 | writer = csv.writer(data)
318 | writer.writerows(sorted(output_rows))
319 | return gzip.compress(data.getvalue().encode()), 200, {
320 | 'Content-Type': 'text/csv',
321 | 'Content-Disposition': 'attachment;filename = RRID-data-%s.csv' % DATE,
322 | 'Content-Encoding': 'gzip'}
323 |
324 | @app.route('/export.json', methods=['GET'])
325 | def export_json():
326 | print('starting json export')
327 | output_json, DATE = export_json_impl()
328 | data = json.dumps(output_json, sort_keys=True, indent=4)
329 |
330 | return gzip.compress(data.encode()), 200, {
331 | 'Content-Type': 'application/json',
332 | 'Content-Encoding': 'gzip'}
333 |
334 | if not local:
335 | return app
336 | else:
337 | from os.path import expanduser
338 | from wsgiref.simple_server import make_server
339 | from scibot.config import test_host, port_bookmarklet
340 |
341 | print('no login detected, running on localhost only')
342 | host = test_host
343 | port = port_bookmarklet
344 |
345 | print('host: %s, port %s' % ( host, port ))
346 | server = make_server(host, port, app)
347 | # openssl req -new -x509 -keyout scibot-self-sign-temp.pem -out scibot-self-sign-temp.pem -days 365 -nodes
348 | #server.socket = ssl.wrap_socket(server.socket,
349 | #keyfile='/etc/letsencrypt/live/scibot.scicrunch.io/privkey.pem',
350 | #certfile='/etc/letsencrypt/live/scibot.scicrunch.io/fullchain.pem',
351 | #server_side=True)
352 | server.socket = ssl.wrap_socket(server.socket,
353 | keyfile=expanduser('~/files/certs/scibot_test/tmp-nginx.key'),
354 | certfile=expanduser('~/files/certs/scibot_test/tmp-nginx.crt'),
355 | server_side=True)
356 | log.debug('serving forever')
357 | server.serve_forever()
358 |
359 |
360 | if __name__ == '__main__':
361 | main(local=True)
362 |
--------------------------------------------------------------------------------
/scibot/bookmarklet_server.py:
--------------------------------------------------------------------------------
1 | import os
2 |
3 | os.sys.stdout.write(f'\x1b]2;{os.path.basename(__name__)}\x07\n')
4 |
5 | from scibot.bookmarklet import main
6 | app = main()
7 |
8 | if __name__ == '__main__':
9 | from scibot import config
10 | app.run(host='localhost', port=config.port_bookmarklet, threaded=True)
11 |
--------------------------------------------------------------------------------
/scibot/check.py:
--------------------------------------------------------------------------------
1 |
2 | def check_already_submitted(exact, exact_for_hypothesis, found_rrids, tags, unresolved_exacts):
3 | if exact in tags or exact_for_hypothesis in unresolved_exacts:
4 | print('\tskipping %s, already annotated' % exact)
5 | found_rrids[exact] = 'Already Annotated'
6 | return True
7 |
8 |
--------------------------------------------------------------------------------
/scibot/cli.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 | """SciBot command line utilities
3 |
4 | Usage:
5 | scibot db-init [options] [',h.text.count('
'))
437 | t = 0
438 | Data = ''
439 | for t in range(0, len(hsplit)):
440 | if text in hsplit[t]:
441 | Data = hsplit[t].replace(text + ': ', '')
442 |
443 | if search.upper() in Data.upper():
444 | hstr += ' %s: %s To install the bookmarklet, drag this link -- SciBot '
7 | 'localhost:4443 -- to your bookmarks bar. If you need to copy/paste the bookmarklet's code into a "
9 | "bookmarklet, it's here:
Anno #:%s
' % h
445 | hstr += ' Anno Link
'
446 | hstr += repr(h)
447 | hstr += "
" + BaseURL(annos[h])
448 | counter += 1
449 | if hstr == '':
450 | return('no results')
451 | return (str(counter) + ' Results:
' + hstr)
452 |
453 | def BaseURL(anno):
454 | URL = anno.uri.replace(".long", "").replace("/abstract", "").replace("/full","").replace(".short", "").replace(".full", "").replace("http://","").replace("https://","").replace("/FullText","").replace("/Abstract","").replace("/enhanced","")
455 | SplitURL = URL.split("/", URL.count("/"))
456 | if SplitURL[-1] == '':
457 | URL = SplitURL[0] + SplitURL[-2]
458 | else:
459 | URL = SplitURL[0] + SplitURL[-1]
460 | return URL
461 |
462 | def Journal(anno):
463 | URL = anno.uri.replace(".long", "").replace("/abstract", "").replace("/full","").replace(".short", "").replace(".full", "").replace("http://","").replace("https://","").replace("/FullText","").replace("/Abstract","").replace("/enhanced","")
464 | SplitURL = URL.split("/", URL.count("/"))
465 | if len(SplitURL) == 1 or len(SplitURL) == 0:
466 | print(URL)
467 | URL = SplitURL[0]
468 | return URL
469 |
470 | def annoSync(memfile, group, helpers=tuple(), world_ok=False):
471 | if group == '__world__' and not world_ok:
472 | raise ValueError('Group is set to __world__ please run the usual `export HYP_ ...` command.')
473 | get_annos = Memoizer(memfile, api_token, username, group)
474 | yield get_annos
475 | prefilter = preFilter(groups=[group]).export()
476 | hsh = type(f'helperSyncHandler{group}',
477 | (helperSyncHandler,),
478 | dict(memoizer=get_annos,
479 | helpers=helpers))
480 | annos = get_annos()
481 | yield annos
482 | stream_thread, exit_loop = AnnotationStream(annos, prefilter, hsh)()
483 | yield stream_thread
484 | yield exit_loop
485 |
486 | def setup():
487 | get_annos, annos, stream_thread, exit_loop = annoSync(memfile, group, (Curation,))
488 | get_pannos, pannos, pstream_thread, pexit_loop = annoSync(pmemfile, group_staging,
489 | (PublicAnno,), world_ok=True)
490 | def close_stuff():
491 | exit_loop()
492 | stream_thread.join()
493 |
494 | atexit.register(close_stuff)
495 |
496 | stream_thread.start()
497 | app = make_app(annos, pannos)
498 | app.debug=False
499 | return app
500 |
501 | def main():
502 | get_annos, annos, stream_thread, exit_loop = annoSync(memfile, group, (Curation,))
503 | get_pannos, pannos, pstream_thread, exit_loop = annoSync(pmemfile, group_staging,
504 | (PublicAnno,), world_ok=True)
505 |
506 | app = make_app(annos, pannos)
507 | #stream_loop.start()
508 | #pstream_loop.start() # FIXME eventloop already running error...
509 | app.secret_key = 'super secret key'
510 | app.config['SESSION_TYPE'] = 'filesystem'
511 | print(app.view_functions)
512 | app.debug = False
513 | from scibot.config import test_host, port_dashboard
514 | app.run(host=test_host, port=port_dashboard)
515 |
516 | if __name__ == '__main__':
517 | main()
518 |
--------------------------------------------------------------------------------
/scibot/db.py:
--------------------------------------------------------------------------------
1 | import atexit
2 | import asyncio
3 | from pathlib import Path
4 | from datetime import datetime
5 | from itertools import chain
6 | from collections import namedtuple, defaultdict
7 | import json
8 | from hyputils.memex import models
9 | from hyputils.memex.db import init
10 | from hyputils.memex.util.uri import normalize as uri_normalize
11 | from hyputils.memex.db.types import _get_hex_from_urlsafe, _get_urlsafe_from_hex, URLSafeUUID
12 | from hyputils.memex.util.user import split_user
13 | from hyputils.memex.models.document import update_document_metadata
14 | from sqlalchemy import create_engine
15 | from sqlalchemy.sql import text
16 | from sqlalchemy.sql.expression import bindparam
17 | from sqlalchemy.orm.session import sessionmaker
18 | from sqlalchemy.dialects.postgresql import ARRAY
19 | from hyputils.hypothesis import Memoizer
20 | from hyputils.handlers import dbSyncHandler
21 | from hyputils.subscribe import setup_websocket, preFilter
22 | from scibot import config
23 | from scibot.anno import quickload, quickuri, add_doc_all, validate
24 | from scibot.utils import makeSimpleLogger, uri_normalization
25 | from interlex.core import makeParamsValues # FIXME probably need a common import ...
26 | try:
27 | breakpoint
28 | except NameError:
29 | from IPython import embed as breakpoint
30 |
31 |
32 | def getSession(dburi=config.dbUri(), echo=False):
33 | engine = create_engine(dburi, echo=echo)
34 |
35 | Session = sessionmaker()
36 | Session.configure(bind=engine)
37 | session = Session()
38 | return session
39 |
40 |
41 | def init_scibot(database):
42 | dburi = config.dbUri(user='scibot-admin', database=database)
43 | #dburi = dbUri('postgres')
44 | engine = create_engine(dburi)
45 | init(engine, should_create=True, authority='scicrunch')
46 |
47 | Session = sessionmaker()
48 | Session.configure(bind=engine)
49 | session = Session()
50 | file = Path(__file__).parent / '../sql/permissions.sql'
51 | with open(file.as_posix(), 'rt') as f:
52 | sql = f.read()
53 | #args = dict(database=database)
54 | # FIXME XXX evil replace
55 | sql_icky = sql.replace(':database', f'"{database}"')
56 | session.execute(sql_icky)
57 | session.commit()
58 |
59 |
60 | class DbQueryFactory:
61 | """ parent class for creating converters for queries with uniform results """
62 |
63 | convert = tuple()
64 | query = ''
65 |
66 | def ___new__(cls, *args, **kwargs):
67 | return super().__new__(cls)
68 |
69 | def __new__(cls, session):
70 | newcls = cls.bindSession(session)
71 | newcls.__new__ = cls.___new__
72 | return newcls
73 |
74 | @classmethod
75 | def bindSession(cls, session):
76 | # this approach seems better than overloading what __new__ does
77 | # and doing unexpected things in new
78 | classTypeInstance = type(cls.__name__.replace('Factory',''),
79 | (cls,),
80 | dict(session=session))
81 | return classTypeInstance
82 |
83 | def __init__(self, condition=''):
84 | self.condition = condition
85 |
86 | def execute(self, params=None, raw=False):
87 | if params is None:
88 | params = {}
89 | gen = self.session.execute(self.query + ' ' + self.condition, params)
90 | first = next(gen)
91 | if raw:
92 | yield first
93 | yield from gen
94 | else:
95 | Result = namedtuple(self.__class__.__name__ + 'Result', list(first.keys())) # TODO check perf, seems ok?
96 | for result in chain((first,), gen):
97 | yield Result(*(c(v) if c else v for c, v in zip(self.convert, result)))
98 |
99 | @staticmethod
100 | def get_cols(model, no_id=True):
101 | cols = model.__table__.columns.keys()
102 | if no_id:
103 | cols.remove('id')
104 |
105 | return cols
106 |
107 | def _table_insert(self, table_things, table, cols):
108 | def do_defaults(thing, cols):
109 | for col in cols:
110 | value = getattr(thing, col)
111 | if value is None:
112 | c = table.columns[col]
113 | if c.default: # TODO nullable check here?
114 | value = c.default.arg
115 |
116 | yield value
117 |
118 | *templates, params = makeParamsValues(
119 | *([list(do_defaults(t, cols))]
120 | for t in table_things))
121 |
122 | col_expr = f'({", ".join(cols)})'
123 | sql = (f'INSERT INTO {table.name} {col_expr} VALUES ' +
124 | ', '.join(templates) +
125 | 'RETURNING id')
126 |
127 | for thing, rp in zip(table_things, self.session.execute(sql, params)):
128 | thing.id = rp.id
129 |
130 | def insert_bulk(self, things, column_mapping=None, keep_id=False):
131 | # this works around the orm so be sure
132 | # to call session.expunge_all when done
133 | tables = set(t.__table__ for t in things)
134 | for table in tables:
135 | table_things = [t for t in things if t.__table__ == table]
136 | if column_mapping and table.name in column_mapping:
137 | cols = column_mapping[table.name]
138 | else:
139 | cols = self.get_cols(table_things[0].__class__)
140 | self._table_insert(table_things, table, cols)
141 |
142 | def __call__(self, params=None):
143 | return self.execute(params)
144 |
145 | def __iter__(self):
146 | """ works for cases without params """
147 | return self.execute()
148 |
149 |
150 | class AnnoSyncFactory(Memoizer, DbQueryFactory):
151 | log = makeSimpleLogger('scibot.db.sync')
152 | convert = (lambda d: datetime.isoformat(d) + '+00:00',) # FIXME hack
153 | query = 'SELECT updated FROM annotation'
154 | condition = 'WHERE groupid = :groupid ORDER BY updated DESC LIMIT 1' # default condition
155 |
156 | def __init__(self, api_token=config.api_token, username=config.username,
157 | group=config.group, memoization_file=None, condition=''):
158 | super().__init__(memoization_file, api_token=api_token, username=username, group=group)
159 | if condition:
160 | self.condition = condition
161 |
162 | def __call__(self):
163 | """ block upstream call which does undesirable things """
164 | raise NotImplemented
165 |
166 | def get_api_rows(self, search_after=None, stop_at=None):
167 | try:
168 | if self.group == '__world__':
169 | self.condition = 'WHERE groupid = :groupid AND userid = :userid ORDER BY updated DESC LIMIT 1'
170 | userid = f'acct:{self.username}@hypothes.is' # FIXME other registration authorities
171 | last_updated = next(self.execute(params={'groupid':self.group,
172 | 'userid':userid})).updated
173 | else:
174 | last_updated = next(self.execute(params={'groupid':self.group})).updated
175 | self.log.debug(f'last updated at {last_updated} for {self.group}')
176 | except StopIteration:
177 | last_updated = None
178 | self.log.debug(f'no annotations in database for {self.group}')
179 |
180 | if self.memoization_file is None:
181 | rows = list(self.yield_from_api(search_after=last_updated, stop_at=stop_at))
182 | else:
183 | if last_updated:
184 | rows = [a._row for a in self.get_annos() if a.updated > last_updated]
185 | else:
186 | rows = [a._row for a in self.get_annos()]
187 |
188 | return rows
189 |
190 | def sync_annos(self, search_after=None, stop_at=None, api_rows=None, check=False):
191 | """ batch sync """
192 |
193 | if not api_rows:
194 | # TODO stream this using generators?
195 | api_rows = self.get_api_rows(search_after, stop_at)
196 | if not api_rows:
197 | self.log.info(f'all annotations are up to date')
198 | return
199 |
200 | anno_records = [quickload(r) for r in api_rows]
201 |
202 | #qsql = 'SELECT distinct(id, updated) FROM annotation WHERE groupid=:groupid' # makes it a string :/
203 | qsql = 'SELECT id, updated, document_id FROM annotation WHERE groupid=:groupid'
204 | params = dict(groupid=api_rows[0]['group'])
205 | existing = self.session.execute(qsql, params)
206 | dext = {_get_urlsafe_from_hex(id.hex):(up, did) for id, up, did in existing}
207 | dupes = [(a, dext[a['id']][0].isoformat() + '+00:00') for a in anno_records if a['id'] in dext]
208 | maybe_update = [a['id'] for a, u in dupes if a['updated'] > u]
209 | assert len(dupes) == len(maybe_update)
210 | #to_update = tuple(_get_hex_from_urlsafe(i) i for i in maybe_update)
211 | to_delete = {f'id{i}':v for i, v in enumerate(maybe_update)}
212 | if to_delete:
213 | names_or = ' OR '.join(f'id = :{p}' for p in to_delete)
214 | _dsql = text(f'DELETE FROM annotation WHERE {names_or}')
215 | bindparams=tuple(bindparam(name, type_=URLSafeUUID) for name in to_delete)
216 | dsql = _dsql.bindparams(*bindparams)
217 | # delete to avoid collisions, they will be added again later and
218 | # then finalized when the transaction finishes
219 | self.session.execute(dsql, to_delete)
220 |
221 | self.log.debug(f'quickload complete for {len(api_rows)} api_rows')
222 |
223 | anno_id_to_doc_id = self.q_create_docs(api_rows)
224 | self.q_create_annos(anno_records, anno_id_to_doc_id)
225 |
226 | def do_check():
227 | api_rows # so that it is accessible in function scope
228 | self.log.debug('checking for consistency')
229 | annos = self.session.query(models.Annotation).\
230 | filter(models.Annotation.groupid == self.group).all()
231 | #docs = self.session.query(models.Document).all()
232 | durs = self.session.query(models.DocumentURI).all()
233 | doc_uris = defaultdict(set)
234 | _ = [doc_uris[d.document_id].add(d.uri) for d in durs]
235 | doc_uris = dict(doc_uris)
236 | #dms = self.session.query(models.DocumentMeta).all()
237 | #doc_mismatch = [a for a in annos if anno_id_to_doc_id[a.id] != a.document.id] # super slow due to orm fetches
238 | doc_missing = [a for a in annos if a.id not in anno_id_to_doc_id]
239 | assert not doc_missing
240 | doc_mismatch = [a for a in annos if anno_id_to_doc_id[a.id] != a.document_id]
241 | assert not doc_mismatch, doc_mismatch
242 | # don't use the orm to do this, it is too slow even if you send the other queries above
243 | #breakpoint()
244 | uri_mismatch = [(a.target_uri, doc_uris[a.document_id], a)
245 | for a in annos
246 | if a.target_uri not in doc_uris[a.document_id]]
247 | # NOTE hypothesis only allows 1 record per normalized uri, so we have to normalize here as well
248 | maybe_mismatch = set(frozenset(s) for u, s, a in uri_mismatch if not s.add(u))
249 | h_mismatch = set(s for s in maybe_mismatch if len(frozenset(uri_normalize(u) for u in s)) > 1)
250 | self.log.debug(f'h mismatch has {len(h_mismatch)} cases')
251 | # the above normalization is not sufficient for cases where there are two
252 | # hypothes.is normalized uris AND a scibot normalized uri as well
253 | super_mismatch = set(s for s in h_mismatch if len(frozenset(uri_normalization(u) for u in s)) > 1)
254 | assert not super_mismatch, super_mismatch
255 |
256 | if check:
257 | self.session.flush() # have to run this to get the doc ids to work?
258 | do_check()
259 |
260 | self.session.commit()
261 | self.log.debug('commit done')
262 | else:
263 | breakpoint()
264 |
265 | def q_create_annos(self, anno_records, anno_id_to_doc_id):
266 | # NOTE values_sets adds the document_id field and
267 | # so self.types must be called after values_sets completes
268 | values_sets = tuple(self.values_sets(anno_records, anno_id_to_doc_id))
269 | *values_templates, values, bindparams = makeParamsValues(*values_sets,
270 | types=self.types(anno_records))
271 | rec_keys = self.get_rec_keys(anno_records)
272 | sql = text(f'INSERT INTO annotation ({", ".join(rec_keys)}) VALUES {", ".join(values_templates)}')
273 | sql = sql.bindparams(*bindparams)
274 |
275 | def debug_type(column):
276 | # FIXME column name collisions
277 | col = models.Annotation.__table__.columns[column]
278 | ctype = col.type.python_type
279 | ind = rec_keys.index(column)
280 | for values, in values_sets:
281 | if type(values[ind]) != ctype:
282 | print('ERROR IN ', values)
283 |
284 | def debug_templates(column):
285 | col = models.Annotation.__table__.columns[column]
286 | ctype = col.type.python_type
287 | for t in values_templates:
288 | for k, ws_c_vn_ws in zip(rec_keys, t.strip('(').rstrip(')').split(',')):
289 | vn = ws_c_vn_ws.strip().rstrip().strip(':')
290 | v = values[vn]
291 | if k == column and type(v) != ctype:
292 | print('ERROR IN', t)
293 |
294 | try:
295 | self.session.execute(sql, values)
296 | self.log.debug('anno execute done')
297 | except BaseException as e:
298 | self.log.error('YOU ARE IN ERROR SPACE')
299 | breakpoint()
300 |
301 | self.session.flush()
302 | self.log.debug('anno flush done')
303 |
304 | def get_rec_keys(self, anno_records):
305 | def fix_reserved(k):
306 | if k == 'references':
307 | k = '"references"'
308 |
309 | return k
310 |
311 | return [fix_reserved(k) for k in anno_records[0].keys()]
312 |
313 | def values_sets(self, anno_records, anno_id_to_doc_id):
314 | def type_fix(k, v): # TODO is this faster or is type_fix?
315 | if isinstance(v, dict):
316 | return json.dumps(v) # FIXME perf?
317 | elif isinstance(v, list):
318 | if any(isinstance(e, dict) for e in v):
319 | return json.dumps(v) # FIXME perf?
320 | return v
321 |
322 | def make_vs(d):
323 | id = d['id']
324 | document_id = anno_id_to_doc_id[id]
325 | d['document_id'] = document_id
326 | # FIXME does ordering matter here!?
327 | return [type_fix(k, v) for k, v in d.items()], # don't miss the , to make this a value set
328 |
329 | yield from (make_vs(d) for d in anno_records)
330 | self.log.debug('anno values sets done')
331 |
332 | def types(self, datas):
333 | def make_types(d):
334 | def inner(k):
335 | if k == 'id':
336 | return URLSafeUUID
337 | elif k == 'references':
338 | return ARRAY(URLSafeUUID)
339 | else:
340 | return None
341 | return [inner(k) for k in d]
342 |
343 | yield from (make_types(d) for d in datas)
344 |
345 | @staticmethod
346 | def uri_records(row):
347 | uri = row['uri']
348 | return uri, uri_normalization(uri), quickuri(row)
349 |
350 | def q_prepare_docs(self, rows):
351 | existing_unnormed = {r.uri:(r.document_id,
352 | self.convert[0](r.created),
353 | self.convert[0](r.updated))
354 | for r in self.session.execute('SELECT uri, document_id, created, '
355 | 'updated FROM document_uri')}
356 | created_updated = {docid:(created, updated)
357 | for _, (docid, created, updated) in existing_unnormed.items()}
358 | _existing = defaultdict(set)
359 | _ = [_existing[uri_normalization(uri)].add(docid)
360 | for uri, (docid, created, updated) in existing_unnormed.items()]
361 | assert not [_ for _ in _existing.values() if len(_) > 1] # TODO proper handling for this case
362 | h_existing_unnormed = {uri_normalize(uri):docid
363 | for uri, (docid, created, updated) in existing_unnormed.items()}
364 | existing = {k:next(iter(v)) for k, v in _existing.items()} # FIXME issues when things get big
365 | latest_existing = max(u for c, u in created_updated.values()) if created_updated else None
366 |
367 | new_docs = {} # FIXME this is completely opaque since it is not persisted anywhere
368 | for row in sorted(rows, key=lambda r: r['created']):
369 | id = row['id']
370 | uri, uri_normed, (created, updated, claims) = self.uri_records(row)
371 | try:
372 | docid = existing[uri_normed]
373 | dc, du = created_updated[docid]
374 | doc = models.Document(id=docid, created=dc, updated=du)
375 | if doc.updated < updated:
376 | # FIXME TODO update the record?
377 | #self.log.warning('YOU ARE NOT UPDATING A DOC WHEN YOU SHOULD!!!!!!\n'
378 | #f'{docid} {doc.updated} {updated}')
379 | pass
380 |
381 | do_claims = False
382 | except KeyError as e:
383 | if existing:
384 | if row['updated'] <= latest_existing:
385 | # only need to worry if we are recreating
386 | raise e
387 | if uri_normed not in new_docs:
388 | do_claims = True
389 | doc = models.Document(created=created, updated=updated)
390 | self.session.add(doc) # TODO perf testing vs add_all
391 | new_docs[uri_normed] = doc
392 | else:
393 | do_claims = False
394 | doc = new_docs[uri_normed]
395 |
396 | #if type(doc.created) == str:
397 | #breakpoint()
398 | yield id, doc
399 |
400 | if uri_normalize(uri) not in h_existing_unnormed:
401 | # NOTE allowing only the normalized uri can cause confusion (i.e. see checks in sync_annos)
402 | h_existing_unnormed[uri_normalize(uri)] = doc
403 | # TODO do these get added automatically if their doc gets added but exists?
404 | doc_uri = models.DocumentURI(document=doc,
405 | claimant=uri,
406 | uri=uri,
407 | type='self-claim',
408 | created=created,
409 | updated=updated)
410 | yield None, doc_uri
411 |
412 | # because of how this schema is designed
413 | # the only way that this can be fast is
414 | # if we assume that all claims are identical
415 | # FIXME if there is a new claim type then we are toast though :/
416 | # the modelling here assumes that title etc can't change
417 | #print(id, uri, uri_normed, row['user'], row['uri'], row['created'])
418 | if do_claims:
419 | for claim in claims:
420 | #print(id, uri, uri_normed, claim['claimant'], claim['type'], claim['value'])
421 | dm = models.DocumentMeta(document=doc,
422 | created=created,
423 | updated=updated,
424 | **claim)
425 | yield None, dm
426 |
427 | def q_create_docs(self, rows):
428 | ids_docs = list(self.q_prepare_docs(rows))
429 | docs = sorted(set(d for i, d in ids_docs if i), key=lambda d:d.created)
430 | uri_meta = list(d for i, d in ids_docs if not i)
431 | assert len(uri_meta) == len(set(uri_meta))
432 |
433 | # TODO skip the ones with document ids
434 | self.insert_bulk(docs, {'document':['created', 'updated']})
435 |
436 | for um in uri_meta:
437 | um.document_id = um.document.id
438 | um.document = None
439 | del um.document # have to have this or doceument overrides document_id
440 |
441 | self.insert_bulk(uri_meta)
442 | self.session.expunge_all() # preven attempts to add unpersisted
443 | self.session.flush()
444 | self.log.debug('finished inserting docs')
445 | anno_id_to_doc_id = {i:d.id for i, d in ids_docs}
446 | return anno_id_to_doc_id
447 |
448 | def sync_anno_stream(self, search_after=None, stop_at=None):
449 | """ streaming one anno at a time version of sync """
450 | for row in self.yield_from_api(search_after=last_updated, stop_at=stop_at):
451 | yield row, 'TODO'
452 | continue
453 | # TODO
454 | datum = validate(row) # roughly 30x slower than quickload
455 | # the h code I'm calling assumes these are new annos
456 | datum['id'] = row['id']
457 | datum['created'] = row['created']
458 | datum['updated'] = row['updated']
459 | document_dict = datum.pop('document')
460 | document_uri_dicts = document_dict['document_uri_dicts']
461 | document_meta_dicts = document_dict['document_meta_dicts']
462 | a = [models.Annotation(**d,
463 | document_id=dbdocs[uri_normalize(d['target_uri'])].id)
464 | for d in datas] # slow
465 | self.log.debug('making annotations')
466 | self.session.add_all(a)
467 | self.log.debug('adding all annotations')
468 |
469 |
470 | class WebsocketSyncFactory(AnnoSyncFactory):
471 |
472 | def __init__(self,
473 | api_token=config.api_token,
474 | username=config.username,
475 | group=config.group,
476 | helpers=tuple(),
477 | threaded=False):
478 | super().__init__(api_token, username, group)
479 | self.prefilter = preFilter(groups=[group]).export()
480 | handler = type(f'dbSyncHandler{group}', # TODO this is where we customize
481 | (dbSyncHandler,),
482 | {}
483 | #dict(session=self.session)
484 | # self.handler gives access to the session
485 | # so in theory don't need this unless treading breaks something
486 | )
487 | self.filter_handlers = [handler(self.handler)]
488 | self.ws_loop, self.exit_loop = setup_websocket(self.api_token, self.prefilter, self.filter_handlers)
489 | self.threaded = threaded
490 | self.loop = asyncio.get_event_loop()
491 | if self.threaded: # yes do this at init and not at call time, you should know by then
492 | self.stream_thread = Thread(target=self.loop_target,
493 | args=(self.loop, self.ws_loop))
494 |
495 | def handler(self, message):
496 | act = message['options']['action']
497 | print('act', act)
498 | print(message)
499 | if act != 'delete':
500 | row = message['payload'][0]
501 | self.create_anno(row)
502 | if act == 'create':
503 | pass
504 | elif act == 'update':
505 | pass
506 | elif act == 'delete':
507 | pass
508 | elif act == 'flag':
509 | 'lol'
510 | else:
511 | raise UnknownAction(act) # email the maintainer basically
512 |
513 | def create_anno(self, row):
514 | datum = validate(row)
515 |
516 | document_dict = datum.pop('document')
517 | document_uri_dicts = document_dict['document_uri_dicts']
518 | document_meta_dicts = document_dict['document_meta_dicts']
519 |
520 | id = row['id']
521 | target_uri = datum['target_uri']
522 | created = row['created']
523 | updated = row['updated']
524 |
525 | annotation = models.Annotation(**datum)
526 |
527 | document = update_document_metadata( # TODO update normalization rules
528 | self.session,
529 | target_uri,
530 | document_meta_dicts,
531 | document_uri_dicts,
532 | created=created, # FIXME doesn't quite seem right, would klobber
533 | updated=updated)
534 |
535 | print(id)
536 | annotation.document = document
537 | annotation.id = id
538 | annotation.target_uri = target_uri
539 | annotation.created = created
540 | annotation.updated = updated
541 | self.session.add(annotation)
542 | self.session.flush()
543 | self.session.commit() # FIXME hypothesis doesn't call this
544 |
545 | @staticmethod
546 | def loop_target(loop, ws_loop):
547 | asyncio.set_event_loop(loop)
548 | loop.run_until_complete(ws_loop(loop))
549 |
550 | def close_stuff(self):
551 | self.exit_loop()
552 | if self.threaded: # FIXME
553 | self.stream_thread.join()
554 |
555 | def run(self):
556 | atexit.register(self.close_stuff)
557 | if self.threaded:
558 | self.stream_thread.start()
559 | else:
560 | try:
561 | self.loop.run_until_complete(self.ws_loop(self.loop))
562 | except KeyboardInterrupt:
563 | return # at exist will deal with it
564 |
565 |
566 | def uuid_to_urlsafe(uuid):
567 | return _get_urlsafe_from_hex(uuid.hex)
568 |
569 |
570 | class AnnoQueryFactory(DbQueryFactory):
571 | convert = (
572 | uuid_to_urlsafe,
573 | lambda d: datetime.isoformat(d) + '+00:00', # FIXME hack WARNING MAKE SURE ALL TIMESTAMPS THAT GO IN
574 | lambda d: datetime.isoformat(d) + '+00:00', # FIXME hack ARE DERIVED FROM datetime.utcnow()
575 | None,
576 | lambda userid: split_user(userid)['username'],
577 | None,
578 | lambda lst: [uuid_to_urlsafe(uuid) for uuid in lst],
579 | )
580 | query = ('SELECT id, created, updated, target_uri, userid, tags, a.references '
581 | 'FROM annotation AS a')
582 |
583 |
584 | def bindSession(cls, session):
585 | return cls.bindSession(session)
586 |
--------------------------------------------------------------------------------
/scibot/export.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 | from __future__ import print_function
3 | import re
4 | import csv
5 | import pickle
6 | from datetime import date
7 | from collections import defaultdict
8 | from collections import namedtuple, defaultdict
9 | from lxml import etree
10 | from hyputils.hypothesis import HypothesisUtils, HypothesisAnnotation, Memoizer
11 | from scibot.config import memfile, api_token, username, group
12 |
13 | bad_tags = {
14 | 'RRID:Incorrect',
15 | 'RRID:InsufficientMetadata',
16 | 'RRID:Missing',
17 | 'RRID:Unrecognized',
18 | 'RRID:Unresolved',
19 | 'RRID:Validated',
20 | 'RRID:Duplicate',
21 | }
22 |
23 | def get_proper_citation(xml):
24 | root = etree.fromstring(xml)
25 | if root.findall('error'):
26 | proper_citation = ''
27 | else:
28 | data_elements = root.findall('data')[0]
29 | data_elements = [(e.find('name').text, e.find('value').text) for e in data_elements] # these shouldn't duplicate
30 | a = [v for n, v in data_elements if n == 'Proper Citation']
31 | proper_citation = a[0] if a else ''
32 |
33 | return proper_citation
34 |
35 | def fix_trailing_slash(annotated_urls):
36 | for key in [k for k in annotated_urls.keys()]:
37 | if key.endswith('/'):
38 | new_key = key.rstrip('/')
39 | print(new_key)
40 | if new_key in annotated_urls:
41 | annotated_urls[key].extend(annotated_urls.pop(new_key))
42 |
43 | def export_impl():
44 | get_annos = Memoizer(memfile, username=username, api_token=api_token, group=group)
45 | annos = get_annos()
46 |
47 | annotated_urls = defaultdict(list)
48 | for anno in annos:
49 | annotated_urls[anno.uri].append(anno)
50 |
51 | fix_trailing_slash(annotated_urls)
52 |
53 | output_rows = []
54 | for annotated_url in annotated_urls.keys():
55 | #print(annotated_url)
56 | annos = annotated_urls[annotated_url]
57 | replies = defaultdict(list)
58 | PMID = []
59 | for anno in annos: # gotta build the reply structure and get pmid
60 | #print('id:', anno.id)
61 | #print('user:', anno.user)
62 | #print('exact:', anno.exact)
63 | #print('text:', anno.text)
64 | #print('tags:', anno.tags)
65 | #print('type:', anno.type)
66 | #print('references:', anno.references)
67 | if anno.references:
68 | for reference in anno.references: # shouldn't there only be one???
69 | replies[reference].append(anno)
70 | PMID.extend([tag for tag in anno.tags if tag.startswith('PMID:') and '_' not in tag]) # bad tags with PMID:SCR_
71 | #curators didn't put the pmid in as tags :(
72 | if anno.text.startswith('PMID:'): # DANGER ZONE
73 | if '_' in anno.text:
74 | print('PMIDS DONT HAVE UNDERSCORES PROBABLY CURATION BUG', anno.text)
75 | else:
76 | PMID.append(anno.text.strip()) # because, yep, when you don't tag sometimes you get \n :/
77 |
78 | if PMID:
79 | if len(PMID) > 1:
80 | print(PMID, annotated_url)
81 | if PMID[0] == PMID[1]:
82 | PMID = PMID[0]
83 | print('WARNING: more than one pmid tag')
84 | else:
85 | print("raise BaseException('more than one pmid tag')") # irritating
86 | PMID = PMID[0] # FIXME
87 | else:
88 | PMID = PMID[0]
89 | #print(PMID)
90 | else:
91 | all_tags = []
92 | for a in annos:
93 | all_tags.extend(a.tags)
94 | #print('NO PMID FOR', annotated_url)
95 | #print(set([a.user for a in annos]))
96 | #print(all_tags)
97 | PMID = annotated_url
98 |
99 | RRIDs = defaultdict(list)
100 | EXACTs = {}
101 | CITEs = {}
102 | #USERs = {}
103 | for anno in annos:
104 | RRID = None
105 | additional = []
106 | for tag in anno.tags:
107 | if re.match('RRID:.+[0-9]+.+', tag): # ARRRRGGGGHHHHHHH ARRRRGGHHHH
108 | #if re.match('RRID:.+', tag): # ARRRRGGGGHHHHHHH ARRRRGGHHHH
109 | if RRID is not None:
110 | raise BaseException('MORE THAN ONE RRID PER ENTRY!')
111 | RRID = tag # :/ this works for now but ARHGHHGHASFHAS
112 | else:
113 | additional.append(tag) # eg Unresolved
114 |
115 | if tag == 'RRIDCUR:Missing': # fix for bad curation process
116 | maybe_rrid = anno.text.strip()
117 | if re.match('RRID:.+[0-9]+', maybe_rrid): # ARRRRGGGGHHHHHHH ARRRRGGHHHH
118 | RRID = maybe_rrid # RRIDCUR:Missing was already added above
119 |
120 | if RRID is not None:
121 | EXACTs[RRID] = anno.exact.strip() if anno.exact else ''
122 | RRIDs[RRID].extend(additional)
123 | #USERs[RRID] = anno.user
124 | if RRID not in CITEs:
125 | if anno.text:
126 | if 'Proper Citation:' in anno.text:
127 | CITEs[RRID] = anno.text.split('Proper Citation:')[1].strip().split('<',1)[0]
128 |
129 | if anno.id in replies:
130 | for r_anno in replies[anno.id]:
131 | RRIDs[RRID].extend(r_anno.tags) # not worrying about the text here
132 | elif not anno.references and PMID not in anno.tags: # this is an independent annotation which will not be included
133 | new = 'NONE:' + anno.id
134 | RRIDs[new].append('')
135 | EXACTs[new] = anno.exact
136 | #USERs[RRID] = anno.user
137 |
138 | for rrid, more in RRIDs.items():
139 | #FIXME TOOOOOO SLOW
140 | #r = requests.get('https://scicrunch.org/resolver/{RRID}.xml'.format(RRID=rrid))
141 | #if r.status_code < 300:
142 | #proper_citation = get_proper_citation(r.content)
143 | #else:
144 | #proper_citation = ''
145 |
146 | try:
147 | proper_citation = CITEs[rrid]
148 | except KeyError: # FIXME this is a hack to avoid some cases of LWW for citations
149 | proper_citation = ''
150 |
151 | if not more:
152 | row = [PMID, rrid, '', annotated_url, EXACTs[rrid], proper_citation]
153 | output_rows.append(row)
154 | else:
155 | for val in set(more): # cull dupes
156 | row = [PMID, rrid, val, annotated_url, EXACTs[rrid], proper_citation]
157 | output_rows.append(row)
158 |
159 | DATE = date.today().strftime('%Y-%m-%d')
160 | return output_rows, DATE
161 |
162 |
163 | class NormalizedAnno(HypothesisAnnotation):
164 | @property
165 | def tags(self):
166 | tags = super().tags
167 | out = []
168 | for tag in tags:
169 | if tag in bad_tags:
170 | # scibot made a mistake early
171 | # might be able to correct tags in bulk someday
172 | out.append(tag.replace('RRID:', 'RRIDCUR:'))
173 | else:
174 | out.append(tag)
175 |
176 | text = self.text
177 | exact = self.exact
178 | if text.startswith('RRID:'):
179 | # catch cases where the RRID was put in text instead of in tags
180 | if 'RRIDCUR:Missing' in out or 'RRIDCUR:Unrecognized' in out:
181 | # trap for cases where there is more text after an RRID...
182 | rtag = text.split(None, 1)[0]
183 | if rtag not in out:
184 | out.append(rtag)
185 | print('TEXT ISSUE for %s at https://hyp.is/%s' % (self.user, self.id))
186 |
187 | elif exact and exact.startswith('RRID:'): # this needs to go second in case of RRIDCUR:Incorrect
188 | if exact.startswith('RRID: '): # deal with nospace first
189 | rtag = exact.replace('RRID: ', 'RRID:')
190 | else:
191 | rtag = exact
192 |
193 | rtag = rtag.split(None, 1)[0] # trap more
194 | if rtag not in out:
195 | if self.user == 'scibot' and len(out) == 1 and tags[0].startswith('RRID:RRID:'): # FIXME HACK
196 | out = [rtag]
197 | else:
198 | # anything else we detect in the data doesn't need
199 | # to be corrected or used to fix tags
200 | pass
201 |
202 | return out
203 |
204 |
205 | def export_json_impl():
206 | get_annos = Memoizer(memfile, username=username, api_token=api_token, group=group)
207 | annos = get_annos()
208 |
209 | # clean up bugs from old curation workflow
210 | for anno in annos:
211 | yield NormalizedAnno(anno)._normalized()
212 | continue
213 |
214 | ### URG
215 |
216 | def oldmain():
217 | output_rows, DATE = export_impl()
218 | with open('RRID-data-%s.csv' % DATE, 'wt') as f:
219 | writer = csv.writer(f, lineterminator='\n')
220 | writer.writerows(sorted(output_rows))
221 |
222 | import json
223 | output_json, DATE = export_json_impl()
224 | with open('RRID-data-%s.json' % DATE, 'wt') as f:
225 | json.dump(output_json, f, sort_keys=True, indent=4)
226 |
227 | def main():
228 | import json
229 | DATE = date.today().strftime('%Y-%m-%d')
230 | with open('RRID-data-%s.json' % DATE, 'wt') as f:
231 | f.write('[')
232 | gen = export_json_impl()
233 | s = json.dumps(next(gen), sort_keys=True, indent=1)
234 | for d in gen:
235 | f.write(s)
236 | f.write(',\n')
237 | s = json.dumps(d, sort_keys=True, indent=1)
238 |
239 | f.write(s)
240 | f.write(']')
241 |
242 | if __name__ == '__main__':
243 | main()
244 |
245 |
--------------------------------------------------------------------------------
/scibot/extract.py:
--------------------------------------------------------------------------------
1 | import re
2 | import requests
3 | from bs4 import BeautifulSoup
4 | from scibot.utils import makeSimpleLogger
5 |
6 | log = makeSimpleLogger('extract')
7 |
8 |
9 | # utility
10 | def col0(pairs): return list(zip(*pairs))[0]
11 | def col1(pairs): return list(zip(*pairs))[1]
12 |
13 | # prefixes
14 |
15 | agprefixes = (
16 | ('Addgene', 'Addgene'),
17 | ('addgene', 'Addgene'),
18 | ('plasmid', 'Addgene'),
19 | ('Plasmid', 'Addgene'),
20 | ('addgene cat', 'Addgene'),
21 | ('addgene.org', 'Addgene'),
22 | ('addgene ID', 'Addgene'), # probably won't work
23 | ('addgene cat. no.', 'Addgene'),
24 | ('Jackson Laboratory Cat', 'IMSR_JAX'),
25 | ('Jackson Laboratory Cat', 'IMSR_JAX'),
26 | ('Jackson Laboratory Stock', 'IMSR_JAX'),
27 | ('Jackson Laboratory Stock', 'IMSR_JAX'),
28 | ('The Jackson Laboratory', 'IMSR_JAX'),
29 | ('The Jackson Laboratory Stock', 'IMSR_JAX'),
30 | ('The Jackson Laboratory Stock', 'IMSR_JAX'),
31 | ('The Jackson Laboratory Cat', 'IMSR_JAX'),
32 | ('The Jackson Laboratory Cat', 'IMSR_JAX'),
33 | ('Jackson Laboratories Cat', 'IMSR_JAX'),
34 | ('Jackson Laboratories Cat', 'IMSR_JAX'),
35 | )
36 |
37 | prefixes = (
38 | ('AB', 'AB'),
39 | ('AGSC', 'AGSC'),
40 | ('ARC', 'IMSR_ARC'),
41 | ('BCBC', 'BCBC'),
42 | ('BDSC', 'BDSC'),
43 | ('DGRC', 'DGRC'),
44 | ('CGC', 'CGC'),
45 | ('CRL', 'IMSR_CRL'),
46 | #('CVCL', 'CVCL'), # numbers + letters :/
47 | ('DGGR', 'DGGR'),
48 | ('FBst', 'FBst'),
49 | ('FlyBase', 'FlyBase'),
50 | ('HAR', 'IMSR_HAR'),
51 | ('JAX', 'IMSR_JAX'),
52 | ('KOMP', 'IMSR_KOMP'),
53 | ('MGI', 'MGI'),
54 | ('MMRRC', 'MMRRC'),
55 | ('NCIMR', 'IMSR_NCIMR'),
56 | ('NSRRC', 'NSRRC'),
57 | ('NXR', 'NXR'),
58 | ('RBRC', 'IMSR_RBRC'),
59 | ('RGD', 'RGD'),
60 | ('SCR', 'SCR'),
61 | ('TAC', 'IMSR_TAC'),
62 | ('TIGM', 'IMSR_TIGM'),
63 | ('TSC', 'TSC'),
64 | ('WB-STRAIN', 'WB-STRAIN'),
65 | ('WTSI', 'IMSR_WTSI'),
66 | ('ZDB', 'ZFIN_ZDB'),
67 | ('ZFIN', 'ZFIN'),
68 | ('ZIRC', 'ZIRC'),
69 | )
70 | prefix_lookup = {k:v for k, v in prefixes}
71 | prefix_lookup['CVCL'] = 'CVCL' # ah special cases
72 | prefix_lookup.update({k:v for k, v in agprefixes})
73 |
74 | # paper identifiers
75 |
76 | def searchSoup(soup):
77 | def search(tag, prop, val, key, additional_prop_vals=None):
78 | if additional_prop_vals is None:
79 | pvs = {prop:val}
80 | else:
81 | additional_prop_vals.update({prop:val})
82 | pvs = additional_prop_vals
83 | matches = soup.find_all(tag, pvs)
84 | if matches:
85 | return matches[0][key]
86 | return search
87 |
88 |
89 | def normalizeDoi(doi):
90 | doi = doi.replace(' ', '')
91 | if 'http' in doi or 'doi.org' in doi:
92 | doi = '10.' + doi.split('.org/10.', 1)[-1]
93 | elif doi.startswith('doi:'):
94 | doi = doi.strip('doi:')
95 | elif doi.startswith('DOI:'):
96 | doi = doi.strip('DOI:')
97 | return doi
98 |
99 |
100 | def getDoi(*soups):
101 | argslist = ( # these go in order so best returns first
102 | # TODO bind a handler for these as well...
103 | ('meta', 'name', 'DC.Identifier', 'content'), # elife pmc etc.
104 | ('meta', 'name', 'DOI', 'content'), # nature pref
105 | ('meta', 'name', 'dc.identifier', 'content'), # nature
106 | ('meta', 'name', 'citation_doi', 'content'), # wiley jove f1000 ok
107 | ('a', 'class', 'doi', 'href'), # evilier
108 | ('a', 'class', 'S_C_ddDoi', 'href'), # evilier
109 | ('a', 'id', 'ddDoi', 'href'), # evilier
110 | ('meta', 'name', 'DC.identifier', 'content'), # f1000 worst
111 | ('meta', 'name', 'dc.Identifier', 'content', {'scheme':'doi'}), # tandf
112 | ('meta', 'name', 'dc.Source', 'content'), # mit press jounals wat
113 | ('meta', 'name', 'dc.identifier', 'content'),
114 | ('meta', 'name', 'prism.doi', 'content'),
115 | )
116 | for soup in soups:
117 | for args in argslist:
118 | doi = searchSoup(soup)(*args)
119 | if doi is not None:
120 | return normalizeDoi(doi)
121 |
122 |
123 | def getUri(uri, *soups):
124 | argslist = (
125 | ('meta', 'property', 'og:url', 'content'), # FIXME mitpressjournals has an idiot in it somewhere
126 | ('link', 'rel', 'canonical', 'href'),
127 | )
128 | for soup in soups:
129 | for args in argslist:
130 | cu = searchSoup(soup)(*args)
131 | if cu is not None and cu.startswith('http'):
132 | if cu != uri:
133 | log.warning('canonical and uri do not match, '
134 | f'preferring canonical\n{cu}\n{uri}')
135 | return cu
136 | return uri
137 |
138 |
139 | def getPmid(*soups):
140 | argslist = (
141 | ('meta', 'property', 'citation_pmid', 'content'),
142 | )
143 | for soup in soups:
144 | for args in argslist:
145 | cu = searchSoup(soup)(*args)
146 | if cu is not None:
147 | return cu # FIXME TODO yeild here
148 |
149 |
150 | def getTitle(*soups):
151 | for soup in soups:
152 | for t in soup.find_all('title'):
153 | yield t.text
154 |
155 |
156 | def chooseTitle(document, titles):
157 | meta_titles = []
158 | for k, d_ in document.items():
159 | rank = 0 # TODO
160 | if 'title' in d_:
161 | t = d_['title'][0] # FIXME warn on > 1 ?
162 | meta_titles.append((rank, t))
163 |
164 | if meta_titles:
165 | title = sorted(meta_titles)[0][1]
166 |
167 | elif titles:
168 | title = sorted(titles, key=lambda t: -len(t))[0]
169 |
170 | else:
171 | log.warning(f'no title for {document}')
172 | title = 'Spooky nameless page'
173 |
174 | document['title'] = title
175 |
176 |
177 | def getLinks(*soups):
178 | for soup in soups:
179 | for l in soup.find_all('link'):
180 | yield l.attrs
181 |
182 |
183 | def chooseLinks(document, links):
184 | meta_links = []
185 | for link in links:
186 | if 'rel' in link and 'canonical' in link['rel']:
187 | l = {'rel': 'canonical', 'href': link['href']}
188 | if 'type' in link and link['type']:
189 | l['type'] = link['type']
190 |
191 | meta_links.append(l)
192 |
193 | # TODO pull out other links as well
194 | document['link'].extend(meta_links)
195 |
196 |
197 | def searchSoups(argslist, *soups):
198 | for soup in soups:
199 | for args in argslist:
200 | cu = searchSoup(soup)(*args)
201 | if cu is not None:
202 | yield cu
203 |
204 |
205 | def getDocument(target_uri, *soups):
206 | # TODO probably want to detect when there are tags in the header that
207 | # we are missing/skipping since this takes a closed world approach to detection
208 | # rather than prefix based detection
209 | # TODO pull these out into a more visible file
210 | dc_fields = 'identifier', 'title', 'publisher', 'format', 'creator', 'date'
211 | eprints_fields = ('title', 'creators_name', 'type', 'datestamp', 'ispublished',
212 | 'date', 'date_type', 'publication', 'volume', 'pagerange')
213 | prism_fields = ('volume', 'number', 'startingPage', 'endingPage', 'publicationName',
214 | 'issn', 'publicationDate', 'doi')
215 | highwire_fields = ('title', 'journal_title', 'publisher', 'issue', 'volume', 'doi',
216 | 'firstpage', 'lastpage', 'date', 'abstract_html_url', 'fulltext_html_url',
217 | 'pdf_url', 'pii', 'article_type', 'online_date', 'publication_date',
218 | 'issn', 'keywords', 'language', 'author', 'author_institution',
219 | )
220 | og_fields = ('title', 'type', 'image', 'url', 'audio', 'description', 'determiner',
221 | 'locale', 'locale:alternate', 'site_name', 'video')
222 |
223 | def dmeta(rexp, fields, props=('name',)):
224 | return {field: [('meta', prop, re.compile(rexp.format(field=field), re.I), 'content')
225 | for prop in props] for field in fields}
226 |
227 | todo = {
228 | 'dc': dmeta('^(dc|dcterms).{field}$', dc_fields),
229 | 'eprints': dmeta('^eprints.{field}$', eprints_fields),
230 | 'facebook': dmeta('^og:{field}$', og_fields, props=('name', 'property')),
231 | # some people use name for og instead of property (spec expects property)
232 | 'highwire': dmeta('^citation_{field}$', highwire_fields),
233 | 'prism': dmeta('^prism.{field}$', prism_fields),
234 | 'twitter':{}, # TODO??
235 | }
236 |
237 | document = {key: {field: results
238 | for field, argslist in dict_.items()
239 | for results in (list(searchSoups(argslist, *soups)),)
240 | if results}
241 | for key, dict_ in todo.items()}
242 |
243 | doi = getDoi(*soups)
244 | pmid = getPmid(*soups)
245 | titles = list(getTitle(*soups))
246 | chooseTitle(document, titles)
247 | links = list(getLinks(*soups))
248 | document['link'] = [{'href': target_uri}]
249 | chooseLinks(document, links)
250 | return document, doi, pmid
251 |
252 |
253 | def document_from_url(url):
254 | resp = requests.get(url)
255 | soup = BeautifulSoup(resp.content, 'lxml')
256 | return getDocument(url, resp, soup), soup
257 |
258 |
259 | # rrids
260 |
261 | def clean_text(text):
262 | # cleanup the inner text
263 | text = text.replace('–','-')
264 | text = text.replace('‐','-') # what the wat
265 | text = text.replace('\xa0',' ') # nonbreaking space fix
266 |
267 | mids = (r'',
268 | r'\ ',
269 | r'_\ ',
270 | r'\ _',
271 | r': ',
272 | r'-',
273 | )
274 | tail = r'([\s,;\)])'
275 | replace = r'\1\2_\3\4'
276 | def make_cartesian_product(prefix, suffix=r'(\d+)'):
277 | return [(prefix + mid + suffix + tail, replace) for mid in mids]
278 |
279 | fixes = []
280 | prefixes_digit = [r'([^\w])(%s)' % _ for _ in ('AB', 'SCR', 'MGI')]
281 | for p in prefixes_digit:
282 | fixes.extend(make_cartesian_product(p))
283 | fixes.extend(make_cartesian_product(r'([^\w])(CVCL)', r'([0-9A-Z]+)')) # FIXME r'(\w{0,5})' better \w+ ok
284 | fixes.append((r'\(RRID\):', r'RRID:'))
285 |
286 | for f, r in fixes:
287 | text = re.sub(f, r, text)
288 | return text
289 |
290 |
291 | def find_rrids(text):
292 | # first round
293 | regex1 = '(.{0,32})(RRID(:|\)*,*)[ \t]*)(\w+[_\-:]+[\w\-]+)([^\w].{0,31})'
294 | matches = re.findall(regex1, text)
295 | for prefix, rrid, sep, id_, suffix in matches:
296 | #print((prefix, rrid, sep, id_, suffix))
297 | exact = 'RRID:' + id_
298 | exact_for_hypothesis = exact
299 | yield prefix, exact, exact_for_hypothesis, suffix
300 |
301 | # second round
302 | orblock = '(' + '|'.join(col0(prefixes)) + ')'
303 | sep = '(:|_)([ \t]*)'
304 | agsep = '([ \t]*#)([ \t]*)' # FIXME doesn't work with "Stock No." or "Stock No:"
305 | agorblock = '(' + '|'.join(col0(agprefixes)) + ')'
306 | regex2 = ('(.{0,32})(?:' + orblock + f'{sep}(\d+)|(CVCL){sep}(\w+)|'
307 | + agorblock + f'{agsep}(\w+))([^\w].{{0,31}})') # the first 0,32 always greedy matches???
308 | matches2 = re.findall(regex2, text) # FIXME this doesn't work since our prefix/suffix can be 'wrong'
309 | for (prefix, namespace, sep, spaces, nums,
310 | cvcl, cvcl_sep, cvcl_spaces, cvcl_nums,
311 | ag, ag_sep, ag_spaces, ag_nums,
312 | suffix) in matches2:
313 | if cvcl:
314 | #print('\t\t', (prefix, namespace, sep, spaces, nums, cvcl, cvcl_sep, cvcl_spaces, cvcl_nums, suffix))
315 | namespace, sep, spaces, nums = cvcl, cvcl_sep, cvcl_spaces, cvcl_nums # sigh
316 | elif ag:
317 | namespace, sep, spaces, nums = ag, ag_sep, ag_spaces, ag_nums # sigh
318 | if re.match(regex1, ''.join((prefix, namespace, sep, spaces, nums, suffix))) is not None:
319 | #print('already matched')
320 | continue # already caught it above and don't want to add it again
321 | if ag: # switch sep for addgene after match
322 | sep = '_'
323 | exact_for_hypothesis = namespace + sep + nums
324 | resolver_namespace = prefix_lookup[namespace]
325 | exact = 'RRID:' + resolver_namespace + sep + nums
326 | yield prefix, exact, exact_for_hypothesis, suffix
327 |
328 | # third round for BDSC
329 | regex3 = '(.{0,32})(BDSC|BL|Bl|Bloomington)(\s?)(stock)?(\s)?(#|no|no\.)?(\s?)([0-9]{2,10})([^\w].{0,31})'
330 | matches3 = re.findall(regex3, text)
331 | for prefix, a, b, c, d, e, f, nums, suffix in matches3:
332 | if nums in ['17', '21']:
333 | # special case to avoid false positives
334 | continue
335 | yield prefix, f'RRID:BDSC_{nums.strip()}', f'{a}{b}{c}{d}{e}{f}{nums}', suffix
336 |
337 | # fourth round for SAMN
338 | regex4 = '(.{0,32})(SAMN)(\s?)([0-9]{3,15})([^\w].{0,31})'
339 | matches4 = re.findall(regex4, text)
340 | for prefix, a, b, nums, suffix in matches4:
341 | yield prefix, f'RRID:SAMN{nums.strip()}', f'{a}{b}{nums}', suffix
342 |
343 |
344 | # extract from post
345 |
346 | def process_POST_request(request):
347 | dict_ = dict(request.form)
348 | def htmlify(thing):
349 | try:
350 | html = dict_[thing]
351 | except KeyError as e:
352 | html = ''
353 | return '' + html + ''
354 | uri = dict_['uri']
355 | head = htmlify('head')
356 | body = htmlify('body')
357 | try:
358 | text = dict_['data']
359 | except KeyError as e:
360 | text = ''
361 |
362 | headsoup = BeautifulSoup(head, 'lxml')
363 | bodysoup = BeautifulSoup(body, 'lxml')
364 |
365 | target_uri = getUri(uri, headsoup, bodysoup)
366 | #doi = getDoi(headsoup, bodysoup)
367 | #pmid = getPmid(headsoup, bodysoup)
368 | document, doi, pmid = getDocument(target_uri, headsoup, bodysoup)
369 | cleaned_text = clean_text(text)
370 | return target_uri, document, doi, pmid, head, body, text, cleaned_text
371 |
372 |
373 | class PaperId:
374 | id_types = (
375 | 'uri_normalized',
376 | 'doi',
377 | 'pmid',
378 | 'hypothesis_normalized',
379 | 'uri',
380 | )
381 | def __init__(self,
382 | uri_normalized,
383 | doi=None,
384 | pmid=None,
385 | hypothesis_normalized=None,
386 | uri=None):
387 |
388 | # names
389 | self.uri_normalized = uri_normalized
390 | self.doi = OntId(doi) if doi else doi
391 | self.pmid = OntId(pmid) if pmid else pmid
392 | self.hypothesis_normalized = hypothesis_normalized
393 | self.uri = uri
394 |
395 | # amusingly the actual identities
396 | self.urn = None # pdf fingerprint
397 | self.text_hash = None
398 | self.html_hash = None
399 | self.head_hash = None
400 | self.body_hash = None
401 | self.jats_hash = None
402 | self.stripped_hash = None
403 |
404 | @property
405 | def _existing_ids(self):
406 | for id_type in self.id_types:
407 | id = getattr(self, id_type, None)
408 | if id is not None:
409 | yield id
410 |
411 | @property
412 | def existing_ids(self):
413 | return set(self._existing_ids)
414 |
415 | @property
416 | def _resolvable_ids(self):
417 | yield self.doi.iri
418 | yield self.uri
419 |
420 | @property
421 | def resolvable_ids(self):
422 | return set(self._resolvable_ids)
423 |
424 | @property
425 | def _chains(self):
426 | for id in self.resolvable_ids:
427 | yield id, tuple(resolution_chain(id))
428 |
429 | @property
430 | def chains(self):
431 | return {id:chain for id, chain in self._chains}
432 |
433 | def idPaper(self):
434 | if self.doi is None:
435 | paper = self
436 | doi = paper['DOI']
437 | pmid = paper['PMID']
438 | log.info(url)
439 | if not self.doi and self.uri.startswith('http'): # we've go some weird ones in there...
440 | doi = scrapeDoi(uri)
441 | # scrapeIds(uri)
442 | if doi is not None:
443 | log.info(doi)
444 | pmid = get_pmid(doi)
445 | log.warning('json malformed in get_pmid')
446 | log.info(pmid)
447 | resp = annotate_doi_pmid(url, doi, pmid, rrcu.h_curation, [])
448 | log.info('new doi')
449 | return resp
450 | else:
451 | log.info(doi)
452 | log.info('already found')
453 |
454 | def scrapeDoi(self):
455 | env = os.environ.copy()
456 | cmd_line = ['timeout', '30s', 'google-chrome-unstable', '--headless', '--dump-dom', url]
457 | p = subprocess.Popen(cmd_line, stdin=subprocess.PIPE,
458 | stdout=subprocess.PIPE,
459 | stderr=subprocess.STDOUT,
460 | env=env)
461 | out, err = p.communicate()
462 | if p.returncode:
463 | log.critical('UTOH')
464 | return None
465 | elif b'ERROR:headless_shell.cc' in out:
466 | log.critical(out)
467 | raise IOError('Something is wrong...')
468 | qurl = quote(url, '')
469 | if len(qurl) > 200:
470 | qurl = qurl[:200]
471 | with open(os.path.expanduser(f'~/files/scibot/{qurl}'), 'wb') as f:
472 | f.write(out)
473 | both = BeautifulSoup(out, 'lxml')
474 | doi = getDoi(both, both)
475 | return doi
476 |
477 |
478 |
479 |
480 |
481 |
--------------------------------------------------------------------------------
/scibot/get_annos.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 | """ Get the 10k most recent annotations from a group. """
3 | import json
4 | import hashlib
5 | from os import environ, chmod
6 | from datetime import date
7 | from hyputils.hypothesis import HypothesisUtils
8 |
9 | def main():
10 | TODAY = date.isoformat(date.today())
11 |
12 | api_token = environ.get('SCIBOT_API_TOKEN', 'TOKEN') # Hypothesis API token
13 | username = environ.get('SCIBOT_USERNAME', 'USERNAME') # Hypothesis username
14 | group = environ.get('SCIBOT_GROUP', '__world__')
15 | group_staging = environ.get('SCIBOT_GROUP_STAGING', '__world__')
16 |
17 | m = hashlib.sha256()
18 | m.update(group.encode())
19 | group_hash = m.hexdigest()[:16] # 16 is 2x the length of the original group...
20 | h = HypothesisUtils(username=username, token=api_token, group=group, max_results=10000)
21 | # FIXME there seems to be an additiona bug here in hyputils
22 | # there will be an error at the end when we overrun 10k
23 | recent_annos = list(h.search_all({'group': h.group}))
24 | fn = f'annos-{group_hash}-{TODAY}.json'
25 | with open(fn, 'wt') as f:
26 | json.dump(recent_annos, f, indent=4)
27 | chmod(fn, 0o600)
28 |
29 | if __name__ == '__main__':
30 | main()
31 |
--------------------------------------------------------------------------------
/scibot/papers.py:
--------------------------------------------------------------------------------
1 | class KeyAccessor:
2 | # XXX whenever annos changes these need to be recreated as well...
3 | # if that is too slow then we will need to implement proper add
4 | # and delete for only affected items...
5 | """ For when your identifiers aren't python compatible. """
6 | prop = 'id'
7 | object_container_class = set
8 | def __init__(self, objects=tuple(), id_prop=None):
9 | self._propagate = issubclass(self.object_container_class, KeyAccessor)
10 | self._objects = {}
11 | errors = []
12 | for o in objects:
13 | k = getattr(o, self.prop)
14 | if k not in self._objects:
15 | self._objects[k] = self._make_cont()
16 |
17 | try:
18 | self._objects[k].add(o)
19 | except BaseException as e:
20 | if errors:
21 | try:
22 | raise e from errors[-1]
23 | except BaseException as ne:
24 | errors.append(ne)
25 | else:
26 | errors.append(e)
27 |
28 | if errors:
29 | raise errors[-1]
30 |
31 | self._id_prop = None
32 | if id_prop is not None:
33 | if objects:
34 | print('setting id prop')
35 | setattr(self, id_prop, getattr(o, id_prop)) # o from the for loop
36 | else:
37 | self._id_prop = id_prop
38 |
39 | def _make_cont(self):
40 | if self._propagate:
41 | cont = self.object_container_class(id_prop=self.prop)
42 | else:
43 | cont = self.object_container_class()
44 | return cont
45 |
46 | def remove(self, object_):
47 | k = getattr(object_, self.prop)
48 | self._objects[k].remove(object_)
49 | if not self._objects[k]:
50 | self._objects.pop(k)
51 |
52 | def discard(self, object_):
53 | k = getattr(object_, self.prop)
54 | self._objects[k].discard(object_)
55 | if not self._objects[k]:
56 | self._objects.pop(k)
57 |
58 | def add(self, object_):
59 | if self._id_prop is not None:
60 | setattr(self, self._id_prop, getattr(object_, self._id_prop))
61 | self._id_prop = None
62 | k = getattr(object_, self.prop)
63 | if k not in self._objects:
64 | self._objects[k] = self._make_cont()
65 |
66 | self._objects[k].add(object_)
67 |
68 | def keys(self):
69 | return sorted(self._objects, key=lambda k: '0' if k is None else k)
70 |
71 | def values(self):
72 | for v in list(self._objects.values()):
73 | yield v
74 |
75 | def items(self):
76 | # we use list() here to simplify synchronization issues with the websocket
77 | # since yield allows the thread to shift
78 | for k, v in list(self._objects.items()):
79 | yield k, v
80 |
81 | def __iter__(self):
82 | for k in self._objects:
83 | yield k
84 |
85 | def __contains__(self, key):
86 | return key in self._objects
87 |
88 | def __getitem__(self, key):
89 | try:
90 | return self._objects[key]
91 | except KeyError:
92 | self.__missing__(key)
93 |
94 | #def __setitem__(self, key):
95 | #raise TypeError('Cannot set values on this class, it is immutable')
96 |
97 | def __missing__(self, key):
98 | raise KeyError(f'{key} not found')
99 |
100 | def __len__(self):
101 | return len(self._objects)
102 |
103 | def __repr__(self):
104 | return repr({k:v for k,v in self.items()})
105 |
106 | def __str__(self):
107 | return str({k:v for k,v in self.items()})
108 |
109 |
110 | class RRIDs(KeyAccessor):
111 | """ AKA a Paper """
112 | prop = 'rrid'
113 | object_container_class = set
114 |
115 | @property
116 | def doi(self):
117 | if None in self._objects:
118 | for o in self._objects[None]:
119 | if o.KillPageNote:
120 | continue
121 | if o._anno.is_page_note or o.user != 'scibot': # FIXME some curators did these as annotations too...
122 | for t in o.tags:
123 | if t.startswith('DOI:') and ' ' not in t and '\n' not in t and t.count(':') == 1:
124 | return t
125 |
126 | @property
127 | def pmid(self):
128 | if None in self._objects:
129 | for o in self._objects[None]:
130 | if o.KillPageNote:
131 | continue
132 | for t in o.tags:
133 | if t.startswith('PMID:') and ' ' not in t and '\n' not in t and t.count(':') == 1:
134 | return t
135 |
136 |
137 | class Papers(KeyAccessor):
138 | prop = 'uri_normalized'
139 | object_container_class = RRIDs
140 |
141 |
142 | class SameDOI(KeyAccessor):
143 | prop = 'doi'
144 | object_container_class = Papers
145 |
146 |
147 | class SamePMID(KeyAccessor):
148 | prop = 'pmid'
149 | object_container_class = Papers
150 |
151 |
152 | class MultiplePMID(KeyAccessor):
153 | prop = 'doi'
154 | object_container_class = SamePMID
155 |
156 |
157 | class MultipleDOI(KeyAccessor):
158 | prop = 'pmid'
159 | object_container_class = SameDOI
160 |
161 |
162 | class RRIDSimple(KeyAccessor):
163 | prop = 'rrid'
164 | object_container_class = set
165 |
166 |
167 | class PMIDRRIDs(KeyAccessor):
168 | prop = 'pmid'
169 | object_container_class = RRIDSimple
170 |
171 |
172 | class DOIRRIDs(KeyAccessor):
173 | prop = 'doi'
174 | object_container_class = RRIDSimple
175 |
176 |
177 | class MPP(KeyAccessor):
178 | prop = 'uri_normalized'
179 | object_container_class = PMIDRRIDs
180 |
181 |
182 | class MPD(KeyAccessor):
183 | prop = 'uri_normalized'
184 | object_container_class = DOIRRIDs
185 |
--------------------------------------------------------------------------------
/scibot/release_report.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 | import csv
3 | import socket
4 | from collections import namedtuple, Counter
5 | from pyontutils.utils import mysql_conn_helper
6 | from scibot.release import Curation, PublicAnno, get_annos, get_pannos
7 | from scibot.utils import uri_normalization
8 | from scibot import config
9 | try:
10 | breakpoint
11 | except NameError:
12 | from IPython import embed as breakpoint
13 |
14 |
15 | def dbUri(user='nif_eelg_secure', host='nif-mysql.crbs.ucsd.edu', port=3306, database='nif_eelg'):
16 | DB_URI = 'mysql+pymysql://{user}:{password}@{host}:{port}/{db}' # FIXME db => pyontutils refactor
17 | if socket.gethostname() in config.dev_remote_hosts:
18 | db_cfg_kwargs = mysql_conn_helper('localhost', database, user, 33060) # see .ssh/config
19 | else:
20 | db_cfg_kwargs = mysql_conn_helper(host, database, user, port)
21 |
22 | return DB_URI.format(**db_cfg_kwargs)
23 |
24 |
25 | class RRIDData:
26 | def __init__(self, session):
27 | self.session = session
28 |
29 | def james_rrids(self):
30 | sql = '''SELECT m.uri, v.pmid, v.rrid, m.annotation_id,
31 | m.hypothesis_user, v.journal, v.title, v.year
32 | FROM rrid_mentions_view2 AS v JOIN rrid_mentions AS m ON v.rrid_mention_id = m.id
33 | WHERE source = 'Hypothesis' '''
34 | yield from self.session.execute(sql)
35 |
36 | def missing_pmids(self):
37 | sql = '''SELECT distinct(pmid) FROM rrid_mentions'''
38 | sql2 = '''SELECT distinct(concat("PMID:", pmid)) FROM rrid_mentions_literature_records'''
39 | all_pmids = set(p for p, in self.session.execute(sql))
40 | pmids_in_lit_table = set(p for p, in self.session.execute(sql2))
41 | missing = all_pmids - pmids_in_lit_table
42 | return missing
43 |
44 | def combine(self, rrid_recs=None):
45 | if rrid_recs is None:
46 | rrid_recs = self.james_rrids()
47 |
48 | header = ('release', 'issue', 'urin', 'uri',
49 | 'pmid', 'cpmid', 'doi', 'rrid', 'crrid',
50 | 'rlink', 'alink', 'plink',
51 | 'user', 'journal', 'title', 'year')#, 'ptext')
52 | rr = namedtuple('reprow', header)
53 | yield rr(*header)
54 | pa_done = set()
55 | for row in rrid_recs:
56 | #print(row)
57 | c = Curation.byId(row.annotation_id)
58 | errorl = []
59 | if c is None:
60 |
61 | yield rr(None, 'BAD-ANNOTATION-ID', None, row.uri,
62 | row.pmid, None, None, row.rrid, None,
63 | None, row.annotation_id, None,
64 | row.hypothesis_user, row.journal, row.title, row.year)#, None)
65 | continue
66 |
67 | else:
68 | if row.uri != c.uri:
69 | errorl.append('URI-mismatch')
70 | if row.pmid != c.pmid:
71 | errorl.append('PMID-mismatch')
72 | if row.rrid != c.rrid:
73 | errorl.append('RRID-mismatch')
74 |
75 | issue = ' '.join(errorl)
76 | pa = c._public_anno if c else None
77 | pasl = pa.shareLink if pa is not None else None
78 | pat = pa.text if pa is not None else None
79 | if pa is not None:
80 | pa_done.add(pa.id)
81 | yield rr(c.isReleaseNode, issue, c.uri_normalized, row.uri,
82 | row.pmid, c.pmid, c.doi, row.rrid, c.rrid,
83 | c.rridLink, c.shareLink, pasl,
84 | row.hypothesis_user, row.journal, row.title, row.year)#, pat)
85 |
86 | rissue = 'RELEASED-BUT-NOT-IN-TABLE'
87 | for pa in PublicAnno:
88 | if pa.id not in pa_done:
89 | cs = list(pa.curation_annos)
90 | if not cs:
91 | irn = None
92 | issue = 'DELETE-THIS'
93 | cpmid = None
94 | cdoi = None
95 | cshareLink = None
96 | curators = '' # FIXME look this up?
97 | else:
98 | issue = rissue
99 | irn = all(c.isReleaseNode if c is not None else c for c in cs)
100 | if not irn:
101 | if None in cs:
102 | issue = '!?-WHAT-HAVE-YOU-DONE!?-WHERE-IS-MY-CURATION-ANNO!? ' + issue
103 | cs = [c for c in cs if c is not None]
104 |
105 | if not cs:
106 | irn = None
107 | issue = 'DELETE-THIS ' + issue
108 | cpmid = None
109 | cdoi = None
110 | cshareLink = None
111 | else:
112 | cdoi = cs[0].doi # NOTE this MASSIVE CHANGES semantics of the rrid column
113 | if cdoi != pa.doi:
114 | issue = 'DOI-mismatch ' + issue
115 |
116 | cpmid = cs[0].pmid # NOTE this changes semantics of the pmid column
117 | if cpmid != pa.pmid:
118 | issue = 'PMID-mismatch ' + issue
119 |
120 |
121 | cshareLink = cs[0].shareLink
122 |
123 | curators = ' '.join([c for ca in cs for c in ca.curators])
124 |
125 | issue = 'z ' + issue # ordering
126 | yield rr(irn, issue, pa.uri_normalized, pa.uri,
127 | pa.pmid, cpmid, cdoi, pa.doi, pa.rrid,
128 | pa.rridLink, cshareLink, pa.shareLink,
129 | curators, None, None, None)#, pa.text)
130 |
131 |
132 | def main():
133 | from sqlalchemy import create_engine
134 | from sqlalchemy.orm.session import sessionmaker
135 | engine = create_engine(dbUri(), echo=True)
136 | Session = sessionmaker()
137 | Session.configure(bind=engine)
138 | session = Session()
139 | rd = RRIDData(session)
140 |
141 | annos = get_annos()
142 | Curation._annos_list = annos
143 | pannos = get_pannos()
144 | PublicAnno._annos_list = pannos
145 |
146 | for helper in (Curation, PublicAnno):
147 | [helper(a, helper._annos_list) for a in helper._annos_list]
148 |
149 | def key(r):
150 | i = r.issue if r.issue else ''
151 | p = r.pmid if r.pmid else ''
152 | u = r.urin if r.urin else ''
153 | c = r.crrid if r.crrid else ''
154 | return i, p, u, c
155 |
156 | rrid_recs = [r for r in rd.james_rrids() if r.uri]
157 |
158 | rd = RRIDData(session)
159 | gen = rd.combine(rrid_recs)
160 | report = [next(gen)] + sorted(gen, key=key)
161 | bads = [r for r in report if r.issue]
162 | noanno = [r for r in bads if not r.urin]
163 | noalink = [r for r in noanno if not r.alink]
164 | noannoalink = [r for r in noanno if r.alink]
165 |
166 | badwanno = [r for r in bads if r.urin]
167 |
168 | irep = {}
169 | for r in report:
170 | if r.alink not in irep:
171 | irep[r.alink] = set()
172 |
173 | irep[r.alink].add(r)
174 |
175 | duplicates = {k:v for k, v in irep.items() if len(v) > 1}
176 |
177 | with open('scibot-rrid-bads.csv', 'wt') as f:
178 | writer = csv.writer(f, lineterminator='\n')
179 | writer.writerows(bads)
180 |
181 | with open('scibot-rrid-all.csv', 'wt') as f:
182 | writer = csv.writer(f, lineterminator='\n')
183 | writer.writerows(report)
184 |
185 | # FIXME PublicAnno pmid and doi do not work correctly ...
186 | # FIXME there are 233 annotations with public ids and public rrids
187 | # that do not have an rrid, meaning that the backlinks got scrambled somehow !
188 | public_report = [(c.pmid, c.canonical_rrid, c.rrid,
189 | c._public_anno.shareLink,
190 | c._anno.prefix, c.exact, c._anno.suffix,)
191 | for c in Curation
192 | if c.isAstNode and c.public_id is not None and
193 | c.pmid is not None and c.rrid is not None and
194 | not c.NotRRID and c.canonical_rrid is not None]
195 | wat = [r for r in public_report if None in r]
196 | public_report.sort()
197 | pheader = 'pmid', 'canonical rrid', 'rrid', 'public share link', 'prefix', 'exact', 'suffix'
198 | public_report = [pheader] + public_report
199 |
200 | with open('scibot-rrid-public-simple.csv', 'wt') as f:
201 | writer = csv.writer(f, lineterminator='\n')
202 | writer.writerows(public_report)
203 |
204 | breakpoint()
205 |
206 |
207 | if __name__ == '__main__':
208 | main()
209 |
--------------------------------------------------------------------------------
/scibot/rridxp.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 | """rridxp: export and crunch RRID data
3 |
4 | Usage:
5 | rridxp csv [options] [
' % resolver_uri
78 | r = h.create_annotation_with_target_using_only_text_quote(url=target_uri,
79 | document=document,
80 | prefix=prefix,
81 | exact=exact_for_hypothesis,
82 | suffix=suffix,
83 | text=s,
84 | tags=new_tags + ['RRIDCUR:Unresolved'],
85 | extra=extra,)
86 | log.error(f'rrid unresolved {exact}')
87 |
88 | else:
89 | s = ''
90 | title = root.findall('title')[0].text
91 | s += f'Title: {title}\n'
92 | data_elements = root.findall('data')[0]
93 | data_elements = [(e.find('name').text, e.find('value').text) for e in data_elements] # these shouldn't duplicate
94 | citation = [(n, v) for n, v in data_elements if n == 'Proper Citation']
95 | rrid = [rrid_from_citation(c) for _, c in citation] if citation else [exact]
96 | name = [(n, v) for n, v in data_elements if n == 'Name']
97 | data_elements = citation + name + sorted([(n, v) for n, v in
98 | data_elements if (n != 'Proper Citation' or
99 | n != 'Name') and v is not None])
100 | for name, value in data_elements:
101 | if ((name == 'Reference' or name == 'Mentioned In Literature')
102 | and value is not None and value.startswith(' 500:
104 | continue # nif-0000-30467 fix keep those pubmed links short!
105 | s += '
' % resolver_uri
107 | r = h.create_annotation_with_target_using_only_text_quote(url=target_uri,
108 | document=document,
109 | prefix=prefix,
110 | exact=exact_for_hypothesis,
111 | suffix=suffix,
112 | text=s,
113 | tags=new_tags + rrid,
114 | extra=extra,)
115 |
116 | elif status_code >= 500:
117 | s = 'Resolver lookup failed due to server error.'
118 | s += '
' % resolver_uri
119 | r = None
120 | log.error(f'{status_code} error for {resolver_uri}')
121 | else:
122 | s = 'Resolver lookup failed.'
123 | s += '
' % resolver_uri
124 | r = h.create_annotation_with_target_using_only_text_quote(url=target_uri,
125 | document=document,
126 | prefix=prefix,
127 | exact=exact_for_hypothesis,
128 | suffix=suffix,
129 | text=s,
130 | tags=new_tags + ['RRIDCUR:Unresolved'],
131 | extra=extra,)
132 | if r is not None:
133 | if r.ok:
134 | found_rrids[exact] = r.json()['links']['incontext']
135 | else:
136 | try:
137 | r.raise_for_status()
138 | except Exception as e:
139 | # FIXME the fact that submission can fail silently and
140 | # that it does not get resubmitted or retried is a
141 | # major oversight ...
142 | log.exception(e)
143 |
144 | return r
145 |
146 | def api_row_to_db(api_row):
147 | # TODO insert the created annotation into our local store
148 | # check for contention/consistency with the websocket
149 | pass
150 |
--------------------------------------------------------------------------------
/scibot/sync.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 | """Run the scibot sync service
3 |
4 | Usage:
5 | scibot-sync [options]
6 |
7 | Options:
8 | -p --port=PORT the port that the service should run on [default: 12345]
9 | """
10 |
11 | from curio import run, socket, UniversalEvent, TaskGroup
12 | from curio.time import timeout_after, sleep
13 | from curio.errors import TaskTimeout, TaskCancelled
14 | from curio.channel import Channel, Connection, AuthenticationError
15 | from scibot.utils import log as _log
16 |
17 | log = _log.getChild('aChannel')
18 | slog = _log.getChild('sync')
19 | clog = slog.getChild('client') # TODO add process info here
20 | mlog = slog.getChild('manager')
21 |
22 | done = UniversalEvent()
23 |
24 |
25 | class aChannel(Channel):
26 | async def connect(self, *, authkey=None, attempts=None):
27 | nattempts = 0
28 | while True:
29 | try:
30 | sock = socket.socket(self.family, socket.SOCK_STREAM)
31 | await sock.connect(self.address)
32 | sock_stream = sock.as_stream()
33 | c = Connection(sock_stream, sock_stream)
34 | #raise BaseException
35 | try:
36 | async with timeout_after(1):
37 | if authkey:
38 | await c.authenticate_client(authkey)
39 | return c
40 | except TaskTimeout:
41 | log.warning('Channel connection to %s timed out', self.address)
42 | await c.close()
43 | del c
44 | del sock_stream
45 |
46 | except OSError as e:
47 | if attempts is not None:
48 | if nattempts >= attempts:
49 | raise e
50 | else:
51 | nattempts += 1
52 | else:
53 | log.error('Channel connection to %s failed', self.address, exc_info=True)
54 |
55 | await sock.close()
56 | await sleep(1)
57 |
58 |
59 | class Locker:
60 | def __init__(self, send):
61 | self.send = send
62 |
63 | def _getQ(self):
64 | asdf = set()
65 | while 1:
66 | try:
67 | asdf.add(self.urls.get_nowait())
68 | print('oh boy')
69 | except Empty:
70 | break
71 | print('current queue', asdf)
72 | #print(id(self))
73 | return asdf
74 |
75 | def _setQ(self, uris):
76 | for uri in uris:
77 | log.info('putting uri', uri)
78 | self.urls.put(uri)
79 | print('done putting', uris, 'in queue')
80 |
81 | def start_uri(self, uri):
82 | val = run(self.send, 'add ' + uri)
83 | if val:
84 | return True
85 | else:
86 | return
87 |
88 | print(self.lock, id(self.urls))
89 | with self.lock:
90 | print(self.lock, id(self.urls))
91 | uris = self._getQ()
92 | if uri in uris:
93 | log.info(uri, 'is already running')
94 | return True
95 | else:
96 | log.info('starting work for', uri)
97 | uris.add(uri)
98 | self._setQ(uris)
99 |
100 | def stop_uri(self, uri):
101 | run(self.send, 'del ' + uri)
102 | return
103 | #print(self.lock, id(self.urls))
104 | with self.lock:
105 | #print(self.lock, id(self.urls))
106 | uris = self._getQ()
107 | uris.discard(uri)
108 | print('completed work for', uri)
109 | self._setQ(uris)
110 |
111 |
112 | async def exit(task_group):
113 | await done.wait()
114 | clog.info(f'sync {task_group} exiting ...') # have to call this before cancel
115 | await task_group.cancel_remaining()
116 |
117 |
118 | async def manage_single_connection(connection, currently_running_urls):
119 | while True:
120 | try:
121 | msg = await connection.recv()
122 | except (EOFError, ConnectionResetError) as e: # in the event that the client closes
123 | mlog.info(f'connection {connection} closed due to {e}')
124 | break
125 | else:
126 | op, uri = msg.split(' ', 1)
127 | mlog.debug(f'{op} :: {uri}')
128 | if op == 'add':
129 | if uri in currently_running_urls:
130 | await connection.send(True)
131 | else:
132 | currently_running_urls.add(uri)
133 | await connection.send(False)
134 | elif op == 'del':
135 | currently_running_urls.discard(uri)
136 | await connection.send(False)
137 | else:
138 | await connection.send('ERROR')
139 | mlog.debug(currently_running_urls)
140 |
141 |
142 | async def manager(chan, syncword):
143 | encoded = syncword.encode()
144 | currently_running_urls = set()
145 | async def listen_for_new_conns(task_group):
146 | while True:
147 | ch = Channel(chan)
148 | try:
149 | connection = await ch.accept(authkey=encoded)
150 | mlog.info(f'new connection created {connection}')
151 | await task_group.spawn(manage_single_connection,
152 | connection,
153 | currently_running_urls)
154 | await ch.close() # sort of strange that we need this? can we connect again later !?
155 | except ConnectionResetError as e:
156 | mlog.warning('client connection attempt did not terminate property')
157 |
158 | async with TaskGroup() as connection_tasks:
159 | await connection_tasks.spawn(exit, connection_tasks)
160 | await connection_tasks.spawn(listen_for_new_conns, connection_tasks)
161 |
162 |
163 | # synchronization setup
164 | async def client(chan, syncword):
165 | encoded = syncword.encode()
166 | async def auth():
167 | ch = Channel(chan)
168 | async def connect(_ch=ch, authkey=encoded):
169 | connection = await _ch.connect(authkey=encoded)
170 | clog.debug(f'got connection {connection}')
171 | return connection
172 |
173 | async with TaskGroup(wait=any) as auth_or_exit:
174 | clog.info('waiting for sync services to start')
175 | exit_task = await auth_or_exit.spawn(exit, auth_or_exit)
176 | conn_task = await auth_or_exit.spawn(connect)
177 |
178 | connection = conn_task.result
179 | clog.debug(str(connection))
180 | return connection
181 |
182 | clog.info('starting auth')
183 | heh = [await auth()]
184 | async def send(uri):
185 | c = heh[0]
186 | async def sendit():
187 | await c.send(uri)
188 | resp = await c.recv()
189 | _uri = uri.split(' ', 1)[-1]
190 | msg = f'not :: {_uri}' if resp else f'run :: {_uri}'
191 | clog.debug(msg)
192 | return resp
193 |
194 | try:
195 | async with TaskGroup(wait=any) as send_or_exit:
196 | exit_task = await send_or_exit.spawn(exit, send_or_exit)
197 | send_task = await send_or_exit.spawn(sendit)
198 |
199 | try:
200 | resp = send_task.result
201 | return resp
202 | except TaskCancelled:
203 | return
204 | except RuntimeError as e: # FIXME not quite right?
205 | clog.error(e) # not eure what is causing this ... maybe a connection error?
206 |
207 | except (EOFError, BrokenPipeError) as e:
208 | c = await auth()
209 | heh[0] = c
210 | return await send(uri)
211 |
212 | return send
213 |
214 |
215 | def main():
216 | from scibot.config import syncword
217 | if syncword is None:
218 | raise KeyError('Please set the SCIBOT_SYNC environment variable')
219 |
220 | import os
221 | try:
222 | # This is in a try block because colorama (used by colorlog) wraps
223 | # redirected stdout to strip certain control codes which can cause an
224 | # AttributeError: 'NoneType' object has no attribute 'set_title'
225 | # because colorama changes os.sys.stdout.write in a way that
226 | # removes the call to set_title
227 | os.sys.stdout.write(f'\x1b]2;{os.path.basename(__file__)}\x07\n')
228 | except AttributeError as e:
229 | slog.exception(e)
230 |
231 | from docopt import docopt
232 | args = docopt(__doc__)
233 |
234 | chan = ('localhost', int(args['--port']))
235 | run(manager, chan, syncword)
236 |
237 |
238 | if __name__ == '__main__':
239 | main()
240 |
--------------------------------------------------------------------------------
/scibot/uri.py:
--------------------------------------------------------------------------------
1 | """
2 | Tools for dealing with URIs within the Hypothesis API.
3 |
4 | There are two main ways of considering the relationship between annotations and
5 | annotated objects:
6 |
7 | 1. Annotations are, practically speaking, made on web pages, and thus they have
8 | a URL associated with them.
9 |
10 | 2. Annotations are made on documents, and the particular HTML or PDF page being
11 | annotated is merely a specific manifestation of the abstract document that
12 | is being annotated. In this scenario, a document may be identified by one or
13 | more URIs.
14 |
15 | The second model is more complex from both a conceptual point of view and in
16 | terms of implementation, but it offers substantial benefits. If we talk of
17 | annotations attached to documents, without regard to presentation format or
18 | location, we are able to do many interesting things:
19 |
20 | - Alice makes an annotation on a PDF; Bob visits an HTML version of the same
21 | document, and sees Alice's annotation.
22 | - Alice makes an annotation on an Instapaper-hosted version of a web page which
23 | contains a tag. Bob visits the original article and sees
24 | Alice's annotation.
25 | - Bob makes an annotation on a PDF which is on his local machine. Alice opens
26 | the same PDF on her machine, and see's Bob's annotations even if the PDF has
27 | never been uploaded to a webserver. (We can do this because of the
28 | immutability of PDF documents -- we can uniquely fingerprint each one and
29 | form a "URN" of the form "urn:x-pdf:
6 | {% for error in field.errors %}
7 |
10 | {% endif %}
11 |
16 |
48 |
49 |
50 |
--------------------------------------------------------------------------------
/templates/results.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | SciBot Curation Dashboard
17 | {{ var }}
18 |
19 |
46 |
47 |
20 |
45 |
21 | Missing ({{ nmissing }})
22 |
23 |
24 | Unresolved ({{ nures }})
25 |
26 |
27 | Incorrect ({{ incor }})
28 |
29 |
30 | Papers ({{ npapers }})
31 |
32 |
33 | No PMID ({{ nnopmid }})
34 |
35 |
36 | No DOI ({{ nnodoi }})
37 |
38 |
39 | No Annos ({{ nnoannos }})
40 |
41 |
42 | All Problems ({{ allp }})
43 |
44 |
10 | {% for message in messages %}
11 |
14 | {% endif %}
15 | {% endwith %}
16 |
17 | Search Results
18 |
19 | {{results}}
20 |
21 |
22 |
--------------------------------------------------------------------------------
/templates/search.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
13 |
35 |
36 |
37 |
--------------------------------------------------------------------------------
/templates/table.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 | Search
14 |
15 | {% with messages = get_flashed_messages() %}
16 | {% if messages %}
17 |
18 | {% for message in messages %}
19 |
22 | {% endif %}
23 | {% endwith %}
24 |
25 | {% from "_formhelpers.html" import render_field %}
26 |
34 |
8 |
17 |
18 |
19 |
--------------------------------------------------------------------------------
/test/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/SciCrunch/scibot/3badc8d1ca4f441958d5062d1c5ba925996d22a2/test/__init__.py
--------------------------------------------------------------------------------
/test/test_extract.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | from scibot.extract import clean_text, find_rrids
3 |
4 |
5 | class TestFind(unittest.TestCase):
6 | def test_regex(self):
7 | text = """
8 | REAGENT or RESOURCE SOURCE IDENTIFIER
9 | Antibodies
10 |
11 |
12 | anti-mouse CD3, biotin, clone 17A2 BioLegend Cat# 100244; RRID: AB_2563947
13 | anti-mouse CD19, biotin, clone 6D5 BioLegend Cat# 115504; RRID: AB_312823
14 | anti-mouse Ly-6G and Ly-6C, biotin, clone RB6-8C5 BD Biosciences Cat# 553124; RRID: AB_394640
15 | anti-mouse CD16/CD32, biotin, clone 2.4G2 BD Biosciences Cat# 553143; RRID: AB_394658
16 | anti-mouse CD11b, biotin, clone M1/70 BD Biosciences Cat# 553309; RRID: AB_394773
17 | Bacterial and Virus Strains
18 |
19 |
20 | Lactobacillus plantarum (Lp 39 [IAM 12477]) ATCC ATCC14917
21 |
22 |
23 | Biological Samples
24 |
25 |
26 | Healthy adult intestine tissue Hospital Universitario de La Princesa (Madrid, Spain) N/A
27 | """
28 | found = list(find_rrids(clean_text(text)))
29 | assert len(found) == 5
30 |
31 |
--------------------------------------------------------------------------------
/test/test_resolver.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | from scibot.services import rrid_resolver_xml
3 |
4 |
5 | class TestResolver(unittest.TestCase):
6 | def test_redirect(self):
7 | found_rrids = {}
8 | out = rrid_resolver_xml('%20rid_000041', found_rrids)
9 |
10 |
--------------------------------------------------------------------------------
/test/test_routes.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | from flask import request
3 | from scibot.bookmarklet import main as bookmarklet_main
4 | from test.testing_data import form_data
5 |
6 |
7 | def start_uri(uri):
8 | # nothing real is being done, so alway return not running
9 | return False
10 |
11 |
12 | class TestRoutes(unittest.TestCase):
13 | @classmethod
14 | def setUpClass(cls):
15 | cls.app = bookmarklet_main()
16 | cls.app.URL_LOCK.start_uri = start_uri
17 | print(list(cls.app.view_functions.keys()))
18 |
19 | def test_bookmarket(self):
20 | func = self.app.view_functions['bookmarklet']
21 | with self.app.test_request_context('/bookmarket'):
22 | hrm = func()
23 |
24 | def test_rrid_post(self):
25 | func = self.app.view_functions['rrid']
26 | with self.app.test_request_context('/rrid', method='POST'):
27 | request.form = form_data
28 | hrm = func()
29 |
30 | def test_rrid_options(self):
31 | func = self.app.view_functions['rrid']
32 | with self.app.test_request_context('/rrid', method='OPTIONS'):
33 | hrm = func()
34 |
--------------------------------------------------------------------------------
/test/test_sync.py:
--------------------------------------------------------------------------------
1 | import unittest
2 | from time import sleep
3 | from threading import Thread
4 | from curio import run
5 | from scibot.sync import manager, client, Locker, done
6 | from scibot.utils import makeSimpleLogger
7 |
8 | log = makeSimpleLogger('sync test log')
9 | host = 'localhost'
10 | port = 11111
11 | syncword = 'syncword!'
12 |
13 |
14 | def manager_main():
15 | run(manager, (host, port), syncword)
16 |
17 |
18 | def client_main(uri='aaaaaaaaaaaaaaaaaaaaaaaaaa'):
19 | send = run(client, (host, port), syncword)
20 | URL_LOCK = Locker(send)
21 | while not done.is_set():
22 | URL_LOCK.start_uri(uri)
23 | sleep(0.5)
24 | URL_LOCK.stop_uri(uri)
25 | sleep(1)
26 |
27 |
28 | def main():
29 | manager_thread = Thread(target=manager_main, args=tuple())
30 | manager_thread.start()
31 | n_clients = 7
32 | messages = [chr(x + 97) * 10 for x in range(n_clients)]
33 | cthreads = []
34 | for i, msg in enumerate(messages):
35 | sleep(0.25)
36 | client_thread = Thread(target=client_main, args=(msg,))
37 | cthreads.append(client_thread)
38 | client_thread.start()
39 |
40 | sleep(4)
41 | done.set()
42 | log.info('waiting on clients')
43 | for i, t in enumerate(cthreads):
44 | t.join()
45 | log.info(f'joined client thread {i}')
46 |
47 | log.info('waiting on manager') # this doesn't work
48 | manager_thread.join()
49 |
50 |
51 | class TestSync(unittest.TestCase):
52 | def test_all(self):
53 | main()
54 |
--------------------------------------------------------------------------------
/test/testing_data.py:
--------------------------------------------------------------------------------
1 | form_data = {'body': ['\n'
2 | '
9 |
15 | {{returnStr}}
16 | Problem
10 | PMID
11 | Link
12 | Annotated By
13 | Notes
14 | SciBot
\n'
3 | '\n'
11 | 'javascript:(function(){var xhr=new XMLHttpRequest();\n'
12 | '\n'
13 | "var params='uri='+location.href+\n"
14 | "'&head='+encodeURIComponent(document.head.innerHTML)+\n"
15 | "'&body='+encodeURIComponent(document.body.innerHTML)+\n"
16 | "'&data='+encodeURIComponent(document.body.innerText);\n"
17 | '\n'
18 | "xhr.open('POST','https://localhost:4443/rrid',true);\n"
19 | "xhr.setRequestHeader('Content-type','application/x-www-form-urlencoded');\n"
20 | "xhr.setRequestHeader('Access-Control-Allow-Origin','*');\n"
21 | "xhr.onreadystatechange=function(){if(xhr.readyState==4)console.log('rrids: "
22 | "'+xhr.responseText)};\n"
23 | 'xhr.send(params)}());\n'
24 | '\n'
25 | '\n'
26 | '\n'],
27 | 'data': ['SciBot\n'
28 | '\n'
29 | 'To install the bookmarklet, drag this link -- SciBot localhost:4443 '
30 | '-- to your bookmarks bar.\n'
31 | '\n'
32 | "If you need to copy/paste the bookmarklet's code into a "
33 | "bookmarklet, it's here:\n"
34 | '\n'
35 | 'javascript:(function(){var xhr=new XMLHttpRequest(); var '
36 | "params='uri='+location.href+ "
37 | "'&head='+encodeURIComponent(document.head.innerHTML)+ "
38 | "'&body='+encodeURIComponent(document.body.innerHTML)+ "
39 | "'&data='+encodeURIComponent(document.body.innerText); "
40 | "xhr.open('POST','https://localhost:4443/rrid',true); "
41 | "xhr.setRequestHeader('Content-type','application/x-www-form-urlencoded'); "
42 | "xhr.setRequestHeader('Access-Control-Allow-Origin','*'); "
43 | "xhr.onreadystatechange=function(){if(xhr.readyState==4)console.log('rrids: "
44 | "'+xhr.responseText)}; xhr.send(params)}());"],
45 | 'head': ['\n'
46 | '\n'
51 | '