├── .gitignore ├── README.md ├── requirements.txt ├── templates ├── field.html ├── activity.html ├── css1 │ └── home.css ├── index.html └── home.html └── main.py /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | user_credentials.txt 3 | __pycache__/ 4 | *.log 5 | .DS_Store 6 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # reprolib server 2 | 3 | `pip3 install -r requirements.txt` 4 | 5 | 6 | to run server: 7 | 8 | `python3 main.py` 9 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | sanic==19.6.3 2 | Sanic-Cors==0.9.9.post3 3 | Jinja2==2.10.1 4 | sanic-jinja2==0.7.5 5 | PyLD==1.0.5 6 | # PyGithub 7 | # requests 8 | # reproschema 9 | 10 | 11 | -------------------------------------------------------------------------------- /templates/field.html: -------------------------------------------------------------------------------- 1 | 2 | 3 |

{{ data['skos:prefLabel']}}

4 |

{{ data.question }}

5 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /templates/activity.html: -------------------------------------------------------------------------------- 1 | 2 | 3 |

{{ data['skos:prefLabel'] }}

4 |

{{ data.preamble }}

5 |
    6 | {% for item in data.ui.order %} 7 |

  1. {{ item }}

    2. 8 | {% endfor %} 9 |
10 | 11 | 12 | -------------------------------------------------------------------------------- /templates/css1/home.css: -------------------------------------------------------------------------------- 1 | @import url(https://fonts.googleapis.com/css?family=Open+Sans); 2 | 3 | body{ 4 | background: #f2f2f2; 5 | font-family: 'Open Sans', sans-serif; 6 | } 7 | 8 | .search { 9 | width: 100%; 10 | position: relative; 11 | display: flex; 12 | } 13 | 14 | .searchTerm { 15 | width: 100%; 16 | border: 3px solid #00B4CC; 17 | border-right: none; 18 | padding: 5px; 19 | height: 20px; 20 | border-radius: 5px 0 0 5px; 21 | outline: none; 22 | color: #9DBFAF; 23 | } 24 | 25 | .searchTerm:focus{ 26 | color: #00B4CC; 27 | } 28 | 29 | .searchButton { 30 | width: 40px; 31 | height: 36px; 32 | border: 1px solid #00B4CC; 33 | background: #00B4CC; 34 | text-align: center; 35 | color: #fff; 36 | border-radius: 0 5px 5px 0; 37 | cursor: pointer; 38 | font-size: 20px; 39 | } 40 | 41 | /*Resize the wrap to see the search bar change!*/ 42 | .wrap{ 43 | width: 30%; 44 | position: absolute; 45 | top: 50%; 46 | left: 50%; 47 | transform: translate(-50%, -50%); 48 | } 49 | 50 | .sitename { 51 | max-width: 500px; 52 | min-width: auto; 53 | display: inline-block; 54 | text-shadow: 0 2px 0 #510000; 55 | padding: 0; 56 | top: 25px; 57 | left: -40px; 58 | } -------------------------------------------------------------------------------- /templates/index.html: -------------------------------------------------------------------------------- 1 |
2 |

LIST OF ACTIVITIES

3 |
4 |
5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 |
ACTIVITY NAMEJSON-LD
14 |
15 |
16 | 17 | 18 | 19 | {% for item in data.activities %} 20 | 21 | 28 | {# #} 33 | {# #} 35 | 36 | {% endfor %} 37 | 38 |
{{ item.name }} 22 | 23 | JSON-LD-logo-48 27 | #} 32 | {# {{ item.name }}#} 34 | {#
39 |
40 |
41 |
42 | 43 | -------------------------------------------------------------------------------- /templates/home.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | Home - Reprolib 4 | {# #} 5 | {# #} 6 | 7 | 8 | 9 | {#
#} 10 | {#
#} 11 | {# #} 21 | {#
#} 22 | {#
#} 23 |
24 |
25 | {# #} 48 |
49 |
50 | {#
#} 51 | {#
#} 52 | {#
#} 53 | {#

reprolib

#} 54 | {#
#} 55 | {#
#} 56 | {#
#} 57 | {# #} 58 | {# #} 61 | {#
#} 62 | {#
#} 63 | {#
#} 64 | {##} 65 | {#
#} 66 |
67 | 68 |
69 |
70 | 71 |
72 | 75 |

List of activites

76 | {% for item in data.activities %} 77 | 80 |
81 | {% endfor %} 82 |
83 |

List of protocols

84 | {% for item in data.protocols %} 85 | 88 |
89 | {% endfor %} 90 | 91 | 92 | 93 | 671 | 672 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | from sanic import Sanic 2 | from sanic import response 3 | from sanic.log import logger 4 | from pyld import jsonld 5 | from sanic_jinja2 import SanicJinja2 6 | from difflib import get_close_matches 7 | import requests 8 | import json 9 | import re 10 | import os, sys 11 | from sanic_cors import CORS 12 | 13 | production = 'DEV8dac6d02a913' not in os.environ 14 | basedir = '/vagrant' 15 | basedir = basedir if production else os.getcwd() 16 | logdir = os.path.join(basedir, "reprolib") 17 | if not os.path.exists(logdir): 18 | os.makedirs(logdir, exist_ok=True) 19 | 20 | LOG_SETTINGS = dict( 21 | version=1, 22 | disable_existing_loggers=False, 23 | loggers={ 24 | "sanic.root": {"level": "INFO", "handlers": ["console", "consolefile"]}, 25 | "sanic.error": { 26 | "level": "INFO", 27 | "handlers": ["error_console", "error_consolefile"], 28 | "propagate": True, 29 | "qualname": "sanic.error", 30 | }, 31 | "sanic.access": { 32 | "level": "INFO", 33 | "handlers": ["access_console", "access_consolefile"], 34 | "propagate": True, 35 | "qualname": "sanic.access", 36 | }, 37 | }, 38 | handlers={ 39 | "console": { 40 | "class": "logging.StreamHandler", 41 | "formatter": "generic", 42 | "stream": sys.stdout, 43 | }, 44 | "error_console": { 45 | "class": "logging.StreamHandler", 46 | "formatter": "generic", 47 | "stream": sys.stderr, 48 | }, 49 | "access_console": { 50 | "class": "logging.StreamHandler", 51 | "formatter": "access", 52 | "stream": sys.stdout, 53 | }, 54 | "consolefile": { 55 | 'class': 'logging.FileHandler', 56 | 'filename': os.path.join(logdir, "console.log"), 57 | "formatter": "generic", 58 | }, 59 | "error_consolefile": { 60 | 'class': 'logging.FileHandler', 61 | 'filename': os.path.join(logdir, "error.log"), 62 | "formatter": "generic", 63 | }, 64 | "access_consolefile": { 65 | 'class': 'logging.FileHandler', 66 | 'filename': os.path.join(logdir, "access.log"), 67 | "formatter": "access", 68 | }, 69 | }, 70 | formatters={ 71 | "generic": { 72 | "format": "%(asctime)s [%(process)d] [%(levelname)s] %(message)s", 73 | "datefmt": "[%Y-%m-%d %H:%M:%S %z]", 74 | "class": "logging.Formatter", 75 | }, 76 | "access": { 77 | "format": "%(asctime)s - (%(name)s)[%(levelname)s][%(host)s]: " 78 | + "%(request)s %(message)s %(status)d %(byte)d", 79 | "datefmt": "[%Y-%m-%d %H:%M:%S %z]", 80 | "class": "logging.Formatter", 81 | }, 82 | }, 83 | ) 84 | 85 | app = Sanic(log_config=LOG_SETTINGS) 86 | CORS(app) 87 | 88 | jinja = SanicJinja2(app) 89 | item_resp = {} 90 | activity_map = {} 91 | activityPrefLabel_map = {} 92 | protocolPrefLabel_map = {} 93 | 94 | async def replace_url(file_content, request): 95 | gh_url = "https://raw.githubusercontent.com/ReproNim/reproschema/master" 96 | hostname = await determine_env(request.headers['host']) 97 | for attribute, value in file_content.items(): 98 | # if value is str, replace substring 99 | if isinstance(value, str) and gh_url in value: 100 | value = value.replace(gh_url, 'https://' + hostname) 101 | # print(107, attribute, '-', value) 102 | file_content[attribute] = value 103 | # if value is list, replace substring in list of strings 104 | if isinstance(value, list): 105 | new_list = [] 106 | is_present = False 107 | for c in value: 108 | if gh_url in c: 109 | is_present = True 110 | c = c.replace(gh_url, 'https://' + hostname) 111 | new_list.append(c) 112 | if is_present: 113 | file_content[attribute] = new_list 114 | else: 115 | file_content[attribute] = value 116 | 117 | # if value is dict, repeat process 118 | if isinstance(value, dict): 119 | file_content[attribute] = await replace_url(value, request) 120 | return file_content 121 | 122 | 123 | async def determine_env(hostname): 124 | if '0.0.0.0' in hostname: 125 | return hostname 126 | else: 127 | return hostname + '/rl' 128 | 129 | 130 | @app.route("/update") 131 | def update(request): 132 | import subprocess as sp 133 | out = sp.run(['git', 'pull'], cwd='/opt/reproschema', capture_output=True) 134 | if out.returncode == 0: 135 | logger.info(out) 136 | else: 137 | logger.error(out) 138 | return response.json(out.__dict__, ensure_ascii=False, escape_forward_slashes=False) 139 | 140 | 141 | @app.route("/") 142 | async def test(request): 143 | hostname = await determine_env(request.headers['host']) 144 | api_list = {'activities': [], 'protocols': []} 145 | for activity in next(os.walk('/opt/reproschema-library/activities'))[1]: 146 | act_walks = next(os.walk('/opt/reproschema-library/activities/' + activity)) 147 | activityAlphaNum = re.sub('[^A-Za-z0-9]+', '', activity) # keep only alphanumeric characters 148 | for file in act_walks[2]: # loop over all files in the activity directory 149 | if file.endswith('_schema') and (file == activity+'_schema' or file == activity.lower()+'_schema' or file == activityAlphaNum+'_schema'): 150 | with open(os.path.join(act_walks[0], file), "r") as fa: 151 | try: 152 | act_schema = json.load(fa) 153 | if 'prefLabel' in act_schema: 154 | if isinstance(act_schema['prefLabel'], str): 155 | activityPrefLabel_map[activity] = act_schema['prefLabel'] 156 | else: 157 | activityPrefLabel_map[activity] = act_schema['prefLabel']['en'] 158 | else: activityPrefLabel_map[activity] = act_schema['prefLabel'] 159 | except Exception as e: 160 | print(153, 'error ---', file, e) 161 | # logger.error('error in json', file, e) 162 | if activity in activityPrefLabel_map: 163 | prefLabel = activityPrefLabel_map[activity] 164 | else: prefLabel = activity 165 | act_dict = { 166 | 'name': prefLabel, 167 | 'html_path': 'https://' + hostname + '/activities/' + 168 | activity, 169 | 'jsonld_path': 'https://' + hostname + '/activities/' + 170 | activity + '.jsonld', 171 | 'ttl_path': 'https://' + hostname + '/activities/' + 172 | activity + '.ttl', 173 | 'ui': 'https://schema.repronim.org/ui/#/activities/0/?url='+'https://' + hostname + '/activities/' + activity 174 | } 175 | 176 | api_list['activities'].append(act_dict) 177 | 178 | # sort in place alphabetically 179 | api_list['activities'].sort(key=lambda i: i['name'].lower()) 180 | 181 | # for protocol in next(os.walk('/opt/reproschema-library/protocols'))[1]: 182 | # protocol_walks = next(os.walk('/opt/reproschema-library/protocols/' + protocol)) 183 | # protocolAlphaNum = re.sub('[^A-Za-z0-9]+', '', protocol) # keep only alphanumeric characters 184 | # for file in protocol_walks[2]: # loop over all files in the protocol directory 185 | # if file.endswith('_schema') and ( 186 | # file == protocol + '_schema' or file == protocol.lower() + '_schema' or file == protocolAlphaNum + '_schema'): 187 | # with open(os.path.join(protocol_walks[0], file), "r") as fp: 188 | # try: 189 | # protocol_schema = json.load(fp) 190 | # protocolPrefLabel_map[protocol] = protocol_schema['skos:prefLabel'] 191 | # except Exception as e: 192 | # print(181, 'error', file, e) 193 | # # logger.error('error in json', file, e) 194 | # if protocol in protocolPrefLabel_map: 195 | # protocol_pref_label = protocolPrefLabel_map[protocol] 196 | # else: 197 | # protocol_pref_label = protocol 198 | # 199 | # protocol_dict = { 200 | # 'name': protocol_pref_label, 201 | # 'html_path': 'https://' + hostname + '/protocols/' + 202 | # protocol, 203 | # 'jsonld_path': 'https://' + hostname + '/protocols/' + 204 | # protocol + '.jsonld', 205 | # 'ttl_path': 'https://' + hostname + '/protocols/' + 206 | # protocol + '.jsonld', 207 | # 'ui': 'https://schema.repronim.org/ui/#/?url='+'https://' + hostname + '/protocols/' + protocol 208 | # } 209 | # api_list['protocols'].append(protocol_dict) 210 | # # sort in place alphabetically 211 | # api_list['protocols'].sort(key=lambda i: i['name'].lower()) 212 | return jinja.render("index.html", request, data=api_list) 213 | 214 | 215 | @app.route('/contexts/generic') 216 | async def get_generic_context(request): 217 | response_headers = {'Content-type': 'application/ld+json'} 218 | with open("/opt/reproschema/contexts/generic", "r") as f1: 219 | file_content = json.load(f1) 220 | new_file = await replace_url(file_content, request) 221 | return response.json(new_file, ensure_ascii=False, 222 | escape_forward_slashes=False, headers=response_headers) 223 | 224 | 225 | @app.route('/activities//items/') 226 | async def get_item(request, act_name, item_id): 227 | view_options = 2 # default view is jsonld 228 | response_headers = {'Content-type': 'application/ld+json'} 229 | filename, file_extension = os.path.splitext(item_id) 230 | if not file_extension: 231 | file_extension = '.jsonld' 232 | if request.headers.get('accept') == 'application/json' or \ 233 | request.headers.get('accept') == 'application/ld+json': 234 | view_options = 2 235 | else: 236 | # if not file_extension: 237 | # view_options = 1 # html view 238 | if file_extension == '.jsonld': 239 | view_options = 2 240 | try: 241 | with open("/opt/reproschema-library/activities/" + act_name 242 | + '/items/' + filename, "r") as f2: 243 | file_content = json.load(f2) 244 | # print(178, file_content) 245 | new_file = await replace_url(file_content, request) 246 | except: 247 | print('error getting contents') 248 | return response.text('Could not fetch data. Check item name') 249 | 250 | # if view_options == 1: 251 | # # render html 252 | # return jinja.render("field.html", request, data=new_file) 253 | 254 | if view_options == 2: 255 | return response.json(new_file, ensure_ascii=False, 256 | escape_forward_slashes=False, headers=response_headers) 257 | 258 | 259 | @app.route('/activities/') 260 | async def get_activity(request, act_name): 261 | hostname = await determine_env(request.headers['host']) 262 | filename, file_extension = os.path.splitext(act_name) 263 | if not file_extension: 264 | file_extension = '.jsonld' 265 | for activity in next(os.walk('/opt/reproschema-library/activities'))[1]: 266 | act_walks = next(os.walk('/opt/reproschema-library/activities/' + activity)) 267 | 268 | for file in act_walks[2]: # loop over all files in the activity directory 269 | if file.endswith('_schema'): 270 | # print(150, file, os.path.join(act_walks[0], file)) 271 | with open(os.path.join(act_walks[0], file), "r") as fa: 272 | try: 273 | act_schema = json.load(fa) 274 | # print(155, act_schema['@id']) 275 | activity_map[act_schema['@id']] = os.path.join(act_walks[0], file) 276 | except Exception as e: 277 | logger.error('error in json', file, e) 278 | matched_id = get_close_matches(filename, list(activity_map.keys()), 3, 0.2)[0] 279 | # print(269, activity_map[matched_id]) # returns path of matched file id 280 | 281 | with open(activity_map[matched_id], "r") as f5: 282 | try: 283 | file_content = json.load(f5) 284 | new_file = await replace_url(file_content, request) 285 | except ValueError: 286 | print('error!!') 287 | 288 | view_options = 2 # default view is html 289 | response_headers = {'Content-type': 'application/ld+json'} 290 | 291 | if 'application/json' in request.headers.get('accept') or \ 292 | 'application/ld+json' in request.headers.get('accept'): 293 | view_options = 2 294 | else: 295 | # if not file_extension: 296 | # #view_options = 1 # html view 297 | # file_extension = '.jsonld' 298 | if file_extension == '.jsonld': 299 | view_options = 2 300 | elif file_extension == '.ttl': 301 | view_options = 3 302 | 303 | if view_options == 2: 304 | # print('in json ') 305 | # jsonld 306 | return response.json(new_file, ensure_ascii=False, 307 | escape_forward_slashes=False, headers=response_headers) 308 | 309 | elif view_options == 3: 310 | try: 311 | #print('in turtle ', new_file) 312 | # turtle 313 | normalized_file = jsonld.normalize( 314 | new_file, {'base': 'https://' + hostname + '/activities/' + filename + '/', 'algorithm': 'URDNA2015', 'format': 315 | 'application/n-quads'}) 316 | return response.text(normalized_file, headers=response_headers) 317 | except Exception as e: 318 | print(e) 319 | raise 320 | 321 | 322 | @app.route('/terms/') 323 | async def get_terms(request, term_name): 324 | # view_options = 1 # default view is html 325 | view_options = 2 # make jsonld default for now 326 | response_headers = {'Content-type': 'application/ld+json'} 327 | filename, file_extension = os.path.splitext(term_name) 328 | if not file_extension: 329 | file_extension = '.jsonld' 330 | if request.headers.get('accept') == 'application/json' or \ 331 | request.headers.get('accept') == 'application/ld+json': 332 | view_options = 2 333 | else: 334 | # if not file_extension: 335 | # view_options = 1 # html view 336 | if file_extension == '.jsonld': 337 | view_options = 2 338 | with open("/opt/reproschema/terms/" + filename, "r") as f1: 339 | file_content = json.load(f1) 340 | new_file = await replace_url(file_content, request) 341 | 342 | if view_options == 2: 343 | # jsonld 344 | return response.json(new_file, ensure_ascii=False, 345 | escape_forward_slashes=False, headers=response_headers) 346 | 347 | # if view_options == 1: 348 | # # html. for time being it renders jsonld 349 | # try: 350 | # # TODO 351 | # return jinja.render("field.html", request, data=new_file) 352 | # except Exception as e: 353 | # logger.error(e) 354 | # # if it raises an Exception then deliver the jsonld 355 | # return response.json(new_file, ensure_ascii=False, 356 | # escape_forward_slashes=False, 357 | # headers=response_headers) 358 | 359 | 360 | @app.route('/resources/') 361 | async def get_resources(request, r_name): 362 | response_headers = {'Content-type': 'application/ld+json'} 363 | with open("/opt/reproschema/resources/" + r_name, "r") as f1: 364 | file_content = json.load(f1) 365 | return response.json(file_content, ensure_ascii=False, 366 | escape_forward_slashes=False, headers=response_headers) 367 | 368 | if __name__ == "__main__": 369 | logger.info("Starting reprolib-server") 370 | app.run(host="0.0.0.0", port=8000) 371 | 372 | --------------------------------------------------------------------------------