├── rdf_io ├── tests │ ├── __init__.py │ ├── test_viewspy.py │ └── test_mappings.py ├── migrations │ └── __init__.py ├── views │ ├── __init__.py │ ├── manage.py │ └── serialize.py ├── protocols │ ├── __init__.py │ ├── ldp.py │ ├── rdf4j.py │ └── api.py ├── __init__.py ├── signals │ ├── __init__.py │ └── utils.py ├── wsgi.py ├── fixtures │ ├── initial_data2.json │ └── default_namespaces.json ├── urls.py ├── settings.py ├── templates │ └── admin │ │ └── admin_publish.html ├── admin.py └── models.py ├── manage.py ├── .gitignore ├── setup.py ├── provision └── marmotta.yml ├── geonode_installation.md ├── AdvancedConfiguration.md ├── LICENSE └── README.md /rdf_io/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /rdf_io/migrations/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /rdf_io/views/__init__.py: -------------------------------------------------------------------------------- 1 | from .serialize import * 2 | from .manage import * 3 | -------------------------------------------------------------------------------- /rdf_io/protocols/__init__.py: -------------------------------------------------------------------------------- 1 | from ..protocols.api import * 2 | from ..protocols import api,rdf4j,ldp 3 | 4 | -------------------------------------------------------------------------------- /rdf_io/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | try: 3 | from django.core import signals 4 | except: 5 | from . import signals 6 | 7 | __version__ = (0, 4) 8 | -------------------------------------------------------------------------------- /manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | if __name__ == "__main__": 6 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "rdf_io.settings") 7 | 8 | from django.core.management import execute_from_command_line 9 | 10 | execute_from_command_line(sys.argv) 11 | -------------------------------------------------------------------------------- /rdf_io/signals/__init__.py: -------------------------------------------------------------------------------- 1 | from rdf_io.signals.utils import * 2 | # originally configured signals automatically - but because this results in circular dependencies with modules that set up object mappings this is deprecated 3 | #signals.post_save.connect(setup_signals, sender=ObjectMapping) 4 | #sync_signals() 5 | -------------------------------------------------------------------------------- /rdf_io/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for rdf_io project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/1.6/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "rdf_io.settings") 12 | 13 | from django.core.wsgi import get_wsgi_application 14 | application = get_wsgi_application() 15 | -------------------------------------------------------------------------------- /rdf_io/fixtures/initial_data2.json: -------------------------------------------------------------------------------- 1 | [ { 2 | "pk" : null, 3 | "model" : "rdf_io.objecttype", 4 | "fields" : { 5 | "uri" : "skos:Concept", 6 | "label" : "SKOS Concept" 7 | } 8 | }, { 9 | "pk" : null, 10 | "model" : "rdf_io.objecttype", 11 | "fields" : { 12 | "uri" : "owl:Class", 13 | "label" : "OWL Class" 14 | } 15 | }, { 16 | "pk" : null, 17 | "model" : "rdf_io.objecttype", 18 | "fields" : { 19 | "uri" : "void:Dataset", 20 | "label" : "VoiD Dataset" 21 | } 22 | }, { 23 | "pk" : null, 24 | "model" : "rdf_io.objecttype", 25 | "fields" : { 26 | "uri" : "skos:ConceptScheme", 27 | "label" : "SKOS ConceptScheme" 28 | } 29 | }, { 30 | "pk" : null, 31 | "model" : "rdf_io.objecttype", 32 | "fields" : { 33 | "uri" : "owl:ontology", 34 | "label" : "OWL Ontology" 35 | } 36 | } 37 | ] 38 | -------------------------------------------------------------------------------- /rdf_io/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls import url 2 | try: 3 | from .views import ctl_signals,show_config,sync_remote,to_rdfbyid,pub_rdf, to_rdfbykey 4 | except: 5 | from views import ctl_signals,show_config,sync_remote,to_rdfbyid,pub_rdf, to_rdfbykey 6 | 7 | from django.contrib import admin 8 | admin.autodiscover() 9 | 10 | urlpatterns = [ 11 | # Examples: 12 | # url(r'^$', 'rdf_io.views.home', name='home'), 13 | # url(r'^blog/', include('blog.urls')), 14 | url(r'to_rdf/(?P[^\/]+)/id/(?P\d+)$', to_rdfbyid, name='to_rdfbyid'), 15 | url(r'to_rdf/(?P[^\/]+)/key/(?P.+)$', to_rdfbykey, name='to_rdfbykey'), 16 | url(r'pub_rdf/(?P[^\/]+)/(?P\d+)$', pub_rdf, name='pub_rdf'), 17 | # management urls - add user auth 18 | url(r'sync_remote/(?P[^\/]+)$', sync_remote, name='sync_remote'), 19 | url(r'show_config$', show_config, name='show_config'), 20 | url(r'ctl_signals/(?P[^\/]+)$', ctl_signals, name='ctl_signals'), 21 | # url(r'^admin/', include(admin.site.urls)), 22 | ] 23 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .coverage.* 42 | .cache 43 | nosetests.xml 44 | coverage.xml 45 | *,cover 46 | .hypothesis/ 47 | 48 | # Translations 49 | *.mo 50 | *.pot 51 | 52 | # Django stuff: 53 | *.log 54 | 55 | # Sphinx documentation 56 | docs/_build/ 57 | 58 | # PyBuilder 59 | target/ 60 | 61 | #Ipython Notebook 62 | .ipynb_checkpoints 63 | rdf_io/migrations/ 64 | *.backup 65 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | try: 5 | from setuptools import setup, find_packages 6 | except ImportError: 7 | import ez_setup 8 | ez_setup.use_setuptools() 9 | from setuptools import setup, find_packages 10 | 11 | VERSION = '0.4' 12 | 13 | import os 14 | def read(fname): 15 | return open(os.path.join(os.path.dirname(__file__), fname)).read() 16 | 17 | setup( 18 | name='django-rdf-io', 19 | version = VERSION, 20 | description='Pluggable application for mapping elements of django models to an external RDF store', 21 | packages=['rdf_io'], 22 | include_package_data=True, 23 | author='Rob Atkinson', 24 | author_email='rob@metalinkage.com,au', 25 | license='BSD', 26 | long_description=read('README.md'), 27 | #download_url = "https://github.com/quinode/django-skosxl/tarball/%s" % (VERSION), 28 | #download_url='git://github.com/quinode/django-skosxl.git', 29 | zip_safe=False, 30 | install_requires = ['rdflib>=4.0', 31 | 'rdflib-jsonld', 32 | 'requests', 33 | 'six' 34 | ] 35 | 36 | ) 37 | 38 | -------------------------------------------------------------------------------- /rdf_io/protocols/ldp.py: -------------------------------------------------------------------------------- 1 | 2 | import requests 3 | from .api import RDFStoreException 4 | 5 | def ldp_push(rdfstore, resttgt, model, obj, gr, mode ): 6 | """ publish using LDP protocol """ 7 | etag = _get_etag(resttgt) 8 | headers = {'Content-Type': 'text/turtle'} 9 | if etag : 10 | headers['If-Match'] = etag 11 | 12 | for h in rdfstore.get('headers') or [] : 13 | headers[h] = resolveTemplate( rdfstore['headers'][h], model, obj ) 14 | 15 | result = requests.put( resttgt, headers=headers , data=gr.serialize(format="turtle"), auth=rdfstore.get('auth')) 16 | #logger.info ( "Updating resource {} {}".format(resttgt,result.status_code) ) 17 | if result.status_code > 400 : 18 | # print "Posting new resource" 19 | # result = requests.post( resttgt, headers=headers , data=gr.serialize(format="turtle")) 20 | # logger.error ( "Failed to publish resource {} {}".format(resttgt,result.status_code) ) 21 | raise RDFStoreException("Failed to publish resource {} {} : {} ".format(resttgt,result.status_code, result.content) ) 22 | return result 23 | 24 | def _get_etag(uri): 25 | """ 26 | Gets the LDP Etag for a resource if it exists 27 | """ 28 | # could put in cache here - but for now just issue a HEAD 29 | result = requests.head(uri) 30 | return result.headers.get('ETag') -------------------------------------------------------------------------------- /rdf_io/tests/test_viewspy.py: -------------------------------------------------------------------------------- 1 | from rdf_io.views import * 2 | from django.contrib.contenttypes.models import ContentType 3 | from django.test import TestCase, RequestFactory 4 | from django.http import HttpRequest,HttpResponse,Http404 5 | from django.contrib.auth.models import AnonymousUser, User 6 | # from rdf_io.tests import ObjectMappingTestCase 7 | from .test_mappings import SerialisationSetupTestCase 8 | 9 | class RequestTestCase(SerialisationSetupTestCase): 10 | """ Test case for a view request 11 | 12 | """ 13 | def setUp(self): 14 | super(RequestTestCase,self).setUp() 15 | self.factory = RequestFactory() 16 | 17 | 18 | def test_ttl_serialise(self): 19 | request = self.factory.get('/rdf_io/to_rdf/objectmapping/id/1?_format=turtle') 20 | request.user = AnonymousUser() 21 | res = to_rdfbyid( request, 'objectmapping',1) 22 | self.assertTrue(res.content.find(b'rdfs:label "Mappings test"') >= 0) 23 | 24 | def test_json_serialise(self): 25 | #import pdb; pdb.set_trace() 26 | request = self.factory.get('/rdf_io/to_rdf/objectmapping/id/1?_format=json') 27 | request.user = AnonymousUser() 28 | res = to_rdfbyid( request, 'objectmapping',1) 29 | self.assertTrue(res.content.find(b'"@value": "Mappings test"') >= 0) 30 | 31 | def test_default_ttl_serialise(self): 32 | request = self.factory.get('/rdf_io/to_rdf/objectmapping/id/1') 33 | request.user = AnonymousUser() 34 | res = to_rdfbyid( request, 'objectmapping',1) 35 | #print res.content 36 | self.assertTrue(res.content.find(b'rdfs:label "Mappings test"') >= 0) -------------------------------------------------------------------------------- /rdf_io/settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | Django settings for rdf_io project. 3 | 4 | For more information on this file, see 5 | https://docs.djangoproject.com/en/1.6/topics/settings/ 6 | 7 | For the full list of settings and their values, see 8 | https://docs.djangoproject.com/en/1.6/ref/settings/ 9 | """ 10 | 11 | # Build paths inside the project like this: os.path.join(BASE_DIR, ...) 12 | import os 13 | BASE_DIR = os.path.dirname(os.path.dirname(__file__)) 14 | 15 | 16 | # Quick-start development settings - unsuitable for production 17 | # See https://docs.djangoproject.com/en/1.6/howto/deployment/checklist/ 18 | 19 | # SECURITY WARNING: keep the secret key used in production secret! 20 | SECRET_KEY = '(@fkjrj2_%ywk395d$q^6h=h44%9j2kb3(%j)+k3#cn3h1=&nf' 21 | 22 | # SECURITY WARNING: don't run with debug turned on in production! 23 | DEBUG = True 24 | 25 | TEMPLATE_DEBUG = True 26 | 27 | ALLOWED_HOSTS = [] 28 | 29 | 30 | # Application definition 31 | 32 | INSTALLED_APPS = ( 33 | 'django.contrib.admin', 34 | 'django.contrib.auth', 35 | 'django.contrib.contenttypes', 36 | 'django.contrib.sessions', 37 | 'django.contrib.messages', 38 | 'django.contrib.staticfiles', 39 | 'rdf_io', 40 | ) 41 | 42 | MIDDLEWARE_CLASSES = ( 43 | 'django.contrib.sessions.middleware.SessionMiddleware', 44 | 'django.middleware.common.CommonMiddleware', 45 | 'django.middleware.csrf.CsrfViewMiddleware', 46 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 47 | 'django.contrib.messages.middleware.MessageMiddleware', 48 | 'django.middleware.clickjacking.XFrameOptionsMiddleware', 49 | ) 50 | 51 | ROOT_URLCONF = 'rdf_io.urls' 52 | 53 | WSGI_APPLICATION = 'rdf_io.wsgi.application' 54 | 55 | 56 | # Database 57 | # https://docs.djangoproject.com/en/1.6/ref/settings/#databases 58 | 59 | DATABASES = { 60 | 'default': { 61 | 'ENGINE': 'django.db.backends.sqlite3', 62 | 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), 63 | } 64 | } 65 | 66 | # Internationalization 67 | # https://docs.djangoproject.com/en/1.6/topics/i18n/ 68 | 69 | LANGUAGE_CODE = 'en-us' 70 | 71 | TIME_ZONE = 'UTC' 72 | 73 | USE_I18N = True 74 | 75 | USE_L10N = True 76 | 77 | USE_TZ = True 78 | 79 | 80 | # Static files (CSS, JavaScript, Images) 81 | # https://docs.djangoproject.com/en/1.6/howto/static-files/ 82 | 83 | STATIC_URL = '/static/' 84 | -------------------------------------------------------------------------------- /rdf_io/protocols/rdf4j.py: -------------------------------------------------------------------------------- 1 | 2 | import requests 3 | from .api import RDFStoreException, resolveTemplate 4 | 5 | 6 | def rdf4j_push(rdfstore, model, obj, gr, bindingtype, mode ): 7 | #import pdb; pdb.set_trace() 8 | from rdf_io.models import ServiceBinding 9 | headers = {'Content-Type': 'application/x-turtle;charset=UTF-8'} 10 | 11 | resttgt = resolveTemplate("".join( ( rdfstore['server'],rdfstore['target'])), model, obj , mode) 12 | for h in rdfstore.get('headers') or [] : 13 | headers[h] = resolveTemplate( rdfstore['headers'][h], model, obj , mode) 14 | 15 | if bindingtype == ServiceBinding.PERSIST_REPLACE : 16 | result = requests.put( resttgt, headers=headers , data=gr.serialize(format="turtle")) 17 | elif bindingtype == ServiceBinding.PERSIST_UPDATE : 18 | result = requests.post( resttgt, headers=headers , data=gr.serialize(format="turtle")) 19 | elif bindingtype == ServiceBinding.PERSIST_PURGE : 20 | result = requests.delete( resttgt, headers=headers ) 21 | else: 22 | raise Exception ("RDF4J store does not yet support mode %s" % (mode,)) 23 | # logger.info ( "Updating resource {} {}".format(resttgt,result.status_code) ) 24 | if result.status_code > 400 : 25 | # print "Posting new resource" 26 | # result = requests.post( resttgt, headers=headers , data=gr.serialize(format="turtle")) 27 | # logger.error ( "Failed to publish resource {} {}".format(resttgt,result.status_code) ) 28 | raise RDFStoreException ("Failed to publish resource {} {}".format(resttgt,result.status_code ) ) 29 | return result 30 | 31 | def rdf4j_get(rdfstore, model,obj , mode ): 32 | """ Gets a response from an RDF4J datastore access method. Returns HTTP request 33 | 34 | """ 35 | headers = {'Content-Type': 'application/x-turtle;charset=UTF-8'} 36 | 37 | resttgt = resolveTemplate("".join( ( rdfstore['server'],rdfstore['target'])), model, obj ,mode) 38 | result = requests.get( resttgt, headers=headers ) 39 | return result 40 | 41 | def rdf4j_delete(rdfstore, model,obj, mode): 42 | """ Gets a response from an RDF4J datastore access method. Returns HTTP request 43 | 44 | """ 45 | headers = {'Content-Type': 'application/x-turtle;charset=UTF-8'} 46 | 47 | resttgt = resolveTemplate("".join( ( rdfstore['server'],rdfstore['target'])), model, obj , mode ) 48 | result = requests.delete( resttgt, headers=headers ) 49 | return result -------------------------------------------------------------------------------- /provision/marmotta.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # YAML script for provisioning a local Marmotta instance 3 | # 4 | # this is untested - recording setup steps 5 | 6 | - name: install java8 7 | apt_repository: repo='ppa:openjdk-r/ppa' 8 | 9 | - name: Install Marmotta Dependencies 10 | apt: name={{item}} state=present update_cache=yes cache_valid_time=3600 11 | with_items: 12 | - {{ tomcat_version }} 13 | # needed if we're not pulling in tomcat 14 | - openjdk-8-jdk 15 | tags: [install] 16 | 17 | - file: path=/tmp/{{ application_name }} state=directory 18 | tags: [ldp] 19 | 20 | - name: fetch marmotta 21 | get_url: url={{ marmotta_download }} dest=/var/lib/{{ tomcat_version }}/webapps/marmotta.war 22 | tags: [ldp] 23 | 24 | 25 | - stat: path=/var/lib/{{ tomcat_version }}/webapps/geoserver/WEB-INF/lib/postgresql-9.4-1201-jdbc41.jar 26 | register: postgresql_source 27 | tags: [postgres] 28 | 29 | - stat: path=/usr/share/{{ tomcat_version }}/lib/postgresql-9.4-1201-jdbc41.jar 30 | register: postgresql_target 31 | tags: [postgres] 32 | 33 | - name: move postgresql-9.4-1201-jdbc41.jar 34 | command: mv [postgresql_source - need YAML syntax for this!!] [postgresql_target] 35 | sudo: yes 36 | when: postgresql_source.stat.exists and not postgresql_target.stat.exists 37 | tags: [postgres] 38 | 39 | 40 | - name: restart tomcat 41 | service: name={{ tomcat_version }} state=restarted 42 | tags: [marmotta] 43 | 44 | - wait_for: path=/var/lib/{{ tomcat_version }}/webapps/marmotta/WEB-INF/ 45 | 46 | ## set up marmotta user and jdbc config 47 | 48 | ## sudo -u postgres createuser -P marmotta 49 | 50 | send something to marmotta as its very first web access to set its hostname! 51 | http://resources.opengeospatial.org/ 52 | 53 | curl -X POST -u admin:pass123 -d '["standard"]' http://localhost:8080/marmotta/config/data/security.profile 54 | curl -iX POST -H "Content-Type: application/json" -d '["standard"]' http://localhost:8080/marmotta/config/data/security.profile 55 | edit /tmp/marmotta/system-config.properties 56 | add security.enabled = true 57 | - if use postgres set database.url and database username password/ 58 | 59 | run syncdb - set DJANGO_SETTINGS_ and create a manage.py above geonode installed path - all a massive pain. 60 | 61 | http://resources.opengeospatial.org/ 62 | 63 | 64 | add voc container to marmotta: 65 | curl -i -X POST -d @../django-skosxl/skosxl/fixtures/voc_ldp_container.ttl -H "Content-Type: text/turtle" -H "Slug: voc" http://resources.opengeospatial.org:8080/marmotta/ldp 66 | cd {project}/../django-skosxl/skosxl/fixtures 67 | curl -i -X POST -u admin:xxx -d @voc_ldp_container.ttl -H "Content-Type: text/turtle" -H "Slug: voc" http://resources.opengeospatial.org:8080/marmotta/ldp 68 | register rules 69 | curl -i -u admin:xxx -H "Content-Type: text/plain" -X POST --data-binary @skos.kwrl http://localhost:8080/marmotta/reasoner/program/skos.kwrl 70 | -------------------------------------------------------------------------------- /geonode_installation.md: -------------------------------------------------------------------------------- 1 | # Geonode installation notes 2 | (may apply to other bundled django applications, where mod_wsgi is used to connect to django) 3 | 4 | # note this is out of date - should be revisited now Geonode has migrated to django 2.2 5 | 6 | # Use Case 7 | an existing django application has been installed on unbuntu using apt-get 8 | 9 | This is challenging as its not obvious where things are configured and how to propagate changes. Here are notes from going through this process - if there is a better way please document it here and throw these away! 10 | 11 | # Installing new apps 12 | 13 | This is standard process - up to a point.. 14 | 15 | install django modules - into the same environment you app is installed - note that using the apt-get method means the app is globally installed, not into a virtualenv, and hence most of the django developer docs are no longer relevant 16 | 17 | set up django-admin to use the default app : 18 | - this is tricky because with apt-get installation you do not get the containing django project directory with manage.py - at least I could not find if/where this was available in the default installation 19 | e.g. 20 | mkdir /usr/share/django-apps 21 | cd /usr/share/django-apps 22 | ln -s /usr/local/lib/python2.7/dist-packages/geonode . 23 | create a manage.py in /usr/share/django-apps 24 | edit so settings are geonode/settings ( note, you cannot run this inside geonode because of conflicts between imported module "geoserver" and local module! - 25 | 26 | 27 | now we need to make the installed app know about the extensions: 28 | edit geonode/settings (eg local_settings.py) to include new packages in INSTALLED_APPS, and other settings required (RDFSTORE) 29 | edit geonode/urls.py 30 | 31 | run python manage.py syncdb 32 | 33 | NB this will load initial data 34 | 35 | ## TODO 36 | work out how signals to publish to remote RDFSTORE survive this installation process 37 | 38 | # to propagate configuration changes to running server: 39 | 40 | touch /var/www/geonode/wsgi/geonode.wsgi 41 | 42 | load some content for which RDF mappings exist 43 | test : 44 | 45 | ## to get into debug mode on the server 46 | 47 | one way to test stuff is: 48 | python manage.py shell 49 | 50 | * import all the things in the module you want to invoke so you can run methods and interact with object types: 51 | `from django.shortcuts import render_to_response, redirect 52 | from rdf_io.models import ObjectMapping,Namespace,AttributeMapping, ObjectType, getattr_path 53 | from django.template import RequestContext 54 | from django.contrib.contenttypes.models import ContentType 55 | from django.conf import settings 56 | from string import Formatter 57 | import requests 58 | ` 59 | 60 | * set up debugger: 61 | * set up breakpoint 62 | * back to shell: 63 | * call function to debug 64 | 65 | eg 66 | ` 67 | import pdb; pdb.set_trace() 68 | b rdf_io/views:217 69 | c 70 | from rdf_io.views import do_sync_remote 71 | do_sync_remote('scheme') 72 | ` 73 | 74 | -------------------------------------------------------------------------------- /rdf_io/templates/admin/admin_publish.html: -------------------------------------------------------------------------------- 1 | {% extends "admin/base_site.html" %} 2 | 3 | {% block content %} 4 | 32 | 33 |
34 | {% csrf_token %} 35 | 36 | {% for scheme in schemes %} 37 | 38 | {% endfor %} 39 | 40 |

41 | 42 |

Publishing in REVIEW or PUBLISH mode uses alternative Config Variables values if scoped to these modes. Publishing executes the configured ServiceBinding chains for these objects with the selected set of config variables.

43 |

Show config vars 44 |

45 |
{% for var in pubvars %} 46 | 47 | {% endfor %} 48 |
PUBLISH
{{ var.var }}{{ var.value }}
49 |
50 |
51 | {% for var in reviewvars %} 52 | 53 | {% endfor %} 54 |
REVIEW
{{ var.var }}{{ var.value }}
55 |
56 |

Choose publishing options

57 |
58 | 59 |
60 | 61 |
62 | 63 | 64 |
65 |
66 | 67 | 68 |
69 |
70 | 71 | 72 |
73 | 74 | 90 | 91 | {% endblock %} -------------------------------------------------------------------------------- /rdf_io/fixtures/default_namespaces.json: -------------------------------------------------------------------------------- 1 | [{ 2 | "pk" : null, 3 | "model" : "rdf_io.namespace", 4 | "fields" : { 5 | "notes" : "RDF", 6 | "prefix" : "rdf", 7 | "uri" : [ "http://www.w3.org/1999/02/22-rdf-syntax-ns#" ] 8 | } 9 | }, 10 | { 11 | "pk" : null, 12 | "model" : "rdf_io.namespace", 13 | "fields" : { 14 | "notes" : "RDFS", 15 | "prefix" : "rdfs", 16 | "uri" : [ "http://www.w3.org/2000/01/rdf-schema#" ] 17 | } 18 | }, 19 | { 20 | "pk" : null, 21 | "model" : "rdf_io.namespace", 22 | "fields" : { 23 | "notes" : "SKOS", 24 | "prefix" : "skos", 25 | "uri" : [ "http://www.w3.org/2004/02/skos/core#" ] 26 | } 27 | }, 28 | { 29 | "pk" : null, 30 | "model" : "rdf_io.namespace", 31 | "fields" : { 32 | "notes" : "SKOS-XL", 33 | "prefix" : "skosxl", 34 | "uri" : [ "http://www.w3.org/2008/05/skos-xl#" ] 35 | } 36 | }, 37 | { 38 | "pk" : null, 39 | "model" : "rdf_io.namespace", 40 | "fields" : { 41 | "notes" : "FOAF", 42 | "prefix" : "foaf", 43 | "uri" : [ "http://xmlns.com/foaf/0.1/" ] 44 | } 45 | }, 46 | { 47 | "pk" : null, 48 | "model" : "rdf_io.namespace", 49 | "fields" : { 50 | "notes" : "Dublin Core", 51 | "prefix" : "dct", 52 | "uri" : [ "http://purl.org/dc/terms/" ] 53 | } 54 | } 55 | , 56 | { 57 | "pk" : null, 58 | "model" : "rdf_io.namespace", 59 | "fields" : { 60 | "notes" : "DCAT", 61 | "prefix" : "dcat", 62 | "uri" : [ "http://www.w3.org/ns/dcat#" ] 63 | } 64 | } 65 | , 66 | { 67 | "pk" : null, 68 | "model" : "rdf_io.namespace", 69 | "fields" : { 70 | "notes" : "VCARD", 71 | "prefix" : "vcard", 72 | "uri" : [ "http://www.w3.org/2006/vcard/ns#" ] 73 | } 74 | } , 75 | { 76 | "pk" : null, 77 | "model" : "rdf_io.namespace", 78 | "fields" : { 79 | "notes" : "XSD", 80 | "prefix" : "xsd", 81 | "uri" : [ "http://www.w3.org/2001/XMLSchema#" ] 82 | } 83 | }, 84 | { 85 | "pk" : null, 86 | "model" : "rdf_io.objecttype", 87 | "fields" : { 88 | "uri" : [ "skos:Concept", ] 89 | "label" : "SKOS Concept" 90 | } 91 | }, { 92 | "pk" : null, 93 | "model" : "rdf_io.objecttype", 94 | "fields" : { 95 | "uri" : [ "owl:Class", ] 96 | "label" : "OWL Class" 97 | } 98 | }, { 99 | "pk" : null, 100 | "model" : "rdf_io.objecttype", 101 | "fields" : { 102 | "uri" : [ "void:Dataset", ] 103 | "label" : "VoiD Dataset" 104 | } 105 | }, { 106 | "pk" : null, 107 | "model" : "rdf_io.objecttype", 108 | "fields" : { 109 | "uri" : [ "skos:ConceptScheme", ] 110 | "label" : "SKOS ConceptScheme" 111 | } 112 | }, { 113 | "pk" : null, 114 | "model" : "rdf_io.objecttype", 115 | "fields" : { 116 | "uri" : [ "owl:ontology", ] 117 | "label" : "OWL Ontology" 118 | } 119 | } 120 | , { 121 | "pk" : null, 122 | "model" : "rdf_io.genericmetaprop", 123 | "fields" : { 124 | "namespace" : ["http://purl.org/dc/terms/" ], 125 | "propname" : "rights", 126 | "definition" : "Information about rights held in and over the resource." 127 | } 128 | } 129 | , { 130 | "pk" : null, 131 | "model" : "rdf_io.genericmetaprop", 132 | "fields" : { 133 | "namespace" : ["http://purl.org/dc/terms/" ], 134 | "propname" : "source", 135 | "definition" : "A related resource from which the described resource is derived." 136 | } 137 | } 138 | ] 139 | 140 | -------------------------------------------------------------------------------- /rdf_io/signals/utils.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | from builtins import str 3 | from django.conf import settings 4 | from django.db.models import signals 5 | 6 | from django.contrib.contenttypes.models import ContentType 7 | from rdf_io.models import ObjectMapping 8 | from rdf_io.views import publish 9 | 10 | import logging 11 | logger = logging.getLogger(__name__) 12 | 13 | def publish_rdf( **kwargs) : 14 | obj = kwargs['instance'] 15 | try: 16 | if obj.skip_post_save : 17 | return 18 | except: # false or not supported 19 | pass 20 | ct = ContentType.objects.get_for_model(obj) 21 | oml = ObjectMapping.objects.filter(content_type=ct) 22 | result = publish( obj, ct.name, oml, None) 23 | logger.debug( 24 | "Persisting RDF for {} of type {} status {} body {}".format(obj,ct,result.status_code,result.content)) 25 | print("Persisting RDF for {} of type {} status {} body {}".format(obj,ct,result.status_code,result.content)) 26 | 27 | def setup_signals( **kwargs) : 28 | objmapping = kwargs['instance'] 29 | #import pdb; pdb.set_trace() 30 | _setup(objmapping) 31 | 32 | def sync_signals(): 33 | """For each ObjectMapping force signal to be added or removed""" 34 | mappedObj = ObjectMapping.objects.all() 35 | for objmapping in mappedObj: 36 | _setup(objmapping) 37 | return "synced RDf publishing signals" 38 | 39 | def _setup(objmapping) : 40 | try: 41 | if objmapping.auto_push : 42 | ct = ContentType.objects.get(id = objmapping.content_type_id).model_class() 43 | signals.post_save.connect(publish_rdf, sender=ct, dispatch_uid=str(ct.__name__)) 44 | print("RDF publishing configured for model {}".format((ct))) 45 | logger.info( 46 | "RDF publishing configured for model {}".format((ct))) 47 | except Exception as e: 48 | print("Error trying to set up auto-publish for object mapping.. %s " % e) 49 | pass 50 | 51 | def clear_signals(): 52 | """For each ObjectMapping force signal to be removed""" 53 | mappedObj = ObjectMapping.objects.all() 54 | for objmapping in mappedObj: 55 | _clear(objmapping) 56 | return "synced RDf publishing signals" 57 | 58 | def _clear(objmapping) : 59 | try: 60 | if objmapping.auto_push : 61 | ct = ContentType.objects.get(id = objmapping.content_type_id).model_class() 62 | signals.post_save.disconnect(publish_rdf, sender=ct, dispatch_uid=str(ct.__name__)) 63 | print("RDF publishing un-configured for model {}".format((ct))) 64 | logger.info( 65 | "RDF publishing un-configured for model {}".format((ct))) 66 | except Exception as e: 67 | print("Error trying to clear auto-publish for object mapping.. %s " % e) 68 | pass 69 | 70 | def list_pubs(): 71 | # import pdb; pdb.set_trace() 72 | import weakref 73 | msg = [] 74 | for receiver in signals.post_save.receivers: 75 | (receiver_class_name,_), receiver = receiver 76 | # django.contrib.contenttypes.generic.GenericForeignKey.instance_pre_init is not weakref 77 | if isinstance(receiver, weakref.ReferenceType): 78 | receiver = receiver() 79 | receiver = getattr(receiver, '__wraps__', receiver) 80 | receiver_name = getattr(receiver, '__name__', str(receiver)) 81 | text = "%s.%s" % (receiver_class_name, receiver_name) 82 | if receiver_name == 'publish_rdf' : 83 | msg.append(text) 84 | 85 | return str(msg) 86 | -------------------------------------------------------------------------------- /rdf_io/views/manage.py: -------------------------------------------------------------------------------- 1 | # # -*- coding:utf-8 -*- 2 | #from django.shortcuts import render_to_response, redirect 3 | from rdf_io.models import ObjectMapping,Namespace,AttributeMapping,EmbeddedMapping, ObjectType, getattr_path, apply_pathfilter, expand_curie, dequote 4 | from rdf_io.views import get_rdfstore,publish 5 | from django.template import RequestContext 6 | from django.contrib.contenttypes.models import ContentType 7 | from django.conf import settings 8 | from string import Formatter 9 | from rdflib import BNode 10 | # TODO make python 3 safe! 11 | import urllib as u 12 | import requests 13 | 14 | from django.db.models import signals 15 | 16 | from django.shortcuts import get_object_or_404 17 | # deprecated since 1.3 18 | # from django.views.generic.list_detail import object_list 19 | # but not used anyway? 20 | # if needed.. from django.views.generic import ListView 21 | 22 | from django.http import HttpResponse,Http404 23 | 24 | from rdflib import Graph,namespace 25 | from rdflib.term import URIRef, Literal 26 | from rdflib.namespace import NamespaceManager,RDF 27 | 28 | import json 29 | 30 | import logging 31 | logger = logging.getLogger(__name__) 32 | 33 | def show_config(request) : 34 | return HttpResponse(json.dumps( ConfigVar.objects.all() )) 35 | 36 | def sync_remote(request,models): 37 | """ 38 | Synchronises the RDF published output for the models, in the order listed (list containers before members!) 39 | """ 40 | if request.GET.get('pdb') : 41 | import pdb; pdb.set_trace() 42 | 43 | for model in models.split(",") : 44 | try: 45 | (app,model) = model.split('.') 46 | ct = ContentType.objects.get(app_label=app,model=model) 47 | except: 48 | ct = ContentType.objects.get(model=model) 49 | if not ct : 50 | raise Http404("No such model found") 51 | 52 | try: 53 | # rdfstore = get_rdfstore(model,name=request.GET.get('rdfstore') ) 54 | do_sync_remote( model, ct , rdfstore=None ) 55 | except Exception as e: 56 | return HttpResponse("Sync to RDF for model %s threw %s" % (model,e) , status=410 ) 57 | 58 | return HttpResponse("sync successful for {}".format(models), status=200) 59 | 60 | def do_sync_remote(formodel, ct ,rdfstore): 61 | 62 | oml = ObjectMapping.objects.filter(content_type=ct) 63 | modelclass = ct.model_class() 64 | for obj in modelclass.objects.all() : 65 | publish( obj, formodel, oml, rdfstore) 66 | # gr.add((URIRef('skos:Concept'), RDF.type, URIRef('foaf:Person'))) 67 | # gr.add((URIRef('rdf:Concept'), RDF.type, URIRef('xxx:Person'))) 68 | 69 | def ctl_signals(request,cmd): 70 | """utility view to control and debug signals""" 71 | from rdf_io.signals import setup_signals,list_pubs,sync_signals 72 | if cmd == 'on': 73 | msg = auto_on() 74 | elif cmd == 'off' : 75 | msg = auto_off() 76 | elif cmd == 'list' : 77 | msg = list_pubs() 78 | elif cmd == 'sync' : 79 | msg = sync_signals() 80 | elif cmd == 'help' : 81 | msg = "usage /rdf_io/ctl_signals/(on|off|list|sync|help)" 82 | else: 83 | msg = "Command %s not understood. Use /rdf_io/ctl_signals/help for valid commands" % cmd 84 | return HttpResponse(msg, status=200) 85 | 86 | 87 | def auto_on(): 88 | """turn Auto push signals on""" 89 | from rdf_io.signals import setup_signals,list_pubs 90 | signals.post_save.connect(setup_signals, sender=ObjectMapping) 91 | return list_pubs() 92 | 93 | def auto_off(): 94 | """turn off all Auto push signals on""" 95 | from rdf_io.signals import clear_signals 96 | return clear_signals() 97 | 98 | 99 | -------------------------------------------------------------------------------- /AdvancedConfiguration.md: -------------------------------------------------------------------------------- 1 | # RDF-IO Advanced 2 | 3 | RDF-IO supports publishing RDF outputs to a triple-store, which can support SPARQL queries and inferencing. 4 | 5 | It also allows inferencing stores to be used in intermediate steps to publishing to non-inferencing stores. _(NB This may be replaced by native inferencing capabilities within the RDF-IO environment in future)_ 6 | 7 | ## RDF store configuration 8 | 9 | 0. Set up any target RDF stores (currently supported are the LDP API and RDF4J/Apache Sesame API) - note RDF_IO can be used to import and serialise django objects as RDF without any RDF stores, and different RDF stores can be used for different objects 10 | 1. Configure variables for use in setting up ServiceBindings to RDF stores and inferencing engines. 11 | 2. Set up ServiceBindings for your target stores (via the admin interface or migrations) 12 | 3. Pre-load any static data and rules required by your reasoner (e.g. SHACL) - or set up migrations to load these using ImportedResource bound to appropriate ServiceBindings 13 | 14 | ## Configure variables 15 | /admin/ConfigVars 16 | 17 | Publishing uses a set of configuration variables which may be scoped to different phases of publication: 18 | 19 | 1. **PUBLISH** - data is send to final destination 20 | 2. **REVIEW** - data is send to an alternative data store 21 | 3. **TEST** - not used yet 22 | 23 | These variables are resolved in URL templates in the ServiceBindings objects. 24 | If scope is not set, the value will be used for all cases. 25 | 26 | ## Publishing to a RDF store (via API) 27 | 28 | 1. Configure one or more ServiceBindings and attach to the relevant ObjectMapping (if updates to that Object are to be published to RDF store - otherwise ServiceBindings can be directly bound to individual ImportedResource objects) 29 | NOTE: A service binding of type VALIDATION will cause checks to be performed - and on failure will abort the service chain and invoke on_fail bindings (not yet implemented) 30 | NOTE: An service binding of type INFERENCING will augment the data to be stored, but not save it. It should be chained to PERSIST binding(s). 31 | 2. To publish a specific object to the configured RDF store: 32 | `{SERVER_URL}/rdf_io/pub_rdf/{model_name}/{model_id}` 33 | (note that this will happen automatically on object save if an object mapping is defined) 34 | 3. To republish all objects for a set of django models: 35 | `{SERVER_URL}/rdf_io/sync_remote/{model_name}[,{model_name}]*` 36 | 37 | 38 | NOTE: for the /rdf_io/to_rdf/{model_name}/key/{model_natural_key} to work the target model must define a manage with a get_by_natural_key method that takes a unique single term - such as a uri - note this will allow use of CURIES such as myns:myterm where the prefix is registered as a namespace in the RDF_IO space. If a CURIE is detected, then RDF_IO will try to match first as a string, then expand to a full URI and match. 39 | 40 | 41 | ### Inferencing 42 | Inferencing allows RDF based reasoning to generate richer views of inter-related data, and potentially derive a range of additional knowledge. This can all be done inside custom logic, but RDF_IO allows standards such as SHACL etc to be used to capture this and avoids hard-coding and hiding all these rules. 43 | 44 | when an object is saved, any enabled inferencing ServiceBindings will be applied before saving (stores may invoke loaded rules post-save) 45 | 46 | 1. Set up a RDF Inferencer - note this may be a matter of enabling inferencing on the default store, or setting up a new store. If rules cannot safely co-exist, then multiple inferencing stores may be configured. 47 | 2. Load inferencing rules to the Inferencer (optionally using ImportedResource objects and PERSIST_REPLACE service bindings) 48 | 3. Create service bindings for inferencing - these may be chained using Next_service 49 | 4. Make sure that the inferencing store is cleared of temporary data - using an appropriate PERSISTENCE_DELETE ServiceBinding as a final step in the chain 50 | 51 | NOTE: Inferencing rules may need to be more complex if a separate inferencing store is set up, but some of the data needed for inferencing resides in the main target repository. Using SPIN, this leads to constructs like: 52 | 53 | ``` 54 | WHERE { 55 | ?mainrepo a service:Repo . 56 | 57 | ?A a skos:Concept . 58 | OPTIONAL { ?A skos:broader ?B . } 59 | OPTIONAL { ?B skos:narrower ?A } 60 | OPTIONAL { SERVICE ?mainrepo 61 | { 62 | OPTIONAL { ?A skos:broader ?B . } 63 | OPTIONAL { ?B skos:narrower ?A . } 64 | 65 | } 66 | } 67 | 68 | ``` 69 | 70 | Where `?mainrepo` is an object loaded to the inferencer to define the target data store, allowing such a rule to be reusable. -------------------------------------------------------------------------------- /rdf_io/protocols/api.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | from django.conf import settings 3 | import os 4 | import rdflib 5 | 6 | import requests 7 | 8 | # from rdf_io.models import ServiceBinding 9 | from string import Formatter 10 | 11 | class RDFConfigNotFoundException(Exception): 12 | """ Cannot find a RDF publish configuration matching object """ 13 | pass 14 | 15 | class RDFConfigException(Exception): 16 | """ RDF store binding configuration exception""" 17 | pass 18 | 19 | class RDFStoreException(Exception): 20 | """ RDF store response exception """ 21 | pass 22 | 23 | def push_to_store(binding, model, obj, gr, mode='PUBLISH' ): 24 | from .rdf4j import rdf4j_push, rdf4j_get 25 | from .ldp import ldp_push 26 | """ push an object via its serialisation rules to a store via a ServiceBinding """ 27 | if not binding: 28 | try: 29 | from rdf_io.models import ServiceBinding 30 | binding = ServiceBinding.get_service_bindings(model,(ServiceBinding.PERSIST_CREATE,ServiceBinding.PERSIST_UPDATE,ServiceBinding.PERSIST_REPLACE)).first() 31 | except: 32 | raise RDFConfigNotFoundException("Cant locate appropriate repository configuration" ) 33 | rdfstore = { 'server_api' : binding.service_api , 'server' : binding.service_url , 'target' : binding.resource } 34 | 35 | resttgt = resolveTemplate("".join( ( rdfstore['server'],rdfstore['target'])), model, obj, mode ) 36 | print (resttgt) 37 | 38 | if binding.service_api == "RDF4JREST" : 39 | return rdf4j_push(rdfstore, model, obj, gr , binding.binding_type, mode) 40 | elif binding.service_api == "LDP" : 41 | return ldp_push(rdfstore, model, obj, gr ,binding_type , mode) 42 | else: 43 | raise RDFConfigException("Unknown server API %s" % binding.service_api ) 44 | 45 | push_to_store.RDFConfigException = RDFConfigException 46 | push_to_store.RDFConfigNotFoundException = RDFConfigNotFoundException 47 | push_to_store.RDFStoreException = RDFStoreException 48 | 49 | 50 | def resolveTemplate(template, model, obj,mode='PUBLISH') : 51 | from rdf_io.models import getattr_path, ConfigVar 52 | try: 53 | from urllib.parse import quote_plus 54 | except: 55 | from urllib import quote_plus 56 | vals = { 'model' : model } 57 | #import pdb; pdb.set_trace() 58 | for (literal,param,repval,conv) in Formatter().parse(template) : 59 | if param and param != 'model' : 60 | if( param[0] == '_' ) : 61 | val = ConfigVar.getval(param[1:],mode) 62 | if val: 63 | vals[param] = val 64 | else: 65 | #import pdb; pdb.set_trace() 66 | raise Exception( "template references unset ConfigVariable %s" % param[1:]) 67 | else: 68 | try: 69 | vals[param] = quote_plus( getattr_path(obj,param).pop()) 70 | except: 71 | if param == 'slug' : 72 | vals[param] = obj.id 73 | 74 | try: 75 | return template.format(**vals) 76 | except KeyError as e : 77 | raise KeyError( 'Property %s of model %s not found when creating API URL' % (e,model)) 78 | 79 | def inference(model, obj, inferencer, gr, mode ): 80 | """ Perform configured inferencing, return graph of new axioms 81 | 82 | use configured service binding to push an object to the inferencer, depending on its API type, perform inferencing using whatever 83 | rules have been set up, and return the new axioms, for disposition using the persistence rules (service bindings) for the resource """ 84 | # import pdb; pdb.set_trace() 85 | from .rdf4j import rdf4j_push,rdf4j_get 86 | from .ldp import ldp_push 87 | from rdf_io.models import ServiceBinding 88 | 89 | if inferencer.service_api == "RDF4JREST" : 90 | rdfstore = { 'server_api' : inferencer.service_api , 'server' : inferencer.service_url ,'target' : inferencer.resource } 91 | 92 | rdf4j_push(rdfstore, model, obj, gr, ServiceBinding.PERSIST_REPLACE, mode ) 93 | rdfstore['target'] = inferencer.inferenced_resource 94 | inference_response = rdf4j_get( rdfstore, model, obj , mode) 95 | graph = rdflib.Graph() 96 | newgr = graph.parse(data=inference_response.content, format='nt') 97 | # elif inferencer.service_api == "LDP" : 98 | # ldp_push(rdfstore, resttgt, model, obj, gr ) 99 | else: 100 | raise RDFConfigException("Unsupported server API %s" % inferencer.service_api ) 101 | 102 | # print "Performed inference with %s - now need to get results!" % (inferencer) 103 | 104 | return newgr 105 | 106 | def rdf_delete( binding, model, obj, mode ): 107 | """ Deletes content from remote RDF store 108 | 109 | Typically used for cleanup after inferencing and on post_delete signals for autompublished models.""" 110 | from rdf_io.models import ServiceBinding 111 | from .rdf4j import rdf4j_delete 112 | # print "Asked to delete from %s %s " % ( binding.service_url, binding.resource) 113 | rdfstore = { 'server_api' : binding.service_api , 'server' : binding.service_url , 'target' : binding.resource } 114 | 115 | if binding.service_api == "RDF4JREST" : 116 | rdf4j_delete( rdfstore, model, obj, mode ) 117 | else: 118 | raise RDFConfigException ("Delete not supported yet for %s " % binding.service_api ) 119 | return True -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | CC0 1.0 Universal 2 | 3 | Statement of Purpose 4 | 5 | The laws of most jurisdictions throughout the world automatically confer 6 | exclusive Copyright and Related Rights (defined below) upon the creator and 7 | subsequent owner(s) (each and all, an "owner") of an original work of 8 | authorship and/or a database (each, a "Work"). 9 | 10 | Certain owners wish to permanently relinquish those rights to a Work for the 11 | purpose of contributing to a commons of creative, cultural and scientific 12 | works ("Commons") that the public can reliably and without fear of later 13 | claims of infringement build upon, modify, incorporate in other works, reuse 14 | and redistribute as freely as possible in any form whatsoever and for any 15 | purposes, including without limitation commercial purposes. These owners may 16 | contribute to the Commons to promote the ideal of a free culture and the 17 | further production of creative, cultural and scientific works, or to gain 18 | reputation or greater distribution for their Work in part through the use and 19 | efforts of others. 20 | 21 | For these and/or other purposes and motivations, and without any expectation 22 | of additional consideration or compensation, the person associating CC0 with a 23 | Work (the "Affirmer"), to the extent that he or she is an owner of Copyright 24 | and Related Rights in the Work, voluntarily elects to apply CC0 to the Work 25 | and publicly distribute the Work under its terms, with knowledge of his or her 26 | Copyright and Related Rights in the Work and the meaning and intended legal 27 | effect of CC0 on those rights. 28 | 29 | 1. Copyright and Related Rights. A Work made available under CC0 may be 30 | protected by copyright and related or neighboring rights ("Copyright and 31 | Related Rights"). Copyright and Related Rights include, but are not limited 32 | to, the following: 33 | 34 | i. the right to reproduce, adapt, distribute, perform, display, communicate, 35 | and translate a Work; 36 | 37 | ii. moral rights retained by the original author(s) and/or performer(s); 38 | 39 | iii. publicity and privacy rights pertaining to a person's image or likeness 40 | depicted in a Work; 41 | 42 | iv. rights protecting against unfair competition in regards to a Work, 43 | subject to the limitations in paragraph 4(a), below; 44 | 45 | v. rights protecting the extraction, dissemination, use and reuse of data in 46 | a Work; 47 | 48 | vi. database rights (such as those arising under Directive 96/9/EC of the 49 | European Parliament and of the Council of 11 March 1996 on the legal 50 | protection of databases, and under any national implementation thereof, 51 | including any amended or successor version of such directive); and 52 | 53 | vii. other similar, equivalent or corresponding rights throughout the world 54 | based on applicable law or treaty, and any national implementations thereof. 55 | 56 | 2. Waiver. To the greatest extent permitted by, but not in contravention of, 57 | applicable law, Affirmer hereby overtly, fully, permanently, irrevocably and 58 | unconditionally waives, abandons, and surrenders all of Affirmer's Copyright 59 | and Related Rights and associated claims and causes of action, whether now 60 | known or unknown (including existing as well as future claims and causes of 61 | action), in the Work (i) in all territories worldwide, (ii) for the maximum 62 | duration provided by applicable law or treaty (including future time 63 | extensions), (iii) in any current or future medium and for any number of 64 | copies, and (iv) for any purpose whatsoever, including without limitation 65 | commercial, advertising or promotional purposes (the "Waiver"). Affirmer makes 66 | the Waiver for the benefit of each member of the public at large and to the 67 | detriment of Affirmer's heirs and successors, fully intending that such Waiver 68 | shall not be subject to revocation, rescission, cancellation, termination, or 69 | any other legal or equitable action to disrupt the quiet enjoyment of the Work 70 | by the public as contemplated by Affirmer's express Statement of Purpose. 71 | 72 | 3. Public License Fallback. Should any part of the Waiver for any reason be 73 | judged legally invalid or ineffective under applicable law, then the Waiver 74 | shall be preserved to the maximum extent permitted taking into account 75 | Affirmer's express Statement of Purpose. In addition, to the extent the Waiver 76 | is so judged Affirmer hereby grants to each affected person a royalty-free, 77 | non transferable, non sublicensable, non exclusive, irrevocable and 78 | unconditional license to exercise Affirmer's Copyright and Related Rights in 79 | the Work (i) in all territories worldwide, (ii) for the maximum duration 80 | provided by applicable law or treaty (including future time extensions), (iii) 81 | in any current or future medium and for any number of copies, and (iv) for any 82 | purpose whatsoever, including without limitation commercial, advertising or 83 | promotional purposes (the "License"). The License shall be deemed effective as 84 | of the date CC0 was applied by Affirmer to the Work. Should any part of the 85 | License for any reason be judged legally invalid or ineffective under 86 | applicable law, such partial invalidity or ineffectiveness shall not 87 | invalidate the remainder of the License, and in such case Affirmer hereby 88 | affirms that he or she will not (i) exercise any of his or her remaining 89 | Copyright and Related Rights in the Work or (ii) assert any associated claims 90 | and causes of action with respect to the Work, in either case contrary to 91 | Affirmer's express Statement of Purpose. 92 | 93 | 4. Limitations and Disclaimers. 94 | 95 | a. No trademark or patent rights held by Affirmer are waived, abandoned, 96 | surrendered, licensed or otherwise affected by this document. 97 | 98 | b. Affirmer offers the Work as-is and makes no representations or warranties 99 | of any kind concerning the Work, express, implied, statutory or otherwise, 100 | including without limitation warranties of title, merchantability, fitness 101 | for a particular purpose, non infringement, or the absence of latent or 102 | other defects, accuracy, or the present or absence of errors, whether or not 103 | discoverable, all to the greatest extent permissible under applicable law. 104 | 105 | c. Affirmer disclaims responsibility for clearing rights of other persons 106 | that may apply to the Work or any use thereof, including without limitation 107 | any person's Copyright and Related Rights in the Work. Further, Affirmer 108 | disclaims responsibility for obtaining any necessary consents, permissions 109 | or other rights required for any use of the Work. 110 | 111 | d. Affirmer understands and acknowledges that Creative Commons is not a 112 | party to this document and has no duty or obligation with respect to this 113 | CC0 or use of the Work. 114 | 115 | For more information, please see 116 | 117 | -------------------------------------------------------------------------------- /rdf_io/views/serialize.py: -------------------------------------------------------------------------------- 1 | # # -*- coding:utf-8 -*- 2 | #from django.shortcuts import render, redirect 3 | from rdf_io.models import * 4 | from ..protocols import push_to_store,inference,rdf_delete 5 | 6 | from django.template import RequestContext 7 | from django.contrib.contenttypes.models import ContentType 8 | from django.conf import settings 9 | 10 | from rdflib import BNode 11 | # TODO make python 3 safe! 12 | import urllib as u 13 | import requests 14 | 15 | from django.db.models import signals 16 | 17 | from django.shortcuts import get_object_or_404 18 | # deprecated since 1.3 19 | # from django.views.generic.list_detail import object_list 20 | # but not used anyway? 21 | # if needed.. from django.views.generic import ListView 22 | 23 | from django.http import HttpResponse,Http404 24 | 25 | from rdflib import Graph,namespace 26 | from rdflib.term import URIRef, Literal 27 | from rdflib.namespace import NamespaceManager,RDF 28 | 29 | import sys 30 | 31 | import logging 32 | logger = logging.getLogger(__name__) 33 | 34 | _nslist = {} 35 | 36 | 37 | def to_rdfbykey(request,model,key): 38 | """ 39 | take a model name + object id reference to an instance and apply any RDF serialisers defined for this 40 | allows a key to de appended to the uri or supplied by parameter (easier for uri values) 41 | """ 42 | if request.GET.get('key'): 43 | key = request.GET.get('key') 44 | try: 45 | return _tordf(request,model,None,key) 46 | except Exception as e: 47 | return HttpResponse("Model not serialisable to RDF: %s" % e, status=500) 48 | 49 | def to_rdfbyid(request,model,id): 50 | """ 51 | take a model name + object id reference to an instance and apply any RDF serialisers defined for this 52 | """ 53 | try: 54 | return _tordf(request,model,id,None) 55 | except Exception as e: 56 | return HttpResponse("Model not serialisable to RDF: %s" % e, status=500) 57 | 58 | def _tordf(request,model,id,key): 59 | if request.GET.get('pdb') : 60 | import pdb; pdb.set_trace() 61 | format = request.GET.get('_format') 62 | if format not in ('turtle','json-ld'): 63 | if format == 'json': 64 | format = "json-ld" 65 | else: 66 | format = "turtle" 67 | 68 | # find the model type referenced 69 | try: 70 | (app,model) = model.split('.') 71 | ct = ContentType.objects.get(app_label=app,model=model) 72 | except: 73 | ct = ContentType.objects.get(model=model) 74 | if not ct : 75 | raise Http404("No such model found") 76 | oml = ObjectMapping.objects.filter(content_type=ct) 77 | if not oml : 78 | return HttpResponse("Model not serialisable to RDF", status=410 ) 79 | if id : 80 | obj = get_object_or_404(ct.model_class(), pk=id) 81 | else : 82 | try: 83 | obj = ct.model_class().objects.get_by_natural_key(key) 84 | except Exception as e: 85 | try: 86 | (prefix,term) = key.split(':') 87 | ns = Namespace.objects.get(prefix=prefix) 88 | urikey = "".join((ns.uri,term)) 89 | obj = ct.model_class().objects.get_by_natural_key(urikey) 90 | except Exception as e2: 91 | raise e 92 | 93 | # ok so object exists and is mappable, better get down to it.. 94 | 95 | includemembers = True 96 | if request.GET.get('skip') : 97 | includemembers = request.GET.get('skip') != 'True' 98 | 99 | gr = Graph() 100 | # import pdb; pdb.set_trace() 101 | # ns_mgr = NamespaceManager(Graph()) 102 | # gr.namespace_manager = ns_mgr 103 | try: 104 | gr = build_rdf(gr, obj, oml, includemembers) 105 | except Exception as e: 106 | raise Http404("Error during serialisation: " + str(e) ) 107 | return HttpResponse(content_type="text/turtle; charset=utf-8", content=gr.serialize(format=format)) 108 | 109 | def pub_rdf(request,model,id): 110 | """ 111 | take a model name + object id reference to an instance serialise and push to the configured triplestore 112 | """ 113 | if request.GET.get('pdb') : 114 | import pdb; pdb.set_trace() 115 | # find the model type referenced 116 | try: 117 | (app,model) = model.split('.') 118 | ct = ContentType.objects.get(app_label=app,model=model) 119 | except: 120 | ct = ContentType.objects.get(model=model) 121 | if not ct : 122 | raise Http404("No such model found") 123 | oml = ObjectMapping.objects.filter(content_type=ct) 124 | if not oml : 125 | raise HttpResponse("Model not serialisable to RDF", status=410 ) 126 | 127 | obj = get_object_or_404(ct.model_class(), pk=id) 128 | # ok so object exists and is mappable, better get down to it.. 129 | 130 | try: 131 | rdfstore = get_rdfstore(model,name=request.GET.get('rdfstore') ) 132 | except: 133 | return HttpResponse("RDF store not configured", status=410 ) 134 | 135 | try: 136 | result = publish(obj, model, oml,rdfstore) 137 | except Exception as e: 138 | return HttpResponse("Exception publishing remote RDF content %s" % e,status=500 ) 139 | return HttpResponse("Server reports %s" % result.content,status=result.status_code ) 140 | 141 | def get_rdfstore(model, name=None ): 142 | # now get the remote store mappings 143 | # deprecated - using ConfigVar and ServiceBindings now.. 144 | # print "Warning - deprecated method invoked - use ServiceBindings instead of static config now" 145 | return None 146 | # if name : 147 | # rdfstore_cfg = settings.RDFSTORES[name] 148 | # else: 149 | # rdfstore_cfg = settings.RDFSTORE 150 | # rdfstore = rdfstore_cfg['default'] 151 | # auth = rdfstore.get('auth') 152 | # server = rdfstore['server'] 153 | # server_api = rdfstore['server_api'] 154 | 155 | # try: 156 | # rdfstore = rdfstore_cfg[model] 157 | # if not rdfstore.has_key('server') : 158 | # rdfstore['server'] = server 159 | # rdfstore['auth'] = auth 160 | # if not rdfstore.has_key('server_api') : 161 | # rdfstore['server_api'] = server_api 162 | # except: 163 | # pass # use default then 164 | 165 | # return rdfstore 166 | 167 | def publish_set(queryset, model,check=False,mode='PUBLISH'): 168 | """ publish select set of objects of type "model" 169 | 170 | Because this may be long running it is a status message generator 171 | """ 172 | oml = ObjectMapping.objects.filter(content_type__model=model) 173 | for obj in queryset : 174 | if check: 175 | try: 176 | yield ("checking %s " % (obj.uri,)) 177 | resp = u.urlopen(obj.uri) 178 | if resp.getcode() == 200 : 179 | continue 180 | except Exception as e : 181 | yield("Exception: %s" % (str(e), ) ) 182 | yield ("publishing %s " % (str(obj),) ) 183 | try: 184 | publish( obj, model, oml,mode=mode) 185 | yield ("... Success") 186 | except Exception as e : 187 | yield("Exception %s" % (str(e), ) ) 188 | 189 | 190 | -------------------------------------------------------------------------------- /rdf_io/tests/test_mappings.py: -------------------------------------------------------------------------------- 1 | from rdf_io.models import * 2 | from django.contrib.contenttypes.models import ContentType 3 | from django.test import TestCase 4 | from rdflib import Literal, URIRef, Graph, XSD 5 | 6 | class SerialisationSetupTestCase(TestCase): 7 | testmapping = None 8 | testobj = None 9 | attachable = None 10 | 11 | def setUp(self): 12 | (object_type,created) = ObjectType.objects.get_or_create(uri="http://metalinkage.com.au/rdfio/ObjectMapping", defaults = { "label" : "Test RDF target type" }) 13 | content_type = ContentType.objects.get(app_label="rdf_io",model="objectmapping") 14 | defaults = { "auto_push" : False , 15 | "id_attr" : "id", 16 | "target_uri_expr" : '"http://metalinkage.com.au/test/id/"', 17 | "content_type" : content_type 18 | } 19 | 20 | (pm,created) = ObjectMapping.objects.get_or_create(name="Mappings test", defaults =defaults) 21 | pm.obj_type.add(object_type) 22 | 23 | # run mapping over myself - i'm a suitable complex object :-) 24 | self.testmapping = pm 25 | self.testobj = pm 26 | # create test with absolute URI with <> 27 | am = AttributeMapping(scope=pm, attr="id_attr", predicate="", is_resource=False).save() 28 | # create test with URI with http string 29 | am = AttributeMapping(scope=pm, attr="id_attr", predicate="http://metalinkage.com.au/rdfio/id_attr", is_resource=False).save() 30 | # create test with CURIE 31 | ns = Namespace.objects.get_or_create(uri='http://www.w3.org/2000/01/rdf-schema#', prefix='rdfs') 32 | am = AttributeMapping(scope=pm, attr="name", predicate="rdfs:label", is_resource=False).save() 33 | 34 | # test with language tag 35 | am = AttributeMapping(scope=pm, attr="name@en", predicate="rdfs:comment", is_resource=False).save() 36 | 37 | # generic metadata properties 38 | ns,created = Namespace.objects.get_or_create(uri='http://example.org/', prefix='eg') 39 | gmp,created = GenericMetaProp.objects.get_or_create(namespace=ns, propname="metaprop") 40 | attachable,created = AttachedMetadata.objects.get_or_create(metaprop=gmp,value="something to change during testing") 41 | 42 | self.attachable = attachable 43 | 44 | class MetaObjectsTestCase(SerialisationSetupTestCase): 45 | """ Tests basic object behaviours needed for RDF content handling """ 46 | 47 | def test_make_metaprop_with_uri_from_ns(self): 48 | """ tests that if a uri is provided for which a namespace is registered then namespace gets set""" 49 | gmp,created = GenericMetaProp.objects.get_or_create(uri="http://example.org/frog") 50 | self.assertEqual(gmp.namespace.prefix,"eg") 51 | self.assertEqual(gmp.propname,"frog") 52 | 53 | def test_get_metaprop_with_curie(self): 54 | gmp = GenericMetaProp.objects.get_by_natural_key("eg:metaprop") 55 | self.assertEqual(gmp.namespace.prefix,"eg") 56 | self.assertEqual(gmp.propname,"metaprop") 57 | 58 | def test_get_metaprop_with_url(self): 59 | gmp = GenericMetaProp.objects.get_by_natural_key("http://example.org/metaprop") 60 | self.assertEqual(gmp.namespace.prefix,"eg") 61 | self.assertEqual(gmp.propname,"metaprop") 62 | 63 | def test_node_string(self): 64 | self.assertEqual(makenode(Graph(),"frog"),Literal("frog")) 65 | 66 | def test_node_string_quoted(self): 67 | self.assertEqual(makenode(Graph(),'"frog"'),Literal("frog")) 68 | 69 | def test_node_string_int(self): 70 | #import pdb; pdb.set_trace() 71 | self.assertEqual(makenode(Graph(),"12"),Literal("12",datatype=XSD.integer)) 72 | 73 | def test_node_string_lang(self): 74 | self.assertEqual(makenode(Graph(),"frog@en"),Literal("frog",lang="en")) 75 | 76 | def test_node_string_lang_quoted(self): 77 | self.assertEqual(makenode(Graph(),'"frog"@en'),Literal("frog",lang="en")) 78 | 79 | def test_node_string_datatype_curie(self): 80 | self.assertEqual(makenode(Graph(),"12^^xsd:int"),Literal("12",datatype=XSD.integer)) 81 | 82 | def test_node_string_datatype_uri(self): 83 | self.assertEqual(makenode(Graph(),"frog^^"),Literal("frog",datatype=URIRef('http://eg.org')) ) 84 | 85 | 86 | def test_quote_string(self): 87 | self.assertEqual(quote('frog'), '"frog"') 88 | 89 | def test_quote_string_prequoted(self): 90 | self.assertEqual(quote('"frog"'), '"frog"') 91 | 92 | def test_quote_string_lang(self): 93 | self.assertEqual(quote('frog@en'), '"frog"@en') 94 | 95 | def test_quote_string_lang_prequoted(self): 96 | self.assertEqual(quote('"frog"@en'), '"frog"@en') 97 | 98 | def test_quote_string_datatype(self): 99 | self.assertEqual(quote('frog^^'), '"frog"^^') 100 | 101 | 102 | class ObjectMappingTestCase(SerialisationSetupTestCase): 103 | """ Test case for object serialisation to rdf 104 | 105 | Creates an object mapping for an ObjectMapping object - which is a suitably complex object with nested M2M and avoid us having 106 | to have dependencies on other fixtures - if all a bit recursive :-) """ 107 | 108 | 109 | 110 | def test_getattr_path_direct_char(self): 111 | vals = getattr_path(self.testobj,"id_attr") 112 | self.assertEqual(vals[0],"id") 113 | 114 | def test_getattr_path_direct_int(self): 115 | vals = getattr_path(self.testobj,"id") 116 | self.assertEqual(vals[0],self.testobj.id) 117 | 118 | def test_getattr_path_nested_FK(self): 119 | vals = getattr_path(self.testobj,"content_type.model") 120 | self.assertEqual(vals[0],"objectmapping") 121 | 122 | def test_getattr_path_nested_M2M(self): 123 | vals = getattr_path(self.testobj,"obj_type.uri") 124 | self.assertEqual(list(vals)[0],"http://metalinkage.com.au/rdfio/ObjectMapping") 125 | 126 | def test_getattr_path_nested_M2M_filter_on_string(self): 127 | (extraobject_type,created) = ObjectType.objects.get_or_create(uri="http://metalinkage.com.au/rdfio/ObjectMapping2", defaults = { "label" : "Test RDF target type - extra" }) 128 | self.testobj.obj_type.add(extraobject_type) 129 | vals = list(getattr_path(self.testobj,"obj_type.uri")) 130 | self.assertEqual(len(vals),2) 131 | vals = list(getattr_path(self.testobj,"obj_type[uri='http://metalinkage.com.au/rdfio/ObjectMapping'].uri")) 132 | self.assertEqual(len(vals),1) 133 | vals = list(getattr_path(self.testobj,"obj_type[uri='http://metalinkage.com.au/rdfio/ObjectMapping2'].uri")) 134 | self.assertEqual(len(vals),1) 135 | vals = list(getattr_path(self.testobj,"obj_type[uri='http://metalinkage.com.au/rdfio/ObjectMapping3'].uri")) 136 | self.assertEqual(len(vals),0) 137 | vals = list(getattr_path(self.testobj,"obj_type[uri!='http://metalinkage.com.au/rdfio/ObjectMapping3'].uri")) 138 | self.assertEqual(len(vals),2) 139 | 140 | def test_getattr_path_nested_M2M_filter_int(self): 141 | (extraobject_type,created) = ObjectType.objects.get_or_create(uri="http://metalinkage.com.au/rdfio/ObjectMapping2", defaults = { "label" : "Test RDF target type - extra" }) 142 | self.testobj.obj_type.add(extraobject_type) 143 | vals = list(getattr_path(self.testobj,"obj_type[id=%s].id" % extraobject_type.id)) 144 | self.assertEqual(list(vals)[0],extraobject_type.id) 145 | # 146 | vals = list(getattr_path(self.testobj,"obj_type[id!=%s].id" % extraobject_type.id)) 147 | self.assertNotEqual(list(vals)[0],extraobject_type.id) 148 | 149 | def test_get_relobjs(self): 150 | """ test case where related objects refer to this object (reverse FK) """ 151 | #import pdb; pdb.set_trace() 152 | relobjs = list(getattr_path(self.testmapping,'attributemapping')) 153 | self.assertEqual(len(relobjs),4) 154 | 155 | -------------------------------------------------------------------------------- /rdf_io/admin.py: -------------------------------------------------------------------------------- 1 | from .models import * 2 | from django.contrib import admin 3 | from django.db.models import Q 4 | from django.contrib.contenttypes.models import ContentType 5 | from .views import * 6 | from django import forms 7 | #from django.contrib.admin.widgets import SelectWidget 8 | from django.utils.safestring import mark_safe 9 | from django.contrib import messages 10 | from django.shortcuts import render 11 | from django.http import HttpResponseRedirect, HttpResponse 12 | 13 | class GenericMetaPropInline(admin.TabularInline): 14 | model = GenericMetaProp 15 | # readonly_fields = ('slug','created') 16 | #fields = ('code','namespace') 17 | # related_search_fields = {'label' : ('name','slug')} 18 | extra=1 19 | 20 | 21 | def publish_set_background(queryset,model,check,mode,logf): 22 | from django.core.files import File 23 | import time 24 | 25 | with open(logf,'w') as f: 26 | proclog = File(f) 27 | f.write("Publishing %s %ss in mode %s at %s
" % ( str(len(queryset)), model, mode, time.asctime())) 28 | for msg in publish_set(queryset,model,check,mode): 29 | if( msg.startswith("Exception") ): 30 | em = "" 31 | emend = "" 32 | else: 33 | em = "" 34 | emend = "" 35 | f.write("".join(("
  • ",em,msg,emend,"
    • "))) 36 | f.flush() 37 | f.write ("
    publish action finished at %s
    " % ( time.asctime(),)) 38 | 39 | 40 | def publish_set_action(queryset,model,check=False,mode='PUBLISH'): 41 | import threading 42 | from django.conf import settings 43 | import os 44 | import time 45 | timestr = time.strftime("%Y%m%d-%H%M%S") 46 | logfname = '{}_batch_publish_{}.html'.format(model,timestr) 47 | try: 48 | logf = os.path.join(settings.BATCH_RDFPUB_LOG, logfname) 49 | except: 50 | logf = os.path.join(settings.STATIC_ROOT if settings.STATIC_ROOT else '' ,logfname) 51 | t = threading.Thread(target=publish_set_background, args=(queryset,model,check,mode,logf), kwargs={}) 52 | t.setDaemon(True) 53 | t.start() 54 | return "/static/" + logfname 55 | 56 | 57 | 58 | def force_prefix_use(modeladmin, request, queryset): 59 | """ update selected Metaprops to use CURIE form with registered prefix """ 60 | for obj in queryset.all() : 61 | obj.save() 62 | force_prefix_use.short_description = "update selected Metaprops to use CURIE form with registered prefix" 63 | 64 | class GenericMetaPropAdmin(admin.ModelAdmin): 65 | search_fields = ['propname' ] 66 | actions= [force_prefix_use] 67 | pass 68 | 69 | class ObjectTypeAdmin(admin.ModelAdmin): 70 | pass 71 | 72 | class AttributeMappingInline(admin.TabularInline): 73 | model = AttributeMapping 74 | # readonly_fields = ('slug','created') 75 | #fields = ('code','namespace') 76 | # related_search_fields = {'label' : ('name','slug')} 77 | extra=1 78 | 79 | class EmbeddedMappingInline(admin.TabularInline): 80 | model = EmbeddedMapping 81 | # readonly_fields = ('slug','created') 82 | #fields = ('code','namespace') 83 | # related_search_fields = {'label' : ('name','slug')} 84 | extra=1 85 | 86 | class ChainedMappingInline(admin.TabularInline): 87 | model = ChainedMapping 88 | fk_name='scope' 89 | # readonly_fields = ('slug','created') 90 | fields = ('attr','predicate','chainedMapping') 91 | # related_search_fields = {'label' : ('name','slug')} 92 | extra=1 93 | 94 | class ObjectMappingAdmin(admin.ModelAdmin): 95 | search_fields = ['content_type__name' ] 96 | inlines = [ AttributeMappingInline, ChainedMappingInline, EmbeddedMappingInline] 97 | filter_horizontal = ('obj_type',) 98 | pass 99 | 100 | class AttributeMappingAdmin(admin.ModelAdmin): 101 | pass 102 | 103 | 104 | class EmbeddedMappingAdmin(admin.ModelAdmin): 105 | pass 106 | 107 | 108 | class NamespaceAdmin(admin.ModelAdmin): 109 | list_display = ('uri','prefix','notes') 110 | fields = ('uri','prefix','notes') 111 | search_fields = ['uri','prefix' ] 112 | # related_search_fields = {'concept' : ('pref_label','definition')} 113 | #list_editable = ('name','slug') 114 | search_fields = ['uri','prefix'] 115 | 116 | 117 | class ConfigVarAdmin(admin.ModelAdmin): 118 | pass 119 | 120 | class ResourceMetaInline(admin.TabularInline): 121 | model = ResourceMeta 122 | verbose_name = 'Additional property' 123 | verbose_name_plural = 'Additional properties' 124 | # list_fields = ('pref_label', ) 125 | show_change_link = True 126 | max_num = 20 127 | fields = ('subject','metaprop','value') 128 | # list_display = ('pref_label',) 129 | extra = 1 130 | 131 | IR = ContentType.objects.get_for_model(ImportedResource) 132 | 133 | class ImportedResourceAdmin(admin.ModelAdmin): 134 | list_display = ('description', 'subtype', '__unicode__') 135 | search_fields = ['description','file','remote'] 136 | inlines = [ ResourceMetaInline , ] 137 | actions= ['publish_options', ] 138 | resourcetype = 'importedresource' 139 | def get_queryset(self, request): 140 | qs = super(ImportedResourceAdmin, self).get_queryset(request) 141 | # import pdb; pdb.set_trace() 142 | return qs.filter(Q(subtype__isnull=True) | Q(subtype=IR )) 143 | 144 | def publish_options(self,request,queryset): 145 | """Batch publish with a set of mode options""" 146 | if 'apply' in request.POST: 147 | # The user clicked submit on the intermediate form. 148 | # Perform our update action: 149 | if request.POST.get('mode') == "CANCEL" : 150 | self.message_user(request, 151 | "Cancelled publish action") 152 | else: 153 | checkuri = 'checkuri' in request.POST 154 | logfile= publish_set_action(queryset,self.resourcetype,check=checkuri,mode=request.POST.get('mode')) 155 | self.message_user(request, 156 | mark_safe('started publishing in {} mode for {} {}s at {}'.format(request.POST.get('mode'),queryset.count(),self.resourcetype, logfile,logfile) ) ) 157 | return HttpResponseRedirect(request.get_full_path()) 158 | return render(request, 159 | 'admin/admin_publish.html', 160 | context={'schemes':queryset, 161 | 'pubvars': ConfigVar.getvars('PUBLISH') , 162 | 'reviewvars': ConfigVar.getvars('REVIEW') , 163 | }) 164 | 165 | 166 | class ObjectBoundListFilter(admin.SimpleListFilter): 167 | title='Chain Start by Object Type' 168 | parameter_name = 'objtype' 169 | 170 | def lookups(self, request, model_admin): 171 | chains = ServiceBinding.objects.filter(object_mapping__content_type__isnull=False) 172 | return set([(c.object_mapping.first().content_type.model, c.object_mapping.first().content_type.model) for c in chains]) 173 | 174 | def queryset(self, request, qs): 175 | try: 176 | #import pdb; pdb.set_trace() 177 | if request.GET.get('objtype') : 178 | qs= qs.filter(object_mapping__content_type__model = request.GET.get('objtype')) 179 | except: 180 | pass 181 | return qs 182 | 183 | class ChainListFilter(admin.SimpleListFilter): 184 | title='Chain members' 185 | parameter_name = 'chain_id' 186 | 187 | def lookups(self, request, model_admin): 188 | chains = ServiceBinding.objects.filter(object_mapping__name__isnull=False) 189 | return [(c.id, c.object_mapping.first().name) for c in chains] 190 | 191 | def queryset(self, request, qs): 192 | try: 193 | pass 194 | #qs= qs.filter(object_mapping__id = request.GET.get('chain_id')) 195 | except: 196 | pass 197 | return qs 198 | 199 | class NextChainWidget( forms.Select): 200 | def render(self, name, value, attrs=None): 201 | self.choices = self.form_instance.fields['next_service'].choices 202 | s = super(forms.Select, self).render(name, value, attrs) 203 | h="
    " 204 | ind= "-> {}
    " 205 | 206 | for next in self.form_instance.instance.next_chain(): 207 | h = h+ ind.format( str(next)) 208 | ind = "--" + ind 209 | 210 | 211 | return mark_safe(s+ h ) 212 | 213 | class ServiceBindingAdminForm (forms.ModelForm): 214 | def __init__(self, *args, **kwargs): 215 | super(ServiceBindingAdminForm, self).__init__(*args, **kwargs) 216 | self.fields['next_service'].widget = NextChainWidget() 217 | self.fields['next_service'].widget.form_instance = self 218 | 219 | class ServiceBindingAdmin(admin.ModelAdmin) : 220 | list_display = ('title', 'binding_type', 'object_mapping_list') 221 | list_filter=(ObjectBoundListFilter,ChainListFilter,'binding_type') 222 | search_fields = ['title','binding_type'] 223 | form = ServiceBindingAdminForm 224 | pass 225 | 226 | admin.site.register(Namespace, NamespaceAdmin) 227 | admin.site.register(GenericMetaProp,GenericMetaPropAdmin) 228 | admin.site.register(ObjectType, ObjectTypeAdmin) 229 | admin.site.register(ObjectMapping, ObjectMappingAdmin) 230 | #admin.site.register(AttributeMapping, AttributeMappingAdmin) 231 | #admin.site.register(EmbeddedMapping, EmbeddedMappingAdmin) 232 | admin.site.register(ImportedResource, ImportedResourceAdmin) 233 | 234 | admin.site.register(ServiceBinding, ServiceBindingAdmin) 235 | admin.site.register(ConfigVar, ConfigVarAdmin) -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # rdf-io 2 | 3 | Utilities to link Django to RDF stores and inferencers. 4 | 5 | Why: Allows semantic data models and rules to be used to generate rich views of content, and expose standardised access and query interfaces - such as SPARQL and the Linked Data Platform. Conversely, allow use of Django to manage content in RDF stores :-) 6 | 7 | ## compatibility 8 | Tested with django 1.11 + python 2.7 and django 3.0 with python 3.8 9 | 10 | ## Features 11 | * RDF serializer for Django objects driven by editable mapping rules 12 | * Configurable metadata properties for arbitrary Django objects. 13 | RDF_IO has initial data that loads up common W3C namespaces and prefixes ready for use (OWL, RDF, DCAT) 14 | * Extensible to support specific information models - such as SKOS 15 | 16 | * Publishing to a persistent external triple-store 17 | * RDF4J and LDP API support, extensible by plugin 18 | * Configureable ServiceBindings to RDF store APIs for different CRUD and inferencing tasks 19 | * RDF source load and push to designated RDF store 20 | * chainable inferencing support and persistence handling 21 | * configuration variables scoped to publishing modes to support publishing to different stores for review and final publication 22 | 23 | 24 | ## installation 25 | 26 | get a working copy with 27 | ``` 28 | git clone https://github.com/rob-metalinkage/django-rdf-io 29 | pip install -e (where you put it) 30 | ``` 31 | in your master django project: 32 | * add 'rdf_io' to the INSTALLED_APPS in settings.py 33 | * add ` url(r"^rdf_io/", include('rdf_io.urls'))` to urls.py 34 | * optionally define setting for RDFSERVER and RDFSERVER_API 35 | * run manage.py makemigrations 36 | * run manage.py migrate 37 | 38 | ## Automated publishing of updated to RDF 39 | This is really only guaranteed for pushing additions and updates - deletions are not handled, although updates will tend to replace statements. 40 | 41 | ### on startup to enable (necessary after django-reload) 42 | NB - TODO a way to force this to happen automatically - needs to happen after both RDF_IO and the target models are installed, so cant go in initialisation for either model. 43 | 44 | `{SERVER_URL}/rdf_io/ctl_signals/sync` 45 | 46 | ### to turn on publishing for a model class 47 | 1) check Auto-push flag is checked in an ObjectMapping for that model class 48 | 2) save - should register a post_save signal for the model class 49 | 50 | ### to turn on/off publishing for all model classes 51 | `{SERVER_URL}/rdf_io/ctl_signals/(on/off)` 52 | 53 | ### 54 | 55 | 56 | ## Usage 57 | 58 | 59 | ### Overview 60 | 1) Define mappings for your target models using the admin interface $SERVER/admin/rdf_io (see below) 61 | 2) To create an online resource use 62 | `{SERVER_URL}/rdf_io/to_rdf/{model_name}/id/{model_id}` 63 | `{SERVER_URL}/rdf_io/to_rdf/{model_name}/key/{model_natural_key}` 64 | 65 | 66 | ### Object Mappings 67 | Mappings to RDF are done for Django models. Each mapping consists of: 68 | 1) an identifier mapping to generate the URI for the object 69 | 2) a set of AttributeMapping elements that map a list of values to a RDF predicate 70 | 3) a set of EmbeddedMapping that map a list of values to complex object property (optionally wrapped in a blank node) 71 | 4) a filter to limit the set of objects the mapping applies to 72 | 73 | More than one object mapping may exist for a Django model. The RDF graph is the union of all the configured Object Mapping outputs. 74 | (Note that a ServiceBinding may be bound to a specific mapping, but the default behaviour is for this to be used to find all ServiceBindings for a gioven django modeltype - and they all get the composite graph (this may be changed to supported publishing different graphs to different RDf stores in future.) 75 | 76 | ### Mapping syntax 77 | Mapping is non trivial - because the elements of your model may need to extracted from related models 78 | 79 | Mapping is from elements in a Django model to a RDF value (a URI or a literal) 80 | 81 | source model elements may be defined using XPath-like syntax, with nesting using django filter style __, a__b .(dot) or / notation, where each element of the path may support an optional filter. 82 | ``` 83 | path = (literal|element([./]element)*) 84 | 85 | literal = "a quoted string" | 'a quoted string' | 86 | 87 | element = (property|related_model_expr)([filter])? 88 | 89 | property = a valid name of a property of a django model 90 | 91 | related_model_expr = model_name(\({property}\))? 92 | 93 | 94 | filter = (field(!)?=literal)((,| AND )field(!)?=literal)* | literal((,| OR )literal)* 95 | ``` 96 | 97 | Notes: 98 | * filters on related models will be evaluated within the database using django filters, filters on property values will be performed during serialisation. 99 | 100 | * literal values of None or NULL are treated as None or empty strings, as per Django practice. 101 | 102 | * filters on properties are a simple list of possible matches (, is the same as " OR " ) and apply to the element of the path 103 | person.title['MRS','MISS','MS'] would match any title with value "MRS", "MISS", "MS" 104 | 105 | * filters on related objects are property=value syntax. Properties use django-stlye paths - i.e. notation.namespace.prefix=skos 106 | 107 | * if a ManyToMany field is used through an intermediary, then use the related_model_expr - and if this is a self-relation then specify the property : eg. 108 | semrelation(origin_concept)[rel_type='1'].target_concept 109 | 110 | ## Status: 111 | beta, functionally complete initial capability: 112 | * TTL serialisation of a given model (for which a mapping has been registered) 113 | * Publishing to remote LDP service using per-model templates to define LDP resources 114 | * Autoconfiguring of signals so that objects with mappings are published on post_ save 115 | * syc_remote method to push all objects of list of model types (push is idempotent - safe to repeat) 116 | * sophisticated property-chains with per-level filter options in attribute marmotta 117 | * tested in context of geonode project under django 1.6 118 | * tested against django 1.8.6 and 1.11 119 | 120 | todo: 121 | * implement global filters for object mappings (to limit mappings to a subset) 122 | * set up signals and methods to delete objects from remote store 123 | 124 | ## API 125 | 126 | ### Serialising within python 127 | ``` 128 | from django.contrib.contenttypes.models import ContentType 129 | 130 | from rdflib import Graph 131 | 132 | from rdf_io.views import build_rdf 133 | from rdf_io.models import ObjectMapping 134 | 135 | from my_app.models import Task 136 | 137 | # This example assumes ... 138 | # * you have created a model called `Task` and there’s at least one task 139 | # * you have created a mapping for the Task model 140 | 141 | object_to_serialize = Task.objects.first() 142 | 143 | content_type = ContentType.objects.get(model='task') 144 | obj_mapping_list = ObjectMapping.objects.filter(content_type=content_type) 145 | 146 | graph = Graph() 147 | 148 | build_rdf(graph, object_to_serialize, obj_mapping_list, includemembers=True) 149 | 150 | print(graph.serialize(format="turtle")) 151 | ``` 152 | ### Serialising using django views: 153 | 154 | `{SERVER_URL}/rdf_io/to_rdf/{model_name}/{model_id}` 155 | 156 | 157 | 158 | 159 | ## Deprecated 160 | ## Marmotta LDP 161 | * deploy marmotta.war and configure as per Marmotta instructions 162 | * define resource container patterns for different models 163 | 164 | e.g. 165 | 166 | ``` 167 | # RDF triplestore settings 168 | RDFSTORE = { 169 | 'default' : { 170 | 'server' : "".join((SITEURL,":8080/marmotta" )), 171 | # model and slug are special - slug will revert to id if not present 172 | 'target' : "/ldp/{model}/{slug}", 173 | # this could be pulled from settings 174 | 'auth' : ('admin', 'pass123') 175 | }, 176 | # define special patterns for nested models 177 | 'scheme' : { 178 | 'target' : "/ldp/voc/{slug}", 179 | }, 180 | 'concept' : { 181 | 'target' : "/ldp/voc/{scheme__slug}/{term}", 182 | } 183 | } 184 | ``` 185 | 186 | * create containers necessary for patterns (eg /ldp/voc) in the example above 187 | * deploy reasoning rules for target models (to generate additional statements that can be inferred from the published data - this is where the power comes in) 188 | - see http://eagle-dev.salzburgresearch.at/reasoner/admin/about.html 189 | e.g. 190 | ``` 191 | curl -i -H "Content-Type: text/plain" -X POST --data-binary @fixtures/skos.kwrl http://localhost:8080/marmotta/reasoner/program/skos.kwrl 192 | curl -i -X GET http://localhost:8080/marmotta/reasoner/program/skos.kwrl 193 | curl -i -X GET 194 | curl -i -H "Content-Type: text/plain" -X POST --data-binary @skos.skwrl http://localhost:8080/marmotta/reasoner/program/skos.skwrl 195 | ``` 196 | ### Operations 197 | 198 | If auto_publish is set in an Object Mapping then the RDF-IO mapping is triggered automatically when saving an object once an ObjectMapping is defined for that object type. 199 | 200 | A bulk load to the RDF store can be achieved with /rdf_io/sync_remote/{model}(,{model})* 201 | 202 | Note that containers need to be create in the right order - so for the SKOS example this must be /rdf_io/sync_remote/scheme,concept 203 | 204 | 205 | # Design Goals 206 | 207 | * Apps could define default settings (the mappings to RDF) - and these will just be ignored if the serializer is not present. 208 | * When bringing in the serializer, if you wanted to be able to serialize a Class in an app for which there are no default mappings, it should be possible to define these (create a rdf_mappings.py file in the top project) 209 | * The top project will allow either the default mappings for an app to be overridden, either as a whole or on a per-mapping basis (i.e. change or add mappings for individual attributes) 210 | * the serialiser would be available as a stand-alone serialiser for dumpdata (and extended to be a deserialiser for loaddata) - but also able to be hooked up to post the serialized data to an external service - so my serialiser app might have a model to capture connection parameters for such services - and other app settings would be able to define connections in this model and bind different model's rdf mappings to different target services. 211 | 212 | We have four types of apps then: 213 | 1 the master project 214 | 1 the RDF serializer utility 215 | 1 imported apps that have default RDF serializations 216 | 1 imported apps that may or may not have RDF serialisations defined in the project settings. 217 | 218 | I suspect that this may all be a fairly common pattern - but I've only seen far more heavyweight approaches to RDF trying to fully model RDF and implement SPARQL - all I want to do is spit some stuff out into an external triple-store. 219 | 220 | default RDF serialisations are handled by loading initial_data fixtures. RDF_IO objects are defined using natural keys to allow default mappings for modules to be loaded in any order. It may be more elegant to use settings so these defaults can be customised more easily. 221 | 222 | Signals are registered when an ObjectMapping is defined for a model. 223 | -------------------------------------------------------------------------------- /rdf_io/models.py: -------------------------------------------------------------------------------- 1 | # from __future__ import unicode_literals 2 | # from __future__ import print_function 3 | from builtins import str 4 | from builtins import next 5 | from builtins import range 6 | from builtins import object 7 | 8 | import logging 9 | logger = logging.getLogger(__name__) 10 | 11 | try: 12 | from django.utils.encoding import python_2_unicode_compatible 13 | except: 14 | from six import python_2_unicode_compatible 15 | 16 | from django.db import models 17 | from django.conf import settings 18 | 19 | from django.utils.translation import ugettext_lazy as _ 20 | # for django 1.7 + 21 | from django.contrib.contenttypes.fields import GenericForeignKey 22 | #from django.contrib.contenttypes.generic import GenericForeignKey 23 | from django.contrib.contenttypes.models import ContentType 24 | from django.core.exceptions import ValidationError 25 | from django.core.validators import RegexValidator, URLValidator 26 | try: 27 | from django.db.models.fields.related import ReverseSingleRelatedObjectDescriptor 28 | except: 29 | from django.db.models.fields.related_descriptors import ForwardManyToOneDescriptor as ReverseSingleRelatedObjectDescriptor 30 | import sys 31 | import re 32 | import requests 33 | import itertools 34 | import os 35 | from rdflib import Graph,namespace, XSD 36 | from rdflib.term import URIRef, Literal 37 | from six import string_types 38 | from rdflib.namespace import NamespaceManager,RDF 39 | from rdf_io.protocols import * 40 | from django.db.models import Q 41 | 42 | def testxx(): 43 | print( __name__) 44 | 45 | # helpers 46 | def getattr_path(obj,path) : 47 | """ Get a list of attribute values matching a nested attribute path with filters 48 | 49 | format of path is a string a.b.c with optional filters a[condition].b[condition] etc 50 | """ 51 | try : 52 | return _getattr_related(obj,obj, pathsplit(path.replace('__','.')), extravals={}) 53 | 54 | except ValueError as e: 55 | import traceback 56 | # import pdb; pdb.set_trace() 57 | raise ValueError("Failed to map '{}' on '{}' (cause {})".format(path, obj, e)) 58 | 59 | def pathsplit(str): 60 | """ takes a path with filters which may include literals, and ignores filter contents when splitting """ 61 | result = [] 62 | tok_start = 0 63 | infilt= False 64 | for i,c in enumerate(str) : 65 | if c == '.' and not infilt : 66 | result.append( str[tok_start:i]) 67 | tok_start = i+1 68 | elif c == '[' : 69 | infilt=True 70 | elif c == ']' : 71 | infilt=False 72 | result.append(str[tok_start:]) 73 | return result 74 | 75 | def getattr_tuple_path(obj,pathlist) : 76 | """ Get a list of attribute value tuples matching a set of nested attribute paths with filters 77 | 78 | format of each path is a string a.b.c with optional filters a[condition].b[condition] etc 79 | 80 | tuples are generated at the level of common path - e.g (a.b.c,a.b.d) generaters the tuple (val(c), val(d)) for objects in path a.b 81 | """ 82 | for p in pathlist: 83 | p.replace('__','.').replace("/",".") 84 | try : 85 | return _getattr_related(obj,obj, pathsplit(pathlist[0]), pathlist=pathlist[1:], extravals={}) 86 | 87 | except ValueError as e: 88 | import traceback 89 | # import pdb; pdb.set_trace() 90 | raise ValueError("Failed to map '{}' on '{}' (cause {})".format(pathlist, obj, e)) 91 | 92 | def dequote(s): 93 | """ Remove outer quotes from string 94 | 95 | If a string has single or double quotes around it, remove them. 96 | todo: Make sure the pair of quotes match. 97 | If a matching pair of quotes is not found, return the string unchanged. 98 | """ 99 | if s.startswith('"""') : 100 | return s[3:-3] 101 | elif s.startswith(("'", '"', '<')): 102 | return s[1:-1] 103 | return s 104 | 105 | def quote(s): 106 | """ quote string so it gets processed as a literal 107 | 108 | leave @lang and ^^datatype qualifiers outside quoting! 109 | """ 110 | 111 | if isinstance(s, string_types) and s[0] not in ('"',"'"): 112 | try: 113 | root,lang = s.split('@') 114 | return ''.join(('"',root,'"','@',lang)) 115 | except: 116 | try: 117 | root,lang = s.split('^^') 118 | return ''.join(('"',root,'"','^^',lang)) 119 | except: 120 | return s.join(('"','"')) 121 | return s 122 | 123 | def _apply_filter(val, filter,localobj, rootobj) : 124 | """ 125 | Apply a simple filter to a specific property, with a list of possible values 126 | """ 127 | for targetval in filter.replace(" OR ",",").split(",") : 128 | tval = dequote(targetval) 129 | if tval.startswith('^') : 130 | tval = getattr(rootobj,tval[1:]) 131 | elif tval.startswith('.') : 132 | tval = getattr(localobj,tval[1:]) 133 | if tval == 'None' : 134 | return bool(val) 135 | elif tval == 'NotNone' : 136 | return not bool(val) 137 | elif val == tval : 138 | return True 139 | return False 140 | 141 | def apply_pathfilter(obj, filter_expr ): 142 | """ does obj match filter expression? 143 | 144 | apply a filter based on a list of path expressions path1=a,b AND path2=c,db 145 | """ 146 | and_clauses = filter_expr.split(" AND ") 147 | 148 | for clause in and_clauses: 149 | 150 | (path,vallist) = clause.split("=") 151 | if path[:-1] == '!' : 152 | negate = True 153 | path = path[0:-1] 154 | else: 155 | negate = False 156 | or_vals = vallist.split(",") 157 | # if multiple values - only one needs to match 158 | matched = False 159 | for val in getattr_path(obj,path): 160 | for match in or_vals : 161 | if match == 'None' : 162 | matched = negate ^ ( not val ) 163 | elif type(val) == bool : 164 | matched = negate ^ (val == (match == 'True')) 165 | break; 166 | else : 167 | if negate ^ (val == dequote(match)) : 168 | matched = True 169 | break 170 | if matched : 171 | # dont need to check any mor - continue with the AND loop 172 | break 173 | # did any value match? 174 | if not matched : 175 | return False 176 | 177 | return True 178 | 179 | def _getattr_related(rootobj,obj, fields, pathlist=None, extravals={} ): 180 | """ recursive walk down object path looking for field values 181 | 182 | get an attribute - if multi-valued will be a list object! 183 | if pathlist is present, then each path in the 184 | fields may include filters. 185 | """ 186 | # print obj, fields 187 | if not len(fields): 188 | return [[obj,] + [ extravals[i] for i in range(0,len(extravals)) ]] if extravals else [obj] 189 | 190 | field = fields.pop(0) 191 | if pathlist: 192 | pathlist2= list(pathlist) 193 | for i,p in enumerate(pathlist): 194 | if not p : 195 | continue 196 | elif p.startswith(field + "."): 197 | pathlist2[i] = p[len(field)+1:] 198 | else : 199 | pathlist2[i] = None 200 | extravals[i] = getattr_path(obj,p) 201 | pathlist = tuple(pathlist2) 202 | filter = None 203 | filters = None 204 | # try to get - then check for django 1.7+ manager for related field 205 | try: 206 | # check for lang 207 | try: 208 | (field,langfield) = field.split('@') 209 | if langfield[0] in ["'" , '"'] : 210 | lang = langfield[1:-1] 211 | else: 212 | lang = _getattr_related(rootobj,obj, [langfield,] + fields).pop(0) 213 | fields = [] 214 | except: 215 | lang = None 216 | # check for datatype ^^type 217 | try: 218 | (field,typefield) = field.split('^^') 219 | if typefield[0] in ["'" , '"'] : 220 | typeuri = typefield[1:-1] 221 | else: 222 | try: 223 | typeuri = _getattr_related(rootobj,obj, [typefield,] + fields).pop(0) 224 | except Exception as e : 225 | raise ValueError("error accessing data type field '{}' in field '{}' : {}".format(typefield, field, e) ) 226 | #have reached end of chain and have used up field list after we hit ^^ 227 | fields = [] 228 | except: 229 | typeuri = None 230 | # check for filt 231 | # check for filter 232 | if "[" in field : 233 | filter = field[ field.index("[") +1 : -1 ] 234 | field = field[0:field.index("[")] 235 | filters=_makefilters(filter, obj, rootobj) 236 | 237 | val = getattr(obj, field) 238 | if not val : 239 | return [] 240 | # import pdb; pdb.set_trace() 241 | try: 242 | # slice the list for fields[:] to force a copy so each iteration starts from top of list in spite of pop() 243 | if filter: 244 | valset = val.filter(**filters['includes']).exclude(**filters['excludes']) 245 | else : 246 | valset = val.all() 247 | return itertools.chain(*(_getattr_related(rootobj,xx, fields[:], pathlist=pathlist,extravals=extravals) for xx in valset)) 248 | except Exception as e: 249 | pass 250 | if filter and not _apply_filter(val, filter, obj, rootobj) : 251 | return [] 252 | if lang: 253 | val = "@".join((val,lang)) 254 | elif typeuri : 255 | val = "^^".join((val,typeuri)) 256 | except AttributeError: 257 | relobjs = _get_relobjs(obj,field,filters) 258 | 259 | # will still throw an exception if val is not set! 260 | try: 261 | # slice the list fo fields[:] to force a copy so each iteration starts from top of list in spite of pop() 262 | return itertools.chain(*(_getattr_related(rootobj,xx, fields[:], pathlist=pathlist) for xx in relobjs.all())) 263 | # !list(itertools.chain(*([[1],[2]]))) 264 | except: 265 | try: 266 | return _getattr_related(obj,val, fields, pathlist=pathlist,extravals=extravals) 267 | except: 268 | raise ValueError( "Object type %s has no related model %s " % ( str(type(obj)), str(fields) )) 269 | 270 | def _get_relobjs(obj,field,filters=None): 271 | """Find related objects that match 272 | 273 | Could be linked using a "related_name" or as _set 274 | 275 | django versions have changed this around so somewhat tricky.. 276 | """ 277 | # then try to find objects of this type with a foreign key property using either (name) supplied or target object type 278 | 279 | if not filters: 280 | filters = { 'includes': {} , 'excludes' : {} } 281 | if field.endswith(")") : 282 | (field, relprop ) = str(field[0:-1]).split("(") 283 | else : 284 | relprop = None 285 | 286 | try: 287 | reltype = ContentType.objects.get(model=field) 288 | except ContentType.DoesNotExist as e : 289 | raise ValueError("Could not locate attribute or related model '{}' in element '{}'".format(field, type(obj)) ) 290 | 291 | # if no related_name set in related model then only one candidate and djanog creates X_set attribute we can use 292 | try: 293 | return getattr(obj, "".join((field,"_set"))).filter(**filters['includes']).exclude(**filters['excludes']) 294 | except: 295 | pass 296 | 297 | # trickier then - need to look at models of the named type 298 | claz = reltype.model_class() 299 | for prop,val in list(claz.__dict__.items()) : 300 | # skip related property names if set 301 | if relprop and prop != relprop : 302 | continue 303 | if relprop or type(val) is ReverseSingleRelatedObjectDescriptor and val.field.related.model == type(obj) : 304 | filters['includes'].update({prop:obj}) 305 | return claz.objects.filter(**filters['includes']).exclude(**filters['excludes']) 306 | 307 | def _makefilters(filter, obj, rootobj): 308 | """Makes a django filter syntax for includes and excludes from provided filter 309 | 310 | allow for filter clauses with references relative to the object being serialised, the root of the path being encoded or the element in the path specifying the filter""" 311 | if not filter : 312 | return {} 313 | filterclauses = dict( [fc.split("=") for fc in filter.replace(" AND ",",").split(",")]) 314 | includes = {} 315 | excludes = {} 316 | for fc in filterclauses : 317 | fval = filterclauses[fc] 318 | if fc[-1] == '!' : 319 | excludes = _add_clause(excludes, fc[0:-1], fval, obj, rootobj) 320 | else: 321 | includes = _add_clause(includes, fc, fval, obj, rootobj) 322 | 323 | return { 'includes': includes , 'excludes' : excludes } 324 | 325 | def _add_clause(extrafilterclauses, fc, fval , obj, rootobj): 326 | if not fval : 327 | extrafilterclauses[ "".join((fc,"__isnull"))] = False 328 | elif fval == 'None' : 329 | extrafilterclauses[ "".join((fc,"__isnull"))] = True 330 | elif fval.startswith('^'): # property value via path from root object being serialised 331 | try: 332 | objvals = getattr_path(rootobj,fval[1:]) 333 | if len(objvals) == 0 : 334 | return [] # non null match against null source fails 335 | extrafilterclauses[fc] = objvals.pop() 336 | except Exception as e: 337 | raise ValueError ("Error in filter clause %s on field %s " % (fc,prop)) 338 | 339 | elif fval.startswith('.'): # property value via path from current path object 340 | try: 341 | objvals = getattr_path(obj,fval[1:]) 342 | if len(objvals) == 0 : 343 | return [] # non null match against null source fails 344 | extrafilterclauses[fc] = objvals.pop() 345 | except Exception as e: 346 | raise ValueError ("Error in filter clause %s on field %s " % (fc,prop)) 347 | elif fval.startswith(("'", '"', '<')) : 348 | extrafilterclauses[fc] = dequote(fval) 349 | elif not str(fval).isnumeric() : 350 | # look for a value 351 | extrafilterclauses[fc] = getattr(obj, fval) 352 | else: 353 | extrafilterclauses[fc] = fval 354 | 355 | return extrafilterclauses 356 | 357 | def expand_curie(value): 358 | try: 359 | parts = value.split(":") 360 | if len(parts) == 2 : 361 | ns = Namespace.objects.get(prefix=parts[0]) 362 | return "".join((ns.uri,parts[1])) 363 | except: 364 | pass 365 | return value 366 | 367 | def as_uri(value): 368 | """ puts <> around if not a CURIE """ 369 | try: 370 | parts = value.split(":") 371 | if len(parts) == 2 : 372 | return value 373 | except: 374 | pass 375 | return value.join("<",">") 376 | 377 | 378 | def as_resource(gr,curie) : 379 | cleaned = dequote(curie) 380 | if cleaned[0:4] == 'http' : 381 | return URIRef(cleaned) 382 | # this will raise error if not valid curie format 383 | try: 384 | (ns,value) = cleaned.split(":",2) 385 | except: 386 | return URIRef(cleaned) # just have to assume its not a problem - URNs are valid uri.s 387 | # raise ValueError("value not value HTTP or CURIE format %s" % curie) 388 | try : 389 | nsuri = Namespace.getNamespace(ns) 390 | if nsuri : 391 | gr.namespace_manager.bind( str(ns), namespace.Namespace(nsuri.uri), override=False) 392 | return URIRef("".join((nsuri.uri,value))) 393 | else : 394 | return URIRef(cleaned) 395 | except: 396 | raise ValueError("prefix " + ns + "not recognised") 397 | 398 | TYPES = { 399 | 'xsd:int' : XSD.integer , 400 | 'xsd:float': XSD.float , 401 | 'xsd:double': XSD.double , 402 | 'xsd:time' : XSD.time , 403 | 'xsd:dateTime' : XSD.dateTime , 404 | 'xsd:boolean' : XSD.boolean , 405 | 'xsd:integer' : XSD.integer , 406 | } 407 | 408 | def makenode(gr,value, is_resource=False): 409 | """ make a RDF node from a string representation 410 | 411 | probably ought to be able to find this function in rdflib but seems hidden""" 412 | if is_resource or value[0] == '<' and '<' not in value[1:] and value [-1] == '>' : 413 | return as_resource(gr,value) 414 | else: 415 | try: 416 | try : 417 | (value,valtype) = value.split("^^") 418 | try: 419 | typeuri = TYPES[valtype] 420 | except: 421 | typeuri = as_resource(gr,valtype) 422 | return Literal(dequote(value),datatype=typeuri) 423 | except: 424 | try : 425 | (value,valtype) = value.split("@") 426 | return Literal(dequote(value),lang=valtype) 427 | except: 428 | try: 429 | value = int(value) 430 | return Literal(value, datatype=XSD.integer) 431 | except: 432 | try: 433 | value = double(value) 434 | return Literal(value, datatype=XSD.double) 435 | except: 436 | return Literal(dequote(value)) 437 | except: 438 | raise ValueError("Value not a convertable type %s" % type(value)) 439 | 440 | 441 | def validate_urisyntax(value): 442 | 443 | if value[0:4] == 'http' : 444 | URLValidator().__call__(value) 445 | else : 446 | parts = value.split(":") 447 | if len(parts) != 2 : 448 | raise ValidationError('invalid syntax - neither http URI or a valid CURIE') 449 | # try: 450 | # ns = Namespace.objects.get(prefix=parts[0]) 451 | # except Exception as e: 452 | # raise ValidationError("Namespace not defined for prefix %s" % parts[0]) 453 | 454 | def validate_propertypath(path): 455 | for value in path.split(): 456 | validate_urisyntax(value) 457 | 458 | 459 | class RDFpath_Field(models.CharField): 460 | """ 461 | Char field for URI with syntax checking for CURIE or http syntax 462 | """ 463 | # validate that prefix is registered if used 464 | validators = [ validate_propertypath, ] 465 | def __init__(self, *args, **kwargs): 466 | kwargs['max_length'] = 500 467 | kwargs['help_text']=_(u'space separated list of RDF property URIs (in form a:b or full URI) representing a nested set of properties in an RDF graph') 468 | super( RDFpath_Field, self).__init__(*args, **kwargs) 469 | 470 | class CURIE_Field(models.CharField): 471 | """ 472 | Char field for URI with syntax checking for CURIE or http syntax 473 | """ 474 | # validate that prefix is registered if used 475 | validators = [ validate_urisyntax, ] 476 | def __init__(self, *args, **kwargs): 477 | kwargs['max_length'] = 200 478 | kwargs['help_text']=_(u'use a:b or full URI') 479 | super( CURIE_Field, self).__init__(*args, **kwargs) 480 | 481 | class EXPR_Field(models.CharField): 482 | """ 483 | Char field for expression - literal or nested atribute with syntax checking for CURIE or http syntax 484 | """ 485 | literal_form=None 486 | def __init__(self, *args, **kwargs): 487 | kwargs['max_length'] = 400 488 | kwargs['help_text']=_(u'for a literal, use "quoted" syntax, for nested attribute use syntax a.b.c') 489 | super( EXPR_Field, self).__init__(*args, **kwargs) 490 | 491 | 492 | class FILTER_Field(models.CharField): 493 | """ 494 | Char field for filter expression: path=value(,value) 495 | """ 496 | 497 | def __init__(self, *args, **kwargs): 498 | kwargs['max_length'] = 400 499 | kwargs['help_text']=_(u'path=value, eg label__label_text="frog"') 500 | super( FILTER_Field, self).__init__(*args, **kwargs) 501 | 502 | 503 | # Need natural keys so can reference in fixtures - let this be the uri 504 | 505 | 506 | class NamespaceManager(models.Manager): 507 | def get_by_natural_key(self, uri): 508 | return self.get(uri=uri) 509 | 510 | class Namespace(models.Model) : 511 | """ 512 | defines a namespace so we can use short prefix where convenient 513 | """ 514 | objects = NamespaceManager() 515 | 516 | uri = models.CharField('uri',max_length=100, unique=True, null=False) 517 | prefix = models.CharField('prefix',max_length=8,unique=True,null=False) 518 | notes = models.TextField(_(u'change note'),blank=True) 519 | 520 | def natural_key(self): 521 | return(self.uri,) 522 | 523 | def get_base_uri(self): 524 | return self.uri[0:-1] 525 | def is_hash_uri(self): 526 | return self.uri[-1] == '#' 527 | 528 | @staticmethod 529 | def getNamespace( prefix) : 530 | try: 531 | return Namespace.objects.get(prefix = prefix) 532 | except: 533 | return None 534 | 535 | class Meta(object): 536 | verbose_name = _(u'namespace') 537 | verbose_name_plural = _(u'namespaces') 538 | 539 | def __unicode__(self): 540 | return self.uri 541 | 542 | def __str__(self): 543 | return str( " = ".join(filter(None,(self.prefix,self.uri)))) 544 | 545 | class GenericMetaPropManager(models.Manager): 546 | def get_by_natural_key(self, curie): 547 | try: 548 | (namespace,prop) = curie.split(":") 549 | return self.get(namespace__prefix=namespace, propname=prop) 550 | except: 551 | return self.get(uri=curie) 552 | 553 | 554 | class GenericMetaProp(models.Model) : 555 | """ 556 | a metadata property that can be attached to any target model to provide extensible metadata. 557 | Works with the namespace object to allow short forms of metadata to be displayed 558 | """ 559 | objects = GenericMetaPropManager() 560 | namespace = models.ForeignKey(Namespace,models.PROTECT,blank=True, null=True, verbose_name=_(u'namespace')) 561 | propname = models.CharField(_(u'name'),blank=True,max_length=250,editable=True) 562 | uri = CURIE_Field(blank=True, unique=True) 563 | definition = models.TextField(_(u'definition'), blank=True) 564 | def natural_key(self): 565 | return ( ":".join((self.namespace.prefix,self.propname)) if self.namespace else self.uri , ) 566 | def __unicode__(self): # __unicode__ on Python 2 567 | return self.natural_key()[0] 568 | def asURI(self): 569 | """ Returns fully qualified uri form of property """ 570 | return uri 571 | 572 | def __str__(self): 573 | return str(self.uri) 574 | 575 | def save(self,*args,**kwargs): 576 | if self.namespace : 577 | self.uri = "".join((self.namespace.uri,self.propname)) 578 | else: 579 | try: 580 | (dummy, base, sep, term) = re.split('(.*)([/#])', self.uri) 581 | ns = Namespace.objects.get(uri="".join((base,sep))) 582 | if ns: 583 | self.namespace = ns 584 | self.propname = term 585 | except: 586 | pass 587 | 588 | super(GenericMetaProp, self).save(*args,**kwargs) 589 | 590 | 591 | class AttachedMetadata(models.Model): 592 | """ metadata property that can be attached using subclass that specificies the subject property FK bining 593 | 594 | extensible metadata using rdf_io managed reusable generic metadata properties 595 | """ 596 | metaprop = models.ForeignKey(GenericMetaProp, models.PROTECT,verbose_name='property') 597 | value = models.CharField(_(u'value'),max_length=2000) 598 | def __unicode__(self): 599 | return str(self.metaprop.__unicode__()) 600 | def getRDFValue(self): 601 | """ returns value in appropriate datatype """ 602 | return makenode(value) 603 | 604 | class Meta(object): 605 | pass 606 | # abstract = True 607 | 608 | class ObjectTypeManager(models.Manager): 609 | def get_by_natural_key(self, uri): 610 | return self.get(uri=uri) 611 | 612 | class ObjectType(models.Model): 613 | """ 614 | Allows for a target object to be declared as multiple object types 615 | Object types may be URI or CURIEs using declared prefixes 616 | """ 617 | objects = ObjectTypeManager() 618 | uri = CURIE_Field(_(u'URI'),blank=False,editable=True) 619 | label = models.CharField(_(u'Label'),blank=False,max_length=250,editable=True) 620 | 621 | def natural_key(self): 622 | return (self.uri,) 623 | 624 | # check short form is registered 625 | def __unicode__(self): # __unicode__ on Python 2 626 | return " -- ".join((self.uri,self.label )) 627 | 628 | class ObjectMappingManager(models.Manager): 629 | def get_by_natural_key(self, name): 630 | return self.get(name=name) 631 | 632 | class ObjectMapping(models.Model): 633 | """ 634 | Maps an instance of a model to a resource (i.e. a URI with a type declaration) 635 | """ 636 | objects = ObjectMappingManager() 637 | content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE) 638 | name = models.CharField(_(u'Name'),help_text=_(u'unique identifying label'),unique=True,blank=False,max_length=250,editable=True) 639 | auto_push = models.BooleanField(_(u'auto_push'),help_text=_(u'set this to push updates to these object to the RDF store automatically')) 640 | id_attr = models.CharField(_(u'ID Attribute'),help_text=_(u'for nested attribute use syntax a.b.c'),blank=False,max_length=250,editable=True) 641 | target_uri_expr = EXPR_Field(_(u'target namespace expression'), blank=False,editable=True) 642 | obj_type = models.ManyToManyField(ObjectType, help_text=_(u'set this to generate a object rdf:type X statement' )) 643 | filter = FILTER_Field(_(u'Filter'), null=True, blank=True ,editable=True) 644 | def natural_key(self): 645 | return (self.name,) 646 | 647 | def __unicode__(self): # __unicode__ on Python 2 648 | return self.name 649 | 650 | def __str__(self): 651 | return str( self.__unicode__()) 652 | @staticmethod 653 | def new_mapping(object_type,content_type_label, title, idfield, tgt,filter=None, auto_push=False, app_label=None): 654 | if not app_label : 655 | try: 656 | (app_label,content_type_label) = content_type_label.split(':') 657 | except: 658 | pass # hope we can find it? 659 | content_type = ContentType.objects.get(app_label=app_label.lower(),model=content_type_label.lower()) 660 | defaults = { "auto_push" : auto_push , 661 | "id_attr" : idfield, 662 | "target_uri_expr" : tgt, 663 | "content_type" : content_type 664 | } 665 | if filter : 666 | defaults['filter']=filter 667 | 668 | (pm,created) = ObjectMapping.objects.get_or_create(name=title, defaults =defaults) 669 | if not created : 670 | AttributeMapping.objects.filter(scope=pm).delete() 671 | 672 | pm.obj_type.add(object_type) 673 | pm.save() 674 | 675 | return pm 676 | 677 | 678 | class AttributeMapping(models.Model): 679 | """ 680 | records a mapping from an object mapping that defines a relation from the object to a value using a predicate 681 | """ 682 | scope = models.ForeignKey(ObjectMapping,models.PROTECT) 683 | attr = EXPR_Field(_(u'source attribute'),help_text=_(u'literal value or path (attribute[filter].)* with optional @element or ^^element eg locationname[language=].name@language. filter values are empty (=not None), None, or a string value'),blank=False,editable=True) 684 | # filter = FILTER_Field(_(u'Filter'), null=True, blank=True,editable=True) 685 | predicate = CURIE_Field(_(u'predicate'),blank=False,editable=True,help_text=_(u'URI or CURIE. Use :prop.prop.prop form to select a property of the mapped object to use as the predicate')) 686 | is_resource = models.BooleanField(_(u'as URI')) 687 | 688 | def __unicode__(self): 689 | return ( ' '.join((self.attr, self.predicate ))) 690 | 691 | class EmbeddedMapping(models.Model): 692 | """ embedded mapping using a template 693 | 694 | records a mapping for a complex data structure 695 | """ 696 | scope = models.ForeignKey(ObjectMapping,models.PROTECT,) 697 | attr = EXPR_Field(_(u'source attribute'),help_text=_(u'attribute - if empty nothing generated, if multivalued will be iterated over')) 698 | predicate = CURIE_Field(_(u'predicate'),blank=False,editable=True, help_text=_(u'URI or CURIE. Use :prop.prop.prop form to select a property of the mapped object to use asthe predicate')) 699 | struct = models.TextField(_(u'object structure'),max_length=2000, help_text=_(u' ";" separated list of predicate attribute expr where attribute expr a model field or "literal" or - in future may be an embedded struct inside {} '),blank=False,editable=True) 700 | use_blank = models.BooleanField(_(u'embed as blank node'), default=True) 701 | 702 | def __unicode__(self): 703 | return ( ' '.join(('struct:',self.attr, self.predicate ))) 704 | 705 | class ChainedMapping(models.Model): 706 | """ nested mapping using another mapping 707 | 708 | Chains to a specific mapping to nest the resulting graph within the current serialisation 709 | """ 710 | scope = models.ForeignKey(ObjectMapping,models.PROTECT,editable=False, ) 711 | attr = EXPR_Field(_(u'source attribute'),help_text=_(u'attribute - if empty nothing generated, if multivalued will be iterated over')) 712 | predicate = CURIE_Field(_(u'predicate'),blank=False,editable=True, help_text=_(u'URI or CURIE. Use :prop.prop.prop form to select a property of the mapped object to use asthe predicate')) 713 | chainedMapping = models.ForeignKey(ObjectMapping, models.PROTECT,blank=False,editable=True, related_name='chained',help_text=_(u'Mapping to nest, for each value of attribute. may be recursive')) 714 | 715 | def __unicode__(self): 716 | return ( ' '.join(('chained mapping:',self.attr, self.predicate, self.chainedMapping.name ))) 717 | 718 | MODE_CHOICES = ( 719 | ( 'REVIEW', 'Persist results in mode for review'), 720 | ( 'TEST', 'Does not persist results' ), 721 | ( 'PUBLISH' , 'Data published to final target' ) 722 | ) 723 | 724 | class ConfigVar(models.Model): 725 | 726 | """ Sets a configuration variable for ServiceBindings templates """ 727 | var=models.CharField(max_length=16, null=False, blank=False , verbose_name='Variable name') 728 | value=models.CharField(max_length=255, null=False, blank=True , verbose_name='Variable value') 729 | mode=models.CharField( verbose_name='Mode scope', choices=MODE_CHOICES,null=True,blank=True,max_length=10 ) 730 | 731 | def __unicode__(self): 732 | return ( ' '.join(('var:',self.var, ' (', str(self.mode), ') = ', self.value ))) 733 | 734 | def __str__(self): 735 | return str( self.__unicode__()) 736 | 737 | @staticmethod 738 | def getval(var,mode): 739 | try: 740 | return ConfigVar.objects.filter(var=var, mode=mode).first().value 741 | except: 742 | try: 743 | return ConfigVar.objects.filter(var=var, mode__isnull=True).first().value 744 | except: 745 | pass 746 | return None 747 | 748 | @staticmethod 749 | def getvars(mode): 750 | return ConfigVar.objects.filter(Q(mode=mode) | Q(mode__isnull=True)).order_by('var') 751 | 752 | 753 | class ServiceBinding(models.Model): 754 | """ Binds object mappings to a RDF handling service 755 | 756 | Services may perform several roles: 757 | * Validation 758 | * INFERENCE 759 | * Persistence 760 | 761 | Bindings may be controlled by status variables in the objects being bound - for example draft content published to a separate directory. 762 | 763 | Bindings, if present for an object, override system defaults. 764 | 765 | Services may be chained with an exception handling clause. API will provide options for choosing starting point of chain and whether to automatically follow chain or report and pause. 766 | """ 767 | VALIDATION='VALIDATION' 768 | INFERENCE='INFERENCE' 769 | PERSIST_CREATE='PERSIST_CREATE' 770 | PERSIST_REPLACE='PERSIST_REPLACE' 771 | PERSIST_UPDATE='PERSIST_UPDATE' 772 | PERSIST_PURGE='PERSIST_PURGE' 773 | DEBUG_SHOW='SHOW - show encoded content or previous service errors' 774 | BINDING_CHOICES = ( 775 | ( VALIDATION, 'VALIDATION - Performs validation check'), 776 | ( INFERENCE, 'INFERENCE - The entailed response replaces the default encoding in downstream services' ), 777 | ( PERSIST_CREATE, 'PERSIST_CREATE - A new resource is created only if not present in the persistence store' ), 778 | ( PERSIST_REPLACE, 'PERSIST_REPLACE - (e.g. HTTP PUT) The resource and its properties are replaced in the persistence store' ), 779 | ( PERSIST_UPDATE, 'PERSIST_UPDATE - (e.g. HTTP POST) The resource and its properties are added to the persistence store' ), 780 | ( PERSIST_PURGE, 'PERSIST_PURGE - (e.g. HTTP DELETE) The resource and its properties are deleted from the persistence store' ), 781 | ) 782 | RDF4JREST = 'RDF4JREST' 783 | LDP = 'LDP' 784 | SHACLAPI = 'SHACLAPI' 785 | SPARQL = 'SPARQL' 786 | GIT = 'GIT' 787 | API_CHOICES=( 788 | (RDF4JREST,'RDF4JREST - a.k.a Sesame'), 789 | (LDP,'LDP: Linked Data Platform'), 790 | (GIT,'GIT'), 791 | (SHACLAPI,'SHACL service'), 792 | (SPARQL,'SPARQL endpoint'), 793 | ) 794 | API_TEMPLATES = { RDF4JREST : "http://localhost:8080/rdf4j-server/repositories/myrepo" } 795 | 796 | title = models.CharField(max_length=255, blank=False, default='' ) 797 | 798 | description = models.TextField(max_length=1000, null=True, blank=True) 799 | binding_type=models.CharField(max_length=16,choices=BINDING_CHOICES, default=PERSIST_REPLACE, help_text='Choose the role of service') 800 | service_api = models.CharField(max_length=16,choices=API_CHOICES, help_text='Choose the API type of service') 801 | service_url = models.CharField(max_length=1000, verbose_name='service url template', help_text='Parameterised service url - {var} where var is an attribute of the object type being mapped (including django nested attributes using a__b syntax) or $model for the short model name') 802 | 803 | resource = models.CharField(max_length=1000, verbose_name='resource path', help_text='Parameterised path to target resource to be persisted - using the target service API syntax - e.g. /statements?context=<{uri}> for a RDF4J named graph.', default="/statements?context=") 804 | inferenced_resource = models.CharField(max_length=1000, verbose_name='generated resource', help_text='Parameterised path to intermediate resource - using the target service API syntax. If this is an inferencing service then this will be the resource identifier for the additional triples generated by the inferencing rules generated for the specific object.', null=True,blank=True) 805 | object_mapping = models.ManyToManyField(ObjectMapping, verbose_name='Object mappings service binding applies to automatically', blank=True) 806 | # use_as_default = models.BooleanField(verbose_name='Use by default', help_text='Set this flag to use this by default') 807 | 808 | object_filter=models.TextField(max_length=2000, verbose_name='filter expression', help_text='A (python dict) filter on the objects that this binding applies to', blank=True, null=True) 809 | next_service=models.ForeignKey('ServiceBinding', models.PROTECT, verbose_name='Next service', blank=True, null=True) 810 | on_delete_service=models.ForeignKey('ServiceBinding', models.PROTECT, related_name='on_delete',verbose_name='Deletion service', blank=True, null=True, help_text='This will be invoked on object deletion if specified, and also if the binding is "replace" - which allows for a specific pre-deletion step if not supported by the repository API natively') 811 | on_fail_service=models.ForeignKey('ServiceBinding', models.PROTECT, related_name='on_fail',verbose_name='On fail service', blank=True, null=True, help_text='Overrides default failure reporting') 812 | 813 | def __unicode__(self): 814 | return self.title + "(" + self.service_api + " : " + self.service_url + ")" 815 | 816 | def __str__(self): 817 | return str( self.__unicode__()) 818 | 819 | @staticmethod 820 | def get_service_bindings(model,bindingtypes): 821 | ct = ContentType.objects.get(model=model) 822 | if bindingtypes: 823 | return ServiceBinding.objects.filter(object_mapping__content_type=ct, binding_type__in=bindingtypes) 824 | else: 825 | return ServiceBinding.objects.filter(object_mapping__content_type=ct) 826 | 827 | def next_chain(self): 828 | obj = self 829 | chain = [] 830 | while obj.next_service : 831 | obj = obj.next_service 832 | chain.append(obj) 833 | return chain 834 | 835 | def object_mapping_list(self): 836 | 837 | return ",".join( self.object_mapping.values_list('name',flat=True) ) 838 | 839 | class ResourceMeta(AttachedMetadata): 840 | """ 841 | extensible metadata using rdf_io managed reusable generic metadata properties 842 | """ 843 | subject = models.ForeignKey("ImportedResource", models.PROTECT,related_name="metaprops") 844 | 845 | 846 | TYPE_RULE='RULE' 847 | TYPE_MODEL='CLASS' 848 | TYPE_INSTANCE='INSTANCE' 849 | TYPE_QUERY='QUERY' 850 | TYPE_VALIDATION='VALID' 851 | TYPE_CHOICES = ( 852 | ( TYPE_RULE, 'Rule (SPIN, SHACL, SKWRL etc)'), 853 | ( TYPE_MODEL, 'Class model - RDFS or OWL' ), 854 | ( TYPE_INSTANCE, 'Instance data - SKOS etc' ), 855 | ( TYPE_QUERY, 'Query template - SPARQL - for future use' ), 856 | ( TYPE_VALIDATION, 'Validation rule - for future use' ), 857 | ) 858 | @python_2_unicode_compatible 859 | class ImportedResource(models.Model): 860 | 861 | 862 | savedgraph = None 863 | 864 | subtype = models.ForeignKey(ContentType,models.PROTECT,editable=False,null=True,verbose_name='Specific Type') 865 | 866 | resource_type=models.CharField(choices=TYPE_CHOICES,default=TYPE_INSTANCE,max_length=10, 867 | help_text='Determines the post processing applied to the uploaded file') 868 | target_repo=models.ForeignKey(ServiceBinding, models.PROTECT, verbose_name='Data disposition',help_text='This is a service binding for the data object, in addition to any service bindings applied to the Imported Resource metadata.' , null=True, blank=True) 869 | description = models.CharField(verbose_name='ImportedResource Name',max_length=255, blank=True) 870 | file = models.FileField(upload_to='resources/',blank=True) 871 | remote = models.URLField(max_length=2000,blank=True,verbose_name='Remote RDF source URI') 872 | graph = models.URLField(max_length=2000,blank=True,null=True,verbose_name='Target RDF graph name') 873 | uploaded_at = models.DateTimeField(auto_now_add=True) 874 | # add per user details? 875 | 876 | def __unicode__(self): 877 | return ( ' '.join( [_f for _f in (self.resource_type,':', self.file.name, self.remote ) if _f])) 878 | 879 | def __str__(self): 880 | return str( self.__unicode__() ) 881 | 882 | # def clean(self): 883 | # import fields; pdb.set_trace() 884 | 885 | def delete(self,*args,**kwargs): 886 | if self.file and os.path.isfile(self.file.path): 887 | os.remove(self.file.path) 888 | if self.target_repo : 889 | logger.info("TODO - delete remote resource in repo %s" % self.target_repo) 890 | super(ImportedResource, self).delete(*args,**kwargs) 891 | 892 | def save(self,*args,**kwargs): 893 | #import pdb; pdb.set_trace() 894 | if(not self.subtype): 895 | self.subtype = ContentType.objects.get_for_model(self.__class__) 896 | if not self.description: 897 | self.description = self.__unicode__() 898 | self.savedgraph = None 899 | super(ImportedResource, self).save(*args,**kwargs) 900 | 901 | def get_publish_service(self): 902 | return [ self.target_repo ] 903 | 904 | def get_graph(self): 905 | # import pdb; pdb.set_trace() 906 | if self.savedgraph : 907 | pass # just return it 908 | elif self.file : 909 | format = rdflib.util.guess_format(self.file.name) 910 | self.savedgraph = rdflib.Graph().parse(self.file.name, format=format ) 911 | elif self.remote : 912 | format = rdflib.util.guess_format(self.remote) 913 | self.savedgraph = rdflib.Graph().parse(self.remote, format=format ) 914 | return self.savedgraph 915 | 916 | def getPathVal(self,gr,rootsubject,path): 917 | 918 | els = path.split() 919 | nels = len(els) 920 | idx = 1 921 | 922 | sparql="SELECT DISTINCT ?p_%s WHERE { <%s> %s ?p_%s ." % (nels,str(rootsubject), as_uri(els[0]), str(idx)) 923 | while idx < nels : 924 | sparql += " ?p_%s %s ?p_%s ." % (str(idx), as_uri(els[idx]), str(idx+1)) 925 | idx += 1 926 | sparql += " } " 927 | # print sparql 928 | results = gr.query(sparql) 929 | # check if a literal now! 930 | for res in results: 931 | return res[0] 932 | 933 | def publish(obj, model, oml, rdfstore=None , mode='PUBLISH'): 934 | """ build RDF graph and execute any postprocessing service chains 935 | 936 | Post processing may be tied to the object type - via Service Binding - or may be directly supported by the object type itself. 937 | Object specific services are executed in advance of general object type bindings - this allows for example uploaded resources to be installed before executing an inferencing chain. 938 | """ 939 | 940 | gr = Graph() 941 | # import pdb; pdb.set_trace() 942 | # ns_mgr = NamespaceManager(Graph()) 943 | # gr.namespace_manager = ns_mgr 944 | try: 945 | gr = build_rdf(gr, obj, oml, True) 946 | if not gr: 947 | return [] 948 | except Exception as e: 949 | logger.exception(sys.exc_info()[0]) 950 | raise Exception("Error during serialisation: " + str(e) ) 951 | 952 | # curl -X POST -H "Content-Type: text/turtle" -d @- http://192.168.56.151:8080/marmotta/import/upload?context=http://mapstory.org/def/featuretypes/gazetteer 953 | inference_chain_results = [] 954 | try: 955 | obj_chain = obj.get_publish_service() 956 | inference_chain_results = inference_chain_results + execute_service_chain(model,obj,mode, obj.get_graph() , obj_chain ) 957 | except: 958 | pass 959 | inference_chain_results = inference_chain_results + execute_service_chain(model,obj,mode, gr, ServiceBinding.get_service_bindings(model,None) ) 960 | 961 | def execute_service_chain(model,obj, mode, gr, chain): 962 | inference_chain_results = [] 963 | for next_binding in chain : 964 | newgr = gr # start off with original RDF graph for each new chain 965 | while next_binding : 966 | logger.info ( " -- ".join( (mode, str(obj), next_binding.__unicode__() ) ) ) 967 | if next_binding.binding_type == ServiceBinding.INFERENCE : 968 | newgr = inference(model, obj, next_binding, newgr, mode) 969 | elif next_binding.binding_type in ( ServiceBinding.PERSIST_UPDATE, ServiceBinding.PERSIST_REPLACE, ServiceBinding.PERSIST_CREATE ) : 970 | push_to_store( next_binding, model, obj, newgr , mode) 971 | elif next_binding.binding_type == ServiceBinding.PERSIST_PURGE : 972 | rdf_delete( next_binding, model, obj , mode) 973 | else: 974 | raise Exception( "service type not supported when post processing inferences") 975 | inference_chain_results.append(str( next_binding) ) 976 | next_binding = next_binding.next_service 977 | 978 | return inference_chain_results 979 | 980 | 981 | def build_rdf( gr,obj, oml, includemembers ) : 982 | 983 | # would be nice to add some comments : as metadata on the graph? '# Turtle generated by django-rdf-io configurable serializer\n' 984 | mappingsused = 0 985 | for om in oml : 986 | # check filter 987 | objfilter = getattr(om,'filter') 988 | if objfilter and not apply_pathfilter(obj, objfilter ) : 989 | continue 990 | mappingsused += 1 991 | try: 992 | tgt_id = getattr_path(obj,om.id_attr)[0] 993 | except (IndexError,ValueError) as e: 994 | raise ValueError("target id attribute {} not found".format( (om.id_attr ,))) 995 | if om.target_uri_expr[0] == '"' : 996 | uribase = om.target_uri_expr[1:-1] 997 | else: 998 | uribase = getattr_path(obj,om.target_uri_expr)[0] 999 | 1000 | tgt_id = str(tgt_id).replace(uribase,"") 1001 | # strip uri base if present in tgt_id 1002 | uribase = expand_curie(uribase) 1003 | 1004 | 1005 | if not tgt_id: 1006 | uri = uribase 1007 | elif uribase[-1] == '/' or uribase[-1] == '#' : 1008 | uri = "".join((uribase,tgt_id)) 1009 | else : 1010 | uri = "/".join((uribase,tgt_id)) 1011 | 1012 | subject = URIRef(uri) 1013 | 1014 | for omt in om.obj_type.all() : 1015 | gr.add( (subject, RDF.type , as_resource(gr,omt.uri)) ) 1016 | 1017 | # now get all the attribute mappings and add these in 1018 | for am in AttributeMapping.objects.filter(scope=om) : 1019 | if am.predicate[0] != ':' : 1020 | _add_vals(gr, obj, subject, am.predicate, am.attr , am.is_resource) 1021 | else: 1022 | for predicate,valuelist in getattr_tuple_path(obj,(am.predicate[1:],am.attr)): 1023 | for value in valuelist: 1024 | is_resource = value[0] == '<' and value[-1:] == '>' 1025 | if is_resource : 1026 | # brute force quote - ignoring any string @lang or ^^ type stuff quote() handles 1027 | value = value[1:-1].join(('"','"')) 1028 | else : 1029 | value = quote(value) 1030 | _add_vals(gr, obj, subject, str(predicate), value, is_resource) 1031 | 1032 | if includemembers: 1033 | for cm in ChainedMapping.objects.filter(scope=om) : 1034 | for val in getattr_path(obj,cm.attr): 1035 | try: 1036 | build_rdf( gr,val, (cm.chainedMapping,), includemembers ) 1037 | except: 1038 | logger.error( "Error serialising object %s as %s " % ( val, cm.attr )) 1039 | 1040 | for em in EmbeddedMapping.objects.filter(scope=om) : 1041 | try: 1042 | # three options - scalar value in which case attributes relative to basic obj, a mulitvalue obj or we have to look for related objects 1043 | try: 1044 | valuelist = getattr_path(obj,em.attr) 1045 | except: 1046 | valuelist = [obj,] 1047 | 1048 | for value in valuelist : 1049 | newnode = None 1050 | 1051 | for element in em.struct.split(";") : 1052 | try: 1053 | (predicate,expr) = element.split() 1054 | except: 1055 | predicate = None 1056 | expr = element 1057 | 1058 | # resolve any internal template parameters {x} 1059 | expr = expr.replace("{$URI}", uri ) 1060 | 1061 | is_resource = False 1062 | if expr.startswith("<") : 1063 | is_resource = True 1064 | expr = expr[1:-1].join(('"','"')) 1065 | elif expr.startswith("/") : 1066 | #value relativeto root obj - retrieve and use as literal 1067 | try: 1068 | expr = next(iter(getattr_path(obj,expr[1:]))) 1069 | if type(expr) == str : 1070 | expr = expr.join( ('"','"')) 1071 | except: 1072 | raise ValueError( "Could not access value of %s from mapped object %s (/ is relative to the object being mapped" % (expr,obj) ) 1073 | else: 1074 | is_resource = False 1075 | 1076 | for (lit,var,x,y) in Formatter().parse(expr) : 1077 | if var : 1078 | try: 1079 | if var.startswith("^"): 1080 | val = next(iter(getattr_path(obj,var[1:]))) 1081 | else: 1082 | val = next(iter(getattr_path(value,var))) 1083 | 1084 | if is_resource: 1085 | try: 1086 | val = u.urlencode({ 'v' : val.encode('utf-8')})[2:] 1087 | except: 1088 | #not a string, just pass 1089 | pass 1090 | val = str(val) 1091 | except: 1092 | val="{!variable not found : %s}" % var 1093 | expr = expr.replace(var.join(("{","}")), val ) 1094 | if predicate : 1095 | # an internal struct has been found so add a new node if not ye done 1096 | if not newnode: 1097 | newnode = BNode() 1098 | gr.add( (subject, as_resource(gr,em.predicate) , newnode) ) 1099 | _add_vals(gr, value, newnode, predicate, expr , is_resource) 1100 | else: 1101 | # add to parent 1102 | _add_vals(gr, value, subject, em.predicate, expr , is_resource) 1103 | except Exception as e: 1104 | #import traceback; import sys; traceback.print_exc() 1105 | logger.error("Could not evaluate extended mapping %s : %s " % (e,em.attr), sys.exc_info()) 1106 | raise ValueError("Could not evaluate extended mapping %s : %s " % (e,em.attr)) 1107 | # do this after looping through all object mappings! 1108 | return gr if mappingsused > 0 else None 1109 | 1110 | def _add_vals(gr, obj, subject, predicate, attr, is_resource ) : 1111 | if type(attr) == float or attr[0] in '\'\"' : # then a literal 1112 | gr.add( (subject, as_resource(gr,predicate) , makenode(gr,attr,is_resource) ) ) 1113 | else : 1114 | values = getattr_path(obj,attr) 1115 | for value in values : 1116 | if not value : 1117 | continue 1118 | gr.add( (subject, as_resource(gr,predicate) , makenode(gr,value,is_resource) ) ) 1119 | 1120 | 1121 | 1122 | 1123 | 1124 | --------------------------------------------------------------------------------