├── .gitignore ├── Makefile ├── Makefile-virtuoso ├── README ├── example_project ├── __init__.py ├── example_app │ ├── __init__.py │ ├── admin.py │ ├── fixtures │ │ ├── auth_user.json │ │ ├── fixture_base_programa.ttl │ │ ├── fixture_base_programa_to_virtuoso_access.ttl │ │ └── fixtures_aggregates.ttl │ ├── models.py │ ├── smodels.py │ ├── tests │ │ ├── __init__.py │ │ ├── functional │ │ │ ├── __init__.py │ │ │ └── test_base_programa.py │ │ ├── integration │ │ │ ├── __init__.py │ │ │ ├── test_aggregates.py │ │ │ └── test_base_programa.py │ │ └── unit │ │ │ ├── __init__.py │ │ │ └── test_base_programa.py │ └── views.py ├── manage.py ├── settings.py ├── settings_test.py └── urls.py ├── requirements.txt ├── semantic ├── __init__.py ├── management │ ├── __init__.py │ └── commands │ │ ├── __init__.py │ │ └── inspectdb_semantic.py ├── rdf │ ├── __init__.py │ ├── backends │ │ ├── __init__.py │ │ └── virtuoso │ │ │ ├── __init__.py │ │ │ ├── base.py │ │ │ ├── client.py │ │ │ ├── creation.py │ │ │ ├── dbapi.py │ │ │ └── instrospection.py │ ├── models │ │ ├── __init__.py │ │ ├── base.py │ │ ├── deletion.py │ │ ├── fields │ │ │ └── __init__.py │ │ ├── manager.py │ │ ├── options.py │ │ ├── query.py │ │ └── sparql │ │ │ ├── __init__.py │ │ │ ├── aggregates.py │ │ │ ├── compiler.py │ │ │ ├── constants.py │ │ │ ├── datastructures.py │ │ │ ├── expressions.py │ │ │ ├── query.py │ │ │ ├── subqueries.py │ │ │ └── where.py │ └── utils.py ├── settings_test.py ├── tests │ ├── __init__.py │ ├── functional │ │ └── __init__.py │ ├── integration │ │ └── __init__.py │ └── unit │ │ ├── __init__.py │ │ ├── simple_test.py │ │ ├── test_models.py │ │ ├── test_rdf_backend_base.py │ │ └── test_sparql_compiler.py └── views.py └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.bkp 3 | *.sw* 4 | .DS_Store 5 | example_project/example.db 6 | semantic/relational.db 7 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | 2 | clean: 3 | @find . -name "*.pyc" -delete 4 | 5 | setup: 6 | @pip install -r requirements.txt 7 | #@sudo chmod 777 /usr/local/virtuoso-opensource/var/lib/virtuoso/db/ 8 | 9 | test-example-project: 10 | @echo "Running example project tests" 11 | @cd example_project; PYTHONPATH='..' python manage.py test example_app --settings=example_project.settings_test 12 | @echo "----------" 13 | @echo 14 | 15 | test-semantic-app: 16 | @echo "Running semantic app tests". 17 | @cd semantic; DJANGO_SETTINGS_MODULE=semantic.settings_test nosetests --verbosity=3 -s . 18 | 19 | test-pep8: 20 | @pep8 example_project --ignore=E501,E126,E127,E128 21 | 22 | test: test-example-project test-semantic-app 23 | 24 | syncdb: 25 | @cd example_project; PYTHONPATH='..' python manage.py syncdb 26 | 27 | shell: 28 | @cd example_project; PYTHONPATH='..' python manage.py shell 29 | 30 | server: syncdb 31 | @cd example_project; PYTHONPATH='..' python manage.py runserver 32 | -------------------------------------------------------------------------------- /Makefile-virtuoso: -------------------------------------------------------------------------------- 1 | # 2 | # Makefile para ajudar na montagem de um ambiente local para o Virtuoso 3 | # e para subida local de ontologias 4 | # 5 | 6 | VIRTUOSO_VERSION := 6.1.6 7 | VIRTUOSO_HOME ?= /usr/local/virtuoso-opensource-$(VIRTUOSO_VERSION) 8 | VIRTUOSO_DB := $(VIRTUOSO_HOME)/var/lib/virtuoso/db 9 | VIRTUOSO_LOAD := $(VIRTUOSO_DB)/ontology 10 | VIRTUOSO_PACKAGE := virtuoso-opensource-$(VIRTUOSO_VERSION).tar.gz 11 | TEMP_DIR := tmp 12 | VIRTUOSO_BUILD_DIR := $(TEMP_DIR)/virtuoso-opensource-$(VIRTUOSO_VERSION) 13 | 14 | all: help 15 | 16 | help: 17 | @echo -n $(blue) 18 | @echo 'USE: make ' 19 | @echo -n $(normal) 20 | @echo '-------' 21 | @echo 'Targets' 22 | @echo '-------' 23 | @echo ' setup-host-ubuntu....................... Install Virtuoso on Ubuntu' 24 | @echo ' setup .................................. Setup Virtuoso' 25 | @echo ' run | start ............................ Start Virtuoso local server' 26 | @echo ' stop ................................... Stop Virtuoso local server' 27 | @echo ' status ................................. Show Virtuoso\'s status' 28 | @echo ' load | base_virtuoso ................... Load ontologys' 29 | @echo ' clean .................................. Remove Virtuoso local' 30 | @echo ' cleanlogs .............................. Remove log\'s files' 31 | 32 | $(TEMP_DIR)/$(VIRTUOSO_PACKAGE): 33 | mkdir -p $(TEMP_DIR) 34 | @echo Downloading the Virtuoso source $(VIRTUOSO_VERSION) 35 | wget -P $(TEMP_DIR) -c --content-disposition "http://sourceforge.net/projects/virtuoso/files/virtuoso/$(VIRTUOSO_VERSION)/$(VIRTUOSO_PACKAGE)/download" 36 | 37 | $(TEMP_DIR)/.configured: $(TEMP_DIR)/$(VIRTUOSO_PACKAGE) 38 | @echo Opening the Virtuoso package 39 | tar xzf $(TEMP_DIR)/$(VIRTUOSO_PACKAGE) -C $(TEMP_DIR) 40 | (cd $(VIRTUOSO_BUILD_DIR); \ 41 | ./configure \ 42 | --disable-bpel-vad \ 43 | --disable-demo-vad \ 44 | --disable-fct-vad \ 45 | --disable-ods-vad \ 46 | --disable-syncml-vad \ 47 | --disable-tutorial-vad \ 48 | --prefix=$(VIRTUOSO_HOME) \ 49 | ) 50 | touch $(TEMP_DIR)/.configured 51 | 52 | unpack: $(TEMP_DIR)/.configured 53 | 54 | $(TEMP_DIR)/.built: $(TEMP_DIR)/.configured 55 | @rm -f $(TEMP_DIR)/.built 56 | $(MAKE) -C $(VIRTUOSO_BUILD_DIR) 57 | @touch $(TEMP_DIR)/.built 58 | 59 | build: $(TEMP_DIR)/.built 60 | 61 | $(VIRTUOSO_HOME)/.installed: $(TEMP_DIR)/.built 62 | sudo $(MAKE) -C $(VIRTUOSO_BUILD_DIR) install 63 | sudo sed -i -e 's/^Load[0-9]/;&/' \ 64 | $(VIRTUOSO_HOME)/var/lib/virtuoso/db/virtuoso.ini 65 | sudo sed -i -e 's/^DirsAllowed.*/&, ./ontology/' \ 66 | $(VIRTUOSO_HOME)/var/lib/virtuoso/db/virtuoso.ini 67 | sudo mkdir -p $(VIRTUOSO_HOME)/var/lib/virtuoso/db/ontology 68 | sudo chmod a+rwx $(VIRTUOSO_HOME)/var/lib/virtuoso/db/ontology 69 | sudo touch $(VIRTUOSO_HOME)/.installed 70 | 71 | sudo cp $(VIRTUOSO_HOME)/var/lib/virtuoso/db/virtuoso.ini $(VIRTUOSO_HOME)/var/lib/virtuoso/db/virtuoso-test.ini 72 | sudo sed -i -e 's/8890/8891/g' \ 73 | $(VIRTUOSO_HOME)/var/lib/virtuoso/db/virtuoso-test.ini 74 | 75 | ifeq ($(origin VIRTUOSO_HOME), file) 76 | 77 | install: $(VIRTUOSO_HOME)/.installed 78 | 79 | else 80 | 81 | $(VIRTUOSO_HOME)/bin/virtuoso-t: 82 | @echo "EXTERNAL_VIRTUOSO esa definido, mas não encontrei $(VIRTUOSO_HOME)/bin/virtuoso-t" 83 | @false 84 | 85 | install: $(VIRTUOSO_HOME)/bin/virtuoso-t 86 | 87 | endif 88 | 89 | setup: install 90 | 91 | run: setup 92 | @echo "$(blue)Stating Virtuoso" 93 | cd $(VIRTUOSO_DB); sudo $(VIRTUOSO_HOME)/bin/virtuoso-t >/dev/null 2>&1 94 | 95 | run-test: setup 96 | @echo "$(blue)Stating Virtuoso" 97 | sudo $(VIRTUOSO_HOME)/bin/virtuoso-t +config $(VIRTUOSO_DB)/virtuoso-test.ini 98 | 99 | test: run-test 100 | @python example_project/manage.py test --settings=example_project.settings_test 101 | make stop 102 | 103 | start: run 104 | 105 | start-test: run-test 106 | 107 | stop: 108 | @echo "$(blue)Stoping Virtuoso" 109 | $(VIRTUOSO_HOME)/bin/isql -H localhost -U dba -P dba -K 2>/dev/null || true 110 | 111 | status: 112 | @if $(VIRTUOSO_HOME)/bin/isql localhost <<<'status();' &>/dev/null ; then \ 113 | echo "Started" ;\ 114 | else \ 115 | echo "Not started" ;\ 116 | fi 117 | 118 | load: 119 | cap LOADER_OPTS="$(LOADER_OPTS)" ambiente:local action:deploylocal 120 | 121 | base_virtuoso: load 122 | 123 | clean: 124 | @echo To remove a local Virtuoso's installatin, use \'make realclean\' 125 | 126 | ifeq ($(origin VIRTUOSO_HOME), file) 127 | 128 | realclean: stop 129 | @echo "$(blue)Removing $(TARGET_DIR) e $(VIRTUOSO_HOME)" 130 | @sleep 5 131 | sudo rm -rf $(VIRTUOSO_HOME) 132 | 133 | else 134 | 135 | realclean: 136 | @echo "$(blue)Your Virtuoso can't be removed, because it not have be installed with this Makefile" 137 | 138 | endif 139 | 140 | cleanlogs: 141 | rm $(VIRTUOSO_DB)/virtuoso.log 142 | 143 | setup-host-ubuntu: 144 | sudo apt-get update 145 | sudo apt-get install \ 146 | build-essential \ 147 | autoconf \ 148 | automake \ 149 | libtool \ 150 | make \ 151 | flex \ 152 | bison \ 153 | gawk \ 154 | gperf \ 155 | m4 \ 156 | libssl-dev \ 157 | libxml2-dev 158 | 159 | setup-host-mac: 160 | @for prog in xcodebuild flex bison gawk gperf m4; do \ 161 | which $$prog >/dev/null 2>&1 || (echo "Você precisa ter $$prog instalado para compilar o Virtuoso"; exit 1) \ 162 | done 163 | 164 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | Semantic-Django is a library for accessing RDF data from Python. It is based on Django models, so you can use the same methods you would use to query a relational database. 2 | 3 | Installing: 4 | 5 | pip install -r requirements.txt 6 | 7 | Currently being developed a Django backend compatible with OpenLink Virtuoso triplestore. 8 | 9 | Features: 10 | 11 | - Integration with Django admin for CRUD operations 12 | - Save 13 | - Update (Not yet) 14 | - Delete 15 | - Aggregate functions (COUNT, AVG, MIN, MAX, SUM) 16 | -------------------------------------------------------------------------------- /example_project/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rfloriano/semantic-django/f35747887507ca6aa08a9bf528c706a214382162/example_project/__init__.py -------------------------------------------------------------------------------- /example_project/example_app/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rfloriano/semantic-django/f35747887507ca6aa08a9bf528c706a214382162/example_project/example_app/__init__.py -------------------------------------------------------------------------------- /example_project/example_app/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | 3 | from example_project.example_app.smodels import BasePrograma 4 | 5 | 6 | class BaseProgramaAdmin(admin.ModelAdmin): 7 | ordering = ['uri'] 8 | 9 | admin.site.register(BasePrograma, BaseProgramaAdmin) 10 | -------------------------------------------------------------------------------- /example_project/example_app/fixtures/auth_user.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "pk": 1, 4 | "model": "auth.user", 5 | "fields": { 6 | "username": "super", 7 | "first_name": "", 8 | "last_name": "", 9 | "is_active": true, 10 | "is_superuser": true, 11 | "is_staff": true, 12 | "last_login": "2012-10-18 09:37:03", 13 | "groups": [], 14 | "user_permissions": [], 15 | "password": "sha1$b73bb$4cf6d53a063679b1cb929c8be0d8172a6d9daaa4", 16 | "email": "root@root.om", 17 | "date_joined": "2012-10-18 09:36:56" 18 | } 19 | } 20 | ] -------------------------------------------------------------------------------- /example_project/example_app/fixtures/fixture_base_programa.ttl: -------------------------------------------------------------------------------- 1 | ; 2 | "Rock in Rio" ; 3 | ; 4 | 5116 . -------------------------------------------------------------------------------- /example_project/example_app/fixtures/fixture_base_programa_to_virtuoso_access.ttl: -------------------------------------------------------------------------------- 1 | ; 2 | "One Program" ; 3 | ; 4 | 123 . -------------------------------------------------------------------------------- /example_project/example_app/fixtures/fixtures_aggregates.ttl: -------------------------------------------------------------------------------- 1 | ; 2 | "Rock in Rio" ; 3 | ; 4 | 5116 . 5 | 6 | ; 7 | "Lollapalooza" ; 8 | ; 9 | 5117 . 10 | -------------------------------------------------------------------------------- /example_project/example_app/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | # Create your models here. 4 | -------------------------------------------------------------------------------- /example_project/example_app/smodels.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | # This is an auto-generated Semantic Globo model module. 3 | # You'll have to do the following manually to clean this up: 4 | # * Rearrange models' order 5 | # Feel free to rename the models, but don't rename semantic_graph values or field names. 6 | # 7 | 8 | from semantic.rdf import models 9 | 10 | 11 | class BasePrograma(models.SemanticModel): 12 | label = models.CharField(graph='rdfs', max_length=200) 13 | foto_perfil = models.CharField(graph='base', max_length=200, blank=True) 14 | id_do_programa_na_webmedia = models.IntegerField(graph='base') 15 | faz_parte_do_canal = models.URIField(graph='base') 16 | tem_edicao_do_programa = models.CharField(graph='base', max_length=200, blank=True) 17 | 18 | class Meta: 19 | # FIXME: Remove this abstract property 20 | # abstract = True 21 | managed = False 22 | graph = 'http://semantica.globo.com/' 23 | namespace = 'http://semantica.globo.com/base/Programa' 24 | 25 | def __unicode__(self): 26 | return self.uri 27 | 28 | -------------------------------------------------------------------------------- /example_project/example_app/tests/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /example_project/example_app/tests/functional/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rfloriano/semantic-django/f35747887507ca6aa08a9bf528c706a214382162/example_project/example_app/tests/functional/__init__.py -------------------------------------------------------------------------------- /example_project/example_app/tests/functional/test_base_programa.py: -------------------------------------------------------------------------------- 1 | from semantic.tests import SemanticTestCase 2 | 3 | 4 | class TestBasePrograma(SemanticTestCase): 5 | fixtures = ["example_app/fixtures/auth_user.json"] 6 | semantic_fixtures = ["example_app/fixtures/fixture_base_programa.ttl"] 7 | 8 | def setUp(self): 9 | self.client.login(username='super', password='secret') 10 | 11 | def tearDown(self): 12 | self.client.logout() 13 | 14 | def test_if_can_edit_a_baseprograma_objects_in_admin(self): 15 | response = self.client.get('/admin/example_app/baseprograma/http_3A_2F_2Fsemantica.globo.com_2Fbase_2FPrograma_5FRock_5Fin_5FRio/') 16 | self.assertContains(response, '') 17 | self.assertContains(response, '') 18 | self.assertContains(response, '') 19 | self.assertContains(response, '') 20 | self.assertContains(response, '') 21 | self.assertContains(response, '') 22 | 23 | def test_if_can_add_baseprograma_objects_in_admin(self): 24 | response = self.client.get('/admin/example_app/baseprograma/add/') 25 | self.assertContains(response, '') 26 | self.assertContains(response, '') 27 | self.assertContains(response, '') 28 | self.assertContains(response, '') 29 | self.assertContains(response, '') 30 | self.assertContains(response, '') 31 | 32 | def test_field_label_has_label_label(self): 33 | # regression test related to bug introduced in b1bb72e6f386bdf25e35d69d01ef1da417e4b20e 34 | response = self.client.get('/admin/example_app/baseprograma/add/') 35 | self.assertContains(response, '') 36 | -------------------------------------------------------------------------------- /example_project/example_app/tests/integration/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rfloriano/semantic-django/f35747887507ca6aa08a9bf528c706a214382162/example_project/example_app/tests/integration/__init__.py -------------------------------------------------------------------------------- /example_project/example_app/tests/integration/test_aggregates.py: -------------------------------------------------------------------------------- 1 | from semantic.tests import SemanticTestCase 2 | 3 | from example_app.smodels import BasePrograma 4 | 5 | 6 | class TestAggregates(SemanticTestCase): 7 | allow_virtuoso_connection = True 8 | semantic_fixtures = ["example_app/fixtures/fixtures_aggregates.ttl"] 9 | 10 | def test_count(self): 11 | expected = 2 12 | actual = BasePrograma.objects.count() 13 | self.assertEquals(expected, actual) 14 | -------------------------------------------------------------------------------- /example_project/example_app/tests/integration/test_base_programa.py: -------------------------------------------------------------------------------- 1 | from semantic.tests import SemanticTestCase 2 | 3 | from example_project.example_app.smodels import BasePrograma 4 | 5 | 6 | class TestBaseProgramaVirtuoso(SemanticTestCase): 7 | fixtures = ["example_app/fixtures/auth_user.json"] 8 | semantic_fixtures = [ 9 | "example_app/fixtures/fixture_base_programa.ttl", 10 | "example_app/fixtures/fixture_base_programa_to_virtuoso_access.ttl" 11 | ] 12 | allow_virtuoso_connection = True 13 | 14 | def setUp(self): 15 | self.client.login(username='super', password='secret') 16 | 17 | def tearDown(self): 18 | self.client.logout() 19 | 20 | def test_filter_from_uri_with_exact(self): 21 | programa = BasePrograma.objects.filter( 22 | uri='http://semantica.globo.com/base/Programa_OneProgram' 23 | ) 24 | self.assertEqual(len(programa), 1) 25 | 26 | def test_object_creation_with_required_data(self): 27 | programa = BasePrograma.objects.create( 28 | uri='http://semantica.globo.com/base/Programa_OneProgram', 29 | label="One Program", 30 | id_do_programa_na_webmedia="123", 31 | faz_parte_do_canal='http://semantica.globo.com/base/Canal_MeuCanal' 32 | ) 33 | self.assertTrue(programa) 34 | 35 | def test_object_delete(self): 36 | programa = BasePrograma.objects.create( 37 | uri='http://semantica.globo.com/base/Programa_AnotherProgram', 38 | label="Another Program", 39 | id_do_programa_na_webmedia="123", 40 | faz_parte_do_canal='http://semantica.globo.com/base/Canal_MeuCanal' 41 | ) 42 | programas = BasePrograma.objects.filter( 43 | uri='http://semantica.globo.com/base/Programa_AnotherProgram' 44 | ) 45 | self.assertEqual(len(programas), 1) 46 | programa.delete() 47 | programas = BasePrograma.objects.filter( 48 | uri='http://semantica.globo.com/base/Programa_AnotherProgram' 49 | ) 50 | self.assertEqual(len(programas), 0) 51 | 52 | def test_object_update(self): 53 | programa = BasePrograma.objects.get( 54 | uri='http://semantica.globo.com/base/Programa_OneProgram' 55 | ) 56 | programa.label = "One ProgramB" 57 | programa.save() 58 | programa = BasePrograma.objects.get( 59 | uri='http://semantica.globo.com/base/Programa_OneProgram' 60 | ) 61 | self.assertEqual(programa.label, "One ProgramB") 62 | 63 | def test_add_a_baseprograma_object_by_admin_and_need_find_in_admin_baseprograma_list(self): 64 | """Regression test""" 65 | 66 | self.client.post('/admin/example_app/baseprograma/add/', { 67 | 'uri': 'http://semantica.globo.com/base/Programa_MyProgram', 68 | 'label': 'My Program', 69 | 'faz_parte_do_canal': 'http://semantica.globo.com/base/Canal_MyChannel', 70 | 'id_do_programa_na_webmedia': '123', 71 | '_save': 'Save', 72 | }) 73 | response = self.client.get('/admin/example_app/baseprograma/') 74 | self.assertContains(response, 'http://semantica.globo.com/base/Programa_MyProgram') 75 | 76 | def test_object_creation_with_required_data_and_without_uri(self): 77 | """Regression test""" 78 | program = BasePrograma.objects.create( 79 | id_do_programa_na_webmedia="123", 80 | faz_parte_do_canal='http://semantica.globo.com/base/Canal_MeuCanal' 81 | ) 82 | self.assertTrue(program) 83 | self.assertIn('semantica.globo.com', program.uri) 84 | 85 | def test_dont_fill_empty_fields(self): 86 | """ Regression test for issue #1 """ 87 | programa = BasePrograma.objects.create( 88 | label='Lollapalloza', 89 | id_do_programa_na_webmedia=1, 90 | faz_parte_do_canal="http://semantica.globo.com/base/Canal_Test" 91 | ) 92 | 93 | self.assertFalse(programa.foto_perfil) 94 | self.assertFalse(programa.tem_edicao_do_programa) 95 | -------------------------------------------------------------------------------- /example_project/example_app/tests/unit/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rfloriano/semantic-django/f35747887507ca6aa08a9bf528c706a214382162/example_project/example_app/tests/unit/__init__.py -------------------------------------------------------------------------------- /example_project/example_app/tests/unit/test_base_programa.py: -------------------------------------------------------------------------------- 1 | from semantic.tests import SemanticTestCase 2 | 3 | from example_project.example_app.smodels import BasePrograma 4 | 5 | 6 | class TestBasePrograma(SemanticTestCase): 7 | semantic_fixtures = ["example_app/fixtures/fixture_base_programa.ttl"] 8 | 9 | def test_filter_from_uri_with_exact(self): 10 | programas = BasePrograma.objects.filter(uri='http://semantica.globo.com/base/Programa_Rock_in_Rio') 11 | self.assertEqual(len(programas), 1) 12 | 13 | def test_get_from_uri(self): 14 | rock_in_rio = BasePrograma.objects.get(uri='http://semantica.globo.com/base/Programa_Rock_in_Rio') 15 | self.assertEqual(rock_in_rio.faz_parte_do_canal, u'http://semantica.globo.com/base/Canal_Multishow') 16 | self.assertEqual(rock_in_rio.foto_perfil, '') 17 | self.assertEqual(rock_in_rio.id_do_programa_na_webmedia, u'5116') 18 | self.assertEqual(rock_in_rio.label, u'Rock in Rio') 19 | self.assertEqual(rock_in_rio.tem_edicao_do_programa, '') 20 | self.assertEqual(rock_in_rio.uri, u'http://semantica.globo.com/base/Programa_Rock_in_Rio') 21 | 22 | def test_get_from_label_with_filter_startswith(self): 23 | programas = BasePrograma.objects.filter(label__startswith='Rock') 24 | self.assertEqual(len(programas), 1) 25 | 26 | def test_get_from_label_with_filter_endswith(self): 27 | programas = BasePrograma.objects.filter(label__endswith='Rio') 28 | self.assertEqual(len(programas), 1) 29 | 30 | def test_get_from_label_with_filter_istartswith(self): 31 | programas = BasePrograma.objects.filter(label__istartswith='rock') 32 | self.assertEqual(len(programas), 1) 33 | 34 | def test_get_from_label_with_filter_iendswith(self): 35 | programas = BasePrograma.objects.filter(label__iendswith='rio') 36 | self.assertEqual(len(programas), 1) 37 | 38 | def test_get_from_label_with_filter_greatest(self): 39 | programas = BasePrograma.objects.filter(id_do_programa_na_webmedia__gt='5115') 40 | self.assertEqual(len(programas), 1) 41 | 42 | def test_get_from_label_with_filter_greatest_or_equal(self): 43 | programas = BasePrograma.objects.filter(id_do_programa_na_webmedia__gte='5116') 44 | self.assertEqual(len(programas), 1) 45 | 46 | def test_get_from_label_with_filter_less_than(self): 47 | programas = BasePrograma.objects.filter(id_do_programa_na_webmedia__lt='5117') 48 | self.assertEqual(len(programas), 1) 49 | 50 | def test_get_from_label_with_filter_less_than_or_equal(self): 51 | programas = BasePrograma.objects.filter(id_do_programa_na_webmedia__lte='5116') 52 | self.assertEqual(len(programas), 1) 53 | 54 | def test_get_from_label_with_filter_exact(self): 55 | programas = BasePrograma.objects.filter(label__exact='Rock in Rio') 56 | self.assertEqual(len(programas), 1) 57 | 58 | def test_get_from_label_with_filter_iexact(self): 59 | programas = BasePrograma.objects.filter(label__iexact='rOcK iN rIo') 60 | self.assertEqual(len(programas), 1) 61 | 62 | def test_get_from_label_with_filter_regex(self): 63 | programas = BasePrograma.objects.filter(label__regex='Rock') 64 | self.assertEqual(len(programas), 1) 65 | 66 | def test_get_from_label_with_filter_iregex(self): 67 | programas = BasePrograma.objects.filter(label__iregex='rock') 68 | self.assertEqual(len(programas), 1) 69 | 70 | def test_get_from_label_with_filter_contains(self): 71 | programas = BasePrograma.objects.filter(label__contains='ck in Rio') 72 | self.assertEqual(len(programas), 1) 73 | 74 | def test_get_from_label_with_filter_icontains(self): 75 | programas = BasePrograma.objects.filter(label__icontains='ck in rio') 76 | self.assertEqual(len(programas), 1) 77 | 78 | 79 | -------------------------------------------------------------------------------- /example_project/example_app/views.py: -------------------------------------------------------------------------------- 1 | # Create your views here. 2 | -------------------------------------------------------------------------------- /example_project/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from django.core.management import execute_manager 3 | import imp 4 | try: 5 | imp.find_module('settings') # Assumed to be in the same directory. 6 | except ImportError: 7 | import sys 8 | sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n" % __file__) 9 | sys.exit(1) 10 | 11 | import settings 12 | 13 | if __name__ == "__main__": 14 | execute_manager(settings) 15 | -------------------------------------------------------------------------------- /example_project/settings.py: -------------------------------------------------------------------------------- 1 | # Django settings for example_project project. 2 | 3 | DEBUG = True 4 | TEMPLATE_DEBUG = DEBUG 5 | 6 | ADMINS = ( 7 | # ('Your Name', 'your_email@example.com'), 8 | ) 9 | 10 | MANAGERS = ADMINS 11 | 12 | DATABASES = { 13 | 'default': { 14 | 'ENGINE': 'django.db.backends.sqlite3', # Add 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'. 15 | 'NAME': 'example.db', # Or path to database file if using sqlite3. 16 | 'USER': '', # Not used with sqlite3. 17 | 'PASSWORD': '', # Not used with sqlite3. 18 | 'HOST': '', # Set to empty string for localhost. Not used with sqlite3. 19 | 'PORT': '', # Set to empty string for default. Not used with sqlite3. 20 | } 21 | } 22 | 23 | SEMANTIC_DATABASES = { 24 | 'default': { 25 | 'ENGINE': 'semantic.rdf.backends.virtuoso', 26 | 'NAME': 'sparql', 27 | 'USER': 'dba', 28 | 'PASSWORD': 'dba', 29 | 'HOST': 'localhost', 30 | 'PORT': '8890', 31 | 'PREFIX': { 32 | 'base': '', 33 | 'g1': '', 34 | } 35 | } 36 | } 37 | 38 | # Local time zone for this installation. Choices can be found here: 39 | # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name 40 | # although not all choices may be available on all operating systems. 41 | # On Unix systems, a value of None will cause Django to use the same 42 | # timezone as the operating system. 43 | # If running in a Windows environment this must be set to the same as your 44 | # system time zone. 45 | TIME_ZONE = 'America/Chicago' 46 | 47 | # Language code for this installation. All choices can be found here: 48 | # http://www.i18nguy.com/unicode/language-identifiers.html 49 | LANGUAGE_CODE = 'en-us' 50 | 51 | SITE_ID = 1 52 | 53 | # If you set this to False, Django will make some optimizations so as not 54 | # to load the internationalization machinery. 55 | USE_I18N = True 56 | 57 | # If you set this to False, Django will not format dates, numbers and 58 | # calendars according to the current locale 59 | USE_L10N = True 60 | 61 | # Absolute filesystem path to the directory that will hold user-uploaded files. 62 | # Example: "/home/media/media.lawrence.com/media/" 63 | MEDIA_ROOT = '' 64 | 65 | # URL that handles the media served from MEDIA_ROOT. Make sure to use a 66 | # trailing slash. 67 | # Examples: "http://media.lawrence.com/media/", "http://example.com/media/" 68 | MEDIA_URL = '' 69 | 70 | # Absolute path to the directory static files should be collected to. 71 | # Don't put anything in this directory yourself; store your static files 72 | # in apps' "static/" subdirectories and in STATICFILES_DIRS. 73 | # Example: "/home/media/media.lawrence.com/static/" 74 | STATIC_ROOT = '' 75 | 76 | # URL prefix for static files. 77 | # Example: "http://media.lawrence.com/static/" 78 | STATIC_URL = '/static/' 79 | 80 | # URL prefix for admin static files -- CSS, JavaScript and images. 81 | # Make sure to use a trailing slash. 82 | # Examples: "http://foo.com/static/admin/", "/static/admin/". 83 | ADMIN_MEDIA_PREFIX = '/static/admin/' 84 | 85 | # Additional locations of static files 86 | STATICFILES_DIRS = ( 87 | # Put strings here, like "/home/html/static" or "C:/www/django/static". 88 | # Always use forward slashes, even on Windows. 89 | # Don't forget to use absolute paths, not relative paths. 90 | ) 91 | 92 | # List of finder classes that know how to find static files in 93 | # various locations. 94 | STATICFILES_FINDERS = ( 95 | 'django.contrib.staticfiles.finders.FileSystemFinder', 96 | 'django.contrib.staticfiles.finders.AppDirectoriesFinder', 97 | # 'django.contrib.staticfiles.finders.DefaultStorageFinder', 98 | ) 99 | 100 | # Make this unique, and don't share it with anybody. 101 | SECRET_KEY = 'h7y1cjy#=(h0h(-ejlhe4rnw7954gbo#jw6hm^k(=rdz*fuuuw' 102 | 103 | # List of callables that know how to import templates from various sources. 104 | TEMPLATE_LOADERS = ( 105 | 'django.template.loaders.filesystem.Loader', 106 | 'django.template.loaders.app_directories.Loader', 107 | # 'django.template.loaders.eggs.Loader', 108 | ) 109 | 110 | MIDDLEWARE_CLASSES = ( 111 | 'django.middleware.common.CommonMiddleware', 112 | 'django.contrib.sessions.middleware.SessionMiddleware', 113 | 'django.middleware.csrf.CsrfViewMiddleware', 114 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 115 | 'django.contrib.messages.middleware.MessageMiddleware', 116 | ) 117 | 118 | ROOT_URLCONF = 'example_project.urls' 119 | 120 | TEMPLATE_DIRS = ( 121 | # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates". 122 | # Always use forward slashes, even on Windows. 123 | # Don't forget to use absolute paths, not relative paths. 124 | ) 125 | 126 | INSTALLED_APPS = ( 127 | 'django.contrib.auth', 128 | 'django.contrib.contenttypes', 129 | 'django.contrib.sessions', 130 | 'django.contrib.sites', 131 | 'django.contrib.messages', 132 | 'django.contrib.staticfiles', 133 | # Uncomment the next line to enable the admin: 134 | 'django.contrib.admin', 135 | # Uncomment the next line to enable admin documentation: 136 | # 'django.contrib.admindocs', 137 | 'semantic', 138 | 'example_app', 139 | ) 140 | 141 | # A sample logging configuration. The only tangible logging 142 | # performed by this configuration is to send an email to 143 | # the site admins on every HTTP 500 error. 144 | # See http://docs.djangoproject.com/en/dev/topics/logging for 145 | # more details on how to customize your logging configuration. 146 | LOGGING = { 147 | 'version': 1, 148 | 'disable_existing_loggers': False, 149 | 'handlers': { 150 | 'mail_admins': { 151 | 'level': 'ERROR', 152 | 'class': 'django.utils.log.AdminEmailHandler' 153 | } 154 | }, 155 | 'loggers': { 156 | 'django.request': { 157 | 'handlers': ['mail_admins'], 158 | 'level': 'ERROR', 159 | 'propagate': True, 160 | }, 161 | } 162 | } 163 | 164 | -------------------------------------------------------------------------------- /example_project/settings_test.py: -------------------------------------------------------------------------------- 1 | from settings import * 2 | 3 | INSTALLED_APPS += ('django_nose',) 4 | 5 | TESTING = True 6 | TEST_SEMANTIC_GRAPH = "http://testgraph.globo.com" 7 | 8 | TEST_RUNNER = 'django_nose.NoseTestSuiteRunner' 9 | 10 | NOSE_ARGS = ['-s', '--verbosity=3'] 11 | -------------------------------------------------------------------------------- /example_project/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls.defaults import patterns, include, url 2 | 3 | # Uncomment the next two lines to enable the admin: 4 | from django.contrib import admin 5 | admin.autodiscover() 6 | 7 | urlpatterns = patterns('', 8 | # Examples: 9 | # url(r'^$', 'example_project.views.home', name='home'), 10 | # url(r'^example_project/', include('example_project.foo.urls')), 11 | 12 | # Uncomment the admin/doc line below to enable admin documentation: 13 | # url(r'^admin/doc/', include('django.contrib.admindocs.urls')), 14 | 15 | # Uncomment the next line to enable the admin: 16 | url(r'^admin/', include(admin.site.urls)), 17 | ) 18 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | django==1.3 2 | rdflib==3.2.1 3 | rdfextras==0.3 4 | isodate==0.4.8 5 | git+git://github.com/tatiana/SPARQL-Wrapper.git#egg=SPARQL-Wrapper 6 | 7 | # this could go in a separate requirements later on 8 | nose==1.2.1 9 | django_nose==1.1 10 | mocker==1.1.1 11 | pep8==1.3.3 -------------------------------------------------------------------------------- /semantic/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rfloriano/semantic-django/f35747887507ca6aa08a9bf528c706a214382162/semantic/__init__.py -------------------------------------------------------------------------------- /semantic/management/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rfloriano/semantic-django/f35747887507ca6aa08a9bf528c706a214382162/semantic/management/__init__.py -------------------------------------------------------------------------------- /semantic/management/commands/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rfloriano/semantic-django/f35747887507ca6aa08a9bf528c706a214382162/semantic/management/commands/__init__.py -------------------------------------------------------------------------------- /semantic/management/commands/inspectdb_semantic.py: -------------------------------------------------------------------------------- 1 | #-*- coding:utf-8 -*- 2 | import sys 3 | import codecs 4 | from optparse import make_option 5 | 6 | from django.core.management.base import BaseCommand, CommandError 7 | 8 | from semantic.rdf import Virtuoso 9 | 10 | # Hack to encode stdout 11 | sys.stdout = codecs.getwriter('utf8')(sys.stdout) 12 | 13 | 14 | class Command(BaseCommand): 15 | help = "Introspects the database tables in the given database and outputs a Django model module." 16 | 17 | option_list = BaseCommand.option_list + ( 18 | make_option('--graph', action='store', dest='graph', 19 | help='Nominates a graph to semantic introspect.'), 20 | ) 21 | 22 | requires_model_validation = False 23 | 24 | rdf_module = 'semantic.rdf' 25 | virtuoso = Virtuoso() 26 | graph = '' 27 | 28 | def handle(self, **options): 29 | self.graph = options.get('graph', None) 30 | 31 | if not self.graph: 32 | raise CommandError("A graph need to be specified to inspection") 33 | 34 | self.graph = options['graph'] 35 | if not self.graph.endswith("/"): 36 | self.graph += "/" 37 | 38 | query = """ 39 | SELECT DISTINCT(?class) WHERE { 40 | { 41 | ?class rdf:type . 42 | ?class rdfs:subClassOf ?parent. 43 | } union { 44 | ?class rdfs:comment ?comment 45 | } 46 | FILTER REGEX (?class, '%(graph)s') 47 | } 48 | """ % {"graph": self.graph} 49 | results = self.get_results(query) 50 | 51 | class_data = {} 52 | for result in results: 53 | klass = self.get_or_blank(result, 'class') 54 | data = self.get_data_class(klass) 55 | properties = self.get_class_properties(klass) 56 | 57 | class_data[klass] = { 58 | 'data': data, 59 | 'properties': properties, 60 | } 61 | 62 | for line in self.make_model(class_data): 63 | self.stdout.write("%s\n".encode('utf-8') % line) 64 | 65 | def get_or_blank(self, data, key): 66 | try: 67 | return data[key]['value'] 68 | except KeyError: 69 | return '' 70 | 71 | def get_results(self, query): 72 | return self.virtuoso._query(query)['results']['bindings'] 73 | 74 | def get_data_class(self, klass): 75 | query = """ 76 | SELECT * WHERE { 77 | <%(class)s> rdfs:subClassOf ?parent . 78 | OPTIONAL { <%(class)s> rdfs:comment ?comment } 79 | FILTER REGEX (?parent, '%(graph)s') 80 | } 81 | """ % {'graph': self.graph, 'class': klass} 82 | return self.get_results(query) 83 | 84 | def get_parent_graph_and_parent_class(self, uri): 85 | parent_graph = '' 86 | parent_class = '' 87 | if self.graph in uri: 88 | uri = uri.replace(uri, '') 89 | parent_graph, parent_class = uri.split('/') 90 | return parent_graph, parent_class 91 | 92 | def get_class_properties(self, klass): 93 | query = """ 94 | SELECT * WHERE { 95 | ?property rdfs:domain <%(class)s> 96 | } 97 | """ % {'class': klass} 98 | return self.get_results(query) 99 | 100 | def get_name_from_uri(self, uri): 101 | path = uri.replace(self.graph, '').strip('/') 102 | name = path \ 103 | .replace('/', '') \ 104 | .replace(' ', '') \ 105 | .replace('_', '') \ 106 | .replace('-', '') 107 | return self.capfirst(name) 108 | 109 | def get_field_property(self, prop): 110 | splited_uri = prop['value'].split('/') 111 | name = splited_uri.pop() 112 | return name, self.capfirst(prop['type']) 113 | 114 | def get_meta(self, graph): 115 | return [' class Meta:', 116 | ' semantic_graph = %r' % graph, 117 | ''] 118 | 119 | def get_parent(self, parent_uri): 120 | return self.get_name_from_uri(parent_uri) 121 | 122 | def capfirst(self, value): 123 | return value and value[0].upper() + value[1:] 124 | 125 | def make_model(self, data): 126 | yield "#-*- coding:utf-8 -*-" 127 | yield "# This is an auto-generated Semantic Globo model module." 128 | yield "# You'll have to do the following manually to clean this up:" 129 | yield "# * Rearrange models' order" 130 | yield "# Feel free to rename the models, but don't rename semantic_graph values or field names." 131 | yield "#" 132 | yield '' 133 | yield 'from %s import models' % self.rdf_module 134 | yield '' 135 | yield '' 136 | for graph_uri, data in data.items(): 137 | klass_name = self.get_name_from_uri(graph_uri) 138 | 139 | if not klass_name: 140 | continue 141 | 142 | parent_class = 'models.SemanticModel' 143 | if len(data['data']) > 0 and 'parent' in data['data'][0]: 144 | parent_class = self.get_parent(self.get_or_blank(data['data'][0], 'parent')) 145 | 146 | yield 'class %s(%s):' % (klass_name, parent_class) 147 | 148 | if len(data['data']) > 0 and 'comment' in data['data'][0]: 149 | yield ' """%s"""' % self.get_or_blank(data['data'][0], 'comment') 150 | 151 | if not data['properties']: 152 | yield ' pass' 153 | else: 154 | for prop in data['properties']: 155 | yield ' %s = models.%sField()' % (self.get_field_property(prop['property'])) 156 | 157 | yield '' 158 | 159 | for l in self.get_meta(graph_uri): 160 | yield l 161 | -------------------------------------------------------------------------------- /semantic/rdf/__init__.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | 3 | from semantic.rdf.utils import ConnectionHandler 4 | # from semantic.rdf.backends.virtuoso import Virtuoso 5 | 6 | connections = ConnectionHandler(settings.SEMANTIC_DATABASES) 7 | -------------------------------------------------------------------------------- /semantic/rdf/backends/__init__.py: -------------------------------------------------------------------------------- 1 | from django.db.backends import BaseDatabaseOperations 2 | from django.utils.importlib import import_module 3 | 4 | 5 | class BaseSemanticDatabaseOperations(BaseDatabaseOperations): 6 | compiler_module = "semantic.rdf.models.sparql.compiler" 7 | 8 | def compiler(self, compiler_name): 9 | """ 10 | Returns the SPARQLCompiler class corresponding to the given name, 11 | in the namespace corresponding to the `compiler_module` attribute 12 | on this backend. 13 | """ 14 | if self._cache is None: 15 | self._cache = import_module(self.compiler_module) 16 | return getattr(self._cache, compiler_name) 17 | -------------------------------------------------------------------------------- /semantic/rdf/backends/virtuoso/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rfloriano/semantic-django/f35747887507ca6aa08a9bf528c706a214382162/semantic/rdf/backends/virtuoso/__init__.py -------------------------------------------------------------------------------- /semantic/rdf/backends/virtuoso/base.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | from django.db import utils 4 | from django.db.backends import * 5 | from django.db.backends.signals import connection_created 6 | from django.db.backends.sqlite3.client import DatabaseClient 7 | from django.db.backends.sqlite3.creation import DatabaseCreation 8 | from django.db.backends.sqlite3.introspection import DatabaseIntrospection 9 | from django.utils.encoding import smart_unicode 10 | 11 | from SPARQLWrapper import SPARQLWrapper as Database 12 | from SPARQLWrapper import JSON 13 | 14 | from semantic.rdf.backends import BaseSemanticDatabaseOperations 15 | from semantic.rdf.backends.virtuoso.dbapi import Cursor 16 | 17 | 18 | class DatabaseFeatures(BaseDatabaseFeatures): 19 | # SPARQLite cannot handle us only partially reading from a cursor's result set 20 | # and then writing the same rows to the database in another cursor. This 21 | # setting ensures we always read result sets fully into memory all in one 22 | # go. 23 | can_use_chunked_reads = False 24 | test_db_allows_multiple_connections = False 25 | supports_unspecified_pk = True 26 | supports_1000_query_parameters = False 27 | supports_mixed_date_datetime_comparisons = False 28 | 29 | def _supports_stddev(self): 30 | """Confirm support for STDDEV and related stats functions 31 | 32 | SPARQLite supports STDDEV as an extension package; so 33 | connection.ops.check_aggregate_support() can't unilaterally 34 | rule out support for STDDEV. We need to manually check 35 | whether the call works. 36 | """ 37 | cursor = self.connection.cursor() 38 | cursor.execute('CREATE TABLE STDDEV_TEST (X INT)') 39 | try: 40 | cursor.execute('SELECT STDDEV(*) FROM STDDEV_TEST') 41 | has_support = True 42 | except utils.DatabaseError: 43 | has_support = False 44 | cursor.execute('DROP TABLE STDDEV_TEST') 45 | return has_support 46 | 47 | 48 | class DatabaseOperations(BaseSemanticDatabaseOperations): 49 | def date_extract_sparql(self, lookup_type, field_name): 50 | # sparqlite doesn't support extract, so we fake it with the user-defined 51 | # function django_extract that's registered in connect(). Note that 52 | # single quotes are used because this is a string (and could otherwise 53 | # cause a collision with a field name). 54 | return "django_extract('%s', %s)" % (lookup_type.lower(), field_name) 55 | 56 | def date_interval_sparql(self, sparql, connector, timedelta): 57 | # It would be more straightforward if we could use the sparqlite strftime 58 | # function, but it does not allow for keeping six digits of fractional 59 | # second information, nor does it allow for formatting date and datetime 60 | # values differently. So instead we register our own function that 61 | # formats the datetime combined with the delta in a manner suitable 62 | # for comparisons. 63 | return u'django_format_dtdelta(%s, "%s", "%d", "%d", "%d")' % (sparql, 64 | connector, timedelta.days, timedelta.seconds, timedelta.microseconds) 65 | 66 | def date_trunc_sparql(self, lookup_type, field_name): 67 | # sparqlite doesn't support DATE_TRUNC, so we fake it with a user-defined 68 | # function django_date_trunc that's registered in connect(). Note that 69 | # single quotes are used because this is a string (and could otherwise 70 | # cause a collision with a field name). 71 | return "django_date_trunc('%s', %s)" % (lookup_type.lower(), field_name) 72 | 73 | def drop_foreignkey_sparql(self): 74 | return "" 75 | 76 | def pk_default_value(self): 77 | return 'NULL' 78 | 79 | def quote_name(self, name): 80 | # if name.startswith('"') and name.endswith('"'): 81 | # return name # Quoting once is enough. 82 | # return '"%s"' % name 83 | return '?%s' % name 84 | 85 | def quote_subject(self, name): 86 | return '<%s>' % name 87 | 88 | def quote_predicate(self, field, predicate): 89 | return '%s:%s %%s' % (field.graph, predicate) 90 | 91 | def no_limit_value(self): 92 | return -1 93 | 94 | def sparql_flush(self, style, tables, sequences): 95 | # NB: The generated SPARQL below is specific to SPARQLite 96 | # Note: The DELETE FROM... SPARQL generated below works for SPARQLite databases 97 | # because constraints don't exist 98 | sparql = ['%s %s %s;' % \ 99 | (style.SPARQL_KEYWORD('DELETE'), 100 | style.SPARQL_KEYWORD('FROM'), 101 | style.SPARQL_FIELD(self.quote_name(table)) 102 | ) for table in tables] 103 | # Note: No requirement for reset of auto-incremented indices (cf. other 104 | # sparql_flush() implementations). Just return SPARQL at this point 105 | return sparql 106 | 107 | def field_cast_sparql(self, db_type): 108 | """ 109 | Given a column type (e.g. 'BLOB', 'VARCHAR'), returns the SQL necessary 110 | to cast it before using it in a WHERE statement. Note that the 111 | resulting string should contain a '%s' placeholder for the column being 112 | searched against. 113 | """ 114 | return '%s' 115 | 116 | def year_lookup_bounds(self, value): 117 | first = '%s-01-01' 118 | second = '%s-12-31 23:59:59.999999' 119 | return [first % value, second % value] 120 | 121 | def convert_values(self, value, field): 122 | """SPARQLite returns floats when it should be returning decimals, 123 | and gets dates and datetimes wrong. 124 | For consistency with other backends, coerce when required. 125 | """ 126 | internal_type = field.get_internal_type() 127 | if internal_type == 'DecimalField': 128 | return util.typecast_decimal(field.format_number(value)) 129 | elif internal_type and internal_type.endswith('IntegerField') or internal_type == 'AutoField': 130 | return int(value) 131 | elif internal_type == 'DateField': 132 | return util.typecast_date(value) 133 | elif internal_type == 'DateTimeField': 134 | return util.typecast_timestamp(value) 135 | elif internal_type == 'TimeField': 136 | return util.typecast_time(value) 137 | 138 | # No field, or the field isn't known to be a decimal or integer 139 | return value 140 | 141 | 142 | class DatabaseWrapper(BaseDatabaseWrapper): 143 | vendor = 'virtuoso' 144 | # SPARQLite requires LIKE statements to include an ESCAPE clause if the value 145 | # being escaped has a percent or underscore in it. 146 | # See http://www.sparqlite.org/lang_expr.html for an explanation. 147 | operators = { 148 | 'exact': '= %s', 149 | 'iexact': "'^%s$', 'i'", # REGEX(?field, '^%s$', 'i') 150 | 151 | 'gt': '> %s', 152 | 'gte': '>= %s', 153 | 154 | 'lt': '< %s', 155 | 'lte': '<= %s', 156 | 157 | 'contains': "'%s'", # REGEX(?field, '%s') 158 | 'icontains': "'%s', 'i'", # REGEX(?field, '%s', 'i') 159 | 160 | 'regex': "'%s'", # 'REGEX (?field, %s) 161 | 'iregex': "'%s', 'i'", # REGEX(?field, '%s', 'i') 162 | 163 | 'startswith': "'^%s'", # REGEX(?field, '^%s') 164 | 'endswith': "'%s$'", # REGEX(?field, '%s$) 165 | 166 | 'istartswith': "'^%s', 'i'", # REGEX(?field, '^%s', 'i') 167 | 'iendswith': "'%s$', 'i'", # REGEX(?field, '%s$', 'i') 168 | } 169 | 170 | def __init__(self, *args, **kwargs): 171 | super(DatabaseWrapper, self).__init__(*args, **kwargs) 172 | 173 | self.features = DatabaseFeatures(self) 174 | self.ops = DatabaseOperations() 175 | self.client = DatabaseClient(self) 176 | self.creation = DatabaseCreation(self) 177 | self.introspection = DatabaseIntrospection(self) 178 | self.validation = BaseDatabaseValidation(self) 179 | if 'PREFIX' in self.settings_dict: 180 | self.prefixes = ['prefix %s: %s' % (key, value) \ 181 | for key, value in self.settings_dict['PREFIX'].items()] 182 | else: 183 | self.prefixes = '' 184 | 185 | def _cursor(self): 186 | if self.connection is None: 187 | settings_dict = self.settings_dict 188 | if not settings_dict['NAME'] or not settings_dict['HOST']: 189 | from django.core.exceptions import ImproperlyConfigured 190 | raise ImproperlyConfigured("Please fill out the database NAME and HOST in the settings module before using the database.") 191 | 192 | host = settings_dict['HOST'] 193 | port = settings_dict['PORT'] 194 | name = settings_dict['NAME'] 195 | 196 | if port: 197 | endpoint = 'http://%s:%s/%s' % (host, port, name) 198 | else: 199 | endpoint = 'http://%s/%s' % (host, port, name) 200 | 201 | self.connection = Database(endpoint) 202 | self.connection.setReturnFormat(JSON) 203 | 204 | connection_created.send(sender=self.__class__, connection=self) 205 | return Cursor(self.connection, self.prefixes) 206 | 207 | def close(self): 208 | # If database is in memory, closing the connection destroys the 209 | # database. To prevent accidental data loss, ignore close requests on 210 | # an in-memory db. 211 | if self.settings_dict['NAME'] != ":memory:": 212 | BaseDatabaseWrapper.close(self) 213 | 214 | def make_debug_cursor(self, cursor): 215 | return util.CursorDebugWrapper(cursor, self) 216 | -------------------------------------------------------------------------------- /semantic/rdf/backends/virtuoso/client.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rfloriano/semantic-django/f35747887507ca6aa08a9bf528c706a214382162/semantic/rdf/backends/virtuoso/client.py -------------------------------------------------------------------------------- /semantic/rdf/backends/virtuoso/creation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rfloriano/semantic-django/f35747887507ca6aa08a9bf528c706a214382162/semantic/rdf/backends/virtuoso/creation.py -------------------------------------------------------------------------------- /semantic/rdf/backends/virtuoso/dbapi.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # implements Python DBAPI 2.0 4 | # see PEP 249 (http://www.python.org/dev/peps/pep-0249/) 5 | import re 6 | import base64 7 | import decimal 8 | 9 | from SPARQLWrapper.Wrapper import SELECT, INSERT, DELETE, MODIFY 10 | # CONSTRUCT 11 | # ASK 12 | # DESCRIBE 13 | 14 | 15 | class Error(Exception): 16 | pass 17 | 18 | 19 | class SparqlSyntaxError(Exception): 20 | def __init__(self, message, line=None): 21 | self.message = message 22 | self.line = line 23 | 24 | 25 | class DatabaseError(Error): 26 | pass 27 | 28 | 29 | class ProgrammingError(DatabaseError): 30 | def __init__(self, msg, **kwargs): 31 | DatabaseError.__init__(self, msg) 32 | 33 | 34 | class Cursor(object): 35 | def __init__(self, connection, prefixes=''): 36 | self.arraysize = 100 37 | self.connection = connection 38 | self.sparql = None 39 | self.results = None 40 | self.pointer = 0 41 | self.prefixes = prefixes 42 | self.rowcount = -1 43 | 44 | def __iter__(self): 45 | result = self.pointer and self.results[self.pointer:] or self.results 46 | return iter(result) 47 | 48 | def _desc(self): 49 | #(name, type_code, display_size, internal_size, precision, scale, null_ok) 50 | #first two are required, supply None for optional values 51 | if not self.results: 52 | return None 53 | 54 | if len(self.results) == 0: 55 | return None 56 | 57 | desc = [] 58 | cols = [self.results.get_binding_name(i) for i in range(self.results.get_bindings_count())] 59 | for col in cols: 60 | desc_col = (col, literal_datatype(self.results.get_binding_value_by_name(col)), None, None, None, None, None) 61 | desc.append(desc_col) 62 | return desc 63 | 64 | #if no return values, or nothing executed 65 | return None 66 | description = property(_desc) 67 | 68 | def close(self): 69 | pass 70 | 71 | def _escape_param(self, param): 72 | # if type(param) in typeToSchema: 73 | # return '%s^^%s' % (param, typeToSchema[type(param)]) 74 | if isinstance(param, (str, unicode)): 75 | return '%s' % param 76 | return unicode(param) 77 | 78 | def escape_params(self, parameters): 79 | #for dict, return dict 80 | if isinstance(parameters, dict): 81 | params = {} 82 | for k, v in parameters.iteritems(): 83 | params[k] = self._escape_param(v) 84 | return params 85 | #for sequence, return tuple 86 | params = [] 87 | for p in parameters: 88 | params.append(self._escape_param(p)) 89 | return tuple(params) 90 | 91 | def execute(self, sparql, params=[]): 92 | params = self.escape_params(params) 93 | sparql = '%s %s' % (' '.join(self.prefixes), sparql) 94 | self.sparql = sparql % params 95 | self.connection.setQuery(self.sparql) 96 | self.results = self.connection.query().convert()["results"]["bindings"] 97 | self.update_rowcount() 98 | 99 | def update_rowcount(self): 100 | if self.connection.queryType == SELECT: 101 | self.rowcount = len(self.results) 102 | elif self.connection.queryType == INSERT \ 103 | or self.connection.queryType == DELETE: 104 | message = self.results[0]['callret-0']['value'] 105 | finder = re.search('(?P\d).*triples', message).groupdict() 106 | self.rowcount = int(finder['number']) 107 | elif self.connection.queryType == MODIFY: 108 | message = self.results[0]['callret-0']['value'] 109 | finder = re.search('delete (?P\d).*insert (?P\d)', message).groupdict() 110 | deleted = int(finder['deleted']) 111 | inserted = int(finder['inserted']) 112 | self.rowcount = deleted or inserted 113 | else: 114 | raise NotImplementedError("Type %s cannot get a rowcount" % self.connection.queryType) 115 | 116 | def executemany(self, operation, seq_of_parameters): 117 | raise NotImplementedError('executemany need to implemented') 118 | 119 | # def next(self): 120 | # row = self.fetchone() 121 | # if row is None: 122 | # raise StopIteration 123 | # return row 124 | 125 | def fetchone(self): 126 | if self.pointer >= len(self.results): 127 | return None 128 | result = self.results[self.pointer] 129 | self.pointer += 1 130 | return _rowfactory(result) 131 | 132 | def fetchmany(self, size=None): 133 | end = self.pointer + (size or self.arraysize) 134 | results = self.results[self.pointer:end] 135 | self.pointer = min(end, len(self.results)) 136 | return tuple([ 137 | _rowfactory(r) for r in results 138 | ]) 139 | 140 | def fetchall(self): 141 | if self.pointer: 142 | results = self.results[self.pointer:] 143 | else: 144 | results = self.results 145 | self.pointer = len(self.results) 146 | return tuple([ 147 | _rowfactory(r) for r in results 148 | ]) 149 | 150 | def dictfetchone(self): 151 | if not self.results: 152 | return None 153 | return self.results[0] 154 | 155 | def nextset(self): 156 | return None 157 | 158 | def setinputsizes(self): 159 | pass 160 | 161 | def setoutputsize(self, size, column=None): 162 | pass 163 | 164 | 165 | def _rowfactory(row): 166 | return tuple([(key, value['value']) for key, value in row.items()]) 167 | 168 | 169 | def literal_datatype(node): 170 | if not node.is_literal(): 171 | return None 172 | dt = node.literal_value['datatype'] 173 | if dt: 174 | return unicode(dt) 175 | return u'http://www.w3.org/2001/XMLSchema#string' 176 | 177 | typeToSchema = { 178 | unicode: '', 179 | bool: '', 180 | decimal: '', 181 | int: '', 182 | long: '', 183 | float: '', 184 | base64: '', 185 | } 186 | 187 | 188 | SchemaToPython = { # (schema->python, python->schema) Does not validate. 189 | 'http://www.w3.org/2001/XMLSchema#string': (unicode, unicode), 190 | 'http://www.w3.org/2001/XMLSchema#normalizedString': (unicode, unicode), 191 | 'http://www.w3.org/2001/XMLSchema#token': (unicode, unicode), 192 | 'http://www.w3.org/2001/XMLSchema#language': (unicode, unicode), 193 | 'http://www.w3.org/2001/XMLSchema#boolean': (bool, lambda i: unicode(i).lower()), 194 | 'http://www.w3.org/2001/XMLSchema#decimal': (decimal.Decimal, unicode), 195 | 'http://www.w3.org/2001/XMLSchema#integer': (int, unicode), 196 | 'http://www.w3.org/2001/XMLSchema#nonPositiveInteger': (int, unicode), 197 | 'http://www.w3.org/2001/XMLSchema#long': (long, unicode), 198 | 'http://www.w3.org/2001/XMLSchema#nonNegativeInteger': (int, unicode), 199 | 'http://www.w3.org/2001/XMLSchema#negativeInteger': (int, unicode), 200 | 'http://www.w3.org/2001/XMLSchema#int': (int, unicode), 201 | 'http://www.w3.org/2001/XMLSchema#unsignedLong': (long, unicode), 202 | 'http://www.w3.org/2001/XMLSchema#positiveInteger': (int, unicode), 203 | 'http://www.w3.org/2001/XMLSchema#short': (int, unicode), 204 | 'http://www.w3.org/2001/XMLSchema#unsignedInt': (long, unicode), 205 | 'http://www.w3.org/2001/XMLSchema#byte': (int, unicode), 206 | 'http://www.w3.org/2001/XMLSchema#unsignedShort': (int, unicode), 207 | 'http://www.w3.org/2001/XMLSchema#unsignedByte': (int, unicode), 208 | 'http://www.w3.org/2001/XMLSchema#float': (float, unicode), 209 | 'http://www.w3.org/2001/XMLSchema#double': (float, unicode), # doesn't do the whole range 210 | # duration 211 | # dateTime 212 | # time 213 | # date 214 | # gYearMonth 215 | # gYear 216 | # gMonthDay 217 | # gDay 218 | # gMonth 219 | # hexBinary 220 | 'http://www.w3.org/2001/XMLSchema#base64Binary': (base64.decodestring, lambda i: base64.encodestring(i)[:-1]), 221 | 'http://www.w3.org/2001/XMLSchema#anyURI': (str, str), 222 | } 223 | -------------------------------------------------------------------------------- /semantic/rdf/backends/virtuoso/instrospection.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rfloriano/semantic-django/f35747887507ca6aa08a9bf528c706a214382162/semantic/rdf/backends/virtuoso/instrospection.py -------------------------------------------------------------------------------- /semantic/rdf/models/__init__.py: -------------------------------------------------------------------------------- 1 | from semantic.rdf.models.base import SemanticModel 2 | from semantic.rdf.models.fields import * 3 | -------------------------------------------------------------------------------- /semantic/rdf/models/base.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import copy 3 | 4 | from django.db import transaction 5 | from django.db import router 6 | from django.db.models import signals 7 | from django.db.models.base import ModelBase, Model, subclass_exception 8 | from django.db.models.fields.related import OneToOneField 9 | from django.db.models.loading import register_models, get_model 10 | from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned, FieldError 11 | 12 | from semantic.rdf import connections 13 | from semantic.rdf.models.deletion import Collector 14 | from semantic.rdf.models.options import SemanticOptions 15 | from semantic.rdf.models.fields import AutoSemanticField 16 | from semantic.rdf.models.manager import SemanticManager 17 | 18 | 19 | class SemanticModelBase(ModelBase): 20 | """ 21 | Metaclass for all models. 22 | """ 23 | def __new__(cls, name, bases, attrs): 24 | super_new = super(ModelBase, cls).__new__ 25 | parents = [b for b in bases if isinstance(b, ModelBase)] 26 | if not parents: 27 | # If this isn't a subclass of Model, don't do anything special. 28 | return super_new(cls, name, bases, attrs) 29 | 30 | # Create the class. 31 | module = attrs.pop('__module__') 32 | new_class = super_new(cls, name, bases, {'__module__': module}) 33 | attr_meta = attrs.pop('Meta', None) 34 | abstract = getattr(attr_meta, 'abstract', False) 35 | if not attr_meta: 36 | meta = getattr(new_class, 'Meta', None) 37 | else: 38 | meta = attr_meta 39 | base_meta = getattr(new_class, '_meta', None) 40 | 41 | if getattr(meta, 'app_label', None) is None: 42 | # Figure out the app_label by looking one level up. 43 | # For 'django.contrib.sites.models', this would be 'sites'. 44 | model_module = sys.modules[new_class.__module__] 45 | kwargs = {"app_label": model_module.__name__.split('.')[-2]} 46 | else: 47 | kwargs = {} 48 | 49 | new_class.add_to_class('_meta', SemanticOptions(meta, **kwargs)) 50 | if not abstract: 51 | new_class.add_to_class('DoesNotExist', subclass_exception('DoesNotExist', 52 | tuple(x.DoesNotExist 53 | for x in parents if hasattr(x, '_meta') and not x._meta.abstract) 54 | or (ObjectDoesNotExist,), module)) 55 | new_class.add_to_class('MultipleObjectsReturned', subclass_exception('MultipleObjectsReturned', 56 | tuple(x.MultipleObjectsReturned 57 | for x in parents if hasattr(x, '_meta') and not x._meta.abstract) 58 | or (MultipleObjectsReturned,), module)) 59 | if base_meta and not base_meta.abstract: 60 | # Non-abstract child classes inherit some attributes from their 61 | # non-abstract parent (unless an ABC comes before it in the 62 | # method resolution order). 63 | if not hasattr(meta, 'ordering'): 64 | new_class._meta.ordering = base_meta.ordering 65 | if not hasattr(meta, 'get_latest_by'): 66 | new_class._meta.get_latest_by = base_meta.get_latest_by 67 | 68 | is_proxy = new_class._meta.proxy 69 | 70 | if getattr(new_class, '_default_manager', None): 71 | if not is_proxy: 72 | # Multi-table inheritance doesn't inherit default manager from 73 | # parents. 74 | new_class._default_manager = None 75 | new_class._base_manager = None 76 | else: 77 | # Proxy classes do inherit parent's default manager, if none is 78 | # set explicitly. 79 | new_class._default_manager = new_class._default_manager._copy_to_model(new_class) 80 | new_class._base_manager = new_class._base_manager._copy_to_model(new_class) 81 | 82 | # Bail out early if we have already created this class. 83 | m = get_model(new_class._meta.app_label, name, False) 84 | if m is not None: 85 | return m 86 | 87 | # Add all attributes to the class. 88 | for obj_name, obj in attrs.items(): 89 | new_class.add_to_class(obj_name, obj) 90 | 91 | # All the fields of any type declared on this model 92 | new_fields = new_class._meta.local_fields + \ 93 | new_class._meta.local_many_to_many + \ 94 | new_class._meta.virtual_fields 95 | field_names = set([f.name for f in new_fields]) 96 | 97 | # Basic setup for proxy models. 98 | if is_proxy: 99 | base = None 100 | for parent in [cls for cls in parents if hasattr(cls, '_meta')]: 101 | if parent._meta.abstract: 102 | if parent._meta.fields: 103 | raise TypeError("Abstract base class containing model fields not permitted for proxy model '%s'." % name) 104 | else: 105 | continue 106 | if base is not None: 107 | raise TypeError("Proxy model '%s' has more than one non-abstract model base class." % name) 108 | else: 109 | base = parent 110 | if base is None: 111 | raise TypeError("Proxy model '%s' has no non-abstract model base class." % name) 112 | if (new_class._meta.local_fields or 113 | new_class._meta.local_many_to_many): 114 | raise FieldError("Proxy model '%s' contains model fields." % name) 115 | while base._meta.proxy: 116 | base = base._meta.proxy_for_model 117 | new_class._meta.setup_proxy(base) 118 | 119 | # Do the appropriate setup for any model parents. 120 | o2o_map = dict([(f.rel.to, f) for f in new_class._meta.local_fields 121 | if isinstance(f, OneToOneField)]) 122 | 123 | for base in parents: 124 | original_base = base 125 | if not hasattr(base, '_meta'): 126 | # Things without _meta aren't functional models, so they're 127 | # uninteresting parents. 128 | continue 129 | 130 | parent_fields = base._meta.local_fields + base._meta.local_many_to_many 131 | # Check for clashes between locally declared fields and those 132 | # on the base classes (we cannot handle shadowed fields at the 133 | # moment). 134 | for field in parent_fields: 135 | if field.name in field_names: 136 | raise FieldError('Local field %r in class %r clashes ' 137 | 'with field of similar name from ' 138 | 'base class %r' % 139 | (field.name, name, base.__name__)) 140 | if not base._meta.abstract: 141 | # Concrete classes... 142 | while base._meta.proxy: 143 | # Skip over a proxy class to the "real" base it proxies. 144 | base = base._meta.proxy_for_model 145 | if base in o2o_map: 146 | field = o2o_map[base] 147 | elif not is_proxy: 148 | attr_name = '%s_ptr' % base._meta.module_name 149 | field = OneToOneField(base, name=attr_name, 150 | auto_created=True, parent_link=True) 151 | new_class.add_to_class(attr_name, field) 152 | else: 153 | field = None 154 | new_class._meta.parents[base] = field 155 | else: 156 | # .. and abstract ones. 157 | for field in parent_fields: 158 | new_class.add_to_class(field.name, copy.deepcopy(field)) 159 | 160 | # Pass any non-abstract parent classes onto child. 161 | new_class._meta.parents.update(base._meta.parents) 162 | 163 | # Inherit managers from the abstract base classes. 164 | new_class.copy_managers(base._meta.abstract_managers) 165 | 166 | # Proxy models inherit the non-abstract managers from their base, 167 | # unless they have redefined any of them. 168 | if is_proxy: 169 | new_class.copy_managers(original_base._meta.concrete_managers) 170 | 171 | # Inherit virtual fields (like GenericForeignKey) from the parent 172 | # class 173 | for field in base._meta.virtual_fields: 174 | if base._meta.abstract and field.name in field_names: 175 | raise FieldError('Local field %r in class %r clashes '\ 176 | 'with field of similar name from '\ 177 | 'abstract base class %r' % \ 178 | (field.name, name, base.__name__)) 179 | new_class.add_to_class(field.name, copy.deepcopy(field)) 180 | 181 | if abstract: 182 | # Abstract base models can't be instantiated and don't appear in 183 | # the list of models for an app. We do the final setup for them a 184 | # little differently from normal models. 185 | attr_meta.abstract = False 186 | new_class.Meta = attr_meta 187 | return new_class 188 | 189 | new_class._prepare() 190 | register_models(new_class._meta.app_label, new_class) 191 | 192 | # Because of the way imports happen (recursively), we may or may not be 193 | # the first time this model tries to register with the framework. There 194 | # should only be one class for each model, so we always return the 195 | # registered version. 196 | return get_model(new_class._meta.app_label, name, False) 197 | 198 | # class SemanticModelState(object): 199 | # """ 200 | # A class for storing instance state 201 | # """ 202 | # def __init__(self, db=None): 203 | # self.db = db 204 | 205 | 206 | class SemanticModel(Model): 207 | __metaclass__ = SemanticModelBase 208 | 209 | uri = AutoSemanticField(graph='base') 210 | objects = SemanticManager() 211 | _base_manager = objects 212 | 213 | class Meta: 214 | abstract = True 215 | 216 | def save_base(self, raw=False, cls=None, origin=None, force_insert=False, 217 | force_update=False, using=None): 218 | # this method is based on Model.save_base from django/db/models/base.py 219 | # changes: 220 | # (+) new line 221 | # (*) changed 222 | # (-) removed (commented) 223 | 224 | using = using or router.db_for_write(self.__class__, instance=self) 225 | connection = connections[using] # (*) 226 | 227 | assert not (force_insert and force_update) 228 | if cls is None: 229 | cls = self.__class__ 230 | meta = cls._meta 231 | if not meta.proxy: 232 | origin = cls 233 | else: 234 | meta = cls._meta 235 | 236 | if origin and not meta.auto_created: 237 | signals.pre_save.send(sender=origin, instance=self, raw=raw, using=using) 238 | 239 | # If we are in a raw save, save the object exactly as presented. 240 | # That means that we don't try to be smart about saving attributes 241 | # that might have come from the parent class - we just save the 242 | # attributes we have been given to the class we have been given. 243 | # We also go through this process to defer the save of proxy objects 244 | # to their actual underlying model. 245 | if not raw or meta.proxy: 246 | if meta.proxy: 247 | org = cls 248 | else: 249 | org = None 250 | for parent, field in meta.parents.items(): 251 | # At this point, parent's primary key field may be unknown 252 | # (for example, from administration form which doesn't fill 253 | # this field). If so, fill it. 254 | if field and getattr(self, parent._meta.pk.attname) is None and getattr(self, field.attname) is not None: 255 | setattr(self, parent._meta.pk.attname, getattr(self, field.attname)) 256 | 257 | self.save_base(cls=parent, origin=org, using=using) 258 | 259 | if field: 260 | setattr(self, field.attname, self._get_pk_val(parent._meta)) 261 | if meta.proxy: 262 | return 263 | 264 | if not meta.proxy: 265 | non_pks = [f for f in meta.local_fields if not f.primary_key] 266 | 267 | # First, try an UPDATE. If that doesn't update anything, do an INSERT. 268 | pk_val = self._get_pk_val(meta) 269 | pk_set = pk_val is not None 270 | record_exists = True 271 | manager = cls._base_manager 272 | if pk_set: 273 | # Determine whether a record with the primary key already exists. 274 | if (force_update or (not force_insert and 275 | manager.using(using).filter(pk=pk_val).exists())): 276 | # It does already exist, so do an UPDATE. 277 | if force_update or non_pks: 278 | values = [(f, None, (raw and getattr(self, f.attname) or f.pre_save(self, False))) for f in non_pks] 279 | rows = manager.using(using).filter(pk=pk_val)._update(values) 280 | if force_update and not rows: 281 | raise DatabaseError("Forced update did not affect any rows.") 282 | else: 283 | record_exists = False 284 | if not pk_set or not record_exists: 285 | if meta.order_with_respect_to: 286 | # If this is a model with an order_with_respect_to 287 | # autopopulate the _order field 288 | field = meta.order_with_respect_to 289 | order_value = manager.using(using).filter(**{field.name: getattr(self, field.attname)}).count() 290 | self._order = order_value 291 | 292 | if not pk_set: 293 | if force_update: 294 | raise ValueError("Cannot force an update in save() with no primary key.") 295 | values = [(f, f.get_db_prep_save(raw and getattr(self, f.attname) or f.pre_save(self, True), connection=connection)) 296 | for f in meta.local_fields if not isinstance(f, AutoField)] 297 | else: 298 | values = [(f, f.get_db_prep_save(raw and getattr(self, f.attname) or f.pre_save(self, True), connection=connection)) 299 | for f in meta.local_fields] 300 | 301 | record_exists = False 302 | 303 | update_pk = bool(meta.has_auto_field and not pk_set) 304 | if values: 305 | # Create a new record. 306 | result = manager._insert(values, return_id=update_pk, using=using) 307 | else: 308 | # Create a new record with defaults for everything. 309 | result = manager._insert([(meta.pk, connection.ops.pk_default_value())], return_id=update_pk, raw_values=True, using=using) 310 | 311 | if update_pk: 312 | setattr(self, meta.pk.attname, result) 313 | transaction.commit_unless_managed(using=using) 314 | 315 | # Store the database on which the object was saved 316 | self._state.db = using 317 | # Once saved, this is no longer a to-be-added instance. 318 | self._state.adding = False 319 | 320 | # Signal that the save is complete 321 | if origin and not meta.auto_created: 322 | signals.post_save.send(sender=origin, instance=self, 323 | created=(not record_exists), raw=raw, using=using) 324 | 325 | save_base.alters_data = True 326 | 327 | def delete(self, using=None): 328 | using = using or router.db_for_write(self.__class__, instance=self) 329 | assert self._get_pk_val() is not None, "%s object can't be deleted because its %s attribute is set to None." % (self._meta.object_name, self._meta.pk.attname) 330 | 331 | collector = Collector(using=using) 332 | collector.collect([self]) 333 | collector.delete() 334 | 335 | # def __init__(self, *args, **kwargs): 336 | # # Set up the storage for instance state 337 | # self._state = SemanticModelState() 338 | -------------------------------------------------------------------------------- /semantic/rdf/models/deletion.py: -------------------------------------------------------------------------------- 1 | from operator import attrgetter 2 | 3 | from django.db import connections, transaction, IntegrityError 4 | from django.db.models import signals 5 | from django.utils.datastructures import SortedDict 6 | from django.utils.functional import wraps 7 | 8 | from semantic.rdf.models import sparql 9 | from semantic.rdf.models.sparql.constants import GET_ITERATOR_CHUNK_SIZE 10 | 11 | 12 | class ProtectedError(IntegrityError): 13 | def __init__(self, msg, protected_objects): 14 | self.protected_objects = protected_objects 15 | # TODO change this to use super() when we drop Python 2.4 16 | IntegrityError.__init__(self, msg, protected_objects) 17 | 18 | 19 | def CASCADE(collector, field, sub_objs, using): 20 | collector.collect(sub_objs, source=field.rel.to, 21 | source_attr=field.name, nullable=field.null) 22 | if field.null and not connections[using].features.can_defer_constraint_checks: 23 | collector.add_field_update(field, None, sub_objs) 24 | 25 | 26 | def PROTECT(collector, field, sub_objs, using): 27 | raise ProtectedError("Cannot delete some instances of model '%s' because " 28 | "they are referenced through a protected foreign key: '%s.%s'" % ( 29 | field.rel.to.__name__, sub_objs[0].__class__.__name__, field.name 30 | ), 31 | sub_objs 32 | ) 33 | 34 | 35 | def SET(value): 36 | if callable(value): 37 | def set_on_delete(collector, field, sub_objs, using): 38 | collector.add_field_update(field, value(), sub_objs) 39 | else: 40 | def set_on_delete(collector, field, sub_objs, using): 41 | collector.add_field_update(field, value, sub_objs) 42 | return set_on_delete 43 | 44 | 45 | SET_NULL = SET(None) 46 | 47 | 48 | def SET_DEFAULT(collector, field, sub_objs, using): 49 | collector.add_field_update(field, field.get_default(), sub_objs) 50 | 51 | 52 | def DO_NOTHING(collector, field, sub_objs, using): 53 | pass 54 | 55 | 56 | def force_managed(func): 57 | @wraps(func) 58 | def decorated(self, *args, **kwargs): 59 | if not transaction.is_managed(using=self.using): 60 | transaction.enter_transaction_management(using=self.using) 61 | forced_managed = True 62 | else: 63 | forced_managed = False 64 | try: 65 | func(self, *args, **kwargs) 66 | if forced_managed: 67 | transaction.commit(using=self.using) 68 | else: 69 | transaction.commit_unless_managed(using=self.using) 70 | finally: 71 | if forced_managed: 72 | transaction.leave_transaction_management(using=self.using) 73 | return decorated 74 | 75 | 76 | class Collector(object): 77 | def __init__(self, using): 78 | self.using = using 79 | # Initially, {model: set([instances])}, later values become lists. 80 | self.data = {} 81 | self.batches = {} # {model: {field: set([instances])}} 82 | self.field_updates = {} # {model: {(field, value): set([instances])}} 83 | self.dependencies = {} # {model: set([models])} 84 | 85 | def add(self, objs, source=None, nullable=False, reverse_dependency=False): 86 | """ 87 | Adds 'objs' to the collection of objects to be deleted. If the call is 88 | the result of a cascade, 'source' should be the model that caused it 89 | and 'nullable' should be set to True, if the relation can be null. 90 | 91 | Returns a list of all objects that were not already collected. 92 | """ 93 | if not objs: 94 | return [] 95 | new_objs = [] 96 | model = objs[0].__class__ 97 | instances = self.data.setdefault(model, set()) 98 | for obj in objs: 99 | if obj not in instances: 100 | new_objs.append(obj) 101 | instances.update(new_objs) 102 | # Nullable relationships can be ignored -- they are nulled out before 103 | # deleting, and therefore do not affect the order in which objects have 104 | # to be deleted. 105 | if new_objs and source is not None and not nullable: 106 | if reverse_dependency: 107 | source, model = model, source 108 | self.dependencies.setdefault(source, set()).add(model) 109 | return new_objs 110 | 111 | def add_batch(self, model, field, objs): 112 | """ 113 | Schedules a batch delete. Every instance of 'model' that is related to 114 | an instance of 'obj' through 'field' will be deleted. 115 | """ 116 | self.batches.setdefault(model, {}).setdefault(field, set()).update(objs) 117 | 118 | def add_field_update(self, field, value, objs): 119 | """ 120 | Schedules a field update. 'objs' must be a homogenous iterable 121 | collection of model instances (e.g. a QuerySet). 122 | """ 123 | if not objs: 124 | return 125 | model = objs[0].__class__ 126 | self.field_updates.setdefault( 127 | model, {}).setdefault( 128 | (field, value), set()).update(objs) 129 | 130 | def collect(self, objs, source=None, nullable=False, collect_related=True, 131 | source_attr=None, reverse_dependency=False): 132 | """ 133 | Adds 'objs' to the collection of objects to be deleted as well as all 134 | parent instances. 'objs' must be a homogenous iterable collection of 135 | model instances (e.g. a QuerySet). If 'collect_related' is True, 136 | related objects will be handled by their respective on_delete handler. 137 | 138 | If the call is the result of a cascade, 'source' should be the model 139 | that caused it and 'nullable' should be set to True, if the relation 140 | can be null. 141 | 142 | If 'reverse_dependency' is True, 'source' will be deleted before the 143 | current model, rather than after. (Needed for cascading to parent 144 | models, the one case in which the cascade follows the forwards 145 | direction of an FK rather than the reverse direction.) 146 | """ 147 | new_objs = self.add(objs, source, nullable, 148 | reverse_dependency=reverse_dependency) 149 | if not new_objs: 150 | return 151 | model = new_objs[0].__class__ 152 | 153 | # Recursively collect parent models, but not their related objects. 154 | # These will be found by meta.get_all_related_objects() 155 | for parent_model, ptr in model._meta.parents.iteritems(): 156 | if ptr: 157 | parent_objs = [getattr(obj, ptr.name) for obj in new_objs] 158 | self.collect(parent_objs, source=model, 159 | source_attr=ptr.rel.related_name, 160 | collect_related=False, 161 | reverse_dependency=True) 162 | 163 | if collect_related: 164 | for related in model._meta.get_all_related_objects(include_hidden=True): 165 | field = related.field 166 | if related.model._meta.auto_created: 167 | self.add_batch(related.model, field, new_objs) 168 | else: 169 | sub_objs = self.related_objects(related, new_objs) 170 | if not sub_objs: 171 | continue 172 | field.rel.on_delete(self, field, sub_objs, self.using) 173 | 174 | # TODO This entire block is only needed as a special case to 175 | # support cascade-deletes for GenericRelation. It should be 176 | # removed/fixed when the ORM gains a proper abstraction for virtual 177 | # or composite fields, and GFKs are reworked to fit into that. 178 | for relation in model._meta.many_to_many: 179 | if not relation.rel.through: 180 | sub_objs = relation.bulk_related_objects(new_objs, self.using) 181 | self.collect(sub_objs, 182 | source=model, 183 | source_attr=relation.rel.related_name, 184 | nullable=True) 185 | 186 | def related_objects(self, related, objs): 187 | """ 188 | Gets a QuerySet of objects related to ``objs`` via the relation ``related``. 189 | 190 | """ 191 | return related.model._base_manager.using(self.using).filter( 192 | **{"%s__in" % related.field.name: objs} 193 | ) 194 | 195 | def instances_with_model(self): 196 | for model, instances in self.data.iteritems(): 197 | for obj in instances: 198 | yield model, obj 199 | 200 | def sort(self): 201 | sorted_models = [] 202 | models = self.data.keys() 203 | while len(sorted_models) < len(models): 204 | found = False 205 | for model in models: 206 | if model in sorted_models: 207 | continue 208 | dependencies = self.dependencies.get(model) 209 | if not (dependencies and dependencies.difference(sorted_models)): 210 | sorted_models.append(model) 211 | found = True 212 | if not found: 213 | return 214 | self.data = SortedDict([(model, self.data[model]) 215 | for model in sorted_models]) 216 | 217 | @force_managed 218 | def delete(self): 219 | # sort instance collections 220 | for model, instances in self.data.items(): 221 | self.data[model] = sorted(instances, key=attrgetter("pk")) 222 | 223 | # if possible, bring the models in an order suitable for databases that 224 | # don't support transactions or cannot defer contraint checks until the 225 | # end of a transaction. 226 | self.sort() 227 | 228 | # send pre_delete signals 229 | for model, obj in self.instances_with_model(): 230 | if not model._meta.auto_created: 231 | signals.pre_delete.send( 232 | sender=model, instance=obj, using=self.using 233 | ) 234 | 235 | # update fields 236 | for model, instances_for_fieldvalues in self.field_updates.iteritems(): 237 | query = sparql.UpdateQuery(model) 238 | for (field, value), instances in instances_for_fieldvalues.iteritems(): 239 | query.update_batch([obj.pk for obj in instances], 240 | {field.name: value}, self.using) 241 | 242 | # reverse instance collections 243 | for instances in self.data.itervalues(): 244 | instances.reverse() 245 | 246 | # delete batches 247 | for model, batches in self.batches.iteritems(): 248 | query = sparql.DeleteQuery(model) 249 | for field, instances in batches.iteritems(): 250 | query.delete_batch([obj.pk for obj in instances], self.using, field) 251 | 252 | # delete instances 253 | for model, instances in self.data.iteritems(): 254 | query = sparql.DeleteQuery(model) 255 | pk_list = [obj.pk for obj in instances] 256 | query.delete_batch(pk_list, self.using) 257 | 258 | # send post_delete signals 259 | for model, obj in self.instances_with_model(): 260 | if not model._meta.auto_created: 261 | signals.post_delete.send( 262 | sender=model, instance=obj, using=self.using 263 | ) 264 | 265 | # update collected instances 266 | for model, instances_for_fieldvalues in self.field_updates.iteritems(): 267 | for (field, value), instances in instances_for_fieldvalues.iteritems(): 268 | for obj in instances: 269 | setattr(obj, field.attname, value) 270 | for model, instances in self.data.iteritems(): 271 | for instance in instances: 272 | setattr(instance, model._meta.pk.attname, None) 273 | -------------------------------------------------------------------------------- /semantic/rdf/models/fields/__init__.py: -------------------------------------------------------------------------------- 1 | # from django.db import models 2 | import re 3 | import uuid 4 | 5 | from django.core import validators 6 | from django.utils.encoding import smart_unicode 7 | from django.utils.translation import ugettext_lazy as _ 8 | from django.db.models.fields import * 9 | 10 | 11 | class SemanticField(Field): 12 | def __init__(self, graph, *args, **kwargs): 13 | super(SemanticField, self).__init__(*args, **kwargs) 14 | self.graph = graph 15 | 16 | def get_db_prep_lookup(self, lookup_type, value, *args, **kwargs): 17 | if lookup_type in ('startswith', 'istartswith', 'endswith', 'iendswith', 'contains', 'icontains'): 18 | return [value] 19 | return super(SemanticField, self).get_db_prep_lookup(lookup_type, value, *args, **kwargs) 20 | 21 | 22 | class CharField(SemanticField): 23 | description = _("String (up to %(max_length)s)") 24 | 25 | def __init__(self, *args, **kwargs): 26 | super(CharField, self).__init__(*args, **kwargs) 27 | self.validators.append(validators.MaxLengthValidator(self.max_length)) 28 | 29 | def get_internal_type(self): 30 | return "CharField" 31 | 32 | def get_prep_value(self, value): 33 | if not value: 34 | return '' 35 | 36 | if isinstance(value, basestring) or value is None: 37 | result = value 38 | else: 39 | result = smart_unicode(value) 40 | 41 | return '"%s"' % result 42 | 43 | def formfield(self, **kwargs): 44 | # Passing max_length to forms.CharField means that the value's length 45 | # will be validated twice. This is considered acceptable since we want 46 | # the value in the form field (to pass into widget for example). 47 | defaults = {'max_length': self.max_length} 48 | defaults.update(kwargs) 49 | return super(CharField, self).formfield(**defaults) 50 | 51 | 52 | class URLField(CharField): 53 | 54 | def __init__(self, *args, **kwargs): 55 | kwargs['max_length'] = kwargs.get('max_length', 200) 56 | verify_exists = kwargs.pop('verify_exists', True) 57 | CharField.__init__(self, *args, **kwargs) 58 | 59 | self.validators.append(validators.URLValidator(verify_exists=verify_exists)) 60 | 61 | 62 | class URIField(URLField): 63 | 64 | def __init__(self, *args, **kwargs): 65 | verify_exists = kwargs.get('verify_exists', False) 66 | super(URIField, self).__init__(verify_exists=verify_exists, *args, **kwargs) 67 | 68 | def get_prep_value(self, value): 69 | if value: 70 | return '<%s>' % value 71 | return '"%s"' % value 72 | 73 | def get_db_prep_value(self, value, connection, prepared=False): 74 | """Returns field's value prepared for interacting with the database 75 | backend. 76 | 77 | Used by the default implementations of ``get_db_prep_save``and 78 | `get_db_prep_lookup``` 79 | """ 80 | if not prepared: 81 | value = self.get_prep_value(value) 82 | return value 83 | 84 | 85 | class AutoSemanticField(URIField): 86 | description = _("Auto Semantic Field") 87 | 88 | def __init__(self, graph, verbose_name='URI', name=None, primary_key=True, verify_exists=False, **kwargs): 89 | super(AutoSemanticField, self).__init__(verbose_name, name, verify_exists, primary_key=primary_key, **kwargs) 90 | self.graph = graph 91 | 92 | def pre_save(self, *args, **kargs): 93 | model_data = args[0] 94 | value = getattr(model_data, self.attname) 95 | if not value: 96 | value = '%s/%s' % (self.model._meta.graph.rstrip('/'), uuid.uuid4()) 97 | setattr(model_data, self.attname, value) 98 | return super(AutoSemanticField, self).pre_save(*args, **kargs) 99 | 100 | 101 | class IntegerField(IntegerField, SemanticField): 102 | pass 103 | -------------------------------------------------------------------------------- /semantic/rdf/models/manager.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | from semantic.rdf.models.query import SemanticQuerySet, SemanticEmptyQuerySet, \ 4 | RawSemanticQuerySet, insert_query 5 | 6 | 7 | class SemanticManager(models.Manager): 8 | def __init__(self): 9 | super(SemanticManager, self).__init__() 10 | self._db = None 11 | 12 | def get_empty_query_set(self): 13 | return SemanticEmptyQuerySet(self.model, using=self._db) 14 | 15 | def get_query_set(self): 16 | """Returns a new QuerySet object. Subclasses can override this method 17 | to easily customize the behavior of the Manager. 18 | """ 19 | return SemanticQuerySet(self.model, using=self._db) 20 | 21 | def raw(self, raw_query, params=None, *args, **kwargs): 22 | return RawSemanticQuerySet(raw_query=raw_query, model=self.model, params=params, using=self._db, *args, **kwargs) 23 | 24 | def _insert(self, values, **kwargs): 25 | return insert_query(self.model, values, **kwargs) 26 | -------------------------------------------------------------------------------- /semantic/rdf/models/options.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | from django.conf import settings 4 | from django.utils.translation import string_concat 5 | from django.db.models.options import Options, DEFAULT_NAMES, get_verbose_name 6 | 7 | DEFAULT_NAMES = list(DEFAULT_NAMES) 8 | 9 | SEMANTIC_DEFAULT_NAMES = tuple( 10 | DEFAULT_NAMES + [ 11 | 'graph', 12 | 'namespace', 13 | ]) 14 | 15 | 16 | class SemanticOptions(Options): 17 | def __init__(self, meta, app_label=None): 18 | super(SemanticOptions, self).__init__(meta, app_label) 19 | self.graph = '' 20 | 21 | def contribute_to_class(self, cls, name): 22 | cls._meta = self 23 | self.installed = re.sub('\.models$', '', cls.__module__) in settings.INSTALLED_APPS 24 | # First, construct the default values for these options. 25 | self.object_name = cls.__name__ 26 | self.module_name = self.object_name.lower() 27 | self.verbose_name = get_verbose_name(self.object_name) 28 | 29 | # Next, apply any overridden values from 'class Meta'. 30 | if self.meta: 31 | meta_attrs = self.meta.__dict__.copy() 32 | for name in self.meta.__dict__: 33 | # Ignore any private attributes that Django doesn't care about. 34 | # NOTE: We can't modify a dictionary's contents while looping 35 | # over it, so we loop over the *original* dictionary instead. 36 | if name.startswith('_'): 37 | del meta_attrs[name] 38 | for attr_name in SEMANTIC_DEFAULT_NAMES: 39 | if attr_name in meta_attrs: 40 | setattr(self, attr_name, meta_attrs.pop(attr_name)) 41 | elif hasattr(self.meta, attr_name): 42 | setattr(self, attr_name, getattr(self.meta, attr_name)) 43 | 44 | # verbose_name_plural is a special case because it uses a 's' 45 | # by default. 46 | if self.verbose_name_plural is None: 47 | self.verbose_name_plural = string_concat(self.verbose_name, 's') 48 | 49 | # Any leftover attributes must be invalid. 50 | if meta_attrs != {}: 51 | raise TypeError("'class Meta' got invalid attribute(s): %s" % ','.join(meta_attrs.keys())) 52 | else: 53 | self.verbose_name_plural = string_concat(self.verbose_name, 's') 54 | del self.meta 55 | -------------------------------------------------------------------------------- /semantic/rdf/models/query.py: -------------------------------------------------------------------------------- 1 | from django.db.models.query import QuerySet, EmptyQuerySet, RawQuerySet 2 | 3 | from semantic.rdf.models import sparql 4 | 5 | 6 | class SemanticQuerySet(QuerySet): 7 | def __init__(self, model=None, query=None, using=None): 8 | super(SemanticQuerySet, self).__init__(model, query, using) 9 | self.query = query or sparql.SparqlQuery(self.model) 10 | 11 | def _update(self, values): 12 | """ 13 | A version of update that accepts field objects instead of field names. 14 | Used primarily for model saving and not intended for use by general 15 | code (it requires too much poking around at model internals to be 16 | useful at that level). 17 | """ 18 | # Based on QuerySet._update, available at django/db/models/query 19 | # changes: 20 | # (+) new line 21 | # (*) changed 22 | # (-) removed (commented) 23 | assert self.query.can_filter(), \ 24 | "Cannot update a query once a slice has been taken." 25 | query = self.query.clone(sparql.UpdateQuery) # (*) 26 | query.add_update_fields(values) 27 | self._result_cache = None 28 | return query.get_compiler(self.db).execute_sparql(None) # (*) 29 | _update.alters_data = True 30 | 31 | 32 | class SemanticEmptyQuerySet(EmptyQuerySet): 33 | pass 34 | 35 | 36 | class RawSemanticQuerySet(RawQuerySet): 37 | """ 38 | Provides an iterator which converts the results of raw SQL queries into 39 | annotated model instances. 40 | """ 41 | 42 | # TODO: improve dbapi to can use this class on object.raw("A SPARQL QUERY") 43 | 44 | def __init__(self, raw_query, model=None, query=None, params=None, 45 | translations=None, using=None): 46 | self.raw_query = raw_query 47 | self.model = model 48 | self._db = using 49 | self.query = query or sparql.RawSemanticQuery(sparql=raw_query, using=self.db, params=params) 50 | self.params = params or () 51 | self.translations = translations or {} 52 | 53 | def using(self, alias): 54 | """ 55 | Selects which database this Raw QuerySet should excecute it's query against. 56 | """ 57 | return RawSemanticQuerySet(self.raw_query, model=self.model, 58 | query=self.query.clone(using=alias), 59 | params=self.params, translations=self.translations, 60 | using=alias) 61 | 62 | 63 | def insert_query(model, values, return_id=False, raw_values=False, using=None): 64 | """ 65 | Inserts a new record for the given model. This provides an interface to 66 | the InsertQuery class and is how Model.save() is implemented. It is not 67 | part of the public API. 68 | """ 69 | query = sparql.InsertQuery(model) 70 | query.insert_values(values, raw_values) 71 | # TODO: use 'using' variable from parameters 72 | return query.get_compiler(using='default').execute_sparql(return_id) 73 | -------------------------------------------------------------------------------- /semantic/rdf/models/sparql/__init__.py: -------------------------------------------------------------------------------- 1 | from query import * 2 | from subqueries import * 3 | from where import AND, OR 4 | from datastructures import EmptyResultSet 5 | 6 | __all__ = ['SemanticQuery', 'AND', 'OR', 'EmptyResultSet'] 7 | 8 | -------------------------------------------------------------------------------- /semantic/rdf/models/sparql/aggregates.py: -------------------------------------------------------------------------------- 1 | """ 2 | Classes to represent the default SPARQL aggregate functions 3 | """ 4 | 5 | 6 | class AggregateField(object): 7 | """An internal field mockup used to identify aggregates in the 8 | data-conversion parts of the database backend. 9 | """ 10 | def __init__(self, internal_type): 11 | self.internal_type = internal_type 12 | 13 | def get_internal_type(self): 14 | return self.internal_type 15 | 16 | ordinal_aggregate_field = AggregateField('IntegerField') 17 | computed_aggregate_field = AggregateField('FloatField') 18 | 19 | 20 | class Aggregate(object): 21 | """ 22 | Default SPARQL Aggregate. 23 | """ 24 | is_ordinal = False 25 | is_computed = False 26 | sparql_template = '%(function)s(%(field)s)' 27 | 28 | def __init__(self, col, source=None, is_summary=False, **extra): 29 | """Instantiate an SPARQL aggregate 30 | 31 | * col is a column reference describing the subject field 32 | of the aggregate. It can be an alias, or a tuple describing 33 | a table and column name. 34 | * source is the underlying field or aggregate definition for 35 | the column reference. If the aggregate is not an ordinal or 36 | computed type, this reference is used to determine the coerced 37 | output type of the aggregate. 38 | * extra is a dictionary of additional data to provide for the 39 | aggregate definition 40 | 41 | Also utilizes the class variables: 42 | * sparql_function, the name of the SPARQL function that implements the 43 | aggregate. 44 | * sparql_template, a template string that is used to render the 45 | aggregate into SPARQL. 46 | * is_ordinal, a boolean indicating if the output of this aggregate 47 | is an integer (e.g., a count) 48 | * is_computed, a boolean indicating if this output of this aggregate 49 | is a computed float (e.g., an average), regardless of the input 50 | type. 51 | 52 | """ 53 | self.col = col 54 | self.lookup = col 55 | self.source = source 56 | self.is_summary = is_summary 57 | self.extra = extra 58 | 59 | # Follow the chain of aggregate sources back until you find an 60 | # actual field, or an aggregate that forces a particular output 61 | # type. This type of this field will be used to coerce values 62 | # retrieved from the database. 63 | tmp = self 64 | 65 | while tmp and isinstance(tmp, Aggregate): 66 | if getattr(tmp, 'is_ordinal', False): 67 | tmp = ordinal_aggregate_field 68 | elif getattr(tmp, 'is_computed', False): 69 | tmp = computed_aggregate_field 70 | else: 71 | tmp = tmp.source 72 | 73 | self.field = tmp 74 | 75 | # Added for semantic_django 76 | self.default_alias = "%s__%s" % (self.sparql_function.lower(), self.col) 77 | self.name = self.__class__.__name__ 78 | 79 | def relabel_aliases(self, change_map): 80 | if isinstance(self.col, (list, tuple)): 81 | self.col = (change_map.get(self.col[0], self.col[0]), self.col[1]) 82 | 83 | def as_sparql(self, qn, connection): 84 | "Return the aggregate, rendered as SPARQL." 85 | 86 | if hasattr(self.col, 'as_sparql'): 87 | field_name = self.col.as_sparql(qn, connection) 88 | elif isinstance(self.col, (list, tuple)): 89 | field_name = qn(self.col[1]) 90 | else: 91 | field_name = self.column 92 | 93 | params = { 94 | 'function': self.sparql_function, 95 | 'field': field_name 96 | } 97 | params.update(self.extra) 98 | 99 | return self.sparql_template % params 100 | 101 | # As in django.db.models.aggregates 102 | def add_to_query(self, query, alias, col, source, is_summary): 103 | """Add the aggregate to the nominated query. 104 | 105 | This method is used to convert the generic Aggregate definition into a 106 | backend-specific definition. 107 | 108 | * query is the backend-specific query instance to which the aggregate 109 | is to be added. 110 | * col is a column reference describing the subject field 111 | of the aggregate. It can be an alias, or a tuple describing 112 | a table and column name. 113 | * source is the underlying field or aggregate definition for 114 | the column reference. If the aggregate is not an ordinal or 115 | computed type, this reference is used to determine the coerced 116 | output type of the aggregate. 117 | * is_summary is a boolean that is set True if the aggregate is a 118 | summary value rather than an annotation. 119 | """ 120 | klass = getattr(query.aggregates_module, self.name) 121 | aggregate = klass(col, source=source, is_summary=is_summary, **self.extra) 122 | query.aggregates[alias] = aggregate 123 | 124 | 125 | 126 | class Avg(Aggregate): 127 | is_computed = True 128 | sparql_function = 'AVG' 129 | 130 | class Count(Aggregate): 131 | is_ordinal = True 132 | sparql_function = 'COUNT' 133 | sparql_template = '%(function)s(%(distinct)s%(field)s)' 134 | 135 | def __init__(self, col, distinct=False, **extra): 136 | super(Count, self).__init__(col, **extra) 137 | self.distinct = distinct and 'DISTINCT ' or '' 138 | 139 | def as_sparql(self, qn, connection): 140 | field_name = self.col 141 | 142 | params = { 143 | 'function': self.sparql_function, 144 | 'field': qn(field_name[1]) if isinstance(field_name, tuple) else field_name, 145 | 'distinct': self.distinct 146 | } 147 | params.update(self.extra) 148 | 149 | return self.sparql_template % params 150 | 151 | 152 | class Max(Aggregate): 153 | sparql_function = 'MAX' 154 | 155 | 156 | class Min(Aggregate): 157 | sparql_function = 'MIN' 158 | 159 | 160 | class Sum(Aggregate): 161 | sparql_function = 'SUM' 162 | -------------------------------------------------------------------------------- /semantic/rdf/models/sparql/compiler.py: -------------------------------------------------------------------------------- 1 | from django.core.exceptions import FieldError 2 | from django.db.backends.util import truncate_name 3 | from django.db.models.sql.constants import * 4 | from django.db.models.sql.datastructures import EmptyResultSet 5 | from django.db.models.sql.query import get_proxied_model, get_order_dir, \ 6 | select_related_descend, Query 7 | from django.conf import settings 8 | 9 | from semantic.rdf.models.sparql.expressions import SPARQLEvaluator 10 | 11 | 12 | class SPARQLCompiler(object): 13 | def __init__(self, query, connection, using): 14 | self.query = query 15 | self.connection = connection 16 | self.using = using 17 | self.quote_cache = {} 18 | 19 | def pre_sparql_setup(self): 20 | """ 21 | Does any necessary class setup immediately prior to producing SPARQL. This 22 | is for things that can't necessarily be done in __init__ because we 23 | might not have all the pieces in place at that time. 24 | """ 25 | # opts = self.query.model._meta 26 | # self.query.where.add_default_where(opts.get_fields_with_model()) 27 | if not self.query.tables: 28 | self.query.join((None, self.query.model._meta.db_table, None, None)) 29 | if (not self.query.select and self.query.default_cols and not 30 | self.query.included_inherited_models): 31 | self.query.setup_inherited_models() 32 | # if self.query.select_related and not self.query.related_select_cols: 33 | # self.fill_related_selections() 34 | 35 | def quote_name_unless_alias(self, name): 36 | """ 37 | A wrapper around connection.ops.quote_name that doesn't quote aliases 38 | for table names. This avoids problems with some SPARQL dialects that treat 39 | quoted strings specially (e.g. PostgreSPARQL). 40 | """ 41 | if name in self.quote_cache: 42 | return self.quote_cache[name] 43 | if ((name in self.query.alias_map and name not in self.query.table_map) or 44 | name in self.query.extra_select): 45 | self.quote_cache[name] = name 46 | return name 47 | r = self.connection.ops.quote_name(name) 48 | self.quote_cache[name] = r 49 | return r 50 | 51 | # def as_sparql(self, with_limits=True, with_col_aliases=False): 52 | # return """ 53 | # prefix base: 54 | 55 | # select ?uri ?id_do_programa_na_webmedia ?faz_parte_do_canal ?foto_perfil from where { 56 | # ?uri 57 | # base:id_do_programa_na_webmedia ?id_do_programa_na_webmedia ; 58 | # base:faz_parte_do_canal ?faz_parte_do_canal . 59 | # optional { 60 | # ?uri base:foto_perfil ?foto_perfil 61 | # } 62 | # } 63 | # """, [] 64 | 65 | def as_sparql(self, with_limits=True, with_col_aliases=False): 66 | """ 67 | Creates the SPARQL for this query. Returns the SPARQL string and list of 68 | parameters. 69 | 70 | If 'with_limits' is False, any limit/offset information is not included 71 | in the query. 72 | """ 73 | if with_limits and self.query.low_mark == self.query.high_mark: 74 | return '', () 75 | 76 | self.pre_sparql_setup() 77 | out_cols = self.get_columns(with_col_aliases) 78 | ordering, ordering_group_by = self.get_ordering() 79 | 80 | # This must come after 'select' and 'ordering' -- see docstring of 81 | # get_from_clause() for details. 82 | # TODO: garantir que o from busque do grafo certo, olhar casos: from g1: g1:candidatura e from base: base:Programa 83 | from_, f_params = self.get_from_clause() 84 | 85 | qn = self.quote_name_unless_alias 86 | 87 | # where, w_params = self.get_where_triples() 88 | opts = self.query.model._meta 89 | fields = opts.get_fields_with_model() 90 | where, w_params = self.query.where.as_sparql(qn=qn, connection=self.connection, fields=fields) 91 | having, h_params = self.query.having.as_sparql(qn=qn, connection=self.connection) 92 | params = [] 93 | for val in self.query.extra_select.itervalues(): 94 | params.extend(val[1]) 95 | 96 | result = ['SELECT'] 97 | if self.query.distinct: 98 | result.append('DISTINCT') 99 | result.append(' '.join(out_cols + self.query.ordering_aliases)) 100 | 101 | if from_: 102 | result.append(from_) 103 | # params.extend(f_params) 104 | 105 | if where: 106 | result.append('WHERE { %s }' % where) 107 | params.extend(w_params) 108 | 109 | grouping, gb_params = self.get_grouping() 110 | if grouping: 111 | if ordering: 112 | # If the backend can't group by PK (i.e., any database 113 | # other than MySPARQL), then any fields mentioned in the 114 | # ordering clause needs to be in the group by clause. 115 | if not self.connection.features.allows_group_by_pk: 116 | for col, col_params in ordering_group_by: 117 | if col not in grouping: 118 | grouping.append(str(col)) 119 | gb_params.extend(col_params) 120 | else: 121 | ordering = self.connection.ops.force_no_ordering() 122 | result.append('GROUP BY %s' % ', '.join(grouping)) 123 | params.extend(gb_params) 124 | 125 | if having: 126 | result.append('HAVING %s' % having) 127 | params.extend(h_params) 128 | 129 | if ordering: 130 | result.append('ORDER BY %s' % ', '.join(ordering)) 131 | 132 | if with_limits: 133 | if self.query.high_mark is not None: 134 | result.append('LIMIT %d' % (self.query.high_mark - self.query.low_mark)) 135 | if self.query.low_mark: 136 | if self.query.high_mark is None: 137 | val = self.connection.ops.no_limit_value() 138 | if val: 139 | result.append('LIMIT %d' % val) 140 | result.append('OFFSET %d' % self.query.low_mark) 141 | 142 | return ' '.join(result), tuple(params) 143 | 144 | def resolve_columns(self, row, fields=()): 145 | index_extra_select = len(self.query.extra_select.keys()) 146 | row_dict = dict(row[index_extra_select:]) 147 | values = [] 148 | for field in fields: 149 | values.append(row_dict.get(field.attname, '')) 150 | return row[:index_extra_select] + tuple(values) 151 | 152 | def as_nested_sparql(self): 153 | """ 154 | Perform the same functionality as the as_sparql() method, returning an 155 | SPARQL string and parameters. However, the alias prefixes are bumped 156 | beforehand (in a copy -- the current query isn't changed), and any 157 | ordering is removed if the query is unsliced. 158 | 159 | Used when nesting this query inside another. 160 | """ 161 | obj = self.query.clone() 162 | if obj.low_mark == 0 and obj.high_mark is None: 163 | # If there is no slicing in use, then we can safely drop all ordering 164 | obj.clear_ordering(True) 165 | obj.bump_prefix() 166 | return obj.get_compiler(connection=self.connection).as_sparql() 167 | 168 | def get_columns(self, with_aliases=False): 169 | """ 170 | Returns the list of columns to use in the select statement. If no 171 | columns have been specified, returns all columns relating to fields in 172 | the model. 173 | 174 | If 'with_aliases' is true, any column names that are duplicated 175 | (without the table names) are given unique aliases. This is needed in 176 | some cases to avoid ambiguity with nested queries. 177 | """ 178 | qn = self.quote_name_unless_alias 179 | qn2 = self.connection.ops.quote_name 180 | result = ['(%s) AS %s' % (col[0], qn2(alias)) for alias, col in self.query.extra_select.iteritems()] 181 | aliases = set(self.query.extra_select.keys()) 182 | if with_aliases: 183 | col_aliases = aliases.copy() 184 | else: 185 | col_aliases = set() 186 | if self.query.select: 187 | only_load = self.deferred_to_columns() 188 | for col in self.query.select: 189 | if isinstance(col, (list, tuple)): 190 | alias, column = col 191 | table = self.query.alias_map[alias][TABLE_NAME] 192 | if table in only_load and col not in only_load[table]: 193 | continue 194 | r = '%s.%s' % (qn(alias), qn(column)) 195 | if with_aliases: 196 | if col[1] in col_aliases: 197 | c_alias = 'Col%d' % len(col_aliases) 198 | result.append('%s AS %s' % (r, c_alias)) 199 | aliases.add(c_alias) 200 | col_aliases.add(c_alias) 201 | else: 202 | result.append('%s AS %s' % (r, qn2(col[1]))) 203 | aliases.add(r) 204 | col_aliases.add(col[1]) 205 | else: 206 | result.append(r) 207 | aliases.add(r) 208 | col_aliases.add(col[1]) 209 | else: 210 | result.append(col.as_sparql(qn, self.connection)) 211 | 212 | if hasattr(col, 'alias'): 213 | aliases.add(col.alias) 214 | col_aliases.add(col.alias) 215 | 216 | elif self.query.default_cols: 217 | cols, new_aliases = self.get_default_columns(with_aliases, 218 | col_aliases) 219 | result.extend(cols) 220 | aliases.update(new_aliases) 221 | 222 | max_name_length = self.connection.ops.max_name_length() 223 | result.extend([ 224 | '%s%s' % ( 225 | aggregate.as_sparql(qn, self.connection), 226 | alias is not None 227 | and ' AS %s' % qn(truncate_name(alias, max_name_length)) 228 | or '' 229 | ) 230 | for alias, aggregate in self.query.aggregate_select.items() 231 | ]) 232 | 233 | for table, col in self.query.related_select_cols: 234 | r = '%s.%s' % (qn(table), qn(col)) 235 | if with_aliases and col in col_aliases: 236 | c_alias = 'Col%d' % len(col_aliases) 237 | result.append('%s AS %s' % (r, c_alias)) 238 | aliases.add(c_alias) 239 | col_aliases.add(c_alias) 240 | else: 241 | result.append(r) 242 | aliases.add(r) 243 | col_aliases.add(col) 244 | 245 | self._select_aliases = aliases 246 | return result 247 | 248 | def get_default_columns(self, with_aliases=False, col_aliases=None, 249 | start_alias=None, opts=None, as_pairs=False, local_only=False): 250 | """ 251 | Computes the default columns for selecting every field in the base 252 | model. Will sometimes be called to pull in related models (e.g. via 253 | select_related), in which case "opts" and "start_alias" will be given 254 | to provide a starting point for the traversal. 255 | 256 | Returns a list of strings, quoted appropriately for use in SPARQL 257 | directly, as well as a set of aliases used in the select statement (if 258 | 'as_pairs' is True, returns a list of (alias, col_name) pairs instead 259 | of strings as the first component and None as the second component). 260 | """ 261 | result = [] 262 | if opts is None: 263 | opts = self.query.model._meta 264 | qn = self.quote_name_unless_alias 265 | qn2 = self.connection.ops.quote_name 266 | aliases = set() 267 | only_load = self.deferred_to_columns() 268 | # Skip all proxy to the root proxied model 269 | proxied_model = get_proxied_model(opts) 270 | 271 | if start_alias: 272 | seen = {None: start_alias} 273 | for field, model in opts.get_fields_with_model(): 274 | if local_only and model is not None: 275 | continue 276 | if start_alias: 277 | try: 278 | alias = seen[model] 279 | except KeyError: 280 | if model is proxied_model: 281 | alias = start_alias 282 | else: 283 | link_field = opts.get_ancestor_link(model) 284 | alias = self.query.join((start_alias, model._meta.db_table, 285 | link_field.column, model._meta.pk.column)) 286 | seen[model] = alias 287 | else: 288 | # If we're starting from the base model of the queryset, the 289 | # aliases will have already been set up in pre_sparql_setup(), so 290 | # we can save time here. 291 | alias = self.query.included_inherited_models[model] 292 | table = self.query.alias_map[alias][TABLE_NAME] 293 | if table in only_load and field.column not in only_load[table]: 294 | continue 295 | if as_pairs: 296 | result.append((alias, field.column)) 297 | aliases.add(alias) 298 | continue 299 | if with_aliases and field.column in col_aliases: 300 | c_alias = 'Col%d' % len(col_aliases) 301 | result.append('%s.%s AS %s' % (qn(alias), 302 | qn2(field.column), c_alias)) 303 | col_aliases.add(c_alias) 304 | aliases.add(c_alias) 305 | else: 306 | r = '?%s' % (field.column) 307 | result.append(r) 308 | aliases.add(r) 309 | if with_aliases: 310 | col_aliases.add(field.column) 311 | return result, aliases 312 | 313 | def get_ordering(self): 314 | """ 315 | Returns a tuple containing a list representing the SPARQL elements in the 316 | "order by" clause, and the list of SPARQL elements that need to be added 317 | to the GROUP BY clause as a result of the ordering. 318 | 319 | Also sets the ordering_aliases attribute on this instance to a list of 320 | extra aliases needed in the select. 321 | 322 | Determining the ordering SPARQL can change the tables we need to include, 323 | so this should be run *before* get_from_clause(). 324 | """ 325 | if self.query.extra_order_by: 326 | ordering = self.query.extra_order_by 327 | elif not self.query.default_ordering: 328 | ordering = self.query.order_by 329 | else: 330 | ordering = self.query.order_by or self.query.model._meta.ordering 331 | qn = self.quote_name_unless_alias 332 | qn2 = self.connection.ops.quote_name 333 | distinct = self.query.distinct 334 | select_aliases = self._select_aliases 335 | result = [] 336 | group_by = [] 337 | ordering_aliases = [] 338 | if self.query.standard_ordering: 339 | asc, desc = ORDER_DIR['ASC'] 340 | else: 341 | asc, desc = ORDER_DIR['DESC'] 342 | 343 | # It's possible, due to model inheritance, that normal usage might try 344 | # to include the same field more than once in the ordering. We track 345 | # the table/column pairs we use and discard any after the first use. 346 | processed_pairs = set() 347 | 348 | for field in ordering: 349 | if field == '?': 350 | result.append(self.connection.ops.random_function_sparql()) 351 | continue 352 | if isinstance(field, int): 353 | if field < 0: 354 | order = desc 355 | field = -field 356 | else: 357 | order = asc 358 | result.append('%s %s' % (field, order)) 359 | group_by.append((field, [])) 360 | continue 361 | col, order = get_order_dir(field, asc) 362 | if col in self.query.aggregate_select: 363 | result.append('%s %s' % (qn(col), order)) 364 | continue 365 | if '.' in field: 366 | # This came in through an extra(order_by=...) addition. Pass it 367 | # on verbatim. 368 | table, col = col.split('.', 1) 369 | if (table, col) not in processed_pairs: 370 | elt = '%s.%s' % (qn(table), col) 371 | processed_pairs.add((table, col)) 372 | if not distinct or elt in select_aliases: 373 | result.append('%s %s' % (elt, order)) 374 | group_by.append((elt, [])) 375 | elif get_order_dir(field)[0] not in self.query.extra_select: 376 | # 'col' is of the form 'field' or 'field1__field2' or 377 | # '-field1__field2__field', etc. 378 | for table, col, order in self.find_ordering_name(field, 379 | self.query.model._meta, default_order=asc): 380 | if (table, col) not in processed_pairs: 381 | elt = '%s' % (qn2(col)) 382 | processed_pairs.add((table, col)) 383 | if distinct and elt not in select_aliases: 384 | ordering_aliases.append(elt) 385 | result.append('%s[%s]' % (order, elt)) 386 | group_by.append((elt, [])) 387 | else: 388 | elt = qn2(col) 389 | if distinct and col not in select_aliases: 390 | ordering_aliases.append(elt) 391 | result.append('%s %s' % (elt, order)) 392 | group_by.append(self.query.extra_select[col]) 393 | self.query.ordering_aliases = ordering_aliases 394 | return result, group_by 395 | 396 | def find_ordering_name(self, name, opts, alias=None, default_order='ASC', 397 | already_seen=None): 398 | """ 399 | Returns the table alias (the name might be ambiguous, the alias will 400 | not be) and column name for ordering by the given 'name' parameter. 401 | The 'name' is of the form 'field1__field2__...__fieldN'. 402 | """ 403 | name, order = get_order_dir(name, default_order) 404 | pieces = name.split(LOOKUP_SEP) 405 | if not alias: 406 | alias = self.query.get_initial_alias() 407 | field, target, opts, joins, last, extra = self.query.setup_joins(pieces, 408 | opts, alias, False) 409 | alias = joins[-1] 410 | col = target.column 411 | if not field.rel: 412 | # To avoid inadvertent trimming of a necessary alias, use the 413 | # refcount to show that we are referencing a non-relation field on 414 | # the model. 415 | self.query.ref_alias(alias) 416 | 417 | # Must use left outer joins for nullable fields and their relations. 418 | self.query.promote_alias_chain(joins, 419 | self.query.alias_map[joins[0]][JOIN_TYPE] == self.query.LOUTER) 420 | 421 | # If we get to this point and the field is a relation to another model, 422 | # append the default ordering for that model. 423 | if field.rel and len(joins) > 1 and opts.ordering: 424 | # Firstly, avoid infinite loops. 425 | if not already_seen: 426 | already_seen = set() 427 | join_tuple = tuple([self.query.alias_map[j][TABLE_NAME] for j in joins]) 428 | if join_tuple in already_seen: 429 | raise FieldError('Infinite loop caused by ordering.') 430 | already_seen.add(join_tuple) 431 | 432 | results = [] 433 | for item in opts.ordering: 434 | results.extend(self.find_ordering_name(item, opts, alias, 435 | order, already_seen)) 436 | return results 437 | 438 | if alias: 439 | # We have to do the same "final join" optimisation as in 440 | # add_filter, since the final column might not otherwise be part of 441 | # the select set (so we can't order on it). 442 | while 1: 443 | join = self.query.alias_map[alias] 444 | if col != join[RHS_JOIN_COL]: 445 | break 446 | self.query.unref_alias(alias) 447 | alias = join[LHS_ALIAS] 448 | col = join[LHS_JOIN_COL] 449 | return [(alias, col, order)] 450 | 451 | def wrap_graph(self, graph): 452 | if graph.startswith('http'): 453 | return 'FROM <%s>' % (graph) 454 | else: 455 | return 'FROM %s' % (graph) 456 | 457 | def get_from_clause(self): 458 | """ 459 | Returns a list of strings that are joined together to go after the 460 | "FROM" part of the query, as well as a list any extra parameters that 461 | need to be included. Sub-classes, can override this to create a 462 | from-clause via a "select". 463 | 464 | This should only be called after any SPARQL construction methods that 465 | might change the tables we need. This means the select columns and 466 | ordering must be done first. 467 | """ 468 | # result = [] 469 | # qn = self.quote_name_unless_alias 470 | # qn2 = self.connection.ops.quote_name 471 | # first = True 472 | # for alias in self.query.tables: 473 | # if not self.query.alias_refcount[alias]: 474 | # continue 475 | # try: 476 | # name, alias, join_type, lhs, lhs_col, col, nullable = self.query.alias_map[alias] 477 | # except KeyError: 478 | # # Extra tables can end up in self.tables, but not in the 479 | # # alias_map if they aren't in a join. That's OK. We skip them. 480 | # continue 481 | # alias_str = (alias != name and ' %s' % alias or '') 482 | # if join_type and not first: 483 | # result.append('%s %s%s ON (%s.%s = %s.%s)' 484 | # % (join_type, qn(name), alias_str, qn(lhs), 485 | # qn2(lhs_col), qn(alias), qn2(col))) 486 | # else: 487 | # connector = not first and ', ' or '' 488 | # result.append('%s%s%s' % (connector, qn(name), alias_str)) 489 | # first = False 490 | # for t in self.query.extra_tables: 491 | # alias, unused = self.query.table_alias(t) 492 | # # Only add the alias if it's not already present (the table_alias() 493 | # # calls increments the refcount, so an alias refcount of one means 494 | # # this is the only reference. 495 | # if alias not in self.query.alias_map or self.query.alias_refcount[alias] == 1: 496 | # connector = not first and ', ' or '' 497 | # result.append('%s%s' % (connector, qn(alias))) 498 | # first = False 499 | # return result, [] 500 | 501 | graph = self.query.model._meta.graph 502 | from_ = [] 503 | 504 | if isinstance(graph, list): 505 | for g in graph: 506 | from_.append('%s' % self.wrap_graph(g)) 507 | return ' '.join(from_), [] 508 | 509 | def get_grouping(self): 510 | """ 511 | Returns a tuple representing the SPARQL elements in the "group by" clause. 512 | """ 513 | qn = self.quote_name_unless_alias 514 | result, params = [], [] 515 | if self.query.group_by is not None: 516 | if (len(self.query.model._meta.fields) == len(self.query.select) and 517 | self.connection.features.allows_group_by_pk): 518 | self.query.group_by = [ 519 | (self.query.model._meta.db_table, self.query.model._meta.pk.column) 520 | ] 521 | 522 | group_by = self.query.group_by or [] 523 | 524 | extra_selects = [] 525 | for extra_select, extra_params in self.query.extra_select.itervalues(): 526 | extra_selects.append(extra_select) 527 | params.extend(extra_params) 528 | cols = (group_by + self.query.select + 529 | self.query.related_select_cols + extra_selects) 530 | for col in cols: 531 | if isinstance(col, (list, tuple)): 532 | result.append('%s.%s' % (qn(col[0]), qn(col[1]))) 533 | elif hasattr(col, 'as_sparql'): 534 | result.append(col.as_sparql(qn, self.connection)) 535 | else: 536 | result.append('(%s)' % str(col)) 537 | return result, params 538 | 539 | def fill_related_selections(self, opts=None, root_alias=None, cur_depth=1, 540 | used=None, requested=None, restricted=None, nullable=None, 541 | dupe_set=None, avoid_set=None): 542 | """ 543 | Fill in the information needed for a select_related query. The current 544 | depth is measured as the number of connections away from the root model 545 | (for example, cur_depth=1 means we are looking at models with direct 546 | connections to the root model). 547 | """ 548 | if not restricted and self.query.max_depth and cur_depth > self.query.max_depth: 549 | # We've recursed far enough; bail out. 550 | return 551 | 552 | if not opts: 553 | opts = self.query.get_meta() 554 | root_alias = self.query.get_initial_alias() 555 | self.query.related_select_cols = [] 556 | self.query.related_select_fields = [] 557 | if not used: 558 | used = set() 559 | if dupe_set is None: 560 | dupe_set = set() 561 | if avoid_set is None: 562 | avoid_set = set() 563 | orig_dupe_set = dupe_set 564 | 565 | # Setup for the case when only particular related fields should be 566 | # included in the related selection. 567 | if requested is None: 568 | if isinstance(self.query.select_related, dict): 569 | requested = self.query.select_related 570 | restricted = True 571 | else: 572 | restricted = False 573 | 574 | for f, model in opts.get_fields_with_model(): 575 | if not select_related_descend(f, restricted, requested): 576 | continue 577 | # The "avoid" set is aliases we want to avoid just for this 578 | # particular branch of the recursion. They aren't permanently 579 | # forbidden from reuse in the related selection tables (which is 580 | # what "used" specifies). 581 | avoid = avoid_set.copy() 582 | dupe_set = orig_dupe_set.copy() 583 | table = f.rel.to._meta.db_table 584 | promote = nullable or f.null 585 | if model: 586 | int_opts = opts 587 | alias = root_alias 588 | alias_chain = [] 589 | for int_model in opts.get_base_chain(model): 590 | # Proxy model have elements in base chain 591 | # with no parents, assign the new options 592 | # object and skip to the next base in that 593 | # case 594 | if not int_opts.parents[int_model]: 595 | int_opts = int_model._meta 596 | continue 597 | lhs_col = int_opts.parents[int_model].column 598 | dedupe = lhs_col in opts.duplicate_targets 599 | if dedupe: 600 | avoid.update(self.query.dupe_avoidance.get((id(opts), lhs_col), 601 | ())) 602 | dupe_set.add((opts, lhs_col)) 603 | int_opts = int_model._meta 604 | alias = self.query.join((alias, int_opts.db_table, lhs_col, 605 | int_opts.pk.column), exclusions=used, 606 | promote=promote) 607 | alias_chain.append(alias) 608 | for (dupe_opts, dupe_col) in dupe_set: 609 | self.query.update_dupe_avoidance(dupe_opts, dupe_col, alias) 610 | if self.query.alias_map[root_alias][JOIN_TYPE] == self.query.LOUTER: 611 | self.query.promote_alias_chain(alias_chain, True) 612 | else: 613 | alias = root_alias 614 | 615 | dedupe = f.column in opts.duplicate_targets 616 | if dupe_set or dedupe: 617 | avoid.update(self.query.dupe_avoidance.get((id(opts), f.column), ())) 618 | if dedupe: 619 | dupe_set.add((opts, f.column)) 620 | 621 | alias = self.query.join((alias, table, f.column, 622 | f.rel.get_related_field().column), 623 | exclusions=used.union(avoid), promote=promote) 624 | used.add(alias) 625 | columns, aliases = self.get_default_columns(start_alias=alias, 626 | opts=f.rel.to._meta, as_pairs=True) 627 | self.query.related_select_cols.extend(columns) 628 | if self.query.alias_map[alias][JOIN_TYPE] == self.query.LOUTER: 629 | self.query.promote_alias_chain(aliases, True) 630 | self.query.related_select_fields.extend(f.rel.to._meta.fields) 631 | if restricted: 632 | next = requested.get(f.name, {}) 633 | else: 634 | next = False 635 | new_nullable = f.null or promote 636 | for dupe_opts, dupe_col in dupe_set: 637 | self.query.update_dupe_avoidance(dupe_opts, dupe_col, alias) 638 | self.fill_related_selections(f.rel.to._meta, alias, cur_depth + 1, 639 | used, next, restricted, new_nullable, dupe_set, avoid) 640 | 641 | if restricted: 642 | related_fields = [ 643 | (o.field, o.model) 644 | for o in opts.get_all_related_objects() 645 | if o.field.unique 646 | ] 647 | for f, model in related_fields: 648 | if not select_related_descend(f, restricted, requested, reverse=True): 649 | continue 650 | # The "avoid" set is aliases we want to avoid just for this 651 | # particular branch of the recursion. They aren't permanently 652 | # forbidden from reuse in the related selection tables (which is 653 | # what "used" specifies). 654 | avoid = avoid_set.copy() 655 | dupe_set = orig_dupe_set.copy() 656 | table = model._meta.db_table 657 | 658 | int_opts = opts 659 | alias = root_alias 660 | alias_chain = [] 661 | chain = opts.get_base_chain(f.rel.to) 662 | if chain is not None: 663 | for int_model in chain: 664 | # Proxy model have elements in base chain 665 | # with no parents, assign the new options 666 | # object and skip to the next base in that 667 | # case 668 | if not int_opts.parents[int_model]: 669 | int_opts = int_model._meta 670 | continue 671 | lhs_col = int_opts.parents[int_model].column 672 | dedupe = lhs_col in opts.duplicate_targets 673 | if dedupe: 674 | avoid.update((self.query.dupe_avoidance.get(id(opts), lhs_col), 675 | ())) 676 | dupe_set.add((opts, lhs_col)) 677 | int_opts = int_model._meta 678 | alias = self.query.join( 679 | (alias, int_opts.db_table, lhs_col, int_opts.pk.column), 680 | exclusions=used, promote=True, reuse=used 681 | ) 682 | alias_chain.append(alias) 683 | for dupe_opts, dupe_col in dupe_set: 684 | self.query.update_dupe_avoidance(dupe_opts, dupe_col, alias) 685 | dedupe = f.column in opts.duplicate_targets 686 | if dupe_set or dedupe: 687 | avoid.update(self.query.dupe_avoidance.get((id(opts), f.column), ())) 688 | if dedupe: 689 | dupe_set.add((opts, f.column)) 690 | alias = self.query.join( 691 | (alias, table, f.rel.get_related_field().column, f.column), 692 | exclusions=used.union(avoid), 693 | promote=True 694 | ) 695 | used.add(alias) 696 | columns, aliases = self.get_default_columns(start_alias=alias, 697 | opts=model._meta, as_pairs=True, local_only=True) 698 | self.query.related_select_cols.extend(columns) 699 | self.query.related_select_fields.extend(model._meta.fields) 700 | 701 | next = requested.get(f.related_query_name(), {}) 702 | new_nullable = f.null or None 703 | 704 | self.fill_related_selections(model._meta, table, cur_depth+1, 705 | used, next, restricted, new_nullable) 706 | 707 | def deferred_to_columns(self): 708 | """ 709 | Converts the self.deferred_loading data structure to mapping of table 710 | names to sets of column names which are to be loaded. Returns the 711 | dictionary. 712 | """ 713 | columns = {} 714 | self.query.deferred_to_data(columns, self.query.deferred_to_columns_cb) 715 | return columns 716 | 717 | def results_iter(self): 718 | """ 719 | Returns an iterator over the results from executing this query. 720 | """ 721 | resolve_columns = hasattr(self, 'resolve_columns') 722 | fields = None 723 | has_aggregate_select = bool(self.query.aggregate_select) 724 | for row in self.execute_sparql(MULTI): 725 | if resolve_columns: 726 | if fields is None: 727 | # We only set this up here because 728 | # related_select_fields isn't populated until 729 | # execute_sparql() has been called. 730 | if self.query.select_fields: 731 | fields = self.query.select_fields + self.query.related_select_fields 732 | else: 733 | fields = self.query.model._meta.fields 734 | # If the field was deferred, exclude it from being passed 735 | # into `resolve_columns` because it wasn't selected. 736 | only_load = self.deferred_to_columns() 737 | if only_load: 738 | db_table = self.query.model._meta.db_table 739 | fields = [f for f in fields if db_table in only_load and 740 | f.column in only_load[db_table]] 741 | row = self.resolve_columns(row, fields) 742 | 743 | if has_aggregate_select: 744 | aggregate_start = len(self.query.extra_select.keys()) + len(self.query.select) 745 | aggregate_end = aggregate_start + len(self.query.aggregate_select) 746 | row = tuple(row[:aggregate_start]) + tuple([ 747 | self.query.resolve_aggregate(value, aggregate, self.connection) 748 | for (alias, aggregate), value 749 | in zip(self.query.aggregate_select.items(), row[aggregate_start:aggregate_end]) 750 | ]) + tuple(row[aggregate_end:]) 751 | 752 | yield row 753 | 754 | def execute_sparql(self, result_type=MULTI): 755 | """ 756 | Run the query against the database and returns the result(s). The 757 | return value is a single data item if result_type is SINGLE, or an 758 | iterator over the results if the result_type is MULTI. 759 | 760 | result_type is either MULTI (use fetchmany() to retrieve all rows), 761 | SINGLE (only retrieve a single row), or None. In this last case, the 762 | cursor is returned if any query is executed, since it's used by 763 | subclasses such as InsertQuery). It's possible, however, that no query 764 | is needed, as the filters describe an empty set. In that case, None is 765 | returned, to avoid any unnecessary database interaction. 766 | """ 767 | try: 768 | sparql, params = self.as_sparql() 769 | if not sparql: 770 | raise EmptyResultSet 771 | except EmptyResultSet: 772 | if result_type == MULTI: 773 | return empty_iter() 774 | else: 775 | return 776 | 777 | cursor = self.connection.cursor() 778 | cursor.execute(sparql, params) 779 | 780 | if not result_type: 781 | return cursor 782 | if result_type == SINGLE: 783 | if self.query.ordering_aliases: 784 | return cursor.fetchone()[:-len(self.query.ordering_aliases)] 785 | return cursor.fetchone() 786 | 787 | # The MULTI case. 788 | if self.query.ordering_aliases: 789 | result = order_modified_iter(cursor, len(self.query.ordering_aliases), 790 | self.connection.features.empty_fetchmany_value) 791 | else: 792 | # result = iter( 793 | # (lambda: cursor.fetchmany(GET_ITERATOR_CHUNK_SIZE)), 794 | # self.connection.features.empty_fetchmany_value 795 | # ) 796 | result = iter(cursor.fetchmany(GET_ITERATOR_CHUNK_SIZE)) 797 | if not self.connection.features.can_use_chunked_reads: 798 | # If we are using non-chunked reads, we return the same data 799 | # structure as normally, but ensure it is all read into memory 800 | # before going any further. 801 | return list(result) 802 | return result 803 | 804 | 805 | class SPARQLInsertCompiler(SPARQLCompiler): 806 | def placeholder(self, field, val): 807 | if field is None: 808 | # A field value of None means the value is raw. 809 | return val 810 | elif hasattr(field, 'get_placeholder'): 811 | # Some fields (e.g. geo fields) need special munging before 812 | # they can be inserted. 813 | return field.get_placeholder(val, self.connection) 814 | else: 815 | # Return the common case for the placeholder 816 | return '%s' 817 | 818 | def as_sparql(self): 819 | # We don't need quote_name_unless_alias() here, since these are all 820 | # going to be column names (so we can avoid the extra overhead). 821 | qn = self.connection.ops.quote_subject 822 | qn2 = self.connection.ops.quote_predicate 823 | # qn3 = self.connection.ops.quote_object 824 | # qn2 = self.connection.ops.quote_name 825 | opts = self.query.model._meta 826 | result = ['INSERT IN GRAPH %s' % qn(opts.graph)] 827 | 828 | predicates = [] 829 | params = [] 830 | uri = None 831 | 832 | for index, data in enumerate(zip(self.query.columns, self.query.values)): 833 | predicate, data = data 834 | field, value = data 835 | 836 | if field.primary_key: 837 | uri = value 838 | continue 839 | 840 | if not value: 841 | continue 842 | 843 | predicates.append(qn2(field, predicate)) 844 | params.append(value) 845 | 846 | # force insert rdf:type 847 | predicates[0] = "%s %s" % (uri, predicates[0]) 848 | predicates += [' rdf:type %s'] 849 | params += [qn(self.query.model._meta.namespace)] 850 | 851 | result.append('{ %s }' % ('; '.join(predicates))) 852 | return ' '.join(result), params 853 | 854 | def execute_sparql(self, return_id=False): 855 | self.return_id = return_id 856 | cursor = super(SPARQLInsertCompiler, self).execute_sparql(None) 857 | if not (return_id and cursor): 858 | return 859 | if self.connection.features.can_return_id_from_insert: 860 | return self.connection.ops.fetch_returned_insert_id(cursor) 861 | return self.connection.ops.last_insert_id(cursor, 862 | self.query.model._meta.db_table, self.query.model._meta.pk.column) 863 | 864 | 865 | class SPARQLDeleteCompiler(SPARQLCompiler): 866 | def as_sparql(self): 867 | """ 868 | Creates the SPARQL for this query. Returns the SPARQL string and list of 869 | parameters. 870 | """ 871 | qn2 = self.connection.ops.quote_subject 872 | result = ['DELETE FROM %s' % qn2(self.query.model._meta.graph)] 873 | 874 | # DELETE FROM { 875 | # ns1:Pessoa_ImportacaoEleicoes2012TSE_100000017695 ns1:naturalidade ns1:Cidade_Russas_CE 876 | # } 877 | 878 | where, params = self.query.where.as_delete_sparql(qn=qn2, connection=self.connection) 879 | result.append('{ %s } WHERE { %s }' % (where, where)) 880 | params += params 881 | return ' '.join(result), tuple(params) 882 | 883 | 884 | class SPARQLUpdateCompiler(SPARQLCompiler): 885 | def as_sparql(self): 886 | """ 887 | Creates the SPARQL for this query. Returns the SPARQL string and list of 888 | parameters. 889 | """ 890 | from django.db.models.base import Model 891 | 892 | self.pre_sparql_setup() 893 | if not self.query.values: 894 | return '', () 895 | 896 | # MODIFY_QUERY = """ 897 | # MODIFY GRAPH <%(graph)s> 898 | # DELETE { <%(uri)s> ?p ?o } 899 | # INSERT { <%(uri)s> ?p ?o } 900 | # WHERE { <%(uri)s> ?p ?o }; 901 | # """ 902 | 903 | # query = MODIFY_QUERY % {"uri": "x", "graph": "y"} 904 | 905 | qn = self.quote_name_unless_alias 906 | qn2 = self.connection.ops.quote_predicate 907 | qn3 = self.connection.ops.quote_subject 908 | graph = self.query.model._meta.graph 909 | result = ['MODIFY GRAPH %s' % qn3(graph)] 910 | insert, insert_params = [], [] 911 | where, where_params = [], [] 912 | uri = self.query.where.as_sparql(qn=qn, connection=self.connection)[1][0] # FIXME: remove this magic numbers 913 | 914 | for field, model, val in self.query.values: 915 | if hasattr(val, 'prepare_database_save'): 916 | val = val.prepare_database_save(field) 917 | else: 918 | val = field.get_db_prep_save(val, connection=self.connection) 919 | 920 | if not val: 921 | continue 922 | 923 | # # Getting the placeholder for the field. 924 | # if hasattr(field, 'get_placeholder'): 925 | # placeholder = field.get_placeholder(val, self.connection) 926 | # else: 927 | # placeholder = '%s' 928 | 929 | if hasattr(val, 'evaluate'): 930 | val = SPARQLEvaluator(val, self.query, allow_joins=False) 931 | name = field.column 932 | if hasattr(val, 'as_sparql'): 933 | sparql, params = val.as_sparql(qn, self.connection) 934 | insert.append('%s %s' % (qn(name), sparql)) 935 | insert_params.extend(params) 936 | elif val is not None: 937 | # insert.append('%s %s' % (qn2(field, name), placeholder)) 938 | insert.append('%s' % qn2(field, name)) 939 | insert_params.append(val) 940 | else: 941 | insert.append('%s NULL' % qn(name)) 942 | 943 | predicate, where_value, insert_value = qn2(field, name), qn(name), val 944 | where.append(predicate) 945 | where_params.append(where_value) 946 | 947 | insert[0] = "%s %s" % (uri, insert[0]) 948 | where[0] = "%s %s" % (uri, where[0]) 949 | 950 | if not insert: 951 | return '', () 952 | result.append('DELETE { %s }' % '; '.join(where)) 953 | result.append('INSERT { %s }' % '; '.join(insert)) 954 | result.append('WHERE { %s }' % '; '.join(where)) 955 | 956 | return ' '.join(result), tuple(where_params + insert_params + where_params) 957 | 958 | def execute_sparql(self, result_type): 959 | """ 960 | Execute the specified update. Returns the number of rows affected by 961 | the primary update query. The "primary update query" is the first 962 | non-empty query that is executed. Row counts for any subsequent, 963 | related queries are not available. 964 | """ 965 | cursor = super(SPARQLUpdateCompiler, self).execute_sparql(result_type) 966 | rows = cursor and cursor.rowcount or 0 967 | is_empty = cursor is None 968 | del cursor 969 | for query in self.query.get_related_updates(): 970 | aux_rows = query.get_compiler(self.using).execute_sparql(result_type) 971 | if is_empty: 972 | rows = aux_rows 973 | is_empty = False 974 | return rows 975 | 976 | def pre_sparql_setup(self): 977 | """ 978 | If the update depends on results from other tables, we need to do some 979 | munging of the "where" conditions to match the format required for 980 | (portable) SPARQL updates. That is done here. 981 | 982 | Further, if we are going to be running multiple updates, we pull out 983 | the id values to update at this point so that they don't change as a 984 | result of the progressive updates. 985 | """ 986 | self.query.select_related = False 987 | self.query.clear_ordering(True) 988 | super(SPARQLUpdateCompiler, self).pre_sparql_setup() 989 | count = self.query.count_active_tables() 990 | if not self.query.related_updates and count == 1: 991 | return 992 | 993 | # We need to use a sub-select in the where clause to filter on things 994 | # from other tables. 995 | query = self.query.clone(klass=Query) 996 | query.bump_prefix() 997 | query.extra = {} 998 | query.select = [] 999 | query.add_fields([query.model._meta.pk.name]) 1000 | must_pre_select = count > 1 and not self.connection.features.update_can_self_select 1001 | 1002 | # Now we adjust the current query: reset the where clause and get rid 1003 | # of all the tables we don't need (since they're in the sub-select). 1004 | self.query.where = self.query.where_class() 1005 | if self.query.related_updates or must_pre_select: 1006 | # Either we're using the idents in multiple update queries (so 1007 | # don't want them to change), or the db backend doesn't support 1008 | # selecting from the updating table (e.g. MySPARQL). 1009 | idents = [] 1010 | for rows in query.get_compiler(self.using).execute_sparql(MULTI): 1011 | idents.extend([r[0] for r in rows]) 1012 | self.query.add_filter(('pk__in', idents)) 1013 | self.query.related_ids = idents 1014 | else: 1015 | # The fast path. Filters and updates in one query. 1016 | self.query.add_filter(('pk__in', query)) 1017 | for alias in self.query.tables[1:]: 1018 | self.query.alias_refcount[alias] = 0 1019 | 1020 | 1021 | class SPARQLAggregateCompiler(SPARQLCompiler): 1022 | def as_sparql(self, qn=None): 1023 | """ 1024 | Creates the SPARQL for this query. Returns the SPARQL string and list of 1025 | parameters. 1026 | """ 1027 | if qn is None: 1028 | qn = self.quote_name_unless_alias 1029 | sparql = ('SELECT %s FROM (%s) subquery' % ( 1030 | ', '.join([ 1031 | aggregate.as_sparql(qn, self.connection) 1032 | for aggregate in self.query.aggregate_select.values() 1033 | ]), 1034 | self.query.subquery) 1035 | ) 1036 | params = self.query.sub_params 1037 | return (sparql, params) 1038 | 1039 | 1040 | class SPARQLDateCompiler(SPARQLCompiler): 1041 | def results_iter(self): 1042 | """ 1043 | Returns an iterator over the results from executing this query. 1044 | """ 1045 | resolve_columns = hasattr(self, 'resolve_columns') 1046 | if resolve_columns: 1047 | from django.db.models.fields import DateTimeField 1048 | fields = [DateTimeField()] 1049 | else: 1050 | from django.db.backends.util import typecast_timestamp 1051 | needs_string_cast = self.connection.features.needs_datetime_string_cast 1052 | 1053 | offset = len(self.query.extra_select) 1054 | for rows in self.execute_sparql(MULTI): 1055 | for row in rows: 1056 | date = row[offset] 1057 | if resolve_columns: 1058 | date = self.resolve_columns(row, fields)[offset] 1059 | elif needs_string_cast: 1060 | date = typecast_timestamp(str(date)) 1061 | yield date 1062 | 1063 | 1064 | def empty_iter(): 1065 | """ 1066 | Returns an iterator containing no results. 1067 | """ 1068 | yield iter([]).next() 1069 | 1070 | 1071 | def order_modified_iter(cursor, trim, sentinel): 1072 | """ 1073 | Yields blocks of rows from a cursor. We use this iterator in the special 1074 | case when extra output columns have been added to support ordering 1075 | requirements. We must trim those extra columns before anything else can use 1076 | the results, since they're only needed to make the SPARQL valid. 1077 | """ 1078 | for rows in iter((lambda: cursor.fetchmany(GET_ITERATOR_CHUNK_SIZE)), 1079 | sentinel): 1080 | yield [r[:-trim] for r in rows] 1081 | -------------------------------------------------------------------------------- /semantic/rdf/models/sparql/constants.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | # Valid query types (a dictionary is used for speedy lookups). 4 | QUERY_TERMS = dict([(x, None) for x in ( 5 | 'exact', 'iexact', 'contains', 'icontains', 'gt', 'gte', 'lt', 'lte', 'in', 6 | 'startswith', 'istartswith', 'endswith', 'iendswith', 'range', 'year', 7 | 'month', 'day', 'week_day', 'isnull', 'search', 'regex', 'iregex', 8 | )]) 9 | 10 | # Size of each "chunk" for get_iterator calls. 11 | # Larger values are slightly faster at the expense of more storage space. 12 | GET_ITERATOR_CHUNK_SIZE = 100 13 | 14 | # Separator used to split filter strings apart. 15 | LOOKUP_SEP = '__' 16 | 17 | # Constants to make looking up tuple values clearer. 18 | # Join lists (indexes into the tuples that are values in the alias_map 19 | # dictionary in the Query class). 20 | TABLE_NAME = 0 21 | RHS_ALIAS = 1 22 | JOIN_TYPE = 2 23 | LHS_ALIAS = 3 24 | LHS_JOIN_COL = 4 25 | RHS_JOIN_COL = 5 26 | NULLABLE = 6 27 | 28 | # How many results to expect from a cursor.execute call 29 | MULTI = 'multi' 30 | SINGLE = 'single' 31 | 32 | ORDER_PATTERN = re.compile(r'\?|[-+]?[.\w]+$') 33 | ORDER_DIR = { 34 | 'ASC': ('ASC', 'DESC'), 35 | 'DESC': ('DESC', 'ASC')} 36 | 37 | 38 | -------------------------------------------------------------------------------- /semantic/rdf/models/sparql/datastructures.py: -------------------------------------------------------------------------------- 1 | """ 2 | Useful auxilliary data structures for query construction. Not useful outside 3 | the SPARQL domain. 4 | """ 5 | 6 | class EmptyResultSet(Exception): 7 | pass 8 | 9 | class FullResultSet(Exception): 10 | pass 11 | 12 | class MultiJoin(Exception): 13 | """ 14 | Used by join construction code to indicate the point at which a 15 | multi-valued join was attempted (if the caller wants to treat that 16 | exceptionally). 17 | """ 18 | def __init__(self, level): 19 | self.level = level 20 | 21 | class Empty(object): 22 | pass 23 | 24 | class RawValue(object): 25 | def __init__(self, value): 26 | self.value = value 27 | 28 | class Date(object): 29 | """ 30 | Add a date selection column. 31 | """ 32 | def __init__(self, col, lookup_type): 33 | self.col = col 34 | self.lookup_type = lookup_type 35 | 36 | def relabel_aliases(self, change_map): 37 | c = self.col 38 | if isinstance(c, (list, tuple)): 39 | self.col = (change_map.get(c[0], c[0]), c[1]) 40 | 41 | def as_sparql(self, qn, connection): 42 | if isinstance(self.col, (list, tuple)): 43 | col = '%s.%s' % tuple([qn(c) for c in self.col]) 44 | else: 45 | col = self.col 46 | return connection.ops.date_trunc_sparql(self.lookup_type, col) 47 | -------------------------------------------------------------------------------- /semantic/rdf/models/sparql/expressions.py: -------------------------------------------------------------------------------- 1 | from django.core.exceptions import FieldError 2 | from django.db.models.fields import FieldDoesNotExist 3 | from django.db.models.sql.constants import LOOKUP_SEP 4 | 5 | class SPARQLEvaluator(object): 6 | def __init__(self, expression, query, allow_joins=True): 7 | self.expression = expression 8 | self.opts = query.get_meta() 9 | self.cols = {} 10 | 11 | self.contains_aggregate = False 12 | self.expression.prepare(self, query, allow_joins) 13 | 14 | def prepare(self): 15 | return self 16 | 17 | def as_sparql(self, qn, connection): 18 | return self.expression.evaluate(self, qn, connection) 19 | 20 | def relabel_aliases(self, change_map): 21 | for node, col in self.cols.items(): 22 | if hasattr(col, "relabel_aliases"): 23 | col.relabel_aliases(change_map) 24 | else: 25 | self.cols[node] = (change_map.get(col[0], col[0]), col[1]) 26 | 27 | ##################################################### 28 | # Vistor methods for initial expression preparation # 29 | ##################################################### 30 | 31 | def prepare_node(self, node, query, allow_joins): 32 | for child in node.children: 33 | if hasattr(child, 'prepare'): 34 | child.prepare(self, query, allow_joins) 35 | 36 | def prepare_leaf(self, node, query, allow_joins): 37 | if not allow_joins and LOOKUP_SEP in node.name: 38 | raise FieldError("Joined field references are not permitted in this query") 39 | 40 | field_list = node.name.split(LOOKUP_SEP) 41 | if (len(field_list) == 1 and 42 | node.name in query.aggregate_select.keys()): 43 | self.contains_aggregate = True 44 | self.cols[node] = query.aggregate_select[node.name] 45 | else: 46 | try: 47 | field, source, opts, join_list, last, _ = query.setup_joins( 48 | field_list, query.get_meta(), 49 | query.get_initial_alias(), False) 50 | col, _, join_list = query.trim_joins(source, join_list, last, False) 51 | 52 | self.cols[node] = (join_list[-1], col) 53 | except FieldDoesNotExist: 54 | raise FieldError("Cannot resolve keyword %r into field. " 55 | "Choices are: %s" % (self.name, 56 | [f.name for f in self.opts.fields])) 57 | 58 | ################################################## 59 | # Vistor methods for final expression evaluation # 60 | ################################################## 61 | 62 | def evaluate_node(self, node, qn, connection): 63 | expressions = [] 64 | expression_params = [] 65 | for child in node.children: 66 | if hasattr(child, 'evaluate'): 67 | sparql, params = child.evaluate(self, qn, connection) 68 | else: 69 | sparql, params = '%s', (child,) 70 | 71 | if len(getattr(child, 'children', [])) > 1: 72 | format = '(%s)' 73 | else: 74 | format = '%s' 75 | 76 | if sparql: 77 | expressions.append(format % sparql) 78 | expression_params.extend(params) 79 | 80 | return connection.ops.combine_expression(node.connector, expressions), expression_params 81 | 82 | def evaluate_leaf(self, node, qn, connection): 83 | col = self.cols[node] 84 | if hasattr(col, 'as_sparql'): 85 | return col.as_sparql(qn, connection), () 86 | else: 87 | return '%s.%s' % (qn(col[0]), qn(col[1])), () 88 | 89 | def evaluate_date_modifier_node(self, node, qn, connection): 90 | timedelta = node.children.pop() 91 | sparql, params = self.evaluate_node(node, qn, connection) 92 | 93 | if timedelta.days == 0 and timedelta.seconds == 0 and \ 94 | timedelta.microseconds == 0: 95 | return sparql, params 96 | 97 | return connection.ops.date_interval_sparql(sparql, node.connector, timedelta), params 98 | -------------------------------------------------------------------------------- /semantic/rdf/models/sparql/subqueries.py: -------------------------------------------------------------------------------- 1 | """ 2 | Query subclasses which provide extra functionality beyond simple data retrieval. 3 | """ 4 | 5 | from django.core.exceptions import FieldError 6 | from django.db.models.fields import DateField, FieldDoesNotExist 7 | from django.db.models.sql.constants import * 8 | from django.db.models.sql.datastructures import Date 9 | 10 | from semantic.rdf.models.sparql.query import SparqlQuery 11 | from semantic.rdf.models.sparql.where import AND, Constraint 12 | 13 | 14 | __all__ = ['DeleteQuery', 'UpdateQuery', 'InsertQuery', 'DateQuery', 15 | 'AggregateQuery'] 16 | 17 | 18 | class DeleteQuery(SparqlQuery): 19 | """ 20 | Delete queries are done through this class, since they are more constrained 21 | than general queries. 22 | """ 23 | 24 | compiler = 'SPARQLDeleteCompiler' 25 | 26 | def do_query(self, table, where, using): 27 | self.tables = [table] 28 | self.where = where 29 | self.get_compiler(using).execute_sparql(None) 30 | 31 | def delete_batch(self, pk_list, using, field=None): 32 | """ 33 | Set up and execute delete queries for all the objects in pk_list. 34 | 35 | More than one physical query may be executed if there are a 36 | lot of values in pk_list. 37 | """ 38 | if not field: 39 | field = self.model._meta.pk 40 | for offset in range(0, len(pk_list), GET_ITERATOR_CHUNK_SIZE): 41 | where = self.where_class() 42 | where.add((Constraint(None, field.column, field), 'in', 43 | pk_list[offset : offset + GET_ITERATOR_CHUNK_SIZE]), AND) 44 | self.do_query(self.model._meta.db_table, where, using=using) 45 | 46 | 47 | class UpdateQuery(SparqlQuery): 48 | """ 49 | Represents an "update" SPARQL query. 50 | """ 51 | # Based on UpdateQuery, available at django/db/models/sql/subqueries.py 52 | 53 | compiler = 'SPARQLUpdateCompiler' 54 | 55 | def __init__(self, *args, **kwargs): 56 | super(UpdateQuery, self).__init__(*args, **kwargs) 57 | self._setup_query() 58 | 59 | def _setup_query(self): 60 | """ 61 | Runs on initialization and after cloning. Any attributes that would 62 | normally be set in __init__ should go in here, instead, so that they 63 | are also set up after a clone() call. 64 | """ 65 | self.values = [] 66 | self.related_ids = None 67 | if not hasattr(self, 'related_updates'): 68 | self.related_updates = {} 69 | 70 | def clone(self, klass=None, **kwargs): 71 | return super(UpdateQuery, self).clone(klass, 72 | related_updates=self.related_updates.copy(), **kwargs) 73 | 74 | 75 | def update_batch(self, pk_list, values, using): 76 | pk_field = self.model._meta.pk 77 | self.add_update_values(values) 78 | for offset in range(0, len(pk_list), GET_ITERATOR_CHUNK_SIZE): 79 | self.where = self.where_class() 80 | self.where.add((Constraint(None, pk_field.column, pk_field), 'in', 81 | pk_list[offset : offset + GET_ITERATOR_CHUNK_SIZE]), 82 | AND) 83 | self.get_compiler(using).execute_sparql(None) 84 | 85 | def add_update_values(self, values): 86 | """ 87 | Convert a dictionary of field name to value mappings into an update 88 | query. This is the entry point for the public update() method on 89 | querysets. 90 | """ 91 | values_seq = [] 92 | for name, val in values.iteritems(): 93 | field, model, direct, m2m = self.model._meta.get_field_by_name(name) 94 | if not direct or m2m: 95 | raise FieldError('Cannot update model field %r (only non-relations and foreign keys permitted).' % field) 96 | if model: 97 | self.add_related_update(model, field, val) 98 | continue 99 | values_seq.append((field, model, val)) 100 | return self.add_update_fields(values_seq) 101 | 102 | def add_update_fields(self, values_seq): 103 | """ 104 | Turn a sequence of (field, model, value) triples into an update query. 105 | Used by add_update_values() as well as the "fast" update path when 106 | saving models. 107 | """ 108 | self.values.extend(values_seq) 109 | 110 | def add_related_update(self, model, field, value): 111 | """ 112 | Adds (name, value) to an update query for an ancestor model. 113 | 114 | Updates are coalesced so that we only run one update query per ancestor. 115 | """ 116 | try: 117 | self.related_updates[model].append((field, None, value)) 118 | except KeyError: 119 | self.related_updates[model] = [(field, None, value)] 120 | 121 | def get_related_updates(self): 122 | """ 123 | Returns a list of query objects: one for each update required to an 124 | ancestor model. Each query will have the same filtering conditions as 125 | the current query but will only update a single table. 126 | """ 127 | if not self.related_updates: 128 | return [] 129 | result = [] 130 | for model, values in self.related_updates.iteritems(): 131 | query = UpdateQuery(model) 132 | query.values = values 133 | if self.related_ids is not None: 134 | query.add_filter(('pk__in', self.related_ids)) 135 | result.append(query) 136 | return result 137 | 138 | 139 | class InsertQuery(SparqlQuery): 140 | compiler = 'SPARQLInsertCompiler' 141 | 142 | def __init__(self, *args, **kwargs): 143 | super(InsertQuery, self).__init__(*args, **kwargs) 144 | self.columns = [] 145 | self.values = [] 146 | self.params = () 147 | 148 | def clone(self, klass=None, **kwargs): 149 | extras = { 150 | 'columns': self.columns[:], 151 | 'values': self.values[:], 152 | 'params': self.params 153 | } 154 | extras.update(kwargs) 155 | return super(InsertQuery, self).clone(klass, **extras) 156 | 157 | def insert_values(self, insert_values, raw_values=False): 158 | """ 159 | Set up the insert query from the 'insert_values' dictionary. The 160 | dictionary gives the model field names and their target values. 161 | 162 | If 'raw_values' is True, the values in the 'insert_values' dictionary 163 | are inserted directly into the query, rather than passed as SPARQL 164 | parameters. This provides a way to insert NULL and DEFAULT keywords 165 | into the query, for example. 166 | """ 167 | placeholders, values = [], [] 168 | for field, val in insert_values: 169 | placeholders.append((field, val)) 170 | self.columns.append(field.column) 171 | values.append(val) 172 | if raw_values: 173 | self.values.extend([(None, v) for v in values]) 174 | else: 175 | self.params += tuple(values) 176 | self.values.extend(placeholders) 177 | 178 | 179 | class DateQuery(SparqlQuery): 180 | """ 181 | A DateQuery is a normal query, except that it specifically selects a single 182 | date field. This requires some special handling when converting the results 183 | back to Python objects, so we put it in a separate class. 184 | """ 185 | 186 | compiler = 'SPARQLDateCompiler' 187 | 188 | def add_date_select(self, field_name, lookup_type, order='ASC'): 189 | """ 190 | Converts the query into a date extraction query. 191 | """ 192 | try: 193 | result = self.setup_joins( 194 | field_name.split(LOOKUP_SEP), 195 | self.get_meta(), 196 | self.get_initial_alias(), 197 | False 198 | ) 199 | except FieldError: 200 | raise FieldDoesNotExist("%s has no field named '%s'" % ( 201 | self.model._meta.object_name, field_name 202 | )) 203 | field = result[0] 204 | assert isinstance(field, DateField), "%r isn't a DateField." \ 205 | % field.name 206 | alias = result[3][-1] 207 | select = Date((alias, field.column), lookup_type) 208 | self.select = [select] 209 | self.select_fields = [None] 210 | self.select_related = False # See #7097. 211 | self.set_extra_mask([]) 212 | self.distinct = True 213 | self.order_by = order == 'ASC' and [1] or [-1] 214 | 215 | if field.null: 216 | self.add_filter(("%s__isnull" % field_name, False)) 217 | 218 | 219 | class AggregateQuery(SparqlQuery): 220 | """ 221 | An AggregateQuery takes another query as a parameter to the FROM 222 | clause and only selects the elements in the provided list. 223 | """ 224 | 225 | compiler = 'SPARQLAggregateCompiler' 226 | 227 | def add_subquery(self, query, using): 228 | self.subquery, self.sub_params = query.get_compiler(using).as_sparql(with_col_aliases=True) 229 | -------------------------------------------------------------------------------- /semantic/rdf/models/sparql/where.py: -------------------------------------------------------------------------------- 1 | """ 2 | Code to manage the creation and SPARQL rendering of 'where' constraints. 3 | """ 4 | import datetime 5 | from itertools import repeat 6 | 7 | from django.utils import tree 8 | from django.db.models.fields import Field 9 | # from django.db.models.query_utils import QueryWrapper 10 | from datastructures import EmptyResultSet, FullResultSet 11 | 12 | # Connection types 13 | AND = '&&' 14 | OR = '||' 15 | 16 | DOT = '.' 17 | SEMICOLON = ';' 18 | SPACE = ' ' 19 | 20 | 21 | class EmptyShortCircuit(Exception): 22 | """ 23 | Internal exception used to indicate that a "matches nothing" node should be 24 | added to the where-clause. 25 | """ 26 | pass 27 | 28 | 29 | class WhereNode(tree.Node): 30 | """ 31 | Used to represent the SPARQL where-clause. 32 | 33 | The class is tied to the Query class that created it (in order to create 34 | the correct SPARQL). 35 | 36 | The children in this tree are usually either Q-like objects or lists of 37 | [table_alias, field_name, db_type, lookup_type, value_annotation, 38 | params]. However, a child could also be any class with as_sparql() and 39 | relabel_aliases() methods. 40 | """ 41 | default = SEMICOLON 42 | 43 | def add_default_where(self, data): 44 | 45 | # def get_where_triples(self): 46 | # # place this method in where.py 47 | # tiple_format = '%(graph)s:%(field_name)s ?%(field_name)s' 48 | # opts = self.query.model._meta 49 | # uri_field = '' 50 | # where = [] 51 | # where_optional = [] 52 | # for field, model in opts.get_fields_with_model(): 53 | # if field.primary_key: 54 | # uri_field = field.name 55 | # else: 56 | # if field.blank: 57 | # where_optional.append(tiple_format % { 58 | # 'graph': field.graph, 59 | # 'field_name': field.name 60 | # }) 61 | # else: 62 | # where.append(tiple_format % { 63 | # 'graph': field.graph, 64 | # 'field_name': field.name 65 | # }) 66 | # where_string = '?%s ' % uri_field 67 | # where_string += '; '.join(where) 68 | # for optional in where_optional: 69 | # where_string += ' OPTIONAL { ?%s %s }' % (uri_field, optional) 70 | 71 | # return where_string, '' 72 | 73 | default = [] 74 | optional = [] 75 | result_params = [] 76 | for field, model in sorted(data, key=lambda field: field[0].blank): 77 | triple = Triple(field) 78 | if not triple.blank: 79 | array = default 80 | else: 81 | array = optional 82 | triple_string, triple_params = triple.as_sparql() 83 | if triple_string: 84 | array.append(triple_string) 85 | result_params.extend(triple_params) 86 | 87 | conn = ' %s ' % SEMICOLON 88 | sparql_string = conn.join(default) 89 | sparql_string += ' ' 90 | conn = '%s' % SPACE 91 | sparql_string += conn.join(optional) 92 | return sparql_string, result_params 93 | 94 | def add(self, data, connector): 95 | """ 96 | Add a node to the where-tree. If the data is a list or tuple, it is 97 | expected to be of the form (obj, lookup_type, value), where obj is 98 | a Constraint object, and is then slightly munged before being stored 99 | (to avoid storing any reference to field objects). Otherwise, the 'data' 100 | is stored unchanged and can be any class with an 'as_sparql()' method. 101 | """ 102 | # (, 'exact', 'bla') 103 | if not isinstance(data, (list, tuple)): 104 | super(WhereNode, self).add(data, connector) 105 | return 106 | 107 | obj, lookup_type, value = data 108 | if hasattr(value, '__iter__') and hasattr(value, 'next'): 109 | # Consume any generators immediately, so that we can determine 110 | # emptiness and transform any non-empty values correctly. 111 | value = list(value) 112 | 113 | # The "annotation" parameter is used to pass auxilliary information 114 | # about the value(s) to the query construction. Specifically, datetime 115 | # and empty values need special handling. Other types could be used 116 | # here in the future (using Python types is suggested for consistency). 117 | if isinstance(value, datetime.datetime): 118 | annotation = datetime.datetime 119 | elif hasattr(value, 'value_annotation'): 120 | annotation = value.value_annotation 121 | else: 122 | annotation = bool(value) 123 | 124 | if hasattr(obj, "prepare"): 125 | value = obj.prepare(lookup_type, value) 126 | super(WhereNode, self).add((obj, lookup_type, annotation, value), 127 | connector) 128 | return 129 | 130 | super(WhereNode, self).add((obj, lookup_type, annotation, value), 131 | connector) 132 | 133 | def as_sparql(self, qn, connection, fields=None): 134 | """ 135 | Returns the SPARQL version of the where clause and the value to be 136 | substituted in. Returns None, None if this node is empty. 137 | 138 | If 'node' is provided, that is the root of the SPARQL generation 139 | (generally not needed except by the internal implementation for 140 | recursion). 141 | """ 142 | 143 | result = [] 144 | result_params = [] 145 | for child in self.children: 146 | try: 147 | if hasattr(child, 'as_sparql'): 148 | sparql, params = child.as_sparql(qn=qn, connection=connection) 149 | else: 150 | # A leaf node in the tree. 151 | sparql, params = self.make_atom(child, qn, connection) 152 | 153 | except EmptyResultSet: 154 | if self.connector == AND and not self.negated: 155 | # We can bail out early in this particular case (only). 156 | raise 157 | continue 158 | except FullResultSet: 159 | if self.connector == OR: 160 | if self.negated: 161 | break 162 | # We match everything. No need for any constraints. 163 | return '', [] 164 | continue 165 | 166 | if sparql: 167 | result.append(sparql) 168 | result_params.extend(params) 169 | 170 | conn = ' %s ' % self.connector 171 | sparql_string = conn.join(result) 172 | if sparql_string: 173 | if self.negated: 174 | sparql_string = '!%s' % sparql_string 175 | elif len(self.children) != 1: 176 | sparql_string = '%s' % sparql_string 177 | 178 | default_params = [] 179 | if fields: 180 | default_where, default_params = self.add_default_where(fields) 181 | if sparql_string: 182 | sparql_string = default_where + ' FILTER(' + sparql_string + ')' 183 | else: 184 | sparql_string = default_where 185 | 186 | default_params.extend(result_params) 187 | return sparql_string, default_params 188 | 189 | def as_delete_sparql(self, qn, connection, fields=None): 190 | """ 191 | Returns the SPARQL version of the where clause and the value to be 192 | substituted in. Returns None, None if this node is empty. 193 | 194 | If 'node' is provided, that is the root of the SPARQL generation 195 | (generally not needed except by the internal implementation for 196 | recursion). 197 | """ 198 | 199 | result = [] 200 | result_params = [] 201 | for child in self.children: 202 | try: 203 | if hasattr(child, 'as_sparql'): 204 | sparql, params = child.as_sparql(qn=qn, connection=connection) 205 | else: 206 | # A leaf node in the tree. 207 | sparql, params = self.make_atom_delete(child, qn, connection) 208 | 209 | except EmptyResultSet: 210 | if self.connector == AND and not self.negated: 211 | # We can bail out early in this particular case (only). 212 | raise 213 | continue 214 | except FullResultSet: 215 | if self.connector == OR: 216 | if self.negated: 217 | break 218 | # We match everything. No need for any constraints. 219 | return '', [] 220 | continue 221 | 222 | if sparql: 223 | result.append(sparql) 224 | result_params.extend(params) 225 | 226 | conn = ' %s ' % self.connector 227 | return conn.join(result), result_params 228 | 229 | def make_atom(self, child, qn, connection): 230 | """ 231 | Turn a tuple (table_alias, column_name, db_type, lookup_type, 232 | value_annot, params) into valid SPARQL. 233 | 234 | Returns the string for the SPARQL fragment and the parameters to use for 235 | it. 236 | """ 237 | lvalue, lookup_type, value_annot, params_or_value = child 238 | if hasattr(lvalue, 'process'): 239 | try: 240 | lvalue, params = lvalue.process(lookup_type, params_or_value, connection) 241 | except EmptyShortCircuit: 242 | raise EmptyResultSet 243 | else: 244 | params = Field().get_db_prep_lookup(lookup_type, params_or_value, 245 | connection=connection, prepared=True) 246 | 247 | if isinstance(lvalue, tuple): 248 | # A direct database column lookup. 249 | field_sparql = self.sparql_for_columns(lvalue, qn, connection) 250 | else: 251 | # A smart object with an as_sparql() method. 252 | field_sparql = lvalue.as_sparql(qn, connection) 253 | 254 | if value_annot is datetime.datetime: 255 | cast_sparql = connection.ops.datetime_cast_sparql() 256 | else: 257 | cast_sparql = '%s' 258 | 259 | if hasattr(params, 'as_sparql'): 260 | extra, params = params.as_sparql(qn, connection) 261 | cast_sparql = '' 262 | else: 263 | extra = '' 264 | 265 | if (len(params) == 1 and params[0] == '' and lookup_type == 'exact' 266 | and connection.features.interprets_empty_strings_as_nulls): 267 | lookup_type = 'isnull' 268 | value_annot = True 269 | 270 | if lookup_type in connection.operators: 271 | if lookup_type in ('iexact', 'icontains', 'istartswith', 'iendswith', 'contains', 'startswith', 'endswith', 'regex', 'iregex'): 272 | format = "REGEX(%s, %%s %%s)" % (connection.ops.lookup_cast(lookup_type),) 273 | else: 274 | format = "%s %%s %%s" % (connection.ops.lookup_cast(lookup_type),) 275 | return (format % (field_sparql, 276 | connection.operators[lookup_type] % cast_sparql, 277 | extra), params) 278 | 279 | if lookup_type == 'in': 280 | if not value_annot: 281 | raise EmptyResultSet 282 | if extra: 283 | return ('%s IN %s' % (field_sparql, extra), params) 284 | max_in_list_size = connection.ops.max_in_list_size() 285 | if max_in_list_size and len(params) > max_in_list_size: 286 | # Break up the params list into an OR of manageable chunks. 287 | in_clause_elements = ['('] 288 | for offset in xrange(0, len(params), max_in_list_size): 289 | if offset > 0: 290 | in_clause_elements.append(' OR ') 291 | in_clause_elements.append('%s IN (' % field_sparql) 292 | group_size = min(len(params) - offset, max_in_list_size) 293 | param_group = ', '.join(repeat('%s', group_size)) 294 | in_clause_elements.append(param_group) 295 | in_clause_elements.append(')') 296 | in_clause_elements.append(')') 297 | return ''.join(in_clause_elements), params 298 | else: 299 | return ('%s IN (%s)' % (field_sparql, 300 | ', '.join(repeat('%s', len(params)))), 301 | params) 302 | elif lookup_type in ('range', 'year'): 303 | return ('%s BETWEEN %%s and %%s' % field_sparql, params) 304 | elif lookup_type in ('month', 'day', 'week_day'): 305 | return ('%s = %%s' % connection.ops.date_extract_sparql(lookup_type, field_sparql), 306 | params) 307 | elif lookup_type == 'isnull': 308 | return ('%s IS %sNULL' % (field_sparql, 309 | (not value_annot and 'NOT ' or '')), ()) 310 | elif lookup_type == 'search': 311 | return (connection.ops.fulltext_search_sparql(field_sparql), params) 312 | elif lookup_type in ('regex', 'iregex'): 313 | return connection.ops.regex_lookup(lookup_type) % (field_sparql, cast_sparql), params 314 | 315 | raise TypeError('Invalid lookup_type: %r' % lookup_type) 316 | 317 | def make_atom_delete(self, child, qn, connection): 318 | """ 319 | Turn a tuple (table_alias, column_name, db_type, lookup_type, 320 | value_annot, params) into valid SPARQL. 321 | 322 | Returns the string for the SPARQL fragment and the parameters to use for 323 | it. 324 | """ 325 | lvalue, lookup_type, value_annot, params_or_value = child 326 | if hasattr(lvalue, 'process'): 327 | try: 328 | lvalue, params = lvalue.process(lookup_type, params_or_value, connection) 329 | except EmptyShortCircuit: 330 | raise EmptyResultSet 331 | else: 332 | params = Field().get_db_prep_lookup(lookup_type, params_or_value, 333 | connection=connection, prepared=True) 334 | 335 | if isinstance(lvalue, tuple): 336 | # A direct database column lookup. 337 | field_sparql = self.sparql_for_columns(lvalue, qn, connection) 338 | else: 339 | # A smart object with an as_sparql() method. 340 | field_sparql = lvalue.as_sparql(qn, connection) 341 | 342 | if value_annot is datetime.datetime: 343 | cast_sparql = connection.ops.datetime_cast_sparql() 344 | else: 345 | cast_sparql = '%s' 346 | 347 | return '%s ?s ?p', params 348 | 349 | def sparql_for_columns(self, data, qn, connection): 350 | """ 351 | Returns the SPARQL fragment used for the left-hand side of a column 352 | constraint (for example, the "T1.foo" portion in the clause 353 | "WHERE ... T1.foo = 6"). 354 | """ 355 | # TODO: improve this method to add a namespace to cases that the propertie exist in more than one namespace 356 | table_alias, name, db_type = data 357 | if table_alias: 358 | lhs = '%s.%s' % (qn(table_alias), qn(name)) 359 | else: 360 | lhs = qn(name) 361 | return connection.ops.field_cast_sparql(db_type) % lhs 362 | 363 | def relabel_aliases(self, change_map, node=None): 364 | """ 365 | Relabels the alias values of any children. 'change_map' is a dictionary 366 | mapping old (current) alias values to the new values. 367 | """ 368 | if not node: 369 | node = self 370 | for pos, child in enumerate(node.children): 371 | if hasattr(child, 'relabel_aliases'): 372 | child.relabel_aliases(change_map) 373 | elif isinstance(child, tree.Node): 374 | self.relabel_aliases(change_map, child) 375 | elif isinstance(child, (list, tuple)): 376 | if isinstance(child[0], (list, tuple)): 377 | elt = list(child[0]) 378 | if elt[0] in change_map: 379 | elt[0] = change_map[elt[0]] 380 | node.children[pos] = (tuple(elt),) + child[1:] 381 | else: 382 | child[0].relabel_aliases(change_map) 383 | 384 | # Check if the query value also requires relabelling 385 | if hasattr(child[3], 'relabel_aliases'): 386 | child[3].relabel_aliases(change_map) 387 | 388 | 389 | class EverythingNode(object): 390 | """ 391 | A node that matches everything. 392 | """ 393 | 394 | def as_sparql(self, qn=None, connection=None): 395 | raise FullResultSet 396 | 397 | def relabel_aliases(self, change_map, node=None): 398 | return 399 | 400 | 401 | class NothingNode(object): 402 | """ 403 | A node that matches nothing. 404 | """ 405 | def as_sparql(self, qn=None, connection=None): 406 | raise EmptyResultSet 407 | 408 | def relabel_aliases(self, change_map, node=None): 409 | return 410 | 411 | 412 | class ExtraWhere(object): 413 | def __init__(self, sparqls, params): 414 | self.sparqls = sparqls 415 | self.params = params 416 | 417 | def as_sparql(self, qn=None, connection=None): 418 | conn = " %s " % AND 419 | return conn.join(self.sparqls), tuple(self.params or ()) 420 | 421 | 422 | class Constraint(object): 423 | """ 424 | An object that can be passed to WhereNode.add() and knows how to 425 | pre-process itself prior to including in the WhereNode. 426 | """ 427 | def __init__(self, alias, col, field): 428 | self.alias, self.col, self.field = alias, col, field 429 | 430 | def __getstate__(self): 431 | """Save the state of the Constraint for pickling. 432 | 433 | Fields aren't necessarily pickleable, because they can have 434 | callable default values. So, instead of pickling the field 435 | store a reference so we can restore it manually 436 | """ 437 | obj_dict = self.__dict__.copy() 438 | if self.field: 439 | obj_dict['model'] = self.field.model 440 | obj_dict['field_name'] = self.field.name 441 | del obj_dict['field'] 442 | return obj_dict 443 | 444 | def __setstate__(self, data): 445 | """Restore the constraint """ 446 | model = data.pop('model', None) 447 | field_name = data.pop('field_name', None) 448 | self.__dict__.update(data) 449 | if model is not None: 450 | self.field = model._meta.get_field(field_name) 451 | else: 452 | self.field = None 453 | 454 | def prepare(self, lookup_type, value): 455 | if self.field: 456 | return self.field.get_prep_lookup(lookup_type, value) 457 | return value 458 | 459 | def process(self, lookup_type, value, connection): 460 | """ 461 | Returns a tuple of data suitable for inclusion in a WhereNode 462 | instance. 463 | """ 464 | # Because of circular imports, we need to import this here. 465 | from django.db.models.base import ObjectDoesNotExist 466 | try: 467 | if self.field: 468 | params = self.field.get_db_prep_lookup(lookup_type, value, 469 | connection=connection, prepared=True) 470 | db_type = self.field.db_type(connection=connection) 471 | else: 472 | # This branch is used at times when we add a comparison to NULL 473 | # (we don't really want to waste time looking up the associated 474 | # field object at the calling location). 475 | params = Field().get_db_prep_lookup(lookup_type, value, 476 | connection=connection, prepared=True) 477 | db_type = None 478 | except ObjectDoesNotExist: 479 | raise EmptyShortCircuit 480 | 481 | return (self.alias, self.col, db_type), params 482 | 483 | def relabel_aliases(self, change_map): 484 | if self.alias in change_map: 485 | self.alias = change_map[self.alias] 486 | 487 | 488 | class Triple(object): 489 | primary_key = False 490 | blank = False 491 | 492 | def __init__(self, field): 493 | self.field = field 494 | if self.field.primary_key: 495 | self.primary_key = True 496 | if self.field.blank: 497 | self.blank = True 498 | 499 | def get_semantic_entity(self): 500 | namespace = self.field.model._meta.namespace 501 | 502 | if namespace.startswith('http'): 503 | return '<%s>' % (namespace) 504 | else: 505 | return '%s' % (namespace) 506 | 507 | def as_sparql(self, qn=None, connection=None): 508 | triple_string = "%s:%s ?%s" % (self.field.graph, self.field.column, self.field.column) 509 | if self.field.primary_key: 510 | triple_string = "?%s %s:%s %s" % (self.field.column, 'rdf', 'type', self.get_semantic_entity()) 511 | 512 | if self.field.blank: 513 | triple_string = 'OPTIONAL { ?uri %s }' % triple_string 514 | 515 | return triple_string, () 516 | -------------------------------------------------------------------------------- /semantic/rdf/utils.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from django.db.utils import ConnectionDoesNotExist 3 | from django.utils.importlib import import_module 4 | 5 | # Define some exceptions that mirror the PEP249 interface. 6 | # We will rethrow any backend-specific errors using these 7 | # common wrappers 8 | 9 | 10 | class DatabaseError(Exception): 11 | pass 12 | 13 | 14 | class IntegrityError(DatabaseError): 15 | pass 16 | 17 | 18 | def load_backend(backend_name): 19 | try: 20 | module = import_module('.base', '%s' % backend_name) 21 | return module 22 | except ImportError: 23 | raise DatabaseError('Could not connect by %s' % backend_name) 24 | 25 | 26 | class ConnectionHandler(object): 27 | def __init__(self, databases): 28 | self.databases = databases 29 | self._connections = {} 30 | 31 | def ensure_defaults(self, alias): 32 | """ 33 | Puts the defaults into the settings dictionary for a given connection 34 | where no settings is provided. 35 | """ 36 | try: 37 | conn = self.databases[alias] 38 | except KeyError: 39 | raise ConnectionDoesNotExist("The connection %s doesn't exist" % alias) 40 | 41 | conn.setdefault('ENGINE', 'django.db.backends.dummy') 42 | if conn['ENGINE'] == 'django.db.backends.' or not conn['ENGINE']: 43 | conn['ENGINE'] = 'django.db.backends.dummy' 44 | conn.setdefault('OPTIONS', {}) 45 | conn.setdefault('TEST_CHARSET', None) 46 | conn.setdefault('TEST_COLLATION', None) 47 | conn.setdefault('TEST_NAME', None) 48 | conn.setdefault('TEST_MIRROR', None) 49 | conn.setdefault('TIME_ZONE', settings.TIME_ZONE) 50 | for setting in ('NAME', 'USER', 'PASSWORD', 'HOST', 'PORT'): 51 | conn.setdefault(setting, '') 52 | 53 | def __getitem__(self, alias): 54 | if alias in self._connections: 55 | return self._connections[alias] 56 | 57 | self.ensure_defaults(alias) 58 | db = self.databases[alias] 59 | backend = load_backend(db['ENGINE']) 60 | conn = backend.DatabaseWrapper(db, alias) 61 | self._connections[alias] = conn 62 | return conn 63 | 64 | def __iter__(self): 65 | return iter(self.databases) 66 | 67 | def all(self): 68 | return [self[alias] for alias in self] 69 | -------------------------------------------------------------------------------- /semantic/settings_test.py: -------------------------------------------------------------------------------- 1 | # A fake settings to compatible tests with django 2 | DATABASES = { 3 | 'default': { 4 | 'ENGINE': 'django.db.backends.sqlite3', 5 | 'NAME': 'relational.db', 6 | } 7 | } 8 | 9 | SEMANTIC_DATABASES = { 10 | 'default': { 11 | 'ENGINE': 'semantic.rdf.backends.virtuoso', 12 | 'NAME': 'sparql', 13 | 'USER': 'dba', 14 | 'PASSWORD': 'dba', 15 | 'HOST': 'localhost', 16 | 'PORT': '8890', 17 | 'PREFIX': { 18 | 'base': '', 19 | 'g1': '', 20 | } 21 | } 22 | } 23 | 24 | TEST_SEMANTIC_GRAPH = "http://testgraph.globo.com" 25 | -------------------------------------------------------------------------------- /semantic/tests/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | import shutil 4 | import subprocess 5 | import rdflib 6 | 7 | from SPARQLWrapper import Wrapper 8 | from django.test import TestCase 9 | from django.conf import settings 10 | 11 | 12 | db = settings.SEMANTIC_DATABASES['default'] 13 | ISQL = "isql" 14 | ISQL_CMD = 'echo "%s" | %s' 15 | ISQL_UP = "DB.DBA.TTLP_MT_LOCAL_FILE('%(ttl)s', '', '%(graph)s');" 16 | ISQL_DOWN = "SPARQL CLEAR GRAPH <%(graph)s>;" 17 | ISQL_SERVER = "select server_root();" 18 | 19 | graph = rdflib.Graph() 20 | allow_virtuoso_connection = False 21 | 22 | 23 | def mocked_query(self): 24 | 25 | qres = graph.query(self.queryString) 26 | 27 | bindings_list = [] 28 | for row in qres.bindings: 29 | row_dict = {} 30 | for key, value in row.items(): 31 | if not value: 32 | continue 33 | 34 | row_item = {} 35 | 36 | if isinstance(value, rdflib.term.URIRef): 37 | type_ = 'uri' 38 | elif isinstance(value, rdflib.term.Literal): 39 | if hasattr(value, 'datatype') and value.datatype: 40 | type_ = 'typed-literal' 41 | row_item["datatype"] = value.datatype 42 | else: 43 | type_ = 'literal' 44 | else: 45 | raise Exception('Unkown type') 46 | 47 | row_item["type"] = type_ 48 | row_item["value"] = str(value) 49 | 50 | row_dict[str(key)] = row_item 51 | 52 | bindings_list.append(row_dict) 53 | 54 | binding_str = { 55 | 'results': { 56 | 'bindings': bindings_list 57 | } 58 | } 59 | return Wrapper.QueryResult(binding_str) 60 | 61 | 62 | def mocked_virtuoso_query(self): 63 | self.queryString = _insert_from_in_test_query(self.queryString) 64 | return Wrapper.QueryResult(self._query()) 65 | 66 | 67 | def _insert_from_in_test_query(query): 68 | graph = settings.TEST_SEMANTIC_GRAPH 69 | if query.find('INSERT') > -1: 70 | query = re.sub('GRAPH [^\s]+', 'GRAPH <%s>' % graph, query) 71 | elif query.find('FROM') == -1 and query.find('WHERE') > -1: 72 | splited_query = query.split('WHERE') 73 | splited_query.insert(1, 'FROM <%s> WHERE' % graph) 74 | return ' '.join(splited_query) 75 | else: 76 | query = re.sub('FROM [^\s]+', 'FROM <%s>' % graph, query) 77 | return query 78 | 79 | 80 | def mocked_convert(self): 81 | return self.response 82 | 83 | 84 | def run_isql(cmd): 85 | isql_cmd = ISQL_CMD % (cmd, ISQL) 86 | process = subprocess.Popen(isql_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 87 | stdout_value, stderr_value = process.communicate() 88 | if stderr_value: 89 | raise Exception(stderr_value) 90 | return stdout_value 91 | 92 | 93 | def copy_ttl_to_virtuoso_dir(ttl): 94 | virtuoso_dir = run_isql(ISQL_SERVER).split('\n\n')[-2] 95 | fixture_dir, fixture_file = os.path.split(ttl) 96 | shutil.copyfile(ttl, os.path.join(virtuoso_dir, fixture_file)) 97 | return fixture_file 98 | 99 | 100 | def remove_ttl_from_virtuoso_dir(ttl): 101 | virtuoso_dir = run_isql(ISQL_SERVER).split('\n\n')[-2] 102 | ttl_path = os.path.join(virtuoso_dir, ttl) 103 | os.remove(ttl_path) 104 | 105 | 106 | class SemanticTestCase(TestCase): 107 | originalSPARQLWrapper = Wrapper.SPARQLWrapper 108 | originalQueryResult = Wrapper.QueryResult 109 | originalQueryResultConver = Wrapper.QueryResult.convert 110 | 111 | allow_virtuoso_connection = False 112 | graph = settings.TEST_SEMANTIC_GRAPH 113 | 114 | def _setup_memory(self): 115 | Wrapper.SPARQLWrapper.query = mocked_query 116 | Wrapper.QueryResult.convert = mocked_convert 117 | 118 | def _teardown_memory(self): 119 | Wrapper.SPARQLWrapper = self.originalSPARQLWrapper 120 | Wrapper.QueryResult = self.originalQueryResult 121 | 122 | def _setup_virtuoso(self): 123 | # assert that the virtuoso wrapper is the original 124 | self._teardown_virtuoso() 125 | 126 | Wrapper.SPARQLWrapper.query = mocked_virtuoso_query 127 | 128 | def _teardown_virtuoso(self): 129 | Wrapper.SPARQLWrapper = self.originalSPARQLWrapper 130 | Wrapper.QueryResult = self.originalQueryResult 131 | Wrapper.QueryResult.convert = self.originalQueryResultConver 132 | 133 | def _upload_fixture_to_memory(self, fixture): 134 | graph.parse(fixture, format="n3") 135 | 136 | def _upload_fixture_to_virtuoso(self, fixture): 137 | fixture = copy_ttl_to_virtuoso_dir(fixture) 138 | isql_up = ISQL_UP % {"ttl": fixture, "graph": self.graph} 139 | run_isql(isql_up) 140 | 141 | def _delete_fixture_from_virtuoso(self): 142 | isql_down = ISQL_DOWN % {"graph": self.graph} 143 | run_isql(isql_down) 144 | 145 | def _fixture_setup(self): 146 | global allow_virtuoso_connection 147 | allow_virtuoso_connection = False 148 | 149 | if self.allow_virtuoso_connection: 150 | allow_virtuoso_connection = True 151 | upload = self._upload_fixture_to_virtuoso 152 | self._setup_virtuoso() 153 | else: 154 | self._setup_memory() 155 | upload = self._upload_fixture_to_memory 156 | 157 | if hasattr(self, 'semantic_fixtures'): 158 | for fixture in self.semantic_fixtures: 159 | upload(fixture) 160 | super(SemanticTestCase, self)._fixture_setup() 161 | 162 | def _fixture_teardown(self): 163 | if self.allow_virtuoso_connection: 164 | self._teardown_virtuoso() 165 | self._delete_fixture_from_virtuoso() 166 | else: 167 | self._teardown_memory() 168 | super(SemanticTestCase, self)._fixture_teardown() 169 | -------------------------------------------------------------------------------- /semantic/tests/functional/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rfloriano/semantic-django/f35747887507ca6aa08a9bf528c706a214382162/semantic/tests/functional/__init__.py -------------------------------------------------------------------------------- /semantic/tests/integration/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rfloriano/semantic-django/f35747887507ca6aa08a9bf528c706a214382162/semantic/tests/integration/__init__.py -------------------------------------------------------------------------------- /semantic/tests/unit/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rfloriano/semantic-django/f35747887507ca6aa08a9bf528c706a214382162/semantic/tests/unit/__init__.py -------------------------------------------------------------------------------- /semantic/tests/unit/simple_test.py: -------------------------------------------------------------------------------- 1 | """ 2 | This file demonstrates writing tests using the unittest module. These will pass 3 | when you run "manage.py test". 4 | 5 | Replace this with more appropriate tests for your application. 6 | """ 7 | 8 | from django.test import TestCase 9 | 10 | 11 | class SimpleTest(TestCase): 12 | def test_basic_addition(self): 13 | """ 14 | Tests that 1 + 1 always equals 2. 15 | """ 16 | self.assertEqual(1 + 1, 2) 17 | -------------------------------------------------------------------------------- /semantic/tests/unit/test_models.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | from semantic.rdf.models.fields import CharField, URLField, URIField 4 | 5 | 6 | class CharFieldTestCase(TestCase): 7 | def setUp(self): 8 | self.char_field = CharField(graph='base') 9 | 10 | def test_get_prepared_value_from_field(self): 11 | value = self.char_field.get_prep_value("test") 12 | 13 | self.assertEqual('"test"', value) 14 | 15 | def test_empty_value_returns_empty_value_after_preparation(self): 16 | """ regression test for issue #1 """ 17 | 18 | value = self.char_field.get_prep_value('') 19 | 20 | self.assertEqual('', value) 21 | 22 | 23 | class URLFieldTestCase(TestCase): 24 | def setUp(self): 25 | self.url = URLField(graph='base') 26 | 27 | def test_url_field_should_have_max_lenght_by_default(self): 28 | self.assertTrue(self.url.max_length) 29 | 30 | def test_url_field_should_have_200_characters_max_lenght(self): 31 | self.assertEqual(200, self.url.max_length) 32 | 33 | def test_url_field_should_have_verify_exists_validator_true_by_default(self): 34 | url_validator = self.url.validators[1] 35 | self.assertTrue(url_validator.verify_exists) 36 | 37 | def test_url_field_should_override_verify_exists(self): 38 | url = URLField(graph='base', verify_exists=False) 39 | url_validator = url.validators[1] 40 | self.assertFalse(url_validator.verify_exists) 41 | 42 | 43 | class URIFieldTestCase(TestCase): 44 | def test_uri_field_should_have_verify_exists_validator_false_by_default(self): 45 | uri = URIField(graph='base') 46 | uri_validator = uri.validators[1] 47 | self.assertFalse(uri_validator.verify_exists) 48 | 49 | def test_uri_field_should_override_verify_exists(self): 50 | url = URLField(graph='base', verify_exists=True) 51 | url_validator = url.validators[1] 52 | self.assertTrue(url_validator.verify_exists) 53 | -------------------------------------------------------------------------------- /semantic/tests/unit/test_rdf_backend_base.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | from semantic.rdf.models.fields import SemanticField 4 | from semantic.rdf.backends.virtuoso.base import DatabaseOperations 5 | 6 | 7 | class BaseBackendTestCase(TestCase): 8 | def test_quote_predicate(self): 9 | field = SemanticField(graph='base') 10 | predicate = 'test_with_sparql' 11 | 12 | database_operations = DatabaseOperations() 13 | quoted_predicate = database_operations.quote_predicate(field, predicate) 14 | 15 | self.assertEqual('base:test_with_sparql %s', quoted_predicate) 16 | -------------------------------------------------------------------------------- /semantic/tests/unit/test_sparql_compiler.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | from example_project.example_app.smodels import BasePrograma 4 | 5 | from semantic.rdf import connections 6 | 7 | from semantic.rdf.models.sparql.subqueries import InsertQuery 8 | 9 | 10 | class SPARQLInsertCompilerTestCase(TestCase): 11 | def test_compile_to_sparql(self): 12 | using = 'default' 13 | query = InsertQuery(BasePrograma) 14 | programa = BasePrograma() 15 | query.insert_values([(programa._meta.get_field('uri'), 'uri'), (programa._meta.get_field('label'), 'test')]) 16 | 17 | connection = connections[using] 18 | 19 | insert_compiler = connection.ops.compiler('SPARQLInsertCompiler')(query, connection, using) 20 | 21 | compiled_query = insert_compiler.as_sparql() 22 | 23 | self.assertEqual(('INSERT IN GRAPH { uri rdfs:label %s; rdf:type %s }', ['test', '']), compiled_query) 24 | -------------------------------------------------------------------------------- /semantic/views.py: -------------------------------------------------------------------------------- 1 | # Create your views here. 2 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | from setuptools import setup, find_packages 3 | 4 | 5 | # Utility function to read the README file. 6 | # Used for the long_description. It's nice, because now 1) we have a top level 7 | # README file and 2) it's easier to type in the README file than to put a raw 8 | # string in below ... 9 | def read(fname): 10 | return open(os.path.join(os.path.dirname(__file__), fname)).read() 11 | 12 | setup( 13 | name="semantic-django", 14 | version="0.0.1", 15 | author="Rafael Floriano da Silva", 16 | author_email="rflorianobr@gmail.com", 17 | description=("Make semantic projects using Django and tiplestore databases"), 18 | license="BSD", 19 | keywords="semantic django", 20 | url="http://github.com/rfloriano/semantic-django", 21 | packages=find_packages( 22 | exclude=['example_project*', '*tests.functional', '*tests.integration', '*tests.unit'] 23 | ), 24 | long_description=read('README'), 25 | classifiers=[ 26 | "Development Status :: 3 - Alpha", 27 | "Topic :: Utilities", 28 | "License :: OSI Approved :: BSD License", 29 | ], 30 | ) 31 | --------------------------------------------------------------------------------