├── .gitignore ├── Jenkinsfile ├── MANIFEST.in ├── README.rst ├── __init__.py ├── docs ├── HISTORY.txt ├── api.rst ├── conf.py └── index.rst ├── setup.py ├── sparql-client ├── __init__.py └── tests │ ├── __init__.py │ ├── big_text.srx │ ├── code.rq │ ├── countries.srx │ ├── genquery.py │ ├── invalid-result.srx │ ├── national.srx │ ├── runtests │ ├── testconversion.py │ ├── testdatatypes.py │ ├── testhttp.py │ ├── testn3parse.py │ ├── testparser.py │ ├── w3-output.srx │ ├── w3-output2.srx │ └── xsdtypes.srx ├── sparql.py └── version.txt /.gitignore: -------------------------------------------------------------------------------- 1 | *.egg-info 2 | *.pyc 3 | *.pyo 4 | *.mo 5 | dist 6 | searchindex.js 7 | search.html 8 | pylint.log 9 | pyflakes.log 10 | -------------------------------------------------------------------------------- /Jenkinsfile: -------------------------------------------------------------------------------- 1 | pipeline { 2 | agent any 3 | 4 | environment { 5 | GIT_NAME = "sparql-client" 6 | SONARQUBE_TAGS = "www.eea.europa.eu" 7 | GIT_VERSIONFILE = "version.txt" 8 | } 9 | 10 | stages { 11 | 12 | stage('Cosmetics') { 13 | steps { 14 | parallel( 15 | 16 | "JS Hint": { 17 | node(label: 'docker') { 18 | script { 19 | catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') { 20 | sh '''docker run -i --rm --name="$BUILD_TAG-jshint" -e GIT_SRC="https://github.com/eea/$GIT_NAME.git" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/jshint''' 21 | } 22 | } 23 | } 24 | }, 25 | 26 | "CSS Lint": { 27 | node(label: 'docker') { 28 | script { 29 | catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') { 30 | sh '''docker run -i --rm --name="$BUILD_TAG-csslint" -e GIT_SRC="https://github.com/eea/$GIT_NAME.git" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/csslint''' 31 | } 32 | } 33 | } 34 | }, 35 | 36 | "PEP8": { 37 | node(label: 'docker') { 38 | script { 39 | catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') { 40 | sh '''docker run -i --rm --name="$BUILD_TAG-pep8" -e GIT_SRC="https://github.com/eea/$GIT_NAME.git" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/pep8''' 41 | } 42 | } 43 | } 44 | }, 45 | 46 | "PyLint": { 47 | node(label: 'docker') { 48 | script { 49 | catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') { 50 | sh '''docker run -i --rm --name="$BUILD_TAG-pylint" -e GIT_SRC="https://github.com/eea/$GIT_NAME.git" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/pylint''' 51 | } 52 | } 53 | } 54 | } 55 | 56 | ) 57 | } 58 | } 59 | 60 | stage('Code') { 61 | steps { 62 | parallel( 63 | 64 | "ZPT Lint": { 65 | node(label: 'docker') { 66 | sh '''docker run -i --rm --name="$BUILD_TAG-zptlint" -e GIT_BRANCH="$BRANCH_NAME" -e ADDONS="$GIT_NAME" -e DEVELOP="src/$GIT_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/plone-test:4 zptlint''' 67 | } 68 | }, 69 | 70 | "JS Lint": { 71 | node(label: 'docker') { 72 | sh '''docker run -i --rm --name="$BUILD_TAG-jslint" -e GIT_SRC="https://github.com/eea/$GIT_NAME.git" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/jslint4java''' 73 | } 74 | }, 75 | 76 | "PyFlakes": { 77 | node(label: 'docker') { 78 | sh '''docker run -i --rm --name="$BUILD_TAG-pyflakes" -e GIT_SRC="https://github.com/eea/$GIT_NAME.git" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/pyflakes''' 79 | } 80 | }, 81 | 82 | "i18n": { 83 | node(label: 'docker') { 84 | sh '''docker run -i --rm --name=$BUILD_TAG-i18n -e GIT_SRC="https://github.com/eea/$GIT_NAME.git" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/i18ndude''' 85 | } 86 | } 87 | ) 88 | } 89 | } 90 | 91 | stage('Tests') { 92 | steps { 93 | parallel( 94 | 95 | "WWW": { 96 | node(label: 'docker') { 97 | script { 98 | sh '''docker run -i --name="$BUILD_TAG-www" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/www-devel /debug.sh coverage''' 99 | sh '''mkdir -p xunit-reports; docker cp $BUILD_TAG-www:/plone/instance/parts/xmltestreport/testreports/. xunit-reports/''' 100 | stash name: "xunit-reports", includes: "xunit-reports/*.xml" 101 | sh '''docker cp $BUILD_TAG-www:/plone/instance/src/$GIT_NAME/coverage.xml coverage.xml''' 102 | stash name: "coverage.xml", includes: "coverage.xml" 103 | junit 'xunit-reports/*.xml' 104 | } 105 | } 106 | }, 107 | 108 | "KGS": { 109 | node(label: 'docker') { 110 | sh '''docker pull eeacms/kgs-devel;docker run -i --rm --name="$BUILD_TAG-kgs" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/kgs-devel /debug.sh bin/test --test-path /plone/instance/src/$GIT_NAME -v -vv -s $GIT_NAME''' 111 | } 112 | }, 113 | 114 | "Plone4": { 115 | node(label: 'docker') { 116 | sh '''docker pull eeacms/plone-test:4;docker run -i --rm --name="$BUILD_TAG-plone4" -e GIT_BRANCH="$BRANCH_NAME" -e ADDONS="$GIT_NAME[test]" -e DEVELOP="src/$GIT_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/plone-test:4 -v -vv -s $GIT_NAME''' 117 | } 118 | }, 119 | 120 | "Plone5 & Python2": { 121 | node(label: 'docker') { 122 | sh '''docker pull eeacms/plone-test:5;docker run -i --rm --name="$BUILD_TAG-plone5py2" -e GIT_BRANCH="$BRANCH_NAME" -e ADDONS="$GIT_NAME[test]" -e DEVELOP="src/$GIT_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/plone-test:5 -v -vv -s $GIT_NAME''' 123 | } 124 | }, 125 | 126 | "Plone5 & Python3": { 127 | node(label: 'docker') { 128 | sh '''docker pull eeacms/plone-test:5-python3;docker run -i --rm --name="$BUILD_TAG-plone5py3" -e GIT_BRANCH="$BRANCH_NAME" -e ADDONS="$GIT_NAME[test]" -e DEVELOP="src/$GIT_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/plone-test:5-python3 -v -vv -s $GIT_NAME''' 129 | } 130 | }, 131 | 132 | "PloneSaaS": { 133 | node(label: 'docker') { 134 | sh '''docker pull eeacms/plonesaas-devel;docker run -i --rm --name="$BUILD_TAG-plonesaas" -e GIT_NAME="$GIT_NAME" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" eeacms/plonesaas-devel /debug.sh bin/test --test-path /plone/instance/src/$GIT_NAME -v -vv -s $GIT_NAME''' 135 | } 136 | } 137 | 138 | ) 139 | } 140 | } 141 | 142 | stage('Report to SonarQube') { 143 | when { 144 | allOf { 145 | environment name: 'CHANGE_ID', value: '' 146 | } 147 | } 148 | steps { 149 | node(label: 'swarm') { 150 | script{ 151 | checkout scm 152 | dir("xunit-reports") { 153 | unstash "xunit-reports" 154 | } 155 | unstash "coverage.xml" 156 | def scannerHome = tool 'SonarQubeScanner'; 157 | def nodeJS = tool 'NodeJS11'; 158 | withSonarQubeEnv('Sonarqube') { 159 | sh '''sed -i "s|/plone/instance/src/$GIT_NAME|$(pwd)|g" coverage.xml''' 160 | sh "export PATH=$PATH:${scannerHome}/bin:${nodeJS}/bin; sonar-scanner -Dsonar.python.xunit.skipDetails=true -Dsonar.python.xunit.reportPath=xunit-reports/*.xml -Dsonar.python.coverage.reportPath=coverage.xml -Dsonar.sources=. -Dsonar.projectKey=$GIT_NAME-$BRANCH_NAME -Dsonar.projectVersion=$BRANCH_NAME-$BUILD_NUMBER" 161 | sh '''try=2; while [ \$try -gt 0 ]; do curl -s -XPOST -u "${SONAR_AUTH_TOKEN}:" "${SONAR_HOST_URL}api/project_tags/set?project=${GIT_NAME}-${BRANCH_NAME}&tags=${SONARQUBE_TAGS},${BRANCH_NAME}" > set_tags_result; if [ \$(grep -ic error set_tags_result ) -eq 0 ]; then try=0; else cat set_tags_result; echo "... Will retry"; sleep 60; try=\$(( \$try - 1 )); fi; done''' 162 | } 163 | } 164 | } 165 | } 166 | } 167 | 168 | stage('Pull Request') { 169 | when { 170 | not { 171 | environment name: 'CHANGE_ID', value: '' 172 | } 173 | environment name: 'CHANGE_TARGET', value: 'master' 174 | } 175 | steps { 176 | node(label: 'docker') { 177 | script { 178 | if ( env.CHANGE_BRANCH != "develop" && !( env.CHANGE_BRANCH.startsWith("hotfix")) ) { 179 | error "Pipeline aborted due to PR not made from develop or hotfix branch" 180 | } 181 | withCredentials([string(credentialsId: 'eea-jenkins-token', variable: 'GITHUB_TOKEN')]) { 182 | sh '''docker run -i --rm --name="$BUILD_TAG-gitflow-pr" -e GIT_CHANGE_BRANCH="$CHANGE_BRANCH" -e GIT_CHANGE_AUTHOR="$CHANGE_AUTHOR" -e GIT_CHANGE_TITLE="$CHANGE_TITLE" -e GIT_TOKEN="$GITHUB_TOKEN" -e GIT_BRANCH="$BRANCH_NAME" -e GIT_CHANGE_ID="$CHANGE_ID" -e GIT_ORG="$GIT_ORG" -e GIT_NAME="$GIT_NAME" -e GIT_VERSIONFILE="$GIT_VERSIONFILE" eeacms/gitflow''' 183 | } 184 | } 185 | } 186 | } 187 | } 188 | 189 | stage('Release') { 190 | when { 191 | allOf { 192 | environment name: 'CHANGE_ID', value: '' 193 | branch 'master' 194 | } 195 | } 196 | steps { 197 | node(label: 'docker') { 198 | withCredentials([[$class: 'UsernamePasswordMultiBinding', credentialsId: 'eea-jenkins', usernameVariable: 'EGGREPO_USERNAME', passwordVariable: 'EGGREPO_PASSWORD'],string(credentialsId: 'eea-jenkins-token', variable: 'GITHUB_TOKEN'),[$class: 'UsernamePasswordMultiBinding', credentialsId: 'pypi-jenkins', usernameVariable: 'PYPI_USERNAME', passwordVariable: 'PYPI_PASSWORD']]) { 199 | sh '''docker run -i --rm --name="$BUILD_TAG-gitflow-master" -e GIT_BRANCH="$BRANCH_NAME" -e EGGREPO_USERNAME="$EGGREPO_USERNAME" -e EGGREPO_PASSWORD="$EGGREPO_PASSWORD" -e GIT_NAME="$GIT_NAME" -e PYPI_USERNAME="$PYPI_USERNAME" -e PYPI_PASSWORD="$PYPI_PASSWORD" -e GIT_ORG="$GIT_ORG" -e GIT_TOKEN="$GITHUB_TOKEN" -e GIT_VERSIONFILE="$GIT_VERSIONFILE" eeacms/gitflow''' 200 | } 201 | } 202 | } 203 | } 204 | 205 | } 206 | 207 | post { 208 | changed { 209 | script { 210 | def url = "${env.BUILD_URL}/display/redirect" 211 | def status = currentBuild.currentResult 212 | def subject = "${status}: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'" 213 | def summary = "${subject} (${url})" 214 | def details = """

${env.JOB_NAME} - Build #${env.BUILD_NUMBER} - ${status}

215 |

Check console output at ${env.JOB_BASE_NAME} - #${env.BUILD_NUMBER}

216 | """ 217 | 218 | def color = '#FFFF00' 219 | if (status == 'SUCCESS') { 220 | color = '#00FF00' 221 | } else if (status == 'FAILURE') { 222 | color = '#FF0000' 223 | } 224 | 225 | emailext (subject: '$DEFAULT_SUBJECT', to: '$DEFAULT_RECIPIENTS', body: details) 226 | } 227 | } 228 | } 229 | } 230 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include *.md *.rst *.txt 2 | graft docs 3 | graft version.txt 4 | global-exclude *pyc 5 | global-exclude *~ 6 | global-exclude *.un~ 7 | global-include *.mo 8 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ************************** 2 | SPARQL HTTP client library 3 | ************************** 4 | .. image:: https://ci.eionet.europa.eu/buildStatus/icon?job=eea/sparql-client/develop 5 | :target: https://ci.eionet.europa.eu/job/eea/job/sparql-client/job/develop/display/redirect 6 | :alt: develop 7 | .. image:: https://ci.eionet.europa.eu/buildStatus/icon?job=eea/sparql-client/master 8 | :target: https://ci.eionet.europa.eu/job/eea/job/sparql-client/job/master/display/redirect 9 | :alt: master 10 | .. image:: https://img.shields.io/github/v/release/eea/sparql-client 11 | :target: https://eggrepo.eea.europa.eu/d/sparql-client/ 12 | :alt: Release 13 | 14 | `sparql-client` is a SPARQL query library that performs SELECT and ASK queries against a SPARQL endpoint via HTTP. 15 | It will automatically convert literals to the coresponding Python types. 16 | 17 | API based on SPARQL_JavaScript_Library_ by Lee Feigenbaum and Elias Torres. Heavy influence from Juan Manuel Caicedo’s SPARQL library. 18 | 19 | .. _SPARQL_JavaScript_Library: https://web.archive.org/web/20120518014957/http://www.thefigtrees.net/lee/sw/sparql.js 20 | 21 | API 22 | --- 23 | 24 | First you open a connection to the endpoint:: 25 | 26 | s = sparql.Service(endpoint, "utf-8", "GET") 27 | 28 | Then you make the query:: 29 | 30 | result = s.query(statement) 31 | 32 | If you have made a SELECT query, then you can read the result with fetchone() or fetchall():: 33 | 34 | for row in result.fetchone(): 35 | 36 | If you have made an ASK query, then you can read the result (a boolean value) with hasresult(): 37 | 38 | works = result.hasresult() 39 | 40 | 41 | How it works 42 | ------------ 43 | 44 | >>> import sparql 45 | 46 | >>> q = ('SELECT DISTINCT ?station, ?orbits WHERE { ' 47 | ... '?station a . ' 48 | ... '?station ?orbits . ' 49 | ... 'FILTER(?orbits > 50000) } ORDER BY DESC(?orbits)') 50 | >>> result = sparql.query('http://dbpedia.org/sparql', q) 51 | 52 | >>> result.variables 53 | [u'station', u'orbits'] 54 | 55 | >>> for row in result: 56 | ... print 'row:', row 57 | ... values = sparql.unpack_row(row) 58 | ... print values[0], "-", values[1], "orbits" 59 | row: (>, >) 60 | http://dbpedia.org/resource/Mir - 86331 orbits 61 | row: (>, >) 62 | http://dbpedia.org/resource/Salyut_7 - 51917 orbits 63 | 64 | sparql module 65 | ------------- 66 | 67 | The ``sparql`` module can be invoked in several different ways. To quickly run a query use ``query()``. Results are encapsulated in a ``_ResultsParser`` instance: 68 | 69 | >>> result = sparql.query(endpoint, query) 70 | >>> for row in result: 71 | >>> print row 72 | 73 | Command-line use 74 | ================ 75 | 76 | >>> sparql.py [-i] endpoint 77 | -i Interactive mode 78 | 79 | If interactive mode is enabled, the program reads queries from the console and then executes them. Use a double line (two ‘enters’) to separate queries. 80 | Otherwise, the query is read from standard input. 81 | 82 | RDF wrapper classes 83 | =================== 84 | 85 | class sparql.RDFTerm 86 | Super class containing methods to override. ``sparql.IRI``, ``sparql.Literal`` and ``sparql.BlankNode`` all inherit from ``sparql.RDFTerm``. 87 | 88 | ``n3()`` 89 | 90 | Return a Notation3 representation of this term. 91 | 92 | ``class sparql.IRI(value)`` 93 | 94 | An RDF resource. 95 | 96 | ``class sparql.Literal(value, datatype=None, lang=None)`` 97 | 98 | Literals. These can take a data type or a language code. 99 | 100 | ``class sparql.BlankNode(value)`` 101 | 102 | Blank node. Similar to IRI but lacks a stable identifier. 103 | 104 | Query utilities 105 | =============== 106 | 107 | ``class sparql.Service(endpoint, qs_encoding='utf-8')`` 108 | 109 | This is the main entry to the library. The user creates a Service, then sends a query to it. If we want to have persistent connections, then open them here. 110 | 111 | ``class sparql._ResultsParser(fp)`` 112 | 113 | Parse the XML result. 114 | 115 | ``__iter__()`` 116 | 117 | Synonim for fetchone(). 118 | 119 | ``fetchall()`` 120 | 121 | Loop through the result to build up a list of all rows. Patterned after DB-API 2.0. 122 | 123 | ``fetchone()`` 124 | 125 | Fetches the next set of rows of a query result, returning a list. An empty list is returned when no more rows are available. If the query was an ASK request, then an empty list is returned as there are no rows available. 126 | 127 | ``hasresult()`` 128 | 129 | ASK queries are used to test if a query would have a result. If the query is an ASK query there won’t be an actual result, and fetchone() will return nothing. Instead, this method can be called to check the result from the ASK query. 130 | 131 | If the query is a SELECT statement, then the return value of hasresult() is None, as the XML result format doesn’t tell you if there are any rows in the result until you have read the first one. 132 | 133 | ``sparql.parse_n3_term(src)`` 134 | 135 | Parse a Notation3 value into a RDFTerm object (IRI or Literal). 136 | 137 | This parser understands IRIs and quoted strings; basic non-string types (integers, decimals, booleans, etc) are not supported yet. 138 | 139 | ``sparql.unpack_row(row, convert=None, convert_type={})`` 140 | 141 | Convert values in the given row from RDFTerm objects to plain Python values: IRI is converted to a unicode string containing the IRI value; BlankNode is converted to a unicode string with the BNode’s identifier, and Literal is converted based on its XSD datatype. 142 | 143 | The library knows about common XSD types (STRING becomes unicode, INTEGER and LONG become int, DOUBLE and FLOAT become float, DECIMAL becomes Decimal, BOOLEAN becomes bool). If the python-dateutil library is found, then DATE, TIME and DATETIME are converted to date, time and datetime respectively. For other conversions, an extra argument convert may be passed. It should be a callable accepting two arguments: the serialized value as a unicode object, and the XSD datatype. 144 | 145 | ``sparql.query(endpoint, query)`` 146 | 147 | Convenient method to execute a query. Exactly equivalent to: 148 | 149 | ``sparql.Service(endpoint).query(query)`` 150 | 151 | Conversion of data types 152 | ------------------------ 153 | 154 | The library will automatically convert typed literals to a coresponding 155 | simple type in Python. Dates are also converted if the dateutil_ library is 156 | available. 157 | 158 | .. _dateutil: http://labix.org/python-dateutil 159 | 160 | 161 | Running the unit tests 162 | ---------------------- 163 | 164 | If you have nose_ installed, just run ``nosetests`` in the top-level directory. 165 | Some tests require the python-dateutil_ (version 1.5) or mock_ libraries. 166 | Tested under Python 2.4 through 2.7. 167 | 168 | .. _nose: http://somethingaboutorange.com/mrl/projects/nose/ 169 | .. _python-dateutil: http://niemeyer.net/python-dateutil 170 | .. _mock: http://www.voidspace.org.uk/python/mock/ 171 | 172 | Installing sparql-client 173 | ------------------------ 174 | 175 | The ``sparql-client`` library is available from PyPI and has no dependencies. Installation is as simple as: 176 | 177 | pip install sparql-client 178 | 179 | We recommend also instlaling ``python-dateutil``, to enable parsing of dates and times from query results 180 | 181 | License 182 | ------- 183 | The contents of this package are subject to the Mozilla Public 184 | License Version 1.1 (the "License"); you may not use this package 185 | except in compliance with the License. You may obtain a copy of 186 | the License at http://www.mozilla.org/MPL/ 187 | 188 | Software distributed under the License is distributed on an "AS 189 | IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or 190 | implied. See the License for the specific language governing 191 | rights and limitations under the License. 192 | 193 | The Original Code is SPARQL client version 1.0. 194 | 195 | The Initial Owner of the Original Code is European Environment 196 | Agency (EEA). Portions created by Eau de Web for EEA are 197 | Copyright (C) European Environment Agency. All Rights Reserved. 198 | 199 | 200 | Authors 201 | ------- 202 | * Søren Roug, EEA 203 | * Alex Morega, Eau de Web 204 | -------------------------------------------------------------------------------- /__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eea/sparql-client/d7914054ef119541833c616684b1cb42ee02b581/__init__.py -------------------------------------------------------------------------------- /docs/HISTORY.txt: -------------------------------------------------------------------------------- 1 | Changelog 2 | ========= 3 | 4 | 3.8 - (2020-06-17) 5 | --------------------------- 6 | * Bug fix: fixed Python3 query execution 7 | [alecghica refs #111217] 8 | * Feature: PEP8 and linting 9 | [alecghica refs #111217] 10 | 11 | 3.7 - (2020-06-17) 12 | --------------------------- 13 | * Feature: added the documentation originally found under eionet.europa.eu 14 | [alecghica refs #111217] 15 | 16 | 3.6 - (2020-03-03) 17 | --------------------------- 18 | * Change: Add jenkins badge 19 | [valentinab25] 20 | 21 | 3.5 - (2020-02-26) 22 | -------------------------- 23 | * Feature: merged source code so it now works on both Python 2 and Python 3 24 | [alecghica refs #110159] 25 | * Feature: updated Jenkins pipeline 26 | [alecghica refs #110159] 27 | 28 | 3.4 - (2019-01-28) 29 | -------------------------- 30 | * Jenkins: Add sonarqube step 31 | [avoinea refs #101552] 32 | 33 | 3.3 - (2018-12-11) 34 | -------------------------- 35 | * Feature: when building the request from an endpoint that followed redirects 36 | the query works fine now 37 | [alecghica refs #100666] 38 | 39 | 3.2 - (2018-06-22) 40 | ----------------------- 41 | * Change: updated URLs pointing to eea.europa.eu with https:// 42 | [alecghica refs #95849] 43 | 44 | 3.1 - (2017-12-12) 45 | ------------------ 46 | * Change: Replace eeacms/zptlint with eeacms/plone-test:4 zptlint 47 | [avoinea refs #90415] 48 | 49 | 3.0 - (2017-11-14) 50 | ------------------ 51 | * Change: Added handler for SAXParseException and wrote a test in order to 52 | cover this exception 53 | [petchesi-iulian refs #88573] 54 | 55 | 2.9 - (2017-09-29) 56 | ------------------ 57 | * Change: Changed the projects tests structure in order to make the tests work 58 | [petchesi-iulian refs #88509] 59 | 60 | 2.8 - (2017-08-16) 61 | ------------------ 62 | * Change: Added sparql export method formats (xml, xmlschema, json) to be stored 63 | and retrieved from cache 64 | [zoltan_andras refs #86464] 65 | 66 | 2.7 - (2017-04-24) 67 | ------------------ 68 | * Change: updated package information 69 | [eduard-fironda] 70 | 71 | 2.6 - (2016-10-04) 72 | ------------------ 73 | * Change: added timeout parameter for _get_response 74 | 75 | 2.5 - (2016-07-29) 76 | ------------------ 77 | * Bug fix: Safely get __version__ from version.txt 78 | [voineali refs #74283] 79 | 80 | 2.4 - (2016-07-29) 81 | ------------------ 82 | * Bug fix: Read __version__ from version.txt 83 | [voineali refs #74283] 84 | 85 | 2.3 - (2016-07-19) 86 | ------------------ 87 | * Change: fixed changelog markup and removed __version__ 88 | [alecghica refs #74151] 89 | 90 | 2.2 - (2016-06-16) 91 | ------------------ 92 | * Feature: Fix Comment in sparql queries 93 | [lucas refs #72876] 94 | 95 | 2.1 - (2016-06-02) 96 | ------------------ 97 | * Bug fix: Fix passing of method from service to query 98 | 99 | 2.0 - (2016-04-18) 100 | ------------------ 101 | * Bug fix: fixed error reporting when running query 102 | [alecghica refs #68990] 103 | 104 | 1.9 - (2016-02-01) 105 | ------------------ 106 | * Bug fix: fix timeout exception call now that we have moved to eventlet 107 | [ichim-david refs #17334] 108 | 109 | 1.8 - (2015-10-05) 110 | ------------------ 111 | * Bug fix: set socket timeout to prevent Zope instances hanging out when the external service is not responding 112 | [lucas refs #29063] 113 | 114 | 1.7 - (2015-07-28) 115 | ------------------ 116 | * Change: Replace pycurl2 with eventlet 117 | [david-batranu refs #25721] 118 | 119 | 1.6 - (2014-09-15) 120 | ------------------ 121 | * Feature: change the default query method from GET to POST. 122 | [pasoviul refs #20501] 123 | 124 | 1.5 - (2013-11-21) 125 | ------------------ 126 | * Feature: added "method" as a parameter for query() and now we can use 127 | either GET or POST methods 128 | [ghicaale refs #14491] 129 | 130 | 1.4 - (2013-07-08) 131 | ------------------ 132 | * Bug fix: Use a temporary file for parsing instead of using a huge string 133 | [szabozo0 refs #14826] 134 | 135 | 1.3 - (2013-05-20) 136 | ------------------ 137 | * Bug fix: fixed ReST merkup under HISTORY.txt 138 | [ghicaale refs #14554] 139 | 140 | 1.2 - (2013-05-20) 141 | ------------------ 142 | * Upgrade step: 143 | - install system dependency libcurl3-dev (Debian/Ubuntu) / curl-devel (CentOS) 144 | - [szabozo0 refs #14349] 145 | * Change: Use pycurl2 instead of pycurl 146 | [szabozo0 refs #14349] 147 | * Bug fix: Use pycurl instead of urllib2, added timeout functionality 148 | [szabozo0 refs #14349] 149 | 150 | 1.1 - (2013-03-15) 151 | ------------------ 152 | * Feature: Removed sparql error handling quickfix 153 | [szabozo0 refs #13705] 154 | 155 | 1.0 - (2013-01-28) 156 | ------------------ 157 | * Feature: Handle sparql errors 158 | [szabozo0 #9608] 159 | 160 | 0.16 - (2013-01-15) 161 | ------------------- 162 | * Bug fix: Strip the endpoint url 163 | [szabozo0] 164 | 165 | 0.15 - (2013-01-10) 166 | ------------------- 167 | * Packaging fix: inline contents of version.txt 168 | [moregale] 169 | 170 | 0.14 - (2013-01-10) 171 | ------------------- 172 | * Bug fix: updated History.txt 173 | [ghicaale] 174 | * Buf fix: unit test 175 | [moregale] 176 | 177 | 0.13 - (2012-09-11) 178 | ------------------- 179 | * Change: changed from HTTP POST to GET 180 | [roug] 181 | 182 | 0.12 - (2012-09-11) 183 | ------------------- 184 | * No changes 185 | 186 | 0.11 - (2012-04-15) 187 | ------------------- 188 | * Feature: added copyright message to sparql.py 189 | [roug] 190 | 191 | 0.10 - (2012-03-05) 192 | ------------------- 193 | * Feature: map the ``xsd:int`` type to Python's ``int`` 194 | [moregale] 195 | 196 | 0.9 - (2011-10-24) 197 | ------------------ 198 | * Feature: new argument `convert_type` for `unpack_row` 199 | [moregale] 200 | 201 | 0.8 - (2011-08-29) 202 | ------------------ 203 | * First version available on PyPI 204 | [roug, moregale] 205 | -------------------------------------------------------------------------------- /docs/api.rst: -------------------------------------------------------------------------------- 1 | :mod:`sparql` module 2 | ==================== 3 | 4 | .. automodule:: sparql 5 | 6 | 7 | RDF wrapper classes 8 | ------------------- 9 | 10 | .. autoclass:: sparql.RDFTerm 11 | :members: 12 | 13 | .. autoclass:: sparql.IRI 14 | 15 | .. autoclass:: sparql.Literal 16 | 17 | .. autoclass:: sparql.BlankNode 18 | 19 | 20 | Query utilities 21 | --------------- 22 | 23 | .. autoclass:: Service 24 | 25 | .. autoclass:: _ResultsParser 26 | :members: 27 | 28 | .. automethod:: __iter__ 29 | 30 | .. autofunction:: sparql.parse_n3_term 31 | 32 | .. autofunction:: unpack_row 33 | 34 | .. autofunction:: query 35 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from sparql import __version__ as version 3 | 4 | extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx'] 5 | 6 | templates_path = ['_templates'] 7 | source_suffix = '.rst' 8 | master_doc = 'index' 9 | 10 | project = u'sparql-client' 11 | copyright = u'2011, European Environment Agency' 12 | 13 | release = version 14 | 15 | exclude_patterns = ['_build'] 16 | pygments_style = 'sphinx' 17 | intersphinx_mapping = {'python': ('http://docs.python.org', None)} 18 | 19 | html_theme = 'default' 20 | html_static_path = ['_static'] 21 | htmlhelp_basename = 'sparql-clientdoc' 22 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | Welcome to sparql-client! 2 | ========================= 3 | 4 | sparql-client is a SPARQL query library that performs `SELECT` and `ASK` 5 | queries against a SPARQL endpoint via HTTP. 6 | 7 | API based on `SPARQL JavaScript Library`_ by Lee Feigenbaum and Elias Torres. 8 | Heavy influence from Juan Manuel Caicedo's SPARQL library 9 | 10 | .. _`SPARQL JavaScript Library`: http://www.thefigtrees.net/lee/sw/sparql.js 11 | 12 | 13 | Briefly, here is how it works:: 14 | 15 | >>> import sparql 16 | 17 | >>> q = ('SELECT DISTINCT ?station, ?orbits WHERE { ' 18 | ... '?station a . ' 19 | ... '?station ?orbits . ' 20 | ... 'FILTER(?orbits > 50000) } ORDER BY DESC(?orbits)') 21 | >>> result = sparql.query('http://dbpedia.org/sparql', q) 22 | 23 | >>> result.variables 24 | [u'station', u'orbits'] 25 | 26 | >>> for row in result: 27 | ... print 'row:', row 28 | ... values = sparql.unpack_row(row) 29 | ... print values[0], "-", values[1], "orbits" 30 | row: (>, >) 31 | http://dbpedia.org/resource/Mir - 86331 orbits 32 | row: (>, >) 33 | http://dbpedia.org/resource/Salyut_7 - 51917 orbits 34 | 35 | 36 | Contents 37 | ======== 38 | 39 | .. toctree:: 40 | :maxdepth: 2 41 | 42 | install 43 | api 44 | 45 | .. include:: HISTORY.txt 46 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | try: 3 | from setuptools import setup 4 | except ImportError: 5 | from distutils.core import setup 6 | 7 | NAME = 'sparql-client' 8 | VERSION = open('version.txt').read().strip() 9 | 10 | 11 | setup(name=NAME, 12 | version=VERSION, 13 | description='Python API to query a SPARQL endpoint', 14 | long_description=open('README.rst').read() + "\n\n" + \ 15 | open(os.path.join("docs", "HISTORY.txt")).read(), 16 | classifiers=[ 17 | 'Environment :: Console', 18 | 'Intended Audience :: Developers', 19 | "Programming Language :: Python", 20 | "Programming Language :: Python :: 3.7", 21 | "Programming Language :: Python :: 2.7", 22 | 'Operating System :: OS Independent', 23 | 'Topic :: Software Development :: Libraries :: Python Modules', 24 | ], 25 | keywords="Sparql Client", 26 | author='European Environment Agency: IDM2 A-Team', 27 | author_email='eea-edw-a-team-alerts@googlegroups.com', 28 | long_description_content_type='text/x-rst', 29 | url='https://github.com/eea/sparql-client', 30 | license="MPL", 31 | py_modules=['sparql'], 32 | install_requires=[ 33 | 'eventlet', 34 | 'six', 35 | 'dnspython < 2.0.0', 36 | ], 37 | extras_require={ 38 | 'test': [ 39 | 'mock', 40 | ] 41 | }, 42 | 43 | ) 44 | -------------------------------------------------------------------------------- /sparql-client/__init__.py: -------------------------------------------------------------------------------- 1 | # See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages 2 | try: 3 | __import__('pkg_resources').declare_namespace(__name__) 4 | except ImportError: 5 | from pkgutil import extend_path 6 | __path__ = extend_path(__path__, __name__) 7 | -------------------------------------------------------------------------------- /sparql-client/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eea/sparql-client/d7914054ef119541833c616684b1cb42ee02b581/sparql-client/tests/__init__.py -------------------------------------------------------------------------------- /sparql-client/tests/big_text.srx: -------------------------------------------------------------------------------- 1 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | multiple<br>paragraphs<br>here 12 | http://example.com/ 13 | bnode.id 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /sparql-client/tests/code.rq: -------------------------------------------------------------------------------- 1 | PREFIX f: 2 | 3 | SELECT * WHERE { 4 | ?subj f:codeCurrency "EUR or €"^^ 5 | } 6 | -------------------------------------------------------------------------------- /sparql-client/tests/countries.srx: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | BE 16 | 17 | 18 | Belgium 19 | 20 | 21 | 44.252934 22 | 23 | 24 | 255 25 | 26 | 27 | 471161.0 28 | 29 | 30 | BE 31 | 32 | 33 | http://rdfdata.eionet.europa.eu/eea/countries/BE 34 | 35 | 36 | 37 | 38 | Albania 39 | 40 | 41 | 3.808241 42 | 43 | 44 | 3 45 | 46 | 47 | 12015.0 48 | 49 | 50 | AL 51 | 52 | 53 | http://rdfdata.eionet.europa.eu/eea/countries/AL 54 | 55 | 56 | 57 | 58 | Japan 59 | 60 | 61 | 5069000.0 62 | 63 | 64 | 65 | 66 | -------------------------------------------------------------------------------- /sparql-client/tests/genquery.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import six.moves.urllib.request 5 | import six.moves.urllib.parse 6 | import six.moves.urllib.error 7 | import six.moves.urllib.request 8 | import six.moves.urllib.error 9 | import six.moves.urllib.parse 10 | 11 | statement = open('code.rq').read() 12 | query = {'query': statement, 'format': 'xml'} 13 | 14 | qs = six.moves.urllib.parse.urlencode(query) 15 | print(qs) 16 | url = 'http://dbpedia.org/sparql?' \ 17 | + six.moves.urllib.parse.urlencode(query) 18 | 19 | opener = \ 20 | six.moves.urllib.request.build_opener(six.moves.urllib.request.HTTPHandler) 21 | six.moves.urllib.request.install_opener(opener) 22 | req = six.moves.urllib.request.Request(url) 23 | 24 | # req.add_header("Accept", "application/xml") 25 | 26 | try: 27 | conn = six.moves.urllib.request.urlopen(req, timeout=10) 28 | except Exception: 29 | conn = None 30 | 31 | if not conn: 32 | raise IOError('Failure in open') 33 | 34 | data = conn.read() 35 | conn.close() 36 | print(data) 37 | -------------------------------------------------------------------------------- /sparql-client/tests/invalid-result.srx: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | barbuanc 24 | 25 | 26 | http://www.eea.europa.eu/portal_types/Assessment#Assessment 27 | 28 | 29 | 2011 2.8.1 30 | 31 | 32 | IND-305-en 33 | 34 | 35 | ENER025 36 | 37 | 38 | http://www.eea.europa.eu/data-and-maps/indicators/energy-efficiency-and-energy-consumption-6/assessment 39 | 40 | 41 | 2015-09-04T19:00:02Z 42 | 43 | 44 | Energy efficiency and energy consumption in industry 45 | 46 | 47 | 2012-04-11T15:18:19Z 48 | 49 | 50 | Over the period 1990-2009, in EU-27 countries, energy efficiency in industry has improved by 30% at an annual average rate of 1.8% per year, with large differences among countries. Energy efficiency improvement has been realized in all industrial branches except textile. Over the period 2005-2009 energy efficiency improved by 1.5%/year with an important deterioration in 2009 due to the economic crisis. 51 | 52 | 53 | en 54 | 55 | 56 | 2012-04-11T15:28:20Z 57 | 58 | 59 | 60 | 61 | vedludia 62 | 63 | 64 | http://www.eea.europa.eu/portal_types/Assessment#Assessment 65 | 66 | 67 | 2016 1.1.2 68 | 69 | 70 | IND-35-en 71 | 72 | 73 | CSI035, TERM012 74 | 75 | 76 | http://www.eea.europa.eu/data-and-maps/indicators/passenger-transport-demand-version-2/assessment-6 77 | 78 | 79 | 2017-08-23T10:21:54Z 80 | 81 | 82 | Passenger transport demand 83 | 84 | 85 | 2016-09-28T10:02:20Z 86 | 87 | 88 | 89 | Passenger transport demand in the EU-28 increased by 1.8 % between 2013 and 2014, after an overall downward trend since its peak in 2009. Car passenger travel remains the dominant transport mode accounting for well over 70 % of total transport. Air transport demand grew by 4 % in 2014 and has a modal share of 9 % (the same as before the economic recession). Rail passenger travel is stable, accounting for 6.5 % of transport demand in 2014. 90 | Land- based  passenger transport demand also grew in the other EEA member countries. Growth in 2014 compared with the previous year was 5.1 % in Iceland , 3.2 % in Norway,  3.1 % in Turkey and 1.8 % in Switzerland . 91 | 92 | 93 | 94 | en 95 | 96 | 97 | 2016-12-01T10:03:43Z 98 | 99 | 100 | 101 | -------------------------------------------------------------------------------- /sparql-client/tests/national.srx: -------------------------------------------------------------------------------- 1 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | http://aims.fao.org/aos/geopolitical.owl#Germany 12 | Germany 13 | Германия 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /sparql-client/tests/runtests: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | #for file in test*.py 3 | for file in testdatatypes.py testparser.py testconversion.py 4 | do 5 | python $file 6 | done 7 | -------------------------------------------------------------------------------- /sparql-client/tests/testconversion.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import unittest 5 | from datetime import datetime, date, time 6 | import os.path 7 | import sparql 8 | from mock import Mock, patch 9 | from six.moves import map 10 | 11 | _dirname = os.path.dirname(__file__) 12 | 13 | class TestConversion(unittest.TestCase): 14 | 15 | def setUp(self): 16 | self._fp = open(os.path.join(_dirname, "xsdtypes.srx")) 17 | self.result = sparql._ResultsParser(self._fp) 18 | 19 | def tearDown(self): 20 | self._fp.close() 21 | 22 | def test_date(self): 23 | """ Simple query with unbound variables """ 24 | # import dateutil # make sure python-dateutil is installed 25 | self.assertEqual([u'name', u'decimalData', u'lastupdated', 26 | u'foundingDate', u'timeexample'], 27 | self.result.variables) 28 | 29 | rows = list(map(sparql.unpack_row, self.result.fetchall())) 30 | row0 = rows[0] 31 | 32 | self.assertEqual(type(row0[2]), datetime) 33 | self.assertEqual(datetime(2009, 11, 0o2, 14, 31, 40), row0[2]) 34 | 35 | self.assertEqual(type(row0[3]), date) 36 | self.assertEqual(date(1991, 8, 20), row0[3]) 37 | 38 | self.assertEqual(type(row0[4]), time) 39 | self.assertEqual(time(18, 58, 21), row0[4]) 40 | 41 | def test_decimal(self): 42 | import decimal 43 | self.assertEqual(self.result.variables[1], 'decimalData') 44 | row0 = list(map(sparql.unpack_row, self.result.fetchall()))[0] 45 | decval = row0[1] 46 | self.assertEqual(type(decval), decimal.Decimal) 47 | self.assertEqual(str(decval), "123.456") 48 | 49 | def test_custom_function(self): 50 | convert = Mock() 51 | no_default_converters = patch('sparql._types', {}) 52 | no_default_converters.start() 53 | try: 54 | row0 = sparql.unpack_row(list(self.result.fetchall())[0], convert) 55 | self.assertTrue(row0[2] is convert.return_value) 56 | convert.assert_called_with("18:58:21", sparql.XSD_TIME) 57 | finally: 58 | no_default_converters.stop() 59 | 60 | def test_custom_mapping(self): 61 | convert_datetime = Mock() 62 | row = list(self.result.fetchall())[0] 63 | unpacked_row = sparql.unpack_row(row, convert_type={ 64 | sparql.XSD_DATETIME: convert_datetime, 65 | }) 66 | self.assertTrue(unpacked_row[2] is convert_datetime.return_value) 67 | convert_datetime.assert_called_with("2009-11-02 14:31:40") 68 | 69 | 70 | if __name__ == '__main__': 71 | unittest.main() 72 | -------------------------------------------------------------------------------- /sparql-client/tests/testdatatypes.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import unittest 5 | import sparql 6 | import six 7 | 8 | 9 | class TestLiterals(unittest.TestCase): 10 | 11 | def test_literal_same(self): 12 | """ Two literals with same language must be the same """ 13 | 14 | l1 = sparql.Literal(u'Hello world', lang=u'en') 15 | l2 = sparql.Literal(u'Hello world', lang=u'en') 16 | self.assertEqual(l1, l2) 17 | 18 | def test_literal_notsame1(self): 19 | """ Two literals different language must be different """ 20 | 21 | l1 = sparql.Literal(u'Hello world', lang=u'en') 22 | l2 = sparql.Literal(u'Hello world', lang=u'en-US') 23 | self.assertNotEqual(l1, l2) 24 | 25 | def test_literal_notsame2(self): 26 | """ Difference on both value and language """ 27 | 28 | l1 = sparql.Literal(u'Hello world', lang=u'en') 29 | l2 = sparql.Literal(u'Hallo Welt', lang=u'de') 30 | self.assertNotEqual(l1, l2) 31 | 32 | def test_literal_notsame3(self): 33 | """ Two literals with same language must be the same """ 34 | 35 | l1 = sparql.Literal(u'Hello world', lang=u'en') 36 | self.assertNotEqual(u'Hello world', l1) 37 | self.assertNotEqual(l1, u'Hello world') 38 | 39 | def test_compare_with_non_literal(self): 40 | l1 = sparql.Literal('hi') 41 | self.assertFalse(l1 == 'hi') 42 | self.assertFalse(l1 == u'hi') 43 | self.assertFalse(l1 == None) 44 | self.assertFalse(l1 == 13) 45 | self.assertFalse(l1 == 13.0) 46 | self.assertFalse(l1 == ['hi']) 47 | self.assertFalse(l1 == {'hi': 'hi'}) 48 | 49 | def test_convert_to_unicode(self): 50 | """ Literals should convert values to unicode when saving them """ 51 | 52 | class SomeType(object): 53 | 54 | def __unicode__(self): 55 | return u'hello world' 56 | 57 | if six.PY2: 58 | l = sparql.Literal(SomeType()) 59 | else: 60 | l = sparql.Literal(SomeType().__unicode__()) 61 | self.assertEqual(str(l), 'hello world') 62 | 63 | def test_repr(self): 64 | """ repr should return the literal in N3 syntax """ 65 | 66 | l = sparql.Literal(u'Hello world') 67 | self.assertEqual(u'', repr(l)) 68 | 69 | 70 | class TestTypedLiterals(unittest.TestCase): 71 | 72 | def test_isinstance(self): 73 | """ Type literals are instances of RDFTerm """ 74 | 75 | l = sparql.Literal(u'Hello world', 76 | u'http://www.w3.org/2001/XMLSchema#string') 77 | assert isinstance(l, sparql.RDFTerm) 78 | 79 | def test_repr(self): 80 | """ repr should return the literal in N3 syntax """ 81 | 82 | l = sparql.Literal(u'Hello world', 83 | u'http://www.w3.org/2001/XMLSchema#string') 84 | self.assertEqual(u'>' 85 | , repr(l)) 86 | 87 | def test_str(self): 88 | """ str should return the literal without type """ 89 | 90 | l = sparql.Literal(u'Hello world', 91 | u'http://www.w3.org/2001/XMLSchema#string') 92 | assert str(l) == u'Hello world' 93 | 94 | def test_literal_same(self): 95 | """ Two literals with same language must be the same """ 96 | 97 | l1 = sparql.Literal(u'Hello world', 98 | u'http://aims.fao.org/aos/geopolitical.owl#MillionUSD' 99 | ) 100 | l2 = sparql.Literal(u'Hello world', 101 | u'http://aims.fao.org/aos/geopolitical.owl#MillionUSD' 102 | ) 103 | self.assertEqual(l1, l2) 104 | 105 | def test_literal_notsame1(self): 106 | """ Two literals different language must be different """ 107 | 108 | l1 = sparql.Literal(u'Hello world', 109 | u'http://aims.fao.org/aos/geopolitical.owl#MillionUSD' 110 | ) 111 | l2 = sparql.Literal(u'Hello world', 112 | u'http://www.w3.org/2001/XMLSchema#string') 113 | self.assertNotEqual(l1, l2) 114 | 115 | def test_literal_notsame2(self): 116 | """ Difference on both value and language """ 117 | 118 | l1 = sparql.Literal(u'Hello world', 119 | u'http://aims.fao.org/aos/geopolitical.owl#MillionUSD' 120 | ) 121 | l2 = sparql.Literal(u'Hallo Welt', 122 | u'http://aims.fao.org/aos/geopolitical.owl#MillionUSD' 123 | ) 124 | self.assertNotEqual(l1, l2) 125 | 126 | def test_literal_notsame3(self): 127 | """ Two literals with same language must be the same """ 128 | 129 | l1 = sparql.Literal(u'Hello world', 130 | u'http://aims.fao.org/aos/geopolitical.owl#MillionUSD' 131 | ) 132 | self.assertNotEqual(u'Hello world', l1) 133 | self.assertNotEqual(l1, u'Hello world') 134 | assert l1 != u'Hello world' 135 | 136 | def test_compare_with_non_literal(self): 137 | l1 = sparql.Literal('hi', 138 | u'http://aims.fao.org/aos/geopolitical.owl#MillionUSD' 139 | ) 140 | self.assertFalse(l1 == 'hi') 141 | self.assertFalse(l1 == u'hi') 142 | self.assertFalse(l1 == None) 143 | self.assertFalse(l1 == 13) 144 | self.assertFalse(l1 == 13.0) 145 | self.assertFalse(l1 == ['hi']) 146 | self.assertFalse(l1 == {'hi': 'hi'}) 147 | 148 | def test_convert_to_unicode(self): 149 | """ Literals should convert values to unicode when saving them """ 150 | 151 | class SomeType(object): 152 | 153 | def __unicode__(self): 154 | return u'hello world' 155 | 156 | if six.PY2: 157 | lt = sparql.Literal(SomeType(), 158 | u'http://aims.fao.org/aos/geopolitical.owl#MillionUSD' 159 | ) 160 | else: 161 | lt = sparql.Literal(SomeType().__unicode__(), 162 | u'http://aims.fao.org/aos/geopolitical.owl#MillionUSD' 163 | ) 164 | self.assertEqual(str(lt), 'hello world') 165 | 166 | 167 | class TestIRIs(unittest.TestCase): 168 | 169 | def test_repr(self): 170 | """ repr should return the literal in N3 syntax """ 171 | 172 | i = sparql.IRI('http://example.com/asdf') 173 | self.assertEqual(repr(i), '>') 174 | 175 | def test_compare_with_non_iri(self): 176 | i1 = sparql.IRI('http://example.com/asdf') 177 | self.assertFalse(i1 == 'http://example.com/asdf') 178 | self.assertFalse(i1 == u'http://example.com/asdf') 179 | self.assertFalse(i1 == None) 180 | self.assertFalse(i1 == 13) 181 | self.assertFalse(i1 == 13.0) 182 | self.assertFalse(i1 == ['http://example.com/asdf']) 183 | self.assertFalse(i1 184 | == {'http://example.com/asdf': 'http://example.com/asdf' 185 | }) 186 | 187 | 188 | _literal_data = [ 189 | ('' , '""'), 190 | (' ' , '" "'), 191 | ('hello' , '"hello"'), 192 | ("back\\slash" , '"back\\\\slash"'), 193 | ('quot"ed' , '"quot\\"ed"'), 194 | ("any\"quot'es" , '"any\\"quot\'es"'), 195 | ("new\nlines" , '"new\\nlines"'), 196 | ("ta\tbs" , '"ta\\tbs"'), 197 | (u"ascii-unicode" , '"ascii-unicode"'), 198 | (u"̈Ünɨcøðé" , '"\\u0308\\u00dcn\\u0268c\\u00f8\\u00f0\\u00e9"') 199 | ] 200 | 201 | if six.PY2: 202 | _literal_data.append((u"\u6f22\u5b57(kanji)" , '"\u6f22\u5b57(kanji)"')) 203 | else: 204 | _literal_data.append((u"\u6f22\u5b57(kanji)" , '"\\u6f22\\u5b57(kanji)"')) 205 | 206 | 207 | class TestNotation3(unittest.TestCase): 208 | 209 | def test_literal(self): 210 | """ Notation3 representation of a literal """ 211 | 212 | for (value, expected) in _literal_data: 213 | self.assertEqual(sparql.Literal(value).n3(), expected) 214 | self.assertEqual(sparql.Literal(value, lang='en').n3(), 215 | expected + '@en') 216 | 217 | def test_typed_literal(self): 218 | """ N3 notation of a typed literal """ 219 | 220 | datatype = u'http://www.w3.org/2001/XMLSchema#string' 221 | for (value, expected) in _literal_data: 222 | tl = sparql.Literal(value, datatype) 223 | self.assertEqual(tl.n3(), '%s^^<%s>' % (expected, datatype)) 224 | 225 | 226 | if __name__ == '__main__': 227 | unittest.main() 228 | -------------------------------------------------------------------------------- /sparql-client/tests/testhttp.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import unittest 5 | import sparql 6 | import six 7 | 8 | QUERIES = { 9 | "SELECT * WHERE {?s ?p ?o} LIMIT 2": """\ 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | http://rdfdata.eionet.europa.eu/eea/languages/en 21 | 22 | 23 | http://www.w3.org/1999/02/22-rdf-syntax-ns#type 24 | 25 | 26 | http://rdfdata.eionet.europa.eu/eea/ontology/Language 27 | 28 | 29 | 30 | 31 | http://rdfdata.eionet.europa.eu/eea/languages/da 32 | 33 | 34 | http://www.w3.org/1999/02/22-rdf-syntax-ns#type 35 | 36 | 37 | http://rdfdata.eionet.europa.eu/eea/ontology/Language 38 | 39 | 40 | 41 | 42 | """ 43 | } 44 | 45 | 46 | class MockResponse(object): 47 | def getcode(self): 48 | return 200 49 | 50 | 51 | class MockQuery(sparql._Query): 52 | def _get_response(self, opener, request, buf, timeout): 53 | if six.PY2: 54 | self.querystring = request.get_data() 55 | else: 56 | if not request.data: 57 | self.querystring = request.selector.split('?')[1] 58 | else: 59 | self.querystring = request.data 60 | if isinstance(self.querystring, six.binary_type): 61 | self.querystring = self.querystring.decode("utf-8") 62 | return MockResponse() 63 | 64 | def _read_response(self, response, buf, timeout): 65 | try: 66 | from six.moves.urllib.parse import parse_qs 67 | except ImportError: 68 | from cgi import parse_qs 69 | query = parse_qs(self.querystring).get('query', [''])[0] 70 | if not six.PY2: 71 | value = QUERIES[query].encode() 72 | else: 73 | value = QUERIES[query] 74 | buf.write(value) 75 | 76 | 77 | class TestSparqlEndpoint(unittest.TestCase): 78 | 79 | def setUp(self): 80 | self.old_Query = sparql._Query 81 | sparql._Query = MockQuery 82 | 83 | def tearDown(self): 84 | sparql._Query = self.old_Query 85 | 86 | def test_simple_query(self): 87 | from sparql import IRI 88 | URI_LANG = 'http://rdfdata.eionet.europa.eu/eea/languages' 89 | URI_TYPE = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type' 90 | URI_LANG_TYPE = 'http://rdfdata.eionet.europa.eu/eea/ontology/Language' 91 | endpoint = "http://cr3.eionet.europa.eu/sparql" 92 | 93 | result = sparql.query(endpoint, "SELECT * WHERE {?s ?p ?o} LIMIT 2") 94 | 95 | self.assertEqual(result.variables, ['s', 'p', 'o']) 96 | self.assertEqual(list(result), [ 97 | (IRI(URI_LANG+'/en'), IRI(URI_TYPE), IRI(URI_LANG_TYPE)), 98 | (IRI(URI_LANG+'/da'), IRI(URI_TYPE), IRI(URI_LANG_TYPE)), 99 | ]) 100 | 101 | 102 | if __name__ == '__main__': 103 | unittest.main() 104 | -------------------------------------------------------------------------------- /sparql-client/tests/testn3parse.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import sparql 3 | import six 4 | 5 | try: 6 | from testdatatypes import _literal_data 7 | except ImportError: 8 | from .testdatatypes import _literal_data 9 | 10 | _string_literals = [ 11 | ('""', ''), # empty string 12 | ("''", ''), # empty string 13 | ('""""""', ''), # triple quotes (") 14 | ("''''''", ''), # triple quotes (') 15 | ('" "', ' '), # one space 16 | ('"hi"', 'hi'), 17 | ("'hi'", 'hi'), 18 | ("'some\\ntext'", 'some\ntext'), # newline 19 | ("'some\\ttext'", 'some\ttext'), # tab 20 | ("'''some\ntext\n with spaces'''", 'some\ntext\n with spaces'), 21 | ] 22 | 23 | for _value, _n3 in _literal_data: 24 | _string_literals.append((_n3, _value)) 25 | 26 | 27 | class N3ParsingTest(unittest.TestCase): 28 | 29 | def test_unicode(self): 30 | value = 'http://example.com/some_iri' 31 | 32 | class Tricky(object): 33 | 34 | def __unicode__(self): 35 | return '<%s>' % value 36 | 37 | if six.PY2: 38 | parsed = sparql.parse_n3_term(Tricky()) 39 | else: 40 | parsed = sparql.parse_n3_term(Tricky().__unicode__()) 41 | self.assertEqual(parsed, sparql.IRI(value)) 42 | 43 | def test_parse_IRI(self): 44 | value = 'http://example.com/some_iri' 45 | result = sparql.parse_n3_term('<%s>' % value) 46 | self.assertTrue(type(result) is sparql.IRI) 47 | self.assertEqual(result.value, value) 48 | 49 | i = sparql.IRI(value) 50 | self.assertEqual(sparql.parse_n3_term(i.n3()), i) 51 | 52 | def test_IRI_error(self): 53 | parse = sparql.parse_n3_term 54 | self.assertRaises(ValueError, parse, '') 56 | self.assertRaises(ValueError, parse, '') 57 | self.assertRaises(ValueError, parse, 'ri>') 58 | 59 | def test_literal(self): 60 | for (n3_value, value) in _string_literals: 61 | result = sparql.parse_n3_term(n3_value) 62 | self.assertTrue(type(result) is sparql.Literal) 63 | self.assertEqual(result.lang, None) 64 | self.assertEqual(result.value, value) 65 | 66 | l = sparql.Literal(value) 67 | self.assertEqual(sparql.parse_n3_term(l.n3()), l) 68 | 69 | def test_literal_with_lang(self): 70 | for (n3_value, value) in _string_literals: 71 | n3_value_with_lang = n3_value + '@en' 72 | result = sparql.parse_n3_term(n3_value_with_lang) 73 | self.assertTrue(type(result) is sparql.Literal) 74 | self.assertEqual(result.lang, 'en') 75 | self.assertEqual(result.value, value) 76 | 77 | l = sparql.Literal(value, lang='en') 78 | self.assertEqual(sparql.parse_n3_term(l.n3()), l) 79 | 80 | def test_typed_literals(self): 81 | million_uri = u'http://aims.fao.org/aos/geopolitical.owl#MillionUSD' 82 | for (n3_value, value) in _string_literals: 83 | n3_value_with_type = n3_value + '^^<' + million_uri + '>' 84 | result = sparql.parse_n3_term(n3_value_with_type) 85 | self.assertTrue(type(result) is sparql.Literal) 86 | self.assertEqual(result.datatype, million_uri) 87 | self.assertEqual(result.value, value) 88 | 89 | l = sparql.Literal(value, million_uri) 90 | self.assertEqual(sparql.parse_n3_term(l.n3()), l) 91 | 92 | def test_evil_literals(self): 93 | parse = sparql.parse_n3_term 94 | self.assertRaises(ValueError, parse, '"hello" + " world"') 95 | self.assertRaises(ValueError, parse, '"hello"\nx = " world"') 96 | self.assertRaises(ValueError, parse, 'hello') -------------------------------------------------------------------------------- /sparql-client/tests/testparser.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import unittest 5 | import sparql 6 | import os.path 7 | from mock import patch 8 | from xml.dom import pulldom 9 | from six.moves import map 10 | import six 11 | 12 | 13 | def _open_datafile(name): 14 | return open(os.path.join(os.path.dirname(__file__), name)) 15 | 16 | 17 | XSD_FAO_MILLION = 'http://aims.fao.org/aos/geopolitical.owl#MillionUSD' 18 | 19 | 20 | class TestParser(unittest.TestCase): 21 | 22 | def test_simple(self): 23 | """ Simple query with unbound variables """ 24 | 25 | resultfp = _open_datafile('countries.srx') 26 | result = sparql._ResultsParser(resultfp) 27 | self.assertEqual([ 28 | u'eeaURI', 29 | u'gdpTotal', 30 | u'eeacode', 31 | u'nutscode', 32 | u'faocode', 33 | u'gdp', 34 | u'name', 35 | ], result.variables) 36 | 37 | rows = result.fetchall() 38 | row0 = rows[0] 39 | self.assertEqual(sparql.IRI(u'http://rdfdata.eionet.europa.eu/eea/countries/BE' 40 | ), row0[0]) 41 | self.assertEqual(sparql.Literal('471161.0', XSD_FAO_MILLION), 42 | row0[1]) 43 | self.assertEqual(sparql.Literal('44.252934', sparql.XSD_FLOAT), 44 | row0[5]) 45 | 46 | def test_unpack(self): 47 | resultfp = _open_datafile('countries.srx') 48 | result = sparql._ResultsParser(resultfp) 49 | self.assertEqual([ 50 | u'eeaURI', 51 | u'gdpTotal', 52 | u'eeacode', 53 | u'nutscode', 54 | u'faocode', 55 | u'gdp', 56 | u'name', 57 | ], result.variables) 58 | 59 | rows = list(map(sparql.unpack_row, result.fetchall())) 60 | row0 = rows[0] 61 | self.assertEqual(u'http://rdfdata.eionet.europa.eu/eea/countries/BE' 62 | , row0[0]) 63 | 64 | # XSD_FAO_MILLION unpacked as string 65 | 66 | self.assertEqual('471161.0', row0[1]) 67 | 68 | # XSD_FLOAT unpacked as float 69 | 70 | self.assertNotEqual('44.252934', row0[5]) 71 | self.assertEqual(44.252934, row0[5]) 72 | 73 | def test_fetchmany(self): 74 | """ Simple query with unbound variables """ 75 | 76 | resultfp = _open_datafile('countries.srx') 77 | result = sparql._ResultsParser(resultfp) 78 | self.assertEqual([ 79 | u'eeaURI', 80 | u'gdpTotal', 81 | u'eeacode', 82 | u'nutscode', 83 | u'faocode', 84 | u'gdp', 85 | u'name', 86 | ], result.variables) 87 | 88 | rows = result.fetchmany(2) 89 | self.assertEqual(2, len(rows)) 90 | row0 = rows[0] 91 | if six.PY2: 92 | self.assertEqual('http://rdfdata.eionet.europa.eu/eea/countries/BE' 93 | , str(row0[0])) 94 | else: 95 | self.assertEqual('http://rdfdata.eionet.europa.eu/eea/countries/BE' 96 | , row0[0].value) 97 | rows = result.fetchmany(2) 98 | self.assertEqual(1, len(rows)) 99 | row0 = rows[0] 100 | assert str(row0[6]) == 'Japan' 101 | 102 | def test_ask_query(self): 103 | """ Check that http://www.w3.org/TR/rdf-sparql-XMLres/output2.srx works """ 104 | 105 | resultfp = _open_datafile('w3-output2.srx') 106 | result = sparql._ResultsParser(resultfp) 107 | rows = result.fetchall() 108 | assert len(rows) == 0 109 | 110 | # for row in result.fetchone(): 111 | # print row 112 | # row1 = result.fetchone() 113 | # print row1[0] 114 | 115 | def test_w3_example(self): 116 | """ Check that http://www.w3.org/TR/rdf-sparql-XMLres/output.srx works """ 117 | 118 | resultfp = _open_datafile('w3-output.srx') 119 | result = sparql._ResultsParser(resultfp) 120 | self.assertEqual([ 121 | u'x', 122 | u'hpage', 123 | u'name', 124 | u'mbox', 125 | u'age', 126 | u'blurb', 127 | u'friend', 128 | ], result.variables) 129 | rows = result.fetchall() 130 | row0 = rows[0] 131 | if six.PY2: 132 | self.assertEqual('http://work.example.org/alice/', 133 | str(row0[1])) 134 | else: 135 | self.assertEqual('http://work.example.org/alice/', 136 | row0[1].value) 137 | 138 | def test_hasresult(self): 139 | """ Check that http://www.w3.org/TR/rdf-sparql-XMLres/output2.srx works """ 140 | 141 | resultfp = _open_datafile('w3-output2.srx') 142 | result = sparql._ResultsParser(resultfp) 143 | assert result.hasresult() == True 144 | 145 | def test_national(self): 146 | """ Simple query with UTF-8 """ 147 | 148 | resultfp = _open_datafile('national.srx') 149 | result = sparql._ResultsParser(resultfp) 150 | self.assertEqual([u'subj', u'nameen', u'nameru'], 151 | result.variables) 152 | 153 | rows = result.fetchall() 154 | row0 = rows[0] 155 | if six.PY2: 156 | self.assertEqual('http://aims.fao.org/aos/geopolitical.owl#Germany' 157 | , str(row0[0])) 158 | else: 159 | self.assertEqual('http://aims.fao.org/aos/geopolitical.owl#Germany' 160 | , row0[0].value) 161 | 162 | self.assertEqual(sparql.IRI(u'http://aims.fao.org/aos/geopolitical.owl#Germany' 163 | ), row0[0]) 164 | self.assertEqual(u"Германия", six.text_type(row0[2])) 165 | 166 | def test_big_text(self): 167 | 168 | # `xml.dom.pulldom` may return several text nodes within a single 169 | # binding. This seems to be triggered especially by entities, e.g. 170 | # "<". 171 | 172 | resultfp = _open_datafile('big_text.srx') 173 | result = sparql._ResultsParser(resultfp) 174 | row0 = result.fetchall()[0] 175 | self.assertEqual('multiple
paragraphs
here', 176 | row0[0].value) 177 | self.assertEqual('http://example.com/', row0[1].value) 178 | self.assertEqual('bnode.id', row0[2].value) 179 | 180 | def side_effect_fetchhead(): 181 | fp = _open_datafile('invalid-result.srx') 182 | return pulldom.parse(fp) 183 | 184 | @patch('sparql._ResultsParser._fetchhead', 185 | side_effect=side_effect_fetchhead) 186 | def test_invalid_fetchone(self, mocked_element): 187 | """ Simple query with invalid characters """ 188 | 189 | resultfp = _open_datafile('invalid-result.srx') 190 | result = sparql._ResultsParser(resultfp) 191 | setattr(result, 'events', result._fetchhead()) 192 | for row in result.fetchone(): 193 | print(row) 194 | 195 | 196 | if __name__ == '__main__': 197 | unittest.main() 198 | -------------------------------------------------------------------------------- /sparql-client/tests/w3-output.srx: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | r1 22 | http://work.example.org/alice/ 23 | Alice 24 | 25 | r2 26 | <p xmlns="http://www.w3.org/1999/xhtml">My name is <b>alice</b></p> 27 | 28 | 29 | 30 | r2 31 | http://work.example.org/bob/ 32 | Bob 33 | mailto:bob@work.example.org 34 | 30 35 | r1 36 | 37 | 38 | 39 | 40 | 41 | -------------------------------------------------------------------------------- /sparql-client/tests/w3-output2.srx: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 7 | 8 | 9 | 10 | true 11 | 12 | 13 | -------------------------------------------------------------------------------- /sparql-client/tests/xsdtypes.srx: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | Estonia 12 | 123.456 13 | 2009-11-02 14:31:40 14 | 1991-08-20 15 | 18:58:21 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /sparql.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # The contents of this file are subject to the Mozilla Public 5 | # License Version 1.1 (the "License"); you may not use this file 6 | # except in compliance with the License. You may obtain a copy of 7 | # the License at http://www.mozilla.org/MPL/ 8 | # 9 | # Software distributed under the License is distributed on an "AS 10 | # IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or 11 | # implied. See the License for the specific language governing 12 | # rights and limitations under the License. 13 | # 14 | # The Original Code is "SPARQL Client" 15 | # 16 | # The Initial Owner of the Original Code is European Environment 17 | # Agency (EEA). Portions created by Eau de Web are 18 | # Copyright (C) 2011 by European Environment Agency. All 19 | # Rights Reserved. 20 | # 21 | # Contributor(s): 22 | # Søren Roug, EEA 23 | # Alex Morega, Eau de Web 24 | # David Bătrânu, Eau de Web 25 | 26 | """ 27 | The `sparql` module can be invoked in several different ways. To quickly run a 28 | query use :func:`query`. Results are encapsulated in a 29 | :class:`_ResultsParser` instance:: 30 | 31 | >>> result = sparql.query(endpoint, query) 32 | >>> for row in result: 33 | >>> print(row) 34 | 35 | 36 | Command-line use 37 | ---------------- 38 | 39 | :: 40 | 41 | sparql.py [-i] endpoint 42 | -i Interactive mode 43 | 44 | If interactive mode is enabled, the program reads queries from the console 45 | and then executes them. Use a double line (two 'enters') to separate queries. 46 | 47 | Otherwise, the query is read from standard input. 48 | """ 49 | 50 | from base64 import encodestring 51 | from six.moves import input, map 52 | from six.moves.urllib.parse import urlencode 53 | from xml.dom import pulldom 54 | from xml.sax import SAXParseException 55 | import copy 56 | import decimal 57 | import re 58 | import tempfile 59 | import eventlet 60 | import six 61 | if six.PY2: 62 | from eventlet.green import urllib2 as ev_request 63 | import compiler 64 | else: 65 | from eventlet.green.urllib import request as ev_request 66 | import ast as astcompiler 67 | 68 | try: 69 | __version__ = open('version.txt').read().strip() 70 | except Exception: 71 | __version__ = "2.6" 72 | 73 | USER_AGENT = "sparql-client/%s +https://www.eionet.europa.eu/software/sparql-client/" % __version__ 74 | 75 | CONTENT_TYPE = { 76 | 'turtle': "application/turtle", 77 | 'n3': "application/n3", 78 | 'rdfxml': "application/rdf+xml", 79 | 'ntriples': "application/n-triples", 80 | 'xml': "application/xml" 81 | } 82 | 83 | 84 | RESULTS_TYPES = { 85 | 'xml': "application/sparql-results+xml", 86 | 'xmlschema': "application/x-ms-access-export+xml", 87 | 'json': "application/sparql-results+json" 88 | } 89 | 90 | # The purpose of this construction is to use shared strings when 91 | # they have the same value. This way comparisons can happen on the 92 | # memory address rather than looping through the content. 93 | XSD_STRING = 'http://www.w3.org/2001/XMLSchema#string' 94 | XSD_INT = 'http://www.w3.org/2001/XMLSchema#int' 95 | XSD_LONG = 'http://www.w3.org/2001/XMLSchema#long' 96 | XSD_DOUBLE = 'http://www.w3.org/2001/XMLSchema#double' 97 | XSD_FLOAT = 'http://www.w3.org/2001/XMLSchema#float' 98 | XSD_INTEGER = 'http://www.w3.org/2001/XMLSchema#integer' 99 | XSD_DECIMAL = 'http://www.w3.org/2001/XMLSchema#decimal' 100 | XSD_DATETIME = 'http://www.w3.org/2001/XMLSchema#dateTime' 101 | XSD_DATE = 'http://www.w3.org/2001/XMLSchema#date' 102 | XSD_TIME = 'http://www.w3.org/2001/XMLSchema#time' 103 | XSD_BOOLEAN = 'http://www.w3.org/2001/XMLSchema#boolean' 104 | 105 | datatype_dict = { 106 | '': '', 107 | XSD_STRING: XSD_STRING, 108 | XSD_INT: XSD_INT, 109 | XSD_LONG: XSD_LONG, 110 | XSD_DOUBLE: XSD_DOUBLE, 111 | XSD_FLOAT: XSD_FLOAT, 112 | XSD_INTEGER: XSD_INTEGER, 113 | XSD_DECIMAL: XSD_DECIMAL, 114 | XSD_DATETIME: XSD_DATETIME, 115 | XSD_DATE: XSD_DATE, 116 | XSD_TIME: XSD_TIME, 117 | XSD_BOOLEAN: XSD_BOOLEAN 118 | } 119 | 120 | # allow import from RestrictedPython 121 | __allow_access_to_unprotected_subobjects__ = { 122 | 'Datatype': 1, 'unpack_row': 1, 123 | 'RDFTerm': 1, 'IRI': 1, 124 | 'Literal': 1, 'BlankNode': 1 125 | } 126 | 127 | 128 | def Datatype(value): 129 | """ 130 | Replace the string with a shared string. 131 | intern() only works for plain strings - not unicode. 132 | We make it look like a class, because it conceptually could be. 133 | """ 134 | if value is None: 135 | r = None 136 | elif value in datatype_dict: 137 | r = datatype_dict[value] 138 | else: 139 | r = datatype_dict[value] = value 140 | return r 141 | 142 | 143 | class RDFTerm(object): 144 | """ 145 | Super class containing methods to override. :class:`IRI`, 146 | :class:`Literal` and :class:`BlankNode` all inherit from :class:`RDFTerm`. 147 | """ 148 | 149 | __allow_access_to_unprotected_subobjects__ = {'n3': 1} 150 | 151 | def __str__(self): 152 | return str(self.value) 153 | 154 | def __unicode__(self): 155 | return self.value 156 | 157 | def n3(self): 158 | """ Return a Notation3 representation of this term. """ 159 | # To override 160 | # See N-Triples syntax: http://www.w3.org/TR/rdf-testcases/#ntriples 161 | raise NotImplementedError("Subclasses of RDFTerm must implement `n3`") 162 | 163 | def __repr__(self): 164 | return '<%s %s>' % (type(self).__name__, self.n3()) 165 | 166 | 167 | class IRI(RDFTerm): 168 | """ An RDF resource. """ 169 | 170 | def __init__(self, value): 171 | self.value = value 172 | 173 | def __str__(self): 174 | return self.value.encode("unicode-escape") 175 | 176 | def __eq__(self, other): 177 | if type(self) != type(other): 178 | return False 179 | if self.value == other.value: 180 | return True 181 | return False 182 | 183 | def n3(self): 184 | return '<%s>' % self.value 185 | 186 | _n3_quote_char = re.compile(r'[^ -~]|["\\]') 187 | _n3_quote_map = { 188 | '"': '\\"', 189 | '\n': '\\n', 190 | '\t': '\\t', 191 | '\\': '\\\\', 192 | } 193 | 194 | 195 | def _n3_quote(string): 196 | def escape(m): 197 | ch = m.group() 198 | if ch in _n3_quote_map: 199 | return _n3_quote_map[ch] 200 | return "\\u%04x" % ord(ch) 201 | return '"' + _n3_quote_char.sub(escape, string) + '"' 202 | 203 | 204 | class Literal(RDFTerm): 205 | """ 206 | Literals. These can take a data type or a language code. 207 | """ 208 | def __init__(self, value, datatype=None, lang=None): 209 | self.value = six.text_type(value) 210 | self.lang = lang 211 | self.datatype = datatype 212 | 213 | def __eq__(self, other): 214 | if type(self) != type(other): 215 | return False 216 | 217 | elif (self.value == other.value and 218 | self.lang == other.lang and 219 | self.datatype == other.datatype): 220 | return True 221 | 222 | return False 223 | 224 | def n3(self): 225 | n3_value = _n3_quote(self.value) 226 | 227 | if self.datatype is not None: 228 | n3_value += '^^<%s>' % self.datatype 229 | 230 | if self.lang is not None: 231 | n3_value += '@' + self.lang 232 | 233 | return n3_value 234 | 235 | 236 | class BlankNode(RDFTerm): 237 | """ Blank node. Similar to `IRI` but lacks a stable identifier. """ 238 | def __init__(self, value): 239 | self.value = value 240 | 241 | def __eq__(self, other): 242 | if type(self) != type(other): 243 | return False 244 | if self.value == other.value: 245 | return True 246 | return False 247 | 248 | def n3(self): 249 | return '_:%s' % self.value 250 | 251 | _n3parser_lang = re.compile(r'@(?P\w+)$') 252 | _n3parser_datatype = re.compile(r'\^\^<(?P[^\^"\'>]+)>$') 253 | 254 | 255 | def parse_n3_term(src): 256 | """ 257 | Parse a Notation3 value into a RDFTerm object (IRI or Literal). 258 | 259 | This parser understands IRIs and quoted strings; basic non-string types 260 | (integers, decimals, booleans, etc) are not supported yet. 261 | """ 262 | 263 | src = six.text_type(src) 264 | 265 | if src.startswith('<'): 266 | # `src` is an IRI 267 | if not src.endswith('>'): 268 | raise ValueError 269 | 270 | value = src[1:-1] 271 | if '<' in value or '>' in value: 272 | raise ValueError 273 | 274 | return IRI(value) 275 | 276 | else: 277 | datatype_match = _n3parser_datatype.search(src) 278 | if datatype_match is not None: 279 | datatype = datatype_match.group('datatype') 280 | src = _n3parser_datatype.sub('', src) 281 | else: 282 | datatype = None 283 | 284 | lang_match = _n3parser_lang.search(src) 285 | if lang_match is not None: 286 | lang = lang_match.group('lang') 287 | src = _n3parser_lang.sub('', src) 288 | else: 289 | lang = None 290 | 291 | # Python literals syntax is mostly compatible with N3. 292 | # We don't execute the code, just turn it into an AST. 293 | try: 294 | if six.PY2: 295 | ast = compiler.parse("value = u" + src) 296 | else: 297 | # should be fixed due to #111217 298 | ast = astcompiler.parse("value = " + src) 299 | except: 300 | raise ValueError 301 | 302 | if six.PY2: 303 | # Don't allow any extra tokens in the AST 304 | if len(ast.node.getChildNodes()) != 1: 305 | raise ValueError 306 | assign_node = ast.node.getChildNodes()[0] 307 | if len(assign_node.getChildNodes()) != 2: 308 | raise ValueError 309 | value_node = assign_node.getChildNodes()[1] 310 | if value_node.getChildNodes(): 311 | raise ValueError 312 | if value_node.__class__ != compiler.ast.Const: 313 | raise ValueError 314 | value = value_node.value 315 | else: 316 | # Don't allow any extra tokens in the AST 317 | if len(ast.body) != 1: 318 | raise ValueError 319 | assign_node = ast.body[0] 320 | 321 | if len(assign_node._fields) != 2: 322 | raise ValueError 323 | 324 | value_node = assign_node.value 325 | if len(value_node._fields) != 1: 326 | raise ValueError 327 | 328 | # if value_node.__class__ != ast.Constant(): 329 | # raise ValueError 330 | value = getattr(value_node, value_node._fields[0]) 331 | 332 | if type(value) is not six.text_type: 333 | raise ValueError 334 | 335 | return Literal(value, datatype, lang) 336 | 337 | ######################################### 338 | # 339 | # _ServiceMixin 340 | # 341 | ######################################### 342 | 343 | 344 | class _ServiceMixin(object): 345 | def __init__(self, endpoint, method="POST", accept=RESULTS_TYPES['xml']): 346 | self._method = method 347 | self.endpoint = endpoint 348 | self._default_graphs = [] 349 | self._named_graphs = [] 350 | self._prefix_map = {} 351 | 352 | self._headers_map = {} 353 | self._headers_map['Accept'] = accept 354 | self._headers_map['User-Agent'] = USER_AGENT 355 | 356 | def _setMethod(self, method): 357 | if method in ("GET", "POST"): 358 | self._method = method 359 | else: 360 | raise ValueError("Only GET or POST is allowed") 361 | 362 | def _getMethod(self): 363 | return self._method 364 | 365 | method = property(_getMethod, _setMethod) 366 | 367 | def addDefaultGraph(self, g): 368 | self._default_graphs.append(g) 369 | 370 | def defaultGraphs(self): 371 | return self._default_graphs 372 | 373 | def addNamedGraph(self, g): 374 | self._named_graphs.append(g) 375 | 376 | def namedGraphs(self): 377 | return self._named_graphs 378 | 379 | def setPrefix(self, prefix, uri): 380 | self._prefix_map[prefix] = uri 381 | 382 | def prefixes(self): 383 | return self._prefix_map 384 | 385 | def headers(self): 386 | return self._headers_map 387 | 388 | ######################################### 389 | # 390 | # Service 391 | # 392 | ######################################### 393 | 394 | 395 | class Service(_ServiceMixin): 396 | """ 397 | This is the main entry to the library. 398 | The user creates a :class:`Service`, then sends a query to it. 399 | If we want to have persistent connections, then open them here. 400 | """ 401 | def __init__(self, endpoint, qs_encoding="utf-8", method="POST", 402 | accept="application/sparql-results+xml"): 403 | _ServiceMixin.__init__(self, endpoint, method, accept) 404 | self.qs_encoding = qs_encoding 405 | 406 | def createQuery(self): 407 | q = _Query(self) 408 | q._default_graphs = copy.deepcopy(self._default_graphs) 409 | q._headers_map = copy.deepcopy(self._headers_map) 410 | q._named_graphs = copy.deepcopy(self._named_graphs) 411 | q._prefix_map = copy.deepcopy(self._prefix_map) 412 | return q 413 | 414 | def query(self, query, timeout=0, raw=False): 415 | q = self.createQuery() 416 | return q.query(query, timeout, raw=raw) 417 | 418 | def authenticate(self, username, password): 419 | # self._headers_map['Authorization'] = "Basic %s" % replace( 420 | # encodestring("%s:%s" % (username, password)), "\012", "") 421 | head = "Basic %s" % encodestring("%s:%s" % (username, password)).replace("\012", "") 422 | self._headers_map['Authorization'] = head 423 | 424 | 425 | def _parseBoolean(val): 426 | return val.lower() in ('true', '1') 427 | 428 | 429 | # XMLSchema types and cast functions 430 | _types = { 431 | XSD_INT: int, 432 | XSD_LONG: int, 433 | XSD_DOUBLE: float, 434 | XSD_FLOAT: float, 435 | XSD_INTEGER: int, # INTEGER is a DECIMAL, but Python `int` has no size 436 | # limit, so it's safe to use 437 | XSD_DECIMAL: decimal.Decimal, 438 | XSD_BOOLEAN: _parseBoolean, 439 | } 440 | 441 | try: 442 | import dateutil.parser 443 | _types[XSD_DATETIME] = dateutil.parser.parse 444 | _types[XSD_DATE] = lambda v: dateutil.parser.parse(v).date() 445 | _types[XSD_TIME] = lambda v: dateutil.parser.parse(v).time() 446 | except ImportError: 447 | pass 448 | 449 | 450 | def unpack_row(row, convert=None, convert_type={}): 451 | """ 452 | Convert values in the given row from :class:`RDFTerm` objects to plain 453 | Python values: :class:`IRI` is converted to a unicode string containing 454 | the IRI value; :class:`BlankNode` is converted to a unicode string with 455 | the BNode's identifier, and :class:`Literal` is converted based on its 456 | XSD datatype. 457 | 458 | The library knows about common XSD types (STRING becomes :class:`unicode`, 459 | INTEGER and LONG become :class:`int`, DOUBLE and FLOAT become 460 | :class:`float`, DECIMAL becomes :class:`~decimal.Decimal`, BOOLEAN becomes 461 | :class:`bool`). If the `python-dateutil` library is found, then DATE, 462 | TIME and DATETIME are converted to :class:`~datetime.date`, 463 | :class:`~datetime.time` and :class:`~datetime.datetime` respectively. For 464 | other conversions, an extra argument `convert` may be passed. It should be 465 | a callable accepting two arguments: the serialized value as a 466 | :class:`unicode` object, and the XSD datatype. 467 | """ 468 | out = [] 469 | known_types = dict(_types) 470 | known_types.update(convert_type) 471 | for item in row: 472 | if item is None: 473 | value = None 474 | elif isinstance(item, Literal): 475 | if item.datatype in known_types: 476 | to_python = known_types[item.datatype] 477 | value = to_python(item.value) 478 | elif convert is not None: 479 | value = convert(item.value, item.datatype) 480 | else: 481 | value = item.value 482 | else: 483 | value = item.value 484 | out.append(value) 485 | return out 486 | 487 | ######################################### 488 | # 489 | # _Query 490 | # 491 | ######################################### 492 | 493 | 494 | class _Query(_ServiceMixin): 495 | 496 | def __init__(self, service): 497 | _ServiceMixin.__init__(self, service.endpoint, service.method) 498 | 499 | def _build_request(self, query): 500 | if self.method == "GET": 501 | if '?' in self.endpoint: 502 | separator = '&' 503 | else: 504 | separator = '?' 505 | uri = self.endpoint.strip() + separator + query 506 | return ev_request.Request(uri) 507 | else: 508 | # uri = self.endpoint.strip().encode('ASCII') 509 | uri = self.endpoint.strip() 510 | return ev_request.Request(uri, data=query) 511 | 512 | def _get_response(self, opener, request, buf, timeout=None): 513 | try: 514 | response = opener.open(request, timeout=timeout) 515 | response_code = response.getcode() 516 | if response_code != 200: 517 | buf.seek(0) 518 | ret = buf.read() 519 | buf.close() 520 | raise SparqlException(response_code, ret) 521 | else: 522 | return response 523 | except SparqlException as error: 524 | raise SparqlException('Error', error.message) 525 | else: 526 | return '' 527 | 528 | def _read_response(self, response, buf, timeout): 529 | if timeout > 0: 530 | with eventlet.timeout.Timeout(timeout): 531 | try: 532 | buf.write(response.read()) 533 | except eventlet.timeout.Timeout as error: 534 | raise SparqlException('Timeout', repr(error)) 535 | else: 536 | buf.write(response.read()) 537 | 538 | def _build_response(self, query, opener, buf, timeout): 539 | request = self._build_request(query) 540 | if type(query) is not bytes and not six.PY2: 541 | query = query.encode() 542 | return self._get_response(opener, request, buf, 543 | timeout if timeout > 0 else None) 544 | 545 | def _request(self, statement, timeout=0): 546 | """ 547 | Builds the query string, then opens a connection to the endpoint 548 | and returns the file descriptor. 549 | """ 550 | query = self._queryString(statement) 551 | buf = tempfile.NamedTemporaryFile() 552 | 553 | opener = ev_request.build_opener(RedirectHandler) 554 | opener.addheaders = list(self.headers().items()) 555 | try: 556 | if type(query) is not bytes and not six.PY2: 557 | query = query.encode() 558 | response = self._build_response(query, opener, buf, timeout) 559 | except SparqlException as error: 560 | self.endpoint = error.message 561 | response = self._build_response(query, opener, buf, timeout) 562 | 563 | self._read_response(response, buf, timeout) 564 | buf.seek(0) 565 | return buf 566 | 567 | def query(self, statement, timeout=0, raw=False): 568 | """ 569 | Sends the request and starts the parser on the response. 570 | """ 571 | response = self._request(statement, timeout) 572 | if raw: 573 | return response 574 | 575 | return _ResultsParser(response) 576 | 577 | def _queryString(self, statement): 578 | """ 579 | Creates the REST query string from the statement and graphs. 580 | """ 581 | args = [] 582 | # refs #72876 removing the replace of newline to allow the comments in sparql queries 583 | # statement = statement.replace("\n", " ").encode('utf-8') 584 | # not needed py3 585 | # statement = statement.encode('utf-8') 586 | 587 | pref = ' '.join(["PREFIX %s: <%s> " % (p, self._prefix_map[p]) for p in self._prefix_map]) 588 | if six.PY2: 589 | statement = pref + statement 590 | else: 591 | statement = pref.encode() + statement.encode() 592 | 593 | args.append(('query', statement)) 594 | 595 | for uri in self.defaultGraphs(): 596 | args.append(('default-graph-uri', uri)) 597 | 598 | for uri in self.namedGraphs(): 599 | args.append(('named-graph-uri', uri)) 600 | if six.PY2: 601 | return urlencode(args).encode('utf-8') 602 | return urlencode(args) 603 | 604 | 605 | class RedirectHandler(ev_request.HTTPRedirectHandler): 606 | """ 607 | Subclass the HTTPRedirectHandler to re-contruct request when follow redirect 608 | """ 609 | def redirect_request(self, req, fp, code, msg, headers, newurl): 610 | if code in (301, 302, 303, 307): 611 | raise SparqlException(code, newurl) 612 | else: 613 | return req 614 | 615 | 616 | class _ResultsParser(object): 617 | """ 618 | Parse the XML result. 619 | """ 620 | 621 | __allow_access_to_unprotected_subobjects__ = { 622 | 'fetchone': 1, 623 | 'fetchmany': 1, 624 | 'fetchall': 1, 625 | 'hasresult': 1, 626 | 'variables': 1 627 | } 628 | 629 | def __init__(self, fp): 630 | self.__fp = fp 631 | self._vals = [] 632 | self._hasResult = None 633 | self.variables = [] 634 | self._fetchhead() 635 | 636 | def __del__(self): 637 | self.__fp.close() 638 | 639 | def _fetchhead(self): 640 | """ 641 | Fetches the head information. If there are no variables in the 642 | , then we also fetch the boolean result. 643 | """ 644 | self.events = pulldom.parse(self.__fp) 645 | 646 | for (event, node) in self.events: 647 | if event == pulldom.START_ELEMENT: 648 | if node.tagName == 'variable': 649 | self.variables.append(node.attributes['name'].value) 650 | elif node.tagName == 'boolean': 651 | self.events.expandNode(node) 652 | self._hasResult = (node.firstChild.data == 'true') 653 | elif node.tagName == 'result': 654 | return # We should not arrive here 655 | elif event == pulldom.END_ELEMENT: 656 | if node.tagName == 'head' and self.variables: 657 | return 658 | elif node.tagName == 'sparql': 659 | return 660 | 661 | def hasresult(self): 662 | """ 663 | ASK queries are used to test if a query would have a result. If the 664 | query is an ASK query there won't be an actual result, and 665 | :func:`fetchone` will return nothing. Instead, this method can be 666 | called to check the result from the ASK query. 667 | 668 | If the query is a SELECT statement, then the return value of 669 | :func:`hasresult` is `None`, as the XML result format doesn't tell you 670 | if there are any rows in the result until you have read the first one. 671 | """ 672 | return self._hasResult 673 | 674 | def __iter__(self): 675 | """ Synonym for :func:`fetchone`. """ 676 | return self.fetchone() 677 | 678 | def fetchone(self): 679 | """ Fetches the next set of rows of a query result, returning a list. 680 | An empty list is returned when no more rows are available. 681 | If the query was an ASK request, then an empty list is returned as 682 | there are no rows available. 683 | """ 684 | idx = -1 685 | 686 | try: 687 | for (event, node) in self.events: 688 | if event == pulldom.START_ELEMENT: 689 | if node.tagName == 'result': 690 | self._vals = [None] * len(self.variables) 691 | elif node.tagName == 'binding': 692 | idx = self.variables.index(node.attributes['name'].value) 693 | elif node.tagName == 'uri': 694 | self.events.expandNode(node) 695 | data = ''.join(t.data for t in node.childNodes) 696 | self._vals[idx] = IRI(data) 697 | elif node.tagName == 'literal': 698 | self.events.expandNode(node) 699 | data = ''.join(t.data for t in node.childNodes) 700 | lang = node.getAttribute('xml:lang') or None 701 | datatype = Datatype(node.getAttribute('datatype')) or None 702 | self._vals[idx] = Literal(data, datatype, lang) 703 | elif node.tagName == 'bnode': 704 | self.events.expandNode(node) 705 | data = ''.join(t.data for t in node.childNodes) 706 | self._vals[idx] = BlankNode(data) 707 | elif event == pulldom.END_ELEMENT: 708 | if node.tagName == 'result': 709 | # print "rtn:", len(self._vals), self._vals 710 | yield tuple(self._vals) 711 | except SAXParseException as e: 712 | if six.PY2: 713 | message = e.message 714 | else: 715 | message = e.getMessage() 716 | faultString = 'The data is ' + message 717 | print(faultString) 718 | yield tuple() 719 | 720 | def fetchall(self): 721 | """ Loop through the result to build up a list of all rows. 722 | Patterned after DB-API 2.0. 723 | """ 724 | result = [] 725 | for row in self.fetchone(): 726 | result.append(row) 727 | return result 728 | 729 | def fetchmany(self, num): 730 | result = [] 731 | for row in self.fetchone(): 732 | result.append(row) 733 | num -= 1 734 | if num <= 0: 735 | return result 736 | return result 737 | 738 | 739 | def query(endpoint, query, timeout=0, qs_encoding="utf-8", method="POST", 740 | accept="application/sparql-results+xml", raw=False): 741 | """ 742 | Convenient method to execute a query. Exactly equivalent to:: 743 | 744 | sparql.Service(endpoint).query(query) 745 | """ 746 | s = Service(endpoint, qs_encoding, method, accept) 747 | return s.query(query, timeout, raw=raw) 748 | 749 | 750 | def _interactive(endpoint): 751 | while True: 752 | try: 753 | lines = [] 754 | while True: 755 | next = input() 756 | if not next: 757 | break 758 | else: 759 | lines.append(next) 760 | 761 | if lines: 762 | sys.stdout.write("Querying...") 763 | result = query(endpoint, " ".join(lines)) 764 | sys.stdout.write(" done\n") 765 | 766 | for row in result.fetchone(): 767 | print("\t".join(map(six.text_type, row))) 768 | lines = [] 769 | 770 | except Exception as e: 771 | sys.stderr.write(str(e)) 772 | 773 | 774 | class SparqlException(Exception): 775 | """ Sparql Exceptions """ 776 | def __init__(self, code, message): 777 | self.code = code 778 | self.message = message 779 | 780 | if __name__ == '__main__': 781 | import sys 782 | import codecs 783 | from optparse import OptionParser 784 | 785 | try: 786 | c = codecs.getwriter(sys.stdout.encoding) 787 | except Exception: 788 | c = codecs.getwriter('ascii') 789 | sys.stdout = c(sys.stdout, 'replace') 790 | 791 | parser = OptionParser(usage="%prog [-i] endpoint", 792 | version="%prog " + str(__version__)) 793 | parser.add_option("-i", dest="interactive", action="store_true", 794 | help="Enables interactive mode") 795 | 796 | (options, args) = parser.parse_args() 797 | 798 | if len(args) != 1: 799 | parser.error("Endpoint must be specified") 800 | 801 | endpoint = args[0] 802 | 803 | if options.interactive: 804 | _interactive(endpoint) 805 | 806 | q = sys.stdin.read() 807 | try: 808 | result = query(endpoint, q) 809 | for row in result.fetchone(): 810 | print("\t".join(map(six.text_type, row))) 811 | except SparqlException as e: 812 | if six.PY2: 813 | message = e.message 814 | else: 815 | message = e.getMessage() 816 | faultString = message 817 | print(faultString) 818 | -------------------------------------------------------------------------------- /version.txt: -------------------------------------------------------------------------------- 1 | 3.8 2 | --------------------------------------------------------------------------------