├── .dockerignore ├── .flake8 ├── .gitattributes ├── .github ├── dependabot.yml └── workflows │ ├── cffconvert.yml │ ├── linting.yml │ ├── publish_pypi.yml │ └── testing.yml ├── .gitignore ├── .reuse └── dep5 ├── CITATION.cff ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── CONTRIBUTORS.md ├── Dockerfile ├── LICENSE.txt ├── LICENSES └── MIT.txt ├── MANIFEST.in ├── README.md ├── bin └── grlc-server ├── codemeta.json ├── config.default.ini ├── doc ├── img │ ├── api-docs.png │ ├── commit.png │ ├── endpoint.png │ ├── file_name.png │ ├── files.png │ ├── github_dashboard.png │ ├── grlc_logo_01.eps │ ├── grlc_logo_01.png │ ├── method.png │ ├── new_file.png │ ├── new_repo.png │ ├── new_repository.png │ ├── new_repository_form.png │ ├── parameter_api.png │ ├── query.png │ ├── results.png │ └── try-it-out.png └── notebooks │ └── GrlcFromNotebook.ipynb ├── docker-assets ├── assets │ ├── build │ │ ├── .gitattributes │ │ └── install.sh │ └── runtime │ │ ├── configs │ │ └── nginx │ │ │ └── grlc │ │ ├── env-defaults │ │ └── functions └── entrypoint.sh ├── paper.bib ├── paper.md ├── requirements-test.txt ├── requirements.txt ├── setup.cfg ├── setup.py ├── src ├── __init__.py ├── __version__.py ├── fileLoaders.py ├── glogging.py ├── gquery.py ├── pagination.py ├── prov.py ├── queryTypes.py ├── server.py ├── sparql.py ├── static.py ├── static │ ├── css │ │ └── grlc.css │ ├── favicon.ico │ ├── grlc_logo_01.png │ ├── grlc_logo_02.png │ ├── js │ │ ├── grlc-api.js │ │ └── grlc-layout.js │ └── toolinfo.json ├── swagger.py ├── templates │ ├── api-docs.html │ └── index.html └── utils.py ├── tests ├── __init__.py ├── mock_data.py ├── repo │ ├── endpoint.txt │ ├── test-endpoint-get.rq │ ├── test-enum.rq │ ├── test-json.json │ ├── test-projection.rq │ ├── test-rq.rq │ ├── test-sparql-jsonconf.sparql │ ├── test-sparql.sparql │ ├── test-tpf.tpf │ └── url.yml ├── test_endpoints.py ├── test_gquery.py ├── test_grlc.py ├── test_loaders.py ├── test_swagger.py └── test_utils.py └── upstart ├── grlc-docker.conf ├── grlc-docker.conf.license └── webhook.py /.dockerignore: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2022 Albert Meroño, Rinke Hoekstra, Carlos Martínez 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | # To be ignored 6 | * 7 | 8 | # To be included 9 | !bin/grlc-server 10 | !LICENSES/ 11 | !docker-assets/ 12 | !src/ 13 | !CITATION.cff 14 | !config.default.ini 15 | !README.md 16 | !requirements-test.txt 17 | !requirements.txt 18 | !setup.cfg 19 | !setup.py 20 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | per-file-ignores = 3 | src/__init__.py:F401 4 | src/prov.py:E203 5 | tests/test_grlc.py:F401 6 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2022 Albert Meroño, Rinke Hoekstra, Carlos Martínez 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | docker-compose.yml merge=ours 6 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | updates: 4 | - package-ecosystem: "pip" 5 | directory: "/" 6 | schedule: 7 | interval: "monthly" 8 | target-branch: "dev" 9 | # Apply only major or minor updates, ignore patches 10 | ignore: 11 | - dependency-name: "*" 12 | update-types: ["version-update:semver-patch"] 13 | # Add reviewers 14 | reviewers: 15 | - "albertmeronyo" 16 | - "c-martinez" 17 | -------------------------------------------------------------------------------- /.github/workflows/cffconvert.yml: -------------------------------------------------------------------------------- 1 | name: cffconvert 2 | 3 | on: 4 | push: 5 | paths: 6 | - CITATION.cff 7 | 8 | jobs: 9 | validate: 10 | name: "validate" 11 | runs-on: ubuntu-latest 12 | steps: 13 | - name: Check out a copy of the repository 14 | uses: actions/checkout@v2 15 | 16 | - name: Check whether the citation metadata from CITATION.cff is valid 17 | uses: citation-file-format/cffconvert-github-action@2.0.0 18 | with: 19 | args: "--validate" 20 | -------------------------------------------------------------------------------- /.github/workflows/linting.yml: -------------------------------------------------------------------------------- 1 | name: Lint 2 | 3 | on: 4 | push: 5 | paths: 6 | - 'src/*.py' 7 | - 'tests/*.py' 8 | jobs: 9 | linter: 10 | runs-on: ubuntu-latest 11 | name: Lint 12 | steps: 13 | - name: Check out source repository 14 | uses: actions/checkout@v3 15 | - name: Set up Python environment 16 | uses: actions/setup-python@v4 17 | with: 18 | python-version: "3.11" 19 | - name: flake8 Lint 20 | uses: py-actions/flake8@v2 21 | with: 22 | max-line-length: "127" 23 | path: "src" 24 | -------------------------------------------------------------------------------- /.github/workflows/publish_pypi.yml: -------------------------------------------------------------------------------- 1 | name: Publish 2 | # Publish to PyPI when a new release is published 3 | on: 4 | release: 5 | types: 6 | - published 7 | jobs: 8 | publish: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@v2 12 | - name: Set up Python 13 | uses: actions/setup-python@v1 14 | with: 15 | python-version: 3.11 16 | - name: Install dependencies 17 | run: | 18 | python -m pip install --upgrade pip 19 | pip install setuptools wheel twine 20 | python setup.py sdist bdist_wheel 21 | - name: Publish package 22 | uses: pypa/gh-action-pypi-publish@v1.4.2 23 | with: 24 | user: __token__ 25 | password: ${{ secrets.PYPI_API_TOKEN }} 26 | # repository_url: https://test.pypi.org/legacy/ 27 | -------------------------------------------------------------------------------- /.github/workflows/testing.yml: -------------------------------------------------------------------------------- 1 | name: Testing 2 | on: [push] 3 | jobs: 4 | build: 5 | runs-on: ${{ matrix.os }} 6 | strategy: 7 | matrix: 8 | os: [ubuntu-latest, macos-latest, windows-latest] 9 | python-version: [3.9, 3.10.x, 3.11, 3.12, 3.13] 10 | steps: 11 | - uses: actions/checkout@master 12 | - name: Set up Python ${{ matrix.python-version }} 13 | uses: actions/setup-python@v2 14 | with: 15 | python-version: ${{ matrix.python-version }} 16 | - name: Install dependencies 17 | run: | 18 | python -m pip install --upgrade pip 19 | pip install . 20 | pip install -r requirements-test.txt 21 | - name: Test with pytest 22 | run: | 23 | pytest ./tests 24 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2022 Albert Meroño, Rinke Hoekstra, Carlos Martínez 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | node_modules 6 | .*!.gitignore 7 | */.DS_Store 8 | .Python 9 | *~ 10 | *.swp 11 | *.log 12 | include/ 13 | lib/ 14 | !static/swagger-ui/dist/lib 15 | local/ 16 | pip-selfcheck.json 17 | db.json 18 | \#*\# 19 | *.pyc 20 | db-cache.json 21 | docker-compose.yml 22 | bin/ 23 | !bin/grlc-server 24 | share/ 25 | .idea 26 | .pytest_cache 27 | .eggs/ 28 | .vscode/ 29 | .ipynb_checkpoints/ 30 | ssl-certificates/ 31 | testQueries/ 32 | grlc.egg-info/ 33 | src/config.ini 34 | src/FileLoaderTesting.ipynb 35 | Dockerfile2 36 | FileLoaders.ipynb 37 | ReleaseProcedure.md 38 | TODOs.md 39 | TwitterAPIKeys.md 40 | config.ini 41 | ink_ext_XXXXXX_img0.png 42 | build/ 43 | -------------------------------------------------------------------------------- /.reuse/dep5: -------------------------------------------------------------------------------- 1 | Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ 2 | 3 | Files: tests/repo/endpoint.txt tests/repo/*.rq tests/repo/*.json tests/repo/*.sparql tests/repo/*.tpf tests/repo/*.yml src/static/toolinfo.json src/static/*.png src/static/favicon.ico requirements-test.txt requirements.txt docker-assets/assets/runtime/functions docker-assets/assets/runtime/env-defaults docker-assets/assets/runtime/configs/nginx/grlc doc/notebooks/GrlcFromNotebook.ipynb doc/img/*.png doc/img/grlc_logo_01.eps CITATION.cff .zenodo.json .travis.yml .github/workflows/tweet-release.yml .dependabot/config.yml 4 | Copyright: 2022 Albert Meroño, Rinke Hoekstra, Carlos Martínez 5 | License: MIT 6 | -------------------------------------------------------------------------------- /CITATION.cff: -------------------------------------------------------------------------------- 1 | # YAML 1.2 2 | # Metadata for citation of this software according to the CFF format (https://citation-file-format.github.io/) 3 | --- 4 | authors: 5 | - 6 | affiliation: "King's College London" 7 | family-names: Meroño-Peñuela 8 | given-names: Albert 9 | orcid: "https://orcid.org/0000-0003-4646-5842" 10 | - 11 | family-names: Hoekstra 12 | given-names: Rinke 13 | - 14 | affiliation: "Netherlands eScience Center" 15 | family-names: Martinez 16 | given-names: Carlos 17 | orcid: "https://orcid.org/0000-0001-5565-7577" 18 | cff-version: "1.0.3" 19 | date-released: 2025-04-01 20 | doi: 10.5281/zenodo.1064391 21 | license: MIT 22 | message: "If you use this software, please cite it as below." 23 | repository-code: "https://github.com/CLARIAH/grlc" 24 | title: "grlc: the git repository linked data API constructor" 25 | abstract: grlc, the git repository linked data API constructor, automatically builds Web APIs using SPARQL queries stored in git repositories. 26 | keywords: 27 | - "swagger-ui" 28 | - sparql 29 | - "linked-data" 30 | - "semantic-web" 31 | - "linked-data-api" 32 | version: "1.3.10" 33 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | 6 | 7 | # Contributor Covenant Code of Conduct 8 | 9 | ## Our Pledge 10 | 11 | In the interest of fostering an open and welcoming environment, we as 12 | contributors and maintainers pledge to making participation in our project and 13 | our community a harassment-free experience for everyone, regardless of age, body 14 | size, disability, ethnicity, gender identity and expression, level of experience, 15 | nationality, personal appearance, race, religion, or sexual identity and 16 | orientation. 17 | 18 | ## Our Standards 19 | 20 | Examples of behavior that contributes to creating a positive environment 21 | include: 22 | 23 | * Using welcoming and inclusive language 24 | * Being respectful of differing viewpoints and experiences 25 | * Gracefully accepting constructive criticism 26 | * Focusing on what is best for the community 27 | * Showing empathy towards other community members 28 | 29 | Examples of unacceptable behavior by participants include: 30 | 31 | * The use of sexualized language or imagery and unwelcome sexual attention or 32 | advances 33 | * Trolling, insulting/derogatory comments, and personal or political attacks 34 | * Public or private harassment 35 | * Publishing others' private information, such as a physical or electronic 36 | address, without explicit permission 37 | * Other conduct which could reasonably be considered inappropriate in a 38 | professional setting 39 | 40 | ## Our Responsibilities 41 | 42 | Project maintainers are responsible for clarifying the standards of acceptable 43 | behavior and are expected to take appropriate and fair corrective action in 44 | response to any instances of unacceptable behavior. 45 | 46 | Project maintainers have the right and responsibility to remove, edit, or 47 | reject comments, commits, code, wiki edits, issues, and other contributions 48 | that are not aligned to this Code of Conduct, or to ban temporarily or 49 | permanently any contributor for other behaviors that they deem inappropriate, 50 | threatening, offensive, or harmful. 51 | 52 | ## Scope 53 | 54 | This Code of Conduct applies both within project spaces and in public spaces 55 | when an individual is representing the project or its community. Examples of 56 | representing a project or community include using an official project e-mail 57 | address, posting via an official social media account, or acting as an appointed 58 | representative at an online or offline event. Representation of a project may be 59 | further defined and clarified by project maintainers. 60 | 61 | ## Enforcement 62 | 63 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 64 | reported by contacting the project team at [INSERT EMAIL ADDRESS]. All 65 | complaints will be reviewed and investigated and will result in a response that 66 | is deemed necessary and appropriate to the circumstances. The project team is 67 | obligated to maintain confidentiality with regard to the reporter of an incident. 68 | Further details of specific enforcement policies may be posted separately. 69 | 70 | Project maintainers who do not follow or enforce the Code of Conduct in good 71 | faith may face temporary or permanent repercussions as determined by other 72 | members of the project's leadership. 73 | 74 | ## Attribution 75 | 76 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 77 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 78 | 79 | [homepage]: https://www.contributor-covenant.org 80 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | 6 | 7 | Thank you very much for your interest in contributing to grlc! It's people like you that truly make the Semantic Web more accessible to everyone :) 8 | 9 | ## Communication channels 10 | 11 | If you would like to get in touch with the grlc developers, and with other users of grlc, you can reach us in two ways: 12 | - Via Twitter, by using the grlc handle (**@grlcldapi**). Follow this account to hear about updates. 13 | - Via the grlc [mailing list](https://groups.google.com/g/grlc-list/). Sign up to the mailing list to ask questions and make suggestions. 14 | 15 | ## Filing bug reports 16 | 17 | The official channel to file bug reports is via our GitHub's [issue tracker](https://github.com/CLARIAH/grlc/issues). When doing so make sure that: 18 | - Your issue title briefly describes the bug 19 | - You include log output (try `docker logs grlc_grlc_1` if you daemonized your instance) 20 | - Name the file/module if known/available 21 | - You tag your issue as **bug** 22 | 23 | ## Sending feature requests 24 | 25 | As with bug reports, for requesting features please use the [issue tracker](https://github.com/CLARIAH/grlc/issues) as well and this time: 26 | - Describe briefly the feature in the title 27 | - Describe the desired feature 28 | - Describe your use case so we understand what you are using grlc for 29 | - Name the file/module if known/available 30 | - Tag the issue as **enhancement** 31 | 32 | ## Sending pull requests 33 | 34 | If you would like to contribute to the code directly, please send in a [pull request (PR)](https://github.com/CLARIAH/grlc/pulls). Please make sure that: 35 | - The title of your PR briefly describes the content 36 | - Describe in detail what your PR contributes 37 | - If your PR addresses a specific issue, indicate the issue number 38 | - Assign @albertmeronyo or @c-martinez as reviewer of your PR. 39 | 40 | ## Testing environment 41 | 42 | To get started with hacking grlc, follow these steps to create a local testing environment (you'll need [docker](https://www.docker.com/) and [docker-compose](https://docs.docker.com/compose/)): 43 | 44 | 1. `docker pull clariah/grlc:latest` 45 | 2. `git clone https://github.com/CLARIAH/grlc` 46 | 3. `cd grlc` 47 | 4. Create a `docker-compose.yml` which matches your needs. For example: 48 | ``` 49 | version: '2' 50 | services: 51 | grlc: 52 | build: ./ 53 | restart: unless-stopped 54 | ports: 55 | - "8001:80" 56 | environment: 57 | - DEBUG=true 58 | - USERMAP_GID=1000 59 | - USERMAP_UID=1000 60 | - GRLC_GITHUB_ACCESS_TOKEN=xxx 61 | - GRLC_GITLAB_ACCESS_TOKEN=yyy 62 | - GRLC_SERVER_NAME=grlc.io 63 | ``` 64 | 65 | 5. `docker-compose up` 66 | 6. Your local grlc instance should be available at http://localhost:8001 and should respond to code modifications you make on `` 67 | 68 | You're good to pick any issue at the [issue tracker](https://github.com/CLARIAH/grlc/issues) marked as **enhancement** and start implementing it :) 69 | 70 | ## Governance model 71 | 72 | As creators of grlc, [@albertmeronyo](https://github.com/albertmeronyo) and [@c-martinez](http://github.com/c-martinez) are benevolent dictators for this project. This means that they have a final say of the direction of the project. This DOES NOT mean they are not willing to listen to suggestion (on the contrary, they *love* to hear new ideas)! 73 | 74 | ## Contributing 75 | 76 | All grlc contributors will be listed in the [CONTRIBUTORS.md](CONTRIBUTORS.md) file. Also, [notes of new releases](https://github.com/CLARIAH/grlc/releases) will mention who contributed to that specific release. 77 | 78 | ## Questions 79 | 80 | Please open an issue at the [issue tracker](https://github.com/CLARIAH/grlc/issues) and tag it as **question** 81 | -------------------------------------------------------------------------------- /CONTRIBUTORS.md: -------------------------------------------------------------------------------- 1 | 6 | 7 | # Contributors 8 | This is a list of all people who have contributed to grlc. Big thanks to everyone. 9 | 10 | [RinkeHoekstra](https://github.com/RinkeHoekstra) 11 | [pasqLisena](https://github.com/pasqLisena) 12 | [rlzijdeman](https://github.com/rlzijdeman) 13 | [RoderickvanderWeerdt](https://github.com/RoderickvanderWeerdt) 14 | [arnikz](https://github.com/arnikz) 15 | [jetschni](https://github.com/jetschni) 16 | [mwigham](https://github.com/mwigham) 17 | [steltenpower](https://github.com/steltenpower) 18 | [jspaaks](https://github.com/jspaaks) 19 | [ecow](https://github.com/ecow) 20 | [rapw3k](https://github.com/rapw3k) 21 | [jaw111](https://github.com/jaw111) 22 | [tkuhn](https://github.com/tkuhn) 23 | [GenEars](https://github.com/GenEars) 24 | [nichtich](https://github.com/nichtich) 25 | [jblom](https://github.com/jblom) 26 | [abelsiqueira](https://github.com/abelsiqueira) 27 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2022 Albert Meroño, Rinke Hoekstra, Carlos Martínez 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | FROM python:3.11-slim 6 | LABEL org.opencontainers.image.authors="albert.merono@vu.nl" 7 | 8 | # Default values for env variables 9 | ARG GRLC_GITHUB_ACCESS_TOKEN= 10 | ARG GRLC_GITLAB_ACCESS_TOKEN= 11 | ARG GRLC_SERVER_NAME=grlc.io 12 | ARG GRLC_SPARQL_ENDPOINT=http://dbpedia.org/sparql 13 | 14 | ENV GRLC_GITHUB_ACCESS_TOKEN=$GRLC_GITHUB_ACCESS_TOKEN \ 15 | GRLC_GITLAB_ACCESS_TOKEN=$GRLC_GITLAB_ACCESS_TOKEN \ 16 | GRLC_SERVER_NAME=$GRLC_SERVER_NAME \ 17 | GRLC_SPARQL_ENDPOINT=$GRLC_SPARQL_ENDPOINT 18 | 19 | ENV GRLC_USER="grlc" \ 20 | GRLC_HOME="/home/grlc" \ 21 | GRLC_LOG_DIR="/var/log/grlc" \ 22 | GITLAB_VERSION=8.10.4 \ 23 | GRLC_CACHE_DIR="/etc/docker-grlc" 24 | 25 | ENV GRLC_INSTALL_DIR="${GRLC_HOME}/grlc" \ 26 | GRLC_DATA_DIR="${GRLC_HOME}/data" \ 27 | GRLC_BUILD_DIR="${GRLC_CACHE_DIR}/build" \ 28 | GRLC_RUNTIME_DIR="${GRLC_CACHE_DIR}/runtime" 29 | 30 | RUN apt-get update \ 31 | && DEBIAN_FRONTEND=noninteractive apt-get install -y nginx git-core logrotate python3-pip locales gettext-base sudo build-essential apt-utils \ 32 | && update-locale LANG=C.UTF-8 LC_MESSAGES=POSIX \ 33 | && locale-gen en_US.UTF-8 \ 34 | && DEBIAN_FRONTEND=noninteractive dpkg-reconfigure locales \ 35 | && rm -rf /var/lib/apt/lists/* 36 | 37 | # RUN curl -sL https://deb.nodesource.com/setup_10.x | sudo -E bash - 38 | RUN apt-get update && apt-get install -y nodejs npm 39 | 40 | COPY ./ ${GRLC_INSTALL_DIR} 41 | 42 | COPY docker-assets/assets/build/ ${GRLC_BUILD_DIR}/ 43 | RUN bash ${GRLC_BUILD_DIR}/install.sh 44 | 45 | COPY docker-assets/assets/runtime/ ${GRLC_RUNTIME_DIR}/ 46 | COPY docker-assets/entrypoint.sh /sbin/entrypoint.sh 47 | 48 | 49 | RUN chmod 755 /sbin/entrypoint.sh 50 | 51 | EXPOSE 80/tcp 52 | 53 | VOLUME ["${GRLC_DATA_DIR}", "${GRLC_LOG_DIR}"] 54 | WORKDIR ${GRLC_INSTALL_DIR} 55 | ENTRYPOINT ["/sbin/entrypoint.sh"] 56 | CMD ["app:start"] 57 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright 2016-2021 Albert Meroño, Rinke Hoekstra, Carlos Martínez 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | -------------------------------------------------------------------------------- /LICENSES/MIT.txt: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 10 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2022 Albert Meroño, Rinke Hoekstra, Carlos Martínez 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | include requirements.txt 6 | include requirements-test.txt 7 | include CITATION.cff 8 | -------------------------------------------------------------------------------- /bin/grlc-server: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # SPDX-FileCopyrightText: 2022 Albert Meroño, Rinke Hoekstra, Carlos Martínez 4 | # 5 | # SPDX-License-Identifier: MIT 6 | 7 | """Grlc server. 8 | 9 | Usage: 10 | grlc-server [--port=PORT] 11 | 12 | Options: 13 | --port=PORT Port the server runs on [default: 8088]. 14 | """ 15 | from docopt import docopt 16 | from grlc.server import app as grlc_app 17 | from grlc import __version__ as grlc_version 18 | from grlc import static 19 | from sys import platform 20 | 21 | 22 | def runViaWaitress(port=8088): 23 | from waitress import serve 24 | serve(grlc_app, listen='*:%d'%port) 25 | 26 | def runViaGunicorn(port=8088): 27 | from gunicorn.app.base import BaseApplication 28 | 29 | class StandaloneApplication(BaseApplication): 30 | def __init__(self, app, options=None): 31 | self.options = options or {} 32 | self.application = app 33 | super(StandaloneApplication, self).__init__() 34 | 35 | def load_config(self): 36 | config = dict([(key, value) for key, value in self.options.items() 37 | if key in self.cfg.settings and value is not None]) 38 | for key, value in config.items(): 39 | self.cfg.set(key.lower(), value) 40 | 41 | def load(self): 42 | return self.application 43 | 44 | options = { 45 | 'bind': '%s:%d' % ('0.0.0.0', port), 46 | 'workers': 20, 47 | 'debug': static.LOG_DEBUG_MODE, 48 | 'timeout': 90 49 | } 50 | StandaloneApplication(grlc_app, options).run() 51 | 52 | if __name__ == '__main__': 53 | args = docopt(__doc__, version='Grlc %s server'%grlc_version) 54 | port = int(args['--port']) 55 | 56 | if platform=='win32': 57 | runViaWaitress(port) 58 | else: 59 | runViaGunicorn(port) 60 | -------------------------------------------------------------------------------- /codemeta.json: -------------------------------------------------------------------------------- 1 | { 2 | "@context": [ 3 | "https://doi.org/10.5063/schema/codemeta-2.0", 4 | "https://w3id.org/software-iodata", 5 | "https://raw.githubusercontent.com/jantman/repostatus.org/master/badges/latest/ontology.jsonld", 6 | "https://schema.org", 7 | "https://w3id.org/software-types" 8 | ], 9 | "@type": "SoftwareSourceCode", 10 | "author": [ 11 | { 12 | "@id": "https://orcid.org/0000-0003-4646-5842", 13 | "@type": "Person", 14 | "affiliation": { 15 | "@type": "Organization", 16 | "legalName": "King's College London" 17 | }, 18 | "familyName": "Meroño-Peñuela", 19 | "givenName": "Albert" 20 | }, 21 | { 22 | "@id": "https://orcid.org/0000-0001-5565-7577", 23 | "@type": "Person", 24 | "affiliation": { 25 | "@type": "Organization", 26 | "legalName": "Netherlands eScience Center" 27 | }, 28 | "familyName": "Martinez", 29 | "givenName": "Carlos" 30 | } 31 | ], 32 | "codeRepository": "https://github.com/CLARIAH/grlc.git", 33 | "contIntegration": "https://travis-ci.org/CLARIAH/grlc", 34 | "contributor": [ 35 | { 36 | "@type": "Person", 37 | "email": "albert.meronyo@gmail.com", 38 | "familyName": "Meroño-Peñuela", 39 | "givenName": "Albert" 40 | }, 41 | { 42 | "@type": "Person", 43 | "email": "c.martinez@esciencecenter.nl", 44 | "familyName": "Martinez", 45 | "givenName": "Carlos" 46 | }, 47 | { 48 | "@type": "Person", 49 | "email": "rinke.hoekstra@vu.nl", 50 | "familyName": "Hoekstra", 51 | "givenName": "Rinke" 52 | }, 53 | { 54 | "@type": "Person", 55 | "email": "pasquale.lisena@eurecom.fr", 56 | "familyName": "Lisena", 57 | "givenName": "Pasquale" 58 | }, 59 | { 60 | "@type": "Person", 61 | "email": "14040777+RoderickvanderWeerdt@users.noreply.github.com", 62 | "familyName": "", 63 | "givenName": "RoderickvanderWeerdt" 64 | }, 65 | { 66 | "@type": "Person", 67 | "email": "richard.zijdeman@iisg.nl", 68 | "familyName": "Zijdeman", 69 | "givenName": "Richard" 70 | }, 71 | { 72 | "@type": "Person", 73 | "email": "jblom@beeldengeluid.nl", 74 | "familyName": "Blom", 75 | "givenName": "Jaap" 76 | }, 77 | { 78 | "@type": "Person", 79 | "email": "a.kuzniar@esciencecenter.nl", 80 | "familyName": "Kuzniar", 81 | "givenName": "Arnold" 82 | }, 83 | { 84 | "@type": "Person", 85 | "email": "38520885+mwigham@users.noreply.github.com", 86 | "familyName": "", 87 | "givenName": "mwigham" 88 | }, 89 | { 90 | "@type": "Person", 91 | "email": "github.com@steltenpower.com", 92 | "familyName": "Steltenpool", 93 | "givenName": "Ruud" 94 | }, 95 | { 96 | "@type": "Person", 97 | "email": "j.spaaks@esciencecenter.nl", 98 | "familyName": "H. Spaaks", 99 | "givenName": "Jurriaan" 100 | }, 101 | { 102 | "@type": "Person", 103 | "email": "jetschni@users.noreply.github.com", 104 | "familyName": "", 105 | "givenName": "Jonas" 106 | }, 107 | { 108 | "@type": "Person", 109 | "email": "abel.s.siqueira@gmail.com", 110 | "familyName": "Siqueira", 111 | "givenName": "Abel" 112 | } 113 | ], 114 | "dateCreated": "2015-11-13T17:17:15Z+0100", 115 | "dateModified": "2022-03-21T22:20:00Z+0100", 116 | "datePublished": "2021-11-03", 117 | "description": "grlc, the git repository linked data API constructor, automatically builds Web APIs using SPARQL queries stored in git repositories.", 118 | "developmentStatus": "https://www.repostatus.org/#inactive", 119 | "identifier": "grlc", 120 | "issueTracker": "https://github.com/CLARIAH/grlc/issues", 121 | "keywords": [ 122 | "linked-data", 123 | "linked-data-api", 124 | "semantic-web", 125 | "sparql", 126 | "swagger-ui" 127 | ], 128 | "license": "http://spdx.org/licenses/MIT", 129 | "maintainer": { 130 | "@id": "https://orcid.org/0000-0003-4646-5842", 131 | "@type": "Person", 132 | "affiliation": { 133 | "@type": "Organization", 134 | "legalName": "King's College London" 135 | }, 136 | "familyName": "Meroño-Peñuela", 137 | "givenName": "Albert" 138 | }, 139 | "name": "grlc: the git repository linked data API constructor", 140 | "producer": { 141 | "@type": "Organization", 142 | "name": "CLARIAH", 143 | "url": "http://www.clariah.nl" 144 | }, 145 | "funding": { 146 | "@type": "Grant", 147 | "name": "CLARIAH-PLUS (NWO grant 184.034.023)", 148 | "funder": { 149 | "@type": "Organization", 150 | "name": "NWO", 151 | "url": "https://www.nwo.nl" 152 | } 153 | }, 154 | "softwareHelp": { 155 | "@id": "https://github.com/CLARIAH/grlc/wiki", 156 | "@type": "WebSite", 157 | "name": "grlc Wiki", 158 | "url": "https://github.com/CLARIAH/grlc/wiki" 159 | }, 160 | "programmingLanguage": "Python", 161 | "readme": "https://github.com/CLARIAH/grlc/blob//README.md", 162 | "runtimePlatform": "Python 3", 163 | "softwareRequirements": [ 164 | { 165 | "@type": "SoftwareApplication", 166 | "identifier": "Flask", 167 | "name": "Flask", 168 | "runtimePlatform": "Python 3", 169 | "version": "1.0.2" 170 | }, 171 | { 172 | "@type": "SoftwareApplication", 173 | "identifier": "Flask-Cors", 174 | "name": "Flask-Cors", 175 | "runtimePlatform": "Python 3", 176 | "version": "3.0.6" 177 | }, 178 | { 179 | "@type": "SoftwareApplication", 180 | "identifier": "MarkupSafe", 181 | "name": "MarkupSafe", 182 | "runtimePlatform": "Python 3", 183 | "version": "0.23" 184 | }, 185 | { 186 | "@type": "SoftwareApplication", 187 | "identifier": "PyGithub", 188 | "name": "PyGithub", 189 | "runtimePlatform": "Python 3", 190 | "version": "1.43.5" 191 | }, 192 | { 193 | "@type": "SoftwareApplication", 194 | "identifier": "PyYAML", 195 | "name": "PyYAML", 196 | "runtimePlatform": "Python 3", 197 | "version": "5.4" 198 | }, 199 | { 200 | "@type": "SoftwareApplication", 201 | "identifier": "SPARQLTransformer", 202 | "name": "SPARQLTransformer", 203 | "runtimePlatform": "Python 3", 204 | "version": "2.1.1" 205 | }, 206 | { 207 | "@type": "SoftwareApplication", 208 | "identifier": "SPARQLWrapper", 209 | "name": "SPARQLWrapper", 210 | "runtimePlatform": "Python 3", 211 | "version": "1.8.2" 212 | }, 213 | { 214 | "@type": "SoftwareApplication", 215 | "identifier": "docopt", 216 | "name": "docopt", 217 | "runtimePlatform": "Python 3", 218 | "version": "0.6.2" 219 | }, 220 | { 221 | "@type": "SoftwareApplication", 222 | "identifier": "docutils", 223 | "name": "docutils", 224 | "runtimePlatform": "Python 3", 225 | "version": "0.17.1" 226 | }, 227 | { 228 | "@type": "SoftwareApplication", 229 | "identifier": "gevent", 230 | "name": "gevent", 231 | "runtimePlatform": "Python 3", 232 | "version": "1.4.0" 233 | }, 234 | { 235 | "@type": "SoftwareApplication", 236 | "identifier": "greenlet", 237 | "name": "greenlet", 238 | "runtimePlatform": "Python 3", 239 | "version": "0.4.15" 240 | }, 241 | { 242 | "@type": "SoftwareApplication", 243 | "identifier": "gunicorn", 244 | "name": "gunicorn", 245 | "runtimePlatform": "Python 3", 246 | "version": "19.6.0" 247 | }, 248 | { 249 | "@type": "SoftwareApplication", 250 | "identifier": "html5lib", 251 | "name": "html5lib", 252 | "runtimePlatform": "Python 3", 253 | "version": "1.0.1" 254 | }, 255 | { 256 | "@type": "SoftwareApplication", 257 | "identifier": "isodate", 258 | "name": "isodate", 259 | "runtimePlatform": "Python 3", 260 | "version": "0.5.4" 261 | }, 262 | { 263 | "@type": "SoftwareApplication", 264 | "identifier": "keepalive", 265 | "name": "keepalive", 266 | "runtimePlatform": "Python 3", 267 | "version": "0.5" 268 | }, 269 | { 270 | "@type": "SoftwareApplication", 271 | "identifier": "pyaml", 272 | "name": "pyaml", 273 | "runtimePlatform": "Python 3", 274 | "version": "18.11.0" 275 | }, 276 | { 277 | "@type": "SoftwareApplication", 278 | "identifier": "pyparsing", 279 | "name": "pyparsing", 280 | "runtimePlatform": "Python 3", 281 | "version": "2.0.7" 282 | }, 283 | { 284 | "@type": "SoftwareApplication", 285 | "identifier": "rdflib", 286 | "name": "rdflib", 287 | "runtimePlatform": "Python 3", 288 | "version": "5.0.0" 289 | }, 290 | { 291 | "@type": "SoftwareApplication", 292 | "identifier": "rdflib-jsonld", 293 | "name": "rdflib-jsonld", 294 | "runtimePlatform": "Python 3", 295 | "version": "0.4.0" 296 | }, 297 | { 298 | "@type": "SoftwareApplication", 299 | "identifier": "requests", 300 | "name": "requests", 301 | "runtimePlatform": "Python 3", 302 | "version": "2.20.0" 303 | }, 304 | { 305 | "@type": "SoftwareApplication", 306 | "identifier": "setuptools", 307 | "name": "setuptools", 308 | "runtimePlatform": "Python 3", 309 | "version": ">= 38.6.0" 310 | }, 311 | { 312 | "@type": "SoftwareApplication", 313 | "identifier": "simplejson", 314 | "name": "simplejson", 315 | "runtimePlatform": "Python 3", 316 | "version": "3.16.0" 317 | }, 318 | { 319 | "@type": "SoftwareApplication", 320 | "identifier": "six", 321 | "name": "six", 322 | "runtimePlatform": "Python 3", 323 | "version": "1.12.0" 324 | }, 325 | { 326 | "@type": "SoftwareApplication", 327 | "identifier": "waitress", 328 | "name": "waitress", 329 | "runtimePlatform": "Python 3", 330 | "version": ">= 1.4.2" 331 | }, 332 | { 333 | "@type": "SoftwareApplication", 334 | "identifier": "werkzeug", 335 | "name": "werkzeug", 336 | "runtimePlatform": "Python 3", 337 | "version": ">= 0.16.0" 338 | } 339 | ], 340 | "targetProduct": { 341 | "@type": "stype:WebApplication", 342 | "name": "grlc: the git repository linked data API constructor" 343 | }, 344 | "url": "https://github.com/CLARIAH/grlc", 345 | "version": "1.3.7" 346 | } 347 | -------------------------------------------------------------------------------- /config.default.ini: -------------------------------------------------------------------------------- 1 | ; SPDX-FileCopyrightText: 2022 Albert Meroño, Rinke Hoekstra, Carlos Martínez 2 | ; 3 | ; SPDX-License-Identifier: MIT 4 | 5 | [auth] 6 | github_access_token = xxx 7 | gitlab_access_token = yyy 8 | 9 | [local] 10 | local_sparql_dir = /home/grlc/queries/ 11 | 12 | [defaults] 13 | # Default endpoint, if none specified elsewhere 14 | sparql_endpoint = http://dbpedia.org/sparql 15 | server_name = grlc.io 16 | 17 | # endpoint default authentication 18 | user = none 19 | password = none 20 | # sparql_access_token = SPARQL endpoint HTTP authorization token 21 | 22 | # Logging level 23 | debug = True 24 | 25 | [api_gitlab] 26 | gitlab_url=https://gitlab.com -------------------------------------------------------------------------------- /doc/img/api-docs.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CLARIAH/grlc/73247dd656d015ebbb567a450b422517f2ef5782/doc/img/api-docs.png -------------------------------------------------------------------------------- /doc/img/commit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CLARIAH/grlc/73247dd656d015ebbb567a450b422517f2ef5782/doc/img/commit.png -------------------------------------------------------------------------------- /doc/img/endpoint.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CLARIAH/grlc/73247dd656d015ebbb567a450b422517f2ef5782/doc/img/endpoint.png -------------------------------------------------------------------------------- /doc/img/file_name.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CLARIAH/grlc/73247dd656d015ebbb567a450b422517f2ef5782/doc/img/file_name.png -------------------------------------------------------------------------------- /doc/img/files.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CLARIAH/grlc/73247dd656d015ebbb567a450b422517f2ef5782/doc/img/files.png -------------------------------------------------------------------------------- /doc/img/github_dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CLARIAH/grlc/73247dd656d015ebbb567a450b422517f2ef5782/doc/img/github_dashboard.png -------------------------------------------------------------------------------- /doc/img/grlc_logo_01.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CLARIAH/grlc/73247dd656d015ebbb567a450b422517f2ef5782/doc/img/grlc_logo_01.png -------------------------------------------------------------------------------- /doc/img/method.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CLARIAH/grlc/73247dd656d015ebbb567a450b422517f2ef5782/doc/img/method.png -------------------------------------------------------------------------------- /doc/img/new_file.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CLARIAH/grlc/73247dd656d015ebbb567a450b422517f2ef5782/doc/img/new_file.png -------------------------------------------------------------------------------- /doc/img/new_repo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CLARIAH/grlc/73247dd656d015ebbb567a450b422517f2ef5782/doc/img/new_repo.png -------------------------------------------------------------------------------- /doc/img/new_repository.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CLARIAH/grlc/73247dd656d015ebbb567a450b422517f2ef5782/doc/img/new_repository.png -------------------------------------------------------------------------------- /doc/img/new_repository_form.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CLARIAH/grlc/73247dd656d015ebbb567a450b422517f2ef5782/doc/img/new_repository_form.png -------------------------------------------------------------------------------- /doc/img/parameter_api.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CLARIAH/grlc/73247dd656d015ebbb567a450b422517f2ef5782/doc/img/parameter_api.png -------------------------------------------------------------------------------- /doc/img/query.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CLARIAH/grlc/73247dd656d015ebbb567a450b422517f2ef5782/doc/img/query.png -------------------------------------------------------------------------------- /doc/img/results.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CLARIAH/grlc/73247dd656d015ebbb567a450b422517f2ef5782/doc/img/results.png -------------------------------------------------------------------------------- /doc/img/try-it-out.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CLARIAH/grlc/73247dd656d015ebbb567a450b422517f2ef5782/doc/img/try-it-out.png -------------------------------------------------------------------------------- /docker-assets/assets/build/.gitattributes: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2022 Albert Meroño, Rinke Hoekstra, Carlos Martínez 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | install.sh merge=ours 6 | -------------------------------------------------------------------------------- /docker-assets/assets/build/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: 2022 Albert Meroño, Rinke Hoekstra, Carlos Martínez 4 | # 5 | # SPDX-License-Identifier: MIT 6 | 7 | set -e 8 | 9 | 10 | ## Execute a command as GITLAB_USER 11 | exec_as_grlc() { 12 | if [[ $(whoami) == ${GRLC_USER} ]]; then 13 | $@ 14 | else 15 | sudo -HEu ${GRLC_USER} "$@" 16 | fi 17 | } 18 | 19 | #add grlc user 20 | adduser --disabled-login --gecos 'grlc' ${GRLC_USER} 21 | passwd -d ${GRLC_USER} 22 | 23 | 24 | cd ${GRLC_INSTALL_DIR} 25 | chown ${GRLC_USER}:${GRLC_USER} ${GRLC_HOME} -R 26 | 27 | pip install --upgrade pip 28 | pip install . 29 | 30 | npm install git2prov 31 | 32 | #move nginx logs to ${GITLAB_LOG_DIR}/nginx 33 | sed -i \ 34 | -e "s|access_log /var/log/nginx/access.log;|access_log ${GRLC_LOG_DIR}/nginx/access.log;|" \ 35 | -e "s|error_log /var/log/nginx/error.log;|error_log ${GRLC_LOG_DIR}/nginx/error.log;|" \ 36 | /etc/nginx/nginx.conf 37 | 38 | # configure gitlab log rotation 39 | cat > /etc/logrotate.d/grlc << EOF 40 | ${GRLC_LOG_DIR}/grlc/*.log { 41 | weekly 42 | missingok 43 | rotate 52 44 | compress 45 | delaycompress 46 | notifempty 47 | copytruncate 48 | } 49 | EOF 50 | 51 | # configure gitlab vhost log rotation 52 | cat > /etc/logrotate.d/grlc-nginx << EOF 53 | ${GRLC_LOG_DIR}/nginx/*.log { 54 | weekly 55 | missingok 56 | rotate 52 57 | compress 58 | delaycompress 59 | notifempty 60 | copytruncate 61 | } 62 | EOF 63 | -------------------------------------------------------------------------------- /docker-assets/assets/runtime/configs/nginx/grlc: -------------------------------------------------------------------------------- 1 | ## grlc 2 | ## 3 | ## Based on https://github.com/sameersbn/docker-gitlab 4 | ## Normal HTTP host 5 | ## Contains grlc additions for caching requests contents 6 | 7 | proxy_cache_path /tmp/grlc-cache levels=1:2 keys_zone=my_zone:10m inactive=60m; 8 | proxy_cache_key "$http_accept$scheme$request_method$host$request_uri"; 9 | 10 | server { 11 | ## Either remove "default_server" from the listen line below, 12 | ## or delete the /etc/nginx/sites-enabled/default file. This will cause gitlab 13 | ## to be served if you visit any address that your server responds to, eg. 14 | ## the ip address of the server (http://x.x.x.x/)n 0.0.0.0:80 default_server; 15 | listen 0.0.0.0:80 default_server; 16 | listen [::]:80 default_server; 17 | server_name {{GRLC_HOST}} *.{{GRLC_HOST}}; ## Replace this with something like gitlab.example.com 18 | server_tokens off; ## Don't show the nginx version number, a security best practice 19 | 20 | ## See app/controllers/application_controller.rb for headers set 21 | add_header X-Accel-Buffering {{NGINX_ACCEL_BUFFERING}}; 22 | add_header Strict-Transport-Security "max-age={{NGINX_HSTS_MAXAGE}};"; 23 | 24 | ## Individual nginx logs for this grlc vhost, with custom log format inc cache info 25 | access_log {{GRLC_LOG_DIR}}/nginx/grlc_access.log; 26 | error_log {{GRLC_LOG_DIR}}/nginx/grlc_error.log; 27 | 28 | ## Enabling cache for this server 29 | proxy_cache my_zone; 30 | add_header X-Proxy-Cache $upstream_cache_status; 31 | 32 | location / { 33 | proxy_pass http://127.0.0.1:8088/; 34 | proxy_set_header Host $host; 35 | proxy_pass_header Server; 36 | client_max_body_size 0; 37 | gzip off; 38 | 39 | ## https://github.com/gitlabhq/gitlabhq/issues/694 40 | ## Some requests take more than 30 seconds. 41 | proxy_read_timeout 600; 42 | proxy_connect_timeout 600; 43 | proxy_send_timeout 600; 44 | send_timeout 600; 45 | proxy_redirect off; 46 | proxy_buffering {{NGINX_PROXY_BUFFERING}}; 47 | 48 | proxy_http_version 1.1; 49 | 50 | proxy_set_header Host $http_host; 51 | proxy_set_header X-Real-IP $remote_addr; 52 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 53 | proxy_set_header X-Forwarded-Proto {{NGINX_X_FORWARDED_PROTO}}; 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /docker-assets/assets/runtime/env-defaults: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | GRLC_HOST=${GRLC_HOST:-localhost} 4 | GRLC_PORT=${GRLC_PORT:-80} 5 | 6 | ## NGINX 7 | NGINX_SERVER_NAMES_HASH_BUCKET_SIZE=${NGINX_SERVER_NAMES_HASH_BUCKET_SIZE:-32}; 8 | NGINX_WORKERS=${NGINX_WORKERS:-1} 9 | NGINX_ACCEL_BUFFERING=${NGINX_ACCEL_BUFFERING:-no} 10 | NGINX_PROXY_BUFFERING=${NGINX_PROXY_BUFFERING:-off} 11 | case ${GRLC_HTTPS} in 12 | true) NGINX_X_FORWARDED_PROTO=${NGINX_X_FORWARDED_PROTO:-https} ;; 13 | *) NGINX_X_FORWARDED_PROTO=${NGINX_X_FORWARDED_PROTO:-\$scheme} ;; 14 | esac 15 | -------------------------------------------------------------------------------- /docker-assets/assets/runtime/functions: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | source ${GRLC_RUNTIME_DIR}/env-defaults 4 | 5 | SYSCONF_TEMPLATES_DIR="${GRLC_RUNTIME_DIR}/configs" 6 | 7 | GITLAB_NGINX_CONFIG="/etc/nginx/sites-enabled/gitlab" 8 | 9 | ## Copies configuration template to the destination as the specified USER 10 | ### Looks up for overrides in ${USERCONF_TEMPLATES_DIR} before using the defaults from ${SYSCONF_TEMPLATES_DIR} 11 | # $1: copy-as user 12 | # $2: source file 13 | # $3: destination location 14 | # $4: mode of destination 15 | install_template() { 16 | local OWNERSHIP=${1} 17 | local SRC=${2} 18 | local DEST=${3} 19 | local MODE=${4:-0644} 20 | if [[ -f ${SYSCONF_TEMPLATES_DIR}/${SRC} ]]; then 21 | cp ${SYSCONF_TEMPLATES_DIR}/${SRC} ${DEST} 22 | elif [[ -f ${SYSCONF_TEMPLATES_DIR}/${SRC} ]]; then 23 | cp ${SYSCONF_TEMPLATES_DIR}/${SRC} ${DEST} 24 | fi 25 | chmod ${MODE} ${DEST} 26 | chown ${OWNERSHIP} ${DEST} 27 | } 28 | 29 | ## Replace placeholders with values 30 | # $1: file with placeholders to replace 31 | # $x: placeholders to replace 32 | update_template() { 33 | local FILE=${1?missing argument} 34 | shift 35 | 36 | [[ ! -f ${FILE} ]] && return 1 37 | 38 | local VARIABLES=($@) 39 | local USR=$(stat -c %U ${FILE}) 40 | local tmp_file=$(mktemp) 41 | cp -a "${FILE}" ${tmp_file} 42 | 43 | local variable 44 | for variable in ${VARIABLES[@]}; do 45 | # Keep the compatibilty: {{VAR}} => ${VAR} 46 | sed -ri "s/[{]{2}$variable[}]{2}/\${$variable}/g" ${tmp_file} 47 | done 48 | 49 | # Replace placeholders 50 | ( 51 | export ${VARIABLES[@]} 52 | local IFS=":"; sudo -HEu ${USR} envsubst "${VARIABLES[*]/#/$}" < ${tmp_file} > ${FILE} 53 | ) 54 | rm -f ${tmp_file} 55 | } 56 | 57 | setup_nginx() { 58 | install_template root: nginx/grlc /etc/nginx/sites-enabled/grlc 59 | update_template /etc/nginx/sites-enabled/grlc \ 60 | GRLC_HOST \ 61 | NGINX_ACCEL_BUFFERING \ 62 | NGINX_HSTS_MAXAGE \ 63 | GRLC_LOG_DIR \ 64 | NGINX_PROXY_BUFFERING \ 65 | NGINX_X_FORWARDED_PROTO 66 | 67 | rm -f /etc/nginx/sites-enabled/default 68 | mkdir -p ${GRLC_LOG_DIR}/nginx 69 | nginx -t 70 | service nginx restart 71 | } 72 | 73 | map_uidgid() { 74 | echo "Mapping UID/GID" 75 | ## Adapt uid and gid for ${REDMINE_USER}:${REDMINE_USER} 76 | USERMAP_ORIG_UID=$(id -u ${GRLC_USER}) 77 | USERMAP_ORIG_GID=$(id -g ${GRLC_USER}) 78 | USERMAP_GID=${USERMAP_GID:-${USERMAP_UID:-$USERMAP_ORIG_GID}} 79 | USERMAP_UID=${USERMAP_UID:-$USERMAP_ORIG_UID} 80 | if [[ ${USERMAP_UID} != ${USERMAP_ORIG_UID} ]] || [[ ${USERMAP_GID} != ${USERMAP_ORIG_GID} ]]; then 81 | echo "Adapting uid and gid for ${GRLC_USER}:${GRLC_USER} to $USERMAP_UID:$USERMAP_GID" 82 | groupmod -g ${USERMAP_GID} ${GRLC_USER} 83 | sed -i -e "s/:${USERMAP_ORIG_UID}:${USERMAP_GID}:/:${USERMAP_UID}:${USERMAP_GID}:/" /etc/passwd 84 | # find ${TRIPLY_HOME} -path ${TRIPLY_DATA_DIR}/\* -prune -o -print0 | xargs -0 chown -h ${TRIPLY_USER}:${TRIPLY_USER} 85 | fi 86 | 87 | # take ownership of directories 88 | chown -R ${GRLC_USER}:${GRLC_USER} ${GRLC_HOME} 89 | chown -R ${GRLC_USER}:${GRLC_USER} ${GRLC_LOG_DIR} 90 | } 91 | -------------------------------------------------------------------------------- /docker-assets/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SPDX-FileCopyrightText: 2022 Albert Meroño, Rinke Hoekstra, Carlos Martínez 4 | # 5 | # SPDX-License-Identifier: MIT 6 | 7 | set -e 8 | source ${GRLC_RUNTIME_DIR}/functions 9 | 10 | [[ $DEBUG == true ]] && set -x 11 | 12 | map_uidgid 13 | 14 | case ${1} in 15 | app:start) 16 | setup_nginx 17 | # initialize_system 18 | # configure_gitlab 19 | # configure_gitlab_shell 20 | # configure_nginx 21 | 22 | case ${1} in 23 | app:start) 24 | cd ${GRLC_INSTALL_DIR} 25 | # put github and gitlab access_tokens in place 26 | cp config.default.ini config.ini 27 | sed -i "s/xxx/${GRLC_GITHUB_ACCESS_TOKEN}/" config.ini 28 | sed -i "s/yyy/${GRLC_GITLAB_ACCESS_TOKEN}/" config.ini 29 | # configure grlc server name 30 | sed -i "s/grlc.io/${GRLC_SERVER_NAME}/" config.ini 31 | # configure default sparql endpoint 32 | sed -i "s|http://dbpedia.org/sparql|${GRLC_SPARQL_ENDPOINT}|" config.ini 33 | # enable/disable debugging 34 | if [ $DEBUG ]; then 35 | sed -i "s/debug = False/debug = True/" config.ini 36 | fi 37 | 38 | grlc-server --port=8088 39 | # migrate_database 40 | # rm -rf /var/run/supervisor.sock 41 | # exec /usr/bin/supervisord -nc /etc/supervisor/supervisord.conf 42 | ;; 43 | # app:init) 44 | # migrate_database 45 | # ;; 46 | # app:sanitize) 47 | # sanitize_datadir 48 | # ;; 49 | # app:rake) 50 | # shift 1 51 | # execute_raketask $@ 52 | # ;; 53 | esac 54 | ;; 55 | app:help) 56 | echo "Available options:" 57 | echo " app:start - Starts the grlc server (default)" 58 | # echo " app:init - Initialize the gitlab server (e.g. create databases, compile assets), but don't start it." 59 | # echo " app:sanitize - Fix repository/builds directory permissions." 60 | # echo " app:rake - Execute a rake task." 61 | echo " app:help - Displays the help" 62 | echo " [command] - Execute the specified command, eg. bash." 63 | ;; 64 | *) 65 | exec "$@" 66 | ;; 67 | esac 68 | -------------------------------------------------------------------------------- /paper.bib: -------------------------------------------------------------------------------- 1 | @Comment{ 2 | SPDX-FileCopyrightText: 2022 Albert Meroño, Rinke Hoekstra, Carlos Martínez 3 | 4 | SPDX-License-Identifier: MIT 5 | } 6 | 7 | @article{Espinoza2021, 8 | title = {Crossing the chasm between ontology engineering and application development: A survey}, 9 | journal = {Journal of Web Semantics}, 10 | volume = {70}, 11 | pages = {100655}, 12 | year = {2021}, 13 | issn = {1570-8268}, 14 | doi = {10.1016/j.websem.2021.100655}, 15 | url = {https://www.sciencedirect.com/science/article/pii/S1570826821000305}, 16 | author = {Paola Espinoza-Arias and Daniel Garijo and Oscar Corcho}, 17 | keywords = {Ontology, OWL, Ontology engineering, Web API, Application development, Knowledge graph}, 18 | abstract = {The adoption of Knowledge Graphs (KGs) by public and private organizations to integrate and publish data has increased in recent years. Ontologies play a crucial role in providing the structure for KGs, but are usually disregarded when designing Application Programming Interfaces (APIs) to enable browsing KGs in a developer-friendly manner. In this paper we provide a systematic review of the state of the art on existing approaches to ease access to ontology-based KG data} 19 | } 20 | 21 | @InProceedings{Merono_ISWC2019, 22 | author="Lisena, Pasquale 23 | and Mero{\~{n}}o-Pe{\~{n}}uela, Albert 24 | and Kuhn, Tobias 25 | and Troncy, Rapha{\"e}l", 26 | editor="Ghidini, Chiara 27 | and Hartig, Olaf 28 | and Maleshkova, Maria 29 | and Sv{\'a}tek, Vojt{\v{e}}ch 30 | and Cruz, Isabel 31 | and Hogan, Aidan 32 | and Song, Jie 33 | and Lefran{\c{c}}ois, Maxime 34 | and Gandon, Fabien", 35 | title="Easy Web API Development with SPARQL Transformer", 36 | booktitle="The Semantic Web -- ISWC 2019", 37 | year="2019", 38 | publisher="Springer International Publishing", 39 | address="Cham", 40 | pages="454--470", 41 | abstract="In a document-based world as the one of Web APIs, the triple-based output of SPARQL endpoints can be a barrier for developers who want to integrate Linked Data in their applications. A different JSON output can be obtained with SPARQL Transformer, which relies on a single JSON object for defining which data should be extracted from the endpoint and which shape should they assume. We propose a new approach that amounts to merge SPARQL bindings on the base of identifiers and the integration in the grlc API framework to create new bridges between the Web of Data and the Web of applications.", 42 | isbn="978-3-030-30796-7", 43 | doi="10.1007/978-3-030-30796-7_28" 44 | } 45 | 46 | @phdthesis{Singh2019, 47 | doi = {10.18174/505685}, 48 | url = {https://doi.org/10.18174/505685}, 49 | publisher = {Wageningen University and Research}, 50 | author = {Gurnoor Singh}, 51 | title = {Genomics data integration for knowledge discovery using genome annotations from molecular databases and scientific literature}, 52 | institution = "Wageningen University", 53 | year = "2019" 54 | } 55 | 56 | @InProceedings{Merono_ISWC2016, 57 | author="Mero{\~{n}}o-Pe{\~{n}}uela, Albert 58 | and Hoekstra, Rinke", 59 | editor="Sack, Harald 60 | and Rizzo, Giuseppe 61 | and Steinmetz, Nadine 62 | and Mladeni{\'{c}}, Dunja 63 | and Auer, S{\"o}ren 64 | and Lange, Christoph", 65 | title="grlc Makes GitHub Taste Like Linked Data APIs", 66 | booktitle="The Semantic Web", 67 | year="2016", 68 | publisher="Springer International Publishing", 69 | address="Cham", 70 | pages="342--353", 71 | abstract="Building Web APIs on top of SPARQL endpoints is becoming common practice. It enables universal access to the integration favorable data space of Linked Data. In the majority of use cases, users cannot be expected to learn SPARQL to query this data space. Web APIs are the most common way to enable programmatic access to data on the Web. However, the implementation of Web APIs around Linked Data is often a tedious and repetitive process. Recent work speeds up this Linked Data API construction by wrapping it around SPARQL queries, which carry out the API functionality under the hood. Inspired by this development, in this paper we present grlc, a lightweight server that takes SPARQL queries curated in GitHub repositories, and translates them to Linked Data APIs on the fly.", 72 | isbn="978-3-319-47602-5", 73 | doi="10.1007/978-3-319-47602-5_48" 74 | } 75 | 76 | @InProceedings{Merono_ESWC2017, 77 | author="Mero{\~{n}}o-Pe{\~{n}}uela, Albert 78 | and Hoekstra, Rinke", 79 | editor="Blomqvist, Eva 80 | and Hose, Katja 81 | and Paulheim, Heiko 82 | and {\L}awrynowicz, Agnieszka 83 | and Ciravegna, Fabio 84 | and Hartig, Olaf", 85 | title="SPARQL2Git: Transparent SPARQL and Linked Data API Curation via Git", 86 | booktitle="The Semantic Web: ESWC 2017 Satellite Events", 87 | year="2017", 88 | publisher="Springer International Publishing", 89 | address="Cham", 90 | pages="143--148", 91 | abstract="In this demo, we show how an effective and application agnostic way of curating SPARQL queries can be achieved by leveraging Git-based architectures. Often, SPARQL queries are hard-coded into Linked Data consuming applications. This tight coupling poses issues in code maintainability, since these queries are prone to change to adapt to new situations; and query reuse, since queries that might be useful in other applications remain inaccessible. In order to enable decoupling, version control, availability and accessibility of SPARQL queries, we propose SPARQL2Git, an interface for editing, curating and storing SPARQL queries that uses cloud based Git repositories (such as GitHub) as a backend. We describe the query management capabilities of SPARQL2Git, its convenience for SPARQL users that lack Git knowledge, and its combination with grlc to easily generate Linked Data APIs.", 92 | isbn="978-3-319-70407-4", 93 | doi="10.1007/978-3-319-70407-4_27" 94 | } 95 | 96 | @InProceedings{Merono_ISWC2017, 97 | author="Mero{\~{n}}o-Pe{\~{n}}uela, Albert 98 | and Hoekstra, Rinke", 99 | editor="d'Amato, Claudia 100 | and Fernandez, Miriam 101 | and Tamma, Valentina 102 | and Lecue, Freddy 103 | and Cudr{\'e}-Mauroux, Philippe 104 | and Sequeda, Juan 105 | and Lange, Christoph 106 | and Heflin, Jeff", 107 | title="Automatic Query-Centric API for Routine Access to Linked Data", 108 | booktitle="The Semantic Web -- ISWC 2017", 109 | year="2017", 110 | publisher="Springer International Publishing", 111 | address="Cham", 112 | pages="334--349", 113 | abstract="Despite the advatages of Linked Data as a data integration paradigm, accessing and consuming Linked Data is still a cumbersome task. Linked Data applications need to use technologies such as RDF and SPARQL that, despite their expressive power, belong to the data integration stack. As a result, applications and data cannot be cleanly separated: SPARQL queries, endpoint addresses, namespaces, and URIs end up as part of the application code. Many publishers address these problems by building RESTful APIs around their Linked Data. However, this solution has two pitfalls: these APIs are costly to maintain; and they blackbox functionality by hiding the queries they use. In this paper we describe grlc, a gateway between Linked Data applications and the LOD cloud that offers a RESTful, reusable and uniform means to routinely access any Linked Data. It generates an OpenAPI compatible API by using parametrized queries shared on the Web. The resulting APIs require no coding, rely on low-cost external query storage and versioning services, contain abundant provenance information, and integrate access to different publishing paradigms into a single API. We evaluate grlc qualitatively, by describing its reported value by current users; and quantitatively, by measuring the added overhead at generating API specifications and answering to calls.", 114 | isbn="978-3-319-68204-4", 115 | doi="10.1007/978-3-319-68204-4_30" 116 | } 117 | 118 | @article{Verborgh2016, 119 | doi = {10.1016/j.websem.2016.03.003}, 120 | url = {https://doi.org/10.1016/j.websem.2016.03.003}, 121 | year = {2016}, 122 | month = mar, 123 | publisher = {Elsevier {BV}}, 124 | volume = {37-38}, 125 | pages = {184--206}, 126 | author = {Ruben Verborgh and Miel Vander Sande and Olaf Hartig and Joachim Van Herwegen and Laurens De Vocht and Ben De Meester and Gerald Haesendonck and Pieter Colpaert}, 127 | title = {Triple Pattern Fragments: A low-cost knowledge graph interface for the Web}, 128 | journal = {Journal of Web Semantics} 129 | } 130 | 131 | @article{Daquino2021, 132 | author = {Marilena Daquino and 133 | Ivan Heibi and 134 | Silvio Peroni and 135 | David M. Shotton}, 136 | title = {Creating Restful APIs over {SPARQL} endpoints with {RAMOSE}}, 137 | journal = {CoRR}, 138 | volume = {abs/2007.16079}, 139 | year = {2020}, 140 | url = {https://arxiv.org/abs/2007.16079}, 141 | archivePrefix = {arXiv}, 142 | eprint = {2007.16079}, 143 | timestamp = {Mon, 03 Aug 2020 14:32:13 +0200}, 144 | biburl = {https://dblp.org/rec/journals/corr/abs-2007-16079.bib}, 145 | bibsource = {dblp computer science bibliography, https://dblp.org} 146 | } 147 | -------------------------------------------------------------------------------- /paper.md: -------------------------------------------------------------------------------- 1 | 6 | 7 | --- 8 | title: 'grlc: the git repository linked data API constructor.' 9 | tags: 10 | - Python 11 | - linked data 12 | - API builder 13 | authors: 14 | - name: Albert Meroño-Peñuela 15 | orcid: 0000-0003-4646-5842 16 | affiliation: 1 17 | - name: Carlos Martinez-Ortiz 18 | orcid: 0000-0001-5565-7577 19 | affiliation: 2 20 | affiliations: 21 | - name: King's College London 22 | index: 1 23 | - name: Netherlands eScience Center 24 | index: 2 25 | date: XX April 2020 26 | bibliography: paper.bib 27 | --- 28 | 29 | # Summary 30 | 31 | RDF is a knowledge representation format (and an enabling technology for Linked Open Data) which has gained popularity over the years and it continues to be adopted in different domains such as life sciences and humanities. RDF data is typically represented as sets of triples, consisting of subject, predicate and object, and is usually stored in a triple store. SPARQL is one of the most commonly used query languages used to retrieve linked data from a triple store. However writing SPARQL queries is not a trivial task and requires some degree of expertise. 32 | 33 | Domain experts usually face the challenge of having to learn SPARQL, when all they want is to be able to access the information contained in the triple store. Knowledge engineers with the necessary expertise can help domain experts write SPARQL queries, but they still need to modify part of the query every time they want to extract new data. 34 | 35 | `grlc` is a lightweight server that takes SPARQL queries (stored in a GitHub repository, local file storage or listed in a URL), and translates them to Linked Data Web APIs. This enables universal access to Linked Data. Users are not required to know SPARQL to query their data, but instead can access a web API. In this way, `grlc` enables researchers to easily access knowledge represented in RDF format. 36 | 37 | `grlc` uses the [BASIL convention](https://github.com/basilapi/basil/wiki/SPARQL-variable-name-convention-for-WEB-API-parameters-mapping) for SPARQL variable mapping and supports LD fragments [@Verborgh2016]. 38 | 39 | `grlc` has been used in a number of scientific publications [@Merono_ISWC2016,@Merono_ISWC2017,@Merono_ESWC2017,@Merono_ISWC2019] and PhD thesis [@Singh2019]. 40 | 41 | Other comparable approaches exist, which allow users to access Linked Open Data without requiring to know SPARQL; for example [OpenPHACTS](https://github.com/openphacts) and RAMOSE [@Daquino2021] are two of the most notable ones. For an extensive review of other related work, a recent survey on API approaches for knowledge graphs [@Espinoza2021]. 42 | 43 | 44 | # Acknowledgements 45 | 46 | We acknowledge contributions from Pasquale Lisena, Richard Zijdeman and Rinke Hoekstra. 47 | 48 | # References 49 | -------------------------------------------------------------------------------- /requirements-test.txt: -------------------------------------------------------------------------------- 1 | mock==5.2.0 2 | pytest==8.3.5 3 | flake8==7.2.0 4 | six==1.17.0 5 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | docopt-ng==0.9.0 2 | Flask==3.1.0 3 | Flask-Cors==5.0.1 4 | pyaml==25.1.0 5 | python-gitlab==5.6.0 6 | rdflib==7.1.3 7 | requests==2.32.3 8 | SPARQLTransformer==2.4.0 9 | PyGithub==2.6.1 10 | gunicorn==23.0.0; sys_platform!="win32" 11 | waitress>=1.4.2; sys_platform=="win32" 12 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2022 Albert Meroño, Rinke Hoekstra, Carlos Martínez 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | [metadata] 6 | description-file = README.md 7 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # SPDX-FileCopyrightText: 2022 Albert Meroño, Rinke Hoekstra, Carlos Martínez 4 | # 5 | # SPDX-License-Identifier: MIT 6 | 7 | # -*- coding: iso-8859-15 -*- 8 | 9 | import codecs 10 | import os 11 | from setuptools import setup 12 | 13 | grlc_base = "src" 14 | grlc_base_dir = os.path.join(grlc_base, "") 15 | grlc_data = [] 16 | for root, dirs, files in os.walk(grlc_base): 17 | if root != grlc_base: 18 | root_dir = root.replace(grlc_base_dir, "") 19 | data_files = os.path.join(root_dir, "*") 20 | grlc_data.append(data_files) 21 | 22 | # To update the package version number, edit CITATION.cff 23 | with open("CITATION.cff", "r") as cff: 24 | for line in cff: 25 | if "version:" in line: 26 | version = line.replace("version:", "").strip().strip('"') 27 | 28 | with codecs.open("requirements.txt", mode="r") as f: 29 | install_requires = f.read().splitlines() 30 | 31 | with codecs.open("requirements-test.txt", mode="r") as f: 32 | tests_require = f.read().splitlines() 33 | 34 | with codecs.open("README.md", mode="r", encoding="utf-8") as f: 35 | long_description = f.read() 36 | 37 | setup( 38 | name="grlc", 39 | description="grlc, the git repository linked data API constructor", 40 | long_description=long_description, 41 | long_description_content_type="text/markdown", 42 | license="Copyright 2017 Albert Meroño", 43 | author="Albert Meroño", 44 | author_email="albert.merono@vu.nl", 45 | url="https://github.com/CLARIAH/grlc", 46 | version=version, 47 | py_modules=["grlc"], 48 | packages=["grlc"], 49 | package_dir={"grlc": grlc_base}, 50 | scripts=["bin/grlc-server"], 51 | install_requires=install_requires, 52 | setup_requires=[ 53 | # dependency for `python setup.py test` 54 | "pytest-runner", 55 | # dependencies for `python setup.py build_sphinx` 56 | # 'sphinx', 57 | # 'recommonmark' 58 | ], 59 | tests_require=tests_require, 60 | package_data={"grlc": grlc_data}, 61 | include_package_data=True, 62 | data_files=[("citation/grlc", ["CITATION.cff"])], 63 | ) 64 | -------------------------------------------------------------------------------- /src/__init__.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2022 Albert Meroño, Rinke Hoekstra, Carlos Martínez 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | from .__version__ import __version__ 6 | -------------------------------------------------------------------------------- /src/__version__.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2022 Albert Meroño, Rinke Hoekstra, Carlos Martínez 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | import os 6 | import sys 7 | import yaml 8 | 9 | # To update the package version number, edit CITATION.cff 10 | citationfile = os.path.join(sys.exec_prefix, "citation/grlc", "CITATION.cff") 11 | with open(citationfile, "r") as f: 12 | data = yaml.safe_load(f) 13 | __version__ = data["version"] 14 | -------------------------------------------------------------------------------- /src/glogging.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2022 Albert Meroño, Rinke Hoekstra, Carlos Martínez 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | import logging 6 | 7 | import grlc.static as static 8 | 9 | 10 | def getGrlcLogger(name): 11 | """Construct a logger for grlc with the logging level specified on `config.ini`.""" 12 | glogger = logging.getLogger(name) 13 | if static.LOG_DEBUG_MODE: 14 | glogger.setLevel(logging.DEBUG) 15 | return glogger 16 | -------------------------------------------------------------------------------- /src/gquery.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # SPDX-FileCopyrightText: 2022 Albert Meroño, Rinke Hoekstra, Carlos Martínez 4 | # 5 | # SPDX-License-Identifier: MIT 6 | 7 | # gquery.py: functions that deal with / transform SPARQL queries in grlc 8 | 9 | import yaml 10 | import json 11 | from rdflib.plugins.sparql.parser import Query, UpdateUnit 12 | from rdflib.plugins.sparql.processor import translateQuery 13 | from flask import request, has_request_context 14 | from pyparsing import ParseException 15 | from pprint import pformat 16 | import traceback 17 | import re 18 | import requests 19 | import SPARQLTransformer 20 | 21 | # grlc modules 22 | import grlc.static as static 23 | import grlc.glogging as glogging 24 | 25 | 26 | glogger = glogging.getGrlcLogger(__name__) 27 | 28 | XSD_PREFIX = "PREFIX xsd: " 29 | 30 | 31 | def guess_endpoint_uri(rq, loader): 32 | """ 33 | Guesses the endpoint URI from (in this order): 34 | - An endpoint parameter in URL 35 | - An #+endpoint decorator 36 | - A endpoint.txt file in the repo 37 | Otherwise assigns a default one 38 | """ 39 | auth = (static.DEFAULT_ENDPOINT_USER, static.DEFAULT_ENDPOINT_PASSWORD) 40 | if auth == ("none", "none"): 41 | auth = None 42 | 43 | if has_request_context() and "endpoint" in request.args: 44 | endpoint = request.args["endpoint"] 45 | glogger.debug("Endpoint provided in request: " + endpoint) 46 | return endpoint, auth 47 | 48 | # Decorator 49 | try: 50 | decorators = get_yaml_decorators(rq) 51 | endpoint = decorators["endpoint"] 52 | auth = None 53 | glogger.debug("Decorator guessed endpoint: " + endpoint) 54 | except (TypeError, KeyError): 55 | # File 56 | try: 57 | endpoint_content = loader.getEndpointText() 58 | endpoint = endpoint_content.strip().splitlines()[0] 59 | auth = None 60 | glogger.debug("File guessed endpoint: " + endpoint) 61 | # TODO: except all is really ugly 62 | except Exception: 63 | # Default 64 | endpoint = static.DEFAULT_ENDPOINT 65 | auth = (static.DEFAULT_ENDPOINT_USER, static.DEFAULT_ENDPOINT_PASSWORD) 66 | if auth == ("none", "none"): 67 | auth = None 68 | glogger.info("No endpoint specified, using default ({})".format(endpoint)) 69 | 70 | return endpoint, auth 71 | 72 | 73 | def count_query_results(query, endpoint): 74 | """ 75 | Returns the total number of results that query 'query' will generate 76 | WARNING: This is too expensive just for providing a number of result pages 77 | Providing a dummy count for now 78 | """ 79 | 80 | # number_results_query, repl = re.subn("SELECT.*FROM", "SELECT COUNT (*) FROM", query) 81 | # if not repl: 82 | # number_results_query = re.sub("SELECT.*{", "SELECT COUNT(*) {", query) 83 | # number_results_query = re.sub("GROUP\s+BY\s+[\?\_\(\)a-zA-Z0-9]+", "", number_results_query) 84 | # number_results_query = re.sub("ORDER\s+BY\s+[\?\_\(\)a-zA-Z0-9]+", "", number_results_query) 85 | # number_results_query = re.sub("LIMIT\s+[0-9]+", "", number_results_query) 86 | # number_results_query = re.sub("OFFSET\s+[0-9]+", "", number_results_query) 87 | # 88 | # glogger.debug("Query for result count: " + number_results_query) 89 | # 90 | # # Preapre HTTP request 91 | # headers = { 'Accept' : 'application/json' } 92 | # data = { 'query' : number_results_query } 93 | # count_json = requests.get(endpoint, params=data, headers=headers).json() 94 | # count = int(count_json['results']['bindings'][0]['callret-0']['value']) 95 | # glogger.info("Paginated query has {} results in total".format(count)) 96 | # 97 | # return count 98 | 99 | return 1000 100 | 101 | 102 | def _getDictWithKey(key, dict_list): 103 | """Returns the first dictionary in dict_list which contains the given key""" 104 | for d in dict_list: 105 | if key in d: 106 | return d 107 | return None 108 | 109 | 110 | def get_parameters(query, endpoint, query_metadata, auth=None): 111 | """ 112 | ?_name The variable specifies the API mandatory parameter name. The value is incorporated in the query as plain literal. 113 | ?__name The parameter name is optional. 114 | ?_name_iri The variable is substituted with the parameter value as a IRI (also: number or literal). 115 | ?_name_en The parameter value is considered as literal with the language 'en' (e.g., en,it,es, etc.). 116 | ?_name_integer The parameter value is considered as literal and the XSD datatype 'integer' is added during substitution. 117 | ?_name_prefix_datatype The parameter value is considered as literal and the datatype 'prefix:datatype' is added during 118 | substitution. The prefix must be specified according to the SPARQL syntax. 119 | """ 120 | # Basil-style variables 121 | re1 = r"\?" # Start with a '?' 122 | re2 = "(?P[_]{1,2})" # ...followed by one (for required vars) or two (for optional vars) '_' 123 | re3 = "(?P[a-zA-Z0-9]+)" # ...then the name of the var 124 | re4 = "([_](?P(iri)|(number)|(literal)|(integer)))?" # ...optionally with a type (iri, number, literal, integer) 125 | re5 = "([_](?P[a-zA-Z0-9]+)[_](?P[a-zA-Z0-9]+))?" # ... OR a user defined type, with a prefix 126 | re6 = "([_](?P[a-zA-Z0-9]+))?" # ...OR a language 127 | variable_matcher = re.compile(re1 + re2 + re3 + re4 + re5 + re6) 128 | 129 | parameters = {} 130 | for match in variable_matcher.finditer(query): 131 | p = {} 132 | vname = match.group("name") 133 | 134 | p["original"] = match.group(0) 135 | p["required"] = len(match.group("required")) == 1 136 | p["name"] = vname 137 | 138 | mtype = match.group("type") 139 | if mtype in ["number", "literal", "string"]: 140 | p["type"] = mtype 141 | elif mtype in ["iri"]: 142 | p["type"] = "string" 143 | p["format"] = "iri" 144 | else: 145 | p["type"] = "string" 146 | if mtype in static.XSD_DATATYPES: 147 | p["datatype"] = "xsd:{}".format(mtype) 148 | elif match.group("prefix") and match.group("userdefined"): 149 | p["datatype"] = "{}:{}".format( 150 | match.group("prefix"), match.group("userdefined") 151 | ) 152 | 153 | vcodes = get_enumeration(query, vname, endpoint, query_metadata, auth) 154 | if vcodes is not None: 155 | p["enum"] = sorted(vcodes) 156 | vdefault = get_defaults(query, vname, query_metadata) 157 | if vdefault is not None: 158 | p["default"] = vdefault 159 | 160 | if match.group("lang") is not None: 161 | p["lang"] = match.group("lang") 162 | 163 | parameters[vname] = p 164 | 165 | glogger.debug("Finished parsing the following parameters: {}".format(parameters)) 166 | return parameters 167 | 168 | 169 | def get_defaults(rq, v, metadata): 170 | """ 171 | Returns the default value for a parameter or None 172 | """ 173 | glogger.debug("Metadata with defaults: {}".format(metadata)) 174 | if "defaults" not in metadata: 175 | return None 176 | defaultsDict = _getDictWithKey(v, metadata["defaults"]) 177 | if defaultsDict: 178 | return defaultsDict[v] 179 | return None 180 | 181 | 182 | def get_enumeration(rq, v, endpoint, metadata={}, auth=None): 183 | """ 184 | Returns a list of enumerated values for variable 'v' in query 'rq' 185 | """ 186 | # glogger.debug("Metadata before processing enums: {}".format(metadata)) 187 | # We only fire the enum filling queries if indicated by the query metadata 188 | if "enumerate" not in metadata: 189 | return None 190 | enumDict = _getDictWithKey(v, metadata["enumerate"]) 191 | if enumDict: 192 | return enumDict[v] 193 | if v in metadata["enumerate"]: 194 | return get_enumeration_sparql(rq, v, endpoint, auth) 195 | return None 196 | 197 | 198 | def get_enumeration_sparql(rq, v, endpoint, auth=None): 199 | """ 200 | Returns a list of enumerated values for variable 'v' in query 'rq' 201 | """ 202 | glogger.debug("Retrieving enumeration for variable {}".format(v)) 203 | vcodes = [] 204 | 205 | # WHERE is optional too!! 206 | tpattern_matcher = re.compile( 207 | r".*?(FROM\s*(?P\<.*\>+))?\s*(WHERE\s*)?\{(?P.*)\}.*", 208 | flags=re.DOTALL, 209 | ) 210 | 211 | glogger.debug(rq) 212 | tp_match = tpattern_matcher.match(rq) 213 | if tp_match: 214 | vtpattern = tp_match.group("tpattern") 215 | gnames = tp_match.group("gnames") 216 | glogger.debug("Detected graph names: {}".format(gnames)) 217 | glogger.debug("Detected BGP: {}".format(vtpattern)) 218 | glogger.debug("Matched triple pattern with parameter") 219 | if gnames: 220 | codes_subquery = re.sub( 221 | r"SELECT.*\{.*\}.*", 222 | r"SELECT DISTINCT ?" 223 | + v 224 | + r" FROM " 225 | + gnames 226 | + r" WHERE { " 227 | + vtpattern 228 | + r" }", 229 | rq, 230 | flags=re.DOTALL, 231 | ) 232 | else: 233 | codes_subquery = re.sub( 234 | r"SELECT.*\{.*\}.*", 235 | r"SELECT DISTINCT ?" + v + r" WHERE { " + vtpattern + " }", 236 | rq, 237 | flags=re.DOTALL, 238 | ) 239 | glogger.debug("Codes subquery: {}".format(codes_subquery)) 240 | glogger.debug(endpoint) 241 | codes_json = requests.get( 242 | endpoint, 243 | params={"query": codes_subquery}, 244 | headers={ 245 | "Accept": static.mimetypes["json"], 246 | "Authorization": "token {}".format(static.SPARQL_ACCESS_TOKEN), 247 | }, 248 | auth=auth, 249 | ).json() 250 | for code in codes_json["results"]["bindings"]: 251 | vcodes.append(list(code.values())[0]["value"]) 252 | else: 253 | glogger.debug("No match between variable name and query.") 254 | 255 | return vcodes 256 | 257 | 258 | def get_yaml_decorators(rq): 259 | """ 260 | Returns the yaml decorator metadata only (this is needed by triple pattern fragments) 261 | """ 262 | # glogger.debug('Guessing decorators for query {}'.format(rq)) 263 | if not rq: 264 | return None 265 | 266 | yaml_string = "" 267 | query_string = "" 268 | query_metadata = None 269 | if isinstance(rq, dict): # json query (sparql transformer) 270 | if "grlc" in rq: 271 | yaml_string = rq["grlc"] 272 | query_string = rq 273 | query_metadata = yaml_string 274 | 275 | else: # classic query 276 | yaml_string = "\n".join( 277 | [row.lstrip("#+") for row in rq.split("\n") if row.startswith("#+")] 278 | ) 279 | query_string = "\n".join( 280 | [row for row in rq.split("\n") if not row.startswith("#+")] 281 | ) 282 | 283 | try: # Invalid YAMLs will produce empty metadata 284 | query_metadata = yaml.safe_load(yaml_string) 285 | except (yaml.parser.ParserError, yaml.scanner.ScannerError): 286 | try: 287 | query_metadata = json.loads(yaml_string) 288 | except json.JSONDecodeError: 289 | glogger.warning( 290 | "Query decorators could not be parsed; check your YAML syntax" 291 | ) 292 | 293 | # If there is no YAML string 294 | if query_metadata is None: 295 | query_metadata = {} 296 | query_metadata["query"] = query_string 297 | 298 | # glogger.debug("Parsed query decorators: {}".format(query_metadata)) 299 | 300 | return query_metadata 301 | 302 | 303 | def enable_custom_function_prefix(rq, prefix): 304 | """Add SPARQL prefixe header if the prefix is used in the given query.""" 305 | if ( 306 | " %s:" % prefix in rq 307 | or "(%s:" % prefix in rq 308 | and not "PREFIX %s:" % prefix in rq 309 | ): 310 | rq = "PREFIX %s: <:%s>\n" % (prefix, prefix) + rq 311 | return rq 312 | 313 | 314 | def get_metadata(rq, endpoint): 315 | """ 316 | Returns the metadata 'exp' parsed from the raw query file 'rq' 317 | 'exp' is one of: 'endpoint', 'tags', 'summary', 'request', 'pagination', 'enumerate' 318 | """ 319 | query_metadata = get_yaml_decorators(rq) 320 | query_metadata["type"] = "UNKNOWN" 321 | query_metadata["original_query"] = rq 322 | 323 | if isinstance(rq, dict): # json query (sparql transformer) 324 | rq, proto, opt = SPARQLTransformer.pre_process(rq) 325 | rq = rq.strip() 326 | query_metadata["proto"] = proto 327 | query_metadata["opt"] = opt 328 | query_metadata["query"] = rq 329 | 330 | rq = enable_custom_function_prefix(rq, "bif") 331 | rq = enable_custom_function_prefix(rq, "sql") 332 | 333 | try: 334 | # THE PARSING 335 | # select, describe, construct, ask 336 | parsed_query = translateQuery(Query.parseString(rq, parseAll=True)) 337 | query_metadata["type"] = parsed_query.algebra.name 338 | if query_metadata["type"] == "SelectQuery": 339 | # Projection variables 340 | query_metadata["variables"] = parsed_query.algebra["PV"] 341 | # Parameters 342 | query_metadata["parameters"] = get_parameters(rq, endpoint, query_metadata) 343 | elif query_metadata["type"] == "ConstructQuery": 344 | # Parameters 345 | query_metadata["parameters"] = get_parameters(rq, endpoint, query_metadata) 346 | else: 347 | glogger.warning( 348 | "Query type {} is currently unsupported and no metadata was parsed!".format( 349 | query_metadata["type"] 350 | ) 351 | ) 352 | except ParseException as pe: 353 | glogger.warning(pe) 354 | glogger.warning( 355 | "Could not parse regular SELECT, CONSTRUCT, DESCRIBE or ASK query" 356 | ) 357 | # glogger.warning(traceback.print_exc()) 358 | 359 | # insert queries won't parse, so we regex 360 | # glogger.info("Trying to parse INSERT query") 361 | # if static.INSERT_PATTERN in rq: 362 | # query_metadata['type'] = 'InsertQuery' 363 | # query_metadata['parameters'] = [u'_g_iri'] 364 | 365 | try: 366 | # update query 367 | glogger.debug("Trying to parse UPDATE query") 368 | parsed_query = UpdateUnit.parseString(rq, parseAll=True) 369 | glogger.debug(parsed_query) 370 | query_metadata["type"] = parsed_query[0]["request"][0].name 371 | if query_metadata["type"] == "InsertData": 372 | query_metadata["parameters"] = { 373 | "g": { 374 | "datatype": None, 375 | "enum": [], 376 | "lang": None, 377 | "name": "g", 378 | "original": "?_g_iri", 379 | "required": True, 380 | "type": "iri", 381 | }, 382 | "data": { 383 | "datatype": None, 384 | "enum": [], 385 | "lang": None, 386 | "name": "data", 387 | "original": "?_data", 388 | "required": True, 389 | "type": "literal", 390 | }, 391 | } 392 | 393 | glogger.debug("Update query parsed with {}".format(query_metadata["type"])) 394 | # if query_metadata['type'] == 'InsertData': 395 | # query_metadata['variables'] = parsed_query.algebra['PV'] 396 | except Exception as e: 397 | glogger.error("Could not parse query") 398 | glogger.error(query_metadata["query"]) 399 | glogger.error(traceback.print_exc()) 400 | raise Exception("could not parse query: {}".format(str(e))) 401 | 402 | glogger.debug("Finished parsing query of type {}".format(query_metadata["type"])) 403 | glogger.debug("All parsed query metadata (from decorators and content): ") 404 | glogger.debug(pformat(query_metadata, indent=32)) 405 | 406 | return query_metadata 407 | 408 | 409 | def paginate_query(query, results_per_page, get_args): 410 | """Modify the given query so that it can be paginated. The paginated query will 411 | split display a maximum of `results_per_page`.""" 412 | page = get_args.get("page", 1) 413 | 414 | glogger.debug( 415 | "Paginating query for page {}, {} results per page".format( 416 | page, results_per_page 417 | ) 418 | ) 419 | 420 | # If contains LIMIT or OFFSET, remove them 421 | glogger.debug("Original query: " + query) 422 | no_limit_query = re.sub(r"((LIMIT|OFFSET)\s+[0-9]+)*", "", query) 423 | glogger.debug("No limit query: " + no_limit_query) 424 | 425 | # Append LIMIT results_per_page OFFSET (page-1)*results_per_page 426 | paginated_query = no_limit_query + " LIMIT {} OFFSET {}".format( 427 | results_per_page, (int(page) - 1) * results_per_page 428 | ) 429 | glogger.debug("Paginated query: " + paginated_query) 430 | 431 | return paginated_query 432 | 433 | 434 | def rewrite_query(query, parameters, get_args): 435 | """Rewrite query to replace query parameters for given values.""" 436 | glogger.debug("Query parameters") 437 | glogger.debug(parameters) 438 | 439 | # Check that all required parameters are present 440 | requiredParams = set( 441 | k for k, v in parameters.items() if v["required"] 442 | ) # Set of required parameters 443 | providedParams = set(get_args.keys()) 444 | glogger.debug( 445 | "Required parameters: {} Request args: {}".format( 446 | requiredParams, providedParams 447 | ) 448 | ) 449 | assert requiredParams.issubset( 450 | providedParams 451 | ), "Provided parameters do not cover the required parameters!" 452 | 453 | if isinstance(query, dict): # json query (sparql transformer) 454 | query = rewrite_query_json(query, parameters, get_args) 455 | else: 456 | query = rewrite_query_standard(query, parameters, get_args) 457 | 458 | glogger.debug("Query rewritten as: " + query) 459 | 460 | return query 461 | 462 | 463 | def rewrite_query_json(query, parameters, get_args): 464 | for pname, p in parameters.items(): 465 | # Get the parameter value from the GET request 466 | v = get_args.get(pname, None) 467 | # If the parameter has a value 468 | if not v: 469 | continue 470 | 471 | if "$values" not in query: 472 | query["$values"] = {} 473 | values = query["$values"] 474 | 475 | if not p["original"] in values: 476 | values[p["original"]] = v 477 | elif isinstance(values[p["original"]], list): 478 | values[p["original"]].append(v) 479 | else: 480 | values[p["original"]] = [values[p["original"]], v] 481 | 482 | rq, proto, opt = SPARQLTransformer.pre_process(query) 483 | query = rq.strip() 484 | return query 485 | 486 | 487 | def rewrite_query_standard(query, parameters, get_args): 488 | requireXSD = False 489 | for pname, p in parameters.items(): 490 | # Get the parameter value from the GET request 491 | v = get_args.get(pname, None) 492 | # If the parameter has a value 493 | if not v: 494 | continue 495 | 496 | # Number (without a datatype) 497 | if p["type"] == "number": 498 | query = query.replace(p["original"], v) 499 | # Literal 500 | elif p["type"] == "literal" or p["type"] == "string": 501 | # If it's a iri 502 | if "format" in p and p["format"] == "iri": 503 | query = query.replace(p["original"], "{}{}{}".format("<", v, ">")) 504 | # If there is a language tag 505 | if "lang" in p and p["lang"]: 506 | query = query.replace(p["original"], '"{}"@{}'.format(v, p["lang"])) 507 | elif "datatype" in p and p["datatype"]: 508 | query = query.replace( 509 | p["original"], '"{}"^^{}'.format(v, p["datatype"]) 510 | ) 511 | if "xsd" in p["datatype"]: 512 | requireXSD = True 513 | else: 514 | query = query.replace(p["original"], '"{}"'.format(v)) 515 | if requireXSD and XSD_PREFIX not in query: 516 | query = query.replace("SELECT", XSD_PREFIX + "\n\nSELECT") 517 | return query 518 | -------------------------------------------------------------------------------- /src/pagination.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2022 Albert Meroño, Rinke Hoekstra, Carlos Martínez 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | from urllib.parse import urlparse, urlunparse, parse_qsl, urlencode, ParseResult 6 | 7 | 8 | def getSwaggerPaginationDef(resultsPerPage): 9 | """Build swagger spec section for pagination""" 10 | return { 11 | "name": "page", 12 | "type": "int", 13 | "in": "query", 14 | "description": "The page number for this paginated query ({} results per page)".format( 15 | resultsPerPage 16 | ), 17 | } 18 | 19 | 20 | def buildPaginationHeader(resultCount, resultsPerPage, pageArg, url): 21 | """Build link header for result pagination""" 22 | lastPage = resultCount / resultsPerPage 23 | 24 | url_parts = urlparse(url) 25 | query = dict( 26 | parse_qsl(url_parts.query) 27 | ) # Use dict parse_qsl instead of parse_qs to ensure 'page' is unique 28 | 29 | first_url = _buildNewUrlWithPage(url_parts, query, page=1) 30 | last_url = _buildNewUrlWithPage(url_parts, query, page=lastPage) 31 | 32 | if not pageArg: 33 | next_url = _buildNewUrlWithPage(url_parts, query, page=1) 34 | prev_url = "" 35 | headerLink = "<{}>; rel=next, <{}>; rel=last".format(next_url, last_url) 36 | else: 37 | page = int(pageArg) 38 | next_url = _buildNewUrlWithPage(url_parts, query, page + 1) 39 | prev_url = _buildNewUrlWithPage(url_parts, query, page - 1) 40 | 41 | if page == lastPage: 42 | headerLink = "<{}>; rel=prev, <{}>; rel=first".format(prev_url, first_url) 43 | else: 44 | headerLink = "<{}>; rel=next, <{}>; rel=prev, <{}>; rel=first, <{}>; rel=last".format( 45 | next_url, prev_url, first_url, last_url 46 | ) 47 | return headerLink 48 | 49 | 50 | def _buildNewUrlWithPage(url_parts, query, page): 51 | query["page"] = page 52 | new_query = urlencode(query) 53 | newParsedUrl = ParseResult( 54 | scheme=url_parts.scheme, 55 | netloc=url_parts.netloc, 56 | path=url_parts.path, 57 | params=url_parts.params, 58 | query=new_query, 59 | fragment=url_parts.fragment, 60 | ) 61 | return urlunparse(newParsedUrl) 62 | -------------------------------------------------------------------------------- /src/prov.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # SPDX-FileCopyrightText: 2022 Albert Meroño, Rinke Hoekstra, Carlos Martínez 4 | # 5 | # SPDX-License-Identifier: MIT 6 | 7 | # prov.py: class generating grlc related W3C prov triples 8 | 9 | from rdflib import Graph, URIRef, Namespace, RDF, Literal 10 | from datetime import datetime 11 | from subprocess import check_output 12 | 13 | # grlc modules 14 | import grlc.static as static 15 | import grlc.glogging as glogging 16 | 17 | glogger = glogging.getGrlcLogger(__name__) 18 | 19 | 20 | class grlcPROV: 21 | """Record provenance of the grlc specification constructed.""" 22 | 23 | def __init__(self, user, repo): 24 | """Default constructor. 25 | 26 | Keyword arguments: 27 | user -- Github user. 28 | repo -- Github repo. 29 | """ 30 | self.user = user 31 | self.repo = repo 32 | self.prov_g = Graph() 33 | prov_uri = URIRef("http://www.w3.org/ns/prov#") 34 | self.prov = Namespace(prov_uri) 35 | self.prov_g.bind("prov", self.prov) 36 | 37 | self.agent = URIRef("http://{}".format(static.SERVER_NAME)) 38 | self.entity_d = URIRef( 39 | "http://{}/api/{}/{}/spec".format(static.SERVER_NAME, self.user, self.repo) 40 | ) 41 | self.activity = URIRef(self.entity_d + "-activity") 42 | 43 | self.init_prov_graph() 44 | 45 | def init_prov_graph(self): 46 | """ 47 | Initialize PROV graph with all we know at the start of the recording 48 | """ 49 | 50 | try: 51 | # Use git2prov to get prov on the repo 52 | repo_prov = check_output( 53 | [ 54 | "node_modules/git2prov/bin/git2prov", 55 | "https://github.com/{}/{}/".format(self.user, self.repo), 56 | "PROV-O", 57 | ] 58 | ).decode("utf-8") 59 | repo_prov = repo_prov[repo_prov.find("@") :] 60 | # glogger.debug('Git2PROV output: {}'.format(repo_prov)) 61 | glogger.debug("Ingesting Git2PROV output into RDF graph") 62 | with open("temp.prov.ttl", "w") as temp_prov: 63 | temp_prov.write(repo_prov) 64 | 65 | self.prov_g.parse("temp.prov.ttl", format="turtle") 66 | except Exception as e: 67 | glogger.error(e) 68 | glogger.error("Couldn't parse Git2PROV graph, continuing without repo PROV") 69 | pass 70 | 71 | self.prov_g.add((self.agent, RDF.type, self.prov.Agent)) 72 | self.prov_g.add((self.entity_d, RDF.type, self.prov.Entity)) 73 | self.prov_g.add((self.activity, RDF.type, self.prov.Activity)) 74 | 75 | # entity_d 76 | self.prov_g.add((self.entity_d, self.prov.wasGeneratedBy, self.activity)) 77 | self.prov_g.add((self.entity_d, self.prov.wasAttributedTo, self.agent)) 78 | # later: entity_d genereated at time (when we know the end time) 79 | 80 | # activity 81 | self.prov_g.add((self.activity, self.prov.wasAssociatedWith, self.agent)) 82 | self.prov_g.add( 83 | (self.activity, self.prov.startedAtTime, Literal(datetime.now())) 84 | ) 85 | # later: activity used entity_o_1 ... entity_o_n 86 | # later: activity endedAtTime (when we know the end time) 87 | 88 | def add_used_entity(self, entity_uri): 89 | """ 90 | Add the provided URI as a used entity by the logged activity 91 | """ 92 | entity_o = URIRef(entity_uri) 93 | self.prov_g.add((entity_o, RDF.type, self.prov.Entity)) 94 | self.prov_g.add((self.activity, self.prov.used, entity_o)) 95 | 96 | def end_prov_graph(self): 97 | """ 98 | Finalize prov recording with end time 99 | """ 100 | endTime = Literal(datetime.now()) 101 | self.prov_g.add((self.entity_d, self.prov.generatedAtTime, endTime)) 102 | self.prov_g.add((self.activity, self.prov.endedAtTime, endTime)) 103 | 104 | def log_prov_graph(self): 105 | """ 106 | Log provenance graph so far 107 | """ 108 | glogger.debug("Spec generation provenance graph:") 109 | glogger.debug(self.prov_g.serialize(format="turtle")) 110 | 111 | def serialize(self, format): 112 | """ 113 | Serialize provenance graph in the specified format 114 | """ 115 | return self.prov_g.serialize(format=format) 116 | -------------------------------------------------------------------------------- /src/queryTypes.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2022 Albert Meroño, Rinke Hoekstra, Carlos Martínez 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | """Definition of grlc query types.""" 6 | 7 | qType = {"SPARQL": "sparql", "TPF": "tpf", "JSON": "json"} 8 | 9 | 10 | def guessQueryType(queryUrl): 11 | queryUrl = queryUrl.lower() 12 | if queryUrl.endswith(".rq"): 13 | return qType["SPARQL"] 14 | elif queryUrl.endswith(".sparql"): 15 | return qType["SPARQL"] 16 | elif queryUrl.endswith(".tpf"): 17 | return qType["TPF"] 18 | elif queryUrl.endswith(".json"): 19 | return qType["JSON"] 20 | else: 21 | raise Exception("Unknown query type: " + queryUrl) 22 | -------------------------------------------------------------------------------- /src/server.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # SPDX-FileCopyrightText: 2022 Albert Meroño, Rinke Hoekstra, Carlos Martínez 4 | # 5 | # SPDX-License-Identifier: MIT 6 | 7 | # server.py: the grlc server 8 | 9 | from flask import Flask, request, jsonify, render_template, make_response 10 | from flask_cors import CORS 11 | 12 | # grlc modules 13 | import grlc.static as static 14 | import grlc.utils as utils 15 | import grlc.glogging as glogging 16 | 17 | glogger = glogging.getGrlcLogger(__name__) 18 | 19 | # The Flask app 20 | app = Flask(__name__) 21 | CORS(app) 22 | 23 | 24 | # Helper functions 25 | def relative_path(): 26 | """Generate relative path for the current route. This is used to build relative paths when rendering templates.""" 27 | path = request.path 28 | path = "." + "/.." * (path.count("/") - 1) 29 | return path 30 | 31 | 32 | def api_docs_template(): 33 | """Generate Grlc API page.""" 34 | return render_template("api-docs.html", relative_path=relative_path()) 35 | 36 | 37 | def swagger_spec( 38 | user, 39 | repo, 40 | subdir=None, 41 | spec_url=None, 42 | sha=None, 43 | content=None, 44 | git_type=None, 45 | branch=None, 46 | ): 47 | """Generate swagger specification""" 48 | glogger.info( 49 | "-----> Generating swagger spec for /{}/{} ({}), subdir {}, params {}, on commit {}".format( 50 | user, repo, git_type, subdir, spec_url, sha 51 | ) 52 | ) 53 | 54 | swag = utils.build_swagger_spec( 55 | user, repo, subdir, spec_url, sha, static.SERVER_NAME, git_type, branch 56 | ) 57 | 58 | resp_spec = make_response(jsonify(swag)) 59 | resp_spec.headers["Content-Type"] = "application/json" 60 | 61 | resp_spec.headers["Cache-Control"] = ( 62 | static.CACHE_CONTROL_POLICY 63 | ) # Caching JSON specs for 15 minutes 64 | 65 | glogger.info( 66 | "-----> API spec generation for /{}/{}, subdir {}, params {}, on commit {} complete".format( 67 | user, repo, subdir, spec_url, sha 68 | ) 69 | ) 70 | return resp_spec 71 | 72 | 73 | def query( 74 | user, 75 | repo, 76 | query_name, 77 | subdir=None, 78 | spec_url=None, 79 | sha=None, 80 | content=None, 81 | git_type=None, 82 | branch=None, 83 | ): 84 | """Execute SPARQL query for a specific grlc-generated API endpoint""" 85 | glogger.info( 86 | "-----> Executing call name at /{}/{} ({})/{}/{} on commit {}".format( 87 | user, repo, git_type, subdir, query_name, sha 88 | ) 89 | ) 90 | glogger.debug("Request accept header: " + request.headers["Accept"]) 91 | 92 | requestArgs = request.args 93 | acceptHeader = request.headers["Accept"] 94 | requestUrl = request.url 95 | formData = request.form 96 | method = request.method 97 | 98 | query_response, status, headers = utils.dispatch_query( 99 | user, 100 | repo, 101 | query_name, 102 | subdir, 103 | spec_url, 104 | sha=sha, 105 | content=content, 106 | requestArgs=requestArgs, 107 | acceptHeader=acceptHeader, 108 | requestUrl=requestUrl, 109 | formData=formData, 110 | method=method, 111 | git_type=git_type, 112 | branch=branch, 113 | ) 114 | if isinstance(query_response, list) or isinstance(query_response, dict): 115 | query_response = jsonify(query_response) 116 | 117 | return make_response(query_response, status, headers) 118 | 119 | 120 | # Server routes 121 | @app.route("/") 122 | def grlc(): 123 | """Grlc landing page.""" 124 | resp = make_response(render_template("index.html")) 125 | return resp 126 | 127 | 128 | # Routes for local APIs 129 | 130 | 131 | # Spec generation, front-end 132 | @app.route("/api-local", methods=["GET"], strict_slashes=False) 133 | @app.route( 134 | "/api/local/local", methods=["GET"], strict_slashes=False 135 | ) # backward compatibility route 136 | def api_docs_local(): 137 | """Grlc API page for local routes.""" 138 | return api_docs_template() 139 | 140 | 141 | # Spec generation, JSON 142 | @app.route("/api-local/swagger", methods=["GET"]) 143 | @app.route( 144 | "/api/local/local/swagger", methods=["GET"], strict_slashes=False 145 | ) # backward compatibility route 146 | def swagger_spec_local(): 147 | """Swagger spec for local routes.""" 148 | return swagger_spec(user=None, repo=None, sha=None, content=None) 149 | 150 | 151 | # Callname execution 152 | @app.route("/api-local/", methods=["GET", "POST"]) 153 | @app.route("/api-local/.", methods=["GET", "POST"]) 154 | @app.route( 155 | "/api/local/local/", methods=["GET", "POST"], strict_slashes=False 156 | ) # backward compatibility route 157 | @app.route( 158 | "/api/local/local/.", 159 | methods=["GET", "POST"], 160 | strict_slashes=False, 161 | ) # backward compatibility route 162 | def query_local(query_name, content=None): 163 | """SPARQL query execution for local routes.""" 164 | return query(user=None, repo=None, query_name=query_name, content=content) 165 | 166 | 167 | # Routes for URL HTTP APIs 168 | 169 | 170 | # Spec generation, front-end 171 | @app.route("/api-url", methods=["POST", "GET"], strict_slashes=False) 172 | def api_docs_param(): 173 | """Grlc API page for specifications loaded via http.""" 174 | # Get queries provided by params 175 | spec_url = request.args["specUrl"] 176 | glogger.info("Spec URL: {}".format(spec_url)) 177 | return api_docs_template() 178 | 179 | 180 | # Spec generation, JSON 181 | @app.route("/api-url/swagger", methods=["GET"]) 182 | def swagger_spec_param(): 183 | """Swagger spec for specifications loaded via http.""" 184 | spec_url = request.args["specUrl"] 185 | glogger.info("Spec URL: {}".format(spec_url)) 186 | return swagger_spec(user=None, repo=None, spec_url=spec_url) 187 | 188 | 189 | # Callname execution 190 | @app.route("/api-url/", methods=["GET", "POST"]) 191 | @app.route("/api-url/.", methods=["GET", "POST"]) 192 | def query_param(query_name, content=None): 193 | """SPARQL query execution for specifications loaded via http.""" 194 | spec_url = request.args["specUrl"] 195 | glogger.debug("Spec URL: {}".format(spec_url)) 196 | return query( 197 | user=None, repo=None, query_name=query_name, spec_url=spec_url, content=content 198 | ) 199 | 200 | 201 | # Routes for GitHub APIs 202 | 203 | 204 | # Spec generation, front-end 205 | @app.route("/api-git//", strict_slashes=False) 206 | @app.route("/api-git///subdir/", strict_slashes=False) 207 | @app.route("/api-git///api-docs") 208 | @app.route("/api-git///commit/") 209 | @app.route("/api-git///commit//api-docs") 210 | @app.route("/api-git///subdir//commit/") 211 | @app.route("/api-git///subdir//commit//api-docs") 212 | @app.route("/api//", strict_slashes=False) # backward compatibility route 213 | @app.route( 214 | "/api///", strict_slashes=False 215 | ) # backward compatibility route 216 | @app.route("/api///api-docs") # backward compatibility route 217 | @app.route("/api///commit/") # backward compatibility route 218 | @app.route("/api///commit//api-docs") # backward compatibility route 219 | @app.route( 220 | "/api////commit/" 221 | ) # backward compatibility route 222 | @app.route( 223 | "/api////commit//api-docs" 224 | ) # backward compatibility route 225 | def api_docs_git(user, repo, subdir=None, sha=None): 226 | """Grlc API page for specifications loaded from a Github repo.""" 227 | return api_docs_template() 228 | 229 | 230 | # Spec generation, JSON 231 | @app.route("/api-git///swagger", methods=["GET"]) 232 | @app.route("/api-git///subdir//swagger", methods=["GET"]) 233 | @app.route("/api-git///commit//swagger") 234 | @app.route("/api-git///subdir//commit//swagger") 235 | @app.route("/api-git////commit//swagger") 236 | @app.route( 237 | "/api///swagger", methods=["GET"] 238 | ) # backward compatibility route 239 | @app.route( 240 | "/api////swagger", methods=["GET"] 241 | ) # backward compatibility route 242 | @app.route("/api///commit//swagger") # backward compatibility route 243 | @app.route( 244 | "/api////commit//swagger" 245 | ) # backward compatibility route 246 | @app.route( 247 | "/api-git////swagger", methods=["GET"] 248 | ) # backward compatibility route 249 | @app.route( 250 | "/api-git////commit//swagger" 251 | ) # backward compatibility route 252 | def swagger_spec_git(user, repo, subdir=None, sha=None): 253 | """Swagger spec for specifications loaded from a Github repo.""" 254 | return swagger_spec( 255 | user, 256 | repo, 257 | subdir=subdir, 258 | spec_url=None, 259 | sha=sha, 260 | content=None, 261 | git_type=static.TYPE_GITHUB, 262 | ) 263 | 264 | 265 | # Callname execution 266 | @app.route("/api-git///", methods=["GET", "POST"]) 267 | @app.route( 268 | "/api-git///subdir//", methods=["GET", "POST"] 269 | ) 270 | @app.route("/api-git///.", methods=["GET", "POST"]) 271 | @app.route( 272 | "/api-git///subdir//.", 273 | methods=["GET", "POST"], 274 | ) 275 | @app.route("/api-git///commit//", methods=["GET", "POST"]) 276 | @app.route( 277 | "/api-git///subdir//commit//", 278 | methods=["GET", "POST"], 279 | ) 280 | @app.route( 281 | "/api-git///commit//.", 282 | methods=["GET", "POST"], 283 | ) 284 | @app.route( 285 | "/api-git///subdir//commit//.", 286 | methods=["GET", "POST"], 287 | ) 288 | @app.route( 289 | "/api///", methods=["GET", "POST"] 290 | ) # backward compatibility route 291 | @app.route( 292 | "/api////", methods=["GET", "POST"] 293 | ) # backward compatibility route 294 | @app.route( 295 | "/api///.", methods=["GET", "POST"] 296 | ) # backward compatibility route 297 | @app.route( 298 | "/api////.", methods=["GET", "POST"] 299 | ) # backward compatibility route 300 | @app.route( 301 | "/api///commit//", methods=["GET", "POST"] 302 | ) # backward compatibility route 303 | @app.route( 304 | "/api////commit//", 305 | methods=["GET", "POST"], 306 | ) # backward compatibility route 307 | @app.route( 308 | "/api///commit//.", methods=["GET", "POST"] 309 | ) # backward compatibility route 310 | @app.route( 311 | "/api////commit//.", 312 | methods=["GET", "POST"], 313 | ) # backward compatibility route 314 | def query_git(user, repo, query_name, subdir=None, sha=None, content=None): 315 | """SPARQL query execution for specifications loaded from a Github repo.""" 316 | return query( 317 | user, 318 | repo, 319 | query_name, 320 | subdir=subdir, 321 | sha=sha, 322 | content=content, 323 | git_type=static.TYPE_GITHUB, 324 | ) 325 | 326 | 327 | # Routes for GitLab APIs 328 | 329 | 330 | # Spec generation, front-end 331 | @app.route("/api-gitlab//", strict_slashes=False) 332 | @app.route("/api-gitlab///branch/", strict_slashes=False) 333 | @app.route("/api-gitlab///subdir/", strict_slashes=False) 334 | @app.route( 335 | "/api-gitlab///branch//subdir/", 336 | strict_slashes=False, 337 | ) 338 | @app.route("/api-gitlab///api-docs") 339 | @app.route("/api-gitlab///commit/") 340 | @app.route("/api-gitlab///commit//api-docs") 341 | @app.route("/api-gitlab///subdir//commit/") 342 | @app.route("/api-gitlab///subdir//commit//api-docs") 343 | def api_docs_gitlab(user, repo, subdir=None, sha=None, branch=None): 344 | """Grlc API page for specifications loaded from a Github repo.""" 345 | glogger.debug("Entry in function: __main__.api_docs_gitlab") 346 | return api_docs_template() 347 | 348 | 349 | # Spec generation, JSON 350 | @app.route("/api-gitlab///swagger", methods=["GET"]) 351 | @app.route("/api-gitlab///branch//swagger", methods=["GET"]) 352 | @app.route("/api-gitlab///subdir//swagger", methods=["GET"]) 353 | @app.route( 354 | "/api-gitlab///branch//subdir//swagger", 355 | methods=["GET"], 356 | ) 357 | @app.route("/api-gitlab///commit//swagger") 358 | @app.route("/api-gitlab///subdir//commit//swagger") 359 | @app.route("/api-gitlab////commit//swagger") 360 | def swagger_spec_gitlab(user, repo, subdir=None, sha=None, branch=None): 361 | """Swagger spec for specifications loaded from a Github repo.""" 362 | glogger.debug("Entry in function: __main__.swagger_spec_gitlab") 363 | return swagger_spec( 364 | user, 365 | repo, 366 | subdir=subdir, 367 | spec_url=None, 368 | sha=sha, 369 | content=None, 370 | git_type=static.TYPE_GITLAB, 371 | branch=branch, 372 | ) 373 | 374 | 375 | # Callname execution 376 | @app.route("/api-gitlab///query/", methods=["GET", "POST"]) 377 | @app.route( 378 | "/api-gitlab///query/branch//", 379 | methods=["GET", "POST"], 380 | ) 381 | @app.route( 382 | "/api-gitlab///query/subdir//", 383 | methods=["GET", "POST"], 384 | ) 385 | @app.route( 386 | "/api-gitlab///query/branch//subdir//", 387 | methods=["GET", "POST"], 388 | ) 389 | @app.route( 390 | "/api-gitlab///query/.", methods=["GET", "POST"] 391 | ) 392 | @app.route( 393 | "/api-gitlab///query/subdir//.", 394 | methods=["GET", "POST"], 395 | ) 396 | @app.route( 397 | "/api-gitlab///query/commit//", methods=["GET", "POST"] 398 | ) 399 | @app.route( 400 | "/api-gitlab///query/subdir//commit//", 401 | methods=["GET", "POST"], 402 | ) 403 | @app.route( 404 | "/api-gitlab///query/commit//.", 405 | methods=["GET", "POST"], 406 | ) 407 | @app.route( 408 | "/api-gitlab///query/subdir//commit//.", 409 | methods=["GET", "POST"], 410 | ) 411 | def query_gitlab( 412 | user, repo, query_name, subdir=None, sha=None, content=None, branch=None 413 | ): 414 | """SPARQL query execution for specifications loaded from a Github repo.""" 415 | glogger.debug("Entry in function: __main__.query_gitlab") 416 | return query( 417 | user, 418 | repo, 419 | query_name, 420 | subdir=subdir, 421 | sha=sha, 422 | content=content, 423 | git_type=static.TYPE_GITLAB, 424 | branch=branch, 425 | ) 426 | 427 | 428 | # Main thread 429 | if __name__ == "__main__": 430 | app.run(host=static.DEFAULT_HOST, port=static.DEFAULT_PORT, debug=True) 431 | -------------------------------------------------------------------------------- /src/sparql.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2022 Albert Meroño, Rinke Hoekstra, Carlos Martínez 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | from SPARQLWrapper import SPARQLWrapper, CSV, JSON 6 | from flask import jsonify 7 | from collections import defaultdict 8 | 9 | import static as static 10 | import grlc.glogging as glogging 11 | 12 | glogger = glogging.getGrlcLogger(__name__) 13 | 14 | # Default value is JSON 15 | SUPPORTED_MIME_FORMATS = defaultdict( 16 | lambda: JSON, {"text/csv": CSV, "application/json": JSON} 17 | ) 18 | 19 | MIME_FORMAT = {format: mime for mime, format in SUPPORTED_MIME_FORMATS.items()} 20 | 21 | 22 | def getResponseText(endpoint, query, requestedMimeType): 23 | """Returns the result and mimetype of executing the given query against 24 | the given endpoint. 25 | 26 | Keyword arguments: 27 | endpoint - URL of sparql endpoint 28 | query - SPARQL query to be executed 29 | requestedMimeType Type of content requested. can be: 30 | 'text/csv; q=1.0, */*; q=0.1' 31 | 'application/json' 32 | etc. 33 | """ 34 | retFormat = _mimeTypeToSparqlFormat(requestedMimeType) 35 | 36 | client = SPARQLWrapper(endpoint) 37 | client.setQuery(query) 38 | client.setReturnFormat(retFormat) 39 | client.setCredentials( 40 | static.DEFAULT_ENDPOINT_USER, static.DEFAULT_ENDPOINT_PASSWORD 41 | ) 42 | result = client.queryAndConvert() 43 | 44 | if retFormat == JSON: 45 | result = jsonify(result) 46 | 47 | return result, MIME_FORMAT[retFormat] 48 | 49 | 50 | def _mimeTypeToSparqlFormat(mimeType): 51 | if ";" in mimeType: 52 | mimeType = mimeType.split(";")[0].strip() 53 | return SUPPORTED_MIME_FORMATS[mimeType] 54 | -------------------------------------------------------------------------------- /src/static.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # SPDX-FileCopyrightText: 2022 Albert Meroño, Rinke Hoekstra, Carlos Martínez 4 | # 5 | # SPDX-License-Identifier: MIT 6 | 7 | # static.py: static values for the grlc Server 8 | import os 9 | from configparser import ConfigParser 10 | 11 | DEFAULT_HOST = None 12 | DEFAULT_PORT = 8088 13 | 14 | # XSD datatypes for parsing queries with parameters 15 | XSD_DATATYPES = [ 16 | "decimal", 17 | "float", 18 | "double", 19 | "integer", 20 | "positiveInteger", 21 | "negativeInteger", 22 | "nonPositiveInteger", 23 | "nonNegativeInteger", 24 | "long", 25 | "int", 26 | "short", 27 | "byte", 28 | "unsignedLong", 29 | "unsignedInt", 30 | "unsignedShort", 31 | "unsignedByte", 32 | "dateTime", 33 | "date", 34 | "gYearMonth", 35 | "gYear", 36 | "duration", 37 | "gMonthDay", 38 | "gDay", 39 | "gMonth", 40 | "string", 41 | "normalizedString", 42 | "token", 43 | "language", 44 | "NMTOKEN", 45 | "NMTOKENS", 46 | "Name", 47 | "NCName", 48 | "ID", 49 | "IDREFS", 50 | "ENTITY", 51 | "ENTITIES", 52 | "QName", 53 | "boolean", 54 | "hexBinary", 55 | "base64Binary", 56 | "anyURI", 57 | "notation", 58 | ] 59 | 60 | # MIME types for content negotiation 61 | mimetypes = { 62 | "csv": "text/csv; q=1.0, */*; q=0.1", 63 | "json": "application/json; q=1.0, application/sparql-results+json; q=0.8, */*; q=0.1", 64 | "html": "text/html; q=1.0, */*; q=0.1", 65 | "ttl": "text/turtle", 66 | } 67 | 68 | # GitHub base URLS 69 | GITHUB_RAW_BASE_URL = "https://raw.githubusercontent.com/" 70 | GITHUB_API_BASE_URL = "https://api.github.com/repos/" 71 | 72 | # Git types 73 | TYPE_GITHUB = "github" 74 | TYPE_GITLAB = "gitlab" 75 | 76 | # Cache control 77 | # CACHE_CONTROL_POLICY = 'public, max-age=60' 78 | # With the new hash retrieveal and redirect caching becomes obsolete 79 | CACHE_CONTROL_POLICY = "no-cache" 80 | 81 | # Setting headers to use access_token for the GitHub API 82 | config_fallbacks = { 83 | "github_access_token": "", 84 | "gitlab_access_token": "", 85 | "sparql_access_token": "", 86 | "sparql_endpoint": "", 87 | "user": "", 88 | "password": "", 89 | "server_name": "", 90 | "local_sparql_dir": "", 91 | "debug": "False", 92 | "gitlab_url": "https://gitlab", 93 | } 94 | config = ConfigParser(config_fallbacks) 95 | config.add_section("auth") 96 | config.add_section("defaults") 97 | config.add_section("local") 98 | config.add_section("api_gitlab") 99 | 100 | config_filename = os.path.join(os.getcwd(), "config.ini") 101 | print("Reading config file: ", config_filename) 102 | config.read(config_filename) 103 | GITHUB_ACCESS_TOKEN = config.get("auth", "github_access_token") 104 | GITLAB_ACCESS_TOKEN = config.get("auth", "gitlab_access_token") 105 | SPARQL_ACCESS_TOKEN = config.get("auth", "sparql_access_token") 106 | 107 | # Default endpoint, if none specified elsewhere 108 | DEFAULT_ENDPOINT = config.get("defaults", "sparql_endpoint") 109 | DEFAULT_ENDPOINT_USER = config.get("defaults", "user") 110 | DEFAULT_ENDPOINT_PASSWORD = config.get("defaults", "password") 111 | 112 | # Local folder where queries are loaded from 113 | LOCAL_SPARQL_DIR = config.get("local", "local_sparql_dir") 114 | 115 | # api_gitlab 116 | GITLAB_URL = config.get("api_gitlab", "gitlab_url") 117 | 118 | # server name, used by the Flask app and in the swagger spec 119 | SERVER_NAME = config.get("defaults", "server_name") 120 | 121 | # Logging format (prettier than the ugly standard in Flask) 122 | LOG_FORMAT = "%(asctime)-15s [%(levelname)s] (%(module)s.%(funcName)s) %(message)s" 123 | LOG_DEBUG_MODE = config.getboolean("defaults", "debug") 124 | 125 | # Pattern for INSERT query call names 126 | INSERT_PATTERN = "INSERT DATA { GRAPH ?_g_iri {

}}" 127 | -------------------------------------------------------------------------------- /src/static/css/grlc.css: -------------------------------------------------------------------------------- 1 | /* 2 | * SPDX-FileCopyrightText: 2022 Albert Meroño, Rinke Hoekstra, Carlos Martínez 3 | * 4 | * SPDX-License-Identifier: MIT 5 | */ 6 | 7 | /* Adapted from bootstrap's cover.css */ 8 | 9 | /* 10 | * Globals 11 | */ 12 | 13 | /* Links */ 14 | a:focus { 15 | color: #373737; 16 | } 17 | a,a:hover { 18 | color: #00a9a4; 19 | } 20 | 21 | /* Custom default button */ 22 | .btn-default:hover,.bton-default:focus { 23 | color: #fff; 24 | text-shadow: none; /* Prevent inheritence from `body` */ 25 | background-color: #008782; 26 | border: 1px solid #01aaa5; 27 | } 28 | .btn-default { 29 | color: #fff; 30 | text-shadow: none; /* Prevent inheritence from `body` */ 31 | background-color: #00a9a4; 32 | border: 1px solid #01aaa5; 33 | } 34 | 35 | 36 | /* 37 | * Base structure 38 | */ 39 | 40 | html, 41 | body { 42 | height: 100%; 43 | background-color: #eee; 44 | } 45 | body { 46 | color: #373737; 47 | text-align: center; 48 | /* text-shadow: 0 1px 3px rgba(0,0,0,.5); */ 49 | } 50 | 51 | /* Extra markup and styles for table-esque vertical and horizontal centering */ 52 | .site-wrapper { 53 | display: table; 54 | width: 100%; 55 | height: 100%; /* For at least Firefox */ 56 | min-height: 100%; 57 | -webkit-box-shadow: inset 0 0 65px rgba(0,0,0,.5); 58 | box-shadow: inset 0 0 65px rgba(0,0,0,.5); 59 | } 60 | .site-wrapper-inner { 61 | display: table-cell; 62 | vertical-align: top; 63 | } 64 | .cover-container { 65 | margin-right: auto; 66 | margin-left: auto; 67 | } 68 | 69 | /* Padding for spacing */ 70 | .inner { 71 | padding: 30px; 72 | } 73 | 74 | .smaller { 75 | font-size: 16px; 76 | } 77 | 78 | 79 | /* 80 | * Header 81 | */ 82 | .masthead-brand { 83 | margin-top: 10px; 84 | margin-bottom: 10px; 85 | } 86 | 87 | .masthead-nav > li { 88 | display: inline-block; 89 | } 90 | .masthead-nav > li + li { 91 | margin-left: 20px; 92 | } 93 | .masthead-nav > li > a { 94 | padding-right: 0; 95 | padding-left: 0; 96 | font-size: 16px; 97 | font-weight: bold; 98 | color: #373737; /* IE8 proofing */ 99 | /* color: rgba(255,255,255,.75); */ 100 | border-bottom: 2px solid transparent; 101 | } 102 | .masthead-nav > li > a:hover, 103 | .masthead-nav > li > a:focus { 104 | background-color: transparent; 105 | border-bottom-color: #373737; 106 | /* border-bottom-color: rgba(255,255,255,.25); */ 107 | } 108 | .masthead-nav > .active > a, 109 | .masthead-nav > .active > a:hover, 110 | .masthead-nav > .active > a:focus { 111 | color: #373737; 112 | border-bottom-color: #00a9a4; 113 | } 114 | 115 | @media (min-width: 768px) { 116 | .masthead-brand { 117 | float: left; 118 | } 119 | .masthead-nav { 120 | float: right; 121 | } 122 | } 123 | 124 | 125 | /* 126 | * Cover 127 | */ 128 | 129 | .cover { 130 | padding: 0 20px; 131 | } 132 | .cover .btn-lg { 133 | padding: 10px 20px; 134 | font-weight: bold; 135 | } 136 | 137 | 138 | /* 139 | * Footer 140 | */ 141 | 142 | .mastfoot { 143 | color: #373737; /* IE8 proofing */ 144 | /* color: rgba(255,255,255,.5); */ 145 | } 146 | 147 | 148 | /* 149 | * Affix and center 150 | */ 151 | 152 | @media (min-width: 768px) { 153 | /* Pull out the header and footer */ 154 | .masthead { 155 | position: fixed; 156 | top: 0; 157 | } 158 | .mastfoot { 159 | position: fixed; 160 | bottom: 0; 161 | } 162 | /* Start the vertical centering */ 163 | .site-wrapper-inner { 164 | vertical-align: middle; 165 | } 166 | /* Handle the widths */ 167 | .masthead, 168 | .mastfoot, 169 | .cover-container { 170 | width: 100%; /* Must be percentage or pixels for horizontal alignment */ 171 | } 172 | } 173 | 174 | @media (min-width: 992px) { 175 | .masthead, 176 | .mastfoot, 177 | .cover-container { 178 | width: 700px; 179 | } 180 | } 181 | 182 | 183 | /* grlc specific styles */ 184 | 185 | .circ-number { 186 | background-color: #fff; 187 | border: 3px solid #00a9a4; 188 | width: 40px; 189 | line-height: 36px; 190 | height: 40px; 191 | display: block; 192 | margin: 0 auto; 193 | border-radius: 50%; 194 | font-weight: bold; 195 | } 196 | 197 | /* Add space between p paragraphs */ 198 | 199 | .lead { 200 | margin-top: 10px; 201 | } 202 | -------------------------------------------------------------------------------- /src/static/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CLARIAH/grlc/73247dd656d015ebbb567a450b422517f2ef5782/src/static/favicon.ico -------------------------------------------------------------------------------- /src/static/grlc_logo_01.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CLARIAH/grlc/73247dd656d015ebbb567a450b422517f2ef5782/src/static/grlc_logo_01.png -------------------------------------------------------------------------------- /src/static/grlc_logo_02.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CLARIAH/grlc/73247dd656d015ebbb567a450b422517f2ef5782/src/static/grlc_logo_02.png -------------------------------------------------------------------------------- /src/static/js/grlc-api.js: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2022 Albert Meroño, Rinke Hoekstra, Carlos Martínez 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | /** 6 | * Log to JavaScript console. 7 | */ 8 | function log() { 9 | if ('console' in window) { 10 | console.log.apply(console, arguments); 11 | } 12 | } 13 | 14 | /** 15 | * Set the links `< Prev` and `Next >` href targets to next and previous commit 16 | * respectively. 17 | */ 18 | function set_commit_links(api) { 19 | let base = api['basePath']; 20 | if(base.includes('commit')) { 21 | base = base.split('commit')[0]; 22 | } 23 | if (api['prev_commit']) { 24 | const prev_target = base + 'commit/' + api['prev_commit']; 25 | $('#prev-commit').attr('onclick', 'redirectTo(" ' + prev_target + '")'); 26 | $('#prev-commit').show(); 27 | } else { 28 | $('#prev-commit').hide(); 29 | } 30 | if (api['next_commit']) { 31 | const next_target = base + 'commit/' + api['next_commit']; 32 | $('#next-commit').attr('onclick', 'redirectTo(" ' + next_target + '")'); 33 | $('#next-commit').show(); 34 | } else { 35 | $('#next-commit').hide(); 36 | } 37 | } 38 | 39 | /** 40 | * Redirects to specified URL 41 | */ 42 | function redirectTo(url) { 43 | location.href= url; 44 | } 45 | 46 | /** 47 | * Set's `Oh yeah ?` link action so it displays provenance when clicked. 48 | */ 49 | function get_prov(api) { 50 | $('#ohyeahdiv').show(); 51 | data = api['prov'].split("\n"); 52 | var html = '

'; 53 | for (var i=0; i < data.length; i++) { 54 | html += '
' + data[i].replace("<", "<").replace(">", ">") + '
'; 55 | } 56 | html += '
'; 57 | $('#prov').html(html).linkify({target: "_blank"}).hide(); 58 | } 59 | 60 | /** 61 | * This function gets called when SwaggerUIBundle finishes loading. 62 | */ 63 | function grlcOnComplete() { 64 | if(typeof ui.initOAuth == "function") { 65 | ui.initOAuth({ 66 | clientId: "your-client-id", 67 | clientSecret: "your-client-secret-if-required", 68 | realm: "your-realms", 69 | appName: "your-app-name", 70 | scopeSeparator: "," 71 | }); 72 | } 73 | // Extract JSON representation of spec 74 | const spec = ui.spec(); 75 | const swaggerApi = JSON.parse(spec.toObject()['spec']) 76 | get_prov(swaggerApi); 77 | set_commit_links(swaggerApi); 78 | 79 | if(window.SwaggerTranslator) { 80 | window.SwaggerTranslator.translate(); 81 | } 82 | 83 | $('pre code').each(function(i, e) { 84 | hljs.highlightBlock(e) 85 | }); 86 | } 87 | 88 | function grlcOnFailure(data) { 89 | log("Unable to Load SwaggerUI"); 90 | } 91 | 92 | /** 93 | * Toggle visualize provenance 94 | */ 95 | function grlcProvToggle(e){ 96 | var prov = $('#prov'); 97 | if (prov.is(":visible")) { 98 | prov.hide(); 99 | } else { 100 | prov.show(); 101 | } 102 | } 103 | 104 | /** 105 | * Called when window is fully loaded. It then triggers creation of SwaggerUIBundle. 106 | * 107 | */ 108 | function grlcOnLoad() { 109 | // Generate the swagger url 110 | let url = document.location.pathname; 111 | if(url.endsWith("/api-docs")) { 112 | url = url.replace('/api-docs', ''); 113 | } 114 | if( ! url.endsWith("/")) { 115 | url += "/"; 116 | } 117 | swagger_url = url + "swagger" + document.location.search; 118 | log('swagger_url: ' + swagger_url); 119 | 120 | // Build a Swagger UI 121 | const ui = SwaggerUIBundle({ 122 | url: swagger_url, 123 | dom_id: '#swagger-ui', 124 | deepLinking: true, 125 | presets: [ 126 | SwaggerUIBundle.presets.apis, 127 | SwaggerUIStandalonePreset 128 | ], 129 | plugins: [ 130 | SwaggerUIBundle.plugins.DownloadUrl, 131 | GrlcLayoutPlugin 132 | ], 133 | layout: "GrlcLayout", 134 | validatorUrl: undefined, // required ?? 135 | supportedSubmitMethods: ['get', 'post', 'put', 'delete', 'patch'], 136 | onComplete: grlcOnComplete, 137 | onFailure: grlcOnFailure, 138 | docExpansion: "list", 139 | apisSorter: "alpha", 140 | showRequestHeaders: true 141 | }); 142 | window.ui = ui; 143 | } 144 | -------------------------------------------------------------------------------- /src/static/js/grlc-layout.js: -------------------------------------------------------------------------------- 1 | // SPDX-FileCopyrightText: 2022 Albert Meroño, Rinke Hoekstra, Carlos Martínez 2 | // 3 | // SPDX-License-Identifier: MIT 4 | 5 | // Create the layout component 6 | class GrlcLayout extends React.Component { 7 | render() { 8 | const { 9 | getComponent 10 | } = this.props; 11 | 12 | const BaseLayout = getComponent("BaseLayout", true); 13 | 14 | // layoutJSX is the compiled version of JSX code: 15 | // (
) 16 | // Compiled using babeljs.io as suggested by https://reactjs.org/docs/react-without-jsx.html 17 | const layoutJSX = React.createElement( 18 | "div", null, 19 | React.createElement(BaseLayout, null) 20 | ); 21 | return layoutJSX; 22 | } 23 | } 24 | 25 | // Create the plugin that provides our layout component 26 | const GrlcLayoutPlugin = () => { 27 | return { 28 | components: { 29 | GrlcLayout: GrlcLayout 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/static/toolinfo.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "grlc", 3 | "title" : "git repository linked data API constructor", 4 | "description" : "grlc makes all your Linked Data accessible to the Web by automatically converting your SPARQL queries into RESTful APIs.", 5 | "url" : "http://grlc.io", 6 | "keywords" : "tools, SPARQL, query, API, Linked Data, LOD", 7 | "author" : "Albert Meroño Peñuela", 8 | "repository" : "https://github.com/clariah/grlc" 9 | } 10 | -------------------------------------------------------------------------------- /src/swagger.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2022 Albert Meroño, Rinke Hoekstra, Carlos Martínez 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | import json 6 | import grlc.utils 7 | import grlc.gquery as gquery 8 | import grlc.pagination as pageUtils 9 | from grlc.fileLoaders import GithubLoader, LocalLoader, URLLoader, GitlabLoader 10 | 11 | import grlc.glogging as glogging 12 | 13 | glogger = glogging.getGrlcLogger(__name__) 14 | 15 | 16 | def get_blank_spec(): 17 | """Creates the base (blank) structure of swagger specification.""" 18 | swag = {} 19 | swag["swagger"] = "2.0" 20 | swag["schemes"] = ( 21 | [] 22 | ) # 'http' or 'https' -- leave blank to make it dependent on how UI is loaded 23 | swag["paths"] = {} 24 | swag["definitions"] = {"Message": {"type": "string"}} 25 | return swag 26 | 27 | 28 | def get_repo_info(loader, sha, prov_g): 29 | """Generate swagger information from the repo being used.""" 30 | user_repo = loader.getFullName() 31 | repo_title = loader.getRepoTitle() 32 | repo_desc = loader.getRepoDescription() 33 | contact_name = loader.getContactName() 34 | contact_url = loader.getContactUrl() 35 | commit_list = loader.getCommitList() 36 | licence_url = loader.getLicenceURL() # This will be None if there is no license 37 | 38 | # Add the API URI as a used entity by the activity 39 | if prov_g: 40 | prov_g.add_used_entity(loader.getRepoURI()) 41 | 42 | prev_commit = None 43 | next_commit = None 44 | version = sha if sha else commit_list[0] 45 | if commit_list.index(version) < len(commit_list) - 1: 46 | prev_commit = commit_list[commit_list.index(version) + 1] 47 | if commit_list.index(version) > 0: 48 | next_commit = commit_list[commit_list.index(version) - 1] 49 | 50 | info = { 51 | "version": version, 52 | "title": repo_title, 53 | "description": repo_desc, 54 | "contact": {"name": contact_name, "url": contact_url}, 55 | } 56 | if licence_url: 57 | info["license"] = {"name": "License", "url": licence_url} 58 | 59 | if type(loader) is GithubLoader: 60 | basePath = "/api-git/" + user_repo + "/" 61 | basePath += ("subdir/" + loader.subdir + "/") if loader.subdir else "" 62 | basePath += ("commit/" + sha + "/") if sha else "" 63 | if type(loader) is GitlabLoader: 64 | basePath = "/api-gitlab/" + user_repo + "/query/" 65 | basePath += ("branch/" + loader.branch + "/") if loader.branch else "" 66 | basePath += ( 67 | ("subdir/" + loader.subdir.strip("/") + "/") if loader.subdir else "" 68 | ) 69 | basePath += ("commit/" + sha + "/") if sha else "" 70 | elif type(loader) is LocalLoader: 71 | basePath = "/api-local/" 72 | elif type(loader) is URLLoader: 73 | basePath = "/api-url/" 74 | else: 75 | # TODO: raise error 76 | glogger.error("Cannot set basePath, loader type unkown") 77 | 78 | return prev_commit, next_commit, info, basePath 79 | 80 | 81 | def get_path_for_item(item): 82 | """Builds the swagger definition for a specific path, based on 83 | the given item.""" 84 | query = item["original_query"] 85 | if isinstance(query, dict): 86 | if "grlc" in query: 87 | del query["grlc"] 88 | query = "\n" + json.dumps(query, indent=2) + "\n" 89 | 90 | description = item["description"] 91 | description += "\n\n```\n{}\n```".format(query) 92 | description += ( 93 | "\n\nSPARQL transformation:\n```json\n{}```".format(item["transform"]) 94 | if "transform" in item 95 | else "" 96 | ) 97 | 98 | item_path = { 99 | item["method"]: { 100 | "tags": item["tags"], 101 | "summary": item["summary"], 102 | "description": description, 103 | "produces": ["text/csv", "application/json", "text/html"], 104 | "parameters": item["params"] if "params" in item else None, 105 | "responses": { 106 | "200": { 107 | "description": "Query response", 108 | "schema": { 109 | "type": "array", 110 | "items": { 111 | "type": "object", 112 | "properties": ( 113 | item["item_properties"] 114 | if "item_properties" in item 115 | else None 116 | ), 117 | }, 118 | }, 119 | }, 120 | "default": { 121 | "description": "Unexpected error", 122 | "schema": {"$ref": "#/definitions/Message"}, 123 | }, 124 | }, 125 | } 126 | } 127 | return item_path 128 | 129 | 130 | def build_spec( 131 | user, 132 | repo, 133 | subdir=None, 134 | query_url=None, 135 | sha=None, 136 | prov=None, 137 | extraMetadata=[], 138 | git_type=None, 139 | branch=None, 140 | ): 141 | """Build grlc specification for the given github user / repo.""" 142 | loader = grlc.utils.getLoader( 143 | user, 144 | repo, 145 | subdir, 146 | query_url, 147 | sha=sha, 148 | prov=prov, 149 | git_type=git_type, 150 | branch=branch, 151 | ) 152 | 153 | files = loader.fetchFiles() 154 | raw_repo_uri = loader.getRawRepoUri() 155 | 156 | # Fetch all .rq files 157 | items = [] 158 | warnings = [] 159 | 160 | allowed_ext = ["rq", "sparql", "json", "tpf"] 161 | for c in files: 162 | glogger.debug(">>>>>>>>>>>>>>>>>>>>>>>>>c_name: {}".format(c["name"])) 163 | extension = c["name"].split(".")[-1] 164 | if ( 165 | extension in allowed_ext or query_url 166 | ): # parameter provided queries may not have extension 167 | 168 | item, warning = _buildItem( 169 | c, extension, query_url, raw_repo_uri, loader, extraMetadata 170 | ) 171 | if item: 172 | items.append(item) 173 | if warning: 174 | warnings.append(warning) 175 | 176 | # Add a warning if no license is found 177 | if loader.getLicenceURL() is None: 178 | warnings.append( 179 | "Queries behind this API do not have a license. You may not be allowed to use them." 180 | ) 181 | 182 | return items, warnings 183 | 184 | 185 | def _buildItem(c, extension, query_url, raw_repo_uri, loader, extraMetadata): 186 | """Collect all the information required to build an item from a file in a repository.""" 187 | item = None 188 | warning = None 189 | 190 | call_name = c["name"].split(".")[0] 191 | 192 | # Retrieve extra metadata from the query decorators 193 | query_text = loader.getTextFor(c) 194 | 195 | if extension == "json": 196 | query_text = json.loads(query_text) 197 | # Validate loaded json is an actual query. 198 | # If it isn't, do not process it further and item is not built 199 | if not grlc.utils.SPARQLTransformer_validJSON(query_text): 200 | glogger.debug( 201 | "===================================================================" 202 | ) 203 | glogger.debug("JSON file not a SPARQL query: {}".format(c["name"])) 204 | glogger.debug( 205 | "===================================================================" 206 | ) 207 | return item, warning 208 | 209 | if extension in ["rq", "sparql", "json"] or query_url: 210 | glogger.debug( 211 | "===================================================================" 212 | ) 213 | glogger.debug("Processing SPARQL query: {}".format(c["name"])) 214 | glogger.debug( 215 | "===================================================================" 216 | ) 217 | try: 218 | item = process_sparql_query_text( 219 | query_text, loader, call_name, extraMetadata 220 | ) 221 | except Exception as e: 222 | warning = str(e) 223 | elif "tpf" == extension: 224 | glogger.debug( 225 | "===================================================================" 226 | ) 227 | glogger.debug("Processing TPF query: {}".format(c["name"])) 228 | glogger.debug( 229 | "===================================================================" 230 | ) 231 | item = process_tpf_query_text( 232 | query_text, raw_repo_uri, call_name, extraMetadata 233 | ) 234 | # TODO: raise exceptions in process_tpf_query_text 235 | else: 236 | glogger.info("Ignoring unsupported source call name: {}".format(c["name"])) 237 | 238 | return item, warning 239 | 240 | 241 | def process_tpf_query_text(query_text, raw_repo_uri, call_name, extraMetadata): 242 | """Generates a swagger specification item based on the given TPF query file.""" 243 | query_metadata = gquery.get_yaml_decorators(query_text) 244 | 245 | tags = query_metadata["tags"] if "tags" in query_metadata else [] 246 | glogger.debug("Read query tags: " + ", ".join(tags)) 247 | 248 | summary = query_metadata["summary"] if "summary" in query_metadata else "" 249 | glogger.debug("Read query summary: " + summary) 250 | 251 | description = ( 252 | query_metadata["description"] if "description" in query_metadata else "" 253 | ) 254 | glogger.debug("Read query description: " + description) 255 | 256 | method = query_metadata["method"].lower() if "method" in query_metadata else "get" 257 | if method not in ["get", "post", "head", "put", "delete", "options", "connect"]: 258 | method = "get" 259 | 260 | pagination = query_metadata["pagination"] if "pagination" in query_metadata else "" 261 | glogger.debug("Read query pagination: " + str(pagination)) 262 | 263 | endpoint = query_metadata["endpoint"] if "endpoint" in query_metadata else "" 264 | glogger.debug("Read query endpoint: " + endpoint) 265 | 266 | # If this query allows pagination, add page number as parameter 267 | params = [] 268 | if pagination: 269 | params.append(pageUtils.getSwaggerPaginationDef(pagination)) 270 | 271 | item = packItem( 272 | "/" + call_name, 273 | method, 274 | tags, 275 | summary, 276 | description, 277 | params, 278 | query_metadata, 279 | extraMetadata, 280 | ) 281 | 282 | return item 283 | 284 | 285 | def process_sparql_query_text(query_text, loader, call_name, extraMetadata): 286 | """Generates a swagger specification item based on the given SPARQL query file.""" 287 | # We get the endpoint name first, since some query metadata fields (eg enums) require it 288 | endpoint, _ = gquery.guess_endpoint_uri(query_text, loader) 289 | glogger.debug("Read query endpoint: {}".format(endpoint)) 290 | 291 | try: 292 | query_metadata = gquery.get_metadata(query_text, endpoint) 293 | except Exception as e: 294 | raise Exception("Could not parse query {}: {}".format(call_name, str(e))) 295 | 296 | tags, summary, description, method, pagination, endpoint_in_url = unpack_metadata( 297 | query_metadata 298 | ) 299 | 300 | # Processing of the parameters 301 | params = [] 302 | 303 | # If this query allows pagination, add page number as parameter 304 | if pagination: 305 | params.append(pageUtils.getSwaggerPaginationDef(pagination)) 306 | 307 | if endpoint_in_url: 308 | params.append(pack_endpoint(endpoint)) 309 | 310 | # If this is a URL generated spec we need to force API calls with the specUrl parameter set 311 | if type(loader) is URLLoader: 312 | params.append(pack_specURL(loader)) 313 | 314 | # ONLY SELECT CONSTRUTCT AND INSERT CURRENTLY SUPPORTED! 315 | if query_metadata["type"] in ["SelectQuery", "ConstructQuery", "InsertData"]: 316 | for _, p in query_metadata["parameters"].items(): 317 | params.append(build_parameter(p)) 318 | elif query_metadata["type"] == "UNKNOWN": 319 | glogger.warning( 320 | "grlc could not parse this query; assuming a plain, non-parametric SELECT in the API spec" 321 | ) 322 | else: 323 | # TODO: process all other kinds of queries 324 | glogger.debug( 325 | "Could not parse query {}: Query of type {} is currently unsupported".format( 326 | call_name, query_metadata["type"] 327 | ) 328 | ) 329 | raise Exception( 330 | "Could not parse query {}: Query of type {} is currently unsupported".format( 331 | call_name, query_metadata["type"] 332 | ) 333 | ) 334 | 335 | # Finally: main structure of the callname spec 336 | item = packItem( 337 | "/" + call_name, 338 | method, 339 | tags, 340 | summary, 341 | description, 342 | params, 343 | query_metadata, 344 | extraMetadata, 345 | ) 346 | 347 | return item 348 | 349 | 350 | def unpack_metadata(query_metadata): 351 | tags = query_metadata["tags"] if "tags" in query_metadata else [] 352 | 353 | summary = query_metadata["summary"] if "summary" in query_metadata else "" 354 | 355 | description = ( 356 | query_metadata["description"] if "description" in query_metadata else "" 357 | ) 358 | 359 | method = query_metadata["method"].lower() if "method" in query_metadata else "" 360 | if method not in ["get", "post", "head", "put", "delete", "options", "connect"]: 361 | if query_metadata["type"] == "InsertData": 362 | method = "post" 363 | else: 364 | method = "get" 365 | 366 | pagination = query_metadata["pagination"] if "pagination" in query_metadata else "" 367 | 368 | endpoint_in_url = ( 369 | query_metadata["endpoint_in_url"] 370 | if "endpoint_in_url" in query_metadata 371 | else True 372 | ) 373 | return tags, summary, description, method, pagination, endpoint_in_url 374 | 375 | 376 | def build_parameter(p): 377 | param = {} 378 | param["name"] = p["name"] 379 | param["type"] = p["type"] 380 | param["required"] = p["required"] 381 | param["in"] = "query" 382 | # TODO: can we simplify the description 383 | param["description"] = ( 384 | "A value of type {} that will substitute {} in the original query".format( 385 | p["type"], p["original"] 386 | ) 387 | ) 388 | if "lang" in p: 389 | param["description"] = ( 390 | "A value of type {}@{} that will substitute {} in the original query".format( 391 | p["type"], p["lang"], p["original"] 392 | ) 393 | ) 394 | if "format" in p: 395 | param["format"] = p["format"] 396 | param["description"] = ( 397 | "A value of type {} ({}) that will substitute {} in the original query".format( 398 | p["type"], p["format"], p["original"] 399 | ) 400 | ) 401 | if "enum" in p: 402 | param["enum"] = p["enum"] 403 | if "default" in p: 404 | param["default"] = p["default"] 405 | return param 406 | 407 | 408 | def pack_endpoint(endpoint): 409 | endpoint_param = {} 410 | endpoint_param["name"] = "endpoint" 411 | endpoint_param["type"] = "string" 412 | endpoint_param["in"] = "query" 413 | endpoint_param["description"] = "Alternative endpoint for SPARQL query" 414 | endpoint_param["default"] = endpoint 415 | return endpoint_param 416 | 417 | 418 | def pack_specURL(loader): 419 | specUrl_param = {} 420 | specUrl_param["name"] = "specUrl" 421 | specUrl_param["type"] = "string" 422 | specUrl_param["in"] = "query" 423 | specUrl_param["description"] = "URL of the API specification" 424 | specUrl_param["default"] = loader.getRawRepoUri() 425 | return specUrl_param 426 | 427 | 428 | def packItem( 429 | call_name, method, tags, summary, description, params, query_metadata, extraMetadata 430 | ): 431 | """Generate a swagger specification item using all the given parameters.""" 432 | item = { 433 | "call_name": call_name, 434 | "method": method, 435 | "tags": tags, 436 | "summary": summary, 437 | "description": description, 438 | "params": params, 439 | "item_properties": None, 440 | "query": query_metadata["query"], 441 | "original_query": query_metadata.get("original_query", query_metadata["query"]), 442 | } 443 | 444 | for extraField in extraMetadata: 445 | if extraField in query_metadata: 446 | item[extraField] = query_metadata[extraField] 447 | 448 | return item 449 | 450 | 451 | def get_warning_div(warn): 452 | return '
{}
'.format(warn) 453 | -------------------------------------------------------------------------------- /src/templates/api-docs.html: -------------------------------------------------------------------------------- 1 | 6 | 7 | 8 | 9 | 10 | 11 | grlc 12 | 13 | 14 | 15 | 36 | 46 | 47 | 48 | 49 | 59 | 60 |
61 | 62 |
63 |
64 |
65 |
66 |

67 | 68 | 69 | 70 | | 71 | 72 | 73 |

74 |
75 | 76 |
77 |
78 |
79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 107 | 108 | 109 | -------------------------------------------------------------------------------- /src/templates/index.html: -------------------------------------------------------------------------------- 1 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | grlc 19 | 20 | 21 | 22 | 23 | 24 | 28 | 38 | 39 | 40 | 41 | 42 |
43 | 44 |
45 | 46 |
47 | 48 |
49 |
50 |

grlc

51 | 59 |
60 |
61 | 62 |
63 | 64 |

grlc makes all your Linked Data accessible to the Web by automatically converting your SPARQL queries into RESTful APIs. With (almost) no effort! Simply:

65 |

1

66 |

Create a GitHub repository, and store all your SPARQL queries in there (like in this example). If you don't have a GitHub account, go get one. You can also just write down the username and the repository name of somebody else :-)

67 |

2

68 |

Go to the address bar of this page, and append /api/github_username/repository_name to it. So if I want the API derived from GitHub's username foo and repository bar, I append /api/foo/bar/ to the domain name (/api/foo/bar/api-docs will work too). Now hit enter. Done!

69 |

70 | Show me an example SPARQL repo 71 | Show me the equivalent API 72 |

73 |
74 | 75 |
76 |

Take a look at the increasing number of users and SPARQL repositories on GitHub that are using grlc to generate APIs!

77 |
78 | 79 |
80 |

Find out more in GitHub and in this paper.

81 |
82 | 83 |
84 |
85 |

Adapted from cover template for Bootstrap, by @mdo.

86 |
87 |
88 | 89 |
90 | 91 |
92 | 93 |
94 | 95 | 97 | 98 | 99 | 100 | 101 | 102 | -------------------------------------------------------------------------------- /src/utils.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2022 Albert Meroño, Rinke Hoekstra, Carlos Martínez 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | import grlc.static as static 6 | import grlc.gquery as gquery 7 | import grlc.pagination as pageUtils 8 | import grlc.swagger as swagger 9 | from grlc.prov import grlcPROV 10 | from grlc.fileLoaders import GithubLoader, LocalLoader, URLLoader, GitlabLoader 11 | from grlc.queryTypes import qType 12 | from grlc import __version__ as grlc_version 13 | 14 | import re 15 | import requests 16 | import json 17 | 18 | from rdflib import Graph 19 | 20 | import SPARQLTransformer 21 | 22 | import grlc.glogging as glogging 23 | 24 | glogger = glogging.getGrlcLogger(__name__) 25 | 26 | 27 | def getLoader( 28 | user, 29 | repo, 30 | subdir=None, 31 | spec_url=None, 32 | sha=None, 33 | prov=None, 34 | git_type=None, 35 | branch=None, 36 | ): 37 | """Build a fileLoader (LocalLoader, GithubLoader, URLLoader) for the given parameters.""" 38 | if user is None and repo is None and not spec_url: 39 | loader = LocalLoader() 40 | elif spec_url: 41 | loader = URLLoader(spec_url) 42 | else: 43 | if git_type == static.TYPE_GITHUB: 44 | glogger.debug("Building GithubLoader....") 45 | loader = GithubLoader(user, repo, subdir, sha, prov) 46 | else: 47 | glogger.debug("Building GitlabLoader....") 48 | loader = GitlabLoader(user, repo, subdir, sha, prov, branch) 49 | return loader 50 | 51 | 52 | def build_spec(user, repo, subdir=None, sha=None, prov=None, extraMetadata=[]): 53 | """Build grlc specification for the given github user / repo. 54 | 55 | Deprecated.""" 56 | glogger.warning( 57 | "grlc.utils.build_spec is deprecated and will " 58 | "be removed in the future. Use grlc.swagger.build_spec instead." 59 | ) 60 | items, _ = swagger.build_spec(user, repo, subdir, sha, prov, extraMetadata) 61 | return items 62 | 63 | 64 | def build_swagger_spec( 65 | user, repo, subdir, spec_url, sha, serverName, git_type, branch=None 66 | ): 67 | """Build grlc specification for the given github user / repo in swagger format.""" 68 | if user and repo: 69 | # Init provenance recording 70 | prov_g = grlcPROV(user, repo) 71 | else: 72 | prov_g = None 73 | 74 | swag = swagger.get_blank_spec() 75 | swag["host"] = serverName 76 | 77 | try: 78 | loader = getLoader(user, repo, subdir, spec_url, sha, prov_g, git_type, branch) 79 | except Exception as e: 80 | # If repo does not exits 81 | swag["info"] = {"title": "ERROR!", "description": str(e)} 82 | swag["paths"] = {} 83 | return swag 84 | 85 | prev_commit, next_commit, info, basePath = swagger.get_repo_info( 86 | loader, sha, prov_g 87 | ) 88 | swag["prev_commit"] = prev_commit 89 | swag["next_commit"] = next_commit 90 | swag["info"] = info 91 | swag["basePath"] = basePath 92 | 93 | # TODO: can we pass loader to build_spec ? --> Ideally yes! 94 | spec, warnings = swagger.build_spec( 95 | user, repo, subdir, spec_url, sha, prov_g, [], git_type, branch 96 | ) 97 | # Use items to build API paths 98 | for item in spec: 99 | swag["paths"][item["call_name"]] = swagger.get_path_for_item(item) 100 | 101 | # TODO: Add bootstrap style to top level HTML 102 | # Without a better place to display warnings, we can make them part of the description. 103 | if "description" not in swag["info"] or swag["info"]["description"] is None: 104 | swag["info"]["description"] = "" 105 | for warn in warnings: 106 | swag["info"]["description"] += swagger.get_warning_div(warn) 107 | 108 | if prov_g: 109 | prov_g.end_prov_graph() 110 | swag["prov"] = prov_g.serialize(format="turtle") 111 | return swag 112 | 113 | 114 | def dispatch_query( 115 | user, 116 | repo, 117 | query_name, 118 | subdir=None, 119 | spec_url=None, 120 | sha=None, 121 | content=None, 122 | requestArgs={}, 123 | acceptHeader="application/json", 124 | requestUrl="http://", 125 | formData={}, 126 | method="POST", 127 | git_type=None, 128 | branch=None, 129 | ): 130 | """Executes the specified SPARQL or TPF query.""" 131 | loader = getLoader( 132 | user, 133 | repo, 134 | subdir, 135 | spec_url, 136 | sha=sha, 137 | prov=None, 138 | git_type=git_type, 139 | branch=branch, 140 | ) 141 | query, q_type = loader.getTextForName(query_name) 142 | 143 | # Call name implemented with SPARQL query 144 | if q_type == qType["SPARQL"] or q_type == qType["JSON"]: 145 | resp, status, headers = dispatchSPARQLQuery( 146 | query, 147 | loader, 148 | requestArgs, 149 | acceptHeader, 150 | content, 151 | formData, 152 | requestUrl, 153 | method, 154 | ) 155 | 156 | if acceptHeader == "application/json": 157 | # TODO: transform JSON result if suitable 158 | pass 159 | 160 | return resp, status, headers 161 | # Call name implemented with TPF query 162 | elif q_type == qType["TPF"]: 163 | resp, status, headers = dispatchTPFQuery(query, loader, acceptHeader, content) 164 | return resp, status, headers 165 | else: 166 | return ( 167 | "Couldn't find a SPARQL, RDF dump, or TPF query with the requested name", 168 | 404, 169 | {}, 170 | ) 171 | 172 | 173 | def _dispatchQueryDump( 174 | raw_sparql_query, endpoint, mime_type, rewritten_query, acceptHeader, content 175 | ): 176 | glogger.debug( 177 | "Detected {} MIME type, proceeding with locally loading remote dump".format( 178 | mime_type 179 | ) 180 | ) 181 | 182 | g = Graph() 183 | try: 184 | g.parse(endpoint, format=mime_type) 185 | glogger.debug( 186 | "Local RDF graph loaded successfully with {} triples".format(len(g)) 187 | ) 188 | except Exception as e: 189 | glogger.error(e) 190 | 191 | results = g.query(rewritten_query, result="sparql") 192 | 193 | # Prepare return format as requested 194 | if "application/json" in acceptHeader or ( 195 | content and "application/json" in static.mimetypes[content] 196 | ): 197 | resp = results.serialize(format="json") 198 | code = 200 199 | glogger.debug( 200 | "Results of SPARQL query against locally loaded dump: {}".format(resp) 201 | ) 202 | elif "text/csv" in acceptHeader or ( 203 | content and "text/csv" in static.mimetypes[content] 204 | ): 205 | resp = results.serialize(format="csv") 206 | code = 200 207 | glogger.debug( 208 | "Results of SPARQL query against locally loaded dump: {}".format(resp) 209 | ) 210 | else: 211 | resp = "Unacceptable requested format" 212 | code = 415 213 | headers = {} 214 | glogger.debug("Finished processing query against RDF dump, end of use case") 215 | del g 216 | return resp, code, headers 217 | 218 | 219 | def _dispatchQueryInsert( 220 | method, rewritten_query, formData, acceptHeader, endpoint, auth, headers 221 | ): 222 | glogger.debug("Processing INSERT query") 223 | if method != "POST": 224 | glogger.debug("INSERT queries must use POST method") 225 | return {"error": "INSERT queries must use POST method"}, 400, headers 226 | 227 | # Rewrite INSERT 228 | rewritten_query = rewritten_query.replace("?_g_iri", "{}".format(formData.get("g"))) 229 | rewritten_query = rewritten_query.replace("

", formData.get("data")) 230 | glogger.debug("INSERT query rewritten as {}".format(rewritten_query)) 231 | 232 | # Prepare HTTP POST request 233 | reqHeaders = { 234 | "Accept": acceptHeader, 235 | "Content-Type": "application/sparql-update", 236 | } 237 | response = requests.post( 238 | endpoint, data=rewritten_query, headers=reqHeaders, auth=auth 239 | ) 240 | glogger.debug("Response header from endpoint: " + response.headers["Content-Type"]) 241 | 242 | # Response headers 243 | resp = response.text 244 | code = 200 245 | headers["Content-Type"] = response.headers["Content-Type"] 246 | 247 | return resp, code, headers 248 | 249 | 250 | def _dispatchQuerySelect( 251 | acceptHeader, content, rewritten_query, endpoint, auth, headers, endpoint_method 252 | ): 253 | reqHeaders = {"Accept": acceptHeader, "Content-Type": "application/sparql-query"} 254 | if content: 255 | reqHeaders = { 256 | "Accept": static.mimetypes[content], 257 | "Content-Type": "application/sparql-query", 258 | } 259 | 260 | glogger.debug("Sending HTTP request to SPARQL endpoint") 261 | glogger.debug("... w/params: {}".format(rewritten_query)) 262 | glogger.debug("... w/headers: {}".format(reqHeaders)) 263 | glogger.debug("... w/auth: {}".format(auth)) 264 | glogger.debug("... via: {}".format(endpoint_method)) 265 | 266 | try: 267 | if endpoint_method == "GET": 268 | data = {"query": rewritten_query} 269 | response = requests.get( 270 | endpoint, params=data, headers=reqHeaders, auth=auth 271 | ) 272 | else: 273 | response = requests.post( 274 | endpoint, data=rewritten_query, headers=reqHeaders, auth=auth 275 | ) 276 | # Response headers 277 | resp = response.text 278 | code = 200 279 | glogger.debug( 280 | "Response header from endpoint: " + response.headers["Content-Type"] 281 | ) 282 | except Exception as e: 283 | # Error contacting SPARQL endpoint 284 | glogger.debug("Exception encountered while connecting to SPARQL endpoint") 285 | return {"error": str(e)}, 400, headers 286 | 287 | glogger.debug("Got HTTP response from to SPARQL endpoint: {}".format(resp)) 288 | headers["Content-Type"] = response.headers["Content-Type"] 289 | 290 | return resp, code, headers 291 | 292 | 293 | def _dispatchTransformerPostprocess(query_metadata, resp): 294 | if "proto" in query_metadata: 295 | resp = SPARQLTransformer.post_process( 296 | json.loads(resp), query_metadata["proto"], query_metadata["opt"] 297 | ) 298 | else: # case ("transform" in query_metadata and acceptHeader == "application/json") 299 | if "@graph" in query_metadata["transform"]: # SPARQLTransformer for JSON-LD 300 | graph = query_metadata["transform"]["@graph"] 301 | proto = graph[0] if isinstance(graph, list) else graph 302 | rq = query_metadata["transform"] 303 | else: # SPARQLTransformer for standard JSON 304 | proto = query_metadata["transform"] 305 | rq = {"proto": proto} 306 | 307 | _, _, opt = SPARQLTransformer.pre_process(rq) 308 | resp = SPARQLTransformer.post_process(json.loads(resp), proto, opt) 309 | return resp 310 | 311 | 312 | def dispatchSPARQLQuery( 313 | raw_sparql_query, 314 | loader, 315 | requestArgs, 316 | acceptHeader, 317 | content, 318 | formData, 319 | requestUrl, 320 | method="GET", 321 | ): 322 | """Executes the specified SPARQL query.""" 323 | endpoint, auth = gquery.guess_endpoint_uri(raw_sparql_query, loader) 324 | if endpoint == "": 325 | return "No SPARQL endpoint indicated", 407, {} 326 | 327 | glogger.debug("=====================================================") 328 | glogger.debug("Sending query to SPARQL endpoint: {}".format(endpoint)) 329 | glogger.debug("=====================================================") 330 | 331 | try: 332 | query_metadata = gquery.get_metadata(raw_sparql_query, endpoint) 333 | except Exception as e: 334 | # extracting metadata 335 | return {"error": str(e)}, 400, {} 336 | 337 | acceptHeader = ( 338 | "application/json" if isinstance(raw_sparql_query, dict) else acceptHeader 339 | ) 340 | pagination = query_metadata["pagination"] if "pagination" in query_metadata else "" 341 | endpoint_method = ( 342 | query_metadata["endpoint-method"] 343 | if "endpoint-method" in query_metadata 344 | else "POST" 345 | ) 346 | rewritten_query = query_metadata["query"] 347 | 348 | # Rewrite query using parameter values 349 | if ( 350 | query_metadata["type"] == "SelectQuery" 351 | or query_metadata["type"] == "ConstructQuery" 352 | ): 353 | rewritten_query = gquery.rewrite_query( 354 | query_metadata["original_query"], query_metadata["parameters"], requestArgs 355 | ) 356 | 357 | # Rewrite query using pagination 358 | if query_metadata["type"] == "SelectQuery" and "pagination" in query_metadata: 359 | rewritten_query = gquery.paginate_query( 360 | rewritten_query, query_metadata["pagination"], requestArgs 361 | ) 362 | 363 | resp = None 364 | code = 0 365 | headers = {} 366 | 367 | # If we have a mime field, we load the remote dump and query it locally 368 | if "mime" in query_metadata and query_metadata["mime"]: 369 | resp, code, headers = _dispatchQueryDump( 370 | raw_sparql_query, 371 | endpoint, 372 | query_metadata["mime"], 373 | rewritten_query, 374 | acceptHeader, 375 | content, 376 | ) 377 | 378 | # Check for INSERT/POST 379 | elif query_metadata["type"] == "InsertData": 380 | resp, code, headers = _dispatchQueryInsert( 381 | method, rewritten_query, formData, acceptHeader, endpoint, auth, headers 382 | ) 383 | 384 | # If there's no mime type, the endpoint is an actual SPARQL endpoint 385 | else: 386 | resp, code, headers = _dispatchQuerySelect( 387 | acceptHeader, 388 | content, 389 | rewritten_query, 390 | endpoint, 391 | auth, 392 | headers, 393 | endpoint_method, 394 | ) 395 | 396 | # If the query is paginated, set link HTTP headers 397 | if pagination: 398 | # Get number of total results 399 | count = gquery.count_query_results(rewritten_query, endpoint) 400 | pageArg = requestArgs.get("page", None) 401 | headerLink = pageUtils.buildPaginationHeader( 402 | count, pagination, pageArg, requestUrl 403 | ) 404 | headers["Link"] = headerLink 405 | 406 | if "proto" in query_metadata or ( 407 | "transform" in query_metadata and acceptHeader == "application/json" 408 | ): 409 | resp = _dispatchTransformerPostprocess(query_metadata, resp) 410 | 411 | headers["Server"] = "grlc/" + grlc_version 412 | return resp, code, headers 413 | 414 | 415 | def dispatchTPFQuery(raw_tpf_query, loader, acceptHeader, content): 416 | """Executes the specified TPF query.""" 417 | endpoint, auth = gquery.guess_endpoint_uri(raw_tpf_query, loader) 418 | glogger.debug("=====================================================") 419 | glogger.debug("Sending query to TPF endpoint: {}".format(endpoint)) 420 | glogger.debug("=====================================================") 421 | 422 | # TODO: pagination for TPF 423 | 424 | # Preapre HTTP request 425 | reqHeaders = { 426 | "Accept": acceptHeader, 427 | "Authorization": "token {}".format(static.SPARQL_ACCESS_TOKEN), 428 | } 429 | if content: 430 | reqHeaders = { 431 | "Accept": static.mimetypes[content], 432 | "Authorization": "token {}".format(static.SPARQL_ACCESS_TOKEN), 433 | } 434 | tpf_list = re.split("\n|=", raw_tpf_query) 435 | subject = tpf_list[tpf_list.index("subject") + 1] 436 | predicate = tpf_list[tpf_list.index("predicate") + 1] 437 | object = tpf_list[tpf_list.index("object") + 1] 438 | data = {"subject": subject, "predicate": predicate, "object": object} 439 | 440 | response = requests.get(endpoint, params=data, headers=reqHeaders, auth=auth) 441 | glogger.debug("Response header from endpoint: " + response.headers["Content-Type"]) 442 | 443 | # Response headers 444 | resp = response.text 445 | headers = {} 446 | headers["Content-Type"] = response.headers["Content-Type"] 447 | headers["Server"] = "grlc/" + grlc_version 448 | return resp, 200, headers 449 | 450 | 451 | def SPARQLTransformer_validJSON(json_file): 452 | """Validate json file (loaded into Python as a dict) is a valid query for 453 | SPARQLTransformer (see https://github.com/D2KLab/py-sparql-transformer/issues/13). 454 | """ 455 | return ("@graph" in json_file) or ("proto" in json_file) 456 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2022 Albert Meroño, Rinke Hoekstra, Carlos Martínez 2 | # 3 | # SPDX-License-Identifier: MIT 4 | -------------------------------------------------------------------------------- /tests/mock_data.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2022 Albert Meroño, Rinke Hoekstra, Carlos Martínez 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | from mock import Mock 6 | from os import path 7 | from glob import glob 8 | 9 | from collections import namedtuple 10 | from grlc.fileLoaders import LocalLoader 11 | from grlc import static 12 | 13 | import base64 14 | 15 | static.GITHUB_ACCESS_TOKEN = ( 16 | "fake-token" # Manually overwrite access token to avoid empty token 17 | ) 18 | 19 | base_url = path.join("tests", "repo") 20 | 21 | 22 | def buildGHEntry(entryName): 23 | entryName = entryName.replace(base_url, "") 24 | 25 | # Named tuple containing properties of mocked github ContentFile 26 | MockGithubContentFile = namedtuple( 27 | "MockGithubContentFile", "download_url name path type decoded_content" 28 | ) 29 | return MockGithubContentFile( 30 | download_url=entryName, 31 | name=entryName, 32 | path=entryName, 33 | type="file", 34 | decoded_content="FAKE FILE CONTENT".encode(), # Because Github ContentFile object contains bytes. 35 | ) 36 | 37 | 38 | def buildGLEntry(entryName): 39 | entryName = entryName.replace(base_url, "") 40 | 41 | return {"type": "blob", "name": entryName} 42 | 43 | 44 | mock_gh_files = [buildGHEntry(f) for f in glob(path.join(base_url, "*"))] 45 | mock_gl_files = [buildGLEntry(f) for f in glob(path.join(base_url, "*"))] 46 | 47 | 48 | class MockGithubRepo: 49 | def get_contents(self, filename, ref=None): 50 | if filename == "": 51 | return mock_gh_files 52 | else: 53 | for f in mock_gh_files: 54 | if filename in f.name: # filenames contain extra / 55 | return f 56 | return None 57 | 58 | 59 | class MockGitlabModule: 60 | def __init__(self) -> None: 61 | gl_repo = Mock() 62 | 63 | gl_repo.repository_tree = Mock(return_value=mock_gl_files) 64 | gl_repo.files.get.side_effect = self.gl_files_content 65 | gl_repo.default_branch = "main" 66 | 67 | self.projects = Mock() 68 | self.projects.get.return_value = gl_repo 69 | 70 | def gl_files_content(self, file_path, ref): 71 | """Returns none if the file is not in the known repo""" 72 | for glf in mock_gl_files: 73 | if file_path in glf["name"]: # filenames contain extra / 74 | f = Mock() 75 | f_content = "The text of a file" 76 | f.content = base64.b64encode(f_content.encode("utf-8")) 77 | return f 78 | return None 79 | 80 | 81 | def mock_requestsUrl(url, headers={}, params={}): 82 | url = url.replace("http://example.org/", "tests/repo/") 83 | f = open(url, "r") 84 | lines = f.readlines() 85 | text = "".join(lines) 86 | return_value = Mock(status_code=200) 87 | return_value.text = text 88 | 89 | return return_value 90 | 91 | 92 | mock_simpleSparqlResponse = { 93 | "head": {"link": [], "vars": ["p", "o"]}, 94 | "results": { 95 | "bindings": [ 96 | { 97 | "p": {"type": "string", "value": "p1"}, 98 | "o": {"type": "string", "value": "o1"}, 99 | }, 100 | { 101 | "p": {"type": "string", "value": "p2"}, 102 | "o": {"type": "string", "value": "o2"}, 103 | }, 104 | { 105 | "p": {"type": "string", "value": "p3"}, 106 | "o": {"type": "string", "value": "o3"}, 107 | }, 108 | { 109 | "p": {"type": "string", "value": "p4"}, 110 | "o": {"type": "string", "value": "o4"}, 111 | }, 112 | { 113 | "p": {"type": "string", "value": "p5"}, 114 | "o": {"type": "string", "value": "o5"}, 115 | }, 116 | ] 117 | }, 118 | } 119 | 120 | 121 | def mock_process_sparql_query_text(query_text, raw_repo_uri, call_name, extraMetadata): 122 | mockItem = {"status": "This is a mock item", "call_name": call_name} 123 | return mockItem 124 | 125 | 126 | filesInRepo = [ 127 | { 128 | "name": "fakeFile1.rq", 129 | "download_url": "https://example.org/path/to/fakeFile.rq", 130 | "decoded_content": "CONTENT ?".encode(), # Because Github ContentFile object contains bytes. 131 | }, 132 | { 133 | "name": "fakeJSONFile1.json", 134 | "download_url": "https://example.org/path/to/fakeJSONFile1.json", 135 | "decoded_content": '{ "x": "y" }'.encode(), # Because Github ContentFile object contains bytes. 136 | }, 137 | ] 138 | 139 | mockLoader = LocalLoader(base_url) 140 | -------------------------------------------------------------------------------- /tests/repo/endpoint.txt: -------------------------------------------------------------------------------- 1 | http://test-endpoint/from-file/sparql 2 | -------------------------------------------------------------------------------- /tests/repo/test-endpoint-get.rq: -------------------------------------------------------------------------------- 1 | #+ summary: Sample query for testing SPARQL endpoint method 2 | #+ endpoint: "http://test-endpoint/transform/sparql/" 3 | #+ transform: { 4 | #+ "key": "?p", 5 | #+ "value": "?o", 6 | #+ "$anchor": "key" 7 | #+ } 8 | #+ endpoint-method: GET 9 | 10 | select ?p ?o where { 11 | ?_id_iri ?p ?o 12 | } LIMIT 5 13 | -------------------------------------------------------------------------------- /tests/repo/test-enum.rq: -------------------------------------------------------------------------------- 1 | #+ summary: Test query for containing pre-defined enumeration. 2 | #+ enumerate: 3 | #+ - o: 4 | #+ - v1 5 | #+ - v2 6 | 7 | SELECT * WHERE { 8 | ?s ?p ?_o . 9 | } 10 | -------------------------------------------------------------------------------- /tests/repo/test-json.json: -------------------------------------------------------------------------------- 1 | { 2 | "proto": { 3 | "id": "?id", 4 | "class": "$rdf:type$required$var:?class", 5 | "label": "$rdfs:label$required", 6 | "o": "?o" 7 | }, 8 | "$prefixes": { 9 | "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", 10 | "rdfs": "http://www.w3.org/2000/01/rdf-schema#", 11 | "dbo": "http://dbpedia.org/ontology/" 12 | }, 13 | "$limit": 100, 14 | "grlc": { 15 | "summary": "Testing DBpedia endpoint", 16 | "endpoint": "http://dbpedia.org/sparql", 17 | "tags": [ 18 | "dbpedia" 19 | ], 20 | "method": "GET", 21 | "pagination": 50, 22 | "defaults": { 23 | "type": "http://dbpedia.org/ontology/Band" 24 | } 25 | } 26 | } -------------------------------------------------------------------------------- /tests/repo/test-projection.rq: -------------------------------------------------------------------------------- 1 | #+ summary: Sample query for testing response transformation 2 | #+ endpoint: "http://test-endpoint/transform/sparql/" 3 | #+ transform: { 4 | #+ "key": "?p", 5 | #+ "value": "?o", 6 | #+ "$anchor": "key" 7 | #+ } 8 | 9 | select ?p ?o where { 10 | ?_id_iri ?p ?o 11 | } LIMIT 5 12 | -------------------------------------------------------------------------------- /tests/repo/test-rq.rq: -------------------------------------------------------------------------------- 1 | #+ summary: This test query contains and endpoint decorator and a query with 2 | #+ some variables with different types using BASIL syntax. 3 | #+ endpoint: http://test-endpoint/from-decorator/sparql 4 | 5 | SELECT * WHERE { 6 | ?s ?p ?_o1 . 7 | ?s ?p ?_o2_iri . 8 | ?s ?p ?_o3_number . 9 | ?s ?p ?_o4_literal . 10 | ?s ?p ?_o5_en . 11 | ?s ?p ?_o6_integer . 12 | ?s ?p ?_o7_xsd_date . 13 | } 14 | -------------------------------------------------------------------------------- /tests/repo/test-sparql-jsonconf.sparql: -------------------------------------------------------------------------------- 1 | #+ { 2 | #+ "summary": "Different types of annotations Adding extra text Just because summaries can be long.", 3 | #+ "tags": [ 4 | #+ "firstTag", 5 | #+ "secondTag" 6 | #+ ], 7 | #+ "endpoint": "http://example.com/sparql", 8 | #+ "method": "GET", 9 | #+ "pagination": 100, 10 | #+ "enumerate": [ 11 | #+ "var1", 12 | #+ "var2" 13 | #+ ] 14 | #+ } 15 | SELECT * 16 | WHERE { 17 | ?s ?p ?o 18 | } 19 | -------------------------------------------------------------------------------- /tests/repo/test-sparql.sparql: -------------------------------------------------------------------------------- 1 | #+ summary: Different types of annotations 2 | #+ Adding extra text 3 | #+ Just because summaries can be long. 4 | #+ tags: 5 | #+ - firstTag 6 | #+ - secondTag 7 | #+ endpoint: http://example.com/sparql 8 | #+ method: GET 9 | #+ pagination: 100 10 | #+ enumerate: 11 | #+ - var1 12 | #+ - var2 13 | SELECT * 14 | WHERE { 15 | ?s ?p ?o 16 | } 17 | -------------------------------------------------------------------------------- /tests/repo/test-tpf.tpf: -------------------------------------------------------------------------------- 1 | # Should add some text to make testable 2 | -------------------------------------------------------------------------------- /tests/repo/url.yml: -------------------------------------------------------------------------------- 1 | title: TestAPI 2 | contact: 3 | name: Name of the author 4 | url: http://example.org/contact.html 5 | licence: http://example.org/licence.html 6 | queries: 7 | - http://example.org/test-rq.rq 8 | - http://example.org/test-sparql.sparql 9 | - http://example.org/test-tpf.tpf 10 | - ./test-rq.rq 11 | -------------------------------------------------------------------------------- /tests/test_endpoints.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2022 Albert Meroño, Rinke Hoekstra, Carlos Martínez 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | import pytest 6 | from mock import patch 7 | from tests.mock_data import mockLoader, mock_requestsUrl 8 | from grlc.server import app 9 | 10 | 11 | @pytest.fixture(scope="class") 12 | def client(request): 13 | """Build http client""" 14 | with app.test_client() as client: 15 | yield client 16 | 17 | 18 | class TestGrlcHome: 19 | """Test all grlc server endpoints.""" 20 | 21 | def test_home(self, client): 22 | """Testing get from grlc home page""" 23 | rv = client.get("/") 24 | assert rv.status_code == 200 25 | assert "text/html" in rv.content_type 26 | body = str(object=rv.data, errors="strict") 27 | assert "grlc" in body 28 | assert ( 29 | "grlc generates RESTful APIs using SPARQL queries stored in GitHub repositories" 30 | in body 31 | ) 32 | 33 | 34 | class TestGrlcFrontEnd: 35 | """Test all grlc api front end generation (swagger html page).""" 36 | 37 | def validate(self, response): 38 | assert response.status_code == 200 39 | assert "text/html" in response.content_type 40 | body = str(object=response.data, errors="strict") 41 | assert '

' in body 42 | 43 | def test_repo(self, client): 44 | """...""" 45 | rv = client.get("/api-git/testuser/testrepo") 46 | self.validate(rv) 47 | 48 | def test_subdir(self, client): 49 | """...""" 50 | rv = client.get("/api-git/testuser/testrepo/subdir/") 51 | self.validate(rv) 52 | 53 | def test_commit(self, client): 54 | """...""" 55 | rv = client.get("/api-git/testuser/testrepo/commit/") 56 | self.validate(rv) 57 | 58 | def test_subdir_commit(self, client): 59 | """...""" 60 | rv = client.get("/api-git/testuser/testrepo/subdir//commit/") 61 | self.validate(rv) 62 | 63 | def test_local(self, client): 64 | """...""" 65 | rv = client.get("/api-local/") 66 | self.validate(rv) 67 | 68 | def test_url(self, client): 69 | """...""" 70 | rv = client.get("/api-url/?specUrl=") 71 | self.validate(rv) 72 | 73 | 74 | class TestGrlcSpec: 75 | """Test all grlc api spec generation.""" 76 | 77 | def validate(self, response): 78 | assert response.status_code == 200 79 | assert "application/json" in response.content_type 80 | spec = response.json 81 | assert spec["swagger"] == "2.0" 82 | assert "paths" in spec 83 | assert spec["info"]["title"] != "ERROR!" 84 | 85 | @patch("grlc.utils.getLoader") 86 | def test_repo(self, mock_loader, client): 87 | """...""" 88 | mock_loader.return_value = mockLoader 89 | 90 | rv = client.get("/api-git/testuser/testrepo/swagger") 91 | self.validate(rv) 92 | 93 | @patch("grlc.utils.getLoader") 94 | def test_subdir(self, mock_loader, client): 95 | """...""" 96 | mock_loader.return_value = mockLoader 97 | 98 | rv = client.get("/api-git/testuser/testrepo/subdir/testsubdir/swagger") 99 | self.validate(rv) 100 | 101 | @patch("grlc.utils.getLoader") 102 | def test_commit(self, mock_loader, client): 103 | """...""" 104 | mock_loader.return_value = mockLoader 105 | 106 | rv = client.get("/api-git/testuser/testrepo/commit/local/swagger") 107 | self.validate(rv) 108 | 109 | @patch("grlc.utils.getLoader") 110 | def test_subdir_commit(self, mock_loader, client): 111 | """...""" 112 | mock_loader.return_value = mockLoader 113 | 114 | rv = client.get( 115 | "/api-git/testuser/testrepo/subdir/testsubdir/commit/local/swagger" 116 | ) 117 | self.validate(rv) 118 | 119 | def test_local(self, client): 120 | """...""" 121 | rv = client.get("/api-local/swagger") 122 | self.validate(rv) 123 | 124 | @patch("requests.get", side_effect=mock_requestsUrl) 125 | def test_url(self, mock_get, client): 126 | """...""" 127 | rv = client.get("/api-url/swagger?specUrl=http://example.org/url.yml") 128 | self.validate(rv) 129 | 130 | 131 | class TestGrlcExec: 132 | """Test all grlc api execution endpoints.""" 133 | 134 | @classmethod 135 | def setup_class(self): 136 | query_response = [{"result": "mock"}] 137 | status = 200 138 | headers = {"Content-Type": "application/json"} 139 | self.mock_response = query_response, status, headers 140 | 141 | def validate(self, response): 142 | assert response.status_code == 200 143 | assert "application/json" in response.content_type 144 | assert len(response.json) > 0 145 | assert "result" in response.json[0] 146 | assert response.json[0]["result"] == "mock" 147 | 148 | @patch("grlc.utils.getLoader") 149 | @patch("grlc.utils.dispatch_query") 150 | def test_repo(self, mock_dispatch, mock_loader, client): 151 | """...""" 152 | mock_dispatch.return_value = self.mock_response 153 | rv = client.get( 154 | "/api-git/testuser/testrepo/query_name", 155 | headers={"Accept": "application/json"}, 156 | ) 157 | self.validate(rv) 158 | 159 | @patch("grlc.utils.getLoader") 160 | @patch("grlc.utils.dispatch_query") 161 | def test_subdir(self, mock_dispatch, mock_loader, client): 162 | """...""" 163 | mock_dispatch.return_value = self.mock_response 164 | 165 | # Check types of data passed to make_response. 166 | # If jsonify(dict) fixes the issue, patch make_response to jsonify(query_response) before 167 | # returning data to rv. 168 | rv = client.get( 169 | "/api-git/testuser/testrepo/subdir/testsubdir/query_name", 170 | headers={"accept": "application/json"}, 171 | ) 172 | self.validate(rv) 173 | 174 | @patch("grlc.utils.getLoader") 175 | @patch("grlc.utils.dispatch_query") 176 | def test_commit(self, mock_dispatch, mock_loader, client): 177 | """...""" 178 | mock_dispatch.return_value = self.mock_response 179 | 180 | rv = client.get( 181 | "/api-git/testuser/testrepo/commit/local/query_name", 182 | headers={"accept": "application/json"}, 183 | ) 184 | self.validate(rv) 185 | 186 | @patch("grlc.utils.getLoader") 187 | @patch("grlc.utils.dispatch_query") 188 | def test_subdir_commit(self, mock_dispatch, mock_loader, client): 189 | """...""" 190 | mock_dispatch.return_value = self.mock_response 191 | 192 | rv = client.get( 193 | "/api-git/testuser/testrepo/subdir/testsubdir/commit/local/query_name", 194 | headers={"accept": "application/json"}, 195 | ) 196 | self.validate(rv) 197 | 198 | @patch("grlc.utils.dispatch_query") 199 | def test_local(self, mock_dispatch, client): 200 | """...""" 201 | mock_dispatch.return_value = self.mock_response 202 | 203 | rv = client.get("/api-local/query_name", headers={"accept": "application/json"}) 204 | self.validate(rv) 205 | 206 | @patch("requests.get", side_effect=mock_requestsUrl) 207 | @patch("grlc.utils.dispatch_query") 208 | def test_url(self, mock_dispatch, mock_get, client): 209 | """...""" 210 | mock_dispatch.return_value = self.mock_response 211 | 212 | rv = client.get( 213 | "/api-url/?specUrl=http://example.org/url.yml", 214 | headers={"accept": "application/json"}, 215 | ) 216 | self.validate(rv) 217 | -------------------------------------------------------------------------------- /tests/test_gquery.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2022 Albert Meroño, Rinke Hoekstra, Carlos Martínez 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | import unittest 6 | import six 7 | import rdflib 8 | from mock import patch, Mock 9 | 10 | from tests.mock_data import mockLoader 11 | 12 | import grlc.gquery as gquery 13 | 14 | from flask import Flask 15 | 16 | 17 | class TestGQuery(unittest.TestCase): 18 | @classmethod 19 | def setUpClass(self): 20 | self.loader = mockLoader 21 | self.app = Flask("unittests") 22 | 23 | def test_guess_endpoint(self): 24 | with self.app.test_request_context( 25 | "/?endpoint=http://url-endpoint/from-url/sparql" 26 | ): 27 | endpoint, _ = gquery.guess_endpoint_uri("", self.loader) 28 | self.assertIn("from-url", endpoint, "Should match endpoint given in url") 29 | 30 | with self.app.test_request_context("/"): 31 | endpoint, _ = gquery.guess_endpoint_uri("", self.loader) 32 | self.assertIn( 33 | "from-file", endpoint, "Should match endpoint in endpoint.txt" 34 | ) 35 | 36 | rq, _ = self.loader.getTextForName("test-rq") 37 | endpoint, _ = gquery.guess_endpoint_uri(rq, self.loader) 38 | self.assertIn( 39 | "from-decorator", endpoint, "Should match endpoint in test-rq.rq" 40 | ) 41 | 42 | def test_get_parameters(self): 43 | rq, _ = self.loader.getTextForName("test-rq") 44 | 45 | params = gquery.get_parameters(rq, "", {}) 46 | 47 | self.assertGreaterEqual(len(params), 7, "Should find some parameters") 48 | for paramName, param in params.items(): 49 | self.assertIn("name", param, "Should have a name") 50 | self.assertIn("type", param, "Should have a type") 51 | self.assertIn("required", param, "Should have a required") 52 | 53 | orig = param["original"] 54 | if "_iri" in orig: 55 | self.assertEqual(param["type"], "string", "Should be type string") 56 | self.assertEqual(param["format"], "iri", "Should be format iri") 57 | if "_number" in orig: 58 | self.assertEqual(param["type"], "number", "Should be type number") 59 | if "_literal" in orig: 60 | self.assertEqual(param["type"], "literal", "Should be type literal") 61 | if "_en" in orig: 62 | self.assertEqual(param["type"], "string", "Should be type literal") 63 | self.assertEqual(param["lang"], "en", "Should be en language") 64 | if "_integer" in orig: 65 | self.assertEqual( 66 | param["datatype"], "xsd:integer", "Should be type xsd:integer" 67 | ) 68 | if "_xsd_date" in orig: 69 | self.assertEqual( 70 | param["datatype"], "xsd:date", "Should be type xsd:date" 71 | ) 72 | 73 | self.assertEqual(params["o1"]["type"], "string", "o1 should be a string") 74 | self.assertEqual(params["o2"]["format"], "iri", "o2 should be format iri") 75 | self.assertEqual(params["o3"]["type"], "number", "o3 should be a number") 76 | self.assertEqual(params["o4"]["type"], "literal", "o4 should be a literal") 77 | self.assertEqual(params["o5"]["lang"], "en", "o5 should be a English") 78 | self.assertEqual( 79 | params["o6"]["datatype"], "xsd:integer", "o6 should be a integer" 80 | ) 81 | self.assertEqual(params["o7"]["datatype"], "xsd:date", "o7 should be a date") 82 | 83 | @patch("requests.get") 84 | def test_get_enumeration(self, mock_get): 85 | mock_get.return_value = Mock(ok=True) 86 | mock_get.return_value.json.return_value = { 87 | "results": {"bindings": [{"o1": {"value": "v1"}}, {"o1": {"value": "v2"}}]} 88 | } 89 | 90 | rq, _ = self.loader.getTextForName("test-rq") 91 | metadata = {"enumerate": "o1"} 92 | enumeration = gquery.get_enumeration( 93 | rq, "o1", "http://mock-endpoint/sparql", metadata 94 | ) 95 | self.assertIsInstance(enumeration, list, "Should return a list of values") 96 | self.assertEqual(len(enumeration), 2, "Should have two elements") 97 | 98 | def test_get_static_enumeration(self): 99 | rq, _ = self.loader.getTextForName("test-enum") 100 | 101 | metadata = gquery.get_yaml_decorators(rq) 102 | self.assertIn("enumerate", metadata, "Should contain enumerate") 103 | 104 | enumeration = gquery.get_enumeration( 105 | rq, "o", "http://mock-endpoint/sparql", metadata 106 | ) 107 | self.assertIsInstance(enumeration, list, "Should return a list of values") 108 | self.assertEqual(len(enumeration), 2, "Should have two elements") 109 | 110 | def test_get_yaml_decorators(self): 111 | rq, _ = self.loader.getTextForName("test-sparql") 112 | 113 | decorators = gquery.get_yaml_decorators(rq) 114 | 115 | # Query always exist -- the rest must be present on the file. 116 | self.assertIn("query", decorators, "Should have a query field") 117 | self.assertIn("summary", decorators, "Should have a summary field") 118 | self.assertIn("pagination", decorators, "Should have a pagination field") 119 | self.assertIn("enumerate", decorators, "Should have a enumerate field") 120 | 121 | self.assertIsInstance( 122 | decorators["summary"], six.string_types, "Summary should be text" 123 | ) 124 | self.assertIsInstance( 125 | decorators["pagination"], int, "Pagination should be numeric" 126 | ) 127 | self.assertIsInstance( 128 | decorators["enumerate"], list, "Enumerate should be a list" 129 | ) 130 | 131 | def test_get_json_decorators(self): 132 | rq, _ = self.loader.getTextForName("test-sparql-jsonconf") 133 | 134 | decorators = gquery.get_yaml_decorators(rq) 135 | 136 | # Query always exist -- the rest must be present on the file. 137 | self.assertIn("query", decorators, "Should have a query field") 138 | self.assertIn("summary", decorators, "Should have a summary field") 139 | self.assertIn("pagination", decorators, "Should have a pagination field") 140 | self.assertIn("enumerate", decorators, "Should have a enumerate field") 141 | 142 | self.assertIsInstance( 143 | decorators["summary"], six.string_types, "Summary should be text" 144 | ) 145 | self.assertIsInstance( 146 | decorators["pagination"], int, "Pagination should be numeric" 147 | ) 148 | self.assertIsInstance( 149 | decorators["enumerate"], list, "Enumerate should be a list" 150 | ) 151 | 152 | def test_get_metadata(self): 153 | rq, _ = self.loader.getTextForName("test-sparql") 154 | 155 | metadata = gquery.get_metadata(rq, "") 156 | self.assertIn("type", metadata, "Should have a type field") 157 | self.assertIn("variables", metadata, "Should have a variables field") 158 | self.assertEqual(metadata["type"], "SelectQuery", "Should be type SelectQuery") 159 | self.assertIsInstance( 160 | metadata["variables"], list, "Should be a list of variables" 161 | ) 162 | for var in metadata["variables"]: 163 | self.assertIsInstance( 164 | var, rdflib.term.Variable, "Should be of type Variable" 165 | ) 166 | 167 | def test_paginate_query(self): 168 | rq, _ = self.loader.getTextForName("test-sparql") 169 | 170 | rq_pag = gquery.paginate_query(rq, 100, {}) 171 | 172 | self.assertNotIn("LIMIT", rq, "Original query should not contain LIMIT keyword") 173 | self.assertIn("LIMIT", rq_pag, "Paginated query should contain LIMIT keyword") 174 | self.assertNotIn( 175 | "OFFSET", rq, "Original query should not contain OFFSET keyword" 176 | ) 177 | self.assertIn("OFFSET", rq_pag, "Paginated query should contain OFFSET keyword") 178 | 179 | @staticmethod 180 | def build_get_parameter(origName, rwName): 181 | """Builds parameter description in the format returned by gquery.get_parameters""" 182 | return { 183 | "original": "?_{}".format(origName), 184 | "name": rwName, 185 | "required": False, 186 | "enum": [], 187 | "type": "literal", 188 | "datatype": "xsd:string", 189 | "lang": "en", 190 | "format": None, 191 | } 192 | 193 | def test_rewrite_query(self): 194 | rq, _ = self.loader.getTextForName("test-rq") 195 | # Parameters on the format returned by gquery.get_parameters 196 | parameters = { 197 | "o1": self.build_get_parameter("o1", "x1"), 198 | "o2": self.build_get_parameter("o2", "x2"), 199 | "o3": self.build_get_parameter("o3", "x3"), 200 | "o4": self.build_get_parameter("o4", "x4"), 201 | "o5": self.build_get_parameter("o5", "x5"), 202 | "o6": self.build_get_parameter("o6", "x6"), 203 | "o7": self.build_get_parameter("o7", "x7"), 204 | } 205 | args = { 206 | "o1": "x1", 207 | "o2": "x2", 208 | "o3": "x3", 209 | "o4": "x4", 210 | "o5": "x5", 211 | "o6": "x6", 212 | "o7": "x7", 213 | } 214 | # Rewritten query will probably be incorrect because parameters are not 215 | # carefully constructed, but that is not the scope of this test 216 | rq_rw = gquery.rewrite_query(rq, parameters, args) 217 | 218 | for pName, pValue in parameters.items(): 219 | self.assertIn( 220 | pName, rq, "Original query should contain original parameter name" 221 | ) 222 | self.assertNotIn( 223 | pName, 224 | rq_rw, 225 | "Rewritten query should not contain original parameter name", 226 | ) 227 | self.assertNotIn( 228 | pValue["name"], 229 | rq, 230 | "Original query should not contain replacement parameter value", 231 | ) 232 | self.assertIn( 233 | pValue["name"], 234 | rq_rw, 235 | "Rewritten query should contain replacement parameter value", 236 | ) 237 | 238 | 239 | if __name__ == "__main__": 240 | unittest.main() 241 | -------------------------------------------------------------------------------- /tests/test_grlc.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2022 Albert Meroño, Rinke Hoekstra, Carlos Martínez 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | import unittest 6 | from mock import patch 7 | 8 | from tests.mock_data import mock_process_sparql_query_text, filesInRepo 9 | 10 | 11 | class TestGrlc(unittest.TestCase): 12 | """Test grlc has been installed""" 13 | 14 | def test_grlc(self): 15 | import grlc 16 | 17 | 18 | class TestGrlcLib(unittest.TestCase): 19 | """Test grlc can be used as a library""" 20 | 21 | @patch("github.Github.get_repo") # Corresponding patch object: mockGithubRepo 22 | @patch( 23 | "grlc.utils.GithubLoader.fetchFiles" 24 | ) # Corresponding patch object: mockLoaderFiles 25 | @patch( 26 | "grlc.swagger.process_sparql_query_text", 27 | side_effect=mock_process_sparql_query_text, 28 | ) 29 | def test_build_spec(self, mockQueryText, mockLoaderFiles, mockGithubRepo): 30 | mockLoaderFiles.return_value = filesInRepo 31 | mockGithubRepo.return_value = [] 32 | 33 | """Using grlc as a library""" 34 | import grlc.swagger as swagger 35 | 36 | user = "testuser" 37 | repo = "testrepo" 38 | spec, warning = swagger.build_spec(user=user, repo=repo, git_type="github") 39 | 40 | # Repo contains one JSON file which is not a query, and should be ignored 41 | self.assertEqual(len(spec), len(filesInRepo) - 1) 42 | 43 | 44 | if __name__ == "__main__": 45 | unittest.main() 46 | -------------------------------------------------------------------------------- /tests/test_loaders.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2022 Albert Meroño, Rinke Hoekstra, Carlos Martínez 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | import unittest 6 | import six 7 | from mock import patch 8 | from os import path 9 | 10 | from grlc.fileLoaders import LocalLoader, GithubLoader, GitlabLoader, URLLoader 11 | from grlc.queryTypes import qType 12 | 13 | from tests.mock_data import MockGithubRepo, MockGitlabModule, mock_requestsUrl 14 | 15 | 16 | class TestGithubLoader(unittest.TestCase): 17 | @classmethod 18 | @patch("grlc.fileLoaders.Github.get_repo", return_value=MockGithubRepo()) 19 | def setUpClass(self, mocked_repo): 20 | self.user = "fakeuser" 21 | self.repo = "fakerepo" 22 | self.loader = GithubLoader( 23 | self.user, self.repo, subdir=None, sha=None, prov=None 24 | ) 25 | 26 | def test_fetchFiles(self): 27 | files = self.loader.fetchFiles() 28 | 29 | # Should return a list of file items 30 | self.assertIsInstance(files, list, "Should return a list of file items") 31 | 32 | # Should have N files (where N=10) 33 | self.assertEqual(len(files), 10, "Should return correct number of files") 34 | 35 | # File items should have a download_url 36 | for fItem in files: 37 | self.assertIn( 38 | "download_url", fItem, "File items should have a download_url" 39 | ) 40 | 41 | def test_getRawRepoUri(self): 42 | repoUri = self.loader.getRawRepoUri() 43 | 44 | # Should be a string 45 | self.assertIsInstance(repoUri, six.string_types, "Should be a string") 46 | 47 | # For URI shoud contain user / repo 48 | self.assertIn(self.user, repoUri, "Should contain user") 49 | self.assertIn(self.repo, repoUri, "Should contain repo") 50 | 51 | def test_getTextFor(self): 52 | files = self.loader.fetchFiles() 53 | 54 | # the contents of each file 55 | for fItem in files: 56 | text = self.loader.getTextFor(fItem) 57 | 58 | # Should be some text 59 | self.assertIsInstance(text, six.string_types, "Should be some text") 60 | 61 | # Should be non-empty for existing items 62 | self.assertGreater(len(text), 0, "Should be non-empty") 63 | 64 | # Should raise exception for invalid file items 65 | with self.assertRaises( 66 | Exception, msg="Should raise exception for invalid file items" 67 | ): 68 | text = self.loader.getTextFor({}) 69 | 70 | def test_getTextForName(self): 71 | testableNames = [ 72 | ("test-rq", qType["SPARQL"]), 73 | ("test-sparql", qType["SPARQL"]), 74 | ("test-tpf", qType["TPF"]), 75 | ] 76 | for name, expectedType in testableNames: 77 | text, actualType = self.loader.getTextForName(name) 78 | self.assertEqual( 79 | expectedType, 80 | actualType, 81 | "Query type should match %s != %s" % (expectedType, actualType), 82 | ) 83 | 84 | def test_getEndpointText(self): 85 | endpoint = self.loader.getEndpointText() 86 | 87 | # Should be some text 88 | self.assertIsInstance(endpoint, six.string_types, "Should be some text") 89 | 90 | 91 | class TestGitlabLoader(unittest.TestCase): 92 | @classmethod 93 | @patch("grlc.fileLoaders.gitlab.Gitlab", return_value=MockGitlabModule()) 94 | def setUpClass(self, mocked_repo): 95 | self.user = "fakeuser" 96 | self.repo = "fakerepo" 97 | self.loader = GitlabLoader( 98 | self.user, self.repo, subdir=None, sha=None, prov=None 99 | ) 100 | 101 | def test_fetchFiles(self): 102 | files = self.loader.fetchFiles() 103 | 104 | # Should return a list of file items 105 | self.assertIsInstance(files, list, "Should return a list of file items") 106 | 107 | # Should have N files (where N=10) 108 | self.assertEqual(len(files), 10, "Should return correct number of files") 109 | 110 | # File items should have a download_url 111 | for fItem in files: 112 | self.assertIn( 113 | "download_url", fItem, "File items should have a download_url" 114 | ) 115 | 116 | def test_getRawRepoUri(self): 117 | repoUri = self.loader.getRawRepoUri() 118 | 119 | # Should be a string 120 | self.assertIsInstance(repoUri, six.string_types, "Should be a string") 121 | 122 | # For URI shoud contain user / repo 123 | self.assertIn(self.user, repoUri, "Should contain user") 124 | self.assertIn(self.repo, repoUri, "Should contain repo") 125 | 126 | def test_getTextFor(self): 127 | files = self.loader.fetchFiles() 128 | 129 | # the contents of each file 130 | for fItem in files: 131 | text = self.loader.getTextFor(fItem) 132 | 133 | # Should be some text 134 | self.assertIsInstance(text, six.string_types, "Should be some text") 135 | 136 | # Should be non-empty for existing items 137 | self.assertGreater(len(text), 0, "Should be non-empty") 138 | 139 | # Should raise exception for invalid file items 140 | with self.assertRaises( 141 | Exception, msg="Should raise exception for invalid file items" 142 | ): 143 | text = self.loader.getTextFor({}) 144 | 145 | def test_getTextForName(self): 146 | testableNames = [ 147 | ("test-rq", qType["SPARQL"]), 148 | ("test-sparql", qType["SPARQL"]), 149 | ("test-tpf", qType["TPF"]), 150 | ] 151 | for name, expectedType in testableNames: 152 | text, actualType = self.loader.getTextForName(name) 153 | self.assertEqual( 154 | expectedType, 155 | actualType, 156 | "Query type should match %s != %s" % (expectedType, actualType), 157 | ) 158 | 159 | def test_getEndpointText(self): 160 | endpoint = self.loader.getEndpointText() 161 | 162 | # Should be some text 163 | self.assertIsInstance(endpoint, six.string_types, "Should be some text") 164 | 165 | 166 | class TestLocalLoader(unittest.TestCase): 167 | @classmethod 168 | def setUpClass(self): 169 | self.loader = LocalLoader(path.join("tests", "repo")) 170 | 171 | def test_fetchFiles(self): 172 | files = self.loader.fetchFiles() 173 | 174 | # Should return a list of file items 175 | self.assertIsInstance(files, list, "Should return a list of file items") 176 | 177 | # Should have N files (where N=10) 178 | self.assertEqual(len(files), 10, "Should return correct number of files") 179 | 180 | # File items should have a download_url 181 | for fItem in files: 182 | self.assertIn( 183 | "download_url", fItem, "File items should have a download_url" 184 | ) 185 | 186 | def test_getRawRepoUri(self): 187 | repoUri = self.loader.getRawRepoUri() 188 | 189 | # Should be a string 190 | self.assertIsInstance(repoUri, six.string_types, "Should be a string") 191 | 192 | # For local repo, should be empty ? 193 | self.assertEqual(repoUri, "", "Should be an empty string") 194 | 195 | def test_getTextFor(self): 196 | files = self.loader.fetchFiles() 197 | 198 | # the contents of each file 199 | for fItem in files: 200 | text = self.loader.getTextFor(fItem) 201 | 202 | # Should be some text 203 | self.assertIsInstance(text, six.string_types, "Should be some text") 204 | 205 | # Should be non-empty for existing items 206 | self.assertGreater(len(text), 0, "Should be non-empty") 207 | 208 | # Should raise exception for invalid file items 209 | with self.assertRaises( 210 | Exception, msg="Should raise exception for invalid file items" 211 | ): 212 | text = self.loader.getTextFor({}) 213 | 214 | def test_getTextForName(self): 215 | testableNames = [ 216 | ("test-rq", qType["SPARQL"]), 217 | ("test-sparql", qType["SPARQL"]), 218 | ("test-tpf", qType["TPF"]), 219 | ] 220 | for name, expectedType in testableNames: 221 | text, actualType = self.loader.getTextForName(name) 222 | self.assertEqual( 223 | expectedType, 224 | actualType, 225 | "Query type should match %s != %s" % (expectedType, actualType), 226 | ) 227 | 228 | def test_getEndpointText(self): 229 | endpoint = self.loader.getEndpointText() 230 | 231 | # Should be some text 232 | self.assertIsInstance(endpoint, six.string_types, "Should be some text") 233 | 234 | 235 | class TestURLLoader(unittest.TestCase): 236 | @classmethod 237 | def setUp(self): 238 | self.patcher = patch( 239 | "grlc.fileLoaders.requests.get", side_effect=mock_requestsUrl 240 | ) 241 | self.patcher.start() 242 | 243 | @classmethod 244 | def tearDown(self): 245 | self.patcher.stop() 246 | 247 | @classmethod 248 | @patch("requests.get", side_effect=mock_requestsUrl) 249 | def setUpClass(self, x): 250 | self.specURL = "http://example.org/url.yml" 251 | self.loader = URLLoader(self.specURL) 252 | 253 | def test_fetchFiles(self): 254 | files = self.loader.fetchFiles() 255 | 256 | # Should return a list of file items 257 | self.assertIsInstance(files, list, "Should return a list of file items") 258 | 259 | # Should have N files (where N=3) 260 | self.assertEqual(len(files), 3, "Should return correct number of files") 261 | 262 | # File items should have a download_url 263 | for fItem in files: 264 | self.assertIn( 265 | "download_url", fItem, "File items should have a download_url" 266 | ) 267 | 268 | def test_getTextFor(self): 269 | files = self.loader.fetchFiles() 270 | 271 | # the contents of each file 272 | for fItem in files: 273 | text = self.loader.getTextFor(fItem) 274 | 275 | # Should be some text 276 | self.assertIsInstance(text, six.string_types, "Should be some text") 277 | 278 | # Should be non-empty for existing items 279 | self.assertGreater(len(text), 0, "Should be non-empty") 280 | 281 | # Should raise exception for invalid file items 282 | with self.assertRaises( 283 | Exception, msg="Should raise exception for invalid file items" 284 | ): 285 | text = self.loader.getTextFor({}) 286 | 287 | def test_getRawRepoUri(self): 288 | repoUri = self.loader.getRawRepoUri() 289 | 290 | # Should be a string 291 | self.assertIsInstance(repoUri, six.string_types, "Should be a string") 292 | 293 | # Should be the same one we used to create the repo 294 | self.assertIn( 295 | self.specURL, repoUri, "Should be the same URL it was initialized with" 296 | ) 297 | 298 | def test_getTextForName(self): 299 | testableNames = [ 300 | ("test-rq", qType["SPARQL"]), 301 | ("test-sparql", qType["SPARQL"]), 302 | ("test-tpf", qType["TPF"]), 303 | ] 304 | for name, expectedType in testableNames: 305 | text, actualType = self.loader.getTextForName(name) 306 | self.assertEqual( 307 | expectedType, 308 | actualType, 309 | "Query type should match %s != %s" % (expectedType, actualType), 310 | ) 311 | 312 | def test_getEndpointText(self): 313 | endpoint = self.loader.getEndpointText() 314 | 315 | # Should be some text 316 | self.assertIsInstance(endpoint, six.string_types, "Should be some text") 317 | 318 | 319 | if __name__ == "__main__": 320 | unittest.main() 321 | -------------------------------------------------------------------------------- /tests/test_swagger.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2022 Albert Meroño, Rinke Hoekstra, Carlos Martínez 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | # Run using `$ pytest -s` 6 | 7 | import unittest 8 | from mock import patch 9 | 10 | from grlc.swagger import build_spec 11 | 12 | from tests.mock_data import mock_process_sparql_query_text, filesInRepo 13 | 14 | 15 | class TestSwagger(unittest.TestCase): 16 | @patch("github.Github.get_repo") # Corresponding patch object: mockGithubRepo 17 | @patch( 18 | "grlc.utils.GithubLoader.fetchFiles" 19 | ) # Corresponding patch object: mockLoaderFiles 20 | @patch( 21 | "grlc.swagger.process_sparql_query_text", 22 | side_effect=mock_process_sparql_query_text, 23 | ) 24 | def test_github(self, mockQueryText, mockLoaderFiles, mockGithubRepo): 25 | mockLoaderFiles.return_value = filesInRepo 26 | mockGithubRepo.return_value = [] 27 | 28 | user = "testuser" 29 | repo = "testrepo" 30 | spec, warnings = build_spec(user, repo, git_type="github") 31 | 32 | # Repo contains one JSON file which is not a query, and should be ignored 33 | self.assertEqual(len(spec), len(filesInRepo) - 1) 34 | 35 | 36 | if __name__ == "__main__": 37 | unittest.main() 38 | -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- 1 | # SPDX-FileCopyrightText: 2022 Albert Meroño, Rinke Hoekstra, Carlos Martínez 2 | # 3 | # SPDX-License-Identifier: MIT 4 | 5 | import unittest 6 | from mock import patch, Mock 7 | import json 8 | 9 | import grlc.utils as utils 10 | 11 | from tests.mock_data import mock_simpleSparqlResponse, mockLoader 12 | 13 | 14 | class TestUtils(unittest.TestCase): 15 | @classmethod 16 | def setUpClass(self): 17 | self.loader = mockLoader 18 | 19 | @patch("requests.post") 20 | def test_sparql_transformer(self, mock_post): 21 | mock_json = { 22 | "head": {}, 23 | "results": { 24 | "bindings": [ 25 | { 26 | "id": { 27 | "type": "uri", 28 | "value": "http://www.w3.org/2001/XMLSchema#anyURI", 29 | }, 30 | "class": { 31 | "type": "uri", 32 | "value": "http://www.w3.org/2000/01/rdf-schema#Datatype", 33 | }, 34 | "v2": { 35 | "type": "literal", 36 | "xml:lang": "en", 37 | "value": "xsd:anyURI", 38 | }, 39 | }, 40 | { 41 | "id": { 42 | "type": "uri", 43 | "value": "http://www.w3.org/2001/XMLSchema#boolean", 44 | }, 45 | "class": { 46 | "type": "uri", 47 | "value": "http://www.w3.org/2000/01/rdf-schema#Datatype", 48 | }, 49 | "v2": { 50 | "type": "literal", 51 | "xml:lang": "en", 52 | "value": "xsd:boolean", 53 | }, 54 | }, 55 | ] 56 | }, 57 | } 58 | 59 | mock_post.return_value = Mock(ok=True) 60 | mock_post.return_value.headers = {"Content-Type": "application/json"} 61 | mock_post.return_value.text = json.dumps(mock_json) 62 | 63 | rq, _ = self.loader.getTextForName("test-json") 64 | 65 | self.assertIn("proto", rq) 66 | 67 | resp, status, headers = utils.dispatchSPARQLQuery( 68 | rq, 69 | self.loader, 70 | content=None, 71 | requestArgs={}, 72 | acceptHeader="application/json", 73 | requestUrl="http://mock-endpoint/sparql", 74 | formData={}, 75 | ) 76 | self.assertEqual(status, 200) 77 | self.assertIsInstance(resp, list) 78 | self.assertIn("http", resp[0]["id"]) 79 | 80 | def validateTestResponse(self, resp): 81 | self.assertIsInstance(resp, list, "Response should be a list") 82 | self.assertEqual(len(resp), 5, "Response should have 5 entries") 83 | for item in resp: 84 | self.assertTrue("key" in item, "Response items should contain a key") 85 | self.assertTrue("value" in item, "Response items should contain a value") 86 | keys = [item["key"] for item in resp] 87 | values = [item["value"] for item in resp] 88 | 89 | self.assertTrue( 90 | all(k in keys for k in ["p1", "p2", "p3", "p4", "p5"]), 91 | "Response should contain all known keys", 92 | ) 93 | self.assertTrue( 94 | all(v in values for v in ["o1", "o2", "o3", "o4", "o5"]), 95 | "Response should contain all known values", 96 | ) 97 | 98 | def setMockGetResponse(self): 99 | return_value = Mock(ok=True) 100 | return_value.headers = {"Content-Type": "application/json"} 101 | return_value.text = json.dumps(mock_simpleSparqlResponse) 102 | return return_value 103 | 104 | @patch("requests.post") 105 | def test_dispatch_SPARQL_query(self, mock_post): 106 | mock_post.return_value = self.setMockGetResponse() 107 | 108 | rq, _ = self.loader.getTextForName("test-projection") 109 | resp, status, headers = utils.dispatchSPARQLQuery( 110 | rq, 111 | self.loader, 112 | content=None, 113 | requestArgs={"id": "http://dbpedia.org/resource/Frida_Kahlo"}, 114 | acceptHeader="application/json", 115 | requestUrl="http://mock-endpoint/sparql", 116 | formData={}, 117 | ) 118 | self.validateTestResponse(resp) 119 | self.assertTrue( 120 | mock_post.called, "Should communicate with SPARQL endpoint via POST" 121 | ) 122 | 123 | @patch("requests.get") 124 | def test_dispatch_SPARQL_query_get(self, mock_get): 125 | """Test that communication with SPARQL endpoint goes via GET method 126 | When the endpoint-method decorator is present and set to GET.""" 127 | mock_get.return_value = self.setMockGetResponse() 128 | 129 | rq, _ = self.loader.getTextForName("test-endpoint-get") 130 | resp, status, headers = utils.dispatchSPARQLQuery( 131 | rq, 132 | self.loader, 133 | content=None, 134 | requestArgs={"id": "http://dbpedia.org/resource/Frida_Kahlo"}, 135 | acceptHeader="application/json", 136 | requestUrl="http://mock-endpoint/sparql", 137 | formData={}, 138 | ) 139 | self.validateTestResponse(resp) 140 | self.assertTrue( 141 | mock_get.called, "Should communicate with SPARQL endpoint via GET" 142 | ) 143 | 144 | @patch("grlc.utils.getLoader") 145 | @patch("requests.post") 146 | def test_dispatch_query(self, mock_post, mock_loader): 147 | mock_post.return_value = self.setMockGetResponse() 148 | mock_loader.return_value = self.loader 149 | 150 | resp, status, headers = utils.dispatch_query( 151 | None, 152 | None, 153 | "test-projection", 154 | requestArgs={"id": "http://dbpedia.org/resource/Frida_Kahlo"}, 155 | ) 156 | 157 | self.validateTestResponse(resp) 158 | self.assertNotEqual(status, 404) 159 | -------------------------------------------------------------------------------- /upstart/grlc-docker.conf: -------------------------------------------------------------------------------- 1 | description "grlc container" 2 | author "albert.merono@vu.nl" 3 | start on filesystem and started docker 4 | stop on runlevel [!2345] 5 | respawn 6 | script 7 | /usr/local/bin/docker-compose -f /home/amp/src/grlc/docker-compose.default.yml up 8 | end script 9 | pre-stop exec /usr/local/bin/docker-compose -f /home/amp/src/grlc/docker-compose.default.yml down 10 | -------------------------------------------------------------------------------- /upstart/grlc-docker.conf.license: -------------------------------------------------------------------------------- 1 | SPDX-FileCopyrightText: 2022 Albert Meroño, Rinke Hoekstra, Carlos Martínez 2 | 3 | SPDX-License-Identifier: MIT 4 | -------------------------------------------------------------------------------- /upstart/webhook.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # SPDX-FileCopyrightText: 2022 Albert Meroño, Rinke Hoekstra, Carlos Martínez 4 | # 5 | # SPDX-License-Identifier: MIT 6 | 7 | from flask import Flask 8 | from subprocess import call 9 | 10 | app = Flask(__name__) 11 | 12 | 13 | @app.route("/", methods=['POST']) 14 | def update(): 15 | print("Starting image update") 16 | call(['docker', 'pull', 'clariah/grlc:dev']) 17 | call(['docker-compose', '-f', '/home/amp/src/grlc-dev/docker-compose.default.yml', 'restart']) 18 | print("All done; exiting...") 19 | 20 | return 200 21 | 22 | 23 | if __name__ == '__main__': 24 | app.run(host='0.0.0.0', port=8004, debug=True) 25 | --------------------------------------------------------------------------------