├── .gitignore
├── .pre-commit-config.yaml
├── .travis.yml
├── DOCKER.md
├── Dockerfile
├── Dockerfile.dev
├── LICENSE.GPL
├── Procfile
├── README.md
├── Vagrantfile
├── ansible
    ├── host_vars
    │   └── vagrant
    ├── production
    ├── requirements.yml
    ├── roles
    │   ├── wikidp.portal
    │   │   ├── defaults
    │   │   │   └── main.yml
    │   │   ├── files
    │   │   │   ├── empty
    │   │   │   └── nginx
    │   │   │   │   └── etc
    │   │   │   │       └── nginx
    │   │   │   │           ├── nginx.conf
    │   │   │   │           └── proxy_params
    │   │   ├── handlers
    │   │   │   └── main.yml
    │   │   ├── meta
    │   │   │   └── main.yml
    │   │   ├── tasks
    │   │   │   ├── main.yml
    │   │   │   ├── nginx
    │   │   │   │   ├── configure.yml
    │   │   │   │   ├── main.yml
    │   │   │   │   ├── server.yml
    │   │   │   │   └── standalone.yml
    │   │   │   ├── portal
    │   │   │   │   ├── configure.yml
    │   │   │   │   ├── dependencies.yml
    │   │   │   │   ├── dirs.yml
    │   │   │   │   ├── install.yml
    │   │   │   │   ├── main.yml
    │   │   │   │   └── users.yml
    │   │   │   ├── service
    │   │   │   │   └── main.yml
    │   │   │   └── uwsgi
    │   │   │   │   ├── main.yml
    │   │   │   │   ├── server.yml
    │   │   │   │   └── standalone.yml
    │   │   ├── templates
    │   │   │   ├── nginx
    │   │   │   │   └── etc
    │   │   │   │   │   └── nginx
    │   │   │   │   │       ├── conf.d
    │   │   │   │   │           └── wikidp.conf.j2
    │   │   │   │   │       └── sites-available
    │   │   │   │   │           └── wikidp.j2
    │   │   │   ├── portal
    │   │   │   │   ├── bin
    │   │   │   │   │   ├── restart.sh.j2
    │   │   │   │   │   ├── show.sh.j2
    │   │   │   │   │   ├── start.sh.j2
    │   │   │   │   │   └── stop.sh.j2
    │   │   │   │   ├── home
    │   │   │   │   │   └── pywikibot
    │   │   │   │   │   │   └── user-config.py.j2
    │   │   │   │   └── install
    │   │   │   │   │   └── setup_venv.sh.j2
    │   │   │   └── uwsgi
    │   │   │   │   ├── etc
    │   │   │   │       └── systemd
    │   │   │   │       │   └── system
    │   │   │   │       │       └── wikidp.service.j2
    │   │   │   │   └── wikidp
    │   │   │   │       └── home
    │   │   │   │           ├── bin
    │   │   │   │               └── start.sh.j2
    │   │   │   │           └── wikidp.ini.j2
    │   │   ├── tests
    │   │   │   ├── inventory
    │   │   │   ├── test.yml
    │   │   │   └── vagrant.yml
    │   │   └── vars
    │   │   │   └── main.yml
    │   └── wikidp.wikibase
    │   │   ├── defaults
    │   │       └── main.yml
    │   │   ├── files
    │   │       ├── empty
    │   │       └── nginx
    │   │       │   └── etc
    │   │       │       └── nginx
    │   │       │           └── nginx.conf
    │   │   ├── handlers
    │   │       └── main.yml
    │   │   ├── meta
    │   │       └── main.yml
    │   │   ├── tasks
    │   │       ├── main.yml
    │   │       ├── nginx
    │   │       │   ├── configure.yml
    │   │       │   └── main.yml
    │   │       └── wikibase
    │   │       │   ├── dependencies.yml
    │   │       │   ├── main.yml
    │   │       │   ├── mediawiki.yml
    │   │       │   ├── php-fpm.yml
    │   │       │   └── wikibase.yml
    │   │   ├── templates
    │   │       ├── nginx
    │   │       │   └── etc
    │   │       │   │   └── nginx
    │   │       │   │       └── sites-available
    │   │       │   │           └── wikibase.conf.j2
    │   │       └── php
    │   │       │   └── etc
    │   │       │       └── php
    │   │       │           └── fpm
    │   │       │               ├── php-fpm.conf.j2
    │   │       │               ├── php.ini.j2
    │   │       │               └── pool.d
    │   │       │                   └── pool.conf.j2
    │   │   ├── tests
    │   │       ├── inventory
    │   │       ├── test.yml
    │   │       └── vagrant.yml
    │   │   └── vars
    │   │       └── main.yml
    ├── vagrant
    └── vagrant.yml
├── apicache-py3
    ├── 6bc28b771fb3b4881b11ac888717488cdd6acc33b08858c3cfd2880f4728b882
    ├── f2bd856f660527e525d4c5a8904717bb5c59d9de2f332d5f5982309888fe9cc7
    ├── f8d9172cb945cc1b53e4cac31649fc8ab1ce5b8cc64c5d1e8b7e8264392c4ba6
    └── fce830cb647985332a95ad135d86777a42b9d298ba90a1f56e4bc7904bbc31b3
├── coverage_report.sh
├── docker-dev.sh
├── docs
    ├── .gitkeep
    ├── CNAME
    ├── OAUTH-SETUP.md
    ├── getting-started
    │   └── index.md
    ├── index.md
    ├── oauth-1.png
    ├── oauth-2.png
    ├── oauth-3.png
    └── oauth-4.png
├── requirements.txt
├── run.sh
├── setup.py
├── setup.sh
├── tests
    ├── __init__.py
    ├── settings.py
    ├── test_controllers.py
    ├── test_reads.py
    └── test_routes.py
├── user-config.py.template
├── venv.sh
└── wikidp
    ├── __init__.py
    ├── config.py
    ├── const.py
    ├── controllers
        ├── __init__.py
        ├── api.py
        ├── auth.py
        ├── pages.py
        └── search.py
    ├── models.py
    ├── routes
        ├── __init__.py
        ├── _converters.py
        ├── _filters.py
        ├── api.py
        ├── forms.py
        ├── oauth.py
        ├── pages.py
        └── search.py
    ├── schemas
        ├── emulator.json
        ├── file_format
        │   ├── file_format_family.json
        │   ├── file_format_id_pattern.json
        │   ├── file_format_minimal.json
        │   └── file_format_references.json
        ├── operating_system.json
        └── software
        │   └── software_extended_references.json
    ├── sparql.py
    ├── static
        ├── .DS_Store
        ├── css
        │   └── wikidp.css
        ├── img
        │   ├── .DS_Store
        │   ├── favicon.ico
        │   └── logo.png
        ├── js
        │   ├── item.js
        │   ├── item_contribute.js
        │   ├── item_preview.js
        │   └── wikidp.js
        └── vendor
        │   ├── clipboard.min.js
        │   └── jquery.qlabel.js
    ├── templates
        ├── about.html
        ├── base.html
        ├── error.html
        ├── item.html
        ├── item_contribute.html
        ├── item_preview.html
        ├── login.html
        ├── navbar.html
        ├── page.html
        ├── profile.html
        ├── puid_results.html
        ├── reports.html
        ├── search_results.html
        ├── snippets
        │   ├── contribute_input_templates.html
        │   ├── item_sidebar.html
        │   ├── item_templates.html
        │   ├── macros.html
        │   ├── property_checklist.html
        │   └── search_form.html
        └── welcome.html
    └── utils
        ├── __init__.py
        └── wd_int_utils.py


/.gitignore:
--------------------------------------------------------------------------------
  1 | # Prevent people adding their username and password by mistake
  2 | run.sh
  3 | 
  4 | # Byte-compiled / optimized / DLL files
  5 | __pycache__/
  6 | *.py[cod]
  7 | *$py.class
  8 | 
  9 | # C extensions
 10 | *.so
 11 | 
 12 | # Distribution / packaging
 13 | .Python
 14 | env/
 15 | build/
 16 | develop-eggs/
 17 | dist/
 18 | downloads/
 19 | eggs/
 20 | .eggs/
 21 | lib/
 22 | lib64/
 23 | parts/
 24 | sdist/
 25 | var/
 26 | wheels/
 27 | *.egg-info/
 28 | .installed.cfg
 29 | *.egg
 30 | 
 31 | # PyInstaller
 32 | #  Usually these files are written by a python script from a template
 33 | #  before PyInstaller builds the exe, so as to inject date/other infos into it.
 34 | *.manifest
 35 | *.spec
 36 | 
 37 | # Installer logs
 38 | pip-log.txt
 39 | pip-delete-this-directory.txt
 40 | 
 41 | # Unit test / coverage reports
 42 | htmlcov/
 43 | .tox/
 44 | .coverage
 45 | .coverage.*
 46 | .cache
 47 | nosetests.xml
 48 | coverage.xml
 49 | *,cover
 50 | .hypothesis/
 51 | 
 52 | # Translations
 53 | *.mo
 54 | *.pot
 55 | 
 56 | # Django stuff:
 57 | *.log
 58 | local_settings.py
 59 | 
 60 | # Flask stuff:
 61 | instance/
 62 | .webassets-cache
 63 | 
 64 | # Scrapy stuff:
 65 | .scrapy
 66 | 
 67 | # Sphinx documentation
 68 | docs/_build/
 69 | 
 70 | # PyBuilder
 71 | target/
 72 | 
 73 | # Jupyter Notebook
 74 | .ipynb_checkpoints
 75 | 
 76 | # pyenv
 77 | .python-version
 78 | 
 79 | # celery beat schedule file
 80 | celerybeat-schedule
 81 | 
 82 | # SageMath parsed files
 83 | *.sage.py
 84 | 
 85 | # dotenv
 86 | .env
 87 | 
 88 | # virtualenv
 89 | .venv
 90 | venv/
 91 | ENV/
 92 | test_venv/
 93 | 
 94 | # Spyder project settings
 95 | .spyderproject
 96 | 
 97 | # Rope project settings
 98 | .ropeproject
 99 | 
100 | # mkdocs documentation
101 | /site
102 | 
103 | # PyCharm files
104 | .idea/
105 | 
106 | # Application odds and ends
107 | .pytest_cache/
108 | apicache/
109 | apicache-py3/
110 | throttle.ctrl
111 | user-config.py
112 | caches/
113 | wikidp/idjc-paper.pages
114 | pywikibot.lwp
115 | 
116 | # Ingore all vagrant artifacts
117 | .vagrant*
118 | 
119 | # Ignore Ansible build directories
120 | ansible/roles/nginxinc.nginx/
121 | ansible/roles/tersmitten.apt/
122 | ansible/roles/tersmitten.timezone/
123 | ansible/roles/tschifftner.hostname/
124 | 
125 | # Ignore Ansible retry files
126 | ansible/*.retry
127 | 
128 | # MAC OS Hidden Files
129 | .DS_Store
130 | 


--------------------------------------------------------------------------------
/.pre-commit-config.yaml:
--------------------------------------------------------------------------------
 1 | -   repo: git://github.com/pre-commit/pre-commit-hooks
 2 |     rev: v1.4.0
 3 |     hooks:
 4 |     -   id: check-byte-order-marker
 5 |     -   id: check-case-conflict
 6 |     -   id: check-docstring-first
 7 | #    -   id: end-of-file-fixer
 8 | #    -   id: trailing-whitespace
 9 | #    -   id: check-xml
10 | 


--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
 1 | language: python
 2 | 
 3 | python:
 4 |   - "3.6"
 5 | 
 6 | services:
 7 |   - docker
 8 | 
 9 | before_install:
10 |   - python --version
11 |   - pip install -U pip pytest pyinstaller pep257 twine codacy-coverage
12 | 
13 | install:
14 |   - pip install .[testing]
15 | 
16 | before_script:
17 |   # Pylint everything with the following tests disabled for now:
18 |   # Disabled in the .pylintrc config file
19 |   #  - C0103: Invalid name ( e.g. camel case)
20 |   # Disabled in the command below
21 |   #  - R0201: Method could be a function
22 |   #  - R0902: Too many instance attributes
23 |   #  - R0904: Too many public methods
24 |   #  - R0912: Too many branches
25 |   #  - R0913: Too many arguments
26 |   #  - R0914: Too many local variables
27 |   #  - R0915: Too many statements
28 |   #  - R1702: Too many nested blocks
29 |   #  - R1703: Simplifiable if statement
30 |   #  - W0511: Warning note in comments
31 |   #  - W0702: No Exception type specified
32 |   #  - W0703: Catching too general an Exception type
33 |   #  - C0200: Consider using enumerate instead of iterating with range
34 |   #  - C0302: Too many lines in module
35 |   - pylint setup.py ./wikidp/
36 | 
37 | script:
38 |   - pre-commit run --all-files --verbose
39 |   - pytest --cov=wikidp
40 |   - pep257 wikidp setup.py
41 | 
42 | after_success:
43 |   - coverage xml
44 |   - python-codacy-coverage -r coverage.xml
45 | 


--------------------------------------------------------------------------------
/DOCKER.md:
--------------------------------------------------------------------------------
 1 | Docker Instructions
 2 | ===================
 3 | There are two docker images for WikiDP, one for use as a local development server
 4 | the other as a demonstration server. Unless you intend to write and test your own
 5 | code the demonstration server is probably the right container for you.
 6 | 
 7 | Docker Development Server
 8 | ------------------------------
 9 | The development server is based on the `python:3.6-stretch` Docker image. It's
10 | an order of magnitude larger than the optimised demonstration server, 1GB to
11 | 100MB. The advantage is it comes with Python and build tools installed.
12 | 
13 | ### Building the development environment image
14 | You'll need to build a copy of the Docker image from the `Dockerfile.dev`
15 | [Dockerfile](https://docs.docker.com/engine/reference/builder/) before starting
16 | the container for the first time. The `-t wikidp-dev` arg means we creating a machine
17 | with tag `wikidp-dev`, in fact the full tag is `wikidp-dev:latest`. The
18 | `-f Dockerfile.dev` arg uses a named Dockerfile, `Dockerfile.dev` as a build source.
19 | 
20 | ```
21 | $ docker build -t wikidp-dev -f Dockerfile.dev .
22 | 
23 | ...
24 | 
25 | Successfully built 0069200c9c28
26 | Successfully tagged wikidp-dev:latest
27 | 
28 | $ docker image ls
29 | 
30 | REPOSITORY    TAG         IMAGE ID            CREATED             SIZE
31 | wikidp-dev    latest      0069200c9c28        4 minutes ago       1.14GB
32 | ```
33 | The same command can be used to update the image if changes are made to `Dockerfile.dev`.
34 | 
35 | ### Run the Docker development environment
36 | This command creates a named container from the `wikidp-dev` image created above.
37 | The `-p 5000:5000` arg maps port 5000 on the host to the same on the container,
38 | `-v "$PWD":/wikidp` arg share's the current host directory (local project root) as
39 | the named volume `/wikidp` in the container. The `--rm` means the container
40 | is cleaned up automatically when stopped, while `--name wikidp-dev` explicitly
41 | names the container (with the image name).
42 | 
43 | From within the project directory:
44 | ```
45 | docker run -p 5000:5000 -v "$PWD":/wikidp --rm --name wikidp-dev wikidp-dev
46 | ```
47 | The server should be available on <http://0.0.0.0:5000/>.
48 | 
49 | Docker Demonstration Server
50 | ---------------------------
51 | This is a lightweight optimised server meant to be easily downloaded from Dockerhub
52 | and quickly deployed. It's easier to use than the development server but is less
53 | flexible.
54 | ### Build a copy the WikiDP server
55 | From the project root directory:
56 | ```
57 | $ docker build -t wikidp .
58 | 
59 | Successfully built 3625ea189aea
60 | Successfully tagged wikidp:latest
61 | 
62 | $ docker image ls
63 | 
64 | REPOSITORY    TAG         IMAGE ID            CREATED              SIZE
65 | wikidp        latest      3625ea189aea        About a minute ago   95.4MB
66 | ```
67 | 
68 | ### Run the demonstration server container
69 | ```
70 | docker run --restart always -p 5000:5000 -e WIKIDP_CONSUMER_KEY='consumer-id'-e WIKIDP_CONSUMER_SECRET='consumer-secret' --name wikidp wikidp/portal:master
71 | ```
72 | 


--------------------------------------------------------------------------------
/Dockerfile:
--------------------------------------------------------------------------------
 1 | FROM python:3.6-alpine as builder
 2 | 
 3 | LABEL maintainer="carl.wilson@openpreservation.org" \
 4 |       org.openpreservation.vendor="Open Preservation Foundation" \
 5 |       version="0.1"
 6 | 
 7 | RUN  apk update && apk --no-cache --update-cache add gcc build-base libxml2-dev libxslt-dev
 8 | 
 9 | WORKDIR /src
10 | 
11 | COPY setup.py setup.py
12 | COPY README.md README.md
13 | COPY wikidp/* wikidp/
14 | 
15 | RUN mkdir /install && pip install -U pip && pip install --prefix=/install .
16 | 
17 | FROM python:3.6-alpine
18 | 
19 | RUN apk update && apk add --no-cache --update-cache libc6-compat libstdc++ bash
20 | RUN install -d -o root -g root -m 755 /opt && adduser --uid 1000 -h /opt/wikidp -S wikidp && pip install -U pip python-dateutil
21 | 
22 | WORKDIR /opt/wikidp
23 | 
24 | COPY --from=builder /install /usr/local
25 | COPY . /opt/wikidp/
26 | RUN chown -R wikidp:users /opt/wikidp
27 | 
28 | USER wikidp
29 | 
30 | EXPOSE 5000
31 | ENV FLASK_APP='wikidp'
32 | ENV PYWIKIBOT_DIR='/pywikibot'
33 | ENTRYPOINT flask run --host "0.0.0.0" --port "5000"
34 | 


--------------------------------------------------------------------------------
/Dockerfile.dev:
--------------------------------------------------------------------------------
 1 | FROM python:3.6-stretch
 2 | 
 3 | LABEL maintainer="carl.wilson@openpreservation.org" \
 4 |       org.openpreservation.vendor="Open Preservation Foundation" \
 5 |       version="0.1"
 6 | 
 7 | WORKDIR /tmp
 8 | 
 9 | COPY requirements.txt requirements.txt
10 | RUN pip install -r /tmp/requirements.txt
11 | 
12 | EXPOSE 5000
13 | ENV FLASK_APP='wikidp'
14 | WORKDIR /wikidp
15 | ENTRYPOINT ["/wikidp/docker-dev.sh"]
16 | 


--------------------------------------------------------------------------------
/Procfile:
--------------------------------------------------------------------------------
1 | web: gunicorn wikidp:APP
2 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # Wikidata DP Portal Prototype
  2 | A prototype for the Wikidata digital preservation portal.
  3 | 
  4 | ## Pre-requisites
  5 |   - Git or a copy of the latest source code
  6 |   - MacOS or Linux. Sorry Windows isn't currently supported.
  7 |   - Python 3. Python 2 isn't supported.
  8 |   - [Python pip](https://pip.pypa.io/en/stable/) for installing Python modules.
  9 | 
 10 | Using virtual environments for Python will save a lot of pain and allow you to
 11 | run Python 3 and Python 2 applications in harmony. If that sounds good then read [this primer](http://docs.python-guide.org/en/latest/dev/virtualenvs/).
 12 | 
 13 | ## Quick Start
 14 | The portal is a Flask web application written in Python. It's currently easy to install as long as you have a Python 3 environment. Be aware that this is currently a local development installation, it's not ready for deployment as a reliable application to a server. We're working on that. That said there's a few stages to getting going:
 15 | 1. Getting the code.
 16 | 2. Setting up a Python 3 virtualenv (optional but reccommended).
 17 | 3. Installing the portal prototype and its dependencies.
 18 | 4. Running the portal.
 19 | 
 20 | Let's take a look in a little more detail. We've provided some helper scripts for virtualenv setup, deployment and running which we'll also point you to.
 21 | 
 22 | ### Getting the code
 23 | There's no helper script for this. Clone this repository and move into the project root directory:
 24 | ````bash
 25 | git clone https://github.com/WikiDP/portal-proto.git
 26 | cd portal-proto
 27 | ````
 28 | Alternatively download and unpack the source archive from this git repository.
 29 | ````bash
 30 | wget https://github.com/WikiDP/portal-proto/archive/master.zip
 31 | unzip master.zip
 32 | rm master.zip
 33 | cd portal-proto-master
 34 | ````
 35 | Once you've done this you can create the virtualenv for installation, or skip the next step if you have a Python 3 environment you're happy to use.
 36 | 
 37 | ### Setting up a Python 3 virtualenv (optional)
 38 | We need to create a Python virtualenv in the project root folder in a `venv` subdirectory and activate it thus:
 39 | ````bash
 40 | virtualenv -p python3 venv
 41 | source ./venv/bin/activate
 42 | ````
 43 | There's a helper script in the root directory you can run instead:
 44 | `$ ./venv.sh`. If this has worked your terminal prompt should be adorned with a venv marker, e.g. `(venv) $`.
 45 | 
 46 | You only need to create the virtualenv once, although you can remove it by simply deleting the `venv` subdirectory. You'll need to activate the virtualenv `source ./venv/bin/activate` every time you start a new terminal session.
 47 | 
 48 | ### Installing the application and dependencies
 49 | Installing the application is straightforward using `pip`:
 50 | ````bash
 51 | (venv) $ source ./venv/bin/activate
 52 | (venv) $ pip install -e .
 53 | ````
 54 | Where the `-e` switch tells pip to monitor the directory and recompile changes. This is useful for development but should be omitted for stable deployments.
 55 | 
 56 | Again there's a helper script for this: `(venv) $ ./setup.sh`.
 57 | 
 58 | ### Running the application
 59 | The following steps need to be followed for every new terminal session.
 60 | 
 61 | You'll need to set up your Wikidata user name and password credentials, these can be exported as environment variables for now along with another variable that sets the flask application name:
 62 | ````bash
 63 | (venv) $ export WIKIDP_BOT_USER='<username>'
 64 | (venv) $ export WIKIDP_BOT_PASSWORD='<password>'
 65 | (venv) $ export FLASK_APP='wikidp'
 66 | ````
 67 | where `<username>` and `<password>` are your user name and password.
 68 | 
 69 | **NOTE** *these will need to be set for every new session for now*.
 70 | 
 71 | Finally run the  Flask application:
 72 | 
 73 | ````bash
 74 | (venv) $ flask run
 75 | ````
 76 | There's a script that you can use for this in the project root, `run.sh`. You'll need to edit it once to provide your Wikdata credentials by replacing the placeholders in the script here:
 77 | ````bash
 78 | export WIKIDP_BOT_USER='<username>'
 79 | export WIKIDP_BOT_PASSWORD='<password>'
 80 | ````
 81 | 
 82 | Point your browser to <http://127.0.0.1:5000> and you should see the prototype of the portal.
 83 | 
 84 | Testing the application
 85 | --------------
 86 | All Tests are currently in the _tests/_ directory. We use py.test to manage our testing interface.
 87 | 
 88 | ### Running Tests
 89 | ````bash
 90 | (venv) $ pytest
 91 | ````
 92 | 
 93 | ### Checking test coverage
 94 | We have provided a bash script to handle the coverage reporting
 95 | ````bash
 96 | (venv) $ ./coverage_report.sh
 97 | ````
 98 | If you would prefer a more visual web-interface for the coverage report,
 99 | this creates an _htmlcov/index.html_ file for browser viewing
100 | ````bash
101 | (venv) $ ./coverage_report.sh visual
102 | ````
103 | 
104 | ## Ansible roll out
105 | 
106 | ### Local vagrant VM
107 | `vagrant up`
108 | 
109 | ### Install Ansible roles locally
110 | ```bash
111 | ansible-galaxy install -r ansible/requirements.yml
112 | ```
113 | 
114 | #### Ansible CLI roll out to local Vagrant VM
115 | ```bash
116 | ansible-playbook --key-file=.vagrant/machines/default/virtualbox/private_key -v --inventory ansible/vagrant vagrant.yml
117 | ```
118 | 
119 | #### First time setup for any server via SSH, needs root password for server
120 | ```bash
121 | ansible-playbook ansible/setup.yml -i ansible/staging.yml -u root -vv -k
122 | ```
123 | <https://unix.stackexchange.com/questions/332641/how-to-install-python-3-6>
124 | wget <https://www.python.org/ftp/python/3.6.3/Python-3.6.3.tgz>
125 | tar xvf Python-3.6.3.tgz
126 | cd Python-3.6.3
127 | ./configure --enable-optimizations
128 | make -j8
129 | sudo make altinstall
130 | python3.6
131 | 
132 | ## Troubleshooting
133 | 
134 | ### Set FLASK_APP env variable
135 | If you see something like:
136 | ```shell
137 | Usage: flask run [OPTIONS]
138 | 
139 | Error: Could not locate Flask application. You did not provide the FLASK_APP environment variable.
140 | 
141 | For more information see http://flask.pocoo.org/docs/latest/quickstart/
142 | ```
143 | You need to set the FLASK_APP environment variable, see Quick Start above.
144 | 
145 | This code is released under the GPLv3 license.
146 | 


--------------------------------------------------------------------------------
/Vagrantfile:
--------------------------------------------------------------------------------
 1 | # -*- mode: ruby -*-
 2 | # vi: set ft=ruby :
 3 | 
 4 | # All Vagrant configuration is done below. The "2" in Vagrant.configure
 5 | # configures the configuration version (we support older styles for
 6 | # backwards compatibility). Please don't change it unless you know what
 7 | # you're doing.
 8 | Vagrant.configure("2") do |config|
 9 |   # The most common configuration options are documented and commented below.
10 |   # For a complete reference, please see the online documentation at
11 |   # https://docs.vagrantup.com.
12 | 
13 |   # Every Vagrant development environment requires a box. You can search for
14 |   # boxes at https://vagrantcloud.com/search.
15 |   # Debian Jessie should give us a stable server platform
16 |   config.vm.box = "debian/contrib-jessie64"
17 | 
18 |   # Disable automatic box update checking. If you disable this, then
19 |   # boxes will only be checked for updates when the user runs
20 |   # `vagrant box outdated`. This is not recommended.
21 |   # config.vm.box_check_update = false
22 | 
23 |   # Create a forwarded port mapping which allows access to a specific port
24 |   # within the machine from a port on the host machine. In the example below,
25 |   # accessing "localhost:8080" will access port 80 on the guest machine.
26 |   # NOTE: This will enable public access to the opened port
27 |   # NOTE: Uncomment below for a quick, Flask only, development deployment.
28 |   #       Simply passes the default Flask port from guest to host
29 |   config.vm.network "forwarded_port", guest: 5000, host: 5000
30 |   config.vm.network "forwarded_port", guest: 80, host: 8080
31 | 
32 |   # Create a forwarded port mapping which allows access to a specific port
33 |   # within the machine from a port on the host machine and only allow access
34 |   # via 127.0.0.1 to disable public access
35 |   # config.vm.network "forwarded_port", guest: 80, host: 8080, host_ip: "127.0.0.1"
36 |   # config.vm.network "private_network", ip: "192.168.10.200"
37 | 
38 |   # Create a private network, which allows host-only access to the machine
39 |   # using a specific IP.
40 |   # config.vm.network "private_network", ip: "192.168.33.10"
41 | 
42 |   # Create a public network, which generally matched to bridged network.
43 |   # Bridged networks make the machine appear as another physical device on
44 |   # your network.
45 |   # config.vm.network "public_network"
46 | 
47 |   # Share an additional folder to the guest VM. The first argument is
48 |   # the path on the host to the actual folder. The second argument is
49 |   # the path on the guest to mount the folder. And the optional third
50 |   # argument is a set of non-required options.
51 |   # config.vm.synced_folder "../data", "/vagrant_data"
52 | 
53 |   # Provider-specific configuration so you can fine-tune various
54 |   # backing providers for Vagrant. These expose provider-specific options.
55 |   # Example for VirtualBox:
56 |   #
57 |   # config.vm.provider "virtualbox" do |vb|
58 |   #   # Display the VirtualBox GUI when booting the machine
59 |   #   vb.gui = true
60 |   #
61 |   #   # Customize the amount of memory on the VM:
62 |   #   vb.memory = "1024"
63 |   # end
64 |   #
65 |   # View the documentation for the provider you are using for more
66 |   # information on available options.
67 |   config.vm.provider "virtualbox" do |vb|
68 |     # Display the VirtualBox GUI when booting the machine
69 |     # vb.gui = true
70 |     vb.name = "wikidp.dev"
71 |     # Customize the amount of memory on the VM:
72 |     vb.memory = "4096"
73 |     vb.cpus = "4"
74 |   end
75 | 
76 |   # Enable provisioning with a shell script. Additional provisioners such as
77 |   # Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the
78 |   # documentation for more information about their specific syntax and use.
79 |   # config.vm.provision "shell", inline: <<-SHELL
80 |   #   apt-get update
81 |   #   apt-get install -y apache2
82 |   # SHELL
83 |   config.vm.provision "ansible" do |ansible|
84 |       ansible.playbook = "ansible/vagrant.yml"
85 |       ansible.verbose = "vv"
86 |       ansible.limit = "all"
87 |       ansible.inventory_path = "ansible/vagrant"
88 |       ansible.galaxy_role_file = "ansible/requirements.yml"
89 |       ansible.galaxy_command = "ansible-galaxy install --role-file=%{role_file} --roles-path=%{roles_path}"
90 |    end
91 | end
92 | 


--------------------------------------------------------------------------------
/ansible/host_vars/vagrant:
--------------------------------------------------------------------------------
 1 | ---
 2 | # Vagrant box setup variables for WikiDP portal.
 3 | # Used by Vagrantfile to set up a local development VM
 4 | 
 5 | # Hostname variables
 6 | #
 7 | # IP Address host machine, MUST be a debian jessie box at present.
 8 | # Set up SSH to use localhost and port 2222 for vagrant.
 9 | ansible_ssh_host: "localhost"
10 | ansible_ssh_port: 2222
11 | # Domain of host, used to derive custom sub-domains perhaps, just localhost for now
12 | hostdomain: localhost
13 | hostname: "{{ hostdomain }}"
14 | 
15 | # Use the default vagrant user
16 | wikidp_user: "vagrant"
17 | wikibase_user: "vagrant"
18 | # Use our shared project root development directory as our local git repo
19 | wikidp_git_local: "/vagrant"
20 | # Please don't update the local repo, the developer should do this
21 | wikidp_git_update: false
22 | # System variables
23 | #
24 | # Set machine timezone
25 | timezone_zone: Europe/London
26 | # Install apt dependencies
27 | apt_install:
28 |   # Debian Jessie lacks the ability to download apt over HTTPS
29 |   - apt-transport-https
30 |   # Install some helper network tools
31 |   - psmisc
32 |   - net-tools
33 | 
34 | # Set true to deploy an nginx proxy
35 | wikidp_nginx_install: false
36 | # Set true to install uwsgi app server in pip mode
37 | wikidp_uwsgi_install: false
38 | 
39 | # Wikidata user name, password and prefered language
40 | wikidata:
41 |   user_name: "my_wikidata_user"
42 |   password: "my_wikidata_password"
43 |   language: "my_wikidata_language"
44 | 


--------------------------------------------------------------------------------
/ansible/production:
--------------------------------------------------------------------------------
1 | # Define all servers children
2 | [servers:children]
3 | wikidp
4 | 
5 | # The child list of WikiDP servers
6 | [wikidp]
7 | production
8 | 


--------------------------------------------------------------------------------
/ansible/requirements.yml:
--------------------------------------------------------------------------------
1 | - src: nginxinc.nginx
2 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.portal/defaults/main.yml:
--------------------------------------------------------------------------------
 1 | ---
 2 | # defaults file for wikidp.portal
 3 | 
 4 | wikidp_tmp: "/tmp/"
 5 | wikidp_app_name: "wikidp"
 6 | wikidp_user: "{{ wikidp_app_name }}"
 7 | wikidp_group: "{{ wikidp_user }}"
 8 | wikidp_user_home: "/home/{{ wikidp_user }}"
 9 | wikidp_app_home: "{{ wikidp_user_home }}/{{ wikidp_app_name }}"
10 | wikidp_host: "{{ opf_server_hostname }}"
11 | 
12 | wikidp_git_remote: "https://github.com/WikiDP/portal-proto.git"
13 | wikidp_git_local: "{{ wikidp_app_home }}/src"
14 | wikidp_git_version: "master"
15 | wikidp_git_update: true
16 | 
17 | wikidp_server_install: false
18 | wikidp_nginx_install: "{{ wikidp_server_install }}"
19 | wikidp_uwsgi_install: "{{ wikidp_server_install }}"
20 | wikidp_nginx_conf_root: "/etc/nginx"
21 | 
22 | ##
23 | # Don't put your wikidata username or password here please.
24 | # Override them in a host_vars or other private file.
25 | # See here for Ansible precendence rules:
26 | #  http://docs.ansible.com/ansible/latest/playbooks_variables.html#variable-precedence-where-should-i-put-a-variable
27 | wikidata:
28 |   user_name: ""
29 |   password: ""
30 |   language: "en"
31 | 
32 | # Namespaced variables you'll unlikely change
33 | wikidp:
34 |   app:
35 |     scripts: "{{ wikidp_app_home }}/bin"
36 |     setup: "setup_venv"
37 |     out_file: "{{ wikidp_tmp }}{{ wikidp_app_name }}.out"
38 |     cache_dir: "{{ wikidp_tmp }}caches"
39 |     venv: "{{ wikidp_app_home }}/venv"
40 |     python_ver: "python3"
41 |   ##
42 |   # dependencies to be installed by apt
43 |   #  - git for wikidp project source control
44 |   #  - python-dev and build-essential for pip
45 |   #  - python pip for python dependencies
46 |   #  - python 3 for our virtualenv
47 |   dependencies:
48 |     apt:
49 |       - "git"
50 |       - "python-dev"
51 |       - "build-essential"
52 |       - "python-pip"
53 |       - "python3"
54 |       - "python3-dev"
55 |     pip:
56 |       - "pip"
57 |       - "virtualenv"
58 |   git:
59 |     temp_archive: "{{ wikidp_tmp }}{{ wikidp_app_name }}.tar.gz"
60 |   src:
61 |     create: false
62 |     location: "/usr/local/src/{{ wikidp_app_name }}/{{ wikidp_git_version }}"
63 |   nginx:
64 |     install: "{{ wikidp_nginx_install }}"
65 |     branch: "stable"
66 |     default_conf: "{{ wikidp_nginx_conf_root }}/nginx.conf"
67 |     default_site: false
68 |     default_site_conf: "{{ wikidp_nginx_conf_root }}/conf.d/default.conf"
69 |     proxy_params: "{{ wikidp_nginx_conf_root }}/proxy_params"
70 |     sites_available: "{{ wikidp_nginx_conf_root }}/sites-available"
71 |     sites_enabled: "{{ wikidp_nginx_conf_root }}/sites-enabled"
72 |     log_dir: "/var/log/nginx"
73 |   flask:
74 |     port: 5000
75 |     bind: "0.0.0.0"
76 |   uwsgi:
77 |    install: "{{ wikidp_uwsgi_install }}"
78 |    socket_file: "/var/www/run/{{ wikidp_app_name }}.sock"
79 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.portal/files/empty:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/WikiDP/wikidp-portal/ab3fe89deacefa9a07ac1aa9046691e31f62eba6/ansible/roles/wikidp.portal/files/empty


--------------------------------------------------------------------------------
/ansible/roles/wikidp.portal/files/nginx/etc/nginx/nginx.conf:
--------------------------------------------------------------------------------
 1 | ##
 2 | # Tweaked nginx config file, heavily based on:
 3 | # https://linode.com/docs/web-servers/nginx/configure-nginx-for-optimized-performance/
 4 | ##
 5 | user              nginx;
 6 | pid               /var/run/nginx.pid;
 7 | worker_processes  auto;
 8 | 
 9 | ##
10 | # Scaleable I/O events
11 | ##
12 | events {
13 |   worker_connections   2048;
14 |   use epoll;
15 |   multi_accept on;
16 | }
17 | 
18 | ##
19 | # Default HTTP configuration
20 | ##
21 | http {
22 | 
23 |   # Keep alive allows for fewer reconnections from the browser.
24 |   keepalive_timeout 15;
25 |   keepalive_requests 100000;
26 | 
27 |   sendfile         on; # optimizes serving static files
28 |   tcp_nopush       on; # optimizes data sent down wire
29 |   tcp_nodelay      on; # send multiple buffers as individual packets
30 | 
31 |   client_body_buffer_size     128k;  # Client buffer size, mostly for POST requests
32 |   client_max_body_size        10m;   # Max body buffer size
33 |   client_header_buffer_size   1k;    # Default sane value
34 |   large_client_header_buffers 4 4k;  #4x4K large client header buffers
35 |   output_buffers              1 32k;
36 |   postpone_output             1460;
37 | 
38 |   ##
39 |   # More aggressive timeouts to help performance
40 |   ##
41 |   client_header_timeout  12;
42 |   client_body_timeout    12;
43 |   send_timeout           10;
44 | 
45 |   ##
46 |   # Cache 1000 files, exclude not accessed in last 20 sec
47 |   ##
48 |   open_file_cache           max=1000 inactive=20s;
49 |   open_file_cache_valid     30s; # Cache for 30s
50 |   open_file_cache_min_uses  5; # Must have been used 5 times in last 20 seconds
51 |   open_file_cache_errors    off;
52 | 
53 |   ##
54 |   # Use gzip to compress content
55 |   # Must be a megabyte at least
56 |   # Use for text MIME types
57 |   ##
58 |   gzip on;
59 |   gzip_min_length  1000;
60 |   gzip_buffers     4 4k;        # 4x4K Gzip buffers
61 |   gzip_disable "MSIE [1-6]\.";  # Disable for old IE versions
62 |   gzip_types       text/css application/x-javascript text/html application/javascript text/javascript text/plain text/xml application/json application/vnd.ms-fontobject application/x-font-opentype application/x-font-truetype application/x-font-ttf application/xml font/eot font/opentype font/otf image/svg+xml image/vnd.microsoft.icon;
63 | 
64 |   # [ debug | info | notice | warn | error | crit | alert | emerg ]
65 |   error_log  /var/log/nginx.error_log  error; # only log errirs
66 | 
67 |   # set main log format
68 |   log_format main      '$remote_addr - $remote_user [$time_local]  '
69 |     '"$request" $status $bytes_sent '
70 |     '"$http_referer" "$http_user_agent" '
71 |  		'"$gzip_ratio"';
72 | 
73 |   log_format download  '$remote_addr - $remote_user [$time_local]  '
74 |     '"$request" $status $bytes_sent '
75 |     '"$http_referer" "$http_user_agent" '
76 |  		'"$http_range" "$sent_http_content_range"';
77 | 
78 |   # Don't log success or redirects
79 |   map $status $loggable {
80 |     ~^[23]  0;
81 |     default 1;
82 |   }
83 | 
84 |   # Include server configs
85 |   include /etc/nginx/conf.d/*.conf;
86 |   # Include virtual hosts
87 |   include /etc/nginx/sites-enabled/*;
88 | }
89 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.portal/files/nginx/etc/nginx/proxy_params:
--------------------------------------------------------------------------------
1 | proxy_set_header Host $http_host;
2 | proxy_set_header X-Real-IP $remote_addr;
3 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
4 | proxy_set_header X-Forwarded-Proto $scheme;
5 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.portal/handlers/main.yml:
--------------------------------------------------------------------------------
1 | # handlers file for wikidp.portal
2 | ---
3 | 
4 | - name: restart nginx
5 |   service:
6 |     name: nginx
7 |     state: restarted
8 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.portal/meta/main.yml:
--------------------------------------------------------------------------------
 1 | # meta file for wikidp.portal
 2 | ---
 3 | galaxy_info:
 4 |   author: Carl Wilson
 5 |   company: The Open Preservation Foundation
 6 |   description: Installs and configures the WikiDP portal server
 7 |   license: GPL v3
 8 |   min_ansible_version: 2.0.0
 9 |   platforms:
10 |     - name: Ubuntu
11 |       versions:
12 |         - xenial
13 |     - name: Debian
14 |       versions:
15 |         - jessie
16 |   galaxy_tags:
17 |     - wikidp
18 |     - portal
19 | dependencies:
20 |   - { role: nginxinc.nginx, branch: "{{ wikidp.nginx.branch }}", when: wikidp.nginx.install }
21 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.portal/tasks/main.yml:
--------------------------------------------------------------------------------
 1 | ---
 2 | # tasks file for wikidp.portal
 3 | 
 4 | - name: "IMPORT | Install and configure the WikiDP Portal application."
 5 |   import_tasks: portal/main.yml
 6 | 
 7 | - name: "IMPORT | Configure and optimize nginx."
 8 |   import_tasks: nginx/main.yml
 9 |   when: wikidp.nginx.install
10 | 
11 | - name: "IMPORT | Configure and optimize uwsgi."
12 |   import_tasks: uwsgi/main.yml
13 |   when: wikidp.uwsgi.install
14 | 
15 | # - name: "IMPORT | Configure portal application as a service."
16 | #   import_tasks: service/main.yml
17 | #   when: wikidp.service.install
18 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.portal/tasks/nginx/configure.yml:
--------------------------------------------------------------------------------
 1 | ---
 2 | # Configure our Nginx installation properly
 3 | 
 4 | - name: "NGINX | Remove default.conf nginx site"
 5 |   file:
 6 |     path: "{{ wikidp.nginx.default_site_conf }}"
 7 |     state: absent
 8 |   notify:
 9 |     - restart nginx
10 |   when: not wikidp.nginx.default_site
11 | 
12 | - name: "NGINX | Add nginx user to wikidp group: {{ wikidp_group }}."
13 |   user:
14 |     name: "nginx"
15 |     group: "{{ wikidp_group }}"
16 | 
17 | - name: "NGINX | Add optimized nginx.conf."
18 |   copy:
19 |     src: "files/nginx/etc/nginx/nginx.conf"
20 |     dest: "{{ wikidp.nginx.default_conf }}"
21 |     mode: 0644
22 |   notify:
23 |     - restart nginx
24 | 
25 | - name: "NGINX | Add our proxy params."
26 |   copy:
27 |     src: "files/nginx/etc/nginx/proxy_params"
28 |     dest: "{{ wikidp.nginx.proxy_params }}"
29 |     mode: 0644
30 |   notify:
31 |     - restart nginx
32 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.portal/tasks/nginx/main.yml:
--------------------------------------------------------------------------------
 1 | ---
 2 | # Tasks to set up WikiDP Portal nginx
 3 | # file: tasks/uwsgi/main.yml
 4 | 
 5 | - name: "IMPORT | NGINX global configuration."
 6 |   import_tasks: configure.yml
 7 | 
 8 | - name: "IMPORT | Standalone NGINX configuration."
 9 |   import_tasks: standalone.yml
10 |   when: not wikidp_server_install
11 | 
12 | - name: "IMPORT | Server NGINX configuration."
13 |   import_tasks: server.yml
14 |   when: wikidp_server_install
15 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.portal/tasks/nginx/server.yml:
--------------------------------------------------------------------------------
 1 | ---
 2 | 
 3 | - name: "NGINX | Remove any standalone configuration."
 4 |   file:
 5 |     path: "{{ wikidp_nginx_conf_root }}/conf.d/{{ wikidp_host }}.conf"
 6 |     state: absent
 7 |   notify:
 8 |     - restart nginx
 9 | 
10 | - name: "NGINX | Create sites-available."
11 |   file:
12 |     path: "{{ wikidp.nginx.sites_available }}"
13 |     state: directory
14 |     mode: 0644
15 | 
16 | - name: "NGINX | Create our sites-enabled."
17 |   file:
18 |     path: "{{ wikidp.nginx.sites_enabled }}"
19 |     state: directory
20 |     mode: 0644
21 | 
22 | - name: "NGINX | Add templated sites-available for wikidp."
23 |   template:
24 |     src: "nginx/etc/nginx/sites-available/wikidp.j2"
25 |     dest: "{{ wikidp.nginx.sites_available }}/{{ wikidp_host }}"
26 |   notify:
27 |     - restart nginx
28 | 
29 | - name: "NGINX | Create sites-enabled symlink"
30 |   file:
31 |     path: "{{ wikidp.nginx.sites_enabled }}/{{ wikidp_host }}"
32 |     src: "{{ wikidp.nginx.sites_available }}/{{ wikidp_host }}"
33 |     state: link
34 |   notify:
35 |     - restart nginx
36 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.portal/tasks/nginx/standalone.yml:
--------------------------------------------------------------------------------
 1 | ---
 2 | 
 3 | - name: "NGINX | Remove any server sites for standalone"
 4 |   file:
 5 |     path: "{{ wikidp.nginx.sites_enabled }}/{{ wikidp_host }}"
 6 |     state: absent
 7 |   notify:
 8 |     - restart nginx
 9 | 
10 | - name: "NGINX | Add templated sites-available for wikidp."
11 |   template:
12 |     src: "nginx/etc/nginx/conf.d/wikidp.conf.j2"
13 |     dest: "{{ wikidp_nginx_conf_root }}/conf.d/{{ wikidp_host }}.conf"
14 |   notify:
15 |     - restart nginx
16 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.portal/tasks/portal/configure.yml:
--------------------------------------------------------------------------------
 1 | ---
 2 | # Configuration of WikiDP Portal
 3 | 
 4 | - name: "CONFIG | Create pywikibot conf in home: {{ wikidp_user_home }}/.pywikibot"
 5 |   file:
 6 |     path: "{{ wikidp_user_home }}/.pywikibot"
 7 |     state: directory
 8 |     mode: 0755
 9 |     owner: "{{ wikidp_user }}"
10 |     group: "{{ wikidp_group }}"
11 | 
12 | - name: "CONFIG | Copy in our templated user-config.py"
13 |   template:
14 |     src: "portal/home/pywikibot/user-config.py.j2"
15 |     dest: "{{ wikidp_user_home }}/.pywikibot/user-config.py"
16 |     owner: "{{ wikidp_user }}"
17 |     group: "{{ wikidp_group }}"
18 |     mode: 0755
19 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.portal/tasks/portal/dependencies.yml:
--------------------------------------------------------------------------------
 1 | ---
 2 | # Pre-installation tasks for WikiDP Portal
 3 | 
 4 | - name: "APT | Install apt package dependencies."
 5 |   apt:
 6 |     name: "{{ item }}"
 7 |     state: "latest"
 8 |   with_items: "{{ wikidp.dependencies.apt }}"
 9 | 
10 | - name: "PIP | Upgrade pip to latest and install dependencies."
11 |   pip:
12 |     name:  "{{ item }}"
13 |     state: latest
14 |   with_items: "{{ wikidp.dependencies.pip }}"
15 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.portal/tasks/portal/dirs.yml:
--------------------------------------------------------------------------------
 1 | # Create directories for portal app
 2 | ---
 3 | 
 4 | - name: "WIKIDP | Create server directory {{ wikidp_app_home }}."
 5 |   file:
 6 |     path: "{{ wikidp_app_home }}"
 7 |     state: directory
 8 |     mode: 0744
 9 |     owner: "{{ wikidp_user }}"
10 |     group: "{{ wikidp_group }}"
11 | 
12 | - name: "WIKIDP | Create scripts directory {{ wikidp.app.scripts }}."
13 |   file:
14 |     path: "{{ wikidp.app.scripts }}"
15 |     state: directory
16 |     mode: 0744
17 |     owner: "{{ wikidp_user }}"
18 |     group: "{{ wikidp_group }}"
19 | 
20 | - name: "WIKIDP | Create the cache dir: {{ wikidp.app.cache_dir }}."
21 |   file:
22 |     path: "{{ wikidp.app.cache_dir }}"
23 |     state: directory
24 |     mode: 0744
25 |     owner: "{{ wikidp_user }}"
26 |     group: "{{ wikidp_group }}"
27 | 
28 | - name: "WIKIDP | Create log file: {{ wikidp.app.out_file }}."
29 |   file:
30 |     path: "{{ wikidp.app.out_file }}"
31 |     state: touch
32 |     mode: 0744
33 |     owner: "{{ wikidp_user }}"
34 |     group: "{{ wikidp_group }}"
35 | 
36 | - name: "WIKIDP | Source directory {{ wikidp.src.location }}."
37 |   file:
38 |     path: "{{ wikidp.src.location }}"
39 |     state: directory
40 |     mode: 0744
41 |   when: wikidp.src.create
42 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.portal/tasks/portal/install.yml:
--------------------------------------------------------------------------------
 1 | ---
 2 | # Installation of WikiDP Portal
 3 | 
 4 | - name: "WIKIDP | Create local git directory {{ wikidp_git_local }}."
 5 |   file:
 6 |     path: "{{ wikidp_git_local }}"
 7 |     state: directory
 8 |     mode: 0744
 9 |     owner: "{{ wikidp_user }}"
10 |     group: "{{ wikidp_group }}"
11 | 
12 | - name: "WIKIDP | Cloning {{ wikidp_git_remote }}/{{ wikidp_git_version }} to {{ wikidp_git_local }}."
13 |   git:
14 |     repo: "{{ wikidp_git_remote }}"
15 |     dest: "{{ wikidp_git_local }}"
16 |     archive: "{{ wikidp.git.temp_archive }}"
17 |     version: "{{ wikidp_git_version }}"
18 |     update: "{{ wikidp_git_update }}"
19 |   become: true
20 |   become_user: "{{ wikidp_user }}"
21 | 
22 | - name: "WIKIDP | Archiving server source to: {{ wikidp.src.dest }}"
23 |   unarchive:
24 |     src: "{{ wikidp.git.temp_archive }}"
25 |     dest: "{{ wikidp.src.dest }}"
26 |     remote_src: yes
27 |   when: wikidp.src.create
28 | 
29 | - name: "WIKIDP | Add the virtualenv install script."
30 |   template:
31 |     src: "portal/install/setup_venv.sh.j2"
32 |     dest: "{{ wikidp.app.scripts }}/{{ wikidp.app.setup }}"
33 |     owner: "{{ wikidp_user }}"
34 |     group: "{{ wikidp_group }}"
35 |     mode: 0755
36 | 
37 | - name: "WIKIDP | Creating virtualenv: {{ wikidp.app.venv }} and installing: {{ wikidp_git_local }}."
38 |   command: "{{ wikidp.app.scripts }}/{{ wikidp.app.setup }}"
39 |   become: true
40 |   become_user: "{{ wikidp_user }}"
41 | 
42 | - name: "CONFIG | Creating server scripts: {{ wikidp.app.scripts }}."
43 |   template:
44 |     src: "portal/bin/{{ item }}.sh.j2"
45 |     dest: "{{ wikidp.app.scripts }}/{{ item }}"
46 |     owner: "{{ wikidp_user }}"
47 |     group: "{{ wikidp_group }}"
48 |     mode: 0755
49 |   with_items:
50 |     - start
51 |     - stop
52 |     - restart
53 |     - show
54 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.portal/tasks/portal/main.yml:
--------------------------------------------------------------------------------
 1 | # tasks file for wikidp.portal.portal
 2 | ---
 3 | 
 4 | - name: "IMPORT | Dependencies for WikiDP Portal."
 5 |   import_tasks: dependencies.yml
 6 | 
 7 | - name: "IMPORT | Create user accounts."
 8 |   import_tasks: users.yml
 9 | 
10 | - name: "IMPORT | Create installation directories."
11 |   import_tasks: dirs.yml
12 | 
13 | - name: "IMPORT | Configure the WikiDP Portal."
14 |   import_tasks: configure.yml
15 | 
16 | - name: "IMPORT | Install the WikiDP Portal."
17 |   import_tasks: install.yml
18 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.portal/tasks/portal/users.yml:
--------------------------------------------------------------------------------
 1 | ---
 2 | # Pre-installation tasks for WikiDP Portal
 3 | 
 4 | - name: "SYS | Create a dedicated user group: {{ wikidp_group }} for server app."
 5 |   group:
 6 |     name: "{{ wikidp_group }}"
 7 |     state: present
 8 | 
 9 | - name: "SYS | Create a dedicated user account: {{ wikidp_user }} for server app."
10 |   user:
11 |     name: "{{ wikidp_user }}"
12 |     comment: "User account for WikiDP server."
13 |     createhome: yes
14 |     home: "{{ wikidp_user_home }}"
15 |     group: "{{ wikidp_group }}"
16 |     shell: "/bin/bash"
17 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.portal/tasks/service/main.yml:
--------------------------------------------------------------------------------
1 | ---
2 | # Tasks to set up WikiDP Portal as a uwsgi service
3 | # file: tasks/service/main.yml
4 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.portal/tasks/uwsgi/main.yml:
--------------------------------------------------------------------------------
 1 | ---
 2 | # Tasks to set up WikiDP Portal uwsgi appliction
 3 | # file: tasks/uwsgi/main.yml
 4 | 
 5 | - name: "IMPORT | Standalone uwsgi configuration."
 6 |   import_tasks: standalone.yml
 7 |   when: not wikidp_server_install
 8 | 
 9 | - name: "IMPORT | Server uwsgi configuration."
10 |   import_tasks: server.yml
11 |   when: wikidp_server_install
12 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.portal/tasks/uwsgi/server.yml:
--------------------------------------------------------------------------------
 1 | ---
 2 | 
 3 | - name: "APT | Install uwsgi from package."
 4 |   apt:
 5 |     name:  "uwsgi"
 6 |     state: latest
 7 | 
 8 | - name: "CONFIG | Creating uswgi ini script: {{ wikidp_git_local }}/wikidp.ini."
 9 |   template:
10 |     src: "uwsgi/wikidp/home/wikidp.ini.j2"
11 |     dest: "{{ wikidp_git_local }}/wikidp.ini"
12 |     owner: "{{ wikidp_user }}"
13 |     group: "{{ wikidp_group }}"
14 |     mode: 0755
15 | 
16 | - name: "CONFIG | Creating WikiDP portal uwsgi service."
17 |   template:
18 |     src: "uwsgi/etc/systemd/system/wikidp.service.j2"
19 |     dest: "/etc/systemd/system/{{ wikidp_app_name }}.service"
20 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.portal/tasks/uwsgi/standalone.yml:
--------------------------------------------------------------------------------
 1 | ---
 2 | 
 3 | - name: "PIP | Install uwsgi."
 4 |   pip:
 5 |     name:  "uwsgi"
 6 |     state: latest
 7 |     virtualenv: "{{ wikidp.app.venv }}"
 8 |     virtualenv_command: "{{ wikidp.app.python_ver }}"
 9 |   become: true
10 |   become_user: "{{ wikidp_user }}"
11 | 
12 | - name: "CONFIG | Creating local start script: {{ wikidp.app.scripts }}."
13 |   template:
14 |     src: "uwsgi/wikidp/home/bin/{{ item }}.sh.j2"
15 |     dest: "{{ wikidp.app.scripts }}/{{ item }}"
16 |     owner: "{{ wikidp_user }}"
17 |     group: "{{ wikidp_group }}"
18 |     mode: 0755
19 |   with_items:
20 |     - start
21 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.portal/templates/nginx/etc/nginx/conf.d/wikidp.conf.j2:
--------------------------------------------------------------------------------
 1 | server {
 2 |     listen 80;
 3 |     server_name {{ wikidp_host }};
 4 | 
 5 |     access_log            {{ wikidp.nginx.log_dir }}/{{ wikidp_host }}.access.log;
 6 |     error_log             {{ wikidp.nginx.log_dir }}/{{ wikidp_host }}.error.log;
 7 | 
 8 |     # --> Custom headers
 9 |     add_header X-Frame-Options "DENY";
10 |     add_header X-XSS-Protection "1; mode=block";
11 |     add_header X-Content-Type-Options "nosniff";
12 |     # <-- Custom headers
13 | 
14 |     location / {
15 |         include {{ wikidp.nginx.proxy_params }};
16 |         proxy_pass http://localhost:{{ wikidp.flask.port }}/;
17 |         proxy_set_header Host $host;
18 |         proxy_set_header X-Real-IP $remote_addr;
19 |         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
20 |         proxy_set_header X-Forwarded-Proto $scheme;
21 |         proxy_max_temp_file_size 0;
22 |         client_max_body_size 100m;
23 |         client_body_buffer_size 90000k;
24 |         proxy_connect_timeout 9000;
25 |         proxy_send_timeout 9000;
26 |         proxy_read_timeout 9000;
27 |         proxy_buffer_size 4000k;
28 |         proxy_buffers 4 32000k;
29 |         proxy_busy_buffers_size 64000k;
30 |         proxy_temp_file_write_size 64000k;
31 |     }
32 | }
33 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.portal/templates/nginx/etc/nginx/sites-available/wikidp.j2:
--------------------------------------------------------------------------------
 1 | server {
 2 |     listen 80;
 3 |     server_name {{ wikidp_host }};
 4 | 
 5 |     access_log            {{ wikidp.nginx.log_dir }}/{{ wikidp_host }}.access.log;
 6 |     error_log             {{ wikidp.nginx.log_dir }}/{{ wikidp_host }}.error.log;
 7 | 
 8 |     # --> Custom headers
 9 |     add_header X-Frame-Options "DENY";
10 |     add_header X-XSS-Protection "1; mode=block";
11 |     add_header X-Content-Type-Options "nosniff";
12 |     # <-- Custom headers
13 | 
14 |     location / {
15 |         include uwsgi_params;
16 |         uwsgi_pass unix:{{ wikidp.uwsgi.socket_file }};
17 |         proxy_set_header Host $host;
18 |         proxy_set_header X-Real-IP $remote_addr;
19 |         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
20 |         proxy_set_header X-Forwarded-Proto $scheme;
21 |         proxy_max_temp_file_size 0;
22 |         client_max_body_size 100m;
23 |         client_body_buffer_size 90000k;
24 |         proxy_connect_timeout 9000;
25 |         proxy_send_timeout 9000;
26 |         proxy_read_timeout 9000;
27 |         proxy_buffer_size 4000k;
28 |         proxy_buffers 4 32000k;
29 |         proxy_busy_buffers_size 64000k;
30 |         proxy_temp_file_write_size 64000k;
31 |     }
32 | }
33 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.portal/templates/portal/bin/restart.sh.j2:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 | SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
3 | bash "$SCRIPT_DIR\stop"
4 | bash "$SCRIPT_DIR\start"
5 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.portal/templates/portal/bin/show.sh.j2:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 | fuser -a {{ wikidp.flask.port }}/tcp
3 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.portal/templates/portal/bin/start.sh.j2:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 | source "{{ wikidp.app.venv }}/bin/activate"
3 | export WIKIDP_BOT_USER='{{ wikidata.user_name }}'
4 | export WIKIDP_BOT_PASSWORD='{{ wikidata.password }}'
5 | export WIKIDP_LANG='{{ wikidata.language }}'
6 | export FLASK_APP='wikidp'
7 | flask run --port="{{ wikidp.flask.port }}" --host="{{ wikidp.flask.bind }}"
8 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.portal/templates/portal/bin/stop.sh.j2:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 | fuser -k {{ wikidp.flask.port }}/tcp
3 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.portal/templates/portal/home/pywikibot/user-config.py.j2:
--------------------------------------------------------------------------------
1 | mylang = 'wikidata'
2 | family = 'wikidata'
3 | usernames[family][mylang] = u'{{ wikidata.user_name }}'
4 | 
5 | authenticate['wikidata.org'] = ('{{ wikidata.user_name }}','{{ wikidata.password }}')
6 | console_encoding = 'utf-8'
7 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.portal/templates/portal/install/setup_venv.sh.j2:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 | virtualenv -p "{{ wikidp.app.python_ver }}" "{{ wikidp.app.venv }}"
3 | source "{{ wikidp.app.venv }}/bin/activate"
4 | pip install -e "{{ wikidp_git_local }}"
5 | deactivate
6 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.portal/templates/uwsgi/etc/systemd/system/wikidp.service.j2:
--------------------------------------------------------------------------------
 1 | 
 2 | [Unit]
 3 | Description=uWSGI instance to serve {{ wikidp_app_name }}
 4 | After=network.target
 5 | 
 6 | [Service]
 7 | User={{ wikidp_user }}
 8 | Group={{ wikidp_group }}
 9 | WorkingDirectory={{ wikidp_app_home }}
10 | Environment="PATH={{ wikidp.app.venv }}/bin"
11 | ExecStart={{ wikidp.app.venv }}/bin/uwsgi --ini {{ wikidp_git_local }}/{{ wikidp_app_name }}.ini
12 | 
13 | [Install]
14 | WantedBy=multi-user.target
15 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.portal/templates/uwsgi/wikidp/home/bin/start.sh.j2:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 | source "{{ wikidp.app.venv }}/bin/activate"
3 | export WIKIDP_BOT_USER='{{ wikidata.user_name }}'
4 | export WIKIDP_BOT_PASSWORD='{{ wikidata.password }}'
5 | export WIKIDP_LANG='{{ wikidata.language }}'
6 | uwsgi --socket "{{ wikidp.flask.bind }}:{{ wikidp.flask.port }}" --protocol=http --wsgi-file "{{ wikidp_git_local }}/wsgi.py" --callable APP -H "{{ wikidp.app.venv }}"
7 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.portal/templates/uwsgi/wikidp/home/wikidp.ini.j2:
--------------------------------------------------------------------------------
 1 | [uwsgi]
 2 | 
 3 | # Python Setup
 4 | virtualenv = {{ wikidp.app.venv }}
 5 | pythonpath = {{ wikidp_app_home }}
 6 | plugins = python
 7 | 
 8 | module = wikidp
 9 | callable = APP
10 | 
11 | # Logging
12 | logto = /var/log/uwsgi/{{ wikidp_app_name }}.log
13 | 
14 | master = true
15 | processes = 5
16 | 
17 | socket = {{ wikidp.uwsgi.socket_file }}
18 | chmod-socket = 660
19 | vacuum = true
20 | 
21 | die-on-term = true
22 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.portal/tests/inventory:
--------------------------------------------------------------------------------
1 | localhost
2 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.portal/tests/test.yml:
--------------------------------------------------------------------------------
1 | # test file for wikidp.portal
2 | ---
3 | - hosts: localhost
4 |   connection: local
5 |   become: true
6 |   roles:
7 |     - ../../
8 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.portal/tests/vagrant.yml:
--------------------------------------------------------------------------------
1 | # test file for wikidp.portal
2 | ---
3 | - hosts: all
4 |   remote_user: vagrant
5 |   become: true
6 |   roles:
7 |     - ../../
8 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.portal/vars/main.yml:
--------------------------------------------------------------------------------
1 | # vars file for wikidp.portal
2 | ---
3 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.wikibase/defaults/main.yml:
--------------------------------------------------------------------------------
  1 | ---
  2 | # defaults file for wikidp.wikibase
  3 | wikidp_tmp: "/tmp/"
  4 | wikibase_app_name: "wikibase"
  5 | wikibase_user: "wikidp"
  6 | wikibase_group: "{{ wikibase_user }}"
  7 | wikibase_user_home: "/home/{{ wikibase_user }}"
  8 | wikibase_app_home: "{{ wikibase_user_home }}/{{ wikibase_app_name }}"
  9 | wikibase_host: "db.{{ hostname }}"
 10 | wikibase_php_version: "7.0"
 11 | wikibase_php_etc: "/etc/php/{{ wikibase_php_version }}"
 12 | wikibase_nginx_conf_root: "/etc/nginx"
 13 | wikibase_mediawiki_version: "1.30"
 14 | wikibase_mediawiki_build: ".0"
 15 | wikibase_mysql_root_password: "vagrant"
 16 | wikibase_mysql_db_schema: "wikidb"
 17 | wikibase_mysql_db_user: "wikiuser"
 18 | wikibase_mysql_db_password: "vagrant"
 19 | wikibase_mysql_host: "localhost"
 20 | wikibase_mediawiki_root: "/srv/www"
 21 | wikibase_mediawiki_dir: "{{ wikibase_mediawiki_root }}/mediawiki-{{ wikibase_mediawiki_version }}{{ wikibase_mediawiki_build}}"
 22 | wikibase:
 23 |   ##
 24 |   # dependencies to be installed by apt
 25 |   #  - php: mediawiki is a PHP application
 26 |   #  - php-apcu, php-intl - further php dependencies
 27 |   dependencies:
 28 |     apt:
 29 |       - "php{{ wikibase_php_version }}-common"
 30 |       - "php{{ wikibase_php_version }}-cli"
 31 |       - "php{{ wikibase_php_version }}-intl"
 32 |       - "php{{ wikibase_php_version }}-curl"
 33 |       - "php{{ wikibase_php_version }}-cgi"
 34 |       - "php{{ wikibase_php_version }}-fpm"
 35 |       - "php{{ wikibase_php_version }}-mysql"
 36 |       - "php{{ wikibase_php_version }}-gd"
 37 |       - "php{{ wikibase_php_version }}-mcrypt"
 38 |       - "php{{ wikibase_php_version }}-posix"
 39 |       - "php{{ wikibase_php_version }}-xml"
 40 |       - "php{{ wikibase_php_version }}-zip"
 41 |       - "php{{ wikibase_php_version }}-xmlreader"
 42 |       - "php{{ wikibase_php_version }}-opcache"
 43 |       - "php{{ wikibase_php_version }}-json"
 44 |       - "php{{ wikibase_php_version }}-dom"
 45 |       - "php{{ wikibase_php_version }}-phar"
 46 |       - "php{{ wikibase_php_version }}-mbstring"
 47 |       - "php{{ wikibase_php_version }}-gettext"
 48 |       - "php{{ wikibase_php_version }}-iconv"
 49 |       - "php{{ wikibase_php_version }}-ctype"
 50 |       - "ssl-cert"
 51 |       - "imagemagick"
 52 |       - "texlive"
 53 |   mediawiki:
 54 |     root: "{{ wikibase_mediawiki_root }}"
 55 |     server: "{{ wikibase_host }}"
 56 |     install:
 57 |       src: "https://releases.wikimedia.org/mediawiki/{{ wikibase_mediawiki_version }}/mediawiki-{{ wikibase_mediawiki_version }}{{ wikibase_mediawiki_build}}.tar.gz"
 58 |       dir: "{{ wikibase_mediawiki_dir }}"
 59 |     extensions_dir: "{{ wikibase_mediawiki_dir}}/extensions"
 60 |     version: "{{ wikibase_mediawiki_version }}"
 61 |     build: "{{ wikibase_mediawiki_build }}"
 62 |     admin_pass: "vagrantwiki"
 63 |   mysql:
 64 |     root_password: "{{ wikibase_mysql_root_password }}"
 65 |     port: "3306"
 66 |     databases:
 67 |      - name: "{{ wikibase_mysql_db_schema }}"
 68 |     users:
 69 |      - name: "{{ wikibase_mysql_db_user }}"
 70 |        password: "{{ wikibase_mysql_db_password }}"
 71 |        priv: "wikidb.*:ALL"
 72 |   nginx:
 73 |     branch: "stable"
 74 |     default_conf: "{{ wikibase_nginx_conf_root }}/nginx.conf"
 75 |     default_site: false
 76 |     default_site_conf: "{{ wikibase_nginx_conf_root }}/conf.d/default.conf"
 77 |     proxy_params: "{{ wikibase_nginx_conf_root }}/proxy_params"
 78 |     sites_available: "{{ wikibase_nginx_conf_root }}/sites-available"
 79 |     sites_enabled: "{{ wikibase_nginx_conf_root }}/sites-enabled"
 80 |     log_dir: "/var/log/nginx"
 81 |   php:
 82 |     apt:
 83 |       repo:
 84 |         main: "deb https://packages.sury.org/php/ {{ ansible_distribution_release }} main"
 85 |         # source: "deb https://packages.sury.org/php/ {{ ansible_distribution_release }} source"
 86 |         key:
 87 |           src: "https://packages.sury.org/php/apt.gpg"
 88 |           dest: "/etc/apt/trusted.gpg.d/php.gpg"
 89 |     composer: "/usr/local/bin/composer"
 90 |     fpm:
 91 |       log: "/var/log/php{{ wikibase_php_version }}-fpm.log"
 92 |       log_level: "notice"
 93 |       syslog_ident: "php-fpm"
 94 |       ini: "{{ wikibase_php_etc }}/fpm/php.ini"
 95 |       conf: "{{ wikibase_php_etc }}/fpm/php-fpm.conf"
 96 |       default_pool: "{{ wikibase_php_etc }}/fpm/pool.d/www.conf"
 97 |       pool_base: "{{ wikibase_php_etc }}/fpm/pool.d/"
 98 |       pools:
 99 |         mediawiki_first:
100 |           name: "mediawiki-first"
101 |           user: "nginx"
102 |           group: "nginx"
103 |           log_file: "/var/log/php{{ wikibase_php_version }}-fpm.log"
104 |           log_errors: "on"
105 |           memory_limit: "512M"
106 |           pm: "dynamic"
107 |           pm_max_children: 5
108 |           pm_min_spare_servers: 1
109 |           pm_max_spare_servers: 3
110 |           pm_max_reqs: 500
111 |           pm_process_timeout: "10s"
112 |           pm_start_servers: 2
113 |     error_reporting: "E_ALL & ~E_DEPRECATED & ~E_STRICT"
114 |     errors: "On"
115 |     max_exec_time: 60
116 |     mem_limit: "128M"
117 |     post_max_size: "20M"
118 |     timezone: "Europe/London"
119 |     upload_max_filesize: "20M"
120 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.wikibase/files/empty:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/WikiDP/wikidp-portal/ab3fe89deacefa9a07ac1aa9046691e31f62eba6/ansible/roles/wikidp.wikibase/files/empty


--------------------------------------------------------------------------------
/ansible/roles/wikidp.wikibase/files/nginx/etc/nginx/nginx.conf:
--------------------------------------------------------------------------------
 1 | ##
 2 | # Tweaked nginx config file, heavily based on:
 3 | # https://linode.com/docs/web-servers/nginx/configure-nginx-for-optimized-performance/
 4 | ##
 5 | user              nginx;
 6 | pid               /var/run/nginx.pid;
 7 | worker_processes  auto;
 8 | 
 9 | ##
10 | # Scaleable I/O events
11 | ##
12 | events {
13 |   worker_connections   2048;
14 |   use epoll;
15 |   multi_accept on;
16 | }
17 | 
18 | ##
19 | # Default HTTP configuration
20 | ##
21 | http {
22 | 
23 |   # Keep alive allows for fewer reconnections from the browser.
24 |   keepalive_timeout 15;
25 |   keepalive_requests 100000;
26 | 
27 |   sendfile         on; # optimizes serving static files
28 |   tcp_nopush       on; # optimizes data sent down wire
29 |   tcp_nodelay      on; # send multiple buffers as individual packets
30 | 
31 |   client_body_buffer_size     128k;  # Client buffer size, mostly for POST requests
32 |   client_max_body_size        10m;   # Max body buffer size
33 |   client_header_buffer_size   1k;    # Default sane value
34 |   large_client_header_buffers 4 4k;  #4x4K large client header buffers
35 |   output_buffers              1 32k;
36 |   postpone_output             1460;
37 | 
38 |   ##
39 |   # More aggressive timeouts to help performance
40 |   ##
41 |   client_header_timeout  12;
42 |   client_body_timeout    12;
43 |   send_timeout           10;
44 | 
45 |   ##
46 |   # Cache 1000 files, exclude not accessed in last 20 sec
47 |   ##
48 |   open_file_cache           max=1000 inactive=20s;
49 |   open_file_cache_valid     30s; # Cache for 30s
50 |   open_file_cache_min_uses  5; # Must have been used 5 times in last 20 seconds
51 |   open_file_cache_errors    off;
52 | 
53 |   ##
54 |   # Use gzip to compress content
55 |   # Must be a megabyte at least
56 |   # Use for text MIME types
57 |   ##
58 |   gzip on;
59 |   gzip_min_length  1000;
60 |   gzip_buffers     4 4k;        # 4x4K Gzip buffers
61 |   gzip_disable "MSIE [1-6]\.";  # Disable for old IE versions
62 |   gzip_types       text/css application/x-javascript text/html application/javascript text/javascript text/plain text/xml application/json application/vnd.ms-fontobject application/x-font-opentype application/x-font-truetype application/x-font-ttf application/xml font/eot font/opentype font/otf image/svg+xml image/vnd.microsoft.icon;
63 | 
64 |   # [ debug | info | notice | warn | error | crit | alert | emerg ]
65 |   error_log  /var/log/nginx.error_log  error; # only log errirs
66 | 
67 |   # set main log format
68 |   log_format main      '$remote_addr - $remote_user [$time_local]  '
69 |     '"$request" $status $bytes_sent '
70 |     '"$http_referer" "$http_user_agent" '
71 |  		'"$gzip_ratio"';
72 | 
73 |   log_format download  '$remote_addr - $remote_user [$time_local]  '
74 |     '"$request" $status $bytes_sent '
75 |     '"$http_referer" "$http_user_agent" '
76 |  		'"$http_range" "$sent_http_content_range"';
77 | 
78 |   # Don't log success or redirects
79 |   map $status $loggable {
80 |     ~^[23]  0;
81 |     default 1;
82 |   }
83 | 
84 |   # Include server configs
85 |   include /etc/nginx/conf.d/*.conf;
86 |   # Include virtual hosts
87 |   include /etc/nginx/sites-enabled/*;
88 | }
89 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.wikibase/handlers/main.yml:
--------------------------------------------------------------------------------
 1 | ---
 2 | # handlers file for wikidp.wikibase
 3 | 
 4 | - name: restart nginx
 5 |   service:
 6 |     name: nginx
 7 |     state: restarted
 8 | 
 9 | - name: restart php7-fpm
10 |   service:
11 |     name: "php{{ wikibase_php_version }}-fpm"
12 |     state: restarted
13 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.wikibase/meta/main.yml:
--------------------------------------------------------------------------------
 1 | # meta file for wikidp.wikibase
 2 | ---
 3 | galaxy_info:
 4 |   author: Carl Wilson
 5 |   company: The Open Preservation Foundation
 6 |   description: Installs and configures a wikibase instance
 7 |   license: GPL v3
 8 |   min_ansible_version: 2.0.0
 9 |   platforms:
10 |     - name: Ubuntu
11 |       versions:
12 |         - xenial
13 |     - name: Debian
14 |       versions:
15 |         - jessie
16 |   galaxy_tags:
17 |     - wikidp
18 |     - portal
19 |     - wikibase
20 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.wikibase/tasks/main.yml:
--------------------------------------------------------------------------------
1 | ---
2 | # tasks file for wikidp.wikibase
3 | 
4 | - name: "IMPORT | Install and configure MediaWiki and WikiBase."
5 |   import_tasks: wikibase/main.yml
6 | 
7 | - name: "IMPORT | Configure and optimize nginx."
8 |   import_tasks: nginx/main.yml
9 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.wikibase/tasks/nginx/configure.yml:
--------------------------------------------------------------------------------
 1 | ---
 2 | # Configure our Nginx installation properly
 3 | 
 4 | - name: "NGINX | Remove default.conf nginx site"
 5 |   file:
 6 |     path: "{{ wikibase.nginx.default_site_conf }}"
 7 |     state: absent
 8 |   notify:
 9 |     - restart nginx
10 |   when: not wikibase.nginx.default_site
11 | 
12 | - name: "NGINX | Add nginx user to wikibase group: {{ wikibase_group }}."
13 |   user:
14 |     name: "nginx"
15 |     group: "{{ wikibase_group }}"
16 | 
17 | - name: "NGINX | Add optimized nginx.conf."
18 |   copy:
19 |     src: "files/nginx/etc/nginx/nginx.conf"
20 |     dest: "{{ wikibase.nginx.default_conf }}"
21 |     mode: 0644
22 |   notify:
23 |     - restart nginx
24 | 
25 | - name: "NGINX | Create sites-available."
26 |   file:
27 |     path: "/etc/nginx/sites-available"
28 |     state: directory
29 |     mode: 0644
30 | 
31 | - name: "NGINX | Create our sites-enabled."
32 |   file:
33 |     path: "/etc/nginx/sites-enabled"
34 |     state: directory
35 |     mode: 0644
36 | 
37 | - name: "NGINX | Create site config for MediaWiki"
38 |   template:
39 |     src: "nginx/etc/nginx/sites-available/wikibase.conf.j2"
40 |     dest: "/etc/nginx/sites-available/wikibase.conf"
41 |     owner: "root"
42 |     mode: 0644
43 | 
44 | - name: "NGINX | Create sites-enabled symlink"
45 |   file:
46 |     path: "/etc/nginx/sites-enabled/wikibase.conf"
47 |     src: "/etc/nginx/sites-available/wikibase.conf"
48 |     state: link
49 |   notify:
50 |     - restart nginx
51 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.wikibase/tasks/nginx/main.yml:
--------------------------------------------------------------------------------
 1 | ---
 2 | # Tasks to set up WikiDP Wikibase nginx
 3 | # file: tasks/nginx/main.yml
 4 | 
 5 | - name: "INCLUDE ROLE | NGINX Setup for WikiMedia"
 6 |   include_role:
 7 |     name: "nginxinc.nginx"
 8 |   vars:
 9 |     - branch: "{{ wikibase.nginx.branch }}"
10 | 
11 | - name: "IMPORT | NGINX global configuration."
12 |   import_tasks: configure.yml
13 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.wikibase/tasks/wikibase/dependencies.yml:
--------------------------------------------------------------------------------
 1 | ---
 2 | # Pre-installation tasks for WikiDP Portal
 3 | - name: "APT | Add key for Debian sury PHP repo."
 4 |   apt_key:
 5 |     url: "{{ wikibase.php.apt.repo.key.src }}"
 6 |     state: present
 7 | 
 8 | - name: "APT | Download key for Debian sury PHP repo {{ wikibase.php.apt.repo.key.src }}."
 9 |   get_url:
10 |     url: "{{ wikibase.php.apt.repo.key.src }}"
11 |     dest: "{{ wikibase.php.apt.repo.key.dest }}"
12 |     mode: 0640
13 | 
14 | - name: "APT | Add alternative PHP repo."
15 |   apt_repository:
16 |     repo: "{{ wikibase.php.apt.repo.main }}"
17 |     state: present
18 | 
19 | - name: "APT | Add alternative PHP repo main {{ wikibase.php.apt.repo.main }}."
20 |   apt_repository:
21 |     repo: "{{ wikibase.php.apt.repo.main }}"
22 |     state: present
23 | 
24 | - name: "APT | Install apt package dependencies."
25 |   apt:
26 |     name: "{{ item }}"
27 |     state: "latest"
28 |   with_items: "{{ wikibase.dependencies.apt }}"
29 |   when: wikibase.dependencies.apt is defined
30 |   notify:
31 |     - restart php7-fpm
32 | 
33 | - name: "INCLUDE | MySQL Setup for Wikimedia"
34 |   include_role:
35 |     name: "geerlingguy.mysql"
36 |   vars:
37 |     mysql_root_password: "{{ wikibase.mysql.root_password }}"
38 |     mysql_port: "{{ wikibase.mysql.port }}"
39 |     mysql_databases: "{{ wikibase.mysql.databases }}"
40 |     mysql_users: "{{ wikibase.mysql.users }}"
41 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.wikibase/tasks/wikibase/main.yml:
--------------------------------------------------------------------------------
 1 | ---
 2 | # tasks file for wikidp.wikibase
 3 | 
 4 | - name: "IMPORT | Install MediaWiki dependencies."
 5 |   import_tasks: dependencies.yml
 6 | 
 7 | - name: "IMPORT | Configure PHP-FPM."
 8 |   import_tasks: php-fpm.yml
 9 | 
10 | - name: "IMPORT | Install MediaWiki."
11 |   import_tasks: mediawiki.yml
12 | 
13 | - name: "IMPORT | Install Wikibase extension."
14 |   import_tasks: wikibase.yml
15 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.wikibase/tasks/wikibase/mediawiki.yml:
--------------------------------------------------------------------------------
 1 | ---
 2 | # tasks file for wikidp.wikibase
 3 | 
 4 | - name: "MEDIAWIKI | Create server directory {{ wikibase.mediawiki.root }}."
 5 |   file:
 6 |     path: "{{ wikibase.mediawiki.root }}"
 7 |     owner: "{{ wikibase_user }}"
 8 |     group: "{{ wikibase_group }}"
 9 |     state: directory
10 |     mode: 0755
11 | 
12 | - name: "MEDIAWIKI | Download and unpack MediaWiki {{ wikibase.mediawiki.install.dir}}."
13 |   unarchive:
14 |     src: "{{ wikibase.mediawiki.install.src }}"
15 |     dest: "{{ wikibase.mediawiki.root }}"
16 |     creates: "{{ wikibase.mediawiki.install.dir}}"
17 |     remote_src: yes
18 |     owner: "{{ wikibase_user }}"
19 |     group: "{{ wikibase_group }}"
20 | 
21 | - name: "MEDIAWIKI | Run installation script."
22 |   command: >
23 |     php maintenance/install.php
24 |     --server "{{ wikibase.mediawiki.server}}"
25 |     --dbuser "{{ wikibase_mysql_db_user }}"
26 |     --dbpass "{{ wikibase_mysql_db_password }}"
27 |     --confpath "{{ wikibase.mediawiki.install.dir}}"
28 |     --dbname "{{ wikibase_mysql_db_schema }}"
29 |     --dbserver "{{ wikibase_mysql_host }}"
30 |     --installdbuser: "root"
31 |     --installdbpass: "{{ wikibase_mysql_root_password}}"
32 |     --pass "{{ wikibase.mediawiki.admin_pass }}"
33 |     --scriptpath=""
34 |     "wikidp"
35 |     "admin"
36 |   args:
37 |     chdir: "{{ wikibase.mediawiki.install.dir}}"
38 |     creates: "{{ wikibase.mediawiki.install.dir}}/LocalSettings.php"
39 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.wikibase/tasks/wikibase/php-fpm.yml:
--------------------------------------------------------------------------------
 1 | ---
 2 | # tasks file for wikidp.wikibase
 3 | 
 4 | - name: "PHP-FPM | Customise PHP ini"
 5 |   template:
 6 |     src: "php/etc/php/fpm/php.ini.j2"
 7 |     dest: "{{ wikibase.php.fpm.ini }}"
 8 |     owner: "root"
 9 |     mode: 0644
10 |   notify: restart php7-fpm
11 | 
12 | - name: "PHP-FPM | Create configuration"
13 |   template:
14 |     src: "php/etc/php/fpm/php-fpm.conf.j2"
15 |     dest: "{{ wikibase.php.fpm.conf }}"
16 |     owner: "root"
17 |     mode: 0644
18 |   notify: restart php7-fpm
19 | 
20 | - name: "PHP | Remove default PHP-FPM pool."
21 |   file:
22 |     path: "{{ wikibase.php.fpm.default_pool }}"
23 |     state: absent
24 |   notify: restart php7-fpm
25 | 
26 | - name: "PHP | Writing PHP-FPM pool configs."
27 |   template:
28 |     src: "php/etc/php/fpm/pool.d/pool.conf.j2"
29 |     dest: "{{ wikibase.php.fpm.pool_base }}/{{ item.key }}.conf"
30 |     owner: "root"
31 |     mode: 0644
32 |   with_dict: "{{ wikibase.php.fpm.pools }}"
33 |   notify: restart php7-fpm
34 |   tags:
35 |     - "php-fpm"
36 |     - "install"
37 | 
38 | - name: "PHP | Install Composer"
39 |   get_url:
40 |     url: https://getcomposer.org/composer.phar
41 |     dest: "{{ wikibase.php.composer }}"
42 |     mode: 0755
43 |     owner: "root"
44 |     group: "staff"
45 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.wikibase/tasks/wikibase/wikibase.yml:
--------------------------------------------------------------------------------
 1 | ---
 2 | # tasks file for wikidp.wikibase
 3 | 
 4 | - name: "WIKIBASE | Create local git directory."
 5 |   file:
 6 |     path: "{{ wikibase.mediawiki.extensions_dir }}/Wikibase"
 7 |     state: directory
 8 |     mode: 0755
 9 |     owner: "{{ wikidp_user }}"
10 |     group: "{{ wikidp_group }}"
11 | 
12 | - name: "WIKIBASE | Cloning {{ wikidp_git_remote }}/{{ wikidp_git_version }} to {{ wikidp_git_local }}."
13 |   git:
14 |     repo: "https://github.com/wikimedia/mediawiki-extensions-Wikibase.git"
15 |     dest: "{{ wikibase.mediawiki.extensions_dir }}/Wikibase"
16 |     version: "REL1_30"
17 |     update: true
18 |   become: true
19 |   become_user: "{{ wikidp_user }}"
20 | 
21 | - name: "WIKIBASE | Composer install Wikibase extension."
22 |   composer:
23 |     command: install
24 |     working_dir: "{{ wikibase.mediawiki.extensions_dir }}/Wikibase"
25 | 
26 | - name: "WIKIBASE | Adding wikibase extension lines to LocalSettings.php"
27 |   blockinfile:
28 |     path: "{{ wikibase.mediawiki.install.dir}}/LocalSettings.php"
29 |     insertafter: "EOF"
30 |     block: |
31 |       $wgEnableWikibaseRepo = true;
32 |       $wgEnableWikibaseClient = true;
33 |       require_once "$IP/extensions/Wikibase/repo/Wikibase.php";
34 |       require_once "$IP/extensions/Wikibase/repo/ExampleSettings.php";
35 |       require_once "$IP/extensions/Wikibase/client/WikibaseClient.php";
36 |       require_once "$IP/extensions/Wikibase/client/ExampleSettings.php";
37 | 
38 | - name: "WIKIBASE | Run WikiMedia maintenance update."
39 |   command: >
40 |     php maintenance/update.php
41 |   args:
42 |     chdir: "{{ wikibase.mediawiki.install.dir}}"
43 |   become: true
44 |   become_user: "{{ wikidp_user }}"
45 | 
46 | - name: "WIKIBASE | Run Wikibase site population script."
47 |   command: >
48 |     php lib/maintenance/populateSitesTable.php
49 |   args:
50 |     chdir: "{{ wikibase.mediawiki.extensions_dir }}/Wikibase"
51 |   become: true
52 |   become_user: "{{ wikidp_user }}"
53 | 
54 | - name: "WIKIBASE | Run Wikibase rebuild maintenance script."
55 |   command: >
56 |     php repo/maintenance/rebuildItemsPerSite.php
57 |   args:
58 |     chdir: "{{ wikibase.mediawiki.extensions_dir }}/Wikibase"
59 |   become: true
60 |   become_user: "{{ wikidp_user }}"
61 | 
62 | - name: "WIKIBASE | Run Wikibase populateInterwiki maintenance script."
63 |   command: >
64 |     php client/maintenance/populateInterwiki.php
65 |   args:
66 |     chdir: "{{ wikibase.mediawiki.extensions_dir }}/Wikibase"
67 |   become: true
68 |   become_user: "{{ wikidp_user }}"
69 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.wikibase/templates/nginx/etc/nginx/sites-available/wikibase.conf.j2:
--------------------------------------------------------------------------------
  1 | server {
  2 |     ## This is to avoid the spurious if for sub-domain name
  3 |     ## "rewriting".
  4 |     listen 80; # IPv4
  5 |     # listen [2a01:7e00::f03c:91ff:fe56:dd46]:443 ipv6only=on;
  6 |     server_name www.{{ wikibase.mediawiki.server }};
  7 |     return 301 $scheme://{{ wikibase.mediawiki.server }}$request_uri;
  8 | } # server domain rewrite.
  9 | 
 10 | 
 11 | server {
 12 |     listen 80; # IPv4
 13 |     server_name {{ wikibase.mediawiki.server }};
 14 |     root {{ wikibase.mediawiki.install.dir }};
 15 |     index index.php;
 16 |     autoindex off;
 17 | 
 18 |     ## Parameterization using hostname of access and log filenames.
 19 |     access_log /var/log/nginx/{{ wikibase.mediawiki.server }}_access.log;
 20 |     error_log /var/log/nginx/{{ wikibase.mediawiki.server }}_error.log;
 21 | 
 22 |     #    Common deny, drop, or internal locations
 23 | 
 24 |     #    Exclude all access from the cache directory
 25 |     location ^~ /cache/ { deny all; }
 26 | 
 27 |     #    Prevent access to any files starting with a dot, like .htaccess
 28 |     #    or text editor temp files
 29 |     location ~ /\. { access_log off; log_not_found off; deny all; }
 30 | 
 31 |     #    Prevent access to any files starting with a $ (usually temp files)
 32 |     location ~ ~$ { access_log off; log_not_found off; deny all; }
 33 | 
 34 |     #    Do not log access to robots.txt, to keep the logs cleaner
 35 |     location = /robots.txt { access_log off; log_not_found off; }
 36 | 
 37 |     #    Do not log access to the favicon, to keep the logs cleaner
 38 |     location = /favicon.ico { access_log off; log_not_found off; }
 39 | 
 40 |     #    Keep images and CSS around in browser cache for as long as possible,
 41 |     #    to cut down on server load
 42 |     location ~* \.(js|css|png|jpg|jpeg|gif|ico)$ {
 43 |         try_files $uri /index.php;
 44 |         expires max;
 45 |         log_not_found off;
 46 |     }
 47 | 
 48 |     #    Mark all of these directories as "internal", which means that they cannot
 49 |     #    be explicitly accessed by clients. However, the web server can still use
 50 |     #    and serve the files inside of them. This keeps people from poking around
 51 |     #    in the wiki's internals.
 52 |     location ^~ /bin/ { internal; }
 53 |     location ^~ /docs/ { internal; }
 54 |     location ^~ /extensions/ { internal; }
 55 |     location ^~ /includes/ { internal; }
 56 |     location ^~ /maintenance/ { internal; }
 57 | #    location ^~ /mw-config/ { internal; } #Uncomment after installation
 58 | #    location ^~ /resources/ { internal; }
 59 |     location ^~ /serialized/ { internal; }
 60 |     location ^~ /tests/ { internal; }
 61 | 
 62 |     #    Force potentially-malicious files in the /images directory to be served
 63 |     #    with a text/plain mime type, to prevent them from being executed by
 64 |     #    the PHP handler
 65 |     location ~* ^/images/.*.(html|htm|shtml|php)$ {
 66 |         types { }
 67 |         default_type text/plain;
 68 |     }
 69 | 
 70 |     #    Redirect all requests for unknown URLs out of images and back to the
 71 |     #    root index.php file
 72 |     location ^~ /images/ {
 73 |         try_files $uri /index.php;
 74 |     }
 75 | 
 76 |     # Uncomment after installation!
 77 |     location / {
 78 |         index index.php;
 79 |         rewrite ^/([^?]*)(?:\?(.*))? /index.php?title=$1&$2 last;
 80 |     }
 81 | 
 82 |     location ~ \.php5?$ {
 83 |         try_files $uri =404;
 84 |         include fastcgi_params;
 85 |         fastcgi_pass unix:/run/php/mediawiki-first-fpm.sock;
 86 |         fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
 87 |         fastcgi_intercept_errors on;
 88 |     }
 89 | 
 90 |     location ~ \.php7?$ {
 91 |         try_files $uri =404;
 92 |         include fastcgi_params;
 93 |         fastcgi_pass unix:/run/php/mediawiki-first-fpm.sock;
 94 |         fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
 95 |         fastcgi_intercept_errors on;
 96 |     }
 97 | 
 98 |     location ~ \.php?$ {
 99 |         try_files $uri =404;
100 |         include fastcgi_params;
101 |         fastcgi_pass unix:/run/php/mediawiki-first-fpm.sock;
102 |         fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
103 |         fastcgi_intercept_errors on;
104 |     }
105 | }
106 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.wikibase/templates/php/etc/php/fpm/php-fpm.conf.j2:
--------------------------------------------------------------------------------
  1 | ;;;;;;;;;;;;;;;;;;;;;
  2 | ; FPM Configuration ;
  3 | ;;;;;;;;;;;;;;;;;;;;;
  4 | 
  5 | ; All relative paths in this configuration file are relative to PHP's install
  6 | ; prefix (/usr). This prefix can be dynamically changed by using the
  7 | ; '-p' argument from the command line.
  8 | 
  9 | ;;;;;;;;;;;;;;;;;;
 10 | ; Global Options ;
 11 | ;;;;;;;;;;;;;;;;;;
 12 | 
 13 | [global]
 14 | ; Pid file
 15 | ; Note: the default prefix is /var
 16 | ; Default Value: none
 17 | 
 18 | ; Error log file
 19 | ; If it's set to "syslog", log is sent to syslogd instead of being written
 20 | ; in a local file.
 21 | ; Note: the default prefix is /var
 22 | ; Default Value: log/php-fpm.log
 23 | error_log = {{ wikibase.php.fpm.log }}
 24 | 
 25 | ; syslog_facility is used to specify what type of program is logging the
 26 | ; message. This lets syslogd specify that messages from different facilities
 27 | ; will be handled differently.
 28 | ; See syslog(3) for possible values (ex daemon equiv LOG_DAEMON)
 29 | ; Default Value: daemon
 30 | ;syslog.facility = daemon
 31 | 
 32 | ; syslog_ident is prepended to every message. If you have multiple FPM
 33 | ; instances running on the same server, you can change the default value
 34 | ; which must suit common needs.
 35 | ; Default Value: php-fpm
 36 | syslog.ident = {{ wikibase.php.fpm.syslog_ident }}
 37 | 
 38 | ; Log level
 39 | ; Possible Values: alert, error, warning, notice, debug
 40 | ; Default Value: notice
 41 | log_level = {{ wikibase.php.fpm.log_level }}
 42 | 
 43 | ; If this number of child processes exit with SIGSEGV or SIGBUS within the time
 44 | ; interval set by emergency_restart_interval then FPM will restart. A value
 45 | ; of '0' means 'Off'.
 46 | ; Default Value: 0
 47 | ;emergency_restart_threshold = 0
 48 | 
 49 | ; Interval of time used by emergency_restart_interval to determine when
 50 | ; a graceful restart will be initiated.  This can be useful to work around
 51 | ; accidental corruptions in an accelerator's shared memory.
 52 | ; Available Units: s(econds), m(inutes), h(ours), or d(ays)
 53 | ; Default Unit: seconds
 54 | ; Default Value: 0
 55 | ;emergency_restart_interval = 0
 56 | 
 57 | ; Time limit for child processes to wait for a reaction on signals from master.
 58 | ; Available units: s(econds), m(inutes), h(ours), or d(ays)
 59 | ; Default Unit: seconds
 60 | ; Default Value: 0
 61 | ;process_control_timeout = 0
 62 | 
 63 | ; The maximum number of processes FPM will fork. This has been design to control
 64 | ; the global number of processes when using dynamic PM within a lot of pools.
 65 | ; Use it with caution.
 66 | ; Note: A value of 0 indicates no limit
 67 | ; Default Value: 0
 68 | ; process.max = 128
 69 | 
 70 | ; Specify the nice(2) priority to apply to the master process (only if set)
 71 | ; The value can vary from -19 (highest priority) to 20 (lower priority)
 72 | ; Note: - It will only work if the FPM master process is launched as root
 73 | ;       - The pool process will inherit the master process priority
 74 | ;         unless it specified otherwise
 75 | ; Default Value: no set
 76 | ; process.priority = -19
 77 | 
 78 | ; Send FPM to background. Set to 'no' to keep FPM in foreground for debugging.
 79 | ; Default Value: yes
 80 | ;daemonize = yes
 81 | 
 82 | ; Set open file descriptor rlimit for the master process.
 83 | ; Default Value: system defined value
 84 | ;rlimit_files = 1024
 85 | 
 86 | ; Set max core size rlimit for the master process.
 87 | ; Possible Values: 'unlimited' or an integer greater or equal to 0
 88 | ; Default Value: system defined value
 89 | ;rlimit_core = 0
 90 | 
 91 | ; Specify the event mechanism FPM will use. The following is available:
 92 | ; - select     (any POSIX os)
 93 | ; - poll       (any POSIX os)
 94 | ; - epoll      (linux >= 2.5.44)
 95 | ; - kqueue     (FreeBSD >= 4.1, OpenBSD >= 2.9, NetBSD >= 2.0)
 96 | ; - /dev/poll  (Solaris >= 7)
 97 | ; - port       (Solaris >= 10)
 98 | ; Default Value: not set (auto detection)
 99 | ;events.mechanism = epoll
100 | 
101 | ; When FPM is build with systemd integration, specify the interval,
102 | ; in second, between health report notification to systemd.
103 | ; Set to 0 to disable.
104 | ; Available Units: s(econds), m(inutes), h(ours)
105 | ; Default Unit: seconds
106 | ; Default value: 10
107 | ;systemd_interval = 10
108 | 
109 | ;;;;;;;;;;;;;;;;;;;;
110 | ; Pool Definitions ;
111 | ;;;;;;;;;;;;;;;;;;;;
112 | 
113 | ; Multiple pools of child processes may be started with different listening
114 | ; ports and different management options.  The name of the pool will be
115 | ; used in logs and stats. There is no limitation on the number of pools which
116 | ; FPM can handle. Your system will tell you anyway :)
117 | 
118 | ; Include one or more files. If glob(3) exists, it is used to include a bunch of
119 | ; files from a glob(3) pattern. This directive can be used everywhere in the
120 | ; file.
121 | ; Relative path can also be used. They will be prefixed by:
122 | ;  - the global prefix if it's been set (-p argument)
123 | ;  - /usr otherwise
124 | include=/etc/php/7.0/fpm/pool.d/*.conf
125 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.wikibase/tests/inventory:
--------------------------------------------------------------------------------
1 | localhost
2 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.wikibase/tests/test.yml:
--------------------------------------------------------------------------------
1 | # test file for wikidp.wikibase
2 | ---
3 | - hosts: localhost
4 |   connection: local
5 |   become: true
6 |   roles:
7 |     - ../../
8 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.wikibase/tests/vagrant.yml:
--------------------------------------------------------------------------------
1 | # test file for wikidp.wikibase
2 | ---
3 | - hosts: all
4 |   remote_user: vagrant
5 |   become: true
6 |   roles:
7 |     - ../../
8 | 


--------------------------------------------------------------------------------
/ansible/roles/wikidp.wikibase/vars/main.yml:
--------------------------------------------------------------------------------
1 | # vars file for wikidp.wikibase
2 | ---
3 | 


--------------------------------------------------------------------------------
/ansible/vagrant:
--------------------------------------------------------------------------------
1 | # Define all servers children
2 | [servers:children]
3 | wikidp
4 | 
5 | # The child list of WikiDP servers
6 | [wikidp]
7 | vagrant
8 | 


--------------------------------------------------------------------------------
/ansible/vagrant.yml:
--------------------------------------------------------------------------------
 1 | ---
 2 | # Playbook to configure a Vagrant box WikiDP Portal instance
 3 | # File: ansible/vagrant.yml
 4 | 
 5 | - name: Store known hosts of 'all' the hosts in the inventory file
 6 |   hosts: localhost
 7 |   connection: local
 8 | 
 9 |   vars:
10 |     ssh_known_hosts_command: "ssh-keyscan -T 10"
11 |     ssh_known_hosts_file: "{{ lookup('env','HOME') + '/.ssh/known_hosts' }}"
12 |     ssh_known_hosts: "{{ groups['all'] }}"
13 | 
14 |   tasks:
15 | 
16 |   - name: For each host, scan for its ssh public key
17 |     shell: "ssh-keyscan {{ item }},`dig +short {{ item }}`"
18 |     with_items: "{{ ssh_known_hosts }}"
19 |     register: ssh_known_host_results
20 |     ignore_errors: yes
21 | 
22 |   - name: Add/update the public key in the '{{ ssh_known_hosts_file }}'
23 |     known_hosts:
24 |       name: "{{ item.item }}"
25 |       key: "{{ item.stdout }}"
26 |       path: "{{ ssh_known_hosts_file }}"
27 |     with_items: "{{ ssh_known_host_results.results }}"
28 | 
29 | - hosts: all
30 |   gather_facts: no
31 |   pre_tasks:
32 |     - name: 'install python2'
33 |       raw: sudo apt-get -y install python-simplejson
34 | 
35 | - hosts: all
36 |   become: true
37 |   roles:
38 |     - { role: opf.server }
39 |     - { role: wikidp.portal }
40 |     - { role: wikidp.wikibase }
41 | 


--------------------------------------------------------------------------------
/apicache-py3/6bc28b771fb3b4881b11ac888717488cdd6acc33b08858c3cfd2880f4728b882:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/WikiDP/wikidp-portal/ab3fe89deacefa9a07ac1aa9046691e31f62eba6/apicache-py3/6bc28b771fb3b4881b11ac888717488cdd6acc33b08858c3cfd2880f4728b882


--------------------------------------------------------------------------------
/apicache-py3/f2bd856f660527e525d4c5a8904717bb5c59d9de2f332d5f5982309888fe9cc7:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/WikiDP/wikidp-portal/ab3fe89deacefa9a07ac1aa9046691e31f62eba6/apicache-py3/f2bd856f660527e525d4c5a8904717bb5c59d9de2f332d5f5982309888fe9cc7


--------------------------------------------------------------------------------
/apicache-py3/f8d9172cb945cc1b53e4cac31649fc8ab1ce5b8cc64c5d1e8b7e8264392c4ba6:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/WikiDP/wikidp-portal/ab3fe89deacefa9a07ac1aa9046691e31f62eba6/apicache-py3/f8d9172cb945cc1b53e4cac31649fc8ab1ce5b8cc64c5d1e8b7e8264392c4ba6


--------------------------------------------------------------------------------
/apicache-py3/fce830cb647985332a95ad135d86777a42b9d298ba90a1f56e4bc7904bbc31b3:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/WikiDP/wikidp-portal/ab3fe89deacefa9a07ac1aa9046691e31f62eba6/apicache-py3/fce830cb647985332a95ad135d86777a42b9d298ba90a1f56e4bc7904bbc31b3


--------------------------------------------------------------------------------
/coverage_report.sh:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env bash
 2 | if (($1 == visual))
 3 | then
 4 |   py.test --cov=wikidp tests --cov-report html
 5 |   open htmlcov/index.html
 6 | else
 7 |   py.test --cov=wikidp tests
 8 |   coverage report
 9 | fi
10 | 


--------------------------------------------------------------------------------
/docker-dev.sh:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 | SCRIPT_DIR="$( dirname "$( readlink -f "${BASH_SOURCE[0]}" )")"
3 | cd "${SCRIPT_DIR}" || exit
4 | pip install -U pip
5 | pip install -U -e .
6 | export FLASK_APP=wikidp
7 | export FLASK_ENV=development
8 | flask run --host 0.0.0.0 --port 5000
9 | 


--------------------------------------------------------------------------------
/docs/.gitkeep:
--------------------------------------------------------------------------------
1 | 
2 | 


--------------------------------------------------------------------------------
/docs/CNAME:
--------------------------------------------------------------------------------
1 | docs.wikidp.org


--------------------------------------------------------------------------------
/docs/OAUTH-SETUP.md:
--------------------------------------------------------------------------------
 1 | Developing With OAuth
 2 | =====================
 3 | 
 4 | Introduction
 5 | ------------
 6 | WikiDP uses the OAuth protocol to allow proxy login to the WikiData site. While this
 7 | is convenient for site users, the key is tied to the main WikiDP domain: wikidp.org.
 8 | This poses problems for developers as it's impossible to use the main site access
 9 | token on a local machine. This is a quick guide to obtaining and setting up an OAuth
10 | token for local development.
11 | 
12 | Pre-requisites
13 | --------------
14 | We assume that you have the software up and running using the [instructions here](../README.md).
15 | You'll also need a [Wikimedia account](https://en.wikipedia.org/wiki/Wikipedia:Why_create_an_account%3F)
16 | both to obtain an OAuth token and to log on to your WikiDP dev instance.
17 | 
18 | Getting an OAuth Token
19 | ----------------------
20 | First you'll need to request an OAuth token from Wikimedia for your development instance.
21 | This special page <https://meta.wikimedia.org/wiki/Special:OAuthConsumerRegistration/propose>.
22 | Wikimedia's general OAuth instructions for developers are here:
23 | <https://www.mediawiki.org/wiki/OAuth/For_Developers> but we show a full example
24 | below with screenshots that meets WikiDP's requirements. The first part of the
25 | OAuth form is shown below. You'll need to propose a unique name in the first form field: "Application name", and use your own email address. Everything else should
26 | be indentical.
27 | 
28 | ![Wikimedia OAuth form part one](./oauth-1.png)
29 | 
30 | The second half of the form specifies the grants for permissions and should look
31 | the same as the diagram below:
32 | 
33 | ![Wikimedia OAuth form part two](./oauth-2.png)
34 | 
35 | Finally there are some details about IP blocking and RSA keys, this should be filled
36 | out as below:
37 | 
38 | ![Wikimedia OAuth form part three](./oauth-3.png)
39 | 
40 | Hi the "Propose consumer" button and you'll get the consumer and secret tokens as
41 | shown below. **PLEASE SCREENSHOT OR CAPTURE THESE NOW, ONCE THE PAGE IS CLOSED YOU
42 | WILL NOT BE ABLE TO RETRIEVE THEM AGAIN**.
43 | 
44 | ![Wikimedia OAuth form part three](./oauth-4.png)
45 | 
46 | Now you will have to wait for confirmation from the Wikimedia team, this usually takes
47 | 24-48 hours maximum and will be returned to you via email. Once you receive the
48 | confirmation you can set up your development environment.
49 | 
50 | Using your Token for Development
51 | --------------------------------
52 | The easiest way of running a development instance is to use the local development
53 | Docker image. You'll need to build a Docker image locally, from the project root:
54 | 
55 | ```
56 | docker build -t wikidp-dev -f Dockerfile.dev .
57 | ```
58 | 
59 | This creates a local image called `wikidp-dev`. You can now run this while adding
60 | your OAuth details as environment variables like so:
61 | ```
62 | docker run -p 5000:5000 -v "$PWD":/wikidp --rm -e WIKIDP_CONSUMER_KEY='<consumer-token>'-e WIKIDP_CONSUMER_SECRET='<secret-token>' --name wikidp-dev wikidp-dev
63 | ```
64 | 


--------------------------------------------------------------------------------
/docs/getting-started/index.md:
--------------------------------------------------------------------------------
1 | Getting started
2 | ===============
3 | 


--------------------------------------------------------------------------------
/docs/index.md:
--------------------------------------------------------------------------------
 1 | Home Page
 2 | =========
 3 | 
 4 | [getting started](getting-started/)
 5 | 
 6 | ## Wikidata for Digital Preservation
 7 | Members of the digital preservation community collate and capture metadata to describe file formats, software, operating systems and hardware, and use it to inform and drive digital preservation processes. The vision of Wikidata for Digital Preservation is to use Wikidata as a technical registry of metadata related to computer software and computing environments. Collaboratively creating this metadata, and making it available as linked open data, will reduce the amount of redundant work digital preservation professionals do in order to describe resources. Having machine-readable, linked open data that describes the digital preservation domain will also allow us to reuse this data in our software applications and information systems, reducing the overhead when building new tools. 
 8 | 
 9 | 
10 | 
11 | 
12 | ## How to Contribute
13 | This application will allow you to find the correct Wikidata item for software or file formats about which you would like to add information.
14 | 
15 | ### Step 1: Find Item
16 | 
17 | #### Searching Tips
18 | Word Perfect
19 | WordPerfect
20 | 
21 | ### Step 2: Contribute statements
22 | 
23 | ### Step 3: Add references
24 | 
25 | 
26 | 
27 | 
28 | 
29 | 
30 | 
31 | 
32 | 
33 | 
34 | 
35 | 
36 | ## Where Can I Find Information to cite? 
37 | 
38 | ### Connecting file formats and software
39 | 
40 | We need to create statements on software items using property [readable file format](https://www.wikidata.org/wiki/Property:P1072) and [writable file format](https://www.wikidata.org/wiki/Property:P1072) to indicate relationships with file formats.
41 | 
42 | This information is available from the software itself. It is also sometimes described in the user manual or on a website about the software.
43 | #### Reference example
44 | This is an item with relatively complete information about how it relates to file formats: [GIMP image processing software](https://www.wikidata.org/wiki/Q8038).
45 | 
46 | ### Leverage external ids
47 | Many software and file format items have identifiers in external systems. These external systems may contain information about the resources you're looking to describe.
48 | #### Video Game Developer example
49 | The Moby Games resource has detailed descriptions about many companies that develop software, especially games.
50 | 
51 | If you find an item for a software developer that does not yet have many statements, you may be able to create additional statements from the info on the Moby Games site.
52 | 
53 | Follow the link from the value for [MobyGames company ID](https://www.wikidata.org/wiki/Property:P4773) on the item for the developer to see the Moby Games page for the company. Thank you for making the information in Wikidata more complete! 
54 | 
55 | Here is a SPARQL query to find all Wikidata items that have a statment using [MobyGames company ID](https://www.wikidata.org/wiki/Property:P4773). Copy the text of the query and paste it into the [Wikidata Query Service SPARQL endpoint](https://query.wikidata.org).
56 | ### Adding external id info to software items
57 | 
58 | External id properties have their own data type and are used to link out to datasets beyond Wikidata.
59 | 
60 |    * [AUR package (P4162)](https://www.wikidata.org/wiki/Property:P4162)
61 |    * [Arch Linux package (P3454)](https://www.wikidata.org/wiki/Property:P3454)
62 |    * [Debian stable package (P3442)](https://www.wikidata.org/wiki/Property:P3442)
63 |   * [DistroWatch ID (P3112)](https://www.wikidata.org/wiki/Property:P3112)
64 |    * [F-Droid package (P3597)](https://www.wikidata.org/wiki/Property:P3597)
65 |    * [Fedora package (P3463)](https://www.wikidata.org/wiki/Property:P3463)
66 |    * [Framalibre ID (P4107)](https://www.wikidata.org/wiki/Property:P4107)
67 |    * [Free Software Directory entry](https://www.wikidata.org/wiki/Property:P2537)
68 |    * [Gentoo package (P3499)](https://www.wikidata.org/wiki/Property:P3499)
69 |    * [Google Play Store App ID (P3418)](https://www.wikidata.org/wiki/Property:P3418)
70 |    * [Hall of Light Amiga database ID (P4671)](https://www.wikidata.org/wiki/Property:P4671)
71 |    * [MAC Address Block Large ID (P4776)](https://www.wikidata.org/wiki/Property:P4776)
72 |    * [Nintendo Game Store ID (P4685)](https://www.wikidata.org/wiki/Property:P4685)
73 |    * [PRONOM software identifier (P2749)](https://www.wikidata.org/wiki/Property:P2749)
74 |    * [Rosetta Code ID (P5047)](https://www.wikidata.org/wiki/Property:P5407)
75 |    * [SourceForge project (P2209)](https://www.wikidata.org/wiki/Property:P2209)
76 |    * [Stack Exchange tag (P1482)](https://www.wikidata.org/wiki/Property:P1482)
77 |    * [Twitter hashtag (P2572)](https://www.wikidata.org/wiki/Property:P2572)
78 |    * [Ubuntu package (P3473)](https://www.wikidata.org/wiki/Property:P3473)
79 |    * [Uniform Resource Identifier Scheme (P4742)](https://www.wikidata.org/wiki/Property:P4742)
80 |    * [crates.io ID (P4763)](https://www.wikidata.org/wiki/Property:P4763)
81 |    * [snap package (P4435)](https://www.wikidata.org/wiki/Property:P4435)
82 |     
83 | ### Work from a list
84 | 
85 | [File formats with info from Just Solve Wiki](https://www.wikidata.org/wiki/User:YULdigitalpreservation/FileFormatsList)
86 | 
87 | ### Source code repository
88 | 


--------------------------------------------------------------------------------
/docs/oauth-1.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/WikiDP/wikidp-portal/ab3fe89deacefa9a07ac1aa9046691e31f62eba6/docs/oauth-1.png


--------------------------------------------------------------------------------
/docs/oauth-2.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/WikiDP/wikidp-portal/ab3fe89deacefa9a07ac1aa9046691e31f62eba6/docs/oauth-2.png


--------------------------------------------------------------------------------
/docs/oauth-3.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/WikiDP/wikidp-portal/ab3fe89deacefa9a07ac1aa9046691e31f62eba6/docs/oauth-3.png


--------------------------------------------------------------------------------
/docs/oauth-4.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/WikiDP/wikidp-portal/ab3fe89deacefa9a07ac1aa9046691e31f62eba6/docs/oauth-4.png


--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
 1 | setuptools
 2 | flask>=1.1.2
 3 | wikidataintegrator==0.8.4
 4 | lxml==4.6.3
 5 | jsonpickle==1.4.1
 6 | numpy==1.19.0
 7 | pandas==1.0.5
 8 | python-dateutil==2.8.1
 9 | flask-mwoauth>=0.4.82
10 | tqdm==4.47.0
11 | gunicorn==19.9.0
12 | validators==0.15.0
13 | 


--------------------------------------------------------------------------------
/run.sh:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 | source ./venv/bin/activate
3 | # Wikidp production token
4 | # export WIKIDP_CONSUMER_KEY=''
5 | # export WIKIDP_CONSUMER_SECRET=''
6 | export FLASK_APP='wikidp'
7 | flask run
8 | 


--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/python
 2 | # coding=UTF-8
 3 | #
 4 | # WikiDP Wikidata Portal
 5 | # Copyright (C) 2020
 6 | # All rights reserved.
 7 | #
 8 | # This code is distributed under the terms of the GNU General Public
 9 | # License, Version 3. See the text file "COPYING" for further details
10 | # about the terms of this license.
11 | #
12 | """Setup for flask app."""
13 | from setuptools import setup, find_packages
14 | 
15 | INSTALL_REQUIRES = [
16 |     'setuptools',
17 |     'flask>=0.12.3',
18 |     'wikidataintegrator==0.8.4',
19 |     'lxml==4.6.3',
20 |     'jsonpickle==1.4.1',
21 |     'numpy==1.18.2',
22 |     'pandas==1.0.3',
23 |     'python-dateutil==2.8.1',
24 |     'flask-mwoauth>=0.4.81',
25 |     'tqdm==4.45.0',
26 |     'validators==0.14.3'
27 | ]
28 | PYTHON_REQUIRES = '>=3.6, <4'
29 | 
30 | TEST_DEPS = [
31 |     'pre-commit',
32 |     'pytest',
33 |     'pylint',
34 |     'pytest-coverage'
35 | ]
36 | EXTRAS = {
37 |     'testing': TEST_DEPS,
38 | }
39 | 
40 | README = open('README.md', 'r')
41 | README_TEXT = README.read()
42 | README.close()
43 | 
44 | setup(
45 |     name='wikidp',
46 |     version='0.1.0',
47 |     packages=find_packages(),
48 |     include_package_data=True,
49 |     install_requires=INSTALL_REQUIRES,
50 |     tests_require=TEST_DEPS,
51 |     extras_require=EXTRAS,
52 |     python_requires=PYTHON_REQUIRES,
53 |     platforms=['POSIX'],
54 |     description='Digital preservation portal for Wikidata',
55 |     long_description=README_TEXT,
56 |     long_description_content_type='text/markdown',
57 |     author='Kenny Seals-Nutt',
58 |     author_email='',
59 |     maintainer='Carl Wilson',
60 |     maintainer_email='carl@openpreservation.org',
61 |     url='http://wikidp.org/',
62 |     classifiers=[
63 |         'Environment :: Console',
64 |         'Programming Language :: Python :: 3',
65 |     ]
66 | )
67 | 
68 | #TO DO: If using python3.6,
69 | #       I had to run 'install certificates.command' inside the directory
70 | #       in order to load url's
71 | 


--------------------------------------------------------------------------------
/setup.sh:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 | source ./venv/bin/activate
3 | pip install -e .
4 | 


--------------------------------------------------------------------------------
/tests/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/WikiDP/wikidp-portal/ab3fe89deacefa9a07ac1aa9046691e31f62eba6/tests/__init__.py


--------------------------------------------------------------------------------
/tests/settings.py:
--------------------------------------------------------------------------------
1 | SAMPLE_QID = 'Q7715973'  # Debian
2 | SAMPLE_QID__TEST = 'Q175461'  # WikiDP Testing Item
3 | SAMPLE_QID_NOT_EXIST = 'Q7'  # Qid that does not have an item (Fake QID)
4 | SAMPLE_QID_NOT_EXIST__TEST = 'Q1'  # Qid that does not have an item (Fake QID)
5 | SAMPLE_PID_WIKIBASEITEM = 'P31'  # Instance of
6 | SAMPLE_PID_WIKIBASEITEM__TEST = 'P80397'  # WikiDP Testing Property (WikibaseItem)
7 | SAMPLE_PID_STRING = 'P31'  # Instance Of
8 | SAMPLE_PID_STRING__TEST = 'P80430'  # WikiDP Testing Property (WikibaseItem)
9 | 


--------------------------------------------------------------------------------
/tests/test_controllers.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/python
 2 | # coding=UTF-8
 3 | #
 4 | # WikiDP Wikidata Portal
 5 | # Copyright (C) 2020
 6 | # All rights reserved.
 7 | #
 8 | # This code is distributed under the terms of the GNU General Public
 9 | # License, Version 3. See the text file "COPYING" for further details
10 | # about the terms of this license.
11 | #
12 | """Unit tests for WikiDP Controllers."""
13 | from unittest import TestCase
14 | 
15 | from tests import settings
16 | from wikidp.controllers import pages as pages_controller
17 | 
18 | 
19 | class ControllerTests(TestCase):
20 |     # Pages Controllers
21 |     def test_controller_pages_get_checklist_context__no_schema(self):
22 |         output = pages_controller.get_property_checklist_from_schema(settings.SAMPLE_QID)
23 |         self.assertEqual(output, [])
24 | 


--------------------------------------------------------------------------------
/tests/test_reads.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/python
 2 | # coding=UTF-8
 3 | #
 4 | # WikiDP Wikidata Portal
 5 | # Copyright (C) 2020
 6 | # All rights reserved.
 7 | #
 8 | # This code is distributed under the terms of the GNU General Public
 9 | # License, Version 3. See the text file "COPYING" for further details
10 | # about the terms of this license.
11 | #
12 | """Unit tests for WikiData reads."""
13 | # TODO: Mock Test Responses for unit tests and make an integration test for live APIs
14 | # from unittest import TestCase
15 | #
16 | # import pywikibot
17 | #
18 | # from wikidp.model import FileFormat
19 | #
20 | # SANDBOX_API_URL = 'https://test.wikidata.org/w/'
21 | # SANDBOX_SPARQL_URL = 'https://test.wikidata.org/proxy/wdqs/bigdata/namespace/wdq/'
22 | # SITE = pywikibot.Site("test", "wikidata")
23 | # REPO = SITE.data_repository()
24 | #
25 | #
26 | # class WikidataTests(TestCase):
27 | #     def test_format_read(self):
28 | #         """Queries Wikidata for formats and returns a list of FileFormat instances."""
29 | #         results = FileFormat.list_formats()
30 | #         assert results, \
31 | #         'Format read returned no results'
32 | #
33 | #     def test_new_item(self):
34 | #         new_item = pywikibot.ItemPage(SITE)
35 | #         labels = {"en": "Carl Wilson", "de": "Carl Wilson"}
36 | #         new_item.editLabels(labels=labels, summary="Adding labels")
37 | #         my_item_id = new_item.getID()
38 | #         item = pywikibot.ItemPage(REPO, my_item_id)
39 | #         item.get()
40 | #         assert item.labels['en'] == 'Carl Wilson'
41 | #         assert item.labels['de'] == 'Carl Wilson'
42 | 


--------------------------------------------------------------------------------
/user-config.py.template:
--------------------------------------------------------------------------------
 1 | mylang = 'en'
 2 | family = 'wikidata'
 3 | usernames[family][mylang] = u'<username>'
 4 | 
 5 | authenticate['wikidata.org'] = ('<username>','<password>')
 6 | console_encoding = 'utf-8'
 7 | 
 8 | usernames[family]['test'] = u'<username>'
 9 | 
10 | authenticate['test.wikidata.org'] = ('<username>','<password>')
11 | 


--------------------------------------------------------------------------------
/venv.sh:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 | virtualenv -p python3 venv
3 | source ./venv/bin/activate
4 | 


--------------------------------------------------------------------------------
/wikidp/__init__.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/python
 2 | # coding=UTF-8
 3 | #
 4 | # WikiDP Wikidata Portal
 5 | # Copyright (C) 2020
 6 | # All rights reserved.
 7 | #
 8 | # This code is distributed under the terms of the GNU General Public
 9 | # License, Version 3. See the text file "COPYING" for further details
10 | # about the terms of this license.
11 | #
12 | # This is a python __init__ script to create the app and import the
13 | # main package contents
14 | """Initialisation module for package, kicks of the flask app."""
15 | import logging
16 | 
17 | from wikidp.config import APP
18 | from . import (
19 |     routes,
20 | )
21 | if __name__ == "__main__":
22 |     logging.debug("Importing %s", routes.__name__)
23 |     logging.debug("Running Flask App on Port %s", APP.config.get('PORT'))
24 |     APP.run(host='0.0.0.0', port=APP.config.get('PORT'))
25 | 


--------------------------------------------------------------------------------
/wikidp/config.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/python
  2 | # coding=UTF-8
  3 | #
  4 | # WikiDP Wikidata Portal
  5 | # Copyright (C) 2020
  6 | # All rights reserved.
  7 | #
  8 | # This code is distributed under the terms of the GNU General Public
  9 | # License, Version 3. See the text file "COPYING" for further details
 10 | # about the terms of this license.
 11 | #
 12 | """Configuration for WikiDP portal Flask app."""
 13 | import logging
 14 | import os
 15 | import tempfile
 16 | 
 17 | from flask import Flask
 18 | 
 19 | from wikidp.const import ConfKey
 20 | 
 21 | 
 22 | # Template these values for flexible install
 23 | HOST = 'localhost'
 24 | TEMP = tempfile.gettempdir()
 25 | 
 26 | 
 27 | # pylint: disable=R0903
 28 | class BaseConfig:
 29 |     """Base / default config, no debug logging and short log format."""
 30 | 
 31 |     CACHE_DIR = os.path.join(TEMP, 'caches')
 32 |     DEBUG = False
 33 |     HOST = HOST
 34 |     ITEM_REGEX = r'(Q|q)\d+'
 35 |     MEDIAWIKI_API_URL = "https://www.wikidata.org/w/api.php"
 36 |     OAUTH_MEDIAWIKI_URL = "https://www.wikidata.org/w/index.php"
 37 |     SPARQL_ENDPOINT_URL = "https://query.wikidata.org/sparql"
 38 |     # Bind to PORT if defined, otherwise default to 5000.
 39 |     PORT = int(os.environ.get('PORT', 5000))
 40 |     PROPERTY_REGEX = r'(P|p)\d+'
 41 |     LOG_FILE = os.path.join(TEMP, 'wikidp.log')
 42 |     LOG_FORMAT = '[%(filename)-15s:%(lineno)-5d] %(message)s'
 43 |     SECRET_KEY = '7d441f27d441f27567d441f2b6176a'
 44 |     WIKIBASE_LANGUAGE = os.getenv('WIKIBASE_LANGUAGE', 'en')
 45 |     WIKIDATA_FB_LANG = os.getenv('WIKIDP_FB_LANG', 'en')
 46 |     WIKIDATA_LANG = os.getenv('WIKIDP_LANG', 'en')
 47 |     WIKIDATA_PASSWORD = os.getenv('WIKIDP_BOT_PASSWORD', '<password>')
 48 |     WIKIDATA_USER_NAME = os.getenv('WIKIDP_BOT_USER', '<username>')
 49 |     WIKIDP_CONSUMER_KEY = os.environ.get('WIKIDP_CONSUMER_KEY', '')
 50 |     WIKIDP_CONSUMER_SECRET = os.environ.get('WIKIDP_CONSUMER_SECRET', '')
 51 |     USER_AGENT = os.environ.get(
 52 |         'USER_AGENT',
 53 |         'wikidp-portal/0.0 (https://www.wikidp.org/; admin@wikidp.org)'
 54 |     )
 55 | 
 56 | 
 57 | # pylint: disable=R0903
 58 | class DevConfig(BaseConfig):
 59 |     """Developer level config, with debug logging and long log format."""
 60 | 
 61 |     DEBUG = True
 62 |     LOG_FORMAT = '[%(asctime)s %(levelname)-8s %(filename)-15s:%(lineno)-5d ' +\
 63 |                  '%(funcName)-30s] %(message)s'
 64 |     MEDIAWIKI_API_URL = "https://wikidp.wiki.opencura.com/w/api.php"
 65 |     SPARQL_ENDPOINT_URL = 'https://wikidp.wiki.opencura.com/query/sparql'
 66 | 
 67 | 
 68 | CONFIGS = {
 69 |     "dev": 'wikidp.config.DevConfig',
 70 |     "default": 'wikidp.config.BaseConfig'
 71 | }
 72 | 
 73 | 
 74 | def configure_app(app):
 75 |     """Grab the environment variable for app config or defaults to dev."""
 76 |     config_name = os.getenv('WIKIDP_CONFIG', 'default')
 77 |     app.config.from_object(CONFIGS[config_name])
 78 |     app.config[ConfKey.STATIC_DIR] = os.path.join(app.root_path, 'static')
 79 |     if os.getenv('WIKIDP_CONFIG_FILE'):
 80 |         app.config.from_envvar('WIKIDP_CONFIG_FILE')
 81 |     app.config['WIKIDATA_SIGN_UP_URL'] = \
 82 |         "https://www.wikidata.org/w/index.php?title=Special:CreateAccount"
 83 |     _lang = app.config[ConfKey .WIKIDATA_LANG]
 84 |     _fb_lang = app.config[ConfKey.WIKIDATA_FB_LANG]
 85 |     # Create the list of unique languages to easy SPARQL queries
 86 |     if _lang != _fb_lang:
 87 |         app.config[ConfKey.WIKIBASE_LANGUAGE] = f"{_lang},{_fb_lang}"
 88 | 
 89 | 
 90 | APP = Flask(__name__)
 91 | # Get the appropriate config
 92 | configure_app(APP)
 93 | # Configure logging across all modules
 94 | logging.basicConfig(filename=APP.config[ConfKey.LOG_FILE], level=logging.DEBUG,
 95 |                     format=APP.config[ConfKey.LOG_FORMAT])
 96 | logging.info("Started Wiki-DP Portal app.")
 97 | logging.debug("Configured logging.")
 98 | logging.debug("Logging in directory %s", APP.config[ConfKey.LOG_FILE])
 99 | logging.debug("Application configured with languages=%s",
100 |               APP.config[ConfKey.WIKIBASE_LANGUAGE])
101 | 


--------------------------------------------------------------------------------
/wikidp/const.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/python
  2 | # coding=UTF-8
  3 | #
  4 | # WikiDP Wikidata Portal
  5 | # Copyright (C) 2020
  6 | # All rights reserved.
  7 | #
  8 | # This code is distributed under the terms of the GNU General Public
  9 | # License, Version 3. See the text file "COPYING" for further details
 10 | # about the terms of this license.
 11 | #
 12 | """Constants used across BitCurator modules.
 13 | 
 14 | These need to map to the names used in the default config file, but better
 15 | than multiple hardcoded strings in code.
 16 | """
 17 | ENTITY_URL_PATTERN = 'http://www.wikidata.org/entity/$1'
 18 | PROPERTY_REGEX = r'(P|p)\d+'
 19 | PUID_REGEX = r"[x-]?fmt/\d+"
 20 | LANG = 'en'
 21 | FALLBACK_LANG = 'en'
 22 | DEFAULT_UI_LANGUAGES = [
 23 |     (LANG, "English"),
 24 |     ("fr", "français (French)"),
 25 |     ("es", "español (Spanish)"),
 26 |     ("de", "Deutsch (German)"),
 27 |     ("da", "dansk (Danish)"),
 28 |     ("nl", "Nederlands (Dutch)"),
 29 |     ("zh", "汉语 (Chinese)"),
 30 |     ("ar", "العربية (Arabic)"),
 31 |     ("it", "italiano (Italian)"),
 32 |     ("lv", "latviešu valoda (Latvian)"),
 33 |     ("et", "eesti keel (Estonian)"),
 34 |     ("fi", "suomi (Finnish)"),
 35 |     ("pt", "português (Portuguese)"),
 36 |     ("sv", "svenska (Swedish)"),
 37 |     ("no", "Norsk (Norwegian)"),
 38 |     ("ja", "日本語 (Japanese)"),
 39 |     ("hi", "हिन्दी (Hindi)"),
 40 |     ("lb", "Lëtzebuergesch (Luxembourgish)")
 41 | ]
 42 | WIKIDATA_ENTITY_BASE_URL = "https://wikidata.org/entity"
 43 | WIKIMEDIA_COMMONS_BASE_URL = "https://commons.wikimedia.org"
 44 | WIKIMEDIA_COMMONS_API_URL = f"{WIKIMEDIA_COMMONS_BASE_URL}/w/api.php"
 45 | WIKIDATA_DATETIME_FORMAT = '+%Y-%m-%dT%H:%M:%SZ'
 46 | 
 47 | 
 48 | # pylint: disable=R0903
 49 | class ConfKey:
 50 |     """Config key string constants."""
 51 | 
 52 |     ITEM_REGEX = 'ITEM_REGEX'
 53 |     LOG_FILE = 'LOG_FILE'
 54 |     LOG_FORMAT = 'LOG_FORMAT'
 55 |     PORT = "PORT"
 56 |     PROPERTY_REGEX = 'PROPERTY_REGEX'
 57 |     STATIC_DIR = "STATIC_DIR"
 58 |     MEDIAWIKI_API_URL = 'MEDIAWIKI_API_URL'
 59 |     OAUTH_MEDIAWIKI_URL = 'OAUTH_MEDIAWIKI_URL'
 60 |     SPARQL_ENDPOINT_URL = 'SPARQL_ENDPOINT_URL'
 61 |     USER_AGENT = 'USER_AGENT'
 62 |     WIKIBASE_LANGUAGE = 'WIKIBASE_LANGUAGE'
 63 |     WIKIDATA_FB_LANG = 'WIKIDATA_FB_LANG'
 64 |     WIKIDATA_LANG = 'WIKIDATA_LANG'
 65 |     WIKIDATA_PASSWORD = 'WIKIDATA_PASSWORD'
 66 |     WIKIDATA_USER_NAME = 'WIKIDATA_USER_NAME'
 67 |     WIKIDP_CONSUMER_KEY = 'WIKIDP_CONSUMER_KEY'
 68 |     WIKIDP_CONSUMER_SECRET = 'WIKIDP_CONSUMER_SECRET'
 69 | 
 70 | 
 71 | class WDEntityField:
 72 |     """Enum of field names for describing Wikidata Entities."""
 73 | 
 74 |     ALIASES = 'aliases'
 75 |     LABEL = 'label'
 76 |     DESCRIPTION = 'description'
 77 |     QID = 'qid'
 78 |     CLAIMS = 'claims'
 79 |     EXTERNAL_LINKS = 'external_links'
 80 |     CATEGORIES = 'categories'
 81 |     REFERENCES = 'references'
 82 |     QUALIFIERS = 'qualifiers'
 83 |     URL = 'url'
 84 | 
 85 | 
 86 | DEFAULT_PID_LIST = [
 87 |     "P9151",
 88 |     "P9100",
 89 |     "P8778",
 90 |     "P8709",
 91 |     "P8443",
 92 |     "P8205",
 93 |     "P7967",
 94 |     "P7966",
 95 |     "P7510",
 96 |     "P7126",
 97 |     "P6931",
 98 |     "P6665",
 99 |     "P6366",
100 |     "P5688",
101 |     "P5398",
102 |     "P5247",
103 |     "P5047",
104 |     "P4969",
105 |     "P4839",
106 |     "P4506",
107 |     "P4460",
108 |     "P4428",
109 |     "P4162",
110 |     "P4152",
111 |     "P3984",
112 |     "P3966",
113 |     "P3827",
114 |     "P3743",
115 |     "P3641",
116 |     "P3499",
117 |     "P3473",
118 |     "P3463",
119 |     "P3454",
120 |     "P3442",
121 |     "P3381",
122 |     "P3374",
123 |     "P3266",
124 |     "P2748",
125 |     "P2537",
126 |     "P2283",
127 |     "P2209",
128 |     "P2179",
129 |     "P2093",
130 |     "P2078",
131 |     "P2037",
132 |     "P1889",
133 |     "P1814",
134 |     "P1813",
135 |     "P1547",
136 |     "P1482",
137 |     "P1366",
138 |     "P1365",
139 |     "P1343",
140 |     "P1065",
141 |     "P973",
142 |     "P856",
143 |     "P854",
144 |     "P348",
145 |     "P144",
146 |     "P856",
147 |     "P527",
148 |     "P577",
149 |     "P1889",
150 |     "P571",
151 |     "P18",
152 |     "P275",
153 |     "P154",
154 |     "P503",
155 |     "P277",
156 |     "P138",
157 | ]
158 | 


--------------------------------------------------------------------------------
/wikidp/controllers/__init__.py:
--------------------------------------------------------------------------------
1 | """Module for application logic controllers."""
2 | 


--------------------------------------------------------------------------------
/wikidp/controllers/auth.py:
--------------------------------------------------------------------------------
 1 | """Methods for user authentication and authorization."""
 2 | from flask import session
 3 | import jsonpickle
 4 | from mwoauth import (
 5 |     ConsumerToken,
 6 |     AccessToken,
 7 |     identify,
 8 | )
 9 | from wikidataintegrator.wdi_login import WDLogin
10 | 
11 | from wikidp.config import APP
12 | from wikidp.const import ConfKey
13 | 
14 | MEDIAWIKI_API_URL = APP.config[ConfKey.MEDIAWIKI_API_URL]
15 | OAUTH_MEDIAWIKI_URL = APP.config[ConfKey.OAUTH_MEDIAWIKI_URL]
16 | WIKIDP_CONSUMER_KEY = APP.config[ConfKey.WIKIDP_CONSUMER_KEY]
17 | WIKIDP_CONSUMER_SECRET = APP.config[ConfKey.WIKIDP_CONSUMER_SECRET]
18 | WIKIDP_CONSUMER_TOKEN = ConsumerToken(WIKIDP_CONSUMER_KEY,
19 |                                       WIKIDP_CONSUMER_SECRET)
20 | USER_AGENT = APP.config[ConfKey.USER_AGENT]
21 | 
22 | 
23 | def identify_user():
24 |     """Return the user identity object obtained from the session WDI login."""
25 |     # Get the WDI login object
26 |     wdi_login_obj = get_wdi_login()
27 |     access_token = AccessToken(
28 |         wdi_login_obj.s.auth.client.resource_owner_key,
29 |         wdi_login_obj.s.auth.client.resource_owner_secret
30 |     )
31 |     return identify(OAUTH_MEDIAWIKI_URL, WIKIDP_CONSUMER_TOKEN, access_token)
32 | 
33 | 
34 | def is_authenticated():
35 |     """Return true if a user is authenticated, otherwise false."""
36 |     if session.get('username'):
37 |         return True
38 |     return False
39 | 
40 | 
41 | def store_wdi_login(wdi_login_obj):
42 |     """
43 |     Store the WDI login object into the session.
44 | 
45 |     Args:
46 |         wdi_login_obj (WDLogin):
47 | 
48 |     Returns:
49 | 
50 |     """
51 |     session['wdilogin'] = jsonpickle.encode(wdi_login_obj)
52 | 
53 | 
54 | def get_wdi_login():
55 |     """
56 |     Get the WDI login object from session.
57 | 
58 |     Args:
59 | 
60 |     Returns (WDLogin):
61 | 
62 |     """
63 |     return jsonpickle.decode(session['wdilogin'])
64 | 
65 | 
66 | def build_wdi_login():
67 |     """
68 |     Build the WDI Login Object with Oauth Consumer.
69 | 
70 |     Returns (WDLogin):
71 | 
72 |     """
73 |     return WDLogin(consumer_key=WIKIDP_CONSUMER_TOKEN.key,
74 |                    consumer_secret=WIKIDP_CONSUMER_TOKEN.secret,
75 |                    callback_url='oob',
76 |                    mediawiki_api_url=MEDIAWIKI_API_URL,
77 |                    user_agent=USER_AGENT)
78 | 
79 | 
80 | def login():
81 |     """Get the current user and store the username in the session."""
82 |     identity = identify_user()
83 |     session["username"] = identity['username']
84 | 
85 | 
86 | def logout():
87 |     """Remove the current user from the session."""
88 |     session.pop('username')
89 | 


--------------------------------------------------------------------------------
/wikidp/controllers/pages.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/python
 2 | # coding=UTF-8
 3 | #
 4 | # WikiDP Wikidata Portal
 5 | # Copyright (C) 2020
 6 | # All rights reserved.
 7 | #
 8 | # This code is distributed under the terms of the GNU General Public
 9 | # License, Version 3. See the text file "COPYING" for further details
10 | # about the terms of this license.
11 | #
12 | # This is a python __init__ script to create the app and import the
13 | # main package contents
14 | """Module for WikiDP pages."""
15 | from flask import request
16 | 
17 | from wikidp.controllers.api import get_property_checklist_from_schema
18 | from wikidp.utils import (
19 |     get_directory_filenames_with_subdirectories,
20 |     get_item_property_counts,
21 |     item_detail_parse,
22 | )
23 | 
24 | SCHEMA_DIRECTORY_PATH = 'wikidp/schemas/'
25 | 
26 | 
27 | def get_item_context(qid, with_claims=True):
28 |     """
29 |     Retrieve an item by QID with accompanying claims if requested.
30 | 
31 |     Args:
32 |         qid (str):
33 |         with_claims (bool):
34 | 
35 |     Returns (Tuple[dict, Optional[list], Optional[list]]):
36 | 
37 |     """
38 |     selected_item = item_detail_parse(qid, with_claims=with_claims)
39 |     options = None
40 |     schemas = None
41 |     if selected_item:
42 |         _options = request.args.get('options', default=None, type=str)
43 |         options = _options.split(',') if _options else [qid]
44 |         schemas = get_schema_list()
45 |     return selected_item, options, schemas
46 | 
47 | 
48 | def get_checklist_context(qid, schema):
49 |     """
50 |     Create a property checklist render context from a schema.
51 | 
52 |     Args:
53 |         qid (str):
54 |         schema (str):
55 | 
56 |     Returns (List[Dict]):
57 | 
58 |     """
59 |     checklist = get_property_checklist_from_schema(schema)
60 |     if checklist:
61 |         counts = get_item_property_counts(qid)
62 |         output = [{
63 |             "id": prop['id'],
64 |             "label": prop['propertyLabel'],
65 |             "hidden": prop.get('hidden', False),
66 |             "description": prop['propertyDescription'],
67 |             "type": prop["value_type"],
68 |             "count": counts.get(prop['id'], 0),
69 |             "qualifiers": prop["qualifiers"],
70 |         } for prop in checklist]
71 |         return output
72 |     return []
73 | 
74 | 
75 | def get_schema_list():
76 |     """Get a flat list of schema files."""
77 |     return get_directory_filenames_with_subdirectories(SCHEMA_DIRECTORY_PATH)
78 | 


--------------------------------------------------------------------------------
/wikidp/controllers/search.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/python
 2 | # coding=UTF-8
 3 | #
 4 | # WikiDP Wikidata Portal
 5 | # Copyright (C) 2020
 6 | # All rights reserved.
 7 | #
 8 | # This code is distributed under the terms of the GNU General Public
 9 | # License, Version 3. See the text file "COPYING" for further details
10 | # about the terms of this license.
11 | #
12 | # This is a python __init__ script to create the app and import the
13 | # main package contents
14 | """Search Controller Functions and Helpers for WikiDP."""
15 | 
16 | import logging
17 | import re
18 | 
19 | from wikidataintegrator.wdi_core import WDItemEngine
20 | 
21 | from wikidp.config import APP
22 | from wikidp.const import (
23 |     ConfKey,
24 |     PUID_REGEX,
25 |     WDEntityField,
26 | )
27 | from wikidp.models import (
28 |     FileFormatExtSearchResult,
29 |     PuidSearchResult,
30 | )
31 | from wikidp.utils import (
32 |     dedupe_by_key,
33 |     item_detail_parse,
34 | )
35 | 
36 | MEDIAWIKI_API_URL = APP.config[ConfKey.MEDIAWIKI_API_URL]
37 | WIKIDATA_LANG = APP.config[ConfKey.WIKIDATA_LANG]
38 | 
39 | 
40 | def get_search_result_context(search_string):
41 |     """
42 |     Get search results from a substring.
43 | 
44 |     Args:
45 |         search_string (str):
46 | 
47 |     Returns (List[Optional[Dict]]):
48 | 
49 |     """
50 |     # Check if searching by file extension
51 |     context = [
52 |         {
53 |             WDEntityField.QID: res.format,
54 |             WDEntityField.LABEL: res.label,
55 |             WDEntityField.DESCRIPTION: res.description,
56 |         }
57 |         for res in FileFormatExtSearchResult.search(search_string)
58 |     ]
59 |     # Check if searching with PUID
60 |     if re.search(PUID_REGEX, search_string):
61 |         context.extend([
62 |             {
63 |                 WDEntityField.QID: res.format,
64 |                 WDEntityField.LABEL: res.label,
65 |                 WDEntityField.DESCRIPTION: res.description,
66 |             }
67 |             for res in PuidSearchResult.search_puid(search_string,
68 |                                                     lang=WIKIDATA_LANG)
69 |         ])
70 |     # Search Wikidata natively
71 |     context.extend(search_result_list(search_string))
72 |     return dedupe_by_key(context, WDEntityField.QID)
73 | 
74 | 
75 | def search_result_list(search_string):
76 |     """
77 |     Use wikidataintegrator to generate a list of similar items.
78 | 
79 |     This is based on a text search and returns a list of
80 |     (qid, Label, description, aliases) dictionaries.
81 |     """
82 |     result_qid_list = WDItemEngine.get_wd_search_results(
83 |         search_string=search_string, language=WIKIDATA_LANG,
84 |         mediawiki_api_url=MEDIAWIKI_API_URL)
85 |     output = []
86 |     for qid in result_qid_list[:10]:
87 |         item = item_detail_parse(qid, with_claims=False)
88 |         if item:
89 |             output.append(item)
90 |     return output
91 | 
92 | 
93 | def get_search_by_puid_context(puid):
94 |     """Perform an item search by PUID."""
95 |     new_puid = puid.replace('_', '/')
96 |     logging.debug("Searching for PUID: %s", new_puid)
97 |     results = PuidSearchResult.search_puid(new_puid, lang=WIKIDATA_LANG)
98 |     return new_puid, results
99 | 


--------------------------------------------------------------------------------
/wikidp/routes/__init__.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/python
 2 | # coding=UTF-8
 3 | #
 4 | # WikiDP Wikidata Portal
 5 | # Copyright (C) 2020
 6 | # All rights reserved.
 7 | #
 8 | # This code is distributed under the terms of the GNU General Public
 9 | # License, Version 3. See the text file "COPYING" for further details
10 | # about the terms of this license.
11 | #
12 | # This is a python __init__ script to create the app and import the
13 | # main package contents
14 | """Package description file for routes."""
15 | from . import (
16 |     _converters,
17 |     _filters,
18 |     api,
19 |     forms,
20 |     oauth,
21 |     pages,
22 |     search,
23 | )
24 | 


--------------------------------------------------------------------------------
/wikidp/routes/_converters.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/python
 2 | # coding=UTF-8
 3 | #
 4 | # WikiDP Wikidata Portal
 5 | # Copyright (C) 2020
 6 | # All rights reserved.
 7 | #
 8 | # This code is distributed under the terms of the GNU General Public
 9 | # License, Version 3. See the text file "COPYING" for further details
10 | # about the terms of this license.
11 | #
12 | """Flask application custom url converters for Wikidata portal."""
13 | from werkzeug.routing import BaseConverter
14 | 
15 | from wikidp.config import APP
16 | from wikidp.const import ConfKey
17 | 
18 | 
19 | ITEM_REGEX = APP.config[ConfKey.ITEM_REGEX]
20 | PROPERTY_REGEX = APP.config[ConfKey.PROPERTY_REGEX]
21 | 
22 | 
23 | class WikidataItemConverter(BaseConverter):
24 |     """Custom Routing Mapping for Wikidata Item Identifiers such as Q1234."""
25 | 
26 |     def __init__(self, url_map):
27 |         """Use default settings with just applied regex."""
28 |         super().__init__(url_map)
29 |         self.regex = ITEM_REGEX
30 | 
31 |     def to_python(self, value):
32 |         """Value should be string with capital letters for consistency."""
33 |         return value.upper()
34 | 
35 |     def to_url(self, value):
36 |         """Value should be string with capital letters for consistency."""
37 |         return value.upper()
38 | 
39 | 
40 | class WikidataPropertyConverter(BaseConverter):
41 |     """Custom Routing Mapping for Wikidata Property Identifiers such as P11."""
42 | 
43 |     def __init__(self, url_map):
44 |         """Use default settings with just applied regex."""
45 |         super().__init__(url_map)
46 |         self.regex = PROPERTY_REGEX
47 | 
48 |     def to_python(self, value):
49 |         """Value should be string with capital letters for consistency."""
50 |         return value.upper()
51 | 
52 |     def to_url(self, value):
53 |         """Value should be string with capital letters for consistency."""
54 |         return value.upper()
55 | 
56 | 
57 | # Custom Routing Converters
58 | APP.url_map.converters['item'] = WikidataItemConverter
59 | APP.url_map.converters['prop'] = WikidataPropertyConverter
60 | 


--------------------------------------------------------------------------------
/wikidp/routes/_filters.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/python
 2 | # coding=UTF-8
 3 | #
 4 | # WikiDP Wikidata Portal
 5 | # Copyright (C) 2020
 6 | # All rights reserved.
 7 | #
 8 | # This code is distributed under the terms of the GNU General Public
 9 | # License, Version 3. See the text file "COPYING" for further details
10 | # about the terms of this license.
11 | #
12 | """Flask application custom template filters for Wikidata portal."""
13 | from urllib.parse import quote_plus
14 | 
15 | from markupsafe import Markup
16 | 
17 | from wikidp.config import APP
18 | from wikidp.const import ENTITY_URL_PATTERN
19 | from wikidp.utils import (
20 |     get_pid_from_string,
21 |     get_qid_from_string,
22 | )
23 | 
24 | 
25 | @APP.template_filter('url_encode')
26 | def template_filter_url_encode(to_encode):
27 |     """Carry out URL encoding on template filter URL."""
28 |     if isinstance(to_encode, Markup):
29 |         to_encode = to_encode.unescape()
30 |     to_encode = to_encode.encode('utf8')
31 |     to_encode = quote_plus(to_encode)
32 |     return Markup(to_encode)
33 | 
34 | 
35 | @APP.template_filter('qlabel_attributes')
36 | def template_qlabel_attributes(url):
37 |     """
38 |     Add tag attributes for qlabel.js.
39 | 
40 |     Args:
41 |         url (str): Text for the label and titles
42 | 
43 |     Returns:
44 |         str: HTML tag attributes
45 |     """
46 |     return "class=qlabel its-ta-ident-ref={}".format(url)
47 | 
48 | 
49 | @APP.template_filter('entity_url')
50 | def template_filter_entity_url(entity_id):
51 |     """
52 |     Convert Item of Property string to Wikidata URL.
53 | 
54 |     Args:
55 |         entity_id (str): Item ('Q1234') or Property ('P1234') identifier
56 | 
57 |     Returns:
58 |         str: http url destination for the entity
59 |     """
60 |     valid_string = get_qid_from_string(entity_id)
61 |     if not valid_string:
62 |         valid_string = get_pid_from_string(entity_id)
63 |     return ENTITY_URL_PATTERN.replace('$1', valid_string) if valid_string else '#'
64 | 


--------------------------------------------------------------------------------
/wikidp/routes/api.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/python
  2 | # coding=UTF-8
  3 | #
  4 | # WikiDP Wikidata Portal
  5 | # Copyright (C) 2020
  6 | # All rights reserved.
  7 | #
  8 | # This code is distributed under the terms of the GNU General Public
  9 | # License, Version 3. See the text file "COPYING" for further details
 10 | # about the terms of this license.
 11 | #
 12 | """Flask application api routes for Wikidata portal."""
 13 | import logging
 14 | 
 15 | from flask import (
 16 |     jsonify,
 17 |     request,
 18 | )
 19 | 
 20 | from wikidp.config import APP
 21 | from wikidp.controllers.api import (
 22 |     get_all_file_formats,
 23 |     get_property_checklist_from_schema,
 24 |     write_claims_to_item,
 25 | )
 26 | from wikidp.controllers.auth import get_wdi_login
 27 | from wikidp.controllers.search import search_result_list
 28 | from wikidp.utils import (
 29 |     get_all_languages,
 30 |     get_all_qualifier_properties,
 31 |     get_all_reference_properties,
 32 |     get_allowed_qualifiers_by_pid,
 33 |     get_property,
 34 |     item_detail_parse,
 35 | )
 36 | 
 37 | 
 38 | @APP.route("/api/")
 39 | def route_api_welcome():
 40 |     """Landing Page API index."""
 41 |     return 'Welcome to the WikiDP API'
 42 | 
 43 | 
 44 | @APP.route("/api/<item:qid>", methods=['GET', 'POST'])
 45 | def route_api_get_item(qid):
 46 |     """User posts a item-id and returns json of (id, label, desc, aliases)."""
 47 |     item = item_detail_parse(qid, with_claims=True)
 48 |     return jsonify(item)
 49 | 
 50 | 
 51 | @APP.route("/api/<item:qid>/summary", methods=['GET', 'POST'])
 52 | def route_api_get_item_summary(qid):
 53 |     """User posts a item-id and returns json of (id, label, desc, aliases)."""
 54 |     item = item_detail_parse(qid, with_claims=False)
 55 |     return jsonify(item)
 56 | 
 57 | 
 58 | @APP.route("/api/items", methods=['GET', 'POST'])
 59 | def route_api_get_item_summary_list():
 60 |     """
 61 |     Get a list of Wikidata Item.
 62 | 
 63 |     Returns (Response): JSON list with id, label, description, and aliases
 64 | 
 65 |     """
 66 |     if request.method == 'GET':
 67 |         qids = request.args.get('qids').split(',')
 68 |     else:
 69 |         qids = request.get_json()
 70 |     items = [item_detail_parse(qid, with_claims=False) for qid in qids]
 71 |     return jsonify(items)
 72 | 
 73 | 
 74 | @APP.route("/api/<item:qid>/claims/write", methods=['POST'])
 75 | def route_api_write_claims_to_item(qid):
 76 |     """User posts a JSON object of claims to contribute to an item."""
 77 |     logging.debug("Processing user POST request.")
 78 |     wdi_login = get_wdi_login()
 79 |     assert wdi_login, "Must be Authenticated"
 80 |     json_data = request.get_json()
 81 |     return write_claims_to_item(qid, json_data, wdi_login)
 82 | 
 83 | 
 84 | @APP.route("/api/<prop:pid>", methods=['GET', 'POST'])
 85 | def route_api_get_property(pid):
 86 |     """Return JSON representation of a property by PID."""
 87 |     prop = get_property(pid)
 88 |     return jsonify(prop)
 89 | 
 90 | 
 91 | @APP.route("/api/<prop:pid>/qualifiers", methods=['GET', 'POST'])
 92 | def route_api_get_allowed_qualifiers_by_pid(pid):
 93 |     """Return JSON representation of allowed property qualifiers by PID."""
 94 |     output = get_allowed_qualifiers_by_pid(pid)
 95 |     return jsonify(output)
 96 | 
 97 | 
 98 | @APP.route("/api/property/qualifiers", methods=['GET', 'POST'])
 99 | def route_api_get_all_qualifier_properties():
100 |     """Return JSON representation of all property qualifiers."""
101 |     output = get_all_qualifier_properties()
102 |     return jsonify(output)
103 | 
104 | 
105 | @APP.route("/api/property/references", methods=['GET', 'POST'])
106 | def route_api_get_all_reference_properties():
107 |     """
108 |     Get JSON representation of all reference properties.
109 | 
110 |     Returns (Response):
111 | 
112 |     """
113 |     output = get_all_reference_properties()
114 |     return jsonify(output)
115 | 
116 | 
117 | @APP.route("/api/language/", methods=['GET', 'POST'])
118 | def route_api_get_all_languages():
119 |     """Return JSON representation of all supported languages."""
120 |     output = get_all_languages()
121 |     return jsonify(output)
122 | 
123 | 
124 | @APP.route("/api/search/<search_string>", methods=['GET', 'POST'])
125 | def route_api_search_item_by_string(search_string):
126 |     """Post string, returns list of json of (id, label, desc, aliases)."""
127 |     _string = search_string.strip()
128 |     output = search_result_list(_string)
129 |     return jsonify(output)
130 | 
131 | 
132 | @APP.route("/api/schema/<path:schema_name>/properties")
133 | def route_api_get_properties_by_schema(schema_name):
134 |     """Return a JSON representation of properties from a particular schema."""
135 |     prop_list = get_property_checklist_from_schema(schema_name)
136 |     return jsonify(prop_list)
137 | 
138 | 
139 | @APP.route("/api/browse/file_format", methods=['GET', 'POST'])
140 | def route_api_browse_file_format():
141 |     """Return a JSON representation of a list of all file formats."""
142 |     format_list = get_all_file_formats()
143 |     return jsonify([x.api_dict() for x in format_list])
144 | 


--------------------------------------------------------------------------------
/wikidp/routes/forms.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/python
 2 | # coding=UTF-8
 3 | #
 4 | # WikiDP Wikidata Portal
 5 | # Copyright (C) 2020
 6 | # All rights reserved.
 7 | #
 8 | # This code is distributed under the terms of the GNU General Public
 9 | # License, Version 3. See the text file "COPYING" for further details
10 | # about the terms of this license.
11 | #
12 | # This is a python __init__ script to create the app and import the
13 | # main package contents
14 | """Module for application form routes."""
15 | import json
16 | 
17 | from flask import (
18 |     redirect,
19 |     request,
20 | )
21 | 
22 | from wikidp.config import APP
23 | 
24 | 
25 | def _get_page_redirect(action):
26 |     """
27 |     Get the Redirect with Formatted Deep-linking.
28 | 
29 |     Args:
30 |         action (Literal['contribute', 'preview']):
31 | 
32 |     Returns (Response):
33 | 
34 |     """
35 |     qid = request.form['qid']
36 |     options = ",".join(json.loads(request.form['optionList']))
37 |     return redirect(f'/{qid}/{action}?qid={qid}&options={options}')
38 | 
39 | 
40 | @APP.route("/preview", methods=['POST'])
41 | def route_form_preview_item():
42 |     """
43 |     Show a preview of a selected search result.
44 | 
45 |     Returns (Response):
46 | 
47 |     """
48 |     return _get_page_redirect('preview')
49 | 
50 | 
51 | @APP.route("/contribute", methods=['POST'])
52 | def route_form_contribute_item():
53 |     """
54 |     Process contribute page into a state-saving url.
55 | 
56 |     Returns (Response):
57 | 
58 |     """
59 |     return _get_page_redirect('contribute')
60 | 


--------------------------------------------------------------------------------
/wikidp/routes/oauth.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/python
  2 | # coding=UTF-8
  3 | #
  4 | # WikiDP Wikidata Portal
  5 | # Copyright (C) 2021
  6 | # All rights reserved.
  7 | #
  8 | # This code is distributed under the terms of the GNU General Public
  9 | # License, Version 3. See the text file "COPYING" for further details
 10 | # about the terms of this license.
 11 | #
 12 | """Flask MWOAuth and WikiDataIntegrator come together."""
 13 | import json
 14 | import logging
 15 | 
 16 | from flask import (
 17 |     jsonify,
 18 |     redirect,
 19 |     render_template,
 20 |     request,
 21 |     session,
 22 | )
 23 | from wikidataintegrator.wdi_core import WDItemEngine
 24 | 
 25 | from wikidp.config import APP
 26 | from wikidp.controllers.auth import (
 27 |     build_wdi_login,
 28 |     get_wdi_login,
 29 |     identify_user,
 30 |     is_authenticated,
 31 |     login,
 32 |     logout,
 33 |     store_wdi_login,
 34 | )
 35 | 
 36 | 
 37 | @APP.route("/", methods=['POST'])
 38 | def route_page_login():
 39 |     """
 40 |     Handle the user login via Oauth.
 41 | 
 42 |     Returns (Response): JSON Response
 43 | 
 44 |     """
 45 |     body = json.loads(request.get_data())
 46 |     if 'url' not in body:
 47 |         return redirect('/')
 48 | 
 49 |     # parse the url from wikidata for the oauth token and secret
 50 |     if is_authenticated():
 51 |         return jsonify(body)
 52 |     wdi_login_obj = get_wdi_login()
 53 |     callback = body['url'].encode("utf-8")
 54 |     wdi_login_obj.continue_oauth(oauth_callback_data=callback)
 55 |     store_wdi_login(wdi_login_obj)
 56 |     login()
 57 |     return jsonify(body)
 58 | 
 59 | 
 60 | @APP.route("/profile", methods=['POST', 'GET'])
 61 | def profile():
 62 |     """Flask OAuth login."""
 63 |     logging.info("Checking user profile")
 64 |     if request.method == 'POST':
 65 |         logging.info("POST so getting data")
 66 |         body = json.loads(request.get_data())
 67 |         if 'initiate' in body.keys():
 68 |             wdi_login_obj = build_wdi_login()
 69 |             store_wdi_login(wdi_login_obj)
 70 |             response_data = {
 71 |                 'wikimediaURL': wdi_login_obj.redirect
 72 |             }
 73 |             return jsonify(response_data)
 74 |     elif request.method == 'GET' and request.args.get('logout'):
 75 |         logout()
 76 | 
 77 |     return render_template('profile.html', username=session.get('username', ''))
 78 | 
 79 | 
 80 | @APP.route("/auth")
 81 | def route_authentication():
 82 |     """
 83 |     Get a simple JSON structure confirming user authentication.
 84 | 
 85 |     Notes:
 86 |         - Simple GET service that returns a tiny dictionary informing the
 87 |         caller as to whether the current session user is authenticated,
 88 |         accompanied by their user name if they are.
 89 | 
 90 |     Returns (Response):
 91 | 
 92 |     """
 93 |     is_user_authenticated = is_authenticated()
 94 |     response_data = {
 95 |         'auth': is_user_authenticated,
 96 |         'username': session.get('username', ''),
 97 |     }
 98 |     return jsonify(response_data)
 99 | 
100 | 
101 | @APP.route("/oauth-write-test")
102 | def _temp_route_oauth_write_test():
103 |     # One-off test to ensure pipes are running, add an alias to WikiDP item
104 |     identity = identify_user()
105 |     for key in identity.keys():
106 |         logging.info('KEY: %s VALUE: %s', key, identity.get(key))
107 |     item = WDItemEngine(wd_item_id="Q51139559")
108 |     item.set_aliases(['WikiDP Application'], append=True)
109 |     # verify the api is working by getting this item
110 |     assert item.get_label() == "Wikidata for Digital Preservation"
111 |     wdi_login = get_wdi_login()
112 |     # verify edit token exists, this is what WDI calls
113 |     assert wdi_login.get_edit_token()
114 |     assert "user" in identity.get('groups')  # verify user in user group
115 |     # verify user in user group
116 |     assert "autoconfirmed" in identity.get('groups')
117 |     assert "edit" in identity.get('rights')  # verify user in user group
118 |     assert "editpage" in identity.get('grants')  # verify user in user group
119 |     updated = item.write(wdi_login)  # fails due to no permissions
120 |     return jsonify(updated)
121 | 


--------------------------------------------------------------------------------
/wikidp/routes/pages.py:
--------------------------------------------------------------------------------
  1 | """Module to hold Web App page routing and parameter handling."""
  2 | import logging
  3 | 
  4 | from flask import (
  5 |     abort,
  6 |     redirect,
  7 |     render_template,
  8 |     send_from_directory,
  9 | )
 10 | 
 11 | from wikidp.config import APP
 12 | from wikidp.const import DEFAULT_UI_LANGUAGES
 13 | from wikidp.controllers.pages import (
 14 |     get_checklist_context,
 15 |     get_item_context,
 16 | )
 17 | 
 18 | 
 19 | @APP.route("/", methods=['GET'])
 20 | def route_page_welcome():
 21 |     """Landing Page for first time."""
 22 |     return render_template('welcome.html')
 23 | 
 24 | 
 25 | @APP.route('/favicon.ico')
 26 | def route_favicon():
 27 |     """Serve the favicon file."""
 28 |     return send_from_directory(APP.config['STATIC_DIR'], 'img/favicon.ico',
 29 |                                mimetype='image/vnd.microsoft.icon')
 30 | 
 31 | 
 32 | @APP.route("/about")
 33 | def route_page_about():
 34 |     """Render the about page."""
 35 |     return render_template('about.html')
 36 | 
 37 | 
 38 | @APP.route("/reports")
 39 | def route_page_reports():
 40 |     """Render the reports page."""
 41 |     return render_template('reports.html')
 42 | 
 43 | 
 44 | @APP.route("/unauthorized")
 45 | def route_page_unauthorized():
 46 |     """Display a 403 error page."""
 47 |     return abort(403)
 48 | 
 49 | 
 50 | @APP.route("/error")
 51 | def route_page_error():
 52 |     """Display a 500 error page."""
 53 |     return abort(500)
 54 | 
 55 | 
 56 | @APP.route("/<item:qid>")
 57 | def route_page_selected_item(qid):
 58 |     """If the item ID is already known, the user can enter in the url."""
 59 |     return redirect('/'+qid+'/preview')
 60 | 
 61 | 
 62 | @APP.route("/<item:qid>/preview")
 63 | def route_item_preview(qid):
 64 |     """If the item ID is already known, the user can enter in the url."""
 65 |     selected_item, options, schemas = get_item_context(qid, with_claims=True)
 66 |     if selected_item:
 67 |         return render_template('item_preview.html', item=selected_item,
 68 |                                options=options, schemas=schemas,
 69 |                                languages=DEFAULT_UI_LANGUAGES,
 70 |                                page='preview')
 71 |     return abort(404)
 72 | 
 73 | 
 74 | @APP.route("/<item:qid>/contribute")
 75 | def route_item_contribute(qid):
 76 |     """Handle a user's contributed statements."""
 77 |     selected_item, options, schemas = get_item_context(qid, with_claims=False)
 78 |     if selected_item:
 79 |         return render_template('item_contribute.html', item=selected_item,
 80 |                                options=options, schemas=schemas,
 81 |                                languages=DEFAULT_UI_LANGUAGES,
 82 |                                page='contribute')
 83 |     return abort(404)
 84 | 
 85 | 
 86 | @APP.route("/<item:qid>/checklist/<path:schema>")
 87 | def route_item_checklist_by_schema(qid, schema):
 88 |     """Render the correct item checklist depending on schema."""
 89 |     properties = get_checklist_context(qid, schema)
 90 |     return render_template('snippets/property_checklist.html',
 91 |                            properties=properties)
 92 | 
 93 | 
 94 | @APP.errorhandler(400)
 95 | @APP.errorhandler(404)
 96 | def route_page_error__not_found(excep):
 97 |     """Handle 404 resource not found problems."""
 98 |     logging.exception('Not Found: %s', excep)
 99 |     return render_template('error.html', message="Page Not Found"), 404
100 | 
101 | 
102 | @APP.errorhandler(403)
103 | def route_page_error__forbidden(excep):
104 |     """Handle HTTP 403, Forbidden."""
105 |     logging.exception('Forbidden: %s', excep)
106 |     message = "You are not authorized to view this page."
107 |     return render_template('error.html', message=message), 403
108 | 
109 | 
110 | @APP.errorhandler(500)
111 | @APP.errorhandler(Exception)
112 | def route_page_error__internal_error(excep):
113 |     """Handle general server errors."""
114 |     logging.exception('Internal Server Error: %s', excep)
115 |     message = "Internal Error. Please Help us by reporting " \
116 |               "this to our admin team! Thank you."
117 |     return render_template('error.html', message=message), 500
118 | 


--------------------------------------------------------------------------------
/wikidp/routes/search.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/python
 2 | # coding=UTF-8
 3 | #
 4 | # WikiDP Wikidata Portal
 5 | # Copyright (C) 2020
 6 | # All rights reserved.
 7 | #
 8 | # This code is distributed under the terms of the GNU General Public
 9 | # License, Version 3. See the text file "COPYING" for further details
10 | # about the terms of this license.
11 | #
12 | # This is a python __init__ script to create the app and import the
13 | # main package contents
14 | """Module to handle web page search functions."""
15 | from flask import redirect, render_template, request
16 | from wikidp.config import APP
17 | from wikidp.controllers import search as search_controller
18 | 
19 | 
20 | @APP.route("/search", methods=['POST'])
21 | def route_process_site_search():
22 |     """
23 |     Process a search request into a state-saving url.
24 | 
25 |     Returns (Response):
26 | 
27 |     """
28 |     query = request.form['userInput'].strip()
29 |     return redirect(f'/search?q={query}')
30 | 
31 | 
32 | @APP.route("/search")
33 | def route_site_search():
34 |     """
35 |     Display the most likely results of a users search.
36 | 
37 |     Notes:
38 |         if only one result returned,
39 |         then user is automatic redirected to preview that item.
40 | 
41 |     Returns (Response):
42 | 
43 |     """
44 |     search_string = request.args.get('q', default='', type=str)
45 |     context = search_controller.get_search_result_context(search_string)
46 |     if len(context) == 1:
47 |         return redirect(f'/{context[0]["qid"]}')
48 |     return render_template('search_results.html', options=context)
49 | 
50 | 
51 | @APP.route("/search/puid/<string:puid>")
52 | def route_search_by_puid(puid):
53 |     """Display a list of extensions and media types."""
54 |     new_puid, results = search_controller.get_search_by_puid_context(puid)
55 |     return render_template('puid_results.html', results=results, puid=new_puid)
56 | 


--------------------------------------------------------------------------------
/wikidp/schemas/emulator.json:
--------------------------------------------------------------------------------
  1 | {
  2 |   "type": "Schema",
  3 |   "start": "https://rawgit.com/shexSpec/shex.js/master/packages/shex-webapp/doc/E91#emulator",
  4 |   "shapes": [
  5 |     {
  6 |       "type": "Shape",
  7 |       "id": "https://rawgit.com/shexSpec/shex.js/master/packages/shex-webapp/doc/E91#emulator",
  8 |       "expression": {
  9 |         "type": "EachOf",
 10 |         "expressions": [
 11 |           {
 12 |             "type": "TripleConstraint",
 13 |             "predicate": "http://www.wikidata.org/prop/P31",
 14 |             "valueExpr": {
 15 |               "type": "Shape",
 16 |               "expression": {
 17 |                 "type": "TripleConstraint",
 18 |                 "predicate": "http://www.wikidata.org/prop/statement/P31",
 19 |                 "valueExpr": {
 20 |                   "type": "NodeConstraint",
 21 |                   "values": [
 22 |                     "http://www.wikidata.org/entity/Q202871"
 23 |                   ]
 24 |                 }
 25 |               }
 26 |             }
 27 |           },
 28 |           {
 29 |             "type": "TripleConstraint",
 30 |             "predicate": "http://www.wikidata.org/prop/P856",
 31 |             "valueExpr": {
 32 |               "type": "Shape",
 33 |               "expression": {
 34 |                 "type": "TripleConstraint",
 35 |                 "predicate": "http://www.wikidata.org/prop/statement/P856",
 36 |                 "valueExpr": {
 37 |                   "type": "NodeConstraint",
 38 |                   "nodeKind": "iri"
 39 |                 }
 40 |               }
 41 |             },
 42 |             "min": 0,
 43 |             "max": -1
 44 |           },
 45 |           {
 46 |             "type": "TripleConstraint",
 47 |             "predicate": "http://www.wikidata.org/prop/P275",
 48 |             "valueExpr": {
 49 |               "type": "Shape",
 50 |               "expression": {
 51 |                 "type": "TripleConstraint",
 52 |                 "predicate": "http://www.wikidata.org/prop/statement/P275",
 53 |                 "valueExpr": {
 54 |                   "type": "NodeConstraint",
 55 |                   "nodeKind": "iri"
 56 |                 }
 57 |               }
 58 |             },
 59 |             "min": 0,
 60 |             "max": -1
 61 |           },
 62 |           {
 63 |             "type": "TripleConstraint",
 64 |             "predicate": "http://www.wikidata.org/prop/P306",
 65 |             "valueExpr": {
 66 |               "type": "Shape",
 67 |               "expression": {
 68 |                 "type": "TripleConstraint",
 69 |                 "predicate": "http://www.wikidata.org/prop/statement/P306",
 70 |                 "valueExpr": {
 71 |                   "type": "NodeConstraint",
 72 |                   "nodeKind": "iri"
 73 |                 }
 74 |               }
 75 |             },
 76 |             "min": 0,
 77 |             "max": -1
 78 |           },
 79 |           {
 80 |             "type": "TripleConstraint",
 81 |             "predicate": "http://www.wikidata.org/prop/P348",
 82 |             "valueExpr": {
 83 |               "type": "Shape",
 84 |               "expression": {
 85 |                 "type": "TripleConstraint",
 86 |                 "predicate": "http://www.wikidata.org/prop/statement/P348",
 87 |                 "valueExpr": {
 88 |                   "type": "NodeConstraint",
 89 |                   "datatype": "http://www.w3.org/2001/XMLSchema#string"
 90 |                 }
 91 |               }
 92 |             },
 93 |             "min": 0,
 94 |             "max": -1
 95 |           },
 96 |           {
 97 |             "type": "TripleConstraint",
 98 |             "predicate": "http://www.wikidata.org/prop/P178",
 99 |             "valueExpr": {
100 |               "type": "Shape",
101 |               "expression": {
102 |                 "type": "TripleConstraint",
103 |                 "predicate": "http://www.wikidata.org/prop/statement/P178",
104 |                 "valueExpr": {
105 |                   "type": "NodeConstraint",
106 |                   "nodeKind": "iri"
107 |                 }
108 |               }
109 |             },
110 |             "min": 0,
111 |             "max": -1
112 |           },
113 |           {
114 |             "type": "TripleConstraint",
115 |             "predicate": "http://www.wikidata.org/prop/P4043",
116 |             "valueExpr": {
117 |               "type": "Shape",
118 |               "expression": {
119 |                 "type": "TripleConstraint",
120 |                 "predicate": "http://www.wikidata.org/prop/statement/P4043",
121 |                 "valueExpr": {
122 |                   "type": "NodeConstraint",
123 |                   "nodeKind": "iri"
124 |                 }
125 |               }
126 |             },
127 |             "min": 0,
128 |             "max": -1
129 |           },
130 |           {
131 |             "type": "TripleConstraint",
132 |             "predicate": "http://www.wikidata.org/prop/direct/P4107",
133 |             "valueExpr": {
134 |               "type": "NodeConstraint",
135 |               "datatype": "http://www.w3.org/2001/XMLSchema#string"
136 |             },
137 |             "min": 0,
138 |             "max": 1
139 |           },
140 |           {
141 |             "type": "TripleConstraint",
142 |             "predicate": "http://www.wikidata.org/prop/direct/P5117",
143 |             "valueExpr": {
144 |               "type": "NodeConstraint",
145 |               "datatype": "http://www.w3.org/2001/XMLSchema#string"
146 |             },
147 |             "min": 0,
148 |             "max": 1
149 |           },
150 |           {
151 |             "type": "TripleConstraint",
152 |             "predicate": "http://www.wikidata.org/prop/direct/P2537",
153 |             "valueExpr": {
154 |               "type": "NodeConstraint",
155 |               "datatype": "http://www.w3.org/2001/XMLSchema#string"
156 |             },
157 |             "min": 0,
158 |             "max": 1
159 |           },
160 |           {
161 |             "type": "TripleConstraint",
162 |             "predicate": "http://www.wikidata.org/prop/direct/P6665",
163 |             "valueExpr": {
164 |               "type": "NodeConstraint",
165 |               "datatype": "http://www.w3.org/2001/XMLSchema#string"
166 |             },
167 |             "min": 0,
168 |             "max": 1
169 |           }
170 |         ]
171 |       }
172 |     }
173 |   ],
174 |   "@context": "http://www.w3.org/ns/shex.jsonld"
175 | }
176 | 


--------------------------------------------------------------------------------
/wikidp/schemas/file_format/file_format_id_pattern.json:
--------------------------------------------------------------------------------
  1 | {
  2 |   "type": "Schema",
  3 |   "start": "https://rawgit.com/shexSpec/shex.js/extends/packages/shex-webapp/doc/shex-simple.html#file_format_with_identification_pattern",
  4 |   "shapes": [
  5 |     {
  6 |       "type": "Shape",
  7 |       "id": "https://rawgit.com/shexSpec/shex.js/extends/packages/shex-webapp/doc/shex-simple.html#file_format_with_identification_pattern",
  8 |       "expression": {
  9 |         "type": "EachOf",
 10 |         "expressions": [
 11 |           {
 12 |             "type": "TripleConstraint",
 13 |             "predicate": "http://www.wikidata.org/prop/P31",
 14 |             "valueExpr": {
 15 |               "type": "Shape",
 16 |               "expression": {
 17 |                 "type": "TripleConstraint",
 18 |                 "predicate": "http://www.wikidata.org/prop/statement/P31",
 19 |                 "valueExpr": {
 20 |                   "type": "NodeConstraint",
 21 |                   "values": [
 22 |                     "http://www.wikidata.org/entity/Q235557"
 23 |                   ]
 24 |                 }
 25 |               }
 26 |             },
 27 |             "min": 0,
 28 |             "max": -1
 29 |           },
 30 |           {
 31 |             "type": "TripleConstraint",
 32 |             "predicate": "http://www.wikidata.org/prop/P279",
 33 |             "valueExpr": {
 34 |               "type": "Shape",
 35 |               "expression": {
 36 |                 "type": "TripleConstraint",
 37 |                 "predicate": "http://www.wikidata.org/prop/statement/P279",
 38 |                 "valueExpr": {
 39 |                   "type": "NodeConstraint",
 40 |                   "nodeKind": "iri"
 41 |                 }
 42 |               }
 43 |             },
 44 |             "min": 0,
 45 |             "max": -1
 46 |           },
 47 |           {
 48 |             "type": "OneOf",
 49 |             "expressions": [
 50 |               {
 51 |                 "type": "EachOf",
 52 |                 "expressions": [
 53 |                   {
 54 |                     "type": "TripleConstraint",
 55 |                     "predicate": "http://www.wikidata.org/prop/P577",
 56 |                     "valueExpr": {
 57 |                       "type": "Shape",
 58 |                       "expression": {
 59 |                         "type": "TripleConstraint",
 60 |                         "predicate": "http://www.wikidata.org/prop/statement/P577",
 61 |                         "valueExpr": {
 62 |                           "type": "NodeConstraint",
 63 |                           "datatype": "http://www.w3.org/2001/XMLSchema#dateTime"
 64 |                         }
 65 |                       }
 66 |                     }
 67 |                   },
 68 |                   {
 69 |                     "type": "TripleConstraint",
 70 |                     "predicate": "http://www.wikidata.org/prop/P571",
 71 |                     "valueExpr": {
 72 |                       "type": "Shape",
 73 |                       "expression": {
 74 |                         "type": "TripleConstraint",
 75 |                         "predicate": "http://www.wikidata.org/prop/statement/P571",
 76 |                         "valueExpr": {
 77 |                           "type": "NodeConstraint",
 78 |                           "datatype": "http://www.w3.org/2001/XMLSchema#dateTime"
 79 |                         }
 80 |                       }
 81 |                     },
 82 |                     "min": 0,
 83 |                     "max": 1
 84 |                   }
 85 |                 ]
 86 |               },
 87 |               {
 88 |                 "type": "TripleConstraint",
 89 |                 "predicate": "http://www.wikidata.org/prop/P571",
 90 |                 "valueExpr": {
 91 |                   "type": "Shape",
 92 |                   "expression": {
 93 |                     "type": "TripleConstraint",
 94 |                     "predicate": "http://www.wikidata.org/prop/statement/P571",
 95 |                     "valueExpr": {
 96 |                       "type": "NodeConstraint",
 97 |                       "datatype": "http://www.w3.org/2001/XMLSchema#dateTime"
 98 |                     }
 99 |                   }
100 |                 }
101 |               }
102 |             ],
103 |             "min": 0,
104 |             "max": -1
105 |           },
106 |           {
107 |             "type": "TripleConstraint",
108 |             "predicate": "http://www.wikidata.org/prop/P4152",
109 |             "valueExpr": "https://rawgit.com/shexSpec/shex.js/extends/packages/shex-webapp/doc/shex-simple.html#id_pattern",
110 |             "min": 1,
111 |             "max": -1
112 |           }
113 |         ]
114 |       },
115 |       "extra": [
116 |         "http://www.wikidata.org/prop/P31"
117 |       ]
118 |     },
119 |     {
120 |       "type": "Shape",
121 |       "id": "https://rawgit.com/shexSpec/shex.js/extends/packages/shex-webapp/doc/shex-simple.html#id_pattern",
122 |       "expression": {
123 |         "type": "EachOf",
124 |         "expressions": [
125 |           {
126 |             "type": "TripleConstraint",
127 |             "predicate": "http://www.wikidata.org/prop/statement/P4152",
128 |             "valueExpr": {
129 |               "type": "NodeConstraint",
130 |               "nodeKind": "literal"
131 |             },
132 |             "min": 1,
133 |             "max": -1
134 |           },
135 |           {
136 |             "type": "TripleConstraint",
137 |             "predicate": "http://www.wikidata.org/prop/qualifier/P4153",
138 |             "valueExpr": {
139 |               "type": "NodeConstraint",
140 |               "nodeKind": "literal"
141 |             },
142 |             "min": 1,
143 |             "max": -1
144 |           },
145 |           {
146 |             "type": "TripleConstraint",
147 |             "predicate": "http://www.wikidata.org/prop/qualifier/P2210",
148 |             "valueExpr": {
149 |               "type": "NodeConstraint",
150 |               "values": [
151 |                 "http://www.wikidata.org/entity/Q1148480",
152 |                 "http://www.wikidata.org/entity/Q35436009"
153 |               ]
154 |             },
155 |             "min": 1,
156 |             "max": -1
157 |           },
158 |           {
159 |             "type": "TripleConstraint",
160 |             "predicate": "http://www.wikidata.org/prop/qualifier/P3294",
161 |             "valueExpr": {
162 |               "type": "NodeConstraint",
163 |               "nodeKind": "iri"
164 |             },
165 |             "min": 1,
166 |             "max": -1
167 |           },
168 |           {
169 |             "type": "TripleConstraint",
170 |             "predicate": "http://www.w3.org/ns/prov#wasDerivedFrom",
171 |             "valueExpr": {
172 |               "type": "NodeConstraint",
173 |               "nodeKind": "iri"
174 |             }
175 |           }
176 |         ]
177 |       }
178 |     }
179 |   ],
180 |   "@context": "http://www.w3.org/ns/shex.jsonld"
181 | }
182 | 


--------------------------------------------------------------------------------
/wikidp/schemas/file_format/file_format_minimal.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "type": "Schema",
 3 |   "start": "https://raw.githubusercontent.com/shexSpec/schemas/master/Wikidata/DigitalPreservation/wikidatafileformatminimal.shex#wikidata-file_format",
 4 |   "shapes": [
 5 |     {
 6 |       "id": "https://raw.githubusercontent.com/shexSpec/schemas/master/Wikidata/DigitalPreservation/wikidatafileformatminimal.shex#wikidata-file_format",
 7 |       "type": "Shape",
 8 |       "expression": {
 9 |         "type": "EachOf",
10 |         "expressions": [
11 |           {
12 |             "type": "TripleConstraint",
13 |             "predicate": "http://www.wikidata.org/prop/P31",
14 |             "valueExpr": "https://raw.githubusercontent.com/shexSpec/schemas/master/Wikidata/DigitalPreservation/wikidatafileformatminimal.shex#P31_instance_of_file_format"
15 |           },
16 |           {
17 |             "type": "TripleConstraint",
18 |             "predicate": "http://www.wikidata.org/prop/P1163",
19 |             "valueExpr": "https://raw.githubusercontent.com/shexSpec/schemas/master/Wikidata/DigitalPreservation/wikidatafileformatminimal.shex#P1163_media_type",
20 |             "min": 0,
21 |             "max": -1
22 |           },
23 |           {
24 |             "type": "TripleConstraint",
25 |             "predicate": "http://www.wikidata.org/prop/P2748",
26 |             "valueExpr": "https://raw.githubusercontent.com/shexSpec/schemas/master/Wikidata/DigitalPreservation/wikidatafileformatminimal.shex#P2748_pronom_id",
27 |             "min": 0,
28 |             "max": -1
29 |           },
30 |           {
31 |             "type": "TripleConstraint",
32 |             "predicate": "http://www.wikidata.org/prop/P3266",
33 |             "valueExpr": "https://raw.githubusercontent.com/shexSpec/schemas/master/Wikidata/DigitalPreservation/wikidatafileformatminimal.shex#P3266_locfdd_id",
34 |             "min": 0,
35 |             "max": -1
36 |           }
37 |         ]
38 |       }
39 |     },
40 |     {
41 |       "id": "https://raw.githubusercontent.com/shexSpec/schemas/master/Wikidata/DigitalPreservation/wikidatafileformatminimal.shex#P31_instance_of_file_format",
42 |       "type": "Shape",
43 |       "expression": {
44 |         "type": "TripleConstraint",
45 |         "predicate": "http://www.wikidata.org/prop/statement/P31",
46 |         "valueExpr": {
47 |           "type": "NodeConstraint",
48 |           "values": [
49 |             "https://www.wikidata.org/wiki/Q235557"
50 |           ]
51 |         }
52 |       }
53 |     },
54 |     {
55 |       "id": "https://raw.githubusercontent.com/shexSpec/schemas/master/Wikidata/DigitalPreservation/wikidatafileformatminimal.shex#P2748_pronom_id",
56 |       "type": "Shape",
57 |       "expression": {
58 |         "type": "TripleConstraint",
59 |         "predicate": "http://www.wikidata.org/prop/statement/P2748",
60 |         "valueExpr": {
61 |           "type": "NodeConstraint",
62 |           "nodeKind": "literal"
63 |         }
64 |       }
65 |     },
66 |     {
67 |       "id": "https://raw.githubusercontent.com/shexSpec/schemas/master/Wikidata/DigitalPreservation/wikidatafileformatminimal.shex#P3266_locfdd_id",
68 |       "type": "Shape",
69 |       "expression": {
70 |         "type": "TripleConstraint",
71 |         "predicate": "http://www.wikidata.org/prop/statement/P3266",
72 |         "valueExpr": {
73 |           "type": "NodeConstraint",
74 |           "nodeKind": "literal"
75 |         }
76 |       }
77 |     },
78 |     {
79 |       "id": "https://raw.githubusercontent.com/shexSpec/schemas/master/Wikidata/DigitalPreservation/wikidatafileformatminimal.shex#P1163_media_type",
80 |       "type": "Shape",
81 |       "expression": {
82 |         "type": "TripleConstraint",
83 |         "predicate": "http://www.wikidata.org/prop/statement/P1163",
84 |         "valueExpr": {
85 |           "type": "NodeConstraint",
86 |           "nodeKind": "literal"
87 |         }
88 |       }
89 |     }
90 |   ],
91 |   "@context": "http://www.w3.org/ns/shex.jsonld"
92 | }
93 | 


--------------------------------------------------------------------------------
/wikidp/schemas/operating_system.json:
--------------------------------------------------------------------------------
  1 | {
  2 |   "type": "Schema",
  3 |   "start": "https://rawgit.com/shexSpec/shex.js/master/packages/shex-webapp/doc/shex-simple.html#operating_system",
  4 |   "shapes": [
  5 |     {
  6 |       "type": "Shape",
  7 |       "id": "https://rawgit.com/shexSpec/shex.js/master/packages/shex-webapp/doc/shex-simple.html#operating_system",
  8 |       "expression": {
  9 |         "type": "EachOf",
 10 |         "expressions": [
 11 |           {
 12 |             "type": "TripleConstraint",
 13 |             "predicate": "http://www.wikidata.org/prop/P31",
 14 |             "valueExpr": {
 15 |               "type": "Shape",
 16 |               "expression": {
 17 |                 "type": "TripleConstraint",
 18 |                 "predicate": "http://www.wikidata.org/prop/statement/P31",
 19 |                 "valueExpr": {
 20 |                   "type": "Shape",
 21 |                   "expression": {
 22 |                     "type": "TripleConstraint",
 23 |                     "predicate": "http://www.wikidata.org/prop/P279",
 24 |                     "valueExpr": {
 25 |                       "type": "Shape",
 26 |                       "expression": {
 27 |                         "type": "TripleConstraint",
 28 |                         "predicate": "http://www.wikidata.org/prop/statement/P279",
 29 |                         "valueExpr": {
 30 |                           "type": "NodeConstraint",
 31 |                           "values": [
 32 |                             "http://www.wikidata.org/entity/Q9135"
 33 |                           ]
 34 |                         }
 35 |                       }
 36 |                     },
 37 |                     "min": 1,
 38 |                     "max": -1
 39 |                   }
 40 |                 }
 41 |               }
 42 |             }
 43 |           },
 44 |           {
 45 |             "type": "TripleConstraint",
 46 |             "predicate": "http://www.wikidata.org/prop/P275",
 47 |             "valueExpr": {
 48 |               "type": "Shape",
 49 |               "expression": {
 50 |                 "type": "TripleConstraint",
 51 |                 "predicate": "http://www.wikidata.org/prop/statement/P275",
 52 |                 "valueExpr": {
 53 |                   "type": "NodeConstraint",
 54 |                   "nodeKind": "iri"
 55 |                 },
 56 |                 "min": 1,
 57 |                 "max": -1
 58 |               }
 59 |             }
 60 |           },
 61 |           {
 62 |             "type": "TripleConstraint",
 63 |             "predicate": "http://www.wikidata.org/prop/P178",
 64 |             "valueExpr": {
 65 |               "type": "Shape",
 66 |               "expression": {
 67 |                 "type": "TripleConstraint",
 68 |                 "predicate": "http://www.wikidata.org/prop/statement/P178",
 69 |                 "valueExpr": {
 70 |                   "type": "NodeConstraint",
 71 |                   "nodeKind": "iri"
 72 |                 }
 73 |               }
 74 |             },
 75 |             "min": 0,
 76 |             "max": 1
 77 |           },
 78 |           {
 79 |             "type": "TripleConstraint",
 80 |             "predicate": "http://www.wikidata.org/prop/P176",
 81 |             "valueExpr": {
 82 |               "type": "Shape",
 83 |               "expression": {
 84 |                 "type": "TripleConstraint",
 85 |                 "predicate": "http://www.wikidata.org/prop/statement/P176",
 86 |                 "valueExpr": {
 87 |                   "type": "NodeConstraint",
 88 |                   "nodeKind": "iri"
 89 |                 }
 90 |               }
 91 |             },
 92 |             "min": 0,
 93 |             "max": 1
 94 |           },
 95 |           {
 96 |             "type": "TripleConstraint",
 97 |             "predicate": "http://www.wikidata.org/prop/P910",
 98 |             "valueExpr": {
 99 |               "type": "Shape",
100 |               "expression": {
101 |                 "type": "TripleConstraint",
102 |                 "predicate": "http://www.wikidata.org/prop/statement/P910",
103 |                 "valueExpr": {
104 |                   "type": "NodeConstraint",
105 |                   "nodeKind": "iri"
106 |                 }
107 |               }
108 |             },
109 |             "min": 0,
110 |             "max": 1
111 |           },
112 |           {
113 |             "type": "TripleConstraint",
114 |             "predicate": "http://www.wikidata.org/prop/P373",
115 |             "valueExpr": {
116 |               "type": "Shape",
117 |               "expression": {
118 |                 "type": "TripleConstraint",
119 |                 "predicate": "http://www.wikidata.org/prop/statement/P373",
120 |                 "valueExpr": {
121 |                   "type": "NodeConstraint",
122 |                   "datatype": "http://www.w3.org/2001/XMLSchema#string"
123 |                 },
124 |                 "min": 0,
125 |                 "max": -1
126 |               }
127 |             },
128 |             "min": 0,
129 |             "max": 1
130 |           },
131 |           {
132 |             "type": "TripleConstraint",
133 |             "predicate": "http://www.wikidata.org/prop/P5795",
134 |             "valueExpr": {
135 |               "type": "Shape",
136 |               "expression": {
137 |                 "type": "TripleConstraint",
138 |                 "predicate": "http://www.wikidata.org/prop/statement/P5795",
139 |                 "valueExpr": {
140 |                   "type": "NodeConstraint",
141 |                   "datatype": "http://www.w3.org/2001/XMLSchema#string"
142 |                 },
143 |                 "min": 0,
144 |                 "max": -1
145 |               }
146 |             },
147 |             "min": 0,
148 |             "max": 1
149 |           },
150 |           {
151 |             "type": "TripleConstraint",
152 |             "predicate": "http://www.wikidata.org/prop/P5868",
153 |             "valueExpr": {
154 |               "type": "Shape",
155 |               "expression": {
156 |                 "type": "TripleConstraint",
157 |                 "predicate": "http://www.wikidata.org/prop/statement/P5868",
158 |                 "valueExpr": {
159 |                   "type": "NodeConstraint",
160 |                   "datatype": "http://www.w3.org/2001/XMLSchema#string"
161 |                 },
162 |                 "min": 0,
163 |                 "max": -1
164 |               }
165 |             },
166 |             "min": 0,
167 |             "max": 1
168 |           },
169 |           {
170 |             "type": "TripleConstraint",
171 |             "predicate": "http://www.wikidata.org/prop/P268",
172 |             "valueExpr": {
173 |               "type": "Shape",
174 |               "expression": {
175 |                 "type": "TripleConstraint",
176 |                 "predicate": "http://www.wikidata.org/prop/statement/P268",
177 |                 "valueExpr": {
178 |                   "type": "NodeConstraint",
179 |                   "datatype": "http://www.w3.org/2001/XMLSchema#string"
180 |                 },
181 |                 "min": 0,
182 |                 "max": -1
183 |               }
184 |             },
185 |             "min": 0,
186 |             "max": 1
187 |           }
188 |         ]
189 |       }
190 |     }
191 |   ],
192 |   "@context": "http://www.w3.org/ns/shex.jsonld"
193 | }
194 | 


--------------------------------------------------------------------------------
/wikidp/sparql.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/python
 2 | # coding=UTF-8
 3 | #
 4 | # WikiDP Wikidata Portal
 5 | # Copyright (C) 2020
 6 | # All rights reserved.
 7 | #
 8 | # This code is distributed under the terms of the GNU General Public
 9 | # License, Version 3. See the text file "COPYING" for further details
10 | # about the terms of this license.
11 | #
12 | # This is a python __init__ script to create the app and import the
13 | # main package contents
14 | """
15 | Collection of sparql queries and related functions turned into python functions.
16 | 
17 | Note: Remove any comments from queries.
18 | """
19 | 
20 | PROPERTY_QUERY = """
21 |     SELECT  (STRAFTER(STR(?property), 'entity/') as ?id) ?property ?propertyType ?propertyLabel
22 |     ?propertyDescription ?propertyAltLabel (STRAFTER(STR(?propertyType), '#') as ?value_type) ?formatter_url
23 |     WHERE {
24 |     VALUES (?property) { $values }
25 |     ?property wikibase:propertyType ?propertyType .
26 |     OPTIONAL {
27 |       ?property wdt:P1630 ?formatter_url.
28 |     }
29 |     SERVICE wikibase:label { bd:serviceParam wikibase:language "[AUTO_LANGUAGE],en". }
30 |     }
31 |     ORDER BY ASC(xsd:integer(STRAFTER(STR(?property), 'P')))
32 | """
33 | 
34 | ALL_QUALIFIER_PROPERTIES = """
35 |     SELECT (STRAFTER(STR(?property), 'entity/') as ?id) ?property ?propertyType ?propertyLabel
36 |     ?propertyDescription ?propertyAltLabel (STRAFTER(STR(?propertyType), '#') as ?value_type)
37 |     WHERE {
38 |       ?property wikibase:propertyType ?propertyType .
39 |       ?property wdt:P31 wd:Q15720608 .
40 |       SERVICE wikibase:label { bd:serviceParam wikibase:language "[AUTO_LANGUAGE],en". }
41 |     }
42 |     ORDER BY ASC(xsd:integer(STRAFTER(STR(?property), 'P')))
43 | """
44 | 
45 | ALL_REFERENCE_PROPERTIES = """
46 |     SELECT (STRAFTER(STR(?property), 'entity/') as ?id)
47 |     ?property ?propertyType ?propertyLabel  ?propertyAltLabel
48 |     (?propertyLabel as ?label) ?propertyDescription
49 |     (?propertyDescription as ?description)
50 |     (STRAFTER(STR(?propertyType), '#') as ?value_type)
51 |     WHERE {
52 |         ?property wikibase:propertyType ?propertyType .
53 |         ?property wdt:P31 wd:Q18608359 .
54 |         SERVICE wikibase:label {
55 |             bd:serviceParam wikibase:language "[AUTO_LANGUAGE],en".
56 |         }
57 |     }
58 | """
59 | 
60 | PROPERTY_ALLOWED_QUALIFIERS = """
61 |     SELECT (STRAFTER(STR(?property), 'entity/') as ?id) ?property ?propertyType ?propertyLabel  ?propertyAltLabel
62 |     (?propertyLabel as ?label) ?propertyDescription (?propertyDescription as ?description)
63 |     (STRAFTER(STR(?propertyType), '#') as ?value_type)
64 |     WHERE {
65 |         VALUES (?main_property) { $values }
66 |         ?main_property p:P2302 ?constraint.
67 |         ?constraint ps:P2302 wd:Q21510851 .
68 |         ?constraint pq:P2306 ?property.
69 |         ?property wikibase:propertyType ?propertyType .
70 |         SERVICE wikibase:label { bd:serviceParam wikibase:language "[AUTO_LANGUAGE],en". }
71 |     }
72 | """
73 | 
74 | ALL_LANGUAGES_QUERY = """
75 |     SELECT ?item ?itemLabel (?itemLabel as ?label) ?code
76 |     (CONCAT("{","{#language:",?code,"}","}") as ?display)
77 |     {
78 |       ?item wdt:P424 ?code .
79 |       MINUS{?item wdt:P31/wdt:P279* wd:Q14827288}
80 |       MINUS{?item wdt:P31/wdt:P279* wd:Q17442446}
81 |       SERVICE wikibase:label { bd:serviceParam wikibase:language "[AUTO_LANGUAGE],en". }
82 |     }
83 | """
84 | 


--------------------------------------------------------------------------------
/wikidp/static/.DS_Store:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/WikiDP/wikidp-portal/ab3fe89deacefa9a07ac1aa9046691e31f62eba6/wikidp/static/.DS_Store


--------------------------------------------------------------------------------
/wikidp/static/img/.DS_Store:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/WikiDP/wikidp-portal/ab3fe89deacefa9a07ac1aa9046691e31f62eba6/wikidp/static/img/.DS_Store


--------------------------------------------------------------------------------
/wikidp/static/img/favicon.ico:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/WikiDP/wikidp-portal/ab3fe89deacefa9a07ac1aa9046691e31f62eba6/wikidp/static/img/favicon.ico


--------------------------------------------------------------------------------
/wikidp/static/img/logo.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/WikiDP/wikidp-portal/ab3fe89deacefa9a07ac1aa9046691e31f62eba6/wikidp/static/img/logo.png


--------------------------------------------------------------------------------
/wikidp/static/js/item.js:
--------------------------------------------------------------------------------
 1 | $(document).ready(() => {
 2 |     // Initialize property checklist based on default schema:
 3 |     loadOptionSelector();
 4 |     $('select#schema-select').change(load_property_checklist_by_schema).change();
 5 |     //TODO: Only call qLabel in Prod Config
 6 |     $.qLabel.switchLanguage('en');
 7 |     $('select#option-select').change(change_item);
 8 | });
 9 | 
10 | 
11 | function get_page_qid(){
12 |     return $('#panel-frame').data('qid');
13 | }
14 | 
15 | 
16 | function get_item_page(){
17 |     return $('#panel-frame').data('item_page');
18 | }
19 | 
20 | 
21 | function get_page_language(){
22 |     return $('#languageSelect').val();
23 | }
24 | 
25 | 
26 | function get_value_from_event(event){
27 |     return event.currentTarget.value;
28 | }
29 | 
30 | 
31 | function pageTransition(form){
32 |     $('div#content-frame').fadeOut(500, () => {
33 |         $('body').append(form);
34 |         form.submit();
35 |     });
36 | }
37 | 
38 | 
39 | function load_property_checklist_by_schema(event){
40 |     let schema = get_value_from_event(event);
41 |     let qid = get_page_qid();
42 |     $.get(`/${qid}/checklist/${schema}`, (response) => {
43 |         $('ul#sidebar-property-list').html(response);
44 |         //  TODO: Move to item_contribute.
45 |         if (typeof initializeStatementPropertySelector !== 'undefined') {
46 |             initializeStatementPropertySelector();
47 |         }
48 |     })
49 | }
50 | 
51 | 
52 | function get_template(template_id, data){
53 |     let template_html = $(template_id).html();
54 |     return $(Mustache.render(template_html, data));
55 | }
56 | 
57 | 
58 | function change_item(event){
59 |     let qid = get_value_from_event(event);
60 |     let page = get_item_page();
61 |     render_item_page(qid, page)
62 | }
63 | 
64 | 
65 | function render_item_page(qid, page){
66 |   const _options = [];
67 |   $('option.item-option').each(function() {
68 |     _options.push($(this).data('qid').replace("'","&#39;"));
69 |   });
70 |   const options = JSON.stringify(_options);
71 |   const ctx = { options, page, qid };
72 |   const $form = get_template('#wikidp-change-item-form', ctx).hide(0);
73 |   pageTransition($form);
74 | }
75 | 
76 | const get_item_options = () => {
77 |   const data = JSON.stringify(QID_OPTIONS);
78 |   return $.ajax({
79 |     data,
80 |     dataType: 'json',
81 |     type: 'POST',
82 |     contentType: 'application/json',
83 |     url: '/api/items',
84 |   });
85 | }
86 | 
87 | const loadOptionSelector = async () => {
88 |   const options = await get_item_options();
89 |   const ctx = { options };
90 |   const $html = get_template('#wikidp-item-options',ctx);
91 |   $html.val(get_page_qid());
92 |   $html.change(change_item);
93 |   $('#item-options-container').html($html);
94 | };
95 | 


--------------------------------------------------------------------------------
/wikidp/static/js/item_preview.js:
--------------------------------------------------------------------------------
 1 | $(document).ready(() => {
 2 |     $('li#contribute-action').click(() => render_item_page(get_page_qid(), 'contribute'));
 3 |     $('.scroller').scroll(() => $.qLabel.switchLanguage(get_page_language()));
 4 |     $('#languageSelect').change(() => $.qLabel.switchLanguage(get_page_language()));
 5 |     $('img.property-image').fadeIn(4000);
 6 | });
 7 | 
 8 | 
 9 | function sidebar_property_click(elm){
10 |     let $elm = $(elm);
11 |     if($elm.data('count') > 0){
12 |         let this_pid = $elm.data('id');
13 |         let isExternalId = $elm.data("value_type") === "ExternalId";
14 |         let tableClass = (isExternalId) ? ".ex-links-table" : ".claims-table";
15 |         let scrollerId = (isExternalId) ? "#other-info-scroller" : "#claims-scroller";
16 |         let $scrollDiv = $(`${tableClass} a[data-entity-id="${this_pid}"]`).parents('tr').addClass('scroll-highlight');
17 | 
18 |         $(scrollerId).scrollTo($scrollDiv, 1000, {
19 |             onAfter: () =>  setTimeout(() => $scrollDiv.removeClass('scroll-highlight'), 1000)
20 |         })
21 |     }
22 |     else{
23 |         alert('There are currently no statements with this property recorded. If you know of some to add, click ' +
24 |             'the contribute button to create a statement for this property.')
25 |     }
26 | }
27 | 


--------------------------------------------------------------------------------
/wikidp/static/js/wikidp.js:
--------------------------------------------------------------------------------
  1 | var oauthController = {
  2 |   authorized: false,
  3 |   initiate: function () {
  4 |     $.ajax({
  5 |       url: '/auth',
  6 |       type: 'GET',
  7 |       dataType: 'json',
  8 |       contentType: false,
  9 |       processData: false,
 10 |       success: function (data, textStatus, jqXHR) {
 11 |         console.log(data)
 12 |         if (data.auth === false) {
 13 |           sessionStorage.setItem('returnTo', window.location.pathname  + window.location.search)
 14 |           oauthController.authorize(null)
 15 |         } else {
 16 |           window.location.href = '/profile'
 17 |         }
 18 |       },
 19 |       // HTTP Error handler
 20 |       error: function (jqXHR, textStatus, errorThrown) {
 21 |         // Log full error to console
 22 |         console.log('Validation Error: ' + textStatus + errorThrown)
 23 |         console.log(jqXHR)
 24 |       }
 25 |     })
 26 |   },
 27 |   authorize: function (callback) {
 28 |     $.ajax({
 29 |       url: '/profile',
 30 |       type: 'POST',
 31 |       data: JSON.stringify({
 32 |         authorization: 'sending',
 33 |         current_path: window.location.pathname,
 34 |         initiate: true
 35 |       }),
 36 |       dataType: 'json',
 37 |       contentType: false,
 38 |       processData: false,
 39 |       success: function (data, textStatus, jqXHR) {
 40 |         console.log(data)
 41 |         window.location.href = data.wikimediaURL
 42 |       },
 43 |       // HTTP Error handler
 44 |       error: function (jqXHR, textStatus, errorThrown) {
 45 |         // Log full error to console
 46 |         console.log('Validation Error: ' + textStatus + errorThrown)
 47 |         console.log(jqXHR)
 48 |       }
 49 |     })
 50 |   },
 51 |   authCheck: function () {
 52 |     $.ajax({
 53 |       url: '/auth',
 54 |       type: 'GET',
 55 |       dataType: 'json',
 56 |       contentType: false,
 57 |       processData: false,
 58 |       success: function (data, textStatus, jqXHR) {
 59 |         if (data.auth === false) {
 60 |           oauthController.forward()
 61 |         } else {
 62 |           window.location.href = '/profile'
 63 |         }
 64 |       },
 65 |       // HTTP Error handler
 66 |       error: function (jqXHR, textStatus, errorThrown) {
 67 |         // Log full error to console
 68 |         console.log('Validation Error: ' + textStatus + errorThrown)
 69 |         console.log(jqXHR)
 70 |       }
 71 |     })
 72 |   },
 73 |   forward: function () {
 74 |     $.ajax({
 75 |       type: 'POST',
 76 |       data: JSON.stringify({
 77 |         url: window.location.pathname + window.location.search
 78 |       }),
 79 |       dataType: 'json',
 80 |       contentType: false,
 81 |       processData: false,
 82 |       success: function (data, textStatus, jqXHR) {
 83 |         var gotoURL = sessionStorage.getItem('returnTo') || null
 84 |         sessionStorage.removeItem('returnTo')
 85 |         if (gotoURL) {
 86 |           window.location = gotoURL
 87 |         }
 88 | 
 89 |         console.log(data)
 90 |       },
 91 |       // HTTP Error handler
 92 |       error: function (jqXHR, textStatus, errorThrown) {
 93 |         alert('forward.fail')
 94 |         // Log full error to console
 95 |         console.log('Validation Error: ' + textStatus + errorThrown)
 96 |         console.log(jqXHR)
 97 |       }
 98 |     })
 99 |   }
100 | }
101 | 
102 | $(document).ready(() => {
103 |   const $pagecontainer = $('div.page-container')
104 |   $('form#navbarSearch').submit(() => $pagecontainer.fadeOut(500))
105 |   $('div.side-icon-div i#searchToggle').click(toggleSearchForm)
106 |   $pagecontainer.fadeIn(500)
107 |   const urlParams = new URLSearchParams(window.location.search)
108 |   if (urlParams.has('oauth_verifier')) {
109 |     oauthController.authCheck()
110 |   }
111 | })
112 | 
113 | function toggleSearchForm () {
114 |   let $icon = $('div.side-icon-div i#searchToggle').fadeOut('slow')
115 |   $('#main-nav-tabs-container').toggle(1000, () => $('#navbarSearch').fadeToggle(500))
116 |   $icon.toggleClass('fa-search').toggleClass('fa-times-circle-o').fadeIn('slow')
117 | }
118 | 
119 | const getItemSummary = (qid, callback) => {
120 |   $.get(`/api/${qid}/summary`, (item) => {
121 |     if (callback) return callback(item)
122 |     return item
123 |   }).fail((error) => console.log(error))
124 | }
125 | 
126 | const selectorToDataArray = (selector) => $(selector).map(
127 |   (index, elem) => $(elem).data()
128 | );
129 | 


--------------------------------------------------------------------------------
/wikidp/templates/about.html:
--------------------------------------------------------------------------------
 1 | {% extends "page.html" %}
 2 | {% block title %}About Wikidata For Digital Preservation{% endblock %}
 3 | {% block page_content %}
 4 | 
 5 |   <div class="about-page middle white" id="content-frame">
 6 |     <h1>About Wikidata for Digital Preservation</h1>
 7 |     <h2>Introduction</h2>
 8 |     <p>Wikidata for Digital Preservation is a portal that provides an enhanced dashboard related to adding data about the domain of computing to Wikidata.
 9 |        We provide a streamlined interface for browsing and contributing structured data related to software and file formats to the Wikidata knowledge base.
10 |        These tools empower us as members of the digital preservation community to edit and maintain our own technical and descriptive metadata.
11 |     </p>
12 |     <h2>The WikiData Team</h2>
13 |     <ul>
14 |       <li>Logo design by <a href="https://www.kaiachessen.com/">Kaia Chessen</a></li>
15 |     </ul>
16 |     <h3>Development Team</h3>
17 |     <ul>
18 |       <li>Katherine Thornton</li>
19 |       <li>Kenneth Seals-Nutt</li>
20 |       <li>Euan Cochrane</li>
21 |       <li>Carl Wilson</li>
22 |     </ul>
23 |     <h2>Background Reading</h2>
24 |     <ul>
25 |       <li><a href="https://scholia.toolforge.org/work/Q27036465">Introducing Wikidata to the Linked Data Web</a></li>
26 |       <li><a href="https://scholia.toolforge.org/work/Q41533080">Modeling the Domain of Digital Preservation in Wikidata</a></li>
27 |       <li><a href="https://scholia.toolforge.org/topic/Q2013">A search via the Scholia platform for scholarly publications about Wikidata</a></li>
28 |       <li><a href="http://www.wikidata.org/entity/Q31742273">Wikidata item of the project</a></li>  
29 |       <li><a href="https://www.wikidata.org/wiki/Wikidata:WikiProject_Informatics">WikiProject Informatics in Wikidata</a></li>
30 |     </ul>
31 |     <h2>Acknowledgements</h2>
32 |     <ul>
33 |       <li><a href="https://meta.wikimedia.org/wiki/WikiCite_2016">Wikicite 2016</a></li>
34 |       <li><a href="https://meta.wikimedia.org/wiki/WikiCite_2017">Wikicite 2017 </a></li>
35 |       <li><a href="http://sulab.org/">The Su Lab</a></li>
36 |       <li><a href="https://www.wikidata.org/wiki/Wikidata:WikiProject_Gene_Wiki">GeneWiki</a></li>
37 |       <li><a href="https://www.clir.org/">Council on Library and Information Resources</a></li>
38 |       <li><a href="https://sloan.org/">The Alfred P. Sloan Foundation</a></li>
39 |       <li><a href="https://mellon.org/">The Andrew W. Mellon Foundation</a></li>
40 |       <li><a href="http://openpreservation.org/">Open Preservation Foundation</a></li>
41 |       <li><a href="http://www.bnf.fr/en/tools/lsp.site_map.html">Bibliothèque nationale de France</a></li>
42 |     </ul>
43 |     <p class="center">
44 |       Looking for inspiration for how to contribute? Visit our inspiration page.
45 |       <img alt="Powered By Wikidata" src="https://upload.wikimedia.org/wikipedia/commons/thumb/2/20/Wikidata_stamp_rec.png/120px-Wikidata_stamp_rec.png">
46 |     </p>
47 |   </div>
48 | 
49 | {% endblock page_content %}
50 | 
51 | {% block page_scripts %}
52 |   <script type="text/javascript">
53 |       $("form").submit(function() {
54 |         $('div.page-container').fadeOut(500);
55 |       });
56 |   </script>
57 | {% endblock page_scripts %}
58 | 


--------------------------------------------------------------------------------
/wikidp/templates/base.html:
--------------------------------------------------------------------------------
 1 | <!DOCTYPE html>
 2 | <html lang="en">
 3 |     <head>
 4 | 
 5 |       <meta charset="utf-8">
 6 |       <meta http-equiv="X-UA-Compatible" content="IE=edge">
 7 |       <meta name="viewport" content="width=device-width, initial-scale=1">
 8 |       <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags -->
 9 | 
10 |       <link rel="shortcut icon" href="{{ url_for('static', filename='img/favicon.ico') }}" type="image/x-icon" />
11 |       <title>{% block title %}{% endblock %} | WikiDP </title>
12 |       <!-- jQuery -->
13 |   <link rel="stylesheet" href="//code.jquery.com/ui/1.12.1/themes/base/jquery-ui.css">
14 |       <script src="https://code.jquery.com/jquery-3.3.1.min.js"></script>
15 |       <script src="https://code.jquery.com/ui/1.12.1/jquery-ui.js"></script>
16 |       <!-- Bootstrap -->
17 |       <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.min.css" integrity="sha384-BVYiiSIFeK1dGmJRAkycuHAHRg32OmUcww7on3RYdg4Va+PmSTsz/K68vbdEjh4u" crossorigin="anonymous">
18 |       <!-- Optional theme -->
19 |       <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap-theme.min.css" integrity="sha384-rHyoN1iRsVXV4nD0JutlnGaslCJuC7uwjduW9SVrLvRYooPp2bWYgmgJQIXwl/Sp" crossorigin="anonymous">
20 |       <!-- WARNING: Respond.js doesn't work if you view the page via file:// -->
21 |       <!--[if lt IE 9]>
22 |         <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
23 |         <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
24 |       <![endif]-->
25 |       <link href="https://maxcdn.bootstrapcdn.com/font-awesome/4.7.0/css/font-awesome.min.css" rel="stylesheet">
26 |       <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha384-Tc5IQib027qvyjSMfHjOMaLkfuWVxZxUPnCJA7l2mCWNIpG9mGCD8wGNIcPD7Txa" crossorigin="anonymous"></script>  <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries -->
27 |       <script src="{{ url_for('static',filename='vendor/jquery.qlabel.js') }}"></script>
28 |       <!-- Load local stylesheet default stylesheet -->
29 |       <link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='css/wikidp.css', cb="20210228") }}" />
30 |     </head>
31 |     <body class="no-pad">
32 | 
33 |         {% block page_header %}{% endblock %}
34 |         {% block page_layout %}{% endblock %}
35 | 
36 |       <!-- Include all compiled plugins (below), or include individual files as needed -->
37 |       <!-- Latest compiled and minified JavaScript -->
38 |       <script src="{{ url_for('static', filename='js/wikidp.js', cb="20210201") }}" type="text/javascript"></script>
39 |       {% block page_scripts %}{% endblock %}
40 |     </body>
41 | </html>
42 | 


--------------------------------------------------------------------------------
/wikidp/templates/error.html:
--------------------------------------------------------------------------------
 1 | {% extends "page.html" %}
 2 | {% block title %}Oops...{% endblock %}
 3 | {% block page_content %}
 4 | 
 5 |   <div class="about-page middle center white" id="content-frame">
 6 |     <h1>Oops...</h1>
 7 |     <h2>The Page You are attempting to reach is not available</h2>
 8 |     <p>
 9 |       {{message}}
10 |     </p>
11 |     <p>
12 |       Please Try Again or <a href="{{url_for('route_page_welcome')}}">Return Home.</a> 
13 |     </p>
14 |   </div>
15 | 
16 | {% endblock page_content %}
17 | 
18 | {% block page_scripts %}
19 |   <script type="text/javascript">
20 |       $("form").submit(function() {
21 |         $('div.page-container').fadeOut(500);
22 |       });
23 |   </script>
24 | {% endblock page_scripts %}
25 | 


--------------------------------------------------------------------------------
/wikidp/templates/item.html:
--------------------------------------------------------------------------------
 1 | {% extends "page.html" %}
 2 | 
 3 | {% block title %}
 4 |     {{item.label}} ({{item.qid}}): {{page|capitalize}}
 5 | {% endblock title %}
 6 | 
 7 | {% block page_content %}
 8 |     <div id="panel-frame" class="{{page}}-page center" data-qid="{{item.qid}}" data-item_page="{{page}}">
 9 |       <div id="sidebar-frame">
10 |             {% include "snippets/item_sidebar.html" %}
11 |       </div>
12 |       <div id="content-frame" class="white">
13 |         <div id="{{page}}-div">
14 |             <div id="header-text" class="center">
15 |                 <span id="preview-item-name">
16 |                     {{wikidata_entity(item.qid, item.label)}}
17 |                 </span>
18 |                 <br>
19 |                 {% if item.description %}
20 |                     <span id="preview-item-description" class="text-medium" title="{{item.description}}">
21 |                         {# TODO: Support Translation of Descriptions #}
22 |                         {{item.description}}
23 |                     </span>
24 |                 {% endif %}
25 |                 <div id="languageSelectDiv">
26 |                     Language:
27 |                     {# TODO: Make API Call to Sync with all Wikidata Languages #}
28 |                     <select name="" id="languageSelect"  aria-label="Select a Language for Item Labels">
29 |                       {% for code, name in languages %}
30 |                         <option value="{{ code }}">{{ name }}</option>
31 |                       {% endfor %}
32 |                     </select>
33 |                 </div>
34 |             </div>
35 |             {% block item_content %}
36 |             {% endblock item_content %}
37 |         </div>
38 |       </div>
39 |     </div>
40 | {% endblock page_content %}
41 | 
42 | {% block page_scripts %}
43 |   <script type="text/javascript">
44 |     const QID_OPTIONS = {{ options|tojson }};
45 |   </script>
46 |     {% include "snippets/item_templates.html" %}
47 |     <script src="https://cdnjs.cloudflare.com/ajax/libs/mustache.js/2.3.0/mustache.min.js"></script>
48 |     <script src="{{ url_for('static', filename='js/item.js', cb="20210201") }}" type="text/javascript"></script>
49 |     {% block item_page_scripts %}
50 |     {% endblock item_page_scripts %}
51 | {% endblock page_scripts %}
52 | 


--------------------------------------------------------------------------------
/wikidp/templates/item_contribute.html:
--------------------------------------------------------------------------------
  1 | {% extends "item.html" %}
  2 | 
  3 | {% block item_content %}
  4 |   {% if session.username  %}
  5 |     <div
  6 |       class="no-pad"
  7 |       id="statement-creation-frame"
  8 |     >
  9 |       <div id="creation-panel">
 10 |         <div id="creation-div">
 11 |           <span class="creation-section-header text-medium">
 12 |             CREATE A CLAIM
 13 |           </span>
 14 |           <br />
 15 |           <span class="creation-section-subtitle">
 16 |             Select a property below and we will provide you
 17 |             with a pre-formatted textbox.
 18 |           </span>
 19 |           <br />
 20 |           <div id="statement-div"></div>
 21 |           <div id="statement-actions"></div>
 22 |         </div>
 23 |         <div id="claims-list">
 24 |           <span class="creation-section-header text-medium">
 25 |             MY CLAIMS
 26 |           </span>
 27 |           <br />
 28 |           <span class="creation-section-subtitle">
 29 |             Currently Signed in as {{ session.username }}
 30 |           </span>
 31 |           <ul id="added-claims">
 32 |             <li id="claimsMessage">
 33 |             </li>
 34 |             <li id="saveClaimsLi" style="display:none">
 35 |               <button
 36 |                 class="white contribute-action-btn"
 37 |                 id="saveClaimsBtn"
 38 |                 onclick="saveClaims()"
 39 |               >
 40 |                 SAVE TO WIKIDATA
 41 |               </button>
 42 |             </li>
 43 |             <li
 44 |               id="goToClaimsLi"
 45 |               style="display:none"
 46 |             >
 47 |               <a
 48 |                 href="{{ item.url }}"
 49 |                 target="_blank"
 50 |               >
 51 |                 <button
 52 |                   class="white contribute-action-btn"
 53 |                   id="goToClaimsBtn"
 54 |                 >
 55 |                   VIEW IN WIKIDATA
 56 |                 </button>
 57 |               </a>
 58 |             </li>
 59 |             <li
 60 |               id="clearClaimsLi"
 61 |               style="display:none"
 62 |             >
 63 |               <button
 64 |                 class="white contribute-action-btn"
 65 |                 id="clearClaimsBtn"
 66 |                 onclick="resetForm()"
 67 |               >
 68 |                 RESET FORM
 69 |               </button>
 70 |             </li>
 71 |           </ul>
 72 |         </div>
 73 |       </div>
 74 |       <div id="lookup-panel">
 75 |         <div
 76 |           id="lookup-header-div"
 77 |           class="text-small"
 78 |         >
 79 |           looking for an item id?
 80 |           <br />
 81 |           <input
 82 |             aria-label="Item Lookup Input"
 83 |             class="center glow white"
 84 |             id="lookup-input"
 85 |             placeholder="search here"
 86 |           >
 87 |         </div>
 88 |         <div id="lookup-results-div">
 89 |           <ul id="lookup-results-list">
 90 |           </ul>
 91 |         </div>
 92 |       </div>
 93 |     </div>
 94 |     {% include "snippets/contribute_input_templates.html" %}
 95 | 	{% else %}
 96 |     <span class="creation-section-header text-medium">
 97 |       You are not currently logged in.
 98 |     </span>
 99 |     <br />
100 | 		<button
101 |       class="white login contribute-action-btn"
102 |       onclick="oauthController.initiate(null, null)"
103 |     >
104 |       Login using Wikidata OAuth
105 |     </button>
106 | 	{% endif %}
107 | {% endblock item_content %}
108 | 
109 | {% block item_page_scripts %}
110 |   <script src="https://cdnjs.cloudflare.com/ajax/libs/selectize.js/0.12.6/js/standalone/selectize.min.js"></script>
111 |   <link rel="stylesheet" type="text/css" href="https://cdnjs.cloudflare.com/ajax/libs/selectize.js/0.12.6/css/selectize.css" />
112 |   <script src="{{ url_for('static', filename='vendor/clipboard.min.js') }}"></script>
113 |   <script src="{{ url_for('static', filename='js/item_contribute.js', cb="20210206") }}" type="text/javascript"></script>
114 | {% endblock item_page_scripts %}
115 | 


--------------------------------------------------------------------------------
/wikidp/templates/item_preview.html:
--------------------------------------------------------------------------------
 1 | {% extends "item.html" %}
 2 | 
 3 | {% from 'snippets/macros.html' import make_value_table, value_table_row, table_row %}
 4 | 
 5 | {% block item_content %}
 6 |     <div id="claims-div">
 7 |         <div class="table-header-div text-medium">
 8 |             <span class="tbl-cat">
 9 |                 Current Data About {{wikidata_entity(item.qid, item.label, label_link=True)}}
10 |             </span>
11 |         </div>
12 |         <div id="claims-scroller" class="scroller">
13 |             {{make_value_table('claims', item.claims, classes="table table-responsive table-striped",
14 |                                internal_link=True, meta=['qualifiers', 'references'], label_link=False) }}
15 |         </div>
16 |     </div>
17 |     <div id="other-info-div">
18 |         <div class="table-header-div text-medium">
19 |             <span class="tbl-cat">
20 |                 other details
21 |             </span>
22 |         </div>
23 |         <div id="other-info-scroller" class="scroller">
24 |             <table class="selected-table table-responsive table-striped table-hover">
25 |                 {{ value_table_row('aliases', item.aliases) if item.aliases}}
26 |                 {{ table_row('description', item.description) if item.description}}
27 |             </table>
28 |             {%if item.external_links %}
29 |                 <br/>
30 |                 <span class="tbl-cat">External Links </span>
31 |                 {{make_value_table('ex-links', item.external_links, classes="table-responsive table-striped")}}
32 |             {%endif%}
33 |         </div>
34 | 	</div>
35 | {% endblock item_content %}
36 | 
37 | {% block item_page_scripts %}
38 |     <script src="//cdn.jsdelivr.net/npm/jquery.scrollto@2.1.2/jquery.scrollTo.min.js"></script>
39 |     <script src="{{ url_for('static', filename='js/item_preview.js') }}" type="text/javascript"></script>
40 | {% endblock item_page_scripts %}
41 | 


--------------------------------------------------------------------------------
/wikidp/templates/login.html:
--------------------------------------------------------------------------------
 1 | {% extends "page.html" %}
 2 | {% block title %}Profile{% endblock %}
 3 | {% block page_content %}
 4 | 
 5 | <div class="middle center white" id="content-frame">
 6 | 	{% if username  %}
 7 | 		<p>You are currently logged in as {{ username }}</p>
 8 | 		<a href="logout">Logout</a>
 9 | 	{% else %}
10 | 		<p>You are not currently logged in.</p>
11 | 		<a href="login">Login using Wikidata OAuth</a>
12 | 	{% endif %}
13 | </div>
14 | 
15 | {% block page_scripts %}
16 | {% endblock page_scripts %}
17 | 
18 | {% endblock page_content %}
19 | 


--------------------------------------------------------------------------------
/wikidp/templates/navbar.html:
--------------------------------------------------------------------------------
 1 | <!-- Fixed navbar -->
 2 | <nav id="main-navbar" class="navbar navbar-inverse navbar-fixed-top white">
 3 |     <div id="nav-container" class="container no-pad">
 4 |         <div id="main-nav-header" class="navbar-header">
 5 |             <img id="nav-logo" alt="Portal logo." class="pull-left" title="portal logo"
 6 |                  src="{{ url_for('static', filename='img/logo.png') }}" />
 7 |             <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar"
 8 |                     aria-expanded="false" aria-controls="navbar">
 9 |                 <span class="sr-only">Toggle navigation</span>
10 |                 <span class="icon-bar"></span>
11 |                 <span class="icon-bar"></span>
12 |                 <span class="icon-bar"></span>
13 |             </button>
14 |             <a id="main-brand" class="navbar-brand text-medium center" href="{{ url_for('route_page_welcome') }}">
15 |                 WIKIDATA</br>
16 |                 <span id="nav-header-subtitle">FOR DIGITAL PRESERVATION</span>
17 |             </a>
18 |         </div>
19 |         <div id="navbar" class="navbar-collapse collapse">
20 |             <div id="main-nav-tabs-container">
21 |                 <ul id="main-nav-tabs" class="nav navbar-nav center">
22 |                     <li><a href="{{ url_for('route_page_about') }}" class="nav-link white effect-underline">about</a></li>
23 |                     <li><a href="{{ url_for('route_page_reports') }}" class="nav-link white effect-underline">reports</a></li>
24 |                     <li><a href="#" class="nav-link white effect-underline">news</a></li>
25 |                     <li><a href="#" class="nav-link white effect-underline">support</a></li>
26 |                 </ul>
27 |             </div>
28 |             <div id="side-icons" class="pull-right">
29 |                 <div class="side-icon-div glow click text-medium white">
30 |                     <i id="searchToggle" class="fa fa-search" aria-hidden="true"></i>
31 |                 </div>
32 |                 <div class="side-icon-div glow click text-medium white">
33 |                     <a href="/profile"><i id="loginBtn" class="fa fa-user-circle" aria-hidden="true"></i></a>
34 |                 </div>
35 |             </div>
36 |             {% include "snippets/search_form.html" %}
37 |         </div><!--/.nav-collapse -->
38 |     </div>
39 | </nav>
40 | 


--------------------------------------------------------------------------------
/wikidp/templates/page.html:
--------------------------------------------------------------------------------
 1 | {% extends "base.html" %}
 2 | 
 3 | {% from 'snippets/macros.html' import wikidata_entity, qlabel %}
 4 | 
 5 | {% block page_header %}
 6 |     {% include "navbar.html" %}
 7 | {% endblock %}
 8 | 
 9 | {% block page_layout %}
10 |     <div class="page-container no-pad" role="main">
11 |         {% block page_content %}
12 |         {% endblock page_content %}
13 |     </div>
14 | {% endblock page_layout %}
15 | 


--------------------------------------------------------------------------------
/wikidp/templates/profile.html:
--------------------------------------------------------------------------------
 1 | {% extends "page.html" %}
 2 | {% block title %}Profile{% endblock %}
 3 | {% block page_content %}
 4 | 
 5 | <div class="middle center white" id="content-frame">
 6 | 	{% if username  %}
 7 | 		<p>You are currently logged in as {{ username }}</p>
 8 | 		<button class="login" onclick="window.location.href='/profile?logout=True';">Logout</button>
 9 | 	{% else %}
10 | 		<p>You are not currently logged in.</p>
11 | 		<button class="login" onclick="oauthController.initiate(null, null)">Login using Wikidata OAuth</button>
12 | 	{% endif %}
13 | </div>
14 | 
15 | {% block page_scripts %}
16 | {% endblock page_scripts %}
17 | 
18 | {% endblock page_content %}
19 | 


--------------------------------------------------------------------------------
/wikidp/templates/puid_results.html:
--------------------------------------------------------------------------------
 1 | {% extends "page.html" %}
 2 | {% block title %}PUID Results{% endblock %}
 3 | {% block page_content %}
 4 |     <!-- Main jumbotron for a primary marketing message or call to action -->
 5 |     <div class="jumbotron media">
 6 |       <div class="media-left">
 7 |         <img alt="Portal logo." title="portal logo" src="{{ url_for('static', filename='img/logo.png') }}" />
 8 |       </div>
 9 |       <div class="media-body">
10 |         <h1 class="media-heading">WikiDP Formats matching search PUID {{ puid }}</h1>
11 |         <p>
12 |           Test search by PUID on Wikidata
13 |         </p>
14 |       </div>
15 |     </div>
16 |     <div class="row">
17 |       <div class="col-md-2"></div>
18 |       <div class="col-md-8">
19 |         <h2>Formats</h2>
20 |         <table class="table table-striped">
21 |           <thead>
22 |             <tr>
23 |               <th>Format</th>
24 |               <th>Label</th>
25 |               <th>PUID</th>
26 |             </tr>
27 |           </thead>
28 |           <tbody>
29 |             {% for result in results %}
30 |             <tr>
31 |               <td>{{ result.format }}</td>
32 |               <td>{{ result.label }}</td>
33 |               <td>{{ result.puid }}</td>
34 |             </tr>
35 |             {% endfor %}
36 |           </tbody>
37 |         </table>
38 |       </div>
39 |       <div class="col-md-2"></div>
40 |     </div>
41 | {% endblock page_content %}
42 | 


--------------------------------------------------------------------------------
/wikidp/templates/reports.html:
--------------------------------------------------------------------------------
  1 | {% extends "page.html" %}
  2 | {% block title %}Reports{% endblock %}
  3 | {% block page_content %}
  4 | <div class="reports-page middle white" id="content-frame">
  5 |   <div id="header-text" class="center">
  6 |     <span id="preview-item-name" class="reports-header">Reports</span>
  7 |     <br>
  8 |     <span id="preview-item-description" class="text-medium">
  9 |       Select a report title below to run a live query to fetch the metric in realtime along with a table of the results.
 10 |     </span>
 11 |   </div>
 12 |   <div class="reports-section" id="queries">
 13 |     <div class="reports-section-header">
 14 |       Wikidata Queries
 15 |     </div>
 16 |     <ul class="no-pad">
 17 |       <li>
 18 |         <div class="query-title glow click" onclick="addQuery('query1', 1); $(this).next().animate({height: 'toggle', opacity: 'toggle',}, 'slow')">
 19 |           Number of software titles recorded <span id="count1" />
 20 |         </div>
 21 |         <div class="center" hidden>
 22 |           <iframe name="query1" id="query1" class="query-frame middle center" src="" referrerpolicy="origin" sandbox="allow-scripts allow-same-origin allow-popups"></iframe>
 23 |         </div>
 24 |       </li>
 25 |       <li>
 26 |         <div class="query-title glow click" onclick="addQuery('query2', 2); $(this).next().animate({height: 'toggle', opacity: 'toggle',}, 'slow')">
 27 |           Number of file formats recorded <span id="count2" />
 28 |         </div>
 29 |         <div class="center" hidden>
 30 |           <iframe name="query2" id="query2"  class="query-frame middle center" src="" referrerpolicy="origin" sandbox="allow-scripts allow-same-origin allow-popups"></iframe>
 31 |         </div>
 32 |       </li>
 33 |       <li>
 34 |         <div class="query-title glow click" onclick="addQuery('query3', 3); $(this).next().animate({height: 'toggle', opacity: 'toggle',}, 'slow')">
 35 |           Number of PRONOM Unique Identifiers recorded <span id="count3" />
 36 |         </div>
 37 |         <div class="center" hidden>
 38 |           <iframe name="query3" id="query3" class="query-frame middle center" src="" referrerpolicy="origin" sandbox="allow-scripts allow-same-origin allow-popups"></iframe>
 39 |         </div>
 40 |       </li>
 41 |       <li>
 42 |         <div class="query-title glow click" onclick="addQuery('query4', 4); $(this).next().animate({height: 'toggle', opacity: 'toggle',}, 'slow')">
 43 |           Number of file signatures described <span id="count4" />
 44 |         </div>
 45 |         <div class="center" hidden>
 46 |           <iframe name="query4" id="query4" class="query-frame middle center" src="" referrerpolicy="origin" sandbox="allow-scripts allow-same-origin allow-popups"></iframe>
 47 |         </div>
 48 |       </li>
 49 |       <li>
 50 |         <div class="query-title glow click" onclick="addQuery('query5', 5); $(this).next().animate({height: 'toggle', opacity: 'toggle',}, 'slow')">
 51 |           Number of emulators recorded <span id="count5" />
 52 |         </div>
 53 |         <div class="center" hidden>
 54 |           <iframe name="query5" id="query5" class="query-frame middle center" src="" referrerpolicy="origin" sandbox="allow-scripts allow-same-origin allow-popups"></iframe>
 55 |         </div>
 56 |       </li>
 57 |       <li>
 58 |         <div class="query-title glow click" onclick="addQuery('query6', 6); $(this).next().animate({height: 'toggle', opacity: 'toggle',}, 'slow')">
 59 |           Number of file systems recorded <span id="count6" />
 60 |         </div>
 61 |         <div class="center" hidden>
 62 |           <iframe name="query6" id="query6" class="query-frame middle center" src="" referrerpolicy="origin" sandbox="allow-scripts allow-same-origin allow-popups"></iframe>
 63 |         </div>
 64 |       </li>
 65 |     </ul></div>
 66 | </div>
 67 |   <script type="text/javascript">
 68 |     var endpointUrl = 'https://query.wikidata.org/sparql';
 69 |     var endpointEmbedUrl = 'https://query.wikidata.org/embed.html#';
 70 |     var sparqlQueries = [
 71 |       "SELECT ?item ?itemLabel \n" +
 72 |       "WHERE { \n" +
 73 |       "  ?item wdt:P31/wdt:P279* wd:Q7397. \n"
 74 |     ,
 75 |       "SELECT ?item ?itemLabel \n" +
 76 |       "WHERE { \n" +
 77 |       "  ?item wdt:P31/wdt:P279* wd:Q235557. \n"
 78 |     ,
 79 |       "SELECT ?item ?itemLabel ?puid \n" +
 80 |       "WHERE { \n" +
 81 |       "  ?item wdt:P2748 ?puid. \n"
 82 |     ,
 83 |       "SELECT ?item ?itemLabel ?signature ?encodingLabel \n" +
 84 |       "WHERE { \n" +
 85 |       "  ?item wdt:P31/wdt:P279* wd:Q235557. \n" +
 86 |       "  ?item p:P4152 ?signatureStmt. \n" +
 87 |       "  ?signatureStmt ps:P4152 ?signature. \n" +
 88 |       "  ?signatureStmt pq:P3294 ?encoding. \n"
 89 |     ,
 90 |       "SELECT ?item ?itemLabel \n" +
 91 |       "WHERE { \n" +
 92 |       "  ?item wdt:P31/wdt:P279* wd:Q202871. \n"
 93 |     ,
 94 |       "SELECT ?item ?itemLabel \n" +
 95 |       "WHERE { \n" +
 96 |       "    ?item wdt:P31/wdt:P279* wd:Q174989. \n"
 97 |     ];
 98 | 
 99 |     var labelServiceStmt =
100 |       "  SERVICE wikibase:label { bd:serviceParam wikibase:language \"[AUTO_LANGUAGE],en\". } ";
101 |     var limitStmt = " LIMIT 100";
102 | 
103 |     function addQuery(iframeName, number) {
104 |       var $iframe = $('#' + iframeName);
105 |       if ( $iframe.length ) {
106 |         var isThereAQuery = $iframe.attr('src');
107 |         if ( isThereAQuery == '') {
108 |           // First time : the query has to be run
109 |           // Make a first request to COUNT the number of items
110 |           var countQuery = "SELECT (COUNT(*) AS ?number) WHERE { " +
111 |             sparqlQueries[number - 1] + labelServiceStmt + " } " + "}";
112 |           var settings = {
113 |             headers: { Accept: 'application/sparql-results+json' },
114 |             data: { query: countQuery }
115 |           };
116 |           $.ajax( endpointUrl, settings ).then( function ( data ) {
117 |               var counter = data['results']['bindings'][0]['number']['value'];
118 |               $('#count' + number).text("(" + counter + ")");
119 |               // Embed the query and show the first 100 items
120 |               var query = sparqlQueries[number - 1] + labelServiceStmt + " } " + limitStmt;
121 |               $iframe.attr('src', encodeURI(endpointEmbedUrl + query));
122 |           } );
123 |           return false;
124 |         }
125 |       }
126 |       return true;
127 |     }
128 | 
129 |     $("form").submit(function(){
130 |     $('div.page-container').fadeOut(500);
131 | });
132 | 
133 |   </script>
134 | 
135 |     {% endblock page_content %}
136 | 


--------------------------------------------------------------------------------
/wikidp/templates/search_results.html:
--------------------------------------------------------------------------------
 1 | {% extends "page.html" %}
 2 | {% block title %}Search Results{% endblock %}
 3 | {% block page_content %}
 4 | <div class="welcome-div middle center">
 5 | 
 6 | 	<div id="searchbar-container" class="glow text-medium center middle">
 7 | 		<form action="/search" method="POST" role="search" id="searchAgain">
 8 | 		<input class="welcome-searchbar white center" name="userInput" aria-label="site search input"
 9 |                placeholder="try a new search...">
10 | 		<button type="submit" aria-label="site search submit">
11 | 		<span class="glyphicon glyphicon-search"></span> </button>
12 | 		</form>
13 | 	</div>
14 | 	<div class="search-results-div">
15 | 		<ul id="search-option-list" class="no-pad">
16 | 		{% for option in options %}
17 |             <li class="option-li glow click" id="li-{{option['qid']}}" data-qid="{{option['qid']}}"
18 |                 data-label="{{option['label'] | url_encode}}" data-description="{{option['description'] | url_encode}}"
19 |                 onclick="selectResult($(this).data('qid'))">
20 |                 <span class="search-result-label">
21 |                     {{option['label']}}<span class="search-result-id text-small"> ({{option['qid']}})</span>
22 |                 </span>
23 |                 <br>
24 |                 &mdash;<span class="search-result-description"> {{option['description']}} </span>&mdash;
25 |                 <br>
26 |                 {% if option.get('aliases')  %}
27 |                     <span class="search-result-aliases"><b>also referred to as:</b> {{option['aliases'][0]}}
28 |                         {% for alias in option['aliases'][1:] %}
29 |                             <b>/</b> {{alias}}
30 |                         {% endfor %}
31 |                     </span>
32 |                 {% endif %}
33 |             </li>
34 | 		{% endfor %}
35 | 		</ul>
36 | 	</div>
37 | </div>
38 | 
39 | 
40 | <script type="text/javascript">
41 | 	$("form#searchAgain").submit(function(){
42 |     $('div.search-results-div').fadeOut(500);
43 | });
44 | 	function pageTransition(form){
45 |   $('div.page-container').fadeOut(500, function(
46 |   ){
47 |   $('body').append(form);
48 | form.submit();
49 | })
50 | }
51 |   const selectResult = (qid) => {
52 |     const _options = [];
53 |     $('.option-li').each(function() {
54 |       _options.push($(this).data('qid'));
55 |     });
56 |     const options = JSON.stringify(_options);
57 |     const formHTML = `
58 |       <form action="/preview" method="post">
59 |         <input type="text" name="qid" value="${qid}" />
60 |         <input type="text" name="optionList" value=${options} />
61 |       </form>
62 |     `;
63 |     const $form = $(formHTML).hide(0);
64 |     pageTransition($form);
65 |   }
66 | 
67 | </script>
68 | {% endblock page_content %}
69 | 


--------------------------------------------------------------------------------
/wikidp/templates/snippets/contribute_input_templates.html:
--------------------------------------------------------------------------------
  1 | {% raw %}
  2 |     <script id="wikidp-contribute-input" type="text/template">
  3 |         <div id="creation-content-div" class="contribute-input-div">
  4 |             <div id="creation-property-div">
  5 |                 <label for="{{uuid}}" class="property-input-label">{{type}}</label><br>
  6 |                 <select id="{{uuid}}" class="property-selector {{type}}-selectize" title="{{type}} Property"
  7 |                         aria-label="Select Property">
  8 |                 </select> :
  9 |             </div>
 10 |             <div id="{{uuid}}-value-div" class="create-claim-value-div">
 11 |             </div>
 12 |         </div>
 13 |     </script>
 14 | 
 15 |   <script
 16 |     id="wikidp-added-claim-li"
 17 |     type="text/template"
 18 |   >
 19 |     <li
 20 |       class="added-claim-li white {{ type }} text-medium"
 21 |       data-pid="{{ pid }}"
 22 |       data-type="{{ type }}"
 23 |       data-value="{{ value }}"
 24 |     >
 25 |       <div class="row-div">
 26 |         <div class="added-property-div">
 27 |           {{ pidLabel }}
 28 |           <a
 29 |             class="wd-id glow"
 30 |             href="https://wikidata.org/wiki/Property:{{ pid }}"
 31 |             target="_blank"
 32 |           >
 33 |               ({{ pid }})
 34 |           </a>
 35 |         </div>
 36 |         <div class="added-value-div">
 37 |           <span class="added-label">
 38 |             {{ label }}
 39 |           </span>
 40 |           {{#description}}
 41 |             <br/>
 42 |             <i class="text-small">
 43 |               &mdash; {{ description }} &mdash;
 44 |             </i>
 45 |           {{/description}}
 46 |           {{#aliases}}
 47 |             <br/>
 48 |             <span class="sub-aka text-small">
 49 |               <b>
 50 |                 Also Referred To As:
 51 |               </b>
 52 |               {{ aliases }}
 53 |             </span>
 54 |           {{/aliases}}
 55 |           <br/>
 56 |           {{#qualifiers}}
 57 |             <div
 58 |               class="claim-li-qualifier"
 59 |               data-property_id="{{ pid }}"
 60 |               data-value="{{ value }}"
 61 |             >
 62 |               {{ pidLabel }}: {{ label }}
 63 |             </div>
 64 |           {{/qualifiers}}
 65 |           {{#references}}
 66 |             <div
 67 |               class="claim-li-reference"
 68 |               data-property_id="{{ pid }}"
 69 |               data-value="{{ value }}"
 70 |             >
 71 |               {{ pidLabel }}: {{ label }}
 72 |             </div>
 73 |           {{/references}}
 74 |         </div>
 75 |       </div>
 76 |     </li>
 77 |   </script>
 78 | 
 79 |     <script id="wikidp-property-autocomplete" type="text/template">
 80 |         <div>
 81 |             <span class="title">
 82 |                 <span class="name">{{label}}</span>
 83 |                 <span class="by text-small">{{id}}</span>
 84 |             </span>
 85 |             <span class="description text-small">{{description}}</span>
 86 |             <ul class="meta no-pad">
 87 |                 {{#language}}
 88 |                     <li class="language no-pad">{{language}}</li>
 89 |                 {{/language}}
 90 |                 <li class="watchers no-pad">
 91 |                     value type: <span>{{value_type}}</span>
 92 |                 </li>
 93 |                 {{#count}}
 94 |                     <li class="forks no-pad">
 95 |                        <br/>
 96 |                        This item currently has <span>{{count}}</span> value(s) with this property.
 97 |                     </li>
 98 |                 {{/count}}
 99 |             </ul>
100 |         </div>
101 |     </script>
102 | 
103 |     <script id="wikidp-lookup-item" type="text/template">
104 |         <li class="lookup-result-li glow">
105 |             <div class="result-main click text-medium" id="result-main-{{qid}}" data-qid="{{qid}}">
106 |                 <button data-clipboard-text="{{qid}}" class="pull-left clipboardBtn glow no-pad"
107 |                         id="clipboardBtn-{{qid}}" data-qid="{{qid}}">
108 |                     <i class="fa fa-clipboard"></i>
109 |                 </button>
110 |                 <span class="result-label">{{label}}</span><br/>
111 |                 {{qid}} <span class="glyphicon glyphicon-menu-down"></span>
112 |             </div>
113 |             <div class="result-sub" id="result-sub-{{qid}}">
114 |                 {{#description}}
115 |                     <i> &mdash; {{description}} &mdash; </i><br/>
116 |                 {{/description}}
117 |                 {{#aliases}}
118 |                     <br/>
119 |                     <span class="sub-aka text-small"><b>Also Referred To As:</b> {{aliases}}</span>
120 |                 {{/aliases}}
121 |             </div>
122 |         </li>
123 |     </script>
124 | {% endraw %}
125 | 


--------------------------------------------------------------------------------
/wikidp/templates/snippets/item_sidebar.html:
--------------------------------------------------------------------------------
 1 | <div id="sidebar-div" class="no-pad">
 2 |     <!-- action buttons-->
 3 |     <div class="actions-div no-pad text-medium">
 4 |         <ul id="sidebar-action-list" class="no-pad">
 5 |             <li class="sidebar-action-li click" id="explore-action">
 6 |                 explore
 7 |             </li>
 8 |             <li class="sidebar-action-li click {{ 'preview-on' if page == 'preview' }}" id="preview-action">
 9 |                 preview
10 |             </li>
11 |             <li class="sidebar-action-li click {{ 'contribute-on' if page == 'contribute' }}" id="contribute-action">
12 |                 contribute
13 |             </li>
14 |         </ul>
15 |     </div>
16 |     <!-- selected item -->
17 |     <div class="selected-item-div">
18 |         <label class="sidebar-header" for="option-select">selected item</label>
19 |         <div id="item-options-container">
20 |           <i class="fa fa-spinner fa-spin"></i> Loading
21 |         </div>
22 |     </div>
23 |     <!-- schema gallery -->
24 |     <div class="schema-gallery-div">
25 |         <label class="sidebar-header" for="schema-select">selected schema</label>
26 |         <select id="schema-select" class="text-medium" aria-label="Select a Property Checklist Schema">
27 |             {% for s in schemas %}
28 |                 {% if s.type == 'file' %}
29 |                     <option class="item-schema" data-schema_name="{{s.name}}" title="{{s.label}}" value="{{s.name}}">
30 |                         {{s.label}}
31 |                     </option>
32 |                 {% else %}
33 |                     <optgroup label="{{s.label}}">
34 |                         {% for sub_s in s.files %}
35 |                             {% if sub_s.type == 'file' %}
36 |                                 <option class="item-schema" data-schema_name="{{sub_s.name}}"  title="{{sub_s.label }}"
37 |                                         value="{{s.name}}/{{sub_s.name}}">
38 |                                     {{sub_s.label}}
39 |                                 </option>
40 |                             {% endif %}
41 |                         {% endfor %}
42 |                     </optgroup>
43 |                 {% endif %}
44 |             {% endfor %}
45 |         </select>
46 |     </div>
47 |     <!-- property overview -->
48 |     <div class="property-overview-div" class="white text-small">
49 |         <div class="sidebar-header">property checklist</div>
50 |         <div id="sidebar-property-scroller" class="scroller">
51 |             <ul id="sidebar-property-list">
52 |             </ul>
53 |         </div>
54 |     </div>
55 | </div>
56 | 


--------------------------------------------------------------------------------
/wikidp/templates/snippets/item_templates.html:
--------------------------------------------------------------------------------
 1 | {% raw %}
 2 |     <script id="wikidp-change-item-form" type="text/template">
 3 |         <form action="/{{page}}" method="post">
 4 |             <input type="text" name="qid" value="{{qid}}" />
 5 |             <input type="text" name="optionList" value="{{options}}"/>
 6 |         </form>
 7 |     </script>
 8 |   <script
 9 |     id="wikidp-item-options"
10 |     type="text/template"
11 |   >
12 |     <select
13 |       aria-label="Select an alternative wikidata item option"
14 |       class="text-medium"
15 |       id="option-select"
16 |     >
17 |       {{#options}}
18 |         <option
19 |           class="item-option"
20 |           data-label="{{ label }}"
21 |           data-qid="{{ qid }}"
22 |           title="{{ description }}"
23 |           value="{{ qid }}"
24 |         >
25 |           {{ label }} ({{ qid }})
26 |         </option>
27 |       {{/options}}
28 |     </select>
29 |   </script>
30 | {% endraw %}
31 | 


--------------------------------------------------------------------------------
/wikidp/templates/snippets/macros.html:
--------------------------------------------------------------------------------
 1 | {% macro wikidata_entity(id, label=None, label_link=False, internal_link=False) %}
 2 |     {% with url=id|entity_url, link=url_for('route_page_selected_item', qid=id) if internal_link else id|entity_url %}
 3 |         {% if label_link %}
 4 |             <a href="{{link}}" data-entity-id="{{id}}" target="_blank">{{qlabel(url, label)}}</a>
 5 |         {% else %}
 6 |             {{qlabel(url, label)}}
 7 |             <a class="wd-id glow {{"p_or_q_id" if internal_link}}" href="{{link}}" data-entity-id="{{id}}" target="_blank">({{id}})</a>
 8 |         {% endif %}
 9 |     {% endwith %}
10 | {% endmacro %}
11 | 
12 | {% macro qlabel(url, label=None) %}
13 |     <span {{url|qlabel_attributes}}>{{label if label}}</span>
14 | {% endmacro %}
15 | 
16 | {% macro parse_claim_value(claim_value, internal_link=False, label_link=False) %}
17 |     {% if claim_value.parse_type == 'item' %}
18 |         {{ wikidata_entity(claim_value.value, internal_link=internal_link, label_link=label_link) }}
19 |     {% elif claim_value.parse_type == 'property' %}
20 |         {{ wikidata_entity(claim_value.value) }}
21 |     {% elif claim_value.parse_type == 'url'  %}
22 |         {{ claim_value.value|urlize(target='_blank') }}
23 |     {% elif claim_value.parse_type == 'external-id' %}
24 |         {% if claim_value.value.url %}
25 |             <a target="_blank" href="{{claim_value.value.url }}">{{claim_value.value.label}}</a>
26 |         {%else%}
27 |             {{claim_value.value.label}}
28 |         {%endif%}
29 |     {% elif claim_value.parse_type == 'image'  %}
30 |         <a target="_blank" href="{{claim_value.value}}">
31 |             <img class="property-image" src="{{claim_value.value}}" alt="Image Provided By {{claim_value.value}}">
32 |         </a>
33 |     {% else %}
34 |         {{ claim_value.value }}
35 |     {% endif %}
36 | {% endmacro %}
37 | 
38 | {% macro metadata_table(table_name, items) %}
39 |     {{make_value_table(table_name, items,
40 |                         classes='metadata-table table-responsive table-striped table-hover text-small')}}
41 | {% endmacro %}
42 | 
43 | {% macro make_value_table(table_name, items, classes=None, internal_link=False, label_link=True, meta=None)  %}
44 |     <table class="{{ classes if classes }} {{table_name}}-table">
45 |         {% for x in items %}
46 |             {% call(claim_value) value_table_row(wikidata_entity(x.pid), x['values'])  %}
47 |                 {{ parse_claim_value(claim_value, internal_link=internal_link, label_link=label_link) }}
48 |                 {% if meta %}
49 |                     {% for meta_type in meta %}
50 |                         {{ metadata_table(meta_type, claim_value[meta_type]) }}
51 |                     {% endfor %}
52 |                 {% endif %}
53 |             {% endcall %}
54 |         {% endfor %}
55 |     </table>
56 | {% endmacro %}
57 | 
58 | {% macro value_table_row(left, right) %}
59 |     {% with value_caller = caller if caller else False %}
60 |         {% call(right) table_row(left, right) %}
61 |             {% for val in right  %}
62 |                 <span class="detail-row">{{value_caller(val) if value_caller else val}}<br/></span>
63 |             {% endfor %}
64 |         {% endcall %}
65 |     {% endwith %}
66 | {% endmacro %}
67 | 
68 | {% macro table_row(left, right) %}
69 |     <tr>
70 |         <td>{{ left }}</td>
71 |         <td>{{caller(right) if caller else right}}</td>
72 |     </tr>
73 | {% endmacro %}
74 | 


--------------------------------------------------------------------------------
/wikidp/templates/snippets/property_checklist.html:
--------------------------------------------------------------------------------
 1 | {% if properties|count > 0 %}
 2 |   {#    TODO: Connect this to qlabel for translation support #}
 3 |   {% for property in properties %}
 4 |     <li
 5 |       class="sidebar-property-li white click {% if property.hidden %}hidden{% elif property.count > 0 %}property-ok{% endif %}"
 6 |       data-id="{{property.id}}"
 7 |       data-label="{{property.label}}"
 8 |       data-count="{{property.count}}"
 9 |       data-value_type="{{property.type}}"
10 |       data-description="{{property.description}}"
11 |       onclick="sidebar_property_click(this)"
12 |     >
13 |       <span
14 |         title="{{property.label}} ({{property.id}}): {{property.description}}"
15 |         data-content-start="{{property.label}} ({{property.id}}) "
16 |         data-content-end=""
17 |       >
18 |       </span>
19 |       <div class="sidebar-property-count">
20 |         : {{property.count}}
21 |       </div>
22 |       <div>
23 |         {% for qualifier in property.qualifiers %}
24 |           <span
25 |             class="sidebar-qualifier"
26 |             data-content-end=""
27 |             data-content-start="↳ {{qualifier.propertyLabel}} ({{qualifier.id}}) "
28 |             title="{{qualifier.propertyLabel}} ({{qualifier.id}}): {{qualifier.propertyDescription}}"
29 |           >
30 |           </span>
31 |           <br/>
32 |         {% endfor %}
33 |       </div>
34 |     </li>
35 |   {% endfor %}
36 | {% else %}
37 |     no suggested properties in this schema
38 | {% endif %}
39 | 


--------------------------------------------------------------------------------
/wikidp/templates/snippets/search_form.html:
--------------------------------------------------------------------------------
 1 | <form class="navbar-form navbar-right center" role="search" action="{{ url_for('route_process_site_search') }}"
 2 |       method="POST" id="navbarSearch">
 3 |     <div class="input-group middle">
 4 |         <input type="text" name="userInput" class="nav-input white form-control no-pad center" placeholder="search here">
 5 |         <span class="input-group-btn">
 6 |             <button type="reset" class="btn btn-nav white no-pad glow center" id="search-reset" title="clear input">
 7 |                 <span class="glyphicon glyphicon-remove">
 8 |                     <span class="sr-only">Clear</span>
 9 |                 </span>
10 |             </button>
11 |             <button type="submit" class="btn btn-nav white no-pad glow center" id="search-btn">
12 |                 <span class="glyphicon glyphicon-search">
13 |                     <span class="sr-only">Search</span>
14 |                 </span>
15 |             </button>
16 |         </span>
17 |     </div>
18 | </form>
19 | 


--------------------------------------------------------------------------------
/wikidp/templates/welcome.html:
--------------------------------------------------------------------------------
 1 | {% extends "page.html" %}
 2 | {% block title %}Home{% endblock %}
 3 | {% block page_content %}
 4 | 
 5 | <div class="welcome-div middle center">
 6 | 	<div class="middle">
 7 |         <img id="welcome-logo" alt="Portal logo." title="portal logo" src="{{ url_for('static', filename='img/logo.png') }}" />
 8 |     </div>
 9 | 	<div id="searchbar-container" class="glow text-medium center middle">
10 | 		<form action="/search" method="POST" role="search">
11 | 		<input class="welcome-searchbar white center" name="userInput" aria-label="site search input" placeholder="search for file formats, software applications, software environments, etc ....">
12 | 		<button name="submit-search" type="submit" aria-label="site search submit">
13 | 		<span class="glyphicon glyphicon-search"></span> </button>
14 | 		</form>
15 | 	</div>
16 | </div>
17 | 
18 | {% block page_scripts %}
19 |   <script type="text/javascript">
20 |       $("form").submit(function() {
21 |         $('div.page-container').fadeOut(500);
22 |       });
23 |   </script>
24 | {% endblock page_scripts %}
25 | 
26 | {% endblock page_content %}
27 | 


--------------------------------------------------------------------------------
/wikidp/utils/wd_int_utils.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/python
 2 | # coding=UTF-8
 3 | #
 4 | # WikiDP Wikidata Portal
 5 | # Copyright (C) 2020
 6 | # All rights reserved.
 7 | #
 8 | # This code is distributed under the terms of the GNU General Public
 9 | # License, Version 3. See the text file "COPYING" for further details
10 | # about the terms of this license.
11 | #
12 | # This is a python __init__ script to create the app and import the
13 | # main package contents
14 | """Module to hold all WikiDataIntegrator routines for dependency management."""
15 | from datetime import datetime
16 | import logging
17 | from wikidataintegrator.wdi_core import WDItemEngine
18 | 
19 | from wikidp.config import APP
20 | from wikidp.const import (
21 |     ConfKey,
22 |     WIKIDATA_DATETIME_FORMAT,
23 | )
24 | 
25 | MEDIAWIKI_API_URL = APP.config[ConfKey.MEDIAWIKI_API_URL]
26 | SPARQL_ENDPOINT_URL = APP.config[ConfKey.SPARQL_ENDPOINT_URL]
27 | 
28 | 
29 | def process_query_string(query):
30 |     """Use WikiDataIntegrator Engine to process a SPARQL Query."""
31 |     result = WDItemEngine.execute_sparql_query(query,
32 |                                                endpoint=SPARQL_ENDPOINT_URL)
33 |     bindings = result['results'].get('bindings')
34 |     return _format_wikidata_bindings(bindings)
35 | 
36 | 
37 | def _format_wikidata_bindings(bindings):
38 |     return [{k: v.get('value') for k, v in res.items()} for res in bindings]
39 | 
40 | 
41 | def get_item_json(qid):
42 |     """
43 |     Get item json dictionary from qid.
44 | 
45 |     Args:
46 |         qid (str): Wikidata Identifier, ex: "Q1234"
47 | 
48 |     Returns:
49 |         Dict: Returned value of WDItemEngine().wd_json_representation
50 |     """
51 |     try:
52 |         item = WDItemEngine(wd_item_id=qid,
53 |                             mediawiki_api_url=MEDIAWIKI_API_URL,
54 |                             sparql_endpoint_url=SPARQL_ENDPOINT_URL)
55 |         return item.wd_json_representation
56 |     except (ValueError, ConnectionAbortedError, KeyError):
57 |         logging.exception("Exception reading QID: %s", qid)
58 |         return None
59 | 
60 | 
61 | def format_date(date_string):
62 |     """
63 |     Format Date String for WDI.
64 | 
65 |     Args:
66 |         date_string (str): ex '2020-01-01'
67 | 
68 |     Returns (str):
69 | 
70 |     """
71 |     date = datetime.fromisoformat(date_string)
72 |     return date.strftime(WIKIDATA_DATETIME_FORMAT)
73 | 


--------------------------------------------------------------------------------