├── .flake8 ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ ├── feature_request.md │ └── other.md ├── dependabot.yml └── workflows │ └── CI.yml ├── .gitignore ├── .gitmodules ├── CONTRIBUTING.rst ├── LICENSE ├── README.rst ├── combined_gen.sh ├── combined_sitemap.rst ├── gen.py ├── gen.sh ├── gen_dss.py ├── gen_html.sh ├── gen_rst.sh ├── gen_solr.py ├── iatirulesets ├── importwiki.sh ├── last_updated_gen.sh ├── requirements.txt ├── templates ├── en │ ├── codelist.rst │ ├── overview.rst │ ├── ruleset.rst │ ├── schema_element.rst │ └── schema_table.rst └── fr ├── tests └── sample_output_test.py └── update_submodules.sh /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | ignore = E501,E731,W504,W503 3 | exclude = conf.py,pyenv,pyenv3,IATI-Codelists,IATI-Extra-Documentation,IATI-Rulesets -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | **Desktop (please complete the following information):** 27 | - OS: [e.g. iOS] 28 | - Browser [e.g. chrome, safari] 29 | - Version [e.g. 22] 30 | 31 | **Smartphone (please complete the following information):** 32 | - Device: [e.g. iPhone6] 33 | - OS: [e.g. iOS8.1] 34 | - Browser [e.g. stock browser, safari] 35 | - Version [e.g. 22] 36 | 37 | **Additional context** 38 | Add any other context about the problem here. 39 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/other.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Other 3 | about: A blank template 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | 11 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | # Enable version updates for GitHub Actions 4 | - package-ecosystem: 'github-actions' 5 | # or GitHub Actions, set the directory to / to check for workflow files in .github/workflows. 6 | directory: '/' 7 | target-branch: "version-2.03" 8 | schedule: 9 | interval: 'daily' 10 | 11 | # Enable version updates for python 12 | - package-ecosystem: 'pip' 13 | # Look for `requirements.txt` file 14 | directory: '/' 15 | target-branch: "version-2.03" 16 | schedule: 17 | interval: 'daily' 18 | 19 | # Enable version updates for GitHub Actions 20 | - package-ecosystem: 'github-actions' 21 | # or GitHub Actions, set the directory to / to check for workflow files in .github/workflows. 22 | directory: '/' 23 | target-branch: "version-2.02" 24 | schedule: 25 | interval: 'daily' 26 | 27 | # Enable version updates for python 28 | - package-ecosystem: 'pip' 29 | # Look for `requirements.txt` file 30 | directory: '/' 31 | target-branch: "version-2.02" 32 | schedule: 33 | interval: 'daily' 34 | 35 | # Enable version updates for GitHub Actions 36 | - package-ecosystem: 'github-actions' 37 | # or GitHub Actions, set the directory to / to check for workflow files in .github/workflows. 38 | directory: '/' 39 | target-branch: "version-2.01" 40 | schedule: 41 | interval: 'daily' 42 | 43 | # Enable version updates for python 44 | - package-ecosystem: 'pip' 45 | # Look for `requirements.txt` file 46 | directory: '/' 47 | target-branch: "version-2.01" 48 | schedule: 49 | interval: 'daily' 50 | -------------------------------------------------------------------------------- /.github/workflows/CI.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a variety of Python versions 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions 3 | 4 | name: CI 5 | 6 | on: 7 | push: 8 | paths-ignore: 9 | - "**/README.rst" 10 | - "**/dependabot.yml" 11 | branches: [version-2.03] 12 | pull_request: 13 | paths-ignore: 14 | - "**/README.rst" 15 | - "**/dependabot.yml" 16 | branches: [version-2.03] 17 | 18 | concurrency: CI-2.03 19 | 20 | jobs: 21 | build: 22 | runs-on: ubuntu-latest 23 | 24 | steps: 25 | - uses: actions/checkout@v4 26 | with: 27 | submodules: recursive 28 | 29 | - name: Set up Python 30 | uses: actions/setup-python@v5 31 | with: 32 | python-version: '3.9' 33 | cache: 'pip' 34 | 35 | - name: Install dependencies 36 | run: | 37 | sudo apt-get update 38 | sudo apt-get -y install libxml2-dev libxslt-dev libxslt1-dev python3-dev python3-lxml libxml2-utils 39 | python -m pip install --upgrade pip setuptools wheel 40 | python -m pip install -r requirements.txt 41 | 42 | - name: flake8 lint 43 | run: flake8 44 | 45 | - name: Build and test 46 | run: | 47 | ./gen.sh 48 | pytest tests 49 | 50 | automerge: 51 | needs: build 52 | runs-on: ubuntu-latest 53 | permissions: 54 | pull-requests: write 55 | contents: write 56 | steps: 57 | - uses: fastify/github-action-merge-dependabot@v3.11 58 | with: 59 | github-token: ${{secrets.GITHUB_TOKEN}} 60 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | *.pyc 3 | pyenv* 4 | .DS_Store 5 | .vscode 6 | **/__pycache__ 7 | ./pytest_cache 8 | 9 | # Output HTML folders 10 | docs/ 11 | docs-copy/ 12 | 13 | # IATI Theme (for generating a local version of the reference HTML with full IATI branding) 14 | IATI-Websites/ 15 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "IATI-Schemas"] 2 | path = IATI-Schemas 3 | url = https://github.com/IATI/IATI-Schemas.git 4 | branch = version-2.03 5 | [submodule "IATI-Rulesets"] 6 | path = IATI-Rulesets 7 | url = https://github.com/IATI/IATI-Rulesets.git 8 | branch = version-2.03 9 | [submodule "IATI-Codelists"] 10 | path = IATI-Codelists 11 | url = https://github.com/IATI/IATI-Codelists.git 12 | branch = version-2.03 13 | [submodule "IATI-Extra-Documentation"] 14 | path = IATI-Extra-Documentation 15 | url = https://github.com/IATI/IATI-Extra-Documentation.git 16 | branch = version-2.03 17 | -------------------------------------------------------------------------------- /CONTRIBUTING.rst: -------------------------------------------------------------------------------- 1 | If you would like to contribute to the IATI Standard SSOT project, you can.... 2 | 3 | * Send us feedback about your user experience. Contact details at: https://github.com/IATI 4 | * Report bugs 5 | * Request new features 6 | * Contribute code, text, or documents to improve the application. See the list of specific tasks below. 7 | 8 | How to report a bug or request a feature 9 | ======================================== 10 | If you are able to work with GitHub then please "Create an issue" 11 | 12 | Before creating a new issue check to see if the issue already exists. If not then please do report it. If you have permissions to assign the issue to people and milestones then try to select suitable options. 13 | 14 | If you are not comfortable working with GitHub, but would still like to contribute, then talk to us. Details at: https://github.com/IATI 15 | 16 | 17 | How to contribute code and documents 18 | ==================================== 19 | 20 | How we use branches in this repository 21 | -------------------------------------- 22 | 23 | * each version of the IATI Standard to which this code/documentation refers has its own branch. e.g. version-2.01 24 | * Other branches represent development work or bug fixes. 25 | 26 | Submitting changes 27 | ------------------ 28 | 29 | * Fork this repository (if you haven't previously) 30 | * Make sure you're working on top of an up to date copy of IATI's master branch 31 | - Create a branch named after the work you're doing (if you're targeting a specific issue, start the branch name with the issue number e.g. ``42-feature-name``) 32 | * Do your work 33 | - If your work addresses a specific issue, reference that issue in your commit message by starting the commit message with ``[#issue number]`` e.g. ``[#64]`` 34 | * Create a pull request 35 | 36 | Specific Tasks: 37 | =============== 38 | 39 | Deployment 40 | ---------- 41 | If you find any issues in deploying your own version of the code/documentation we'd love to hear about it and try to improve our deployment documentation. 42 | 43 | 44 | Documentation 45 | ------------- 46 | We would welcome any improvements to how the code or the application is documented. 47 | 48 | Unit Tests 49 | ---------- 50 | Can you improve the unit testing to make deployment more robust? 51 | 52 | Fix a Bitesize issue 53 | -------------------- 54 | We mark some of issues as 'Bitesize'. Generally these will help ease you into the code and help you find your way around. 55 | 56 | Talk to us 57 | ========== 58 | We'd love to hear from you. Details at: https://github.com/IATI 59 | 60 | 61 | For general guidance on contributing to IATI Code please see http://iatistandard.org/developer/contribute/ 62 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | This is the license for the software in this repository. For the license of the 2 | text, which is stored in other repositories, please see 3 | https://github.com/IATI/IATI-Guidance/blob/master/en/license.rst 4 | 5 | 6 | 7 | 8 | The MIT License (MIT) 9 | 10 | Copyright (c) 2013-2014 Ben Webb 11 | 12 | Permission is hereby granted, free of charge, to any person obtaining a copy of 13 | this software and associated documentation files (the "Software"), to deal in 14 | the Software without restriction, including without limitation the rights to 15 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 16 | the Software, and to permit persons to whom the Software is furnished to do so, 17 | subject to the following conditions: 18 | 19 | The above copyright notice and this permission notice shall be included in all 20 | copies or substantial portions of the Software. 21 | 22 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 23 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 24 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 25 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 26 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 27 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 28 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | IATI Standard SSOT 2 | ================== 3 | 4 | .. image:: https://github.com/IATI/IATI-Standard-SSOT/workflows/CI/badge.svg 5 | :target: https://github.com/IATI/IATI-Standard-SSOT/actions 6 | 7 | .. image:: https://requires.io/github/IATI/IATI-Standard-SSOT/requirements.svg?branch=version-2.03 8 | :target: https://requires.io/github/IATI/IATI-Standard-SSOT/requirements/?branch=version-2.03 9 | :alt: Requirements Status 10 | .. image:: https://img.shields.io/badge/license-MIT-blue.svg 11 | :target: https://github.com/IATI/IATI-Standard-SSOT/blob/version-2.03/LICENSE 12 | 13 | Introduction 14 | ------------ 15 | 16 | This is the main github repository for the IATI Standard Single Source of Truth (SSOT). For more detailed information about the SSOT, please see http://iatistandard.org/developer/ssot/ 17 | 18 | Building the documentation 19 | ========================== 20 | 21 | Requirements: 22 | 23 | * Git 24 | * Unix based setup (e.g. Linux, Mac OS X) with bash etc. 25 | * Python 3.x 26 | * gcc 27 | * Development files for libxml and libxslt e.g. libxml2-dev, libxslt-dev 28 | 29 | Fetch the source code::: 30 | 31 | git clone https://github.com/IATI/IATI-Standard-SSOT.git 32 | 33 | Pull in the git submodules::: 34 | 35 | git submodule init 36 | git submodule update 37 | 38 | Set up a virtual environment: 39 | 40 | .. code-block:: bash 41 | 42 | # Create a virtual environment (recommended) 43 | python3 -m venv pyenv 44 | 45 | # Activate the virtual environment if you created one 46 | # This must repeated each time you open a new shell 47 | source pyenv/bin/activate 48 | 49 | # Install python requirements 50 | pip install -r requirements.txt 51 | 52 | Build the documentation::: 53 | 54 | ./gen.sh 55 | 56 | The built documentation is now in ``docs//_build/dirhtml`` 57 | 58 | 59 | Editing the documentation 60 | ========================= 61 | 62 | Make any changes in ``IATI-Extra-Documentation``, as the ``docs`` directory is generated from 63 | this and other sources each time ``./gen.sh`` is run. 64 | 65 | 66 | Building a website that also includes additonal guidance 67 | ======================================================== 68 | 69 | There is additonal guidance in the following git repositories: 70 | 71 | * https://github.com/IATI/IATI-Guidance 72 | * https://github.com/IATI/IATI-Developer-Documentation/ 73 | 74 | These are not versioned with the standard, so are not included in this repository (`IATI-Standard-SSOT `__) or its submodules. 75 | 76 | To generate a copy of the website with these extra repositories included, run: 77 | 78 | .. code-block:: bash 79 | 80 | # If you have not done already create the docs directory as a git repository 81 | # (more info below) 82 | mkdir docs 83 | cd docs 84 | git init 85 | cd .. 86 | # Actually run the generate script 87 | ./combined_gen.sh 88 | 89 | This generates the website in the ``docs`` directory, but then copies it to ``docs-copy`` at the end, so that a webserver can be pointed to ``docs-copy/en/_build/dirhtml`` and not be interrupted when the site is being rebuilt. 90 | 91 | The ``docs`` directory should be a git repository in order to support adding the "Last updated" line to the bottom of the page. We build the live and dev websites in different directories so that the last updated date corresponds to when the site was actually changed, not when the relevant commit was added to the source git respository. 92 | 93 | 94 | Generating a local version with the IATI theme 95 | ============================================== 96 | 97 | A local version of the website (with the full IATI theme) can be generated after cloning the theme files and setting up the required symlinks for Sphinx to follow when generating the HTML files. 98 | 99 | .. code-block:: bash 100 | 101 | # Clone the repository containing the IATI theme at the same level where you cloned IATI-Standard-SSOT 102 | git clone https://github.com/IATI/IATI-Websites.git IATI-Websites 103 | 104 | # Set-up symlinks to the template file/folders 105 | # for the symlinks to work, you'll have to be inside the IATI-Extra-Documentation folder cloned inside IATI-Standard-SSOT 106 | cd IATI-Extra-Documentation/en 107 | ln -s ../../../IATI-Websites/iatistandard/_templates/ ./ 108 | ln -s ../../../IATI-Websites/iatistandard/_static/ ./ 109 | ln -s ../../../IATI-Websites/iatistandard/_templates/layout_dev.html ./_templates/layout.html 110 | 111 | # Generate a version of the documentation 112 | cd ../../ 113 | ./combined_gen.sh 114 | 115 | # The documentation HTML files are stored in the `docs-copy/en/_build/dirhtml` folder 116 | # You can navigate around your generated version of the site using a Python HTTP webserver 117 | cd docs-copy/en/_build/dirhtml 118 | python3 -m http.server 119 | # View the site in a browser at http://0.0.0.0:8000/ 120 | -------------------------------------------------------------------------------- /combined_gen.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # This script pulls in the Developer Documetnation and Guidance to build the full iatistandard.org website 3 | # See the README for more information 4 | ./gen_rst.sh || exit $? 5 | 6 | cd docs || exit 1 7 | 8 | mkdir en/developer 9 | cp -n ../../IATI-Developer-Documentation/*.rst en/developer 10 | cp -rn ../../IATI-Developer-Documentation/*/ en/developer 11 | mkdir en/guidance 12 | cp -n ../..//IATI-Guidance/en/*.rst en/guidance 13 | cp -rn ../../IATI-Guidance/en/*/ en/guidance 14 | mv en/guidance/404.rst en/ 15 | mv en/guidance/upgrades* en/ 16 | mv en/guidance/introduction* en/ 17 | mv en/guidance/key-considerations* en/ 18 | mv en/guidance/license* en/ 19 | cp ../combined_sitemap.rst en/sitemap.rst 20 | 21 | git add . 22 | git commit -a -m 'Auto' 23 | git ls-tree -r --name-only HEAD | grep 'rst$' | while read filename; do 24 | echo $'\n\n\n'"*Last updated on $(git log -1 --format="%ad" --date=short -- $filename)*" >> $filename 25 | done 26 | 27 | cd .. || exit 1 28 | ./gen_html.sh || exit $? 29 | 30 | echo '' > docs/en/_build/dirhtml/sitemap.xml 31 | find docs/en/_build/dirhtml | grep -v _static | grep index.html$ | sed 's|index.html$||' | sed "s|docs/en/_build/dirhtml|http://`cat URL`|" >> docs/en/_build/dirhtml/sitemap.xml 32 | echo '' >> docs/en/_build/dirhtml/sitemap.xml 33 | 34 | cp -r docs docs-copy.new 35 | mv docs-copy docs-copy.old 36 | mv docs-copy.new docs-copy 37 | rm -rf docs-copy.old 38 | sed -i 's/\.\.\//\//g' docs-copy/en/_build/dirhtml/404/index.html 39 | 40 | -------------------------------------------------------------------------------- /combined_sitemap.rst: -------------------------------------------------------------------------------- 1 | Sitemap 2 | ======= 3 | 4 | .. toctree:: 5 | :includehidden: 6 | :titlesonly: 7 | 8 | introduction 9 | key-considerations 10 | guidance/index 11 | reference 12 | upgrades 13 | developer/index 14 | 15 | -------------------------------------------------------------------------------- /gen.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import os 3 | import json 4 | import shutil 5 | import textwrap 6 | import jinja2 7 | from lxml import etree as ET 8 | from collections import defaultdict 9 | from iatirulesets.text import rules_text 10 | 11 | languages = ['en'] 12 | 13 | # Define the namespaces necessary for opening schema files 14 | namespaces = { 15 | 'xsd': 'http://www.w3.org/2001/XMLSchema' 16 | } 17 | # Define attributes that have documentation that differs to that in the schema 18 | custom_attributes = { 19 | } 20 | 21 | 22 | def get_github_url(repo, path=''): 23 | """Return a link to the Github UI for a given repository and filepath. 24 | 25 | Args: 26 | repo (str): The repository that contains the file at the input path. 27 | path (str): The path (within the repository) to the file. There should be no preceeding slash ('/'). 28 | 29 | Returns: 30 | str: Link to the Github UI page. 31 | """ 32 | github_branches = { 33 | 'IATI-Schemas': 'version-2.03', 34 | 'IATI-Codelists': 'version-2.03', 35 | 'IATI-Rulesets': 'version-2.03', 36 | 'IATI-Extra-Documentation': 'version-2.03', 37 | 'IATI-Codelists-NonEmbedded': 'master', 38 | } 39 | return 'https://github.com/IATI/{0}/blob/{1}/{2}'.format(repo, github_branches[repo], path) 40 | 41 | 42 | def human_list(original_list): 43 | """Return a human-friendly version of a list. 44 | 45 | Currently seperates list items with commas, but could be extended to insert 'and'/'or' correctly. 46 | 47 | Args: 48 | original_list (list): The list to be made human-friendly. 49 | 50 | Returns: 51 | str: The human-friendly represention of the list. 52 | """ 53 | return ', '.join(original_list) 54 | 55 | 56 | def lookup_see_also(standard, mapping, path): 57 | """Return a generator object containing paths relating to the current element as defined by overview-mapping.json. 58 | 59 | Args: 60 | standard (str): Can be either organisation-standard or activity-standard) 61 | mapping (list): List for all templates elements within [standard]/overview-mapping.json 62 | path (str): Last sections of the path passed to see_also, if shorter than 3 sections it will just be the entire path 63 | 64 | Returns: 65 | generator or str: Yields paths of elements related to the current element 66 | """ 67 | if path == '': 68 | return 69 | for overview, elements in mapping.items(): 70 | if path in elements: 71 | yield '/' + standard + '/overview/' + overview 72 | for x in lookup_see_also(standard, mapping, '/'.join(path.split('/')[:-1])): 73 | yield x 74 | 75 | 76 | def see_also(path, lang): 77 | return list() 78 | 79 | 80 | standard_ruleset = json.load(open('./IATI-Rulesets/rulesets/standard.json')) 81 | 82 | 83 | def ruleset_page(lang): 84 | jinja_env = jinja2.Environment(loader=jinja2.FileSystemLoader('templates')) 85 | ruleset = {xpath: rules_text(rules) for xpath, rules in standard_ruleset.items()} 86 | rst_filename = os.path.join(lang, 'rulesets', 'standard-ruleset.rst') 87 | 88 | try: 89 | os.mkdir(os.path.join('docs', lang, 'rulesets')) 90 | except OSError: 91 | pass 92 | 93 | with open(os.path.join('docs', rst_filename), 'w') as fp: 94 | t = jinja_env.get_template(lang + '/ruleset.rst') 95 | fp.write(t.render( 96 | ruleset=ruleset, 97 | extra_docs=get_extra_docs(rst_filename) 98 | )) 99 | 100 | 101 | def ruleset_text(path): 102 | """Return a list of text describing the rulesets for a given path (xpath)""" 103 | out = [] 104 | for xpath, rules in standard_ruleset.items(): 105 | try: 106 | # Use slice 1: to ensure we match /budget/ but not /total-budget/ 107 | reduced_path = path.split(xpath[1:] + '/')[1] 108 | except IndexError: 109 | continue 110 | out += rules_text(rules, reduced_path) 111 | return out 112 | 113 | 114 | codelists_paths = defaultdict(list) 115 | # TODO - This function should be moved into the IATI-Codelists submodule 116 | codelist_mappings = ET.parse('./IATI-Codelists/mapping.xml').getroot().findall('mapping') 117 | 118 | 119 | def match_codelists(path): 120 | """ 121 | Looks up the codelist that the given path (xpath) should be on. 122 | Returns a tuble of the codelist name, and a boolean as describing whether any conditions apply. 123 | If there is no codelist for the given path, the first part of the tuple is None. 124 | 125 | """ 126 | codelist_tuples = [] 127 | for mapping in codelist_mappings: 128 | if mapping.find('path').text.startswith('//'): 129 | if path.endswith(mapping.find('path').text.strip('/')): 130 | codelist = mapping.find('codelist').attrib['ref'] 131 | if path not in codelists_paths[codelist]: 132 | codelists_paths[codelist].append(path) 133 | tup = (codelist, mapping.find('condition') is not None) 134 | codelist_tuples.append(tup) 135 | else: 136 | pass # FIXME 137 | return codelist_tuples 138 | 139 | 140 | def is_complete_codelist(codelist_name): 141 | """Determine whether the specified Codelist is complete. 142 | 143 | Args: 144 | codelist_name (str): The name of the Codelist. This is case-sensitive and must match the mapping file. 145 | 146 | Returns: 147 | bool: Whether the Codelist is complete. 148 | 149 | Note: 150 | Need to manually specify which Codelists are incomplete - it is not auto-detected. This is due to the surrounding architecture making it a challenge to auto-detect this information. 151 | 152 | """ 153 | # use a list of incomplete Codelists since it is shorter 154 | incomplete_codelists = [ 155 | 'Country', 156 | 'HumanitarianScopeType', 157 | 'HumanitarianScopeVocabulary', 158 | 'IndicatorVocabulary', 159 | 'Language', 160 | 'OrganisationIdentifier', 161 | 'OrganisationRegistrationAgency' 162 | ] 163 | return codelist_name not in incomplete_codelists 164 | 165 | 166 | def path_to_ref(path): 167 | return path.replace('//', '_').replace('@', '.') 168 | 169 | 170 | def get_extra_docs(rst_filename): 171 | extra_docs_file = os.path.join('IATI-Extra-Documentation', rst_filename) 172 | if os.path.isfile(extra_docs_file): 173 | with open(extra_docs_file) as fp: 174 | return fp.read() 175 | else: 176 | return '' 177 | 178 | 179 | class Schema2Doc(object): 180 | """Class for converting an IATI XML schema to documentation in the reStructuredText format.""" 181 | def __init__(self, schema, lang): 182 | """ 183 | Args: 184 | schema (str): The filename of the schema to use, e.g. 'iati-activities-schema.xsd' 185 | lang (str): A two-letter (ISO 639-1) language code to build the documentation for (e.g. 'en') 186 | 187 | Sets: 188 | self.tree (lxml.etree._ElementTree): Representing the input schema. 189 | self.tree2 (lxml.etree._ElementTree): Representing the iati-common.xsd schema. 190 | self.jinja_env (jinja2.environment.Environment): The templates contained within the 'templates' folder. 191 | self.lang (str): The input language. 192 | """ 193 | self.tree = ET.parse("./IATI-Schemas/" + schema) 194 | self.tree2 = ET.parse("./IATI-Schemas/iati-common.xsd") 195 | self.jinja_env = jinja2.Environment(loader=jinja2.FileSystemLoader('templates')) 196 | self.lang = lang 197 | 198 | self.jinja_env.filters['is_complete_codelist'] = is_complete_codelist 199 | 200 | def get_schema_element(self, tag_name, name_attribute): 201 | """Returns the xsd definition for a given element from schemas defined in `self.tree` (or `self.tree2` if nothing found). 202 | 203 | Args: 204 | tag_name (str): The name of the tag in the schema - will typically be 'element'. 205 | name_attribute (str): The value of the 'name' attribute in the schema - i.e. the name of the element/type etc. being described, e.g. 'iati-activities'. 206 | 207 | Returns: 208 | None / lxml.etree._Element: The element tree representng the xsd definition for the given inputs. None if no match found. 209 | """ 210 | schema_element = self.tree.find("xsd:{0}[@name='{1}']".format(tag_name, name_attribute), namespaces=namespaces) 211 | if schema_element is None: 212 | schema_element = self.tree2.find("xsd:{0}[@name='{1}']".format(tag_name, name_attribute), namespaces=namespaces) 213 | return schema_element 214 | 215 | def schema_documentation(self, element, ref_element, type_element=None): 216 | """Return a documention string for either a given ref_element (if not None) or an element. 217 | 218 | If the element is a document-link, it will obtain the documentation string from its extension root instead of the extension itself. 219 | 220 | Args: 221 | element (lxml.etree._Element): An xsd element definition. 222 | ref_element (lxml.etree._Element): An element that the `element` inherits properties definitions from (using the xsd `ref` inheritance). If set to None, the documention string for the element is returned. 223 | type_element (lxml.etree._Element): An element that the `element` inherits properties definitions from (using the xsd `type` inheritance). Defaults to None, implying element properties/definitions are defined within `element` or `ref_element`. 224 | 225 | Returns: 226 | str: The documentation string, extracted from the input ref_element or element. 227 | """ 228 | if ref_element is not None: 229 | xsd_docuementation = ref_element.find(".//xsd:documentation", namespaces=namespaces) 230 | if xsd_docuementation is not None: 231 | return xsd_docuementation.text 232 | 233 | xsd_documentation = element.find(".//xsd:documentation", namespaces=namespaces) 234 | if xsd_documentation is not None: 235 | return xsd_documentation.text 236 | 237 | if type_element is not None: 238 | xsd_documentation = type_element.find(".//xsd:documentation", namespaces=namespaces) 239 | if xsd_documentation is not None: 240 | return xsd_documentation.text 241 | 242 | extension = type_element.find(".//xsd:extension", namespaces=namespaces) 243 | if extension is not None: 244 | base_name = type_element.find(".//xsd:extension", namespaces=namespaces).get("base") 245 | base_element = self.get_schema_element('complexType', base_name) 246 | if base_element is not None: 247 | return base_element.find(".//xsd:documentation", namespaces=namespaces).text 248 | 249 | def output_docs(self, element_name, path, element=None, minOccurs='', maxOccurs='', ref_element=None, type_element=None): 250 | """Output documentation for the given element, and it's children. 251 | 252 | If element is not given, we try to find it in the schema using it's 253 | element_name. 254 | 255 | Args: 256 | element_name (str): 257 | path (str): The xpath of the context where this element was found. For the root context (i.e. iati-activities), this is an empty string. 258 | element (lxml.etree._Element): If element is not given, we try to find it in the schema using it's element_name. 259 | minOccurs (str): The number of minimum occurances for the given element_name / element. 260 | maxOccurs (str): The number of minimum occurances for the given element_name / element. 261 | ref_element (lxml.etree._Element): An element that the `element` inherits properties definitions from (using the xsd `ref` inheritance). Defaults to None, implying element properties are defined within `element` or `type_element`. 262 | type_element (lxml.etree._Element): An element that the `element` inherits properties definitions from (using the xsd `type` inheritance). Defaults to None, implying element properties are defined within `element` or `ref_element`. 263 | """ 264 | if element is None: 265 | element = self.get_schema_element('element', element_name) 266 | if element is None: 267 | return 268 | 269 | github_urls = { 270 | 'schema': element.base.replace('./IATI-Schemas/', get_github_url('IATI-Schemas')) + '#L' + str(element.sourceline), 271 | 'extra_documentation': get_github_url('IATI-Extra-Documentation', self.lang + '/' + path + element_name + '.rst') 272 | } 273 | try: 274 | os.makedirs(os.path.join('docs', self.lang, path)) 275 | except OSError: 276 | pass 277 | 278 | rst_filename = os.path.join(self.lang, path, element_name + '.rst') 279 | 280 | children = self.element_loop(element, path) 281 | for child_name, child_element, child_ref_element, child_type_element, child_minOccurs, child_maxOccurs in children: 282 | self.output_docs(child_name, path + element.attrib['name'] + '/', child_element, child_minOccurs, child_maxOccurs, child_ref_element, child_type_element) 283 | 284 | min_occurss = element.xpath('xsd:complexType/xsd:choice/@minOccur', namespaces=namespaces) 285 | # Note that this min_occurs is different to the python variables 286 | # minOccurs and maxOccurs, because this is read from a choice element, 287 | # whereas those are read from the individual element definitions (only 288 | # possible within a sequence element) 289 | if min_occurss: 290 | min_occurs = int(min_occurss[0]) 291 | else: 292 | min_occurs = 0 293 | 294 | with open('docs/' + rst_filename, 'w') as fp: 295 | t = self.jinja_env.get_template(self.lang + '/schema_element.rst') 296 | fp.write(t.render( 297 | element_name=element_name, 298 | element_name_underline='=' * len(element_name), 299 | element=element, 300 | path='/'.join(path.split('/')[1:]), # Strip e.g. activity-standard/ from the path 301 | github_urls=github_urls, 302 | schema_documentation=textwrap.dedent(self.schema_documentation(element, ref_element, type_element)), 303 | extended_types=element.xpath('xsd:complexType/xsd:simpleContent/xsd:extension/@base', namespaces=namespaces), 304 | attributes=self.attribute_loop(element), 305 | textwrap=textwrap, 306 | match_codelists=match_codelists, 307 | path_to_ref=path_to_ref, 308 | ruleset_text=ruleset_text, 309 | childnames=[x[0] for x in children], 310 | extra_docs=get_extra_docs(rst_filename), 311 | min_occurs=min_occurs, 312 | minOccurs=minOccurs, 313 | maxOccurs=maxOccurs, 314 | see_also=see_also(path + element_name, self.lang) 315 | )) 316 | 317 | def output_schema_table(self, element_name, path, element=None, output=False, filename='', title='', minOccurs='', maxOccurs='', ref_element=None, type_element=None): 318 | if element is None: 319 | element = self.get_schema_element('element', element_name) 320 | if element is None: 321 | return 322 | 323 | extended_types = element.xpath('xsd:complexType/xsd:simpleContent/xsd:extension/@base', namespaces=namespaces) 324 | base_type = element.get('type') if element.get('type') and element.get('type').startswith('xsd:') else '' 325 | if type_element is not None: 326 | complex_base_types = [x for x in type_element.xpath('xsd:simpleContent/xsd:extension/@base', namespaces=namespaces) if x.startswith('xsd:')] 327 | if len(complex_base_types) and base_type == '': 328 | base_type = complex_base_types[0] 329 | 330 | rows = [{ 331 | 'name': element_name, 332 | 'path': '/'.join(path.split('/')[1:]) + element_name, 333 | 'doc': '/' + path + element_name, 334 | 'description': textwrap.dedent(self.schema_documentation(element, ref_element, type_element)), 335 | 'type': base_type, 336 | 'occur': (minOccurs or '') + '..' + ('*' if maxOccurs == 'unbounded' else maxOccurs or ''), 337 | 'section': len(path.split('/')) < 5 338 | }] 339 | 340 | if element.xpath('xsd:complexType[@mixed="true"] or xsd:complexType/xsd:simpleContent', namespaces=namespaces): 341 | rows.append({ 342 | 'path': '/'.join(path.split('/')[1:]) + element_name + '/text()', 343 | 'description': '', 344 | 'type': 'mixed' if element.xpath('xsd:complexType[@mixed="true"]', namespaces=namespaces) else ','.join([x for x in extended_types if x.startswith('xsd:')]), 345 | }) 346 | 347 | for a_name, a_type, a_description, a_required in self.attribute_loop(element): 348 | rows.append({ 349 | 'attribute_name': a_name, 350 | 'path': '/'.join(path.split('/')[1:]) + element_name + '/@' + a_name, 351 | 'description': textwrap.dedent(a_description), 352 | 'type': a_type, 353 | 'occur': '1..1' if a_required else '0..1' 354 | }) 355 | 356 | for child_name, child_element, child_ref_element, child_type_element, minOccurs, maxOccurs in self.element_loop(element, path): 357 | rows += self.output_schema_table(child_name, path + element.attrib['name'] + '/', child_element, minOccurs=minOccurs, maxOccurs=maxOccurs, ref_element=child_ref_element, type_element=child_type_element) 358 | 359 | if output: 360 | with open(os.path.join('docs', self.lang, filename), 'w') as fp: 361 | t = self.jinja_env.get_template(self.lang + '/schema_table.rst') 362 | fp.write(t.render( 363 | rows=rows, 364 | title=title, 365 | root_path='/'.join(path.split('/')[1:]), # Strip e.g. activity-standard/ from the path 366 | match_codelists=match_codelists, 367 | ruleset_text=ruleset_text, 368 | description=self.tree.xpath('xsd:annotation/xsd:documentation[@xml:lang="en"]', namespaces=namespaces)[0].text, 369 | extra_docs=get_extra_docs(os.path.join(self.lang, filename)) 370 | )) 371 | else: 372 | return rows 373 | 374 | def output_overview_pages(self, standard): 375 | if self.lang == 'en': # FIXME 376 | try: 377 | os.mkdir(os.path.join('docs', self.lang, standard, 'overview')) 378 | except OSError: 379 | pass 380 | 381 | mapping = json.load(open(os.path.join('IATI-Extra-Documentation', self.lang, standard, 'overview-mapping.json'))) 382 | for page, reference_pages in mapping.items(): 383 | self.output_overview_page(standard, page, reference_pages) 384 | 385 | def output_overview_page(self, standard, page, reference_pages): 386 | if standard == 'activity-standard': 387 | f = lambda x: x if x.startswith('iati-activities') else 'iati-activities/iati-activity/' + x 388 | else: 389 | f = lambda x: x if x.startswith('iati-organisations') else 'iati-organisations/iati-organisation/' + x 390 | reference_pages = [(x, '/' + standard + '/' + f(x)) for x in reference_pages] 391 | with open(os.path.join('docs', self.lang, standard, 'overview', page + '.rst'), 'w') as fp: 392 | t = self.jinja_env.get_template(self.lang + '/overview.rst') 393 | fp.write(t.render( 394 | extra_docs=get_extra_docs(os.path.join(self.lang, standard, 'overview', page + '.rst')), 395 | reference_pages=reference_pages 396 | )) 397 | 398 | def element_loop(self, element, path): 399 | """Find child elements for a given input element. 400 | 401 | Args: 402 | element (lxml.etree._Element): The base element to find child elements for. 403 | path (str): Unused. 404 | 405 | Returns: 406 | list: A list containing tuples for each child element found. Each tuple takes the form of: 407 | str: Element name, 408 | lxml.etree._Element: Represention of the element, 409 | Unknown: ref element, 410 | lxml.etree._Element or None: type_element, 411 | str: minimum number of occurances, 412 | str: maximum number of occurances (could be a number or 'unbounded') 413 | """ 414 | a = element.attrib 415 | type_elements = [] 416 | if 'type' in a: 417 | complexType = self.get_schema_element('complexType', a['type']) 418 | if complexType is not None: 419 | # If this complexType is an extension of another complexType, find the base element and include any child elements 420 | try: 421 | base_name = complexType.find('xsd:complexContent/xsd:extension', namespaces=namespaces).attrib.get('base') 422 | base_type_element = self.get_schema_element('complexType', base_name) 423 | type_elements = ( 424 | base_type_element.findall('xsd:choice/xsd:element', namespaces=namespaces) + 425 | base_type_element.findall('xsd:sequence/xsd:element', namespaces=namespaces)) 426 | except AttributeError: 427 | pass 428 | # This complexType is not extended from a complexType base 429 | 430 | type_elements += ( 431 | complexType.findall('xsd:choice/xsd:element', namespaces=namespaces) + 432 | complexType.findall('xsd:sequence/xsd:element', namespaces=namespaces) + 433 | complexType.findall('xsd:complexContent/xsd:extension/xsd:sequence/xsd:element', namespaces=namespaces)) 434 | 435 | children = ( 436 | element.findall('xsd:complexType/xsd:choice/xsd:element', namespaces=namespaces) 437 | + element.findall('xsd:complexType/xsd:sequence/xsd:element', namespaces=namespaces) 438 | + element.findall("xsd:complexType/xsd:all/xsd:element", namespaces=namespaces) 439 | + type_elements 440 | ) 441 | child_tuples = [] 442 | for child in children: 443 | a = child.attrib 444 | if 'type' in a: 445 | type_element = self.get_schema_element('complexType', a['type']) 446 | else: 447 | type_element = None 448 | 449 | if 'name' in a: 450 | child_tuples.append((a['name'], child, None, type_element, a.get('minOccurs'), a.get('maxOccurs'))) 451 | else: 452 | child_tuples.append((a['ref'], None, child, type_element, a.get('minOccurs'), a.get('maxOccurs'))) 453 | 454 | return child_tuples 455 | 456 | def attribute_loop(self, element): 457 | """Returns a list containing a tuple for each attribute that the input element can have. 458 | 459 | Args: 460 | element (lxml.etree._Element): The base element to find attributes for. 461 | 462 | Returns: 463 | list: A list containing tuples for each attribute found. Each tuple takes the form of: 464 | str: The name of the attribute. 465 | str: The xsd type of the attribute. 466 | str: The documentation string for the given attribute. 467 | bool: A boolean value representing if the attribute is required. 468 | """ 469 | # if element.find("xsd:complexType[@mixed='true']", namespaces=namespaces) is not None: 470 | # print_column_info('text', indent) 471 | 472 | a = element.attrib 473 | type_attributes = [] 474 | type_attributeGroups = [] 475 | if 'type' in a: 476 | complexType = self.get_schema_element('complexType', a['type']) 477 | 478 | # If this complexType is an extension of another complexType, find the base element and use this to find any attributes 479 | try: 480 | base_name = complexType.find('.//xsd:complexContent/xsd:extension', namespaces=namespaces).attrib.get('base') 481 | complexType = self.get_schema_element('complexType', base_name) 482 | except AttributeError: 483 | pass 484 | # This complexType is not extended from a complexType base 485 | 486 | if complexType is None: 487 | print('Notice: No attributes for', a['type']) 488 | else: 489 | type_attributes = ( 490 | complexType.findall('xsd:attribute', namespaces=namespaces) + 491 | complexType.findall('xsd:simpleContent/xsd:extension/xsd:attribute', namespaces=namespaces) 492 | ) 493 | type_attributeGroups = ( 494 | complexType.findall('xsd:attributeGroup', namespaces=namespaces) + 495 | complexType.findall('xsd:simpleContent/xsd:extension/xsd:attributeGroup', namespaces=namespaces) 496 | ) 497 | 498 | group_attributes = [] 499 | for attributeGroup in ( 500 | element.findall('xsd:complexType/xsd:attributeGroup', namespaces=namespaces) + 501 | element.findall('xsd:complexType/xsd:simpleContent/xsd:extension/xsd:attributeGroup', namespaces=namespaces) + 502 | type_attributeGroups 503 | ): 504 | group_attributes += self.get_schema_element('attributeGroup', attributeGroup.attrib['ref']).findall('xsd:attribute', namespaces=namespaces) 505 | 506 | out = [] 507 | for attribute in ( 508 | element.findall('xsd:complexType/xsd:attribute', namespaces=namespaces) + 509 | element.findall('xsd:complexType/xsd:simpleContent/xsd:extension/xsd:attribute', namespaces=namespaces) + 510 | type_attributes + group_attributes 511 | ): 512 | doc = attribute.find(".//xsd:documentation", namespaces=namespaces) 513 | occurs = attribute.get('use') 514 | if 'ref' in attribute.attrib: 515 | if attribute.get('ref') in custom_attributes: 516 | out.append((attribute.get('ref'), '', custom_attributes[attribute.get('ref')], attribute.get('use') == 'required')) 517 | continue 518 | referenced_attribute = self.get_schema_element('attribute', attribute.get('ref')) 519 | if referenced_attribute is not None: 520 | attribute = referenced_attribute 521 | if doc is None: 522 | # Only fetch the documentation of the referenced definition 523 | # if we don't already have documentation. 524 | doc = attribute.find(".//xsd:documentation", namespaces=namespaces) 525 | if occurs is None: 526 | occurs = attribute.get('use') 527 | out.append((attribute.get('name') or attribute.get('ref'), attribute.get('type'), doc.text if doc is not None else '', occurs == 'required')) 528 | return out 529 | 530 | 531 | def codelists_to_docs(lang): 532 | dirname = 'IATI-Codelists/out/clv2/json/' + lang 533 | try: 534 | os.mkdir('docs/' + lang + '/codelists/') 535 | except OSError: 536 | pass 537 | 538 | for fname in os.listdir(dirname): 539 | json_file = os.path.join(dirname, fname) 540 | if not fname.endswith('.json'): 541 | continue 542 | with open(json_file, 'r+') as fp: 543 | codelist_json = json.load(fp) 544 | 545 | fname = fname[:-5] 546 | embedded = os.path.exists(os.path.join('IATI-Codelists', 'xml', fname + '.xml')) 547 | if embedded: 548 | github_url = get_github_url('IATI-Codelists', 'xml/{0}.xml'.format(fname)) 549 | else: 550 | github_url = get_github_url('IATI-Codelists-NonEmbedded', 'xml/{0}.xml'.format(fname)) 551 | 552 | rst_filename = os.path.join(lang, 'codelists', fname + '.rst') 553 | with open(os.path.join('docs', rst_filename), 'w') as fp: 554 | jinja_env = jinja2.Environment(loader=jinja2.FileSystemLoader('templates')) 555 | t = jinja_env.get_template(lang + '/codelist.rst') 556 | fp.write(t.render( 557 | codelist_json=codelist_json, 558 | show_category_column=not all('category' not in x for x in codelist_json['data']), 559 | show_url_column=not all('url' not in x for x in codelist_json['data']), 560 | show_withdrawn=any('status' in x and x['status'] != 'active' for x in codelist_json['data']), 561 | fname=fname, 562 | len=len, 563 | github_url=github_url, 564 | codelist_paths=codelists_paths.get(fname), 565 | path_to_ref=path_to_ref, 566 | extra_docs=get_extra_docs(rst_filename), 567 | dedent=textwrap.dedent, 568 | lang=lang)) 569 | 570 | 571 | def extra_extra_docs(): 572 | """ 573 | Copy over files from IATI-Extra-Documentation that haven't been created in 574 | the docs folder by another function. 575 | 576 | """ 577 | for dirname, dirs, files in os.walk('IATI-Extra-Documentation', followlinks=True): 578 | if dirname.startswith('.'): 579 | continue 580 | for fname in files: 581 | if fname.startswith('.'): 582 | continue 583 | if len(dirname.split(os.path.sep)) == 1: 584 | rst_dirname = '' 585 | else: 586 | rst_dirname = os.path.join(*dirname.split(os.path.sep)[1:]) 587 | rst_filename = os.path.join(rst_dirname, fname) 588 | if not os.path.exists(os.path.join('docs', rst_filename)): 589 | try: 590 | os.makedirs(os.path.join('docs', rst_dirname)) 591 | except OSError: 592 | pass 593 | if fname.endswith('.rst'): 594 | with open(os.path.join('docs', rst_filename), 'w') as fp: 595 | fp.write(get_extra_docs(rst_filename)) 596 | else: 597 | shutil.copy(os.path.join(dirname, fname), os.path.join('docs', rst_filename)) 598 | 599 | 600 | if __name__ == '__main__': 601 | for language in languages: 602 | activities = Schema2Doc('iati-activities-schema.xsd', lang=language) 603 | activities.output_docs('iati-activities', 'activity-standard/') 604 | activities.output_schema_table( 605 | 'iati-activities', 'activity-standard/', output=True, 606 | filename='activity-standard/summary-table.rst', 607 | title='Activity Standard Summary Table' 608 | ) 609 | # activities.output_overview_pages('activity-standard') 610 | 611 | orgs = Schema2Doc('iati-organisations-schema.xsd', lang=language) 612 | orgs.output_docs('iati-organisations', 'organisation-standard/') 613 | orgs.output_schema_table( 614 | 'iati-organisations', 'organisation-standard/', output=True, 615 | filename='organisation-standard/summary-table.rst', 616 | title='Organisation Standard Summary Table' 617 | ) 618 | # orgs.output_overview_pages('organisation-standard') 619 | 620 | ruleset_page(lang=language) 621 | codelists_to_docs(lang=language) 622 | extra_extra_docs() 623 | -------------------------------------------------------------------------------- /gen.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -o nounset 3 | 4 | ./gen_rst.sh || exit $? 5 | ./gen_html.sh || exit $? 6 | -------------------------------------------------------------------------------- /gen_dss.py: -------------------------------------------------------------------------------- 1 | import re 2 | import textwrap 3 | import json 4 | import sys 5 | from gen import Schema2Doc, codelist_mappings, codelists_paths 6 | 7 | 8 | # Define the namespaces necessary for opening schema files 9 | namespaces = { 10 | 'xsd': 'http://www.w3.org/2001/XMLSchema' 11 | } 12 | 13 | 14 | def match_codelists(path): 15 | """ 16 | Looks up the codelist that the given path (xpath) should be on. 17 | Returns a tuple of the codelist name, and a boolean as describing whether any conditions apply. 18 | If there is no codelist for the given path, the first part of the tuple is None. 19 | 20 | """ 21 | codelist_tuples = [] 22 | for mapping in codelist_mappings: 23 | if mapping.find('path').text.startswith('//'): 24 | if path.endswith(mapping.find('path').text.strip('/')): 25 | codelist = mapping.find('codelist').attrib['ref'] 26 | if path not in codelists_paths[codelist]: 27 | codelists_paths[codelist].append(path) 28 | condition = mapping.find('condition') 29 | tup = (codelist, '' if condition is None else condition.text) 30 | codelist_tuples.append(tup) 31 | else: 32 | pass 33 | return codelist_tuples 34 | 35 | 36 | def field_to_label(field): 37 | up_str = '' 38 | for word in field.split('_'): 39 | if up_str == '': 40 | up_str = word.capitalize() 41 | else: 42 | up_str += " " + word.capitalize() 43 | return up_str 44 | 45 | 46 | def get_codelist_json(name): 47 | return json.load(open('IATI-Codelists/out/clv3/json/en/' + name + '.json')) 48 | 49 | 50 | def path_to_solr(path): 51 | final = path 52 | if 'iati-activities/iati-activity/@' in path: 53 | final = path.replace('iati-activities/iati-activity/@', '') 54 | elif 'iati-activities/iati-activity/' in path: 55 | final = path.replace('iati-activities/iati-activity/', '') 56 | elif 'iati-activities' in path: 57 | final = path.replace('iati-activities', 'dataset') 58 | return final.replace('/@', '_').replace('/', '_').replace('-', '_').replace(':', '_') 59 | 60 | 61 | def xsd_type_to_search(element_name=None, xsd_type=None): 62 | if (element_name is not None and re.search('_narrative$', element_name) is not None): 63 | return "text" 64 | 65 | if (element_name == 'location_administrative_level'): 66 | return "text" 67 | 68 | switch = { 69 | 'xsd:string': 'text', 70 | 'xsd:NMTOKEN': 'text', 71 | 'xsd:anyURI': 'text', 72 | 'xsd:decimal': 'number', 73 | 'xsd:dateTime': 'date', 74 | 'xsd:date': 'date', 75 | 'xsd:boolean': 'boolean', 76 | 'xsd:nonNegativeInteger': 'integer', 77 | 'xsd:positiveInteger': 'integer', 78 | 'xsd:int': 'integer' 79 | } 80 | return switch.get(xsd_type, "text") 81 | 82 | 83 | def filter_columns(row): 84 | if row['field'] in ['dataset', 'dataset_iati_activity']: 85 | return False 86 | return True 87 | 88 | 89 | class Schema2Solr(Schema2Doc): 90 | 91 | def output_solr(self, element_name, path, element=None, output=False, template_path='', filename='', codelist_dest='', collection='', out_type='order', minOccurs='', maxOccurs='', ref_element=None, type_element=None, parent_req=True, parent_multi=False): 92 | if element is None: 93 | element = self.get_schema_element('element', element_name) 94 | if element is None: 95 | return 96 | 97 | full_path = '/'.join(path.split('/')[1:]) + element_name 98 | solr_name = path_to_solr(full_path) 99 | xsd_type = element.get('type') if element.get('type') and element.get('type').startswith('xsd:') else '' 100 | if type_element is not None: 101 | complex_base_types = [x for x in type_element.xpath('xsd:simpleContent/xsd:extension/@base', namespaces=namespaces) if x.startswith('xsd:')] 102 | if len(complex_base_types) and xsd_type == '': 103 | xsd_type = complex_base_types[0] 104 | required = (minOccurs == '1') and parent_req 105 | if element_name == 'iati-activity': 106 | maxOccurs = '1' 107 | multivalued = (maxOccurs == 'unbounded') or parent_multi 108 | 109 | rows = [] 110 | # elements should only be in solr if they contain something with a type, otherwise they wouldn't have a flattened value 111 | if element.xpath('xsd:complexType[@mixed="true"] or xsd:complexType/xsd:simpleContent', namespaces=namespaces) or xsd_type != '': 112 | rows = [{ 113 | "field": solr_name, 114 | "label": field_to_label(solr_name), 115 | 'type': xsd_type_to_search(solr_name, xsd_type), 116 | "description": textwrap.dedent(self.schema_documentation(element, ref_element, type_element)), 117 | "name": element_name, 118 | 'path': full_path, 119 | 'xsd_type': xsd_type, 120 | 'solr_required': 'true' if required else 'false', 121 | 'solr_multivalued': 'true' if multivalued else 'false' 122 | }] 123 | 124 | for a_name, a_type, a_description, a_required in self.attribute_loop(element): 125 | full_path = '/'.join(path.split('/')[1:]) + element_name + '/@' + a_name 126 | solr_name = path_to_solr(full_path) 127 | codelist_name_tup = match_codelists(full_path) 128 | codelist_name = '' 129 | codelist_condition = '' 130 | if len(codelist_name_tup) != 0: 131 | codelist_name = codelist_name_tup[0][0] 132 | codelist_condition = codelist_name_tup[0][1] 133 | 134 | # use parent description if attribute description is blank (mainly for @iso-date) 135 | description = '' 136 | if a_description == '': 137 | description = self.schema_documentation(element, ref_element, type_element) 138 | else: 139 | description = a_description 140 | rows.append({ 141 | 'field': solr_name, 142 | "label": field_to_label(solr_name), 143 | 'type': 'select' if codelist_name != '' else xsd_type_to_search(solr_name, xsd_type=a_type), 144 | 'description': textwrap.dedent(description), 145 | 'codelist_name': codelist_name, 146 | 'codelist_condition': codelist_condition, 147 | 'attribute_name': a_name, 148 | 'path': full_path, 149 | 'xsd_type': a_type, 150 | 'solr_required': 'true' if required and a_required else 'false', 151 | 'solr_multivalued': 'true' if multivalued else 'false' 152 | }) 153 | 154 | for child_name, child_element, child_ref_element, child_type_element, minOccurs, maxOccurs in self.element_loop(element, path): 155 | rows += self.output_solr(child_name, path + element.attrib['name'] + '/', child_element, minOccurs=minOccurs, maxOccurs=maxOccurs, ref_element=child_ref_element, type_element=child_type_element, parent_req=required, parent_multi=multivalued) 156 | 157 | if output: 158 | out = '' 159 | if out_type == 'filter': 160 | out = list(filter(filter_columns, rows)) 161 | with open(filename, 'w') as fp: 162 | json.dump(out, fp, indent=2) 163 | codelists = {} 164 | for row in out: 165 | if 'codelist_name' in row and row['codelist_name'] != '': 166 | name = row['codelist_name'] 167 | codelists[name] = (get_codelist_json(name)) 168 | with open(codelist_dest, 'w') as fp: 169 | json.dump(codelists, fp, indent=2) 170 | return rows 171 | 172 | 173 | if __name__ == '__main__': 174 | filter_dest = sys.argv[1] 175 | codelist_dest = sys.argv[2] 176 | 177 | activities = Schema2Solr('iati-activities-schema.xsd', lang='en') 178 | activities.output_solr( 179 | 'iati-activities', 'activity-standard/', minOccurs='1', maxOccurs='1', output=True, 180 | filename=filter_dest, codelist_dest=codelist_dest, out_type='filter' 181 | ) 182 | -------------------------------------------------------------------------------- /gen_html.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -o nounset 3 | 4 | cd docs/en || exit 1 5 | mkdir _static 6 | cd _static || exit 1 7 | ln -s ../../_static/* . 8 | cd .. || exit 1 9 | make dirhtml 10 | cd ../../ || exit 1 11 | cd docs/fr || exit 1 12 | mkdir _static 13 | cd _static || exit 1 14 | ln -s ../../_static/* . 15 | cd .. || exit 1 16 | make dirhtml 17 | 18 | cd ../.. || exit 1 19 | mkdir -p docs/en/_build/dirhtml/schema/downloads/ 20 | cp -r IATI-Schemas/* docs/en/_build/dirhtml/schema/downloads/ 21 | cp -r IATI-Codelists/out/ docs/en/_build/dirhtml/codelists/downloads/ 22 | mkdir -p docs/fr/_build/dirhtml/schema/downloads/ 23 | cp -r IATI-Schemas/* docs/fr/_build/dirhtml/schema/downloads/ 24 | cp -r IATI-Codelists/out/ docs/fr/_build/dirhtml/codelists/downloads/ 25 | 26 | -------------------------------------------------------------------------------- /gen_rst.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -o nounset 3 | 4 | # Remove docs (the output directory), and recreate 5 | rm -r docs/* 6 | 7 | # Generate csvs etc. from codelists 8 | cd IATI-Codelists || exit 1 9 | ./gen.sh || exit 1 10 | cd .. || exit 1 11 | 12 | 13 | # Generate documentation from the Schema and Codelists etc 14 | python gen.py || exit 1 15 | 16 | # Copy rulesets SPEC 17 | cp IATI-Rulesets/SPEC.rst docs/en/rulesets/ruleset-spec.rst 18 | 19 | -------------------------------------------------------------------------------- /gen_solr.py: -------------------------------------------------------------------------------- 1 | import re 2 | import sys 3 | from pathlib import Path 4 | from gen import Schema2Doc 5 | 6 | # Define the namespaces necessary for opening schema files 7 | namespaces = { 8 | 'xsd': 'http://www.w3.org/2001/XMLSchema' 9 | } 10 | 11 | 12 | def path_to_solr(path): 13 | final = path 14 | if 'iati-activities/iati-activity/@' in path: 15 | final = path.replace('iati-activities/iati-activity/@', '') 16 | elif 'iati-activities/iati-activity/' in path: 17 | final = path.replace('iati-activities/iati-activity/', '') 18 | elif 'iati-activities' in path: 19 | final = path.replace('iati-activities', 'dataset') 20 | return final.replace('/@', '_').replace('/', '_').replace('-', '_').replace(':', '_') 21 | 22 | 23 | def xsd_type_to_solr(element_name=None, xsd_type=None): 24 | if (element_name is not None and re.search('_narrative$', element_name) is not None): 25 | return "text_general" 26 | 27 | if (element_name == 'location_administrative_level'): 28 | return "text_gen_sort" 29 | 30 | switch = { 31 | 'xsd:string': 'text_gen_sort', 32 | 'xsd:NMTOKEN': 'text_gen_sort', 33 | 'xsd:anyURI': 'text_general', 34 | 'xsd:decimal': 'pdoubles', 35 | 'xsd:dateTime': 'pdate', 36 | 'xsd:date': 'pdate', 37 | 'xsd:boolean': 'boolean', 38 | 'xsd:nonNegativeInteger': 'pint', 39 | 'xsd:positiveInteger': 'pint', 40 | 'xsd:int': 'pint' 41 | } 42 | return switch.get(xsd_type, "text_gen_sort") 43 | 44 | 45 | class Schema2Solr(Schema2Doc): 46 | 47 | def output_solr(self, element_name, path, element=None, output=False, template_path='', filename='', collection='', out_type='order', minOccurs='', maxOccurs='', ref_element=None, type_element=None, parent_req=True, parent_multi=False): 48 | if element is None: 49 | element = self.get_schema_element('element', element_name) 50 | if element is None: 51 | return 52 | 53 | full_path = '/'.join(path.split('/')[1:]) + element_name 54 | solr_name = path_to_solr(full_path) 55 | xsd_type = element.get('type') if element.get('type') and element.get('type').startswith('xsd:') else '' 56 | if type_element is not None: 57 | complex_base_types = [x for x in type_element.xpath('xsd:simpleContent/xsd:extension/@base', namespaces=namespaces) if x.startswith('xsd:')] 58 | if len(complex_base_types) and xsd_type == '': 59 | xsd_type = complex_base_types[0] 60 | required = (minOccurs == '1') and parent_req 61 | if element_name == 'iati-activity': 62 | maxOccurs = '1' 63 | multivalued = (maxOccurs == 'unbounded') or parent_multi 64 | 65 | rows = [] 66 | # elements should only be in solr if they contain something with a type, otherwise they wouldn't have a flattened value 67 | if element.xpath('xsd:complexType[@mixed="true"] or xsd:complexType/xsd:simpleContent', namespaces=namespaces) or xsd_type != '': 68 | rows = [{ 69 | "name": element_name, 70 | 'path': full_path, 71 | "solr_field_name": solr_name, 72 | 'type': xsd_type, 73 | 'solr_type': xsd_type_to_solr(solr_name, xsd_type), 74 | 'required': required, 75 | 'solr_required': 'true' if required else 'false', 76 | 'solr_multivalued': 'true' if multivalued else 'false' 77 | }] 78 | 79 | for a_name, a_type, a_description, a_required in self.attribute_loop(element): 80 | full_path = '/'.join(path.split('/')[1:]) + element_name + '/@' + a_name 81 | solr_name = path_to_solr(full_path) 82 | 83 | rows.append({ 84 | 'attribute_name': a_name, 85 | 'path': full_path, 86 | 'solr_field_name': solr_name, 87 | 'type': a_type, 88 | 'solr_type': xsd_type_to_solr(solr_name, xsd_type=a_type), 89 | 'solr_required': 'true' if required and a_required else 'false', 90 | 'solr_multivalued': 'true' if multivalued else 'false' 91 | }) 92 | 93 | for child_name, child_element, child_ref_element, child_type_element, minOccurs, maxOccurs in self.element_loop(element, path): 94 | rows += self.output_solr(child_name, path + element.attrib['name'] + '/', child_element, minOccurs=minOccurs, maxOccurs=maxOccurs, ref_element=child_ref_element, type_element=child_type_element, parent_req=required, parent_multi=multivalued) 95 | 96 | if output: 97 | template = Path(template_path).read_text() 98 | out = '' 99 | if out_type == 'order': 100 | full_order_out = '' 101 | partial_order_out = '' 102 | stop = len(rows) 103 | for i, row in enumerate(rows): 104 | if row['solr_field_name'] in ['dataset', 'dataset_iati_activity']: 105 | continue 106 | full_order_out += row['solr_field_name'] 107 | if i < stop - 1: 108 | full_order_out += ',' 109 | if collection == 'transaction' and row['solr_field_name'].startswith('budget_') and row['solr_field_name'] != 'budget_not_provided': 110 | continue 111 | if collection == 'budget' and row['solr_field_name'].startswith('transaction_'): 112 | continue 113 | partial_order_out += row['solr_field_name'] 114 | if i < stop - 1: 115 | partial_order_out += ',' 116 | full_order_out += '' 117 | partial_order_out += '' 118 | out = template.replace("#SEARCHDEFAULTS#", full_order_out).replace("#PARTIALSEARCHDEFAULTS#", partial_order_out) 119 | if out_type == 'schema': 120 | schema_rows = '' 121 | for row in rows: 122 | if row['solr_field_name'] in ['dataset', 'dataset_iati_activity']: 123 | continue 124 | 125 | line = ' tmp; mv tmp $f; done 9 | #for f in *.rst; do sed -e '/./{H;$!d;}' -e 'x;/Page for revision/d' $f > tmp; mv tmp $f; done 10 | 11 | 12 | -------------------------------------------------------------------------------- /last_updated_gen.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ./gen_rst.sh || exit $? 3 | cd docs || exit 1 4 | git add . 5 | git commit -a -m 'Auto' 6 | git ls-tree -r --name-only HEAD | grep 'rst$' | while read filename; do 7 | echo $'\n\n\n'"*Last updated on $(git log -1 --format="%ad" --date=short -- $filename)*" >> $filename 8 | done 9 | cd .. || exit 1 10 | ./gen_html.sh || exit $? 11 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pip==25.1.1 2 | Sphinx==7.4.7 3 | lxml==5.4.0 4 | flake8==7.2.0 5 | Jinja2==3.1.6 6 | pytz==2025.2 7 | python-dateutil==2.9.0.post0 8 | # These are requirements above, added here to pin versions 9 | docutils==0.21.2 10 | Pygments==2.19.1 11 | pytest==8.3.5 -------------------------------------------------------------------------------- /templates/en/codelist.rst: -------------------------------------------------------------------------------- 1 | {{codelist_json.metadata.name}} 2 | {{'='*len(codelist_json.metadata.name)}} 3 | 4 | {% if codelist_json.metadata.description %} 5 | {{dedent(codelist_json.metadata.description)}} 6 | {% endif %} 7 | 8 | {% if codelist_json.metadata.url %} 9 | External URL: {{codelist_json.metadata.url}} 10 | {% endif %} 11 | 12 | {% if codelist_json.metadata.category=="Core" %} 13 | This is a :ref:`Core codelist `. 14 | {% else %} 15 | This is a :ref:`{{codelist_json.metadata.category}} codelist `. 16 | {% endif %} 17 | 18 | {% if codelist_paths %} 19 | Use this codelist for 20 | --------------------- 21 | {% for path in codelist_paths %} 22 | * :ref:`{{path}} <{{path_to_ref(path)}}>` 23 | {% endfor %} 24 | {% endif %} 25 | 26 | Download this codelist 27 | ---------------------- 28 | 29 | .. list-table:: 30 | :header-rows: 1 31 | 32 | * - :ref:`CLv1 `: 33 | - :ref:`CLv2 `: 34 | - :ref:`CLv3 `: 35 | - :ref:`CLv3 (french) `: 36 | 37 | * - `CSV <../downloads/clv1/codelist/{{fname}}.csv>`__ 38 | - `CSV <../downloads/clv2/csv/{{lang}}/{{fname}}.csv>`__ 39 | - `CSV <../downloads/clv3/csv/{{lang}}/{{fname}}.csv>`__ 40 | - `CSV <../downloads/clv3/csv/fr/{{fname}}.csv>`__ 41 | 42 | * - `JSON <../downloads/clv1/codelist/{{fname}}.json>`__ 43 | - `JSON <../downloads/clv2/json/{{lang}}/{{fname}}.json>`__ 44 | - `JSON <../downloads/clv3/json/{{lang}}/{{fname}}.json>`__ 45 | - `JSON <../downloads/clv3/json/fr/{{fname}}.json>`__ 46 | 47 | * - `XML <../downloads/clv1/codelist/{{fname}}.xml>`__ 48 | - `XML <../downloads/clv2/xml/{{fname}}.xml>`__ 49 | - `XML <../downloads/clv3/xml/{{fname}}.xml>`__ 50 | - `XML <../downloads/clv3/xml/{{fname}}.xml>`__ 51 | 52 | `GitHub Source <{{github_url}}>`__ 53 | 54 | {% if show_withdrawn and embedded==False %} 55 | 56 | This codelist has some withdrawn elements, for details on these check the `Non-Core Codelist changelog record `__ 57 | {% endif %} 58 | 59 | The codelists were translated in French in April 2018 with the support of the Government of Canada. Please note that if any codelists have been added since then, they may not be available in French. 60 | 61 | Codes 62 | ----- 63 | 64 | .. _{{fname}}: 65 | .. list-table:: 66 | :header-rows: 1 67 | 68 | 69 | * - Code 70 | - Name 71 | - Description{% if show_category_column %} 72 | - Category{% endif %}{% if show_url_column %} 73 | - URL{% endif %}{% if fname == 'OrganisationRegistrationAgency' %} 74 | - Public Database?{% endif %} 75 | 76 | {% for codelist_item in codelist_json.data %} 77 | {% if codelist_item.status == 'withdrawn' %} 78 | .. rst-class:: withdrawn 79 | * - {{codelist_item.code + " (withdrawn)"}} 80 | {% else %} 81 | * - {{codelist_item.code}} 82 | {% endif %} 83 | - {{codelist_item.name}} 84 | - {% if codelist_item.description %}{{codelist_item.description}}{% endif %}{% if show_category_column %} 85 | - {% if codelist_item.category %}{% if codelist_json.attributes['category-codelist'] %}:ref:`{{codelist_item.category}} <{{codelist_json.attributes['category-codelist']}}>`{%else%}{{codelist_item.category}}{%endif%}{% endif %}{% endif %}{% if show_url_column %} 86 | - {% if codelist_item.url %}{{codelist_item.url}}{% endif %}{% if fname == 'OrganisationRegistrationAgency' %} 87 | - {{codelist_item['public-database']}}{% endif %}{% endif %} 88 | {% endfor %} 89 | 90 | {{extra_docs}} 91 | -------------------------------------------------------------------------------- /templates/en/overview.rst: -------------------------------------------------------------------------------- 1 | {{extra_docs}} 2 | 3 | Reference pages 4 | --------------- 5 | 6 | {% for name, path in reference_pages %} 7 | * :doc:`{{name}} <{{path}}>` 8 | {% endfor %} 9 | -------------------------------------------------------------------------------- /templates/en/ruleset.rst: -------------------------------------------------------------------------------- 1 | {{extra_docs}} 2 | 3 | {% for xpath, rule_texts in ruleset.items() %} 4 | 5 | {{xpath}} 6 | {{'-'*(xpath|count)}} 7 | 8 | {% for rule_text in rule_texts %} 9 | * `{{ rule_text[0] }} <{{ rule_text[2] }}>`_: {{ rule_text[1] }} 10 | {% endfor %} 11 | 12 | {% endfor %} 13 | -------------------------------------------------------------------------------- /templates/en/schema_element.rst: -------------------------------------------------------------------------------- 1 | {{element_name}} 2 | {{element_name_underline}} 3 | 4 | ``{{path}}{{element_name}}`` 5 | 6 | This is the reference page for the XML element ``{{element_name}}``. {% if see_also %}See also the relevant overview page{% if see_also|count > 1%}s{%endif%}: {% for page in see_also %}:doc:`{{page}}`{% if not loop.last %}, {% endif %} {% endfor %}{% endif %} 7 | 8 | .. index:: 9 | single: {{element_name}} 10 | 11 | Definition 12 | ~~~~~~~~~~ 13 | 14 | {{schema_documentation}} 15 | 16 | Rules 17 | ~~~~~ 18 | {% for extended_type in extended_types %} 19 | {% if extended_type.startswith('xsd:') %}The text in this element must be of type {{extended_type}}.{% endif %} 20 | {% endfor %} 21 | 22 | {% if element.get('type') and element.get('type').startswith('xsd:') %}The text in this element must be of type {{element.get('type')}}. 23 | {% endif %} 24 | 25 | {% if min_occurs > 0 %} 26 | The schema states that this element must have at least {{min_occurs}} subelement{% if min_occurs > 1 %}s{% endif %}. 27 | {% endif %} 28 | 29 | {% if minOccurs and maxOccurs %} 30 | {% if minOccurs=='1' and maxOccurs=='1' %} 31 | This element must occur once and only once (within each parent element). 32 | {% elif minOccurs=='0' and maxOccurs=='1' %} 33 | This element must occur no more than once (within each parent element). 34 | {% elif minOccurs=='0' and maxOccurs=='unbounded' %} 35 | This element may occur any number of times. 36 | {% elif minOccurs=='1' and maxOccurs=='unbounded' %} 37 | This element must occur at least once (within each parent element). 38 | {% else %} 39 | This element must occur {{minOccurs}} to {{maxOccurs}} times. 40 | {% endif %} 41 | {% endif %} 42 | 43 | {% set rtexts = ruleset_text(path+element_name) %} 44 | {% if rtexts %} 45 | 46 | {% for rtext in rtexts %} 47 | `{{ rtext[0] }} <{{ rtext[2 ]}}>`_: {{ rtext[1] }}{{ '\n' }} 48 | {% endfor %} 49 | 50 | {%endif%} 51 | 52 | {% if attributes %} 53 | Attributes 54 | ~~~~~~~~~~ 55 | 56 | {% for attribute, attribute_type, text, required in attributes %} 57 | .. _{{path_to_ref(path+element_name+'/@'+attribute)}}: 58 | 59 | @{{attribute}} 60 | {{ textwrap.dedent(text).strip().replace('\n','\n ') }} 61 | {% if required %} 62 | This attribute is required. 63 | 64 | {% endif %}{% set codelist_tuples = match_codelists(path+element_name+'/@'+attribute) %}{% if attribute_type %} 65 | 66 | This value must be of type {{attribute_type}}. 67 | 68 | {% endif %}{% for codelist_tuple in codelist_tuples %} 69 | This value {% if codelist_tuple[0]|is_complete_codelist() %}must{% else %}should{% endif %} be on the :doc:`{{codelist_tuple[0]}} codelist `{% if codelist_tuple[1] %}, if the relevant vocabulary is used{% endif %}. 70 | 71 | {% endfor %} 72 | 73 | {% set rtexts_attrib = ruleset_text(path+element_name+'/@'+attribute) %} 74 | {% if rtexts_attrib %} 75 | 76 | {% for rtext_attrib in rtexts_attrib %} 77 | `{{ rtext_attrib[0] }} <{{ rtext_attrib[2] }}>`_: {{ rtext_attrib[1] }}{{ '\n' }} 78 | {% endfor %} 79 | 80 | {% endif %} 81 | 82 | 83 | {% endfor %} 84 | 85 | {% endif %} 86 | 87 | {{extra_docs}} 88 | 89 | Developer tools 90 | ~~~~~~~~~~~~~~~ 91 | 92 | Find the source of this documentation on github: 93 | 94 | * `Schema <{{github_urls.schema}}>`_ 95 | * `Extra Documentation <{{github_urls.extra_documentation}}>`_ 96 | 97 | {% if childnames %} 98 | Subelements 99 | ~~~~~~~~~~~ 100 | 101 | .. toctree:: 102 | :titlesonly: 103 | :maxdepth: 1 104 | 105 | {% for childname in childnames %} {{element_name}}/{{childname}} 106 | {%endfor%} 107 | {% endif %} 108 | -------------------------------------------------------------------------------- /templates/en/schema_table.rst: -------------------------------------------------------------------------------- 1 | {{title}} 2 | {{'='*(title|count)}} 3 | 4 | .. list-table:: 5 | :header-rows: 1 6 | 7 | * - Section 8 | - Item 9 | - Description 10 | - Type 11 | - Codelist 12 | - XML 13 | - Occur 14 | - Rules 15 | 16 | {% for row in rows %} 17 | * - {%if row.section%}:doc:`{{row.name}} <{{row.doc}}>`{%endif%} 18 | - {%if not row.section%}{%if row.name%}:doc:`{{row.name}} <{{row.doc}}>`{%else%}{{row.attribute_name}}{%endif%}{%endif%} 19 | - {{row.description.replace('\n', '\n ').strip(' \n')}} 20 | - {% if row.type %}{{row.type}}{% endif %} 21 | - {% set codelist_tuples = match_codelists(root_path+row.path) %}{% for codelist_tuple in codelist_tuples %}{%if codelist_tuple[1]%}({%endif%}:doc:`/codelists/{{codelist_tuple[0]}}`{%if codelist_tuple[1]%}){%endif%}{% endfor %} 22 | - {{row.path.replace('@','\@')}} 23 | - {{row.occur}} 24 | - {% set rtexts = ruleset_text(row.path) %}{% for rtext in rtexts %}`{{ rtext[0] }} <{{ rtext[2] }}>`_: {{ rtext[1] }}{{ ' |br| \n ' }}{% endfor %} 25 | {% endfor %} 26 | 27 | :: 28 | 29 | {{description.replace('\n','\n ')}} 30 | 31 | {{extra_docs}} 32 | 33 | .. |br| raw:: html 34 | 35 |
-------------------------------------------------------------------------------- /templates/fr: -------------------------------------------------------------------------------- 1 | en -------------------------------------------------------------------------------- /tests/sample_output_test.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | 4 | class TestSampleOutput: 5 | @pytest.mark.parametrize( 6 | ('file_path', 'expected_snippet'), 7 | [ 8 | ("docs/en/_build/dirhtml/index.html", "IATI Standard 2.03 documentation"), 9 | ] 10 | ) 11 | def test_sample_output(self, file_path, expected_snippet): 12 | with open(file_path, "r") as open_file: 13 | file_contents = open_file.read() 14 | assert expected_snippet in file_contents 15 | -------------------------------------------------------------------------------- /update_submodules.sh: -------------------------------------------------------------------------------- 1 | # Script to update each of the 4 submodules in the IATI-SSOT repository 2 | # 3 | 4 | timestamp=$(date +%s) 5 | 6 | for version in 2.01 2.02 2.03; do 7 | # fetch the latest version from the remote 8 | git fetch origin version-$version 9 | 10 | # Checkout to the specified version for the SSOT directory 11 | git checkout --force origin/version-$version 12 | 13 | # Discard local changes to submodules 14 | # See: https://stackoverflow.com/a/27415757/2323348 15 | git submodule deinit -f . 16 | git submodule update --init 17 | 18 | # Check out a new branch to get around branch protection 19 | git checkout -b update-submodules-$timestamp-$version 20 | 21 | # Pull the latest versions of submodules 22 | git submodule update --remote 23 | 24 | # Git add submodules 25 | git add IATI-Codelists IATI-Extra-Documentation IATI-Schemas IATI-Rulesets 26 | 27 | # Commit updated submodules 28 | git commit -m "Updated submodules (using script) "$version 29 | 30 | # Push to the server 31 | git push origin update-submodules-$timestamp-$version 32 | done 33 | --------------------------------------------------------------------------------