├── .gitignore ├── CONTRIBUTING.md ├── LICENSE ├── MANIFEST.in ├── README.md ├── SECURITY.md ├── bin ├── preview.py └── push_pypi.sh ├── changelog.txt ├── code_of_conduct.md ├── docs ├── assets │ ├── favicon.png │ ├── logo.graffle │ └── logo.png ├── biblio.jinja ├── glossary.jinja ├── index.md ├── javascripts │ └── config.js ├── mkrefs.ttl ├── mkrefs.yml ├── ref.jinja └── stylesheets │ └── extra.css ├── mkdocs.yml ├── mkrefs ├── __init__.py ├── apidocs.py ├── biblio.py ├── cli.py ├── glossary.py ├── plugin.py ├── util.py └── version.py ├── pylintrc ├── requirements-dev.txt ├── requirements.txt └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | docs/biblio.md 2 | docs/glossary.md 3 | docs/ref.md 4 | *~ 5 | 6 | # Byte-compiled / optimized / DLL files 7 | __pycache__/ 8 | *.py[cod] 9 | *$py.class 10 | 11 | # C extensions 12 | *.so 13 | 14 | # Distribution / packaging 15 | .Python 16 | build/ 17 | develop-eggs/ 18 | dist/ 19 | downloads/ 20 | eggs/ 21 | .eggs/ 22 | lib/ 23 | lib64/ 24 | parts/ 25 | sdist/ 26 | var/ 27 | wheels/ 28 | pip-wheel-metadata/ 29 | share/python-wheels/ 30 | *.egg-info/ 31 | .installed.cfg 32 | *.egg 33 | MANIFEST 34 | 35 | # PyInstaller 36 | # Usually these files are written by a python script from a template 37 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 38 | *.manifest 39 | *.spec 40 | 41 | # Installer logs 42 | pip-log.txt 43 | pip-delete-this-directory.txt 44 | 45 | # Unit test / coverage reports 46 | htmlcov/ 47 | .tox/ 48 | .nox/ 49 | .coverage 50 | .coverage.* 51 | .cache 52 | nosetests.xml 53 | coverage.xml 54 | *.cover 55 | *.py,cover 56 | .hypothesis/ 57 | .pytest_cache/ 58 | 59 | # Translations 60 | *.mo 61 | *.pot 62 | 63 | # Django stuff: 64 | *.log 65 | local_settings.py 66 | db.sqlite3 67 | db.sqlite3-journal 68 | 69 | # Flask stuff: 70 | instance/ 71 | .webassets-cache 72 | 73 | # Scrapy stuff: 74 | .scrapy 75 | 76 | # Sphinx documentation 77 | docs/_build/ 78 | 79 | # PyBuilder 80 | target/ 81 | 82 | # Jupyter Notebook 83 | .ipynb_checkpoints 84 | 85 | # IPython 86 | profile_default/ 87 | ipython_config.py 88 | 89 | # pyenv 90 | .python-version 91 | 92 | # pipenv 93 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 94 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 95 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 96 | # install all needed dependencies. 97 | #Pipfile.lock 98 | 99 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 100 | __pypackages__/ 101 | 102 | # Celery stuff 103 | celerybeat-schedule 104 | celerybeat.pid 105 | 106 | # SageMath parsed files 107 | *.sage.py 108 | 109 | # Environments 110 | .env 111 | .venv 112 | env/ 113 | venv/ 114 | ENV/ 115 | env.bak/ 116 | venv.bak/ 117 | 118 | # Spyder project settings 119 | .spyderproject 120 | .spyproject 121 | 122 | # Rope project settings 123 | .ropeproject 124 | 125 | # mkdocs documentation 126 | /site 127 | 128 | # mypy 129 | .mypy_cache/ 130 | .dmypy.json 131 | dmypy.json 132 | 133 | # Pyre type checker 134 | .pyre/ 135 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | # Welcome! 5 | 6 | Thanks for your interest in contributing to **MkRefs** 🎉 7 | 8 | This page gives a quick overview of how things are organized and most 9 | importantly, how to get involved. 10 | 11 | 12 | ## Issues and bug reports 13 | 14 | First, if you want to report a potential issue with this library, please 15 | [do a quick search](https://github.com/DerwenAI/mkrefs/issues) 16 | to see if the issue has already been reported. 17 | If so, it's best to simply leave a comment on an existing issue, 18 | rather than create a new one. 19 | Older issues may also include helpful info and show solutions to 20 | commonly encountered questions. 21 | 22 | 23 | ## Opening new issues 24 | 25 | When opening a 26 | [new issue](https://github.com/DerwenAI/mkrefs/issues/new/choose), 27 | please use a **descriptive title** and include information about your 28 | **environment** and library **installation**: 29 | 30 | * Which operating system and version number? 31 | * Which version of Python? 32 | * How did you install? `pip`, `conda`, clone repo then `setup.py`, etc. 33 | 34 | Try to provide as many details as possible. 35 | What exactly is going wrong? 36 | _How_ is it failing? 37 | Is there an error? 38 | 39 | Please understand that in general our developer community does not 40 | provide support via email, Twitter DMs, and other 1:1 messaging. 41 | We believe that help is much more valuable when it gets **shared 42 | publicly**, so that more people can benefit. 43 | 44 | 45 | ## Code of conduct 46 | 47 | In all communications and collaborations, we adhere to the 48 | [Contributor Covenant Code of Conduct](https://github.com/DerwenAI/mkrefs/blob/main/code_of_conduct.md). 49 | By participating, you are expected to follow this code. 50 | 51 | 52 | ## Developer community 53 | 54 | If you'd like to contribute to this open source project, the best way 55 | to get involved with our developer community is to participate in our 56 | [public office hours](https://www.notion.so/KG-Community-Events-Calendar-8aacbe22efa94d9b8b39b7288e22c2d3) 57 | events. 58 | First join the 59 | [*Graph-Based Data Science*](https://www.linkedin.com/groups/6725785/) 60 | group on LinkedIn where these meetingsget announced. 61 | We'll also have other developer discussions on that forum, along with 62 | related updates, news, conference coupons, etc. 63 | 64 | The 65 | [Knowledge Graph Conference](https://derwen.ai/docs/kgl/glossary/#knowledge-graph-conference) 66 | hosts several community resources where you can post questions and get 67 | help about **MkRefs** and related topics. 68 | Many of our developers are involved there too: 69 | 70 | * [community Slack](https://knowledgegraphconf.slack.com/ssb/redirect) – specifically on the `#ask` channel 71 | 72 | * [Knowledge Tech Q&A site](https://answers.knowledgegraph.tech/) for extended questions posed to experts 73 | 74 | 75 | ## Contributing to the code base 76 | 77 | You don't have to be an expert to contribute, and we're happy to help 78 | you get started. 79 | We'll try to use the 80 | [`good first issue`](https://github.com/DerwenAI/mkrefs/labels/good%20first%20issue) 81 | tags to mark bugs and feature requests that are easy and self-contained. 82 | 83 | If you've decided to take on one of these problems, it's best to 84 | [fork the repo](https://docs.github.com/en/github/collaborating-with-issues-and-pull-requests/about-forks) 85 | and do development and testing in your own fork first. 86 | 87 | Please follow the conventions for code formatting, type annotations, 88 | unit tests, code linting, naming conventions, and so on. 89 | Understand that we will not be able to accept pull requests that make 90 | *major overhauls* of the code base or completely change our shared 91 | work on formatting, testing, etc. 92 | 93 | If you need to incorporate other libraries, please discuss this with 94 | the other developers. 95 | There may be issues regarding point releases and compatibility that 96 | would have impact on other parts of the code base. 97 | 98 | Once you're making good progress, don't forget to add a quick comment 99 | to the original issue. 100 | You can also use the issue to ask questions, or share your work in 101 | progress. 102 | Then when you're ready to submit code for review, please use a 103 | [pull request](https://docs.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request) 104 | on our `main` repo branch. 105 | 106 | 107 | ## Project roadmap 108 | 109 | The 110 | ["Graph-Based Data Science"](https://derwen.ai/s/kcgh) 111 | talk describes the **MkRefs** open source project in much more detail, 112 | and discusses some about our roadmap. 113 | In other words, what new features and integrations are we working toward? 114 | 115 | See also our: 116 | 117 | * [Project Board](https://github.com/DerwenAI/mkrefs/projects/1) 118 | * [Milestones](https://github.com/DerwenAI/mkrefs/milestones) 119 | 120 | Suggestions and contributions for our documentation and tutorial are 121 | always welcomed. 122 | These tend to be good starting points for new contributors: you'll get 123 | familiar with our code samples and other resources through that. 124 | 125 | Many thanks! 126 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Derwen, Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.md 2 | include changelog.txt 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # MkRefs 2 | 3 | The **MkRefs** [plugin](http://www.mkdocs.org/user-guide/plugins/) 4 | for [`MkDocs`](https://www.mkdocs.org/) 5 | generates reference Markdown pages from a knowledge graph, 6 | based on the [`kglab`](https://github.com/DerwenAI/kglab) project. 7 | 8 | No graph database is required; however, let us know if you'd like to 9 | use one in particular. 10 | 11 | There are several planned use cases for the **MkRefs** plugin, 12 | including: 13 | 14 | * *biblio* – semantic bibliography entries, generated from RDF 15 | * *glossary* – semantic glossary entries, generated from RDF 16 | * *apidocs* – semantic [*apidocs*](https://pypi.org/search/?q=apidocs) supporting the [Diátaxis](https://derwen.ai/docs/kgl/learn/#a-grammar-of-learning) grammar for documentation, generated as RDF from Python source code 17 | * *depend* – semantic dependency graph for Python libraries, generated as RDF from `setup.py` 18 | * *index* – semantic search index, generated as RDF from MkDocs content 19 | 20 | Only the *apidocs*, *biblio*, and *glossary* components have been 21 | added to **MkRefs** so far, although the other mentioned components 22 | exist in separate projects and are being integrated. 23 | 24 | 25 |
26 | Contributing Code 27 | 28 | We welcome people getting involved as contributors to this open source 29 | project! 30 | 31 | For detailed instructions please see: 32 | [CONTRIBUTING.md](https://github.com/DerwenAI/mkrefs/blob/main/CONTRIBUTING.md) 33 |
34 | 35 |
36 | Semantic Versioning 37 | 38 | Before MkRefs reaches release v1.0.0 the 39 | types and classes may undergo substantial changes and the project is 40 | not guaranteed to have a consistent API. 41 | 42 | Even so, we'll try to minimize breaking changes. 43 | We'll also be sure to provide careful notes. 44 | 45 | See: 46 | [changelog.txt](https://github.com/DerwenAI/mkrefs/blob/main/changelog.txt) 47 |
48 | 49 | MkRefs, for semantic references 54 | 55 | 56 | ## Why does this matter? 57 | 58 | A key takeaway is that many software engineering aspects of open 59 | source projects involve graphs, therefore a knowledge graph can 60 | provide an integral part of an open source repository. 61 | Moreover, by using semantic representation (RDF) projects that 62 | integrate with each other can share (i.e., federate) common resources, 63 | for example to share definitions, analyze mutual dependencies, etc. 64 | 65 | 66 | ## Installation 67 | 68 | To install the plugin using `pip`: 69 | 70 | ``` 71 | python3 -m pip install mkrefs 72 | ``` 73 | 74 | Then add the plugin into the `mkdocs.yml` file: 75 | ```yaml 76 | plugins: 77 | - mkrefs 78 | ``` 79 | In addition, the following configuration parameter is expected: 80 | 81 | * `mkrefs_config` - YAML configuration file for **MkRefs**; e.g., `mkrefs.yml` 82 | 83 | --- 84 | 85 | ## API Docs 86 | 87 | An `apidocs` parameter within the configuration file expects four 88 | required sub-parameters: 89 | 90 | * `page` – name of the generated Markdown page, e.g., `ref.md` 91 | * `template` – a [Jinja2 template](https://jinja.palletsprojects.com/en/3.0.x/) to generate Markdown, e.g., `ref.jinja` 92 | * `package` – name of the package being documented 93 | * `git` – base URL for source modules in Git, e.g., `https://github.com/DerwenAI/mkrefs/blob/main` 94 | 95 | There is an optional `includes` parameter, as a list of class 96 | definitions to include. 97 | If this is used, then all other classes get ignored. 98 | 99 | See the source code in this repo for examples of how to format 100 | Markdown within *docstrings*. 101 | Specifically see the parameter documentation per method or function, 102 | which differs slightly from pre-exisiting frameworks. 103 | 104 | Note that the name of the generated Markdown page for the 105 | apidocs must appear in the `nav` section of your `mkdocs.yml` 106 | configuration file. 107 | See the structure used in this repo for an example. 108 | 109 | ### Best Practices: RDF representation 110 | 111 | You can use this library outside of MkDocs, i.e., calling it 112 | programmatically, to generate an RDF graph to represent your package 113 | API reference: 114 | 115 | ``` 116 | package_name = "mkrefs" 117 | git_url = "https://github.com/DerwenAI/mkrefs/blob/main" 118 | includes = [ "MkRefsPlugin", "PackageDoc" ] 119 | 120 | pkg_doc = PackageDoc(package_name, git_url, includes) 121 | pkg_doc.build() 122 | 123 | kg = pkg_doc.get_rdf() 124 | ``` 125 | 126 | The `PackageDoc.get_rdf()` method returns an RDF graph as an instance 127 | of an `kglab.KnowledgeGraph` object. 128 | For more details, see 129 | 130 | 131 | ## Bibliography 132 | 133 | A `biblio` parameter within the configuration file expects four 134 | required sub-parameters: 135 | 136 | * `graph` – an RDF graph represented as a Turtle (TTL) file, e.g., `mkrefs.ttl` 137 | * `page` – name of the generated Markdown page, e.g., `biblio.md` 138 | * `template` – a [Jinja2 template](https://jinja.palletsprojects.com/en/3.0.x/) to generate Markdown, e.g., `biblio.jinja` 139 | * `queries` – [SPARQL queries](https://rdflib.readthedocs.io/en/stable/intro_to_sparql.html) used to extract bibliography data from the knowledge graph 140 | 141 | See the [`mkrefs.ttl`](https://github.com/DerwenAI/mkrefs/blob/main/docs/mkrefs.ttl) 142 | file for an example bibliography represented in RDF. 143 | This comes from the documentation for the [`pytextrank`](https://derwen.ai/docs/ptr/biblio/) 144 | open source project. 145 | 146 | In the example RDF, the [*bibo*](http://bibliontology.com/) vocabulary represents 147 | bibliographic entries, and the [*FOAF*](http://xmlns.com/foaf/spec/) vocabulary 148 | represents authors. 149 | This also uses two custom OWL relations from the [*derwen*](https://derwen.ai/ns/v1) 150 | vocabulary: 151 | 152 | * `derw:citeKey` – citekey used to identify a bibliography entry within the documentation 153 | * `derw:openAccess` – open access URL for a bibliography entry (if any) 154 | 155 | The `queries` parameter has three required SPARQL queries: 156 | 157 | * `entry` – to select the identifiers for all of the bibliograpy entries 158 | * `entry_author` – a mapping to identify author links for each bibliography entry 159 | * `entry_publisher` - the publisher link for each bibliography entry (if any) 160 | 161 | Note that the name of the generated Markdown page for the 162 | bibliography must appear in the `nav` section of your `mkdocs.yml` 163 | configuration file. 164 | See the structure used in this repo for an example. 165 | 166 | You may use any valid RDF representation for a bibliography. 167 | Just be sure to change the three SPARQL queries and the Jinja2 168 | template accordingly. 169 | 170 | While this example uses an adaptation of the 171 | [MLA Citation Style](https://www.easybib.com/guides/citation-guides/mla-format/mla-citation/), 172 | feel free to modify the Jinja2 template to generate whatever 173 | bibliographic style you need. 174 | 175 | 176 | ### Best Practices: constructing bibliographies 177 | 178 | As much as possible, bibliography entries should use the conventions at 179 | 180 | for their [*citation keys*](https://bibdesk.sourceforge.io/manual/BibDeskHelp_2.html). 181 | 182 | Journal abbreviations should use 183 | [*ISO 4*](https://en.wikipedia.org/wiki/ISO_4) standards, 184 | for example from 185 | 186 | Links to online versions of cited works should use 187 | [DOI](https://www.doi.org/) 188 | for [*persistent identifiers*](https://www.crossref.org/education/metadata/persistent-identifiers/). 189 | 190 | When available, 191 | [*open access*](https://peerj.com/preprints/3119v1/) 192 | URLs should be listed as well. 193 | 194 | 195 | ### What is going on here? 196 | 197 | For example with the bibliography use case, when the plugin runs... 198 | 199 | 1. It parses its configuration file to identify the target Markdown page to generate and the Jinja2 template 200 | 2. The plugin also loads an RDF graph from the indicated TTL file 201 | 3. Three SPARQL queries are run to identify the unique entities to extract from the graph 202 | 4. The graph is serialized as [JSON-LD](https://derwen.ai/docs/kgl/ref/#kglab.KnowledgeGraph.save_jsonld) 203 | 5. The `author`, `publisher`, and bibliography `entry` entities are used to *denormalize* the graph into a JSON data object 204 | 6. The JSON is rendered using the Jinja2 template to generate the Markdown 205 | 7. The Markdown page is parsed and rendered by MkDocs as HTML, etc. 206 | 207 | 208 | ## Glossary 209 | 210 | A `glossary` parameter within the configuration file expects four 211 | required sub-parameters: 212 | 213 | * `graph` – an RDF graph represented as a Turtle (TTL) file, e.g., `mkrefs.ttl` 214 | * `page` – name of the generated Markdown page, e.g., `glossary.md` 215 | * `template` – a [Jinja2 template](https://jinja.palletsprojects.com/en/3.0.x/) to generate Markdown, e.g., `glossary.jinja` 216 | * `queries` – [SPARQL queries](https://rdflib.readthedocs.io/en/stable/intro_to_sparql.html) used to extract glossary data from the knowledge graph 217 | 218 | See the [`mkrefs.ttl`](https://github.com/DerwenAI/mkrefs/blob/main/docs/mkrefs.ttl) 219 | file for an example glossary represented in RDF. 220 | This example RDF comes from documentation for the 221 | [`pytextrank`](https://derwen.ai/docs/ptr/glossary/) 222 | open source project. 223 | 224 | In the example RDF, the [*cito*](http://purl.org/spar/cito/) 225 | vocabulary represents citations to locally represented bibliographic 226 | entries. 227 | The [*skos*](http://www.w3.org/2004/02/skos/core#) vocabulary 228 | provides support for [*taxonomy*](http://accidental-taxonomist.blogspot.com/) 229 | features, e.g., semantic relations among glossary entries. 230 | This example RDF also uses a definition from the 231 | [*derwen*](https://derwen.ai/ns/v1) vocabulary: 232 | 233 | * `derw:Topic` – a `skos:Concept` used to represent glossary entries 234 | 235 | The `queries` parameter has three required SPARQL queries: 236 | 237 | * `entry` – to select the identifiers for all of the bibliograpy entries 238 | * `entry_syn` – a mapping of synonyms (if any) 239 | * `entry_ref` – a mapping of external references (if any) 240 | * `entry_cite` – citations to the local bibliography citekeys (if any) 241 | * `entry_hyp` – a mapping of [*hypernyms*](https://en.wikipedia.org/wiki/Hyponymy_and_hypernymy) (if any) 242 | 243 | Note that the name of the generated Markdown page for the glossary 244 | must appear in the `nav` section of your `mkdocs.yml` configuration 245 | file. 246 | See the structure used in this repo for an example. 247 | 248 | You may use any valid RDF representation for a glossary. 249 | Just be sure to change the three SPARQL queries and the Jinja2 250 | template accordingly. 251 | 252 | 253 | ## Usage 254 | 255 | The standard way to generate documentation with MkDocs is: 256 | ``` 257 | mkdocs build 258 | ``` 259 | 260 | If you'd prefer to generate reference pages programmatically using 261 | Python scripts, see the code for usage of the `MkRefsPlugin` class, 262 | plus some utility functions: 263 | 264 | * `load_kg()` 265 | * `render_apidocs()` 266 | * `render_biblio()` 267 | * `render_glossary()` 268 | 269 | There are also command line *entry points* provided, which can be 270 | helpful during dev/test cycles on the semantic representation of your 271 | content: 272 | ``` 273 | mkrefs apidocs docs/mkrefs.yml 274 | mkrefs biblio docs/mkrefs.yml 275 | mkrefs glossary docs/mkrefs.yml 276 | ``` 277 | 278 | 279 | ## Caveats 280 | 281 | While the [`MkDocs`](https://www.mkdocs.org/) utility is astoundingly useful, 282 | its documentation (and coding style) leave much room for improvement. 283 | The [documentation for developing plugins](https://www.mkdocs.org/user-guide/plugins/#developing-plugins) 284 | is not even close to what happens when its code executes. 285 | 286 | Consequently, the **MkRefs** project is an attempt to reverse-engineer 287 | the code from many other MkDocs plugins, while documenting its observed 288 | event sequence, required parameters, limitations and workarounds, etc. 289 | 290 | Two issues persist, where you will see warnings even though the **MkRefs** 291 | code is handling configuration as recommended: 292 | 293 | ``` 294 | WARNING - Config value: 'mkrefs_config'. Warning: Unrecognised configuration name: mkrefs_config 295 | ``` 296 | 297 | and 298 | 299 | ``` 300 | INFO - The following pages exist in the docs directory, but are not included in the "nav" configuration: 301 | - biblio.md 302 | - glossary.md 303 | - ref.md 304 | ``` 305 | 306 | For now, you can simply ignore both of these warnings. 307 | Meanwhile, we'll work on eliminating them. 308 | 309 | 310 | ## Feature roadmap 311 | 312 | Let us know if you need features to parse and generate 313 | [BibTeX](http://www.bibtex.org/). 314 | 315 | 316 | ## License and Copyright 317 | 318 | Source code for **MkRefs** plus its logo, documentation, and examples 319 | have an [MIT license](https://spdx.org/licenses/MIT.html) which is 320 | succinct and simplifies use in commercial applications. 321 | 322 | All materials herein are Copyright © 2021 Derwen, Inc. 323 | 324 | 325 | ## Acknowledgements 326 | 327 | Many thanks to our open source [sponsors](https://github.com/sponsors/ceteri); 328 | and to our contributors: 329 | [@ceteri](https://github.com/ceteri) 330 | 331 | This plugin code is based on the marvelous examples in 332 | 333 | with kudos to [@byrnereese](https://github.com/byrnereese/), 334 | and also many thanks to 335 | [@louisguitton](https://github.com/louisguitton), 336 | [@dmccreary](https://github.com/dmccreary), 337 | and 338 | [@LarrySwanson](https://github.com/LarrySwanson) 339 | for their inspiration and insights. 340 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Supported Versions 4 | 5 | Versions which are currently being supported with security updates: 6 | 7 | | Version | Supported | 8 | | ------- | ------------------ | 9 | | > 0.1 | :white_check_mark: | 10 | 11 | ## Reporting a Vulnerability 12 | 13 | To report a vulnerability, please create a new [*issue*](https://github.com/DerwenAI/mkrefs/issues). 14 | We will be notified immediately, and will attempt to respond on the reported issue immediately. 15 | -------------------------------------------------------------------------------- /bin/preview.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | from flask import Flask, redirect, send_from_directory, url_for 5 | from pathlib import PurePosixPath 6 | import os 7 | import sys 8 | 9 | DOCS_ROUTE = "/docs/" 10 | DOCS_FILES = "../site" 11 | DOCS_PORT = 8000 12 | 13 | APP = Flask(__name__, static_folder=DOCS_FILES, template_folder=DOCS_FILES) 14 | 15 | APP.config["DEBUG"] = False 16 | APP.config["MAX_CONTENT_LENGTH"] = 52428800 17 | APP.config["SECRET_KEY"] = "Technically, I remain uncommitted." 18 | APP.config["SEND_FILE_MAX_AGE_DEFAULT"] = 3000 19 | 20 | 21 | @APP.route(DOCS_ROUTE, methods=["GET"]) 22 | @APP.route(DOCS_ROUTE + "", methods=["GET"], defaults={"path": None}) 23 | @APP.route(DOCS_ROUTE + "", methods=["GET"]) 24 | def static_proxy (path=""): 25 | if not path: 26 | suffix = "" 27 | else: 28 | suffix = PurePosixPath(path).suffix 29 | 30 | if suffix not in [".css", ".js", ".map", ".png", ".svg", ".xml", ".tff", ".woff", ".woff2"]: 31 | path = os.path.join(path, "index.html") 32 | 33 | return send_from_directory(DOCS_FILES, path) 34 | 35 | 36 | @APP.route("/index.html") 37 | @APP.route("/home/") 38 | @APP.route("/") 39 | def home_redirects (): 40 | return redirect(url_for("static_proxy")) 41 | 42 | 43 | if __name__ == "__main__": 44 | APP.run(host="0.0.0.0", port=DOCS_PORT, debug=True) 45 | -------------------------------------------------------------------------------- /bin/push_pypi.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | ## debugging the uploaded README: 4 | # pandoc README.md --from markdown --to rst -s -o README.rst 5 | 6 | rm -rf dist 7 | python setup.py sdist bdist_wheel 8 | twine upload --verbose dist/* 9 | -------------------------------------------------------------------------------- /changelog.txt: -------------------------------------------------------------------------------- 1 | # `MkRefs` changelog 2 | 3 | ## 0.3.0 4 | 5 | 2021-07-?? 6 | 7 | * migrated to `mkdocs-material` as part of development 8 | * fixed bug in RDF for function parameters 9 | * using `imporlib` approach to find local source module 10 | * fixed bug when apidocs is not configured 11 | 12 | ## 0.2.0 13 | 14 | 2021-06-27 15 | 16 | * added glossary support 17 | * added apidocs support 18 | 19 | 20 | ## 0.1.0 21 | 22 | 2021-05-22 23 | 24 | * initial release, bibliography only 25 | -------------------------------------------------------------------------------- /code_of_conduct.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to make participation in our 7 | project and our community a harassment-free experience for everyone, 8 | regardless of age, body size, disability, ethnicity, sex 9 | characteristics, gender identity and expression, level of experience, 10 | education, socio-economic status, nationality, personal appearance, 11 | race, or sexual identity and orientation. 12 | 13 | 14 | ## Our Standards 15 | 16 | Examples of behavior that contributes to creating a positive 17 | environment include: 18 | 19 | * Using welcoming and inclusive language 20 | * Being respectful of differing viewpoints and experiences 21 | * Gracefully accepting constructive criticism 22 | * Focusing on what is best for the community 23 | * Showing empathy towards other community members 24 | 25 | Examples of unacceptable behavior by participants include: 26 | 27 | * The use of sexualized language or imagery and unwelcome sexual attention or 28 | advances 29 | * Trolling, insulting/derogatory comments, and personal or political attacks 30 | * Public or private harassment 31 | * Publishing others' private information, such as a physical or electronic 32 | address, without explicit permission 33 | * Other conduct which could reasonably be considered inappropriate in a 34 | professional setting 35 | 36 | 37 | ## Our Responsibilities 38 | 39 | Project maintainers are responsible for clarifying the standards of 40 | acceptable behavior and are expected to take appropriate and fair 41 | corrective action in response to any instances of unacceptable 42 | behavior. 43 | 44 | Project maintainers have the right and responsibility to remove, edit, 45 | or reject comments, commits, code, wiki edits, issues, and other 46 | contributions that are not aligned to this Code of Conduct, or to ban 47 | temporarily or permanently any contributor for other behaviors that 48 | they deem inappropriate, threatening, offensive, or harmful. 49 | 50 | 51 | ## Scope 52 | 53 | This Code of Conduct applies within all project spaces, and it also 54 | applies when an individual is representing the project or its 55 | community in public spaces. 56 | Examples of representing a project or community include using an 57 | official project e-mail address, posting via an official social media 58 | account, or acting as an appointed representative at an online or 59 | offline event. 60 | Representation of a project may be further defined and clarified by 61 | project maintainers. 62 | 63 | 64 | ## Enforcement 65 | 66 | Instances of abusive, harassing, or otherwise unacceptable behavior 67 | may be reported by contacting the project team at the 68 | address. 69 | All complaints will be reviewed and investigated and will result in a 70 | response that is deemed necessary and appropriate to the 71 | circumstances. The project team is obligated to maintain 72 | confidentiality with regard to the reporter of an incident. 73 | Further details of specific enforcement policies may be posted 74 | separately. 75 | Project maintainers who do not follow or enforce the Code of Conduct 76 | in good faith may face temporary or permanent repercussions as 77 | determined by other members of the project's leadership. 78 | 79 | 80 | ## Attribution 81 | 82 | This Code of Conduct is adapted from version `1.4` of the 83 | [Contributor Covenant](http://contributor-covenant.org/), 84 | available at 85 | 86 | 87 | For answers to common questions about this code of conduct, see 88 | 89 | -------------------------------------------------------------------------------- /docs/assets/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DerwenAI/mkrefs/72df6a4d9d724f40728d34a135d0b9a116d548ce/docs/assets/favicon.png -------------------------------------------------------------------------------- /docs/assets/logo.graffle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DerwenAI/mkrefs/72df6a4d9d724f40728d34a135d0b9a116d548ce/docs/assets/logo.graffle -------------------------------------------------------------------------------- /docs/assets/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DerwenAI/mkrefs/72df6a4d9d724f40728d34a135d0b9a116d548ce/docs/assets/logo.png -------------------------------------------------------------------------------- /docs/biblio.jinja: -------------------------------------------------------------------------------- 1 | # Bibliography 2 | 3 | Where possible, the bibliography entries use conventions at 4 | 5 | for [*citation keys*](https://bibdesk.sourceforge.io/manual/BibDeskHelp_2.html). 6 | 7 | Journal abbreviations come from 8 | 9 | based on [*ISO 4*](https://en.wikipedia.org/wiki/ISO_4) standards. 10 | 11 | Links to online versions of cited works use 12 | [DOI](https://www.doi.org/) 13 | for [*persistent identifiers*](https://www.crossref.org/education/metadata/persistent-identifiers/). 14 | When available, 15 | [*open access*](https://peerj.com/preprints/3119v1/) 16 | URLs are listed as well. 17 | 18 | {% for letter, item_list in groups.items() %} 19 | ## – {{ letter.upper() }} – 20 | {% for item in item_list %} 21 | ### {{ item.citeKey }} 22 | 23 | ["{{ item.title }}"]({{ item.id }}) 24 | {% for auth in item.authorList %}{% if loop.index > 1 %}, {% endif %}[**{{ auth.name }}**]({{ auth.id }}){% endfor %} 25 | {% if item.isPartOf %}[*{{ item.isPartOf[0].shortTitle }}*]({{ item.isPartOf[0].identifier.id }}){% if item.volume %} **{{ item.volume.value }}**{% endif %}{% if item.issue %} {{ item.issue.value }}{% endif %}{% if item.pageStart %} pp. {{ item.pageStart.value }}-{{ item.pageEnd.value }}{% endif %} {% endif %}({{ item.Date.value }}) {% if item.doi %} 26 | DOI: {{ item.doi.value }} {% endif %}{% if item.openAccess %} 27 | open: {{ item.openAccess.id }} {% endif %} 28 | > {{ item.abstract }} 29 | {% endfor %} 30 | {% endfor %} 31 | -------------------------------------------------------------------------------- /docs/glossary.jinja: -------------------------------------------------------------------------------- 1 | # Glossary 2 | 3 | {% for letter, item_list in groups.items() %} 4 | ## – {{ letter.upper() }} – 5 | {% for item in item_list %} 6 | {% if item.redirect %}### {{ item.label }} 7 | See also: [{{ item.redirect }}](#{{ item.redirect|replace(" ", "-") }}) 8 | {% else %}### {{ item.prefLabel }} 9 | > {{ item.definition }} 10 | {% for cite_uri in item.citeKey %}{% if loop.index == 1 %} 11 | Described in: {% else %}, {% endif %}{{ cite_uri }}{% endfor %} 12 | {% if item.hypernym|length > 1 %} 13 | Broader: 14 | 15 | {% for hyp_uri in item.hypernym %} * {{ hyp_uri|safe }} 16 | {% endfor %}{% endif %} 17 | {% if item.closeMatch|length > 1 %} 18 | References: 19 | 20 | {% for ref_uri in item.closeMatch %} * {{ ref_uri }} 21 | {% endfor %}{% endif %} 22 | {% endif %} 23 | {% endfor %}{% endfor %} 24 | -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | # Here is a title 2 | 3 | Some placeholder stuff. 4 | 5 | ``` 6 | ls -lth * 7 | ``` 8 | 9 | The other pages here are generated by **MkRefs** 10 | 11 | -------------------------------------------------------------------------------- /docs/javascripts/config.js: -------------------------------------------------------------------------------- 1 | document$.subscribe(() => { 2 | hljs.highlightAll() 3 | }) 4 | 5 | window.MathJax = { 6 | tex: { 7 | inlineMath: [["\\(", "\\)"]], 8 | displayMath: [["\\[", "\\]"]], 9 | processEscapes: true, 10 | processEnvironments: true 11 | }, 12 | options: { 13 | ignoreHtmlClass: ".*|", 14 | processHtmlClass: "arithmatex" 15 | } 16 | }; 17 | -------------------------------------------------------------------------------- /docs/mkrefs.ttl: -------------------------------------------------------------------------------- 1 | @prefix derw: . 2 | 3 | @prefix bibo: . 4 | @prefix cito: . 5 | @prefix dct: . 6 | @prefix foaf: . 7 | @prefix lcsh: . 8 | @prefix madsrdf: . 9 | @prefix owl: . 10 | @prefix rdf: . 11 | @prefix rdfs: . 12 | @prefix skos: . 13 | @prefix wd: . 14 | @prefix xsd: . 15 | 16 | 17 | derw:citeKey rdfs:domain bibo:Document ; 18 | rdfs:range xsd:string ; 19 | skos:definition "bibliographic citekey for team use in related publications"@en ; 20 | . 21 | 22 | derw:openAccess rdfs:domain bibo:Document ; 23 | rdfs:range dct:identifier ; 24 | skos:definition "open access URL for a cited document"@en ; 25 | . 26 | 27 | derw:Topic a skos:Concept , 28 | madsrdf:Topic , 29 | madsrdf:Authority ; 30 | skos:prefLabel "Topic"@en ; 31 | dct:identifier wd:Q1969448 ; 32 | skos:definition "Subject heading used for classifying content and navigating discovery within it."@en ; 33 | . 34 | 35 | 36 | 37 | a foaf:Person ; 38 | foaf:name "David Gleich"@en ; 39 | dct:identifier 40 | . 41 | 42 | 43 | a foaf:Person ; 44 | foaf:name "Mike Williams"@en ; 45 | dct:identifier 46 | . 47 | 48 | 49 | a foaf:Person ; 50 | foaf:name "Cornelia Caragea"@en ; 51 | dct:identifier 52 | . 53 | 54 | 55 | a foaf:Person ; 56 | foaf:name "Corina Florescu"@en ; 57 | dct:identifier 58 | . 59 | 60 | 61 | a foaf:Person ; 62 | foaf:name "Ashkan Kazemi"@en ; 63 | dct:identifier 64 | . 65 | 66 | 67 | a foaf:Person ; 68 | foaf:name "Verónica Pérez-Rosas"@en ; 69 | dct:identifier 70 | . 71 | 72 | 73 | a foaf:Person ; 74 | foaf:name "Rada Mihalcea"@en ; 75 | dct:identifier 76 | . 77 | 78 | 79 | a foaf:Person ; 80 | foaf:name "Paul Tarau"@en ; 81 | dct:identifier 82 | . 83 | 84 | 85 | a foaf:Person ; 86 | foaf:name "Lawrence Page"@en ; 87 | dct:identifier 88 | . 89 | 90 | 91 | a foaf:Person ; 92 | foaf:name "Sergey Brin"@en ; 93 | dct:identifier 94 | . 95 | 96 | 97 | a foaf:Person ; 98 | foaf:name "Rajeev Motwani"@en ; 99 | dct:identifier 100 | . 101 | 102 | 103 | a foaf:Person ; 104 | foaf:name "Terry Winograd"@en ; 105 | dct:identifier 106 | . 107 | 108 | 109 | 110 | a bibo:Journal ; 111 | bibo:shortTitle "Comput Linguist Assoc Comput Linguis"@en ; 112 | dct:title "Computational Linguistics"@en ; 113 | dct:identifier 114 | . 115 | 116 | 117 | a bibo:Journal ; 118 | bibo:shortTitle "SIAM Review"@en ; 119 | dct:title "Society for Industrial and Applied Mathematics"@en ; 120 | dct:identifier 121 | . 122 | 123 | 124 | a bibo:Proceedings ; 125 | bibo:shortTitle "EMNLP"@en ; 126 | dct:title "Proceedings of the 2004 Conference on Empirical Methods in Natural Language Processing"@en ; 127 | dct:Date "2004" ; 128 | dct:identifier ; 129 | dct:publisher 130 | . 131 | 132 | 133 | a bibo:Journal ; 134 | bibo:shortTitle "COLING"@en ; 135 | dct:title "Proceedings - International Conference on Computational Linguistics"@en ; 136 | dct:identifier 137 | . 138 | 139 | 140 | a bibo:Collection ; 141 | bibo:shortTitle "Stanford InfoLab"@en ; 142 | dct:title "Stanford InfoLab"@en ; 143 | dct:identifier 144 | . 145 | 146 | 147 | 148 | a bibo:Article ; 149 | derw:citeKey "florescuc17"@en ; 150 | derw:openAccess ; 151 | dct:title "PositionRank: An Unsupervised Approach to Keyphrase Extraction from Scholarly Documents"@en ; 152 | dct:isPartOf ; 153 | dct:language "en" ; 154 | dct:Date "2017-07-30" ; 155 | bibo:pageStart "1105" ; 156 | bibo:pageEnd "1115" ; 157 | bibo:authorList ( 158 | 159 | 160 | ) ; 161 | bibo:doi "10.18653/v1/P17-1102" ; 162 | bibo:abstract "The large and growing amounts of online scholarly data present both challenges and opportunities to enhance knowledge discovery. One such challenge is to automatically extract a small set of keyphrases from a document that can accurately describe the document’s content and can facilitate fast information processing. In this paper, we propose PositionRank, an unsupervised model for keyphrase extraction from scholarly documents that incorporates information from all positions of a word’s occurrences into a biased PageRank. Our model obtains remarkable improvements in performance over PageRank models that do not take into account word positions as well as over strong baselines for this task. Specifically, on several datasets of research papers, PositionRank achieves improvements as high as 29.09%."@en 163 | . 164 | 165 | 166 | a bibo:Article ; 167 | derw:citeKey "gleich15"@en ; 168 | derw:openAccess ; 169 | dct:title "PageRank Beyond the Web"@en ; 170 | dct:language "en" ; 171 | dct:Date "2015-08-06" ; 172 | dct:isPartOf ; 173 | bibo:volume "57" ; 174 | bibo:issue "3" ; 175 | bibo:pageStart "321" ; 176 | bibo:pageEnd "363" ; 177 | bibo:authorList ( 178 | 179 | ) ; 180 | bibo:doi "10.1137/140976649" ; 181 | bibo:abstract "Google's PageRank method was developed to evaluate the importance of web-pages via their link structure. The mathematics of PageRank, however, are entirely general and apply to any graph or network in any domain. Thus, PageRank is now regularly used in bibliometrics, social and information network analysis, and for link prediction and recommendation. It's even used for systems analysis of road networks, as well as biology, chemistry, neuroscience, and physics. We'll see the mathematics and ideas that unite these diverse applications."@en 182 | . 183 | 184 | 185 | a bibo:Article ; 186 | derw:citeKey "kazemi-etal-2020-biased"@en ; 187 | derw:openAccess ; 188 | dct:title "Biased TextRank: Unsupervised Graph-Based Content Extraction"@en ; 189 | dct:language "en" ; 190 | dct:Date "2020-12-08" ; 191 | dct:isPartOf ; 192 | bibo:volume "28" ; 193 | bibo:pageStart "1642" ; 194 | bibo:pageEnd "1652" ; 195 | bibo:authorList ( 196 | 197 | 198 | 199 | ) ; 200 | bibo:doi "10.18653/v1/2020.coling-main.144" ; 201 | bibo:abstract "We introduce Biased TextRank, a graph-based content extraction method inspired by the popular TextRank algorithm that ranks text spans according to their importance for language processing tasks and according to their relevance to an input 'focus'. Biased TextRank enables focused content extraction for text by modifying the random restarts in the execution of TextRank. The random restart probabilities are assigned based on the relevance of the graph nodes to the focus of the task. We present two applications of Biased TextRank: focused summarization and explanation extraction, and show that our algorithm leads to improved performance on two different datasets by significant ROUGE-N score margins. Much like its predecessor, Biased TextRank is unsupervised, easy to implement and orders of magnitude faster and lighter than current state-of-the-art Natural Language Processing methods for similar tasks."@en 202 | . 203 | 204 | 205 | a bibo:Article ; 206 | derw:citeKey "page1998"@en ; 207 | derw:openAccess ; 208 | dct:title "The PageRank Citation Ranking: Bringing Order to the Web"@en ; 209 | dct:language "en" ; 210 | dct:Date "1999-11-11" ; 211 | dct:isPartOf ; 212 | bibo:authorList ( 213 | 214 | 215 | 216 | 217 | ) ; 218 | bibo:abstract "The importance of a Web page is an inherently subjective matter, which depends on the readers interests, knowledge and attitudes. But there is still much that can be said objectively about the relative importance of Web pages. This paper describes PageRank, a method for rating Web pages objectively and mechanically, effectively measuring the human interest and attention devoted to them. We compare PageRank to an idealized random Web surfer. We show how to efficiently compute PageRank for large numbers of pages. And, we show how to apply PageRank to search and to user navigation."@en 219 | . 220 | 221 | 222 | a bibo:Article ; 223 | derw:citeKey "mihalcea04textrank"@en ; 224 | derw:openAccess ; 225 | dct:title "TextRank: Bringing Order into Text"@en ; 226 | dct:language "en" ; 227 | dct:Date "2004-07-25" ; 228 | dct:isPartOf ; 229 | bibo:pageStart "404" ; 230 | bibo:pageEnd "411" ; 231 | bibo:authorList ( 232 | 233 | 234 | ) ; 235 | bibo:abstract "In this paper, the authors introduce TextRank, a graph-based ranking model for text processing, and show how this model can be successfully used in natural language applications."@en 236 | . 237 | 238 | 239 | a bibo:Slideshow ; 240 | derw:citeKey "williams2016"@en ; 241 | dct:title "Summarizing documents"@en ; 242 | dct:language "en" ; 243 | dct:Date "2016-09-25" ; 244 | bibo:presentedAt ; 245 | bibo:authorList ( 246 | 247 | ) ; 248 | bibo:abstract "I've recently given a couple of talks (PyGotham video, PyGotham slides, Strata NYC slides) about text summarization. I cover three ways of automatically summarizing text. One is an extremely simple algorithm from the 1950s, one uses Latent Dirichlet Allocation, and one uses skipthoughts and recurrent neural networks. The talk is conceptual, and avoids code and mathematics. So here is a list of resources if you're interested in text summarization and want to dive deeper. This list useful is hopefully also useful if you're interested in topic modelling or neural networks for other reasons."@en 249 | . 250 | 251 | 252 | derw:topic_Natural_Language 253 | a derw:Topic ; 254 | skos:prefLabel "natural language"@en ; 255 | skos:altLabel "NLP"@en ; 256 | skos:closeMatch wd:Q30642 , 257 | lcsh:sh88002425 , 258 | ; 259 | skos:broader ; 260 | skos:definition "Intersection of computer science and linguistics, used to leverage data in the form of text, speech, and images to identify structure and meaning. Also used for enabling people and computer-based agents to interact using natural language."@en 261 | . 262 | 263 | derw:topic_Summarization 264 | a derw:Topic ; 265 | skos:prefLabel "summarization"@en ; 266 | skos:altLabel "text summarization"@en ; 267 | skos:closeMatch wd:Q1394144 , 268 | ; 269 | skos:broader derw:topic_Natural_Language ; 270 | skos:definition "Producing a shorter version of one or more documents, while preserving most of the input's meaning."@en 271 | . 272 | 273 | derw:topic_Abstractive_Summarization 274 | a derw:Topic ; 275 | skos:prefLabel "abstractive summarization"@en ; 276 | skos:closeMatch ; 277 | skos:broader derw:topic_Summarization ; 278 | skos:definition "Generating a short, concise summary which captures salient ideas of the source text, potentially using new phrases and sentences that may not appear in the source."@en 279 | . 280 | 281 | derw:topic_Extractive_Summarization 282 | a derw:Topic ; 283 | skos:prefLabel "extractive summarization"@en ; 284 | skos:closeMatch ; 285 | skos:broader derw:topic_Summarization ; 286 | skos:definition "Summarizing the source text by identifying a subset of the sentences as the most important excerpts, then generating a sequence of them verbatim."@en 287 | . 288 | 289 | derw:topic_Coreference_Resolution 290 | a derw:Topic ; 291 | skos:prefLabel "coreference resolution"@en ; 292 | skos:closeMatch wd:Q63087 , 293 | , 294 | ; 295 | skos:broader derw:topic_Natural_Language ; 296 | skos:definition "Clustering mentions within a text that refer to the same underlying entities."@en 297 | . 298 | 299 | derw:topic_Deep_Learning 300 | a derw:Topic ; 301 | skos:prefLabel "deep learning"@en ; 302 | skos:altLabel "DL"@en ; 303 | skos:closeMatch wd:Q197536 , 304 | ; 305 | skos:broader ; 306 | skos:definition "A family of machine learning methods based on artificial neural networks which use representation learning."@en 307 | . 308 | 309 | derw:topic_Graph_Algorithms 310 | a derw:Topic ; 311 | skos:prefLabel "graph algorithms"@en ; 312 | skos:closeMatch lcsh:sh2002004605 , 313 | ; 314 | skos:broader ; 315 | skos:definition "A family of algorithms that operation on graphs for network analysis, measurement, ranking, partitioning, and other methods that leverage graph theory."@en 316 | . 317 | 318 | derw:topic_Eigenvector_Centrality 319 | a derw:Topic ; 320 | skos:prefLabel "eigenvector centrality"@en ; 321 | skos:closeMatch wd:Q28401090 , 322 | ; 323 | skos:broader derw:topic_Graph_Algorithms ; 324 | skos:definition "Measuring the influence of a node within a network."@en 325 | . 326 | 327 | derw:topic_Personalized_PageRank 328 | a derw:Topic ; 329 | skos:prefLabel "personalized pagerank"@en ; 330 | skos:broader derw:topic_Eigenvector_Centrality ; 331 | cito:usesMethodIn "page1998"@en , "gleich15"@en ; 332 | skos:definition "Using the *personalized teleportation behaviors* originally described for the PageRank algorithm to focus ranked results within a neighborhood of the graph, given a set of nodes as input."@en 333 | . 334 | 335 | derw:topic_Language_Model 336 | a derw:Topic ; 337 | skos:prefLabel "language model"@en ; 338 | skos:broader derw:topic_Natural_Language , 339 | ; 340 | skos:closeMatch wd:Q3621696 , 341 | , 342 | ; 343 | skos:definition "A statistical model used for predicting the next word or character within a document."@en 344 | . 345 | 346 | derw:topic_Transformers 347 | a derw:Topic ; 348 | skos:prefLabel "transformers"@en ; 349 | skos:broader derw:topic_Language_Model , 350 | derw:topic_Deep_Learning ; 351 | skos:closeMatch wd:Q85810444 , 352 | ; 353 | skos:definition "A family of deep learning models, mostly used in NLP, which adopts the mechanism of *attention* to weigh the influence of different parts of the input data."@en 354 | . 355 | 356 | derw:topic_Stop_Words 357 | a derw:Topic ; 358 | skos:prefLabel "stop words"@en ; 359 | skos:broader derw:topic_Natural_Language ; 360 | skos:closeMatch wd:Q80735 , 361 | lcsh:sh85046249 ; 362 | skos:definition "Words to be filtered out during natural language processing."@en 363 | . 364 | 365 | derw:topic_Phrase_Extraction 366 | a derw:Topic ; 367 | skos:prefLabel "phrase extraction"@en ; 368 | skos:broader derw:topic_Natural_Language ; 369 | skos:closeMatch wd:Q66709886 ; 370 | skos:definition "Selecting representative phrases from a document as its characteristic entities; in contrast to *keyword* analysis."@en 371 | . 372 | 373 | derw:topic_Named_Entity_Recognition 374 | a derw:Topic ; 375 | skos:prefLabel "named entity recognition"@en ; 376 | skos:altLabel "NER"@en ; 377 | skos:broader derw:topic_Phrase_Extraction ; 378 | skos:closeMatch wd:Q403574 , 379 | , 380 | ; 381 | skos:definition "Extracting mentions of *named entities* from unstructured text, then annotating them with pre-defined categories."@en 382 | . 383 | 384 | derw:topic_Entity_Linking 385 | a derw:Topic ; 386 | skos:prefLabel "entity linking"@en ; 387 | skos:broader derw:topic_Named_Entity_Recognition , 388 | derw:topic_Knowledge_Graph ; 389 | skos:closeMatch wd:Q17012245 , 390 | , 391 | ; 392 | skos:definition "Recognizing named entities within a text, then disambiguating them by linking to specific contexts in a knowledge graph."@en 393 | . 394 | 395 | derw:topic_Knowledge_Graph 396 | a derw:Topic ; 397 | skos:prefLabel "knowledge graph"@en ; 398 | skos:altLabel "KG"@en ; 399 | skos:closeMatch wd:Q33002955 , 400 | ; 401 | skos:broader ; 402 | skos:definition "A knowledge base that uses a graph-structured data model, representing and annotating interlinked descriptions of entities, with an overlay of semantic metadata."@en 403 | . 404 | 405 | derw:topic_Semantic_Relations 406 | a derw:Topic ; 407 | skos:prefLabel "semantic relations"@en ; 408 | skos:closeMatch wd:Q2268906 ; 409 | skos:broader derw:topic_Knowledge_Graph ; 410 | skos:definition "Associations that exist between the meanings of phrases."@en 411 | . 412 | 413 | derw:topic_Textgraphs 414 | a derw:Topic ; 415 | skos:prefLabel "textgraphs"@en ; 416 | skos:closeMatch wd:Q18388823 , 417 | , 418 | ; 419 | skos:broader derw:topic_Natural_Language , 420 | derw:topic_Graph_Algorithms ; 421 | skos:definition "Use of graph algorithms for NLP, based on a graph representation of a source text."@en 422 | . 423 | 424 | derw:topic_Lemma_Graph 425 | a derw:Topic ; 426 | skos:prefLabel "lemma graph"@en ; 427 | skos:broader derw:topic_Textgraphs ; 428 | cito:usesMethodIn "mihalcea04textrank"@en ; 429 | skos:definition "A graph data structure used to represent links among phrase extracted from a source text, during the operation of the TextRank algorithm."@en 430 | . 431 | -------------------------------------------------------------------------------- /docs/mkrefs.yml: -------------------------------------------------------------------------------- 1 | apidocs: 2 | page: ref.md 3 | template: ref.jinja 4 | package: mkrefs 5 | git: https://github.com/DerwenAI/mkrefs/blob/main 6 | includes: MkRefsPlugin, PackageDoc 7 | 8 | glossary: 9 | graph: mkrefs.ttl 10 | page: glossary.md 11 | template: glossary.jinja 12 | queries: 13 | entry: SELECT ?entry ?label WHERE { ?entry a derw:Topic . ?entry skos:prefLabel ?label } 14 | entry_syn: SELECT ?entry ?synonym WHERE { ?entry a derw:Topic . ?entry skos:altLabel ?synonym } 15 | entry_ref: SELECT ?entry ?closeMatch WHERE { ?entry a derw:Topic . ?entry skos:closeMatch ?closeMatch } 16 | entry_cite: SELECT ?entry ?citeKey WHERE { ?entry a derw:Topic . ?entry cito:usesMethodIn ?citeKey } 17 | entry_hyp: SELECT ?entry ?hypernym WHERE { ?entry a derw:Topic . ?entry skos:broader ?hypernym } 18 | 19 | biblio: 20 | graph: mkrefs.ttl 21 | page: biblio.md 22 | template: biblio.jinja 23 | queries: 24 | entry: SELECT ?entry ?citeKey WHERE { VALUES ?kind { bibo:Article bibo:Slideshow } ?entry a ?kind . ?entry derw:citeKey ?citeKey } 25 | entry_author: SELECT ?entry ?authorList WHERE { VALUES ?kind { bibo:Article bibo:Slideshow } ?entry a ?kind . ?entry bibo:authorList/rdf:rest*/rdf:first ?authorList } 26 | entry_publisher: SELECT ?entry ?isPartOf WHERE { VALUES ?kind { bibo:Article bibo:Slideshow } ?entry a ?kind . ?entry dct:isPartOf ?isPartOf } 27 | -------------------------------------------------------------------------------- /docs/ref.jinja: -------------------------------------------------------------------------------- 1 | # Package Reference: `{{ groups.package[0].package }}` 2 | {{ groups.package[0].docstring }} 3 | 4 | {% for class_name, class_item in groups.package[0].class.items() %} 5 | ## [`{{ class_name }}` class](#{{ class_name }}) 6 | {{ class_item.docstring | safe }} 7 | 8 | {% for method_name, method_item in class_item.method.items() %} 9 | --- 10 | #### [`{{ method_name }}` method](#{{ method_item.ns_path }}) 11 | [*\[source\]*]({{ groups.package[0].git_url }}{{ method_item.file }}#L{{ method_item.line_num }}) 12 | 13 | ```python 14 | {{ method_name }}({{ method_item.arg_list_str | safe }}) 15 | ``` 16 | {{ method_item.docstring | safe }} 17 | {{ method_item.arg_docstring | safe }} 18 | {% endfor %} 19 | {% endfor %} 20 | 21 | --- 22 | ## [module functions](#{{ groups.package[0].package }}_functions) 23 | 24 | {% for func_name, func_item in groups.package[0].function.items() %} 25 | --- 26 | #### [`{{ func_name }}` method](#{{ func_item.ns_path }}) 27 | [*\[source\]*]({{ groups.package[0].git_url }}{{ func_item.file }}#L{{ func_item.line_num }}) 28 | 29 | ```python 30 | {{ func_name }}({{ func_item.arg_list_str | safe }}) 31 | ``` 32 | {{ func_item.docstring | safe }} 33 | {{ func_item.arg_docstring | safe }} 34 | {% endfor %} 35 | 36 | --- 37 | ## [module types](#{{ groups.package[0].package }}_types) 38 | 39 | {% for type_name, type_item in groups.package[0].type.items() %} 40 | #### [`{{ type_name }}` type](#{{ type_item.ns_path }}) 41 | ```python 42 | {{ type_name }} = {{ type_item.obj | safe }} 43 | ``` 44 | {{ type_item.docstring | safe }} 45 | {% endfor %} 46 | -------------------------------------------------------------------------------- /docs/stylesheets/extra.css: -------------------------------------------------------------------------------- 1 | .md-typeset a { 2 | color: hsl(66deg 100% 31%); 3 | } 4 | 5 | .md-typeset a:focus, .md-typeset a:hover { 6 | color: hsl(306, 45%, 57%); 7 | } 8 | 9 | :root { 10 | --md-primary-fg-color: hsl(65, 46%, 58%); 11 | --md-primary-fg-color--light: #000; 12 | --md-primary-fg-color--dark: #FFF; 13 | } -------------------------------------------------------------------------------- /mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: MkRefs 2 | site_description: MkRefs Plugin – semantic references for research tools 3 | site_url: https://github.com/DerwenAI/mkrefs 4 | 5 | mkrefs_config: mkrefs.yml 6 | 7 | repo_url: https://github.com/DerwenAI/mkrefs 8 | repo_name: DerwenAI/mkrefs 9 | 10 | copyright: Source code and documentation are licensed under an MIT License; Copyright © 2021 Derwen, Inc. 11 | 12 | nav: 13 | - Home: index.md 14 | - Reference: ref.md 15 | - Glossary: glossary.md 16 | - Bibliography: biblio.md 17 | 18 | theme: 19 | name: material 20 | icon: 21 | repo: fontawesome/brands/github 22 | favicon: assets/favicon.png 23 | logo: assets/logo.png 24 | features: 25 | - navigation.instant 26 | 27 | plugins: 28 | - git-revision-date 29 | - mkrefs 30 | 31 | extra_css: 32 | - stylesheets/extra.css 33 | - https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.7.2/styles/default.min.css 34 | 35 | extra_javascript: 36 | - javascripts/config.js 37 | - https://polyfill.io/v3/polyfill.min.js?features=es6 38 | - https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js 39 | - https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.7.2/highlight.min.js 40 | 41 | use_directory_urls: true 42 | 43 | markdown_extensions: 44 | - admonition 45 | - codehilite 46 | - footnotes 47 | - toc: 48 | toc_depth: 3 49 | permalink: true 50 | - pymdownx.highlight: 51 | use_pygments: false 52 | -------------------------------------------------------------------------------- /mkrefs/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # see license https://github.com/DerwenAI/mkrefs#license-and-copyright 4 | 5 | from .plugin import MkRefsPlugin 6 | 7 | from .apidocs import render_apidocs, PackageDoc 8 | 9 | from .biblio import render_biblio 10 | 11 | from .glossary import render_glossary 12 | 13 | from .util import load_kg 14 | 15 | from .cli import cli 16 | 17 | from .version import __version__ 18 | -------------------------------------------------------------------------------- /mkrefs/apidocs.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # see license https://github.com/DerwenAI/mkrefs#license-and-copyright 4 | 5 | """ 6 | Implementation of apidoc-ish documentation which generates actual 7 | Markdown that can be used with MkDocs, and fits with Diátaxis design 8 | principles for effective documentation. 9 | 10 | Because the others really don't. 11 | 12 | In particular, this library... 13 | 14 | * is aware of type annotations (PEP 484, etc.) 15 | * provides non-bassackwards parameter descriptions (eyes on *you*, GOOG) 16 | * handles forward references (prior to Python 3.8) 17 | * links to source lines in a Git repo 18 | * fixes bugs in `typing` and `inspect` 19 | * does not require use of a separate apidocs plugin 20 | * uses `icecream` for debugging 21 | * exists b/c Sphinx sucks 22 | 23 | You're welcome. 24 | """ 25 | 26 | import importlib 27 | import inspect 28 | import os 29 | import re 30 | import sys 31 | import traceback 32 | import typing 33 | 34 | from icecream import ic # type: ignore # pylint: disable=E0401 35 | import kglab 36 | import pathlib 37 | import rdflib # type: ignore # pylint: disable=E0401 38 | 39 | from .util import render_reference 40 | 41 | 42 | class PackageDoc: 43 | """ 44 | There doesn't appear to be any other Markdown-friendly docstring support in Python. 45 | 46 | See also: 47 | 48 | * [PEP 256](https://www.python.org/dev/peps/pep-0256/) 49 | * [`inspect`](https://docs.python.org/3/library/inspect.html) 50 | """ 51 | PAT_PARAM = re.compile(r"( \S+.*\:\n(?:\S.*\n)+)", re.MULTILINE) 52 | PAT_NAME = re.compile(r"^\s+(.*)\:\n(.*)") 53 | PAT_FWD_REF = re.compile(r"ForwardRef\('(.*)'\)") 54 | 55 | 56 | def __init__ ( 57 | self, 58 | package_name: str, 59 | git_url: str, 60 | class_list: typing.List[str], 61 | ) -> None: 62 | """ 63 | Constructor, to configure a `PackageDoc` object. 64 | 65 | package_name: 66 | name of the Python package 67 | 68 | git_url: 69 | URL for the Git source repository 70 | 71 | class_list: 72 | list of the classes to include in the apidocs 73 | """ 74 | self.package_name = package_name 75 | self.git_url = git_url 76 | self.class_list = class_list 77 | 78 | # hunt for the package 79 | spec = importlib.util.spec_from_file_location(self.package_name, self.package_name + "/__init__.py") 80 | 81 | #self.package_obj = sys.modules[self.package_name] 82 | self.package_obj = importlib.util.module_from_spec(spec) 83 | spec.loader.exec_module(self.package_obj) # type: ignore 84 | sys.modules[spec.name] = self.package_obj 85 | 86 | # prepare a file path prefix (to remove later, per file) 87 | pkg_path = os.path.dirname(inspect.getfile(self.package_obj)) 88 | self.file_prefix = "/".join(pkg_path.split("/")[0:-1]) 89 | 90 | self.md: typing.List[str] = [ 91 | "# Reference: `{}` package".format(self.package_name), 92 | ] 93 | 94 | self.meta: dict = { 95 | "package": self.package_name, 96 | "git_url": self.git_url, 97 | "class": {}, 98 | "function": {}, 99 | "type": {}, 100 | } 101 | 102 | 103 | def show_all_elements ( 104 | self 105 | ) -> None: 106 | """ 107 | Show all possible elements from `inspect` for the given package, for 108 | debugging purposes. 109 | """ 110 | for name, obj in inspect.getmembers(self.package_obj): 111 | for n, o in inspect.getmembers(obj): 112 | ic(name, n, o) 113 | ic(type(o)) 114 | 115 | 116 | def get_todo_list ( 117 | self 118 | ) -> typing.Dict[ str, typing.Any]: 119 | """ 120 | Walk the package tree to find class definitions to document. 121 | 122 | returns: 123 | a dictionary of class objects which need apidocs generated 124 | """ 125 | todo_list: typing.Dict[ str, typing.Any] = { 126 | class_name: class_obj 127 | for class_name, class_obj in inspect.getmembers(self.package_obj, inspect.isclass) 128 | if class_name in self.class_list 129 | } 130 | 131 | return todo_list 132 | 133 | 134 | def build ( 135 | self 136 | ) -> None: 137 | """ 138 | Build the apidocs documentation as markdown. 139 | """ 140 | todo_list: typing.Dict[ str, typing.Any] = self.get_todo_list() 141 | 142 | # markdown for top-level package description 143 | docstring = self.get_docstring(self.package_obj) 144 | self.md.extend(docstring) 145 | 146 | self.meta["docstring"] = "\n".join(docstring).strip() 147 | 148 | # find and format the class definitions 149 | for class_name in self.class_list: 150 | class_obj = todo_list[class_name] 151 | self.format_class(class_name, class_obj) 152 | 153 | # format the function definitions and types 154 | self.format_functions() 155 | self.format_types() 156 | 157 | 158 | def get_docstring ( # pylint: disable=W0102 159 | self, 160 | obj: typing.Any, 161 | parse: bool = False, 162 | arg_dict: dict = {}, 163 | ) -> typing.List[str]: 164 | """ 165 | Get the docstring for the given object. 166 | 167 | obj: 168 | class definition for which its docstring will be inspected and parsed 169 | 170 | parse: 171 | flag to parse docstring or use the raw text; defaults to `False` 172 | 173 | arg_dict: 174 | optional dictionary of forward references, if parsed 175 | 176 | returns: 177 | list of lines of markdown 178 | """ 179 | local_md: typing.List[str] = [] 180 | raw_docstring = obj.__doc__ 181 | 182 | if raw_docstring: 183 | docstring = inspect.cleandoc(raw_docstring) 184 | 185 | if parse: 186 | local_md.append(self.parse_method_docstring(obj, docstring, arg_dict)) 187 | else: 188 | local_md.append(docstring) 189 | 190 | local_md.append("\n") 191 | 192 | return local_md 193 | 194 | 195 | def parse_method_docstring ( 196 | self, 197 | obj: typing.Any, 198 | docstring: str, 199 | arg_dict: dict, 200 | ) -> str: 201 | """ 202 | Parse the given method docstring. 203 | 204 | obj: 205 | class definition currently being inspected 206 | 207 | docstring: 208 | input docstring to be parsed 209 | 210 | arg_dict: 211 | optional dictionary of forward references 212 | 213 | returns: 214 | parsed/fixed docstring, as markdown 215 | """ 216 | local_md: typing.List[str] = [] 217 | 218 | for chunk in self.PAT_PARAM.split(docstring): 219 | m_param = self.PAT_PARAM.match(chunk) 220 | 221 | if m_param: 222 | param = m_param.group() 223 | m_name = self.PAT_NAME.match(param) 224 | 225 | if m_name: 226 | name = m_name.group(1).strip() 227 | 228 | if name not in arg_dict: 229 | code = obj.__code__ 230 | line_num = code.co_firstlineno 231 | module = code.co_filename 232 | raise Exception(f"argument `{name}` described at line {line_num} in {module} is not in the parameter list") 233 | 234 | anno = self.fix_fwd_refs(arg_dict[name]) 235 | descrip = m_name.group(2).strip() 236 | 237 | if name == "returns": 238 | local_md.append("\n * *{}* : `{}` \n{}".format(name, anno, descrip)) 239 | elif name == "yields": 240 | local_md.append("\n * *{}* : \n{}".format(name, descrip)) 241 | else: 242 | local_md.append("\n * `{}` : `{}` \n{}".format(name, anno, descrip)) 243 | else: 244 | chunk = chunk.strip() 245 | 246 | if len(chunk) > 0: 247 | local_md.append(chunk) 248 | 249 | return "\n".join(local_md) 250 | 251 | 252 | def fix_fwd_refs ( 253 | self, 254 | anno: str, 255 | ) -> typing.Optional[str]: 256 | """ 257 | Substitute the quoted forward references for a given package class. 258 | 259 | anno: 260 | raw annotated type for the forward reference 261 | 262 | returns: 263 | fixed forward reference, as markdown; or `None` if no annotation is supplied 264 | """ 265 | results: list = [] 266 | 267 | if not anno: 268 | return None 269 | 270 | for term in anno.split(", "): 271 | for chunk in self.PAT_FWD_REF.split(term): 272 | if len(chunk) > 0: 273 | results.append(chunk) 274 | 275 | return ", ".join(results) 276 | 277 | 278 | def document_method ( 279 | self, 280 | path_list: list, 281 | name: str, 282 | obj: typing.Any, 283 | func_kind: str, 284 | func_meta: dict, 285 | ) -> typing.Tuple[int, typing.List[str]]: 286 | """ 287 | Generate apidocs markdown for the given class method. 288 | 289 | path_list: 290 | elements of a class path, as a list 291 | 292 | name: 293 | class method name 294 | 295 | obj: 296 | class method object 297 | 298 | func_kind: 299 | function kind 300 | 301 | func_meta: 302 | function metadata 303 | 304 | returns: 305 | line number, plus apidocs for the method as a list of markdown lines 306 | """ 307 | local_md: typing.List[str] = ["---"] 308 | 309 | # format a header + anchor 310 | frag = ".".join(path_list + [ name ]) 311 | anchor = "#### [`{}` {}](#{})".format(name, func_kind, frag) 312 | local_md.append(anchor) 313 | 314 | func_meta["ns_path"] = frag 315 | 316 | # link to source code in Git repo 317 | code = obj.__code__ 318 | line_num = code.co_firstlineno 319 | file = code.co_filename.replace(self.file_prefix, "") 320 | 321 | src_url = "[*\[source\]*]({}{}#L{})\n".format(self.git_url, file, line_num) # pylint: disable=W1401 322 | local_md.append(src_url) 323 | 324 | func_meta["file"] = file 325 | func_meta["line_num"] = line_num 326 | 327 | # format the callable signature 328 | sig = inspect.signature(obj) 329 | arg_list = self.get_arg_list(sig) 330 | arg_list_str = "{}".format(", ".join([ a[0] for a in arg_list ])) 331 | 332 | local_md.append("```python") 333 | local_md.append("{}({})".format(name, arg_list_str)) 334 | local_md.append("```") 335 | 336 | func_meta["arg_list_str"] = arg_list_str 337 | 338 | # include the docstring, with return annotation 339 | arg_dict: dict = { 340 | name.split("=")[0]: anno 341 | for name, anno in arg_list 342 | } 343 | 344 | arg_dict["yields"] = None 345 | 346 | ret = sig.return_annotation 347 | 348 | if ret: 349 | arg_dict["returns"] = self.extract_type_annotation(ret) 350 | 351 | arg_docstring = self.get_docstring(obj, parse=True, arg_dict=arg_dict) 352 | local_md.extend(arg_docstring) 353 | local_md.append("") 354 | 355 | func_meta["arg_dict"] = arg_dict 356 | func_meta["arg_docstring"] = "\n".join(arg_docstring).strip() 357 | 358 | return line_num, local_md 359 | 360 | 361 | def get_arg_list ( 362 | self, 363 | sig: inspect.Signature, 364 | ) -> list: 365 | """ 366 | Get the argument list for a given method. 367 | 368 | sig: 369 | inspect signature for the method 370 | 371 | returns: 372 | argument list of `(arg_name, type_annotation)` pairs 373 | """ 374 | arg_list: list = [] 375 | 376 | for param in sig.parameters.values(): 377 | #ic(param.name, param.empty, param.default, param.annotation, param.kind) 378 | 379 | if param.name == "self": 380 | pass 381 | else: 382 | if param.kind == inspect.Parameter.VAR_POSITIONAL: 383 | name = "*{}".format(param.name) 384 | elif param.kind == inspect.Parameter.VAR_KEYWORD: 385 | name = "**{}".format(param.name) 386 | elif param.default == inspect.Parameter.empty: 387 | name = param.name 388 | else: 389 | if isinstance(param.default, str): 390 | default_repr = repr(param.default).replace("'", '"') 391 | else: 392 | default_repr = param.default 393 | 394 | name = "{}={}".format(param.name, default_repr) 395 | 396 | anno = self.extract_type_annotation(param.annotation) 397 | arg_list.append((name, anno)) 398 | 399 | return arg_list 400 | 401 | 402 | @classmethod 403 | def extract_type_annotation ( 404 | cls, 405 | sig: inspect.Signature, 406 | ): 407 | """ 408 | Extract the type annotation for a given method, correcting `typing` 409 | formatting problems as needed. 410 | 411 | sig: 412 | inspect signature for the method 413 | 414 | returns: 415 | corrected type annotation 416 | """ 417 | type_name = str(sig) 418 | type_class = sig.__class__.__module__ 419 | 420 | try: 421 | if type_class != "typing": 422 | if type_name.startswith(" typing.List[str]: 443 | """ 444 | Generate apidocs markdown for the given type definition. 445 | 446 | path_list: 447 | elements of a class path, as a list 448 | 449 | name: 450 | type name 451 | 452 | obj: 453 | type object 454 | 455 | returns: 456 | apidocs for the type, as a list of lines of markdown 457 | """ 458 | local_md: typing.List[str] = [] 459 | 460 | type_meta: typing.Dict[str, str] = {} 461 | self.meta["type"][name] = type_meta 462 | 463 | # format a header + anchor 464 | frag = ".".join(path_list + [ name ]) 465 | anchor = "#### [`{}` {}](#{})".format(name, "type", frag) 466 | local_md.append(anchor) 467 | 468 | type_meta["ns_path"] = frag 469 | 470 | # show type definition 471 | local_md.append("```python") 472 | local_md.append("{} = {}".format(name, obj)) 473 | local_md.append("```") 474 | local_md.append("") 475 | 476 | type_meta["obj"] = repr(obj) 477 | 478 | return local_md 479 | 480 | 481 | @classmethod 482 | def find_line_num ( 483 | cls, 484 | src: typing.Tuple[typing.List[str], int], 485 | member_name: str, 486 | ) -> int: 487 | """ 488 | Corrects for the error in parsing source line numbers of class methods that have decorators: 489 | 490 | 491 | src: 492 | list of source lines for the class being inspected 493 | 494 | member_name: 495 | name of the class member to locate 496 | 497 | returns: 498 | corrected line number of the method definition 499 | """ 500 | correct_line_num = -1 501 | 502 | for line_num, line in enumerate(src[0]): 503 | tokens = line.strip().split(" ") 504 | 505 | if tokens[0] == "def" and tokens[1] == member_name: 506 | correct_line_num = line_num 507 | 508 | return correct_line_num 509 | 510 | 511 | def format_class ( 512 | self, 513 | class_name: str, 514 | class_obj: typing.Any, 515 | ) -> None: 516 | """ 517 | Format apidocs as markdown for the given class. 518 | 519 | class_name: 520 | name of the class to document 521 | 522 | class_obj: 523 | class object 524 | """ 525 | self.md.append("## [`{}` class](#{})".format(class_name, class_name)) # pylint: disable=W1308 526 | 527 | docstring = class_obj.__doc__ 528 | src = inspect.getsourcelines(class_obj) 529 | 530 | class_meta = { 531 | "docstring": docstring, 532 | "method": {}, 533 | } 534 | 535 | self.meta["class"][class_name] = class_meta 536 | 537 | if docstring: 538 | # add the raw docstring for a class 539 | self.md.append(docstring) 540 | 541 | obj_md_pos: typing.Dict[int, typing.List[str]] = {} 542 | 543 | for member_name, member_obj in inspect.getmembers(class_obj): 544 | path_list = [self.package_name, class_name] 545 | 546 | if member_name.startswith("__") or not member_name.startswith("_"): 547 | if member_name not in class_obj.__dict__: 548 | # inherited method 549 | continue 550 | 551 | if inspect.isfunction(member_obj): 552 | func_kind = "method" 553 | elif inspect.ismethod(member_obj): 554 | func_kind = "classmethod" 555 | else: 556 | continue 557 | 558 | func_meta: typing.Dict[str, str] = {} 559 | class_meta["method"][member_name] = func_meta 560 | 561 | _, obj_md = self.document_method(path_list, member_name, member_obj, func_kind, func_meta) 562 | line_num = self.find_line_num(src, member_name) 563 | obj_md_pos[line_num] = obj_md 564 | 565 | for _, obj_md in sorted(obj_md_pos.items()): 566 | self.md.extend(obj_md) 567 | 568 | 569 | def format_functions ( 570 | self 571 | ) -> None: 572 | """ 573 | Walk the package tree, and for each function definition format its 574 | apidocs as markdown. 575 | """ 576 | self.md.append("---") 577 | self.md.append("## [package functions](#{})".format(self.package_name)) 578 | 579 | for func_name, func_obj in inspect.getmembers(self.package_obj, inspect.isfunction): 580 | if not func_name.startswith("_"): 581 | func_meta: typing.Dict[str, str] = {} 582 | self.meta["function"][func_name] = func_meta 583 | 584 | _, obj_md = self.document_method([self.package_name], func_name, func_obj, "function", func_meta) 585 | self.md.extend(obj_md) 586 | 587 | 588 | def format_types ( 589 | self 590 | ) -> None: 591 | """ 592 | Walk the package tree, and for each type definition format its apidocs 593 | as markdown. 594 | """ 595 | self.md.append("---") 596 | self.md.append("## [package types](#{})".format(self.package_name)) 597 | 598 | for name, obj in inspect.getmembers(self.package_obj): 599 | if obj.__class__.__module__ == "typing": 600 | if not str(obj).startswith("~"): 601 | obj_md = self.document_type([self.package_name], name, obj) 602 | self.md.extend(obj_md) 603 | 604 | 605 | @classmethod 606 | def entity_template ( 607 | cls, 608 | kg: kglab.KnowledgeGraph, 609 | node: kglab.RDF_Node, 610 | kind: kglab.RDF_Node, 611 | name: str, 612 | descrip: str, 613 | parent: typing.Optional[kglab.RDF_Node], 614 | ) -> None: 615 | """ 616 | Represent the given entity in RDF. 617 | 618 | kg: 619 | graph object 620 | 621 | node: 622 | entity node being represented 623 | 624 | kind: 625 | RDF type of the entity 626 | 627 | name: 628 | name of the entity 629 | 630 | descrip: 631 | Markdown description from docstring 632 | 633 | parent: 634 | parent node of the entity 635 | """ 636 | kg.add(node, kg.get_ns("rdf").type, kind) 637 | kg.add(node, kg.get_ns("rdfs").label, rdflib.Literal(name, lang=kg.language)) 638 | kg.add(node, kg.get_ns("dct").description, rdflib.Literal(descrip, lang=kg.language)) 639 | 640 | if parent: 641 | kg.add(node, kg.get_ns("dct").isPartOf, parent) 642 | 643 | 644 | @classmethod 645 | def function_template ( 646 | cls, 647 | kg: kglab.KnowledgeGraph, 648 | node: kglab.RDF_Node, 649 | meta: dict, 650 | ) -> None: 651 | """ 652 | Represent additional metadata for a function in RDF. 653 | 654 | kg: 655 | graph object 656 | 657 | node: 658 | entity node being represented 659 | 660 | meta: 661 | additional metadata 662 | """ 663 | kg.add(node, kg.get_ns("derw").apidocs_ns_path, rdflib.Literal(meta["ns_path"], lang=kg.language)) 664 | kg.add(node, kg.get_ns("derw").apidocs_args, rdflib.Literal(meta["arg_list_str"], lang=kg.language)) 665 | kg.add(node, kg.get_ns("derw").apidocs_file, rdflib.Literal(meta["file"], lang=kg.language)) 666 | kg.add(node, kg.get_ns("derw").apidocs_line, rdflib.Literal(meta["line_num"], datatype=rdflib.XSD.integer)) 667 | 668 | if "yields" in meta["arg_dict"] and meta["arg_dict"]["yields"]: 669 | kg.add(node, kg.get_ns("derw").apidocs_yields, rdflib.Literal(meta["arg_dict"]["yields"], lang=kg.language)) 670 | 671 | if "returns" in meta["arg_dict"] and meta["arg_dict"]["returns"]: 672 | kg.add(node, kg.get_ns("derw").apidocs_returns, rdflib.Literal(meta["arg_dict"]["returns"], lang=kg.language)) 673 | 674 | for param_name, param_type in meta["arg_dict"].items(): 675 | if param_name not in ["yields", "returns"]: 676 | param_node = rdflib.URIRef("derw:apidocs:param:" + meta["ns_path"] + "." + re.sub("[^0-9a-zA-Z_]", "", param_name)) 677 | kg.add(param_node, kg.get_ns("rdf").type, kg.get_ns("derw").PythonParam) 678 | kg.add(param_node, kg.get_ns("rdfs").label, rdflib.Literal(param_name, lang=kg.language)) 679 | kg.add(param_node, kg.get_ns("derw").apidocs_type, rdflib.Literal(param_type, lang=kg.language)) 680 | kg.add(param_node, kg.get_ns("dct").isPartOf, node) 681 | 682 | 683 | def get_rdf ( 684 | self 685 | ) -> kglab.KnowledgeGraph: 686 | """ 687 | Generate an RDF graph from the apidocs descriptions. 688 | 689 | returns: 690 | generated knowledge graph 691 | """ 692 | kg = kglab.KnowledgeGraph( 693 | namespaces={ 694 | "dct": "http://purl.org/dc/terms/", 695 | "derw": "https://derwen.ai/ns/v1#", 696 | "owl": "http://www.w3.org/2002/07/owl#", 697 | "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", 698 | "rdfs": "http://www.w3.org/2000/01/rdf-schema#", 699 | "skos": "http://www.w3.org/2004/02/skos/core#", 700 | "xsd": "http://www.w3.org/2001/XMLSchema#", 701 | }) 702 | 703 | package_node = rdflib.URIRef(f"derw:apidocs:package:{self.package_name}") 704 | kg.add(package_node, kg.get_ns("dct").identifier, rdflib.URIRef(self.git_url)) 705 | 706 | self.entity_template( 707 | kg, 708 | package_node, 709 | kg.get_ns("derw").PythonPackage, 710 | self.package_name, 711 | self.meta["docstring"], 712 | None, 713 | ) 714 | 715 | for class_name, class_obj in self.meta["class"].items(): 716 | class_node = rdflib.URIRef(f"derw:apidocs:class:{self.package_name}.{class_name}") 717 | 718 | self.entity_template( 719 | kg, 720 | class_node, 721 | kg.get_ns("derw").PythonClass, 722 | class_name, 723 | class_obj["docstring"], 724 | package_node, 725 | ) 726 | 727 | for method_name, method_obj in class_obj["method"].items(): 728 | method_node = rdflib.URIRef(f"derw:apidocs:method:{self.package_name}.{class_name}.{method_name}") 729 | 730 | self.entity_template( 731 | kg, 732 | method_node, 733 | kg.get_ns("derw").PythonMethod, 734 | method_name, 735 | method_obj["arg_docstring"], 736 | class_node, 737 | ) 738 | 739 | self.function_template( 740 | kg, 741 | method_node, 742 | method_obj, 743 | ) 744 | 745 | for function_name, function_obj in self.meta["function"].items(): 746 | function_node = rdflib.URIRef(f"derw:apidocs:function:{self.package_name}.{function_name}") 747 | 748 | self.entity_template( 749 | kg, 750 | function_node, 751 | kg.get_ns("derw").PythonMethod, 752 | function_name, 753 | function_obj["arg_docstring"], 754 | package_node, 755 | ) 756 | 757 | self.function_template( 758 | kg, 759 | function_node, 760 | function_obj, 761 | ) 762 | 763 | return kg 764 | 765 | 766 | def render_apidocs ( # pylint: disable=R0914 767 | local_config: dict, 768 | template_path: pathlib.Path, 769 | markdown_path: pathlib.Path, 770 | ) -> typing.Dict[str, list]: 771 | """ 772 | Render the Markdown for an apidocs reference page, based on the given 773 | Jinja2 template. 774 | 775 | local_config: 776 | local configuration, including user-configurable includes/excludes 777 | 778 | template_path: 779 | file path for Jinja2 template for rendering an apidocs reference page in MkDocs 780 | 781 | markdown_path: 782 | file path for the rendered Markdown file 783 | 784 | returns: 785 | rendered Markdown 786 | """ 787 | groups: typing.Dict[str, list] = {} 788 | 789 | package_name = local_config["apidocs"]["package"] 790 | git_url = local_config["apidocs"]["git"] 791 | 792 | includes = [ 793 | name.strip() 794 | for name in local_config["apidocs"]["includes"].split(",") 795 | ] 796 | 797 | pkg_doc = PackageDoc( 798 | package_name, 799 | git_url, 800 | includes, 801 | ) 802 | 803 | try: 804 | # hardcore debug only: 805 | #pkg_doc.show_all_elements() 806 | 807 | # build the apidocs markdown 808 | pkg_doc.build() 809 | 810 | # render the JSON into Markdown using the Jinja2 template 811 | groups = { 812 | "package": [ pkg_doc.meta ], 813 | } 814 | 815 | render_reference( 816 | template_path, 817 | markdown_path, 818 | groups, 819 | ) 820 | except Exception as e: # pylint: disable=W0703 821 | print(f"Error rendering apidocs: {e}") 822 | traceback.print_exc() 823 | 824 | return groups 825 | -------------------------------------------------------------------------------- /mkrefs/biblio.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # see license https://github.com/DerwenAI/mkrefs#license-and-copyright 4 | 5 | from collections import OrderedDict 6 | import json 7 | import tempfile 8 | import typing 9 | 10 | import kglab 11 | import pathlib 12 | 13 | from .util import abbrev_iri, denorm_entity, get_item_list, render_reference 14 | 15 | 16 | def render_biblio ( # pylint: disable=R0914 17 | local_config: dict, 18 | kg: kglab.KnowledgeGraph, 19 | template_path: pathlib.Path, 20 | markdown_path: pathlib.Path, 21 | ) -> typing.Dict[str, list]: 22 | """ 23 | Render the Markdown for a bibliography, based on the given KG and 24 | Jinja2 template. 25 | 26 | local_config: 27 | local configuration, including user-configurable SPARQL queries 28 | 29 | kg: 30 | the KG graph object 31 | 32 | template_path: 33 | file path for Jinja2 template for rendering a bibliography page in MkDocs 34 | 35 | markdown_path: 36 | file path for the rendered Markdown file 37 | 38 | returns: 39 | rendered Markdown 40 | """ 41 | # get the bibliography entry identifiers 42 | sparql = local_config["biblio"]["queries"]["entry"] 43 | df = kg.query_as_df(sparql) 44 | entry_ids = denorm_entity(df, "entry") 45 | 46 | # get the entity maps 47 | entity_map: dict = {} 48 | 49 | sparql = local_config["biblio"]["queries"]["entry_author"] 50 | list_name, list_ids = get_item_list(kg, sparql) 51 | entity_map[list_name] = list_ids 52 | 53 | sparql = local_config["biblio"]["queries"]["entry_publisher"] 54 | list_name, list_ids = get_item_list(kg, sparql) 55 | entity_map[list_name] = list_ids 56 | 57 | # extract content as JSON-LD 58 | items: dict = {} 59 | 60 | json_path = pathlib.Path(tempfile.NamedTemporaryFile().name) 61 | kg.save_jsonld(json_path) 62 | 63 | with open(json_path, "r") as f: # pylint: disable=W0621 64 | bib_j = json.load(f) 65 | 66 | items = { 67 | item["@id"]: abbrev_iri(item) 68 | for item in bib_j["@graph"] 69 | } 70 | 71 | # denormalize the JSON-LD for bibliography entries 72 | entries: dict = {} 73 | 74 | for id, val_dict in entry_ids.items(): 75 | citekey = val_dict["citeKey"] 76 | entries[citekey] = items[id] 77 | 78 | for key in entries[citekey].keys(): 79 | if key in entity_map: 80 | entries[citekey][key] = [ 81 | items[mapped_id] 82 | for mapped_id in entity_map[key][id] 83 | ] 84 | 85 | # initialize the `groups` grouping of entries 86 | entries = OrderedDict(sorted(entries.items())) 87 | letters = sorted(list({ 88 | key[0].lower() 89 | for key in entries.keys() 90 | })) 91 | 92 | groups: typing.Dict[str, list] = { # pylint: disable=W0621 93 | letter: [] 94 | for letter in letters 95 | } 96 | 97 | # build the grouping of content entries 98 | for citekey, entry in entries.items(): 99 | letter = citekey[0].lower() 100 | groups[letter].append(entry) 101 | 102 | # render the JSON into Markdown using the Jinja2 template 103 | _ = render_reference(template_path, markdown_path, groups) 104 | 105 | return groups 106 | -------------------------------------------------------------------------------- /mkrefs/cli.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # see license https://github.com/DerwenAI/mkrefs#license-and-copyright 4 | 5 | from pprint import pprint 6 | import pathlib 7 | import typer 8 | import yaml 9 | 10 | from .apidocs import render_apidocs 11 | from .biblio import render_biblio 12 | from .glossary import render_glossary 13 | from .util import load_kg 14 | 15 | 16 | APP = typer.Typer() 17 | 18 | 19 | @APP.command() 20 | def apidocs ( 21 | config_file: str, 22 | ) -> None: 23 | """ 24 | Command to generate a package reference apidocs. 25 | """ 26 | config_path = pathlib.Path(config_file) 27 | docs_dir = config_path.parent 28 | local_config = yaml.safe_load(config_path.read_text()) 29 | 30 | template_path = docs_dir / local_config["apidocs"]["template"] 31 | markdown_path = docs_dir / local_config["apidocs"]["page"] 32 | 33 | groups = render_apidocs(local_config, template_path, markdown_path) 34 | pprint(groups) 35 | 36 | 37 | @APP.command() 38 | def biblio ( 39 | config_file: str, 40 | ) -> None: 41 | """ 42 | Command to generate a bibliography. 43 | """ 44 | config_path = pathlib.Path(config_file) 45 | docs_dir = config_path.parent 46 | local_config = yaml.safe_load(config_path.read_text()) 47 | 48 | graph_path = docs_dir / local_config["biblio"]["graph"] 49 | kg = load_kg(graph_path) 50 | 51 | template_path = docs_dir / local_config["biblio"]["template"] 52 | markdown_path = docs_dir / local_config["biblio"]["page"] 53 | 54 | groups = render_biblio(local_config, kg, template_path, markdown_path) 55 | pprint(groups) 56 | 57 | 58 | @APP.command() 59 | def glossary ( # pylint: disable=W0613 60 | config_file: str, 61 | ) -> None: 62 | """ 63 | Command to generate a glossary. 64 | """ 65 | config_path = pathlib.Path(config_file) 66 | docs_dir = config_path.parent 67 | local_config = yaml.safe_load(config_path.read_text()) 68 | 69 | graph_path = docs_dir / local_config["glossary"]["graph"] 70 | kg = load_kg(graph_path) 71 | 72 | template_path = docs_dir / local_config["glossary"]["template"] 73 | markdown_path = docs_dir / local_config["glossary"]["page"] 74 | 75 | groups = render_glossary(local_config, kg, template_path, markdown_path) 76 | pprint(groups) 77 | 78 | 79 | def cli () -> None: 80 | """ 81 | Entry point for Typer-based CLI. 82 | """ 83 | APP() 84 | -------------------------------------------------------------------------------- /mkrefs/glossary.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # see license https://github.com/DerwenAI/mkrefs#license-and-copyright 4 | 5 | from collections import OrderedDict 6 | import json 7 | import tempfile 8 | import typing 9 | 10 | import kglab 11 | import pathlib 12 | 13 | from .util import abbrev_iri, denorm_entity, get_item_list, render_reference 14 | 15 | 16 | def render_glossary ( # pylint: disable=R0914 17 | local_config: dict, 18 | kg: kglab.KnowledgeGraph, 19 | template_path: pathlib.Path, 20 | markdown_path: pathlib.Path, 21 | ) -> typing.Dict[str, list]: 22 | """ 23 | Render the Markdown for a glossary, based on the given KG and 24 | Jinja2 template. 25 | 26 | local_config: 27 | local configuration, including user-configurable SPARQL queries 28 | 29 | kg: 30 | the KG graph object 31 | 32 | template_path: 33 | file path for Jinja2 template for rendering a glossary page in MkDocs 34 | 35 | markdown_path: 36 | file path for the rendered Markdown file 37 | 38 | returns: 39 | rendered Markdown 40 | """ 41 | # get the glossary entry identifiers 42 | sparql = local_config["glossary"]["queries"]["entry"] 43 | df = kg.query_as_df(sparql, simplify=False, pythonify=True) 44 | entry_ids = denorm_entity(df, "entry") 45 | 46 | sparql = local_config["glossary"]["queries"]["entry_syn"] 47 | _, syn_labels = get_item_list(kg, sparql) 48 | 49 | # get the entity maps 50 | entity_map: dict = {} 51 | 52 | sparql = local_config["glossary"]["queries"]["entry_ref"] 53 | list_name, list_ids = get_item_list(kg, sparql) 54 | entity_map[list_name] = list_ids 55 | 56 | ## localize the taxonomy for hypernyms 57 | sparql = local_config["glossary"]["queries"]["entry_hyp"] 58 | list_name, list_ids = get_item_list(kg, sparql) 59 | localized_hyp_ids: dict = {} 60 | 61 | for topic_uri, items in list_ids.items(): 62 | hyp_ids: list = [] 63 | 64 | for hypernym in items: 65 | if hypernym in entry_ids: 66 | hyp_label = entry_ids[hypernym]["label"] 67 | hyp_link = hyp_label.replace(" ", "-") 68 | 69 | hyp_ids.append(f"[{hyp_label}](#{hyp_link})") 70 | else: 71 | hyp_ids.append(f"{hypernym}") 72 | 73 | localized_hyp_ids[topic_uri] = hyp_ids 74 | 75 | entity_map[list_name] = localized_hyp_ids 76 | 77 | ## localize the citekey entries for the bibliography 78 | sparql = local_config["glossary"]["queries"]["entry_cite"] 79 | list_name, list_ids = get_item_list(kg, sparql) 80 | biblio_page = "../{}/".format(local_config["biblio"]["page"].replace(".md", "")) 81 | localized_cite_ids: dict = {} 82 | 83 | for topic_uri, items in list_ids.items(): 84 | localized_cite_ids[topic_uri] = [ 85 | f"[[{citekey}]]({biblio_page}#{citekey})" 86 | for citekey in items 87 | ] 88 | 89 | entity_map[list_name] = localized_cite_ids 90 | 91 | # extract content as JSON-LD 92 | entries: dict = {} 93 | 94 | json_path = pathlib.Path(tempfile.NamedTemporaryFile().name) 95 | kg.save_jsonld(json_path) 96 | 97 | with open(json_path, "r") as f: # pylint: disable=W0621 98 | gloss_j = json.load(f) 99 | 100 | entries = { 101 | entry_ids[item["@id"]]["label"]: abbrev_iri(item) 102 | for item in gloss_j["@graph"] 103 | if item["@id"] in entry_ids 104 | } 105 | 106 | # denormalize the JSON-LD for glossary entries 107 | for id, val_dict in entry_ids.items(): 108 | definition = val_dict["label"] 109 | 110 | for key, entry in entity_map.items(): 111 | for topic_uri, items in entry.items(): 112 | if topic_uri == id: 113 | entries[definition][key] = items 114 | 115 | # add redirects for the synonyms 116 | for topic_uri, items in syn_labels.items(): 117 | for synonym in items: 118 | entries[synonym] = { 119 | "label": synonym, 120 | "redirect": entry_ids[topic_uri]["label"], 121 | } 122 | 123 | # initialize the `groups` grouping of entries 124 | entries = OrderedDict(sorted(entries.items())) 125 | letters = sorted(list({ 126 | key[0].lower() 127 | for key in entries.keys() 128 | })) 129 | 130 | groups: typing.Dict[str, list] = { # pylint: disable=W0621 131 | letter: [] 132 | for letter in letters 133 | } 134 | 135 | # build the grouping of content entries 136 | for definition, entry in entries.items(): 137 | letter = definition[0].lower() 138 | groups[letter].append(entry) 139 | 140 | # render the JSON into Markdown using the Jinja2 template 141 | _ = render_reference(template_path, markdown_path, groups) 142 | 143 | return groups 144 | -------------------------------------------------------------------------------- /mkrefs/plugin.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # see license https://github.com/DerwenAI/mkrefs#license-and-copyright 4 | 5 | """ 6 | MkDocs Plugin. 7 | 8 | https://www.mkdocs.org/ 9 | https://github.com/DerwenAI/mkrefs 10 | """ 11 | 12 | from collections import defaultdict 13 | from pprint import pprint # pylint: disable=W0611 14 | import pathlib 15 | import sys 16 | import typing 17 | 18 | from mkdocs.config import config_options # type: ignore # pylint: disable=E0401 19 | import mkdocs.plugins # type: ignore # pylint: disable=E0401 20 | import mkdocs.structure.files # type: ignore # pylint: disable=E0401 21 | import mkdocs.structure.nav # type: ignore # pylint: disable=E0401 22 | import mkdocs.structure.pages # type: ignore # pylint: disable=E0401 23 | 24 | import jinja2 25 | import livereload # type: ignore # pylint: disable=E0401 26 | import yaml 27 | 28 | from .apidocs import render_apidocs 29 | from .biblio import render_biblio 30 | from .glossary import render_glossary 31 | from .util import load_kg 32 | 33 | 34 | class MkRefsPlugin (mkdocs.plugins.BasePlugin): 35 | """ 36 | MkDocs plugin for semantic reference pages, partly constructed from an 37 | input knowledge graph, and partly by parsing code and dependencies to 38 | construct portions of a knowledge graph. 39 | """ 40 | _LOCAL_CONFIG_KEYS: dict = { 41 | "apidocs": { 42 | "page": "the generated Markdown page; e.g., `ref.md`", 43 | "template": "a Jinja2 template; e.g., `ref.jinja`", 44 | "package": "the Python package name", 45 | "git": "URL to the source code in a public Git repository", 46 | "includes": "class and function names to include", 47 | }, 48 | 49 | "glossary": { 50 | "graph": "an RDF graph in Turtle (TTL) format; e.g., `mkrefs.ttl`", 51 | "page": "the generated Markdown page; e.g., `glossary.md`", 52 | "template": "a Jinja2 template; e.g., `glossary.jinja`", 53 | "queries": "a list of SPARQL queries to extract [definition, taxonomy, citation] entities", 54 | }, 55 | 56 | "biblio": { 57 | "graph": "an RDF graph in Turtle (TTL) format; e.g., `mkrefs.ttl`", 58 | "page": "the generated Markdown page; e.g., `biblio.md`", 59 | "template": "a Jinja2 template; e.g., `biblio.jinja`", 60 | "queries": "a list of SPARQL queries to extract [author, publisher, entry] entities", 61 | }, 62 | } 63 | 64 | config_scheme = ( 65 | ("mkrefs_config", config_options.Type(str, default="mkrefs.yml")), 66 | ) 67 | 68 | def __init__ (self): 69 | #print("__init__", self.config_scheme) 70 | self.enabled = True 71 | self.local_config: dict = defaultdict() 72 | 73 | self.apidocs_used = False 74 | self.apidocs_file = None 75 | 76 | self.glossary_kg = None 77 | self.glossary_file = None 78 | 79 | self.biblio_kg = None 80 | self.biblio_file = None 81 | 82 | 83 | def _valid_component_config ( 84 | self, 85 | yaml_path: pathlib.Path, 86 | component: str, 87 | ) -> bool: 88 | """ 89 | Semiprivate helper method to run error checking, for the given MkRefs 90 | component, of all of its expected fields in the local configuration. 91 | 92 | yaml_path: 93 | path for the MkRefs local configuration YAML file 94 | 95 | component: 96 | MkRefs plugin component, e.g. `["biblio", "glossary", "apidocs", "depend", "index"]` 97 | 98 | returns: 99 | boolean flag, for whether the component is configured properly 100 | """ 101 | if component in self.local_config: 102 | for param, message in self._LOCAL_CONFIG_KEYS[component].items(): 103 | if param not in self.local_config[component]: 104 | print(f"ERROR: `{yaml_path}` is missing the `{component}:{param}` parameter, which should be {message}") 105 | sys.exit(-1) 106 | 107 | return True 108 | 109 | return False 110 | 111 | 112 | def on_config ( # pylint: disable=W0613 113 | self, 114 | config: config_options.Config, 115 | **kwargs: typing.Any, 116 | ) -> typing.Dict[str, typing.Any]: 117 | """ 118 | The `config` event is the first event called on MkDocs build and gets 119 | run immediately after the user configuration is loaded and validated. 120 | Any alterations to the configuration should be made here. 121 | 122 | 123 | 124 | config: 125 | the default global configuration object 126 | 127 | returns: 128 | the possibly modified global configuration object 129 | """ 130 | #print("on_config") 131 | #pprint(config) 132 | 133 | if "mkrefs_config" not in config: 134 | print("ERROR missing `mkrefs_config` parameter") 135 | sys.exit(-1) 136 | 137 | # load the MkRefs local configuration 138 | yaml_path = pathlib.Path(config["docs_dir"]) / config["mkrefs_config"] 139 | 140 | try: 141 | with open(yaml_path, "r") as f: 142 | self.local_config = yaml.safe_load(f) 143 | #print(self.local_config) 144 | except Exception as e: # pylint: disable=W0703 145 | print(f"ERROR loading local config: {e}") 146 | sys.exit(-1) 147 | 148 | reuse_graph_path = None 149 | 150 | if self._valid_component_config(yaml_path, "apidocs"): 151 | self.apidocs_used = True 152 | 153 | if self._valid_component_config(yaml_path, "glossary"): 154 | # load the KG for the glossary 155 | try: 156 | graph_path = pathlib.Path(config["docs_dir"]) / self.local_config["glossary"]["graph"] 157 | reuse_graph_path = graph_path 158 | self.glossary_kg = load_kg(graph_path) 159 | except Exception as e: # pylint: disable=W0703 160 | print(f"ERROR loading graph: {e}") 161 | sys.exit(-1) 162 | 163 | if self._valid_component_config(yaml_path, "biblio"): 164 | # load the KG for the bibliography 165 | try: 166 | graph_path = pathlib.Path(config["docs_dir"]) / self.local_config["biblio"]["graph"] 167 | 168 | if graph_path == reuse_graph_path: 169 | self.biblio_kg = self.glossary_kg 170 | else: 171 | self.biblio_kg = load_kg(graph_path) 172 | except Exception as e: # pylint: disable=W0703 173 | print(f"ERROR loading graph: {e}") 174 | sys.exit(-1) 175 | 176 | return config 177 | 178 | 179 | def on_files ( # pylint: disable=W0613 180 | self, 181 | files: mkdocs.structure.files.Files, 182 | config: config_options.Config, 183 | **kwargs: typing.Any, 184 | ) -> mkdocs.structure.files.Files: 185 | """ 186 | The `files` event is called after the files collection is populated 187 | from the `docs_dir` parameter. 188 | Use this event to add, remove, or alter files in the collection. 189 | Note that `Page` objects have not yet been associated with the file 190 | objects in the collection. 191 | Use [Page Events](https://www.mkdocs.org/user-guide/plugins/#page-events) 192 | to manipulate page-specific data. 193 | 194 | files: 195 | default global files collection 196 | 197 | config: 198 | the default global configuration object 199 | 200 | returns: 201 | the possibly modified global files collection 202 | """ 203 | if self.apidocs_used and self.local_config["apidocs"]["page"]: 204 | self.apidocs_file = mkdocs.structure.files.File( 205 | path = self.local_config["apidocs"]["page"], 206 | src_dir = config["docs_dir"], 207 | dest_dir = config["site_dir"], 208 | use_directory_urls = config["use_directory_urls"], 209 | ) 210 | 211 | files.append(self.apidocs_file) 212 | 213 | template_path = pathlib.Path(config["docs_dir"]) / self.local_config["apidocs"]["template"] 214 | markdown_path = pathlib.Path(config["docs_dir"]) / self.apidocs_file.src_path 215 | 216 | try: 217 | _ = render_apidocs(self.local_config, template_path, markdown_path) 218 | except Exception as e: # pylint: disable=W0703 219 | print(f"Error rendering apidocs: {e}") 220 | sys.exit(-1) 221 | 222 | if self.glossary_kg: 223 | self.glossary_file = mkdocs.structure.files.File( 224 | path = self.local_config["glossary"]["page"], 225 | src_dir = config["docs_dir"], 226 | dest_dir = config["site_dir"], 227 | use_directory_urls = config["use_directory_urls"], 228 | ) 229 | 230 | files.append(self.glossary_file) 231 | 232 | template_path = pathlib.Path(config["docs_dir"]) / self.local_config["glossary"]["template"] 233 | markdown_path = pathlib.Path(config["docs_dir"]) / self.glossary_file.src_path 234 | 235 | try: 236 | _ = render_glossary(self.local_config, self.glossary_kg, template_path, markdown_path) 237 | except Exception as e: # pylint: disable=W0703 238 | print(f"Error rendering glossary: {e}") 239 | sys.exit(-1) 240 | 241 | if self.biblio_kg: 242 | self.biblio_file = mkdocs.structure.files.File( 243 | path = self.local_config["biblio"]["page"], 244 | src_dir = config["docs_dir"], 245 | dest_dir = config["site_dir"], 246 | use_directory_urls = config["use_directory_urls"], 247 | ) 248 | 249 | files.append(self.biblio_file) 250 | 251 | template_path = pathlib.Path(config["docs_dir"]) / self.local_config["biblio"]["template"] 252 | markdown_path = pathlib.Path(config["docs_dir"]) / self.biblio_file.src_path 253 | 254 | try: 255 | _ = render_biblio(self.local_config, self.biblio_kg, template_path, markdown_path) 256 | except Exception as e: # pylint: disable=W0703 257 | print(f"Error rendering bibliography: {e}") 258 | sys.exit(-1) 259 | 260 | return files 261 | 262 | 263 | ###################################################################### 264 | ## other events 265 | 266 | def on_pre_build ( # pylint: disable=R0201,W0613 267 | self, 268 | config: config_options.Config, 269 | **kwargs: typing.Any, 270 | ) -> None: 271 | """ 272 | This event does not alter any variables. 273 | Use this event to call pre-build scripts. 274 | 275 | 276 | 277 | config: 278 | global configuration object 279 | """ 280 | #print("on_pre_build") 281 | return 282 | 283 | 284 | def on_nav ( # pylint: disable=R0201,W0613 285 | self, 286 | nav: mkdocs.structure.nav.Navigation, 287 | config: config_options.Config, 288 | files: mkdocs.structure.files.Files, 289 | **kwargs: typing.Any, 290 | ) -> mkdocs.structure.nav.Navigation: 291 | """ 292 | The `nav` event is called after the site navigation is created and can 293 | be used to alter the site navigation. 294 | 295 | 296 | 297 | nav: 298 | the default global navigation object 299 | 300 | config: 301 | global configuration object 302 | 303 | files: 304 | global files collection 305 | 306 | returns: 307 | the possibly modified global navigation object 308 | """ 309 | #print("on_nav") 310 | #pprint(vars(nav)) 311 | return nav 312 | 313 | 314 | def on_env ( # pylint: disable=R0201,W0613 315 | self, 316 | env: jinja2.Environment, 317 | config: config_options.Config, 318 | files: mkdocs.structure.files.Files, 319 | **kwargs: typing.Any, 320 | ) -> jinja2.Environment: 321 | """ 322 | The `env` event is called after the Jinja template environment is 323 | created and can be used to alter the Jinja environment. 324 | 325 | 326 | 327 | env: 328 | global Jinja environment 329 | 330 | config: 331 | global configuration object 332 | 333 | files: 334 | global files collection 335 | 336 | returns: 337 | the possibly modified global Jinja environment 338 | """ 339 | #print("on_env") 340 | #pprint(vars(env)) 341 | return env 342 | 343 | 344 | def on_post_build ( # pylint: disable=R0201,W0613 345 | self, 346 | config: config_options.Config, 347 | **kwargs: typing.Any, 348 | ) -> None: 349 | """ 350 | This event does not alter any variables. 351 | Use this event to call post-build scripts. 352 | 353 | 354 | 355 | config: 356 | global configuration object 357 | """ 358 | return 359 | 360 | 361 | def on_pre_template ( # pylint: disable=R0201,W0613 362 | self, 363 | template: str, 364 | template_name: str, 365 | config: config_options.Config, 366 | **kwargs: typing.Any, 367 | ) -> str: 368 | """ 369 | The `pre_template` event is called immediately after the subject 370 | template is loaded and can be used to alter the content of the 371 | template. 372 | 373 | 374 | 375 | template: 376 | the template contents, as a string 377 | 378 | template_name: 379 | filename for the template, as a string 380 | 381 | config: 382 | global configuration object 383 | 384 | returns: 385 | the possibly modified template contents, as string 386 | """ 387 | return template 388 | 389 | 390 | def on_template_context ( # pylint: disable=R0201,W0613 391 | self, 392 | context: dict, 393 | template_name: str, 394 | config: config_options.Config, 395 | **kwargs: typing.Any, 396 | ) -> dict: 397 | """ 398 | The `template_context` event is called immediately after the context 399 | is created for the subject template and can be used to alter the 400 | context for that specific template only. 401 | 402 | 403 | 404 | context: 405 | template context variables, as a dict 406 | 407 | template_name: 408 | filename for the template, as a string 409 | 410 | config: 411 | global configuration object 412 | 413 | returns: 414 | the possibly modified template context variables, as a dict 415 | """ 416 | return context 417 | 418 | 419 | def on_post_template ( # pylint: disable=R0201,W0613 420 | self, 421 | output_content: str, 422 | template_name: str, 423 | config: config_options.Config, 424 | **kwargs: typing.Any, 425 | ) -> str: 426 | """ 427 | The `post_template` event is called after the template is rendered, 428 | but before it is written to disc and can be used to alter the output 429 | of the template. 430 | If an empty string is returned, the template is skipped and nothing is 431 | is written to disc. 432 | 433 | 434 | 435 | output_content: 436 | output of rendered the template, as string 437 | 438 | template_name: 439 | filename for the template, as a string 440 | 441 | config: global configuration object 442 | 443 | returns: 444 | the possibly modified output of the rendered template, as string 445 | """ 446 | return output_content 447 | 448 | 449 | def on_pre_page ( # pylint: disable=R0201,W0613 450 | self, 451 | page: mkdocs.structure.pages.Page, 452 | config: config_options.Config, 453 | files: mkdocs.structure.files.Files, 454 | **kwargs: typing.Any, 455 | ) -> mkdocs.structure.pages.Page: 456 | """ 457 | The `pre_page` event is called before any actions are taken on the 458 | subject page and can be used to alter the `Page` instance. 459 | 460 | 461 | 462 | page: 463 | the default Page instance 464 | 465 | config: 466 | global configuration object 467 | 468 | files: 469 | global files collection 470 | 471 | returns: 472 | the possibly Page instance 473 | """ 474 | return page 475 | 476 | 477 | def on_page_read_source ( # pylint: disable=R0201,W0613 478 | self, 479 | page: mkdocs.structure.pages.Page, 480 | config: config_options.Config, 481 | **kwargs: typing.Any, 482 | ) -> typing.Optional[str]: 483 | """ 484 | The `on_page`_read_source event can replace the default mechanism to 485 | read the contents of a page's source from the filesystem. 486 | 487 | 488 | 489 | page: 490 | the default Page instance 491 | 492 | config: 493 | global configuration object 494 | 495 | returns: 496 | The raw source for a page as unicode string; if `None` is returned, the default loading from a file will be performed. 497 | """ 498 | return None 499 | 500 | 501 | def on_page_markdown ( # pylint: disable=R0201,W0613 502 | self, 503 | markdown: str, 504 | page: mkdocs.structure.pages.Page, 505 | config: config_options.Config, 506 | files: mkdocs.structure.files.Files, 507 | **kwargs: typing.Any, 508 | ) -> str: 509 | """ 510 | The `page_markdown` event is called after the page's Markdown gets loaded 511 | from its file, and can be used to alter the Markdown source text. 512 | The metadata has been parsed and is available as `page.meta` at this 513 | point. 514 | 515 | 516 | 517 | markdown: 518 | Markdown source text of the page, as a string 519 | 520 | page: 521 | Page instance 522 | 523 | config: 524 | global configuration object 525 | 526 | files: 527 | file list 528 | 529 | returns: 530 | the possibly modified Markdown source text of this page, as a string 531 | """ 532 | return markdown 533 | 534 | 535 | def on_page_content ( # pylint: disable=R0201,W0613 536 | self, 537 | html: str, 538 | page: mkdocs.structure.pages.Page, 539 | config: config_options.Config, 540 | files: mkdocs.structure.files.Files, 541 | **kwargs: typing.Any, 542 | ) -> str: 543 | """ 544 | The `page_content` event is called after the Markdown text is rendered 545 | to HTML (but before being passed to a template) and can be used to 546 | alter the HTML body of the page. 547 | 548 | 549 | 550 | html: 551 | the HTML rendered from Markdown source, as string 552 | 553 | page: 554 | Page instance 555 | 556 | config: 557 | global configuration object 558 | 559 | files: 560 | global files collection 561 | 562 | returns: 563 | the possibly modified HTML rendered from Markdown source, as string 564 | """ 565 | return html 566 | 567 | 568 | def on_page_context ( # pylint: disable=R0201,W0613 569 | self, 570 | context: dict, 571 | page: mkdocs.structure.pages.Page, 572 | config: config_options.Config, 573 | nav: mkdocs.structure.nav.Navigation, 574 | **kwargs: typing.Any, 575 | ) -> dict: 576 | """ 577 | The `page_context` event is called after the context for a page is 578 | created and can be used to alter the context for that specific page 579 | only. 580 | 581 | 582 | 583 | context: 584 | template context variables, as a dict 585 | 586 | page: 587 | Page instance 588 | 589 | config: 590 | global configuration object 591 | 592 | nav: 593 | global navigation object 594 | 595 | returns: 596 | the possibly modified template context variables, as a dict 597 | """ 598 | return context 599 | 600 | 601 | def on_post_page ( # pylint: disable=R0201,W0613 602 | self, 603 | output_content: str, 604 | page: mkdocs.structure.pages.Page, 605 | config: config_options.Config, 606 | **kwargs: typing.Any, 607 | ) -> str: 608 | """ 609 | The `post_page` event is called after the template is rendered, but 610 | before it is written to disc and can be used to alter the output of 611 | the page. 612 | If an empty string is returned, the page is skipped and nothing gets 613 | written to disk. 614 | 615 | 616 | 617 | output_content: 618 | the default output of the rendered template, as string 619 | 620 | page: 621 | Page instance 622 | 623 | config: 624 | global configuration object 625 | 626 | returns: 627 | the possibly modified output of the rendered template, as string 628 | """ 629 | return output_content 630 | 631 | 632 | def on_serve ( # pylint: disable=R0201,W0613 633 | self, 634 | server: livereload.Server, 635 | config: config_options.Config, 636 | builder: typing.Any, 637 | **kwargs: typing.Any, 638 | ) -> livereload.Server: 639 | """ 640 | The `serve` event is only called when the serve command is used during 641 | development. 642 | It is passed the `Server` instance which can be modified before it is 643 | activated. 644 | For example, additional files or directories could be added to the 645 | list of "watched" files for auto-reloading. 646 | 647 | 648 | 649 | server: 650 | default livereload.Server instance 651 | 652 | config: 653 | global configuration object 654 | 655 | builder: 656 | a callable which gets passed to each call to `server.watch()` 657 | 658 | returns: 659 | the possibly modified livereload.Server instance 660 | """ 661 | return server 662 | -------------------------------------------------------------------------------- /mkrefs/util.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # see license https://github.com/DerwenAI/mkrefs#license-and-copyright 4 | 5 | from collections import defaultdict 6 | import re 7 | import typing 8 | 9 | import jinja2 # type: ignore # pylint: disable=E0401 10 | import kglab 11 | import pathlib 12 | import pandas as pd # type: ignore # pylint: disable=E0401 13 | 14 | 15 | def load_kg ( 16 | path: pathlib.Path, 17 | ) -> kglab.KnowledgeGraph: 18 | """ 19 | Load a KG from an RDF file in "Turtle" (TTL) format. 20 | 21 | path: 22 | path to the RDF file 23 | 24 | returns: 25 | populated KG 26 | """ 27 | kg = kglab.KnowledgeGraph() 28 | kg.load_rdf(path, format="ttl") 29 | 30 | return kg 31 | 32 | 33 | def get_jinja2_template ( 34 | template_file: str, 35 | dir: str, 36 | ) -> jinja2.Template: 37 | """ 38 | Load a Jinja2 template. 39 | Because MkDocs runs the `on_env` event way too late in the lifecycle to use it to generate markdown files. 40 | 41 | template_file: 42 | file name of the Jinja2 template file 43 | 44 | dir: 45 | subdirectory in which the template file is located 46 | 47 | returns: 48 | loaded Jinja2 template 49 | """ 50 | env = jinja2.Environment( 51 | loader=jinja2.FileSystemLoader(dir), 52 | autoescape=True, 53 | ) 54 | 55 | return env.get_template(template_file) 56 | 57 | 58 | def abbrev_key ( 59 | key: str, 60 | ) -> str: 61 | """ 62 | Abbreviate the IRI, if any 63 | 64 | key: 65 | string content to abbreviate 66 | 67 | returns: 68 | abbreviated IRI content 69 | """ 70 | if key.startswith("@"): 71 | return key[1:] 72 | 73 | key = key.split(":")[-1] 74 | key = key.split("/")[-1] 75 | key = key.split("#")[-1] 76 | 77 | return key 78 | 79 | 80 | def abbrev_iri ( 81 | item: typing.Any, 82 | ) -> typing.Any: 83 | """ 84 | Abbreviate the IRIs in JSON-LD graph content, so that Jinja2 templates 85 | can use it. 86 | 87 | item: 88 | scalar, list, or dictionary to iterate through 89 | 90 | returns: 91 | data with abbreviated IRIs 92 | """ 93 | if isinstance(item, dict): 94 | d = { 95 | abbrev_key(k): abbrev_iri(v) 96 | for k, v in item.items() 97 | } 98 | 99 | return d 100 | 101 | if isinstance(item, list): 102 | l = [ 103 | abbrev_iri(x) 104 | for x in item 105 | ] 106 | 107 | return l 108 | 109 | return item 110 | 111 | 112 | def denorm_entity ( 113 | df: pd.DataFrame, 114 | entity_name: str, 115 | ) -> dict: 116 | """ 117 | Denormalize the result set from a SPARQL query, to collect a specific 118 | class of entities from the KG, along with attribute for each instance. 119 | 120 | df: 121 | SPARQL query result set, as a dataframe 122 | 123 | entity_name: 124 | column name for the entity 125 | 126 | returns: 127 | denormalized entity list with attributes, as a dict 128 | """ 129 | col_names = list(df.columns) 130 | denorm = {} 131 | 132 | for rec in df.to_records(index=False): 133 | values = {} 134 | 135 | for i, val in enumerate(rec): 136 | if pd.isna(val): 137 | val = None 138 | else: 139 | s = re.search(r"\<(.*)\>", val) 140 | 141 | if s: 142 | val = s.group(1) 143 | 144 | if col_names[i] == entity_name: 145 | entity = str(val) 146 | else: 147 | values[col_names[i]] = None if pd.isna(val) else str(val) 148 | 149 | denorm[entity] = values 150 | 151 | return denorm 152 | 153 | 154 | def de_bracket ( 155 | url: str, 156 | ) -> str: 157 | """ 158 | Extract the URL value from its RDF "bracketed" representation. 159 | 160 | url: 161 | input URL string 162 | 163 | returns: 164 | de-bracketed URL string 165 | """ 166 | s = re.search(r"\<(.*)\>", url) 167 | 168 | if s: 169 | url = s.group(1) 170 | 171 | return url 172 | 173 | 174 | def get_item_list ( 175 | kg: kglab.KnowledgeGraph, 176 | sparql: str, 177 | ) -> typing.Tuple[str, dict]: 178 | """ 179 | Query to get a list of entity identifiers to substitute in JSON-LD. 180 | 181 | kg: 182 | the KG graph object 183 | 184 | sparql: 185 | SPARQL query 186 | 187 | returns: 188 | a tuple of the list relation to replace, and the identifier values 189 | """ 190 | df = kg.query_as_df(sparql, simplify=False, pythonify=True) 191 | list_name = df.columns[1] 192 | 193 | list_ids: typing.Dict[str, list] = defaultdict(list) 194 | 195 | for rec in df.to_records(index=False): 196 | key = str(de_bracket(rec[0])) 197 | val = str(de_bracket(rec[1])) 198 | list_ids[key].append(val) 199 | 200 | return list_name, list_ids 201 | 202 | 203 | def render_reference ( 204 | template_path: pathlib.Path, 205 | markdown_path: pathlib.Path, 206 | groups: typing.Dict[str, list], 207 | ) -> str: 208 | """ 209 | Render the Markdown for a MkRefs reference component, based on the 210 | given Jinja2 template. 211 | 212 | template_path: 213 | file path for Jinja2 template for rendering a reference page in MkDocs 214 | 215 | markdown_path: 216 | file path for the rendered Markdown file 217 | 218 | groups: 219 | JSON denomalized content data 220 | 221 | returns: 222 | rendered Markdown 223 | """ 224 | template_file = str(template_path.relative_to(template_path.parent)) 225 | template = get_jinja2_template(template_file, str(template_path.parent)) 226 | 227 | # render the JSON into Markdown using the Jinja2 template 228 | with open(markdown_path, "w") as f: 229 | f.write(template.render(groups=groups)) 230 | 231 | return template.render(groups=groups) 232 | -------------------------------------------------------------------------------- /mkrefs/version.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # see license https://github.com/DerwenAI/mkrefs#license-and-copyright 4 | 5 | import typing 6 | 7 | 8 | ###################################################################### 9 | ## Python version checking 10 | 11 | MIN_PY_VERSION: typing.Tuple = (3, 6,) 12 | __version__: str = "0.3.0" 13 | -------------------------------------------------------------------------------- /pylintrc: -------------------------------------------------------------------------------- 1 | [MASTER] 2 | 3 | # A comma-separated list of package or module names from where C extensions may 4 | # be loaded. Extensions are loading into the active Python interpreter and may 5 | # run arbitrary code. 6 | extension-pkg-whitelist= 7 | 8 | # Specify a score threshold to be exceeded before program exits with error. 9 | fail-under=10.0 10 | 11 | # Add files or directories to the blacklist. They should be base names, not 12 | # paths. 13 | ignore=CVS 14 | 15 | # Add files or directories matching the regex patterns to the blacklist. The 16 | # regex matches against base names, not paths. 17 | ignore-patterns= 18 | 19 | # Python code to execute, usually for sys.path manipulation such as 20 | # pygtk.require(). 21 | #init-hook= 22 | 23 | # Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the 24 | # number of processors available to use. 25 | jobs=1 26 | 27 | # Control the amount of potential inferred values when inferring a single 28 | # object. This can help the performance when dealing with large functions or 29 | # complex, nested conditions. 30 | limit-inference-results=100 31 | 32 | # List of plugins (as comma separated values of python module names) to load, 33 | # usually to register additional checkers. 34 | load-plugins= 35 | 36 | # Pickle collected data for later comparisons. 37 | persistent=yes 38 | 39 | # When enabled, pylint would attempt to guess common misconfiguration and emit 40 | # user-friendly hints instead of false-positive error messages. 41 | suggestion-mode=yes 42 | 43 | # Allow loading of arbitrary C extensions. Extensions are imported into the 44 | # active Python interpreter and may run arbitrary code. 45 | unsafe-load-any-extension=no 46 | 47 | 48 | [MESSAGES CONTROL] 49 | 50 | # Only show warnings with the listed confidence levels. Leave empty to show 51 | # all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED. 52 | confidence= 53 | 54 | # Disable the message, report, category or checker with the given id(s). You 55 | # can either give multiple identifiers separated by comma (,) or put this 56 | # option multiple times (only on the command line, not in the configuration 57 | # file where it should appear only once). You can also use "--disable=all" to 58 | # disable everything first and then reenable specific checks. For example, if 59 | # you want to run only the similarities checker, you can use "--disable=all 60 | # --enable=similarities". If you want to run only the classes checker, but have 61 | # no Warning level messages displayed, use "--disable=all --enable=classes 62 | # --disable=W". 63 | disable=C0103, 64 | C0114, 65 | C0301, 66 | C0411, 67 | C0413, 68 | C0415, 69 | R0401, 70 | R0801, 71 | W0622, 72 | W0707 73 | 74 | # Enable the message, report, category or checker with the given id(s). You can 75 | # either give multiple identifier separated by comma (,) or put this option 76 | # multiple time (only on the command line, not in the configuration file where 77 | # it should appear only once). See also the "--disable" option for examples. 78 | enable=c-extension-no-member 79 | 80 | 81 | [REPORTS] 82 | 83 | # Python expression which should return a score less than or equal to 10. You 84 | # have access to the variables 'error', 'warning', 'refactor', and 'convention' 85 | # which contain the number of messages in each category, as well as 'statement' 86 | # which is the total number of statements analyzed. This score is used by the 87 | # global evaluation report (RP0004). 88 | evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) 89 | 90 | # Template used to display messages. This is a python new-style format string 91 | # used to format the message information. See doc for all details. 92 | #msg-template= 93 | 94 | # Set the output format. Available formats are text, parseable, colorized, json 95 | # and msvs (visual studio). You can also give a reporter class, e.g. 96 | # mypackage.mymodule.MyReporterClass. 97 | output-format=text 98 | 99 | # Tells whether to display a full report or only the messages. 100 | reports=no 101 | 102 | # Activate the evaluation score. 103 | score=yes 104 | 105 | 106 | [REFACTORING] 107 | 108 | # Maximum number of nested blocks for function / method body 109 | max-nested-blocks=5 110 | 111 | # Complete name of functions that never returns. When checking for 112 | # inconsistent-return-statements if a never returning function is called then 113 | # it will be considered as an explicit return statement and no message will be 114 | # printed. 115 | never-returning-functions=sys.exit 116 | 117 | 118 | [LOGGING] 119 | 120 | # The type of string formatting that logging methods do. `old` means using % 121 | # formatting, `new` is for `{}` formatting. 122 | logging-format-style=old 123 | 124 | # Logging modules to check that the string format arguments are in logging 125 | # function parameter format. 126 | logging-modules=logging 127 | 128 | 129 | [SPELLING] 130 | 131 | # Limits count of emitted suggestions for spelling mistakes. 132 | max-spelling-suggestions=4 133 | 134 | # Spelling dictionary name. Available dictionaries: none. To make it work, 135 | # install the python-enchant package. 136 | spelling-dict= 137 | 138 | # List of comma separated words that should not be checked. 139 | spelling-ignore-words= 140 | 141 | # A path to a file that contains the private dictionary; one word per line. 142 | spelling-private-dict-file= 143 | 144 | # Tells whether to store unknown words to the private dictionary (see the 145 | # --spelling-private-dict-file option) instead of raising a message. 146 | spelling-store-unknown-words=no 147 | 148 | 149 | [MISCELLANEOUS] 150 | 151 | # List of note tags to take in consideration, separated by a comma. 152 | notes=FIXME, 153 | XXX, 154 | TODO 155 | 156 | # Regular expression of note tags to take in consideration. 157 | #notes-rgx= 158 | 159 | 160 | [TYPECHECK] 161 | 162 | # List of decorators that produce context managers, such as 163 | # contextlib.contextmanager. Add to this list to register other decorators that 164 | # produce valid context managers. 165 | contextmanager-decorators=contextlib.contextmanager 166 | 167 | # List of members which are set dynamically and missed by pylint inference 168 | # system, and so shouldn't trigger E1101 when accessed. Python regular 169 | # expressions are accepted. 170 | generated-members= 171 | 172 | # Tells whether missing members accessed in mixin class should be ignored. A 173 | # mixin class is detected if its name ends with "mixin" (case insensitive). 174 | ignore-mixin-members=yes 175 | 176 | # Tells whether to warn about missing members when the owner of the attribute 177 | # is inferred to be None. 178 | ignore-none=yes 179 | 180 | # This flag controls whether pylint should warn about no-member and similar 181 | # checks whenever an opaque object is returned when inferring. The inference 182 | # can return multiple potential results while evaluating a Python object, but 183 | # some branches might not be evaluated, which results in partial inference. In 184 | # that case, it might be useful to still emit no-member and other checks for 185 | # the rest of the inferred objects. 186 | ignore-on-opaque-inference=yes 187 | 188 | # List of class names for which member attributes should not be checked (useful 189 | # for classes with dynamically set attributes). This supports the use of 190 | # qualified names. 191 | ignored-classes=optparse.Values,thread._local,_thread._local 192 | 193 | # List of module names for which member attributes should not be checked 194 | # (useful for modules/projects where namespaces are manipulated during runtime 195 | # and thus existing member attributes cannot be deduced by static analysis). It 196 | # supports qualified module names, as well as Unix pattern matching. 197 | ignored-modules= 198 | 199 | # Show a hint with possible names when a member name was not found. The aspect 200 | # of finding the hint is based on edit distance. 201 | missing-member-hint=yes 202 | 203 | # The minimum edit distance a name should have in order to be considered a 204 | # similar match for a missing member name. 205 | missing-member-hint-distance=1 206 | 207 | # The total number of similar names that should be taken in consideration when 208 | # showing a hint for a missing member. 209 | missing-member-max-choices=1 210 | 211 | # List of decorators that change the signature of a decorated function. 212 | signature-mutators= 213 | 214 | 215 | [VARIABLES] 216 | 217 | # List of additional names supposed to be defined in builtins. Remember that 218 | # you should avoid defining new builtins when possible. 219 | additional-builtins= 220 | 221 | # Tells whether unused global variables should be treated as a violation. 222 | allow-global-unused-variables=yes 223 | 224 | # List of strings which can identify a callback function by name. A callback 225 | # name must start or end with one of those strings. 226 | callbacks=cb_, 227 | _cb 228 | 229 | # A regular expression matching the name of dummy variables (i.e. expected to 230 | # not be used). 231 | dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ 232 | 233 | # Argument names that match this expression will be ignored. Default to name 234 | # with leading underscore. 235 | ignored-argument-names=_.*|^ignored_|^unused_ 236 | 237 | # Tells whether we should check for unused import in __init__ files. 238 | init-import=no 239 | 240 | # List of qualified module names which can have objects that can redefine 241 | # builtins. 242 | redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io 243 | 244 | 245 | [FORMAT] 246 | 247 | # Expected format of line ending, e.g. empty (any line ending), LF or CRLF. 248 | expected-line-ending-format= 249 | 250 | # Regexp for a line that is allowed to be longer than the limit. 251 | ignore-long-lines=^\s*(# )??$ 252 | 253 | # Number of spaces of indent required inside a hanging or continued line. 254 | indent-after-paren=4 255 | 256 | # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 257 | # tab). 258 | indent-string=' ' 259 | 260 | # Maximum number of characters on a single line. 261 | max-line-length=100 262 | 263 | # Maximum number of lines in a module. 264 | max-module-lines=2000 265 | 266 | # Allow the body of a class to be on the same line as the declaration if body 267 | # contains single statement. 268 | single-line-class-stmt=no 269 | 270 | # Allow the body of an if to be on the same line as the test if there is no 271 | # else. 272 | single-line-if-stmt=no 273 | 274 | 275 | [SIMILARITIES] 276 | 277 | # Ignore comments when computing similarities. 278 | ignore-comments=yes 279 | 280 | # Ignore docstrings when computing similarities. 281 | ignore-docstrings=yes 282 | 283 | # Ignore imports when computing similarities. 284 | ignore-imports=no 285 | 286 | # Minimum lines number of a similarity. 287 | min-similarity-lines=4 288 | 289 | 290 | [BASIC] 291 | 292 | # Naming style matching correct argument names. 293 | argument-naming-style=snake_case 294 | 295 | # Regular expression matching correct argument names. Overrides argument- 296 | # naming-style. 297 | #argument-rgx= 298 | 299 | # Naming style matching correct attribute names. 300 | attr-naming-style=snake_case 301 | 302 | # Regular expression matching correct attribute names. Overrides attr-naming- 303 | # style. 304 | #attr-rgx= 305 | 306 | # Bad variable names which should always be refused, separated by a comma. 307 | bad-names=foo, 308 | bar, 309 | baz, 310 | toto, 311 | tutu, 312 | tata 313 | 314 | # Bad variable names regexes, separated by a comma. If names match any regex, 315 | # they will always be refused 316 | bad-names-rgxs= 317 | 318 | # Naming style matching correct class attribute names. 319 | class-attribute-naming-style=any 320 | 321 | # Regular expression matching correct class attribute names. Overrides class- 322 | # attribute-naming-style. 323 | #class-attribute-rgx= 324 | 325 | # Naming style matching correct class names. 326 | class-naming-style=PascalCase 327 | 328 | # Regular expression matching correct class names. Overrides class-naming- 329 | # style. 330 | #class-rgx= 331 | 332 | # Naming style matching correct constant names. 333 | const-naming-style=UPPER_CASE 334 | 335 | # Regular expression matching correct constant names. Overrides const-naming- 336 | # style. 337 | #const-rgx= 338 | 339 | # Minimum line length for functions/classes that require docstrings, shorter 340 | # ones are exempt. 341 | docstring-min-length=-1 342 | 343 | # Naming style matching correct function names. 344 | function-naming-style=snake_case 345 | 346 | # Regular expression matching correct function names. Overrides function- 347 | # naming-style. 348 | #function-rgx= 349 | 350 | # Good variable names which should always be accepted, separated by a comma. 351 | good-names=_, 352 | d, 353 | df, 354 | e, 355 | f, 356 | g, 357 | i, 358 | j, 359 | k, 360 | kg, 361 | nm, 362 | ns, 363 | o, 364 | p, 365 | s, 366 | v, 367 | x 368 | 369 | # Good variable names regexes, separated by a comma. If names match any regex, 370 | # they will always be accepted 371 | good-names-rgxs= 372 | 373 | # Include a hint for the correct naming format with invalid-name. 374 | include-naming-hint=no 375 | 376 | # Naming style matching correct inline iteration names. 377 | inlinevar-naming-style=any 378 | 379 | # Regular expression matching correct inline iteration names. Overrides 380 | # inlinevar-naming-style. 381 | #inlinevar-rgx= 382 | 383 | # Naming style matching correct method names. 384 | method-naming-style=snake_case 385 | 386 | # Regular expression matching correct method names. Overrides method-naming- 387 | # style. 388 | #method-rgx= 389 | 390 | # Naming style matching correct module names. 391 | module-naming-style=snake_case 392 | 393 | # Regular expression matching correct module names. Overrides module-naming- 394 | # style. 395 | #module-rgx= 396 | 397 | # Colon-delimited sets of names that determine each other's naming style when 398 | # the name regexes allow several styles. 399 | name-group= 400 | 401 | # Regular expression which should only match function or class names that do 402 | # not require a docstring. 403 | no-docstring-rgx=^_ 404 | 405 | # List of decorators that produce properties, such as abc.abstractproperty. Add 406 | # to this list to register other decorators that produce valid properties. 407 | # These decorators are taken in consideration only for invalid-name. 408 | property-classes=abc.abstractproperty 409 | 410 | # Naming style matching correct variable names. 411 | variable-naming-style=snake_case 412 | 413 | # Regular expression matching correct variable names. Overrides variable- 414 | # naming-style. 415 | #variable-rgx= 416 | 417 | 418 | [STRING] 419 | 420 | # This flag controls whether inconsistent-quotes generates a warning when the 421 | # character used as a quote delimiter is used inconsistently within a module. 422 | check-quote-consistency=no 423 | 424 | # This flag controls whether the implicit-str-concat should generate a warning 425 | # on implicit string concatenation in sequences defined over several lines. 426 | check-str-concat-over-line-jumps=no 427 | 428 | 429 | [IMPORTS] 430 | 431 | # List of modules that can be imported at any level, not just the top level 432 | # one. 433 | allow-any-import-level= 434 | 435 | # Allow wildcard imports from modules that define __all__. 436 | allow-wildcard-with-all=no 437 | 438 | # Analyse import fallback blocks. This can be used to support both Python 2 and 439 | # 3 compatible code, which means that the block might have code that exists 440 | # only in one or another interpreter, leading to false positives when analysed. 441 | analyse-fallback-blocks=no 442 | 443 | # Deprecated modules which should not be used, separated by a comma. 444 | deprecated-modules=optparse,tkinter.tix 445 | 446 | # Create a graph of external dependencies in the given file (report RP0402 must 447 | # not be disabled). 448 | ext-import-graph= 449 | 450 | # Create a graph of every (i.e. internal and external) dependencies in the 451 | # given file (report RP0402 must not be disabled). 452 | import-graph= 453 | 454 | # Create a graph of internal dependencies in the given file (report RP0402 must 455 | # not be disabled). 456 | int-import-graph= 457 | 458 | # Force import order to recognize a module as part of the standard 459 | # compatibility libraries. 460 | known-standard-library= 461 | 462 | # Force import order to recognize a module as part of a third party library. 463 | known-third-party=enchant 464 | 465 | # Couples of modules and preferred modules, separated by a comma. 466 | preferred-modules= 467 | 468 | 469 | [CLASSES] 470 | 471 | # List of method names used to declare (i.e. assign) instance attributes. 472 | defining-attr-methods=__init__, 473 | __new__, 474 | setUp, 475 | __post_init__ 476 | 477 | # List of member names, which should be excluded from the protected access 478 | # warning. 479 | exclude-protected=_asdict, 480 | _fields, 481 | _replace, 482 | _source, 483 | _make 484 | 485 | # List of valid names for the first argument in a class method. 486 | valid-classmethod-first-arg=cls 487 | 488 | # List of valid names for the first argument in a metaclass class method. 489 | valid-metaclass-classmethod-first-arg=cls 490 | 491 | 492 | [DESIGN] 493 | 494 | # Maximum number of arguments for function / method. 495 | max-args=10 496 | 497 | # Maximum number of attributes for a class (see R0902). 498 | max-attributes=10 499 | 500 | # Maximum number of boolean expressions in an if statement (see R0916). 501 | max-bool-expr=5 502 | 503 | # Maximum number of branch for function / method body. 504 | max-branches=12 505 | 506 | # Maximum number of locals for function / method body. 507 | max-locals=20 508 | 509 | # Maximum number of parents for a class (see R0901). 510 | max-parents=7 511 | 512 | # Maximum number of public methods for a class (see R0904). 513 | max-public-methods=40 514 | 515 | # Maximum number of return / yield for function / method body. 516 | max-returns=6 517 | 518 | # Maximum number of statements in function / method body. 519 | max-statements=50 520 | 521 | # Minimum number of public methods for a class (see R0903). 522 | min-public-methods=1 523 | 524 | 525 | [EXCEPTIONS] 526 | 527 | # Exceptions that will emit a warning when being caught. Defaults to 528 | # "BaseException, Exception". 529 | overgeneral-exceptions=BaseException, 530 | Exception 531 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | bandit 2 | codespell 3 | coverage 4 | flask 5 | mkdocs-git-revision-date-plugin 6 | mkdocs-material 7 | mypy 8 | pre-commit 9 | pylint >= 2.7.0 10 | pytest 11 | pymdown-extensions 12 | twine 13 | wheel 14 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | Jinja2 >= 2.10.3 2 | PyYAML >= 5.1 3 | #kglab >= 0.4 4 | livereload >= 2.6.1 5 | mkdocs >= 1.0.4 6 | typer >= 0.3.2 7 | 8 | 9 | tornado>=6.3.2 # not directly required, pinned by Snyk to avoid a vulnerability 10 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # type: ignore 2 | 3 | import importlib.util 4 | import pathlib 5 | import setuptools 6 | import typing 7 | 8 | 9 | KEYWORDS = [ 10 | "apidocs", 11 | "bibliography", 12 | "documentation", 13 | "glossary", 14 | "kglab", 15 | "knowledge graph", 16 | "mkdocs", 17 | "plugin", 18 | "reference", 19 | ] 20 | 21 | 22 | def parse_requirements_file (filename: str) -> typing.List: 23 | """read and parse a Python `requirements.txt` file, returning as a list of str""" 24 | with pathlib.Path(filename).open() as f: 25 | return [ l.strip().replace(" ", "") for l in f.readlines() ] 26 | 27 | 28 | if __name__ == "__main__": 29 | spec = importlib.util.spec_from_file_location("mkrefs.version", "mkrefs/version.py") 30 | mkrefs_version = importlib.util.module_from_spec(spec) 31 | spec.loader.exec_module(mkrefs_version) 32 | 33 | setuptools.setup( 34 | name = "mkrefs", 35 | version = mkrefs_version.__version__, 36 | 37 | description = "MkDocs plugin to generate semantic reference Markdown pages", 38 | long_description = pathlib.Path("README.md").read_text(), 39 | long_description_content_type = "text/markdown", 40 | 41 | author = "Paco Nathan", 42 | author_email = "paco@derwen.ai", 43 | license = "MIT", 44 | url = "", 45 | 46 | python_requires = ">=3.6", 47 | packages = setuptools.find_packages(exclude=[ "docs" ]), 48 | install_requires = parse_requirements_file("requirements.txt"), 49 | 50 | entry_points = { 51 | "mkdocs.plugins": [ 52 | "mkrefs = mkrefs.plugin:MkRefsPlugin", 53 | ], 54 | 55 | "console_scripts": [ 56 | "mkrefs = mkrefs.cli:cli", 57 | ], 58 | }, 59 | 60 | keywords = ", ".join(KEYWORDS), 61 | classifiers = [ 62 | "Development Status :: 4 - Beta", 63 | "Intended Audience :: Developers", 64 | "Intended Audience :: Information Technology", 65 | "Intended Audience :: Science/Research", 66 | "License :: OSI Approved :: MIT License", 67 | "Programming Language :: Python", 68 | "Programming Language :: Python :: 3 :: Only", 69 | "Topic :: Documentation", 70 | "Topic :: Software Development :: Documentation", 71 | "Topic :: Text Processing :: Markup :: Markdown", 72 | ] 73 | ) 74 | --------------------------------------------------------------------------------