├── .codeclimate.yml ├── .coveragerc ├── .dockerignore ├── .gitignore ├── .pylintrc ├── .travis.yml ├── CONTRIBUTING.md ├── COPYING.txt ├── Dockerfile ├── MANIFEST.in ├── README.md ├── TERMS.md ├── docs ├── Makefile ├── conf.py ├── index.rst ├── make.bat ├── regcore.db.rst ├── regcore.management.commands.rst ├── regcore.management.rst ├── regcore.migrations.rst ├── regcore.rst ├── regcore.settings.rst ├── regcore.tests.management.commands.rst ├── regcore.tests.management.rst ├── regcore.tests.rst ├── regcore_pgsql.management.commands.rst ├── regcore_pgsql.management.rst ├── regcore_pgsql.migrations.rst ├── regcore_pgsql.rst ├── regcore_pgsql.tests.rst ├── regcore_read.rst ├── regcore_read.tests.rst ├── regcore_read.views.rst ├── regcore_write.rst ├── regcore_write.tests.rst └── regcore_write.views.rst ├── manage.py ├── regcore ├── __init__.py ├── db │ ├── __init__.py │ ├── django_models.py │ ├── es.py │ ├── interface.py │ └── storage.py ├── fields.py ├── index.py ├── layer.py ├── management │ ├── __init__.py │ └── commands │ │ ├── __init__.py │ │ └── import_docs.py ├── migrations │ ├── 0001_initial.py │ ├── 0002_mptt_add_fields.py │ ├── 0002_preamble.py │ ├── 0003_mptt_copy_children.py │ ├── 0004_mptt_delete_fields.py │ ├── 0005_merge.py │ ├── 0006_auto_20160314_1126.py │ ├── 0007_auto_20160314_1135.py │ ├── 0008_auto_20160314_1144.py │ ├── 0009_auto_20160322_1646.py │ ├── 0010_auto_20160322_1704.py │ ├── 0011_create_document.py │ ├── 0012_migrate_documents.py │ ├── 0013_remove_models.py │ ├── 0014_auto_20160504_0101.py │ └── __init__.py ├── models.py ├── responses.py ├── search_indexes.py ├── settings │ ├── __init__.py │ ├── base.py │ ├── elastic.py │ └── pgsql.py ├── templates │ └── search │ │ └── indexes │ │ └── regcore │ │ └── document_text.txt ├── tests │ ├── __init__.py │ ├── db_django_models_tests.py │ ├── db_es_tests.py │ ├── fields_tests.py │ ├── index_tests.py │ ├── layer_tests.py │ ├── management │ │ ├── __init__.py │ │ └── commands │ │ │ ├── __init__.py │ │ │ └── import_docs_tests.py │ ├── recipes.py │ └── responses_tests.py ├── urls.py └── urls_utils.py ├── regcore_pgsql ├── __init__.py ├── management │ ├── __init__.py │ └── commands │ │ ├── __init__.py │ │ └── rebuild_pgsql_index.py ├── migrations │ ├── 0001_initial.py │ ├── 0002_documentindex_doc_root.py │ └── __init__.py ├── models.py ├── tests │ ├── __init__.py │ ├── rebuild_pgsql_index_tests.py │ └── views_tests.py └── views.py ├── regcore_read ├── __init__.py ├── tests │ ├── __init__.py │ ├── urls.py │ ├── views_diff_tests.py │ ├── views_es_search_tests.py │ ├── views_haystack_search_tests.py │ ├── views_layer_tests.py │ ├── views_notice_tests.py │ ├── views_preamble_tests.py │ ├── views_regulation_tests.py │ └── views_seach_utils_tests.py └── views │ ├── __init__.py │ ├── diff.py │ ├── document.py │ ├── es_search.py │ ├── haystack_search.py │ ├── layer.py │ ├── notice.py │ └── search_utils.py ├── regcore_write ├── __init__.py ├── tests │ ├── __init__.py │ ├── views_diff_tests.py │ ├── views_layer_tests.py │ ├── views_notice_tests.py │ ├── views_preamble_tests.py │ ├── views_regulation_tests.py │ └── views_security_tests.py └── views │ ├── __init__.py │ ├── diff.py │ ├── document.py │ ├── layer.py │ ├── notice.py │ └── security.py ├── setup.py └── tox.ini /.codeclimate.yml: -------------------------------------------------------------------------------- 1 | engines: 2 | duplication: 3 | enabled: true 4 | config: 5 | languages: 6 | - python 7 | markdownlint: 8 | enabled: true 9 | pep8: 10 | enabled: true 11 | radon: 12 | enabled: true 13 | shellcheck: 14 | enabled: true 15 | ratings: 16 | paths: 17 | - "**.py" 18 | exclude_paths: [] 19 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [report] 2 | omit = 3 | bin/* 4 | develop-eggs/* 5 | eggs/* 6 | */site-packages/* 7 | exclude_lines = 8 | pragma: no cover 9 | 10 | # Don't complain if tests don't hit defensive assertion code: 11 | raise AssertionError 12 | raise NotImplementedError 13 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | *.py[co] 2 | *.swp 3 | eregs.db 4 | regcore.egg-info/ 5 | .coverage 6 | .tox 7 | # local_settings.py 8 | docs/_build/ 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[co] 2 | *.swp 3 | eregs.db 4 | regcore.egg-info/ 5 | .coverage 6 | .tox 7 | local_settings.py 8 | docs/_build/ 9 | -------------------------------------------------------------------------------- /.pylintrc: -------------------------------------------------------------------------------- 1 | [TYPECHECK] 2 | generated-members=objects 3 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | sudo: false 3 | python: 4 | - "2.7" 5 | - "3.4" 6 | - "3.5" 7 | - "3.6" 8 | install: 9 | - pip install coveralls tox-travis 10 | script: 11 | - tox 12 | after_success: 13 | - if [[ $INTEGRATION_TARGET = '' ]] && [[ $TRAVIS_PYTHON_VERSION = '3.6' ]]; then coveralls; fi 14 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ## Public domain 2 | 3 | The project is in the public domain within the United States, and 4 | copyright and related rights in the work worldwide are waived through 5 | the [CC0 1.0 Universal public domain dedication][CC0]. 6 | 7 | All contributions to this project will be released under the CC0 8 | dedication. By submitting a pull request, you are agreeing to comply 9 | with this waiver of copyright interest. 10 | 11 | [CC0]: http://creativecommons.org/publicdomain/zero/1.0/ 12 | 13 | -------------------------------------------------------------------------------- /COPYING.txt: -------------------------------------------------------------------------------- 1 | Creative Commons Legal Code 2 | 3 | CC0 1.0 Universal 4 | 5 | CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE 6 | LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN 7 | ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS 8 | INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES 9 | REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS 10 | PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM 11 | THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED 12 | HEREUNDER. 13 | 14 | Statement of Purpose 15 | 16 | The laws of most jurisdictions throughout the world automatically confer 17 | exclusive Copyright and Related Rights (defined below) upon the creator 18 | and subsequent owner(s) (each and all, an "owner") of an original work of 19 | authorship and/or a database (each, a "Work"). 20 | 21 | Certain owners wish to permanently relinquish those rights to a Work for 22 | the purpose of contributing to a commons of creative, cultural and 23 | scientific works ("Commons") that the public can reliably and without fear 24 | of later claims of infringement build upon, modify, incorporate in other 25 | works, reuse and redistribute as freely as possible in any form whatsoever 26 | and for any purposes, including without limitation commercial purposes. 27 | These owners may contribute to the Commons to promote the ideal of a free 28 | culture and the further production of creative, cultural and scientific 29 | works, or to gain reputation or greater distribution for their Work in 30 | part through the use and efforts of others. 31 | 32 | For these and/or other purposes and motivations, and without any 33 | expectation of additional consideration or compensation, the person 34 | associating CC0 with a Work (the "Affirmer"), to the extent that he or she 35 | is an owner of Copyright and Related Rights in the Work, voluntarily 36 | elects to apply CC0 to the Work and publicly distribute the Work under its 37 | terms, with knowledge of his or her Copyright and Related Rights in the 38 | Work and the meaning and intended legal effect of CC0 on those rights. 39 | 40 | 1. Copyright and Related Rights. A Work made available under CC0 may be 41 | protected by copyright and related or neighboring rights ("Copyright and 42 | Related Rights"). Copyright and Related Rights include, but are not 43 | limited to, the following: 44 | 45 | i. the right to reproduce, adapt, distribute, perform, display, 46 | communicate, and translate a Work; 47 | ii. moral rights retained by the original author(s) and/or performer(s); 48 | iii. publicity and privacy rights pertaining to a person's image or 49 | likeness depicted in a Work; 50 | iv. rights protecting against unfair competition in regards to a Work, 51 | subject to the limitations in paragraph 4(a), below; 52 | v. rights protecting the extraction, dissemination, use and reuse of data 53 | in a Work; 54 | vi. database rights (such as those arising under Directive 96/9/EC of the 55 | European Parliament and of the Council of 11 March 1996 on the legal 56 | protection of databases, and under any national implementation 57 | thereof, including any amended or successor version of such 58 | directive); and 59 | vii. other similar, equivalent or corresponding rights throughout the 60 | world based on applicable law or treaty, and any national 61 | implementations thereof. 62 | 63 | 2. Waiver. To the greatest extent permitted by, but not in contravention 64 | of, applicable law, Affirmer hereby overtly, fully, permanently, 65 | irrevocably and unconditionally waives, abandons, and surrenders all of 66 | Affirmer's Copyright and Related Rights and associated claims and causes 67 | of action, whether now known or unknown (including existing as well as 68 | future claims and causes of action), in the Work (i) in all territories 69 | worldwide, (ii) for the maximum duration provided by applicable law or 70 | treaty (including future time extensions), (iii) in any current or future 71 | medium and for any number of copies, and (iv) for any purpose whatsoever, 72 | including without limitation commercial, advertising or promotional 73 | purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each 74 | member of the public at large and to the detriment of Affirmer's heirs and 75 | successors, fully intending that such Waiver shall not be subject to 76 | revocation, rescission, cancellation, termination, or any other legal or 77 | equitable action to disrupt the quiet enjoyment of the Work by the public 78 | as contemplated by Affirmer's express Statement of Purpose. 79 | 80 | 3. Public License Fallback. Should any part of the Waiver for any reason 81 | be judged legally invalid or ineffective under applicable law, then the 82 | Waiver shall be preserved to the maximum extent permitted taking into 83 | account Affirmer's express Statement of Purpose. In addition, to the 84 | extent the Waiver is so judged Affirmer hereby grants to each affected 85 | person a royalty-free, non transferable, non sublicensable, non exclusive, 86 | irrevocable and unconditional license to exercise Affirmer's Copyright and 87 | Related Rights in the Work (i) in all territories worldwide, (ii) for the 88 | maximum duration provided by applicable law or treaty (including future 89 | time extensions), (iii) in any current or future medium and for any number 90 | of copies, and (iv) for any purpose whatsoever, including without 91 | limitation commercial, advertising or promotional purposes (the 92 | "License"). The License shall be deemed effective as of the date CC0 was 93 | applied by Affirmer to the Work. Should any part of the License for any 94 | reason be judged legally invalid or ineffective under applicable law, such 95 | partial invalidity or ineffectiveness shall not invalidate the remainder 96 | of the License, and in such case Affirmer hereby affirms that he or she 97 | will not (i) exercise any of his or her remaining Copyright and Related 98 | Rights in the Work or (ii) assert any associated claims and causes of 99 | action with respect to the Work, in either case contrary to Affirmer's 100 | express Statement of Purpose. 101 | 102 | 4. Limitations and Disclaimers. 103 | 104 | a. No trademark or patent rights held by Affirmer are waived, abandoned, 105 | surrendered, licensed or otherwise affected by this document. 106 | b. Affirmer offers the Work as-is and makes no representations or 107 | warranties of any kind concerning the Work, express, implied, 108 | statutory or otherwise, including without limitation warranties of 109 | title, merchantability, fitness for a particular purpose, non 110 | infringement, or the absence of latent or other defects, accuracy, or 111 | the present or absence of errors, whether or not discoverable, all to 112 | the greatest extent permissible under applicable law. 113 | c. Affirmer disclaims responsibility for clearing rights of other persons 114 | that may apply to the Work or any use thereof, including without 115 | limitation any person's Copyright and Related Rights in the Work. 116 | Further, Affirmer disclaims responsibility for obtaining any necessary 117 | consents, permissions or other rights required for any use of the 118 | Work. 119 | d. Affirmer understands and acknowledges that Creative Commons is not a 120 | party to this document and has no duty or obligation with respect to 121 | this CC0 or use of the Work. 122 | 123 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3-alpine 2 | 3 | COPY [".", "/app/src/"] 4 | 5 | WORKDIR /app/src/ 6 | RUN apk --update add ca-certificates \ 7 | && update-ca-certificates \ 8 | && rm -rf /var/cache/apk/* 9 | RUN pip install --no-cache-dir --upgrade pip \ 10 | && pip install --no-cache-dir -e .[backend-haystack] \ 11 | && python manage.py migrate 12 | 13 | ENV PYTHONUNBUFFERED="1" 14 | EXPOSE 8080 15 | 16 | CMD ["python", "manage.py", "runserver", "0.0.0.0:8080"] 17 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.md 2 | include TERMS.md 3 | recursive-include regcore/templates *.txt 4 | -------------------------------------------------------------------------------- /TERMS.md: -------------------------------------------------------------------------------- 1 | As a work of the United States Government, this package is in the 2 | public domain within the United States. Additionally, we waive 3 | copyright and related rights in the work worldwide through the CC0 1.0 4 | Universal public domain dedication. 5 | 6 | ## CC0 1.0 Universal Summary 7 | 8 | This is a human-readable summary of the 9 | [Legal Code (read the full text)](http://creativecommons.org/publicdomain/zero/1.0/legalcode). 10 | 11 | ### No Copyright 12 | 13 | The person who associated a work with this deed has dedicated the work to 14 | the public domain by waiving all of his or her rights to the work worldwide 15 | under copyright law, including all related and neighboring rights, to the 16 | extent allowed by law. 17 | 18 | You can copy, modify, distribute and perform the work, even for commercial 19 | purposes, all without asking permission. See Other Information below. 20 | 21 | ### Other Information 22 | 23 | In no way are the patent or trademark rights of any person affected by CC0, 24 | nor are the rights that other persons may have in the work or in how the 25 | work is used, such as publicity or privacy rights. 26 | 27 | Unless expressly stated otherwise, the person who associated a work with 28 | this deed makes no warranties about the work, and disclaims liability for 29 | all uses of the work, to the fullest extent permitted by applicable law. 30 | When using or citing the work, you should not imply endorsement by the 31 | author or the affirmer. 32 | 33 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = /home/vagrant/regulations-core/bin/sphinx-build 7 | PAPER = 8 | BUILDDIR = /home/vagrant/regulations-core/docs 9 | 10 | # Internal variables. 11 | PAPEROPT_a4 = -D latex_paper_size=a4 12 | PAPEROPT_letter = -D latex_paper_size=letter 13 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) /home/vagrant/regulations-core/docs-source 14 | # the i18n builder cannot share the environment and doctrees with the others 15 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) /home/vagrant/regulations-core/docs-source 16 | 17 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 18 | 19 | help: 20 | @echo "Please use \`make ' where is one of" 21 | @echo " html to make standalone HTML files" 22 | @echo " warnings-html to make standalone HTML files (warnings become errors)" 23 | @echo " dirhtml to make HTML files named index.html in directories" 24 | @echo " singlehtml to make a single large HTML file" 25 | @echo " pickle to make pickle files" 26 | @echo " json to make JSON files" 27 | @echo " htmlhelp to make HTML files and a HTML help project" 28 | @echo " qthelp to make HTML files and a qthelp project" 29 | @echo " devhelp to make HTML files and a Devhelp project" 30 | @echo " epub to make an epub" 31 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 32 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 33 | @echo " text to make text files" 34 | @echo " man to make manual pages" 35 | @echo " texinfo to make Texinfo files" 36 | @echo " info to make Texinfo files and run them through makeinfo" 37 | @echo " gettext to make PO message catalogs" 38 | @echo " changes to make an overview of all changed/added/deprecated items" 39 | @echo " linkcheck to check all external links for integrity" 40 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 41 | 42 | clean: 43 | -rm -rf $(BUILDDIR)/* 44 | 45 | html: 46 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 47 | @echo 48 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 49 | 50 | warnings-html: 51 | $(SPHINXBUILD) -W -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 52 | @echo 53 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 54 | 55 | dirhtml: 56 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 57 | @echo 58 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 59 | 60 | singlehtml: 61 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 62 | @echo 63 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 64 | 65 | pickle: 66 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 67 | @echo 68 | @echo "Build finished; now you can process the pickle files." 69 | 70 | json: 71 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 72 | @echo 73 | @echo "Build finished; now you can process the JSON files." 74 | 75 | htmlhelp: 76 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 77 | @echo 78 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 79 | ".hhp project file in $(BUILDDIR)/htmlhelp." 80 | 81 | qthelp: 82 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 83 | @echo 84 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 85 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 86 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/sphinxbuilder.qhcp" 87 | @echo "To view the help file:" 88 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/sphinxbuilder.qhc" 89 | 90 | devhelp: 91 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 92 | @echo 93 | @echo "Build finished." 94 | @echo "To view the help file:" 95 | @echo "# mkdir -p $$HOME/.local/share/devhelp/sphinxbuilder" 96 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/sphinxbuilder" 97 | @echo "# devhelp" 98 | 99 | epub: 100 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 101 | @echo 102 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 103 | 104 | latex: 105 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 106 | @echo 107 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 108 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 109 | "(use \`make latexpdf' here to do that automatically)." 110 | 111 | latexpdf: 112 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 113 | @echo "Running LaTeX files through pdflatex..." 114 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 115 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 116 | 117 | text: 118 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 119 | @echo 120 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 121 | 122 | man: 123 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 124 | @echo 125 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 126 | 127 | texinfo: 128 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 129 | @echo 130 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 131 | @echo "Run \`make' in that directory to run these through makeinfo" \ 132 | "(use \`make info' here to do that automatically)." 133 | 134 | info: 135 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 136 | @echo "Running Texinfo files through makeinfo..." 137 | make -C $(BUILDDIR)/texinfo info 138 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 139 | 140 | gettext: 141 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 142 | @echo 143 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 144 | 145 | changes: 146 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 147 | @echo 148 | @echo "The overview file is in $(BUILDDIR)/changes." 149 | 150 | linkcheck: 151 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 152 | @echo 153 | @echo "Link check complete; look for any errors in the above output " \ 154 | "or in $(BUILDDIR)/linkcheck/output.txt." 155 | 156 | doctest: 157 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 158 | @echo "Testing of doctests in the sources finished, look at the " \ 159 | "results in $(BUILDDIR)/doctest/output.txt." 160 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. regcore documentation master file, created by 2 | sphinx-quickstart on Fri Oct 4 13:26:04 2013. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to regcore's documentation! 7 | =================================== 8 | 9 | Contents: 10 | 11 | .. toctree:: 12 | :maxdepth: 4 13 | 14 | regcore 15 | regcore_read 16 | regcore_write 17 | 18 | 19 | Indices and tables 20 | ================== 21 | 22 | * :ref:`genindex` 23 | * :ref:`modindex` 24 | * :ref:`search` 25 | 26 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | REM Command file for Sphinx documentation 4 | 5 | if "%SPHINXBUILD%" == "" ( 6 | set SPHINXBUILD=sphinx-build 7 | ) 8 | set BUILDDIR=/home/vagrant/regulations-core/docs 9 | set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% /home/vagrant/regulations-core/docs-source 10 | set I18NSPHINXOPTS=%SPHINXOPTS% /home/vagrant/regulations-core/docs-source 11 | if NOT "%PAPER%" == "" ( 12 | set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% 13 | set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% 14 | ) 15 | 16 | if "%1" == "" goto help 17 | 18 | if "%1" == "help" ( 19 | :help 20 | echo.Please use `make ^` where ^ is one of 21 | echo. html to make standalone HTML files 22 | echo. warnings-html to make standalone HTML files (turn warnings into errors) 23 | echo. dirhtml to make HTML files named index.html in directories 24 | echo. singlehtml to make a single large HTML file 25 | echo. pickle to make pickle files 26 | echo. json to make JSON files 27 | echo. htmlhelp to make HTML files and a HTML help project 28 | echo. qthelp to make HTML files and a qthelp project 29 | echo. devhelp to make HTML files and a Devhelp project 30 | echo. epub to make an epub 31 | echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter 32 | echo. text to make text files 33 | echo. man to make manual pages 34 | echo. texinfo to make Texinfo files 35 | echo. gettext to make PO message catalogs 36 | echo. changes to make an overview over all changed/added/deprecated items 37 | echo. linkcheck to check all external links for integrity 38 | echo. doctest to run all doctests embedded in the documentation if enabled 39 | goto end 40 | ) 41 | 42 | if "%1" == "clean" ( 43 | for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i 44 | del /q /s %BUILDDIR%\* 45 | goto end 46 | ) 47 | 48 | if "%1" == "html" ( 49 | %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html 50 | if errorlevel 1 exit /b 1 51 | echo. 52 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 53 | goto end 54 | ) 55 | 56 | if "%1" == "warnings-html" ( 57 | %SPHINXBUILD% -W -b html %ALLSPHINXOPTS% %BUILDDIR%/html 58 | if errorlevel 1 exit /b 1 59 | echo. 60 | echo.Build finished. The HTML pages are in %BUILDDIR%/html. 61 | goto end 62 | ) 63 | 64 | if "%1" == "dirhtml" ( 65 | %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml 66 | if errorlevel 1 exit /b 1 67 | echo. 68 | echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. 69 | goto end 70 | ) 71 | 72 | if "%1" == "singlehtml" ( 73 | %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml 74 | if errorlevel 1 exit /b 1 75 | echo. 76 | echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. 77 | goto end 78 | ) 79 | 80 | if "%1" == "pickle" ( 81 | %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle 82 | if errorlevel 1 exit /b 1 83 | echo. 84 | echo.Build finished; now you can process the pickle files. 85 | goto end 86 | ) 87 | 88 | if "%1" == "json" ( 89 | %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json 90 | if errorlevel 1 exit /b 1 91 | echo. 92 | echo.Build finished; now you can process the JSON files. 93 | goto end 94 | ) 95 | 96 | if "%1" == "htmlhelp" ( 97 | %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp 98 | if errorlevel 1 exit /b 1 99 | echo. 100 | echo.Build finished; now you can run HTML Help Workshop with the ^ 101 | .hhp project file in %BUILDDIR%/htmlhelp. 102 | goto end 103 | ) 104 | 105 | if "%1" == "qthelp" ( 106 | %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp 107 | if errorlevel 1 exit /b 1 108 | echo. 109 | echo.Build finished; now you can run "qcollectiongenerator" with the ^ 110 | .qhcp project file in %BUILDDIR%/qthelp, like this: 111 | echo.^> qcollectiongenerator %BUILDDIR%\qthelp\sphinxbuilder.qhcp 112 | echo.To view the help file: 113 | echo.^> assistant -collectionFile %BUILDDIR%\qthelp\sphinxbuilder.ghc 114 | goto end 115 | ) 116 | 117 | if "%1" == "devhelp" ( 118 | %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp 119 | if errorlevel 1 exit /b 1 120 | echo. 121 | echo.Build finished. 122 | goto end 123 | ) 124 | 125 | if "%1" == "epub" ( 126 | %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub 127 | if errorlevel 1 exit /b 1 128 | echo. 129 | echo.Build finished. The epub file is in %BUILDDIR%/epub. 130 | goto end 131 | ) 132 | 133 | if "%1" == "latex" ( 134 | %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex 135 | if errorlevel 1 exit /b 1 136 | echo. 137 | echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. 138 | goto end 139 | ) 140 | 141 | if "%1" == "text" ( 142 | %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text 143 | if errorlevel 1 exit /b 1 144 | echo. 145 | echo.Build finished. The text files are in %BUILDDIR%/text. 146 | goto end 147 | ) 148 | 149 | if "%1" == "man" ( 150 | %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man 151 | if errorlevel 1 exit /b 1 152 | echo. 153 | echo.Build finished. The manual pages are in %BUILDDIR%/man. 154 | goto end 155 | ) 156 | 157 | if "%1" == "texinfo" ( 158 | %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo 159 | if errorlevel 1 exit /b 1 160 | echo. 161 | echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. 162 | goto end 163 | ) 164 | 165 | if "%1" == "gettext" ( 166 | %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale 167 | if errorlevel 1 exit /b 1 168 | echo. 169 | echo.Build finished. The message catalogs are in %BUILDDIR%/locale. 170 | goto end 171 | ) 172 | 173 | if "%1" == "changes" ( 174 | %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes 175 | if errorlevel 1 exit /b 1 176 | echo. 177 | echo.The overview file is in %BUILDDIR%/changes. 178 | goto end 179 | ) 180 | 181 | if "%1" == "linkcheck" ( 182 | %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck 183 | if errorlevel 1 exit /b 1 184 | echo. 185 | echo.Link check complete; look for any errors in the above output ^ 186 | or in %BUILDDIR%/linkcheck/output.txt. 187 | goto end 188 | ) 189 | 190 | if "%1" == "doctest" ( 191 | %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest 192 | if errorlevel 1 exit /b 1 193 | echo. 194 | echo.Testing of doctests in the sources finished, look at the ^ 195 | results in %BUILDDIR%/doctest/output.txt. 196 | goto end 197 | ) 198 | 199 | :end 200 | -------------------------------------------------------------------------------- /docs/regcore.db.rst: -------------------------------------------------------------------------------- 1 | regcore\.db package 2 | =================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | regcore\.db\.django\_models module 8 | ---------------------------------- 9 | 10 | .. automodule:: regcore.db.django_models 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | regcore\.db\.es module 16 | ---------------------- 17 | 18 | .. automodule:: regcore.db.es 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | regcore\.db\.interface module 24 | ----------------------------- 25 | 26 | .. automodule:: regcore.db.interface 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | regcore\.db\.storage module 32 | --------------------------- 33 | 34 | .. automodule:: regcore.db.storage 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | 40 | Module contents 41 | --------------- 42 | 43 | .. automodule:: regcore.db 44 | :members: 45 | :undoc-members: 46 | :show-inheritance: 47 | -------------------------------------------------------------------------------- /docs/regcore.management.commands.rst: -------------------------------------------------------------------------------- 1 | regcore\.management\.commands package 2 | ===================================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | regcore\.management\.commands\.import\_docs module 8 | -------------------------------------------------- 9 | 10 | .. automodule:: regcore.management.commands.import_docs 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | 16 | Module contents 17 | --------------- 18 | 19 | .. automodule:: regcore.management.commands 20 | :members: 21 | :undoc-members: 22 | :show-inheritance: 23 | -------------------------------------------------------------------------------- /docs/regcore.management.rst: -------------------------------------------------------------------------------- 1 | regcore\.management package 2 | =========================== 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | 9 | regcore.management.commands 10 | 11 | Module contents 12 | --------------- 13 | 14 | .. automodule:: regcore.management 15 | :members: 16 | :undoc-members: 17 | :show-inheritance: 18 | -------------------------------------------------------------------------------- /docs/regcore.migrations.rst: -------------------------------------------------------------------------------- 1 | regcore\.migrations package 2 | =========================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | regcore\.migrations\.0001\_initial module 8 | ----------------------------------------- 9 | 10 | .. automodule:: regcore.migrations.0001_initial 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | regcore\.migrations\.0002\_mptt\_add\_fields module 16 | --------------------------------------------------- 17 | 18 | .. automodule:: regcore.migrations.0002_mptt_add_fields 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | regcore\.migrations\.0002\_preamble module 24 | ------------------------------------------ 25 | 26 | .. automodule:: regcore.migrations.0002_preamble 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | regcore\.migrations\.0003\_mptt\_copy\_children module 32 | ------------------------------------------------------ 33 | 34 | .. automodule:: regcore.migrations.0003_mptt_copy_children 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | regcore\.migrations\.0004\_mptt\_delete\_fields module 40 | ------------------------------------------------------ 41 | 42 | .. automodule:: regcore.migrations.0004_mptt_delete_fields 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | regcore\.migrations\.0005\_merge module 48 | --------------------------------------- 49 | 50 | .. automodule:: regcore.migrations.0005_merge 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | 55 | regcore\.migrations\.0006\_auto\_20160314\_1126 module 56 | ------------------------------------------------------ 57 | 58 | .. automodule:: regcore.migrations.0006_auto_20160314_1126 59 | :members: 60 | :undoc-members: 61 | :show-inheritance: 62 | 63 | regcore\.migrations\.0007\_auto\_20160314\_1135 module 64 | ------------------------------------------------------ 65 | 66 | .. automodule:: regcore.migrations.0007_auto_20160314_1135 67 | :members: 68 | :undoc-members: 69 | :show-inheritance: 70 | 71 | regcore\.migrations\.0008\_auto\_20160314\_1144 module 72 | ------------------------------------------------------ 73 | 74 | .. automodule:: regcore.migrations.0008_auto_20160314_1144 75 | :members: 76 | :undoc-members: 77 | :show-inheritance: 78 | 79 | regcore\.migrations\.0009\_auto\_20160322\_1646 module 80 | ------------------------------------------------------ 81 | 82 | .. automodule:: regcore.migrations.0009_auto_20160322_1646 83 | :members: 84 | :undoc-members: 85 | :show-inheritance: 86 | 87 | regcore\.migrations\.0010\_auto\_20160322\_1704 module 88 | ------------------------------------------------------ 89 | 90 | .. automodule:: regcore.migrations.0010_auto_20160322_1704 91 | :members: 92 | :undoc-members: 93 | :show-inheritance: 94 | 95 | regcore\.migrations\.0011\_create\_document module 96 | -------------------------------------------------- 97 | 98 | .. automodule:: regcore.migrations.0011_create_document 99 | :members: 100 | :undoc-members: 101 | :show-inheritance: 102 | 103 | regcore\.migrations\.0012\_migrate\_documents module 104 | ---------------------------------------------------- 105 | 106 | .. automodule:: regcore.migrations.0012_migrate_documents 107 | :members: 108 | :undoc-members: 109 | :show-inheritance: 110 | 111 | regcore\.migrations\.0013\_remove\_models module 112 | ------------------------------------------------ 113 | 114 | .. automodule:: regcore.migrations.0013_remove_models 115 | :members: 116 | :undoc-members: 117 | :show-inheritance: 118 | 119 | regcore\.migrations\.0014\_auto\_20160504\_0101 module 120 | ------------------------------------------------------ 121 | 122 | .. automodule:: regcore.migrations.0014_auto_20160504_0101 123 | :members: 124 | :undoc-members: 125 | :show-inheritance: 126 | 127 | 128 | Module contents 129 | --------------- 130 | 131 | .. automodule:: regcore.migrations 132 | :members: 133 | :undoc-members: 134 | :show-inheritance: 135 | -------------------------------------------------------------------------------- /docs/regcore.rst: -------------------------------------------------------------------------------- 1 | regcore package 2 | =============== 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | 9 | regcore.db 10 | regcore.management 11 | regcore.migrations 12 | regcore.settings 13 | regcore.tests 14 | 15 | Submodules 16 | ---------- 17 | 18 | regcore\.fields module 19 | ---------------------- 20 | 21 | .. automodule:: regcore.fields 22 | :members: 23 | :undoc-members: 24 | :show-inheritance: 25 | 26 | regcore\.index module 27 | --------------------- 28 | 29 | .. automodule:: regcore.index 30 | :members: 31 | :undoc-members: 32 | :show-inheritance: 33 | 34 | regcore\.layer module 35 | --------------------- 36 | 37 | .. automodule:: regcore.layer 38 | :members: 39 | :undoc-members: 40 | :show-inheritance: 41 | 42 | regcore\.models module 43 | ---------------------- 44 | 45 | .. automodule:: regcore.models 46 | :members: 47 | :undoc-members: 48 | :show-inheritance: 49 | 50 | regcore\.responses module 51 | ------------------------- 52 | 53 | .. automodule:: regcore.responses 54 | :members: 55 | :undoc-members: 56 | :show-inheritance: 57 | 58 | regcore\.search\_indexes module 59 | ------------------------------- 60 | 61 | .. automodule:: regcore.search_indexes 62 | :members: 63 | :undoc-members: 64 | :show-inheritance: 65 | 66 | regcore\.urls module 67 | -------------------- 68 | 69 | .. automodule:: regcore.urls 70 | :members: 71 | :undoc-members: 72 | :show-inheritance: 73 | 74 | regcore\.urls\_utils module 75 | --------------------------- 76 | 77 | .. automodule:: regcore.urls_utils 78 | :members: 79 | :undoc-members: 80 | :show-inheritance: 81 | 82 | 83 | Module contents 84 | --------------- 85 | 86 | .. automodule:: regcore 87 | :members: 88 | :undoc-members: 89 | :show-inheritance: 90 | -------------------------------------------------------------------------------- /docs/regcore.settings.rst: -------------------------------------------------------------------------------- 1 | regcore\.settings package 2 | ========================= 3 | 4 | Submodules 5 | ---------- 6 | 7 | regcore\.settings\.base module 8 | ------------------------------ 9 | 10 | .. automodule:: regcore.settings.base 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | regcore\.settings\.elastic module 16 | --------------------------------- 17 | 18 | .. automodule:: regcore.settings.elastic 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | regcore\.settings\.pgsql module 24 | ------------------------------- 25 | 26 | .. automodule:: regcore.settings.pgsql 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | 32 | Module contents 33 | --------------- 34 | 35 | .. automodule:: regcore.settings 36 | :members: 37 | :undoc-members: 38 | :show-inheritance: 39 | -------------------------------------------------------------------------------- /docs/regcore.tests.management.commands.rst: -------------------------------------------------------------------------------- 1 | regcore\.tests\.management\.commands package 2 | ============================================ 3 | 4 | Submodules 5 | ---------- 6 | 7 | regcore\.tests\.management\.commands\.import\_docs\_tests module 8 | ---------------------------------------------------------------- 9 | 10 | .. automodule:: regcore.tests.management.commands.import_docs_tests 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | 16 | Module contents 17 | --------------- 18 | 19 | .. automodule:: regcore.tests.management.commands 20 | :members: 21 | :undoc-members: 22 | :show-inheritance: 23 | -------------------------------------------------------------------------------- /docs/regcore.tests.management.rst: -------------------------------------------------------------------------------- 1 | regcore\.tests\.management package 2 | ================================== 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | 9 | regcore.tests.management.commands 10 | 11 | Module contents 12 | --------------- 13 | 14 | .. automodule:: regcore.tests.management 15 | :members: 16 | :undoc-members: 17 | :show-inheritance: 18 | -------------------------------------------------------------------------------- /docs/regcore.tests.rst: -------------------------------------------------------------------------------- 1 | regcore\.tests package 2 | ====================== 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | 9 | regcore.tests.management 10 | 11 | Submodules 12 | ---------- 13 | 14 | regcore\.tests\.db\_django\_models\_tests module 15 | ------------------------------------------------ 16 | 17 | .. automodule:: regcore.tests.db_django_models_tests 18 | :members: 19 | :undoc-members: 20 | :show-inheritance: 21 | 22 | regcore\.tests\.db\_es\_tests module 23 | ------------------------------------ 24 | 25 | .. automodule:: regcore.tests.db_es_tests 26 | :members: 27 | :undoc-members: 28 | :show-inheritance: 29 | 30 | regcore\.tests\.fields\_tests module 31 | ------------------------------------ 32 | 33 | .. automodule:: regcore.tests.fields_tests 34 | :members: 35 | :undoc-members: 36 | :show-inheritance: 37 | 38 | regcore\.tests\.index\_tests module 39 | ----------------------------------- 40 | 41 | .. automodule:: regcore.tests.index_tests 42 | :members: 43 | :undoc-members: 44 | :show-inheritance: 45 | 46 | regcore\.tests\.layer\_tests module 47 | ----------------------------------- 48 | 49 | .. automodule:: regcore.tests.layer_tests 50 | :members: 51 | :undoc-members: 52 | :show-inheritance: 53 | 54 | regcore\.tests\.recipes module 55 | ------------------------------ 56 | 57 | .. automodule:: regcore.tests.recipes 58 | :members: 59 | :undoc-members: 60 | :show-inheritance: 61 | 62 | regcore\.tests\.responses\_tests module 63 | --------------------------------------- 64 | 65 | .. automodule:: regcore.tests.responses_tests 66 | :members: 67 | :undoc-members: 68 | :show-inheritance: 69 | 70 | 71 | Module contents 72 | --------------- 73 | 74 | .. automodule:: regcore.tests 75 | :members: 76 | :undoc-members: 77 | :show-inheritance: 78 | -------------------------------------------------------------------------------- /docs/regcore_pgsql.management.commands.rst: -------------------------------------------------------------------------------- 1 | regcore\_pgsql\.management\.commands package 2 | ============================================ 3 | 4 | Submodules 5 | ---------- 6 | 7 | regcore\_pgsql\.management\.commands\.rebuild\_pgsql\_index module 8 | ------------------------------------------------------------------ 9 | 10 | .. automodule:: regcore_pgsql.management.commands.rebuild_pgsql_index 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | 16 | Module contents 17 | --------------- 18 | 19 | .. automodule:: regcore_pgsql.management.commands 20 | :members: 21 | :undoc-members: 22 | :show-inheritance: 23 | -------------------------------------------------------------------------------- /docs/regcore_pgsql.management.rst: -------------------------------------------------------------------------------- 1 | regcore\_pgsql\.management package 2 | ================================== 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | 9 | regcore_pgsql.management.commands 10 | 11 | Module contents 12 | --------------- 13 | 14 | .. automodule:: regcore_pgsql.management 15 | :members: 16 | :undoc-members: 17 | :show-inheritance: 18 | -------------------------------------------------------------------------------- /docs/regcore_pgsql.migrations.rst: -------------------------------------------------------------------------------- 1 | regcore\_pgsql\.migrations package 2 | ================================== 3 | 4 | Submodules 5 | ---------- 6 | 7 | regcore\_pgsql\.migrations\.0001\_initial module 8 | ------------------------------------------------ 9 | 10 | .. automodule:: regcore_pgsql.migrations.0001_initial 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | regcore\_pgsql\.migrations\.0002\_documentindex\_doc\_root module 16 | ----------------------------------------------------------------- 17 | 18 | .. automodule:: regcore_pgsql.migrations.0002_documentindex_doc_root 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | 24 | Module contents 25 | --------------- 26 | 27 | .. automodule:: regcore_pgsql.migrations 28 | :members: 29 | :undoc-members: 30 | :show-inheritance: 31 | -------------------------------------------------------------------------------- /docs/regcore_pgsql.rst: -------------------------------------------------------------------------------- 1 | regcore\_pgsql package 2 | ====================== 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | 9 | regcore_pgsql.management 10 | regcore_pgsql.migrations 11 | regcore_pgsql.tests 12 | 13 | Submodules 14 | ---------- 15 | 16 | regcore\_pgsql\.models module 17 | ----------------------------- 18 | 19 | .. automodule:: regcore_pgsql.models 20 | :members: 21 | :undoc-members: 22 | :show-inheritance: 23 | 24 | regcore\_pgsql\.views module 25 | ---------------------------- 26 | 27 | .. automodule:: regcore_pgsql.views 28 | :members: 29 | :undoc-members: 30 | :show-inheritance: 31 | 32 | 33 | Module contents 34 | --------------- 35 | 36 | .. automodule:: regcore_pgsql 37 | :members: 38 | :undoc-members: 39 | :show-inheritance: 40 | -------------------------------------------------------------------------------- /docs/regcore_pgsql.tests.rst: -------------------------------------------------------------------------------- 1 | regcore\_pgsql\.tests package 2 | ============================= 3 | 4 | Submodules 5 | ---------- 6 | 7 | regcore\_pgsql\.tests\.rebuild\_pgsql\_index\_tests module 8 | ---------------------------------------------------------- 9 | 10 | .. automodule:: regcore_pgsql.tests.rebuild_pgsql_index_tests 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | regcore\_pgsql\.tests\.views\_tests module 16 | ------------------------------------------ 17 | 18 | .. automodule:: regcore_pgsql.tests.views_tests 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | 24 | Module contents 25 | --------------- 26 | 27 | .. automodule:: regcore_pgsql.tests 28 | :members: 29 | :undoc-members: 30 | :show-inheritance: 31 | -------------------------------------------------------------------------------- /docs/regcore_read.rst: -------------------------------------------------------------------------------- 1 | regcore\_read package 2 | ===================== 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | 9 | regcore_read.tests 10 | regcore_read.views 11 | 12 | Module contents 13 | --------------- 14 | 15 | .. automodule:: regcore_read 16 | :members: 17 | :undoc-members: 18 | :show-inheritance: 19 | -------------------------------------------------------------------------------- /docs/regcore_read.tests.rst: -------------------------------------------------------------------------------- 1 | regcore\_read\.tests package 2 | ============================ 3 | 4 | Submodules 5 | ---------- 6 | 7 | regcore\_read\.tests\.urls module 8 | --------------------------------- 9 | 10 | .. automodule:: regcore_read.tests.urls 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | regcore\_read\.tests\.views\_diff\_tests module 16 | ----------------------------------------------- 17 | 18 | .. automodule:: regcore_read.tests.views_diff_tests 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | regcore\_read\.tests\.views\_es\_search\_tests module 24 | ----------------------------------------------------- 25 | 26 | .. automodule:: regcore_read.tests.views_es_search_tests 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | regcore\_read\.tests\.views\_haystack\_search\_tests module 32 | ----------------------------------------------------------- 33 | 34 | .. automodule:: regcore_read.tests.views_haystack_search_tests 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | regcore\_read\.tests\.views\_layer\_tests module 40 | ------------------------------------------------ 41 | 42 | .. automodule:: regcore_read.tests.views_layer_tests 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | regcore\_read\.tests\.views\_notice\_tests module 48 | ------------------------------------------------- 49 | 50 | .. automodule:: regcore_read.tests.views_notice_tests 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | 55 | regcore\_read\.tests\.views\_preamble\_tests module 56 | --------------------------------------------------- 57 | 58 | .. automodule:: regcore_read.tests.views_preamble_tests 59 | :members: 60 | :undoc-members: 61 | :show-inheritance: 62 | 63 | regcore\_read\.tests\.views\_regulation\_tests module 64 | ----------------------------------------------------- 65 | 66 | .. automodule:: regcore_read.tests.views_regulation_tests 67 | :members: 68 | :undoc-members: 69 | :show-inheritance: 70 | 71 | regcore\_read\.tests\.views\_seach\_utils\_tests module 72 | ------------------------------------------------------- 73 | 74 | .. automodule:: regcore_read.tests.views_seach_utils_tests 75 | :members: 76 | :undoc-members: 77 | :show-inheritance: 78 | 79 | 80 | Module contents 81 | --------------- 82 | 83 | .. automodule:: regcore_read.tests 84 | :members: 85 | :undoc-members: 86 | :show-inheritance: 87 | -------------------------------------------------------------------------------- /docs/regcore_read.views.rst: -------------------------------------------------------------------------------- 1 | regcore\_read\.views package 2 | ============================ 3 | 4 | Submodules 5 | ---------- 6 | 7 | regcore\_read\.views\.diff module 8 | --------------------------------- 9 | 10 | .. automodule:: regcore_read.views.diff 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | regcore\_read\.views\.document module 16 | ------------------------------------- 17 | 18 | .. automodule:: regcore_read.views.document 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | regcore\_read\.views\.es\_search module 24 | --------------------------------------- 25 | 26 | .. automodule:: regcore_read.views.es_search 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | regcore\_read\.views\.haystack\_search module 32 | --------------------------------------------- 33 | 34 | .. automodule:: regcore_read.views.haystack_search 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | regcore\_read\.views\.layer module 40 | ---------------------------------- 41 | 42 | .. automodule:: regcore_read.views.layer 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | regcore\_read\.views\.notice module 48 | ----------------------------------- 49 | 50 | .. automodule:: regcore_read.views.notice 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | 55 | regcore\_read\.views\.search\_utils module 56 | ------------------------------------------ 57 | 58 | .. automodule:: regcore_read.views.search_utils 59 | :members: 60 | :undoc-members: 61 | :show-inheritance: 62 | 63 | 64 | Module contents 65 | --------------- 66 | 67 | .. automodule:: regcore_read.views 68 | :members: 69 | :undoc-members: 70 | :show-inheritance: 71 | -------------------------------------------------------------------------------- /docs/regcore_write.rst: -------------------------------------------------------------------------------- 1 | regcore\_write package 2 | ====================== 3 | 4 | Subpackages 5 | ----------- 6 | 7 | .. toctree:: 8 | 9 | regcore_write.tests 10 | regcore_write.views 11 | 12 | Module contents 13 | --------------- 14 | 15 | .. automodule:: regcore_write 16 | :members: 17 | :undoc-members: 18 | :show-inheritance: 19 | -------------------------------------------------------------------------------- /docs/regcore_write.tests.rst: -------------------------------------------------------------------------------- 1 | regcore\_write\.tests package 2 | ============================= 3 | 4 | Submodules 5 | ---------- 6 | 7 | regcore\_write\.tests\.views\_diff\_tests module 8 | ------------------------------------------------ 9 | 10 | .. automodule:: regcore_write.tests.views_diff_tests 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | regcore\_write\.tests\.views\_layer\_tests module 16 | ------------------------------------------------- 17 | 18 | .. automodule:: regcore_write.tests.views_layer_tests 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | regcore\_write\.tests\.views\_notice\_tests module 24 | -------------------------------------------------- 25 | 26 | .. automodule:: regcore_write.tests.views_notice_tests 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | regcore\_write\.tests\.views\_preamble\_tests module 32 | ---------------------------------------------------- 33 | 34 | .. automodule:: regcore_write.tests.views_preamble_tests 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | regcore\_write\.tests\.views\_regulation\_tests module 40 | ------------------------------------------------------ 41 | 42 | .. automodule:: regcore_write.tests.views_regulation_tests 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | regcore\_write\.tests\.views\_security\_tests module 48 | ---------------------------------------------------- 49 | 50 | .. automodule:: regcore_write.tests.views_security_tests 51 | :members: 52 | :undoc-members: 53 | :show-inheritance: 54 | 55 | 56 | Module contents 57 | --------------- 58 | 59 | .. automodule:: regcore_write.tests 60 | :members: 61 | :undoc-members: 62 | :show-inheritance: 63 | -------------------------------------------------------------------------------- /docs/regcore_write.views.rst: -------------------------------------------------------------------------------- 1 | regcore\_write\.views package 2 | ============================= 3 | 4 | Submodules 5 | ---------- 6 | 7 | regcore\_write\.views\.diff module 8 | ---------------------------------- 9 | 10 | .. automodule:: regcore_write.views.diff 11 | :members: 12 | :undoc-members: 13 | :show-inheritance: 14 | 15 | regcore\_write\.views\.document module 16 | -------------------------------------- 17 | 18 | .. automodule:: regcore_write.views.document 19 | :members: 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | regcore\_write\.views\.layer module 24 | ----------------------------------- 25 | 26 | .. automodule:: regcore_write.views.layer 27 | :members: 28 | :undoc-members: 29 | :show-inheritance: 30 | 31 | regcore\_write\.views\.notice module 32 | ------------------------------------ 33 | 34 | .. automodule:: regcore_write.views.notice 35 | :members: 36 | :undoc-members: 37 | :show-inheritance: 38 | 39 | regcore\_write\.views\.security module 40 | -------------------------------------- 41 | 42 | .. automodule:: regcore_write.views.security 43 | :members: 44 | :undoc-members: 45 | :show-inheritance: 46 | 47 | 48 | Module contents 49 | --------------- 50 | 51 | .. automodule:: regcore_write.views 52 | :members: 53 | :undoc-members: 54 | :show-inheritance: 55 | -------------------------------------------------------------------------------- /manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | if __name__ == "__main__": 6 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "regcore.settings.base") 7 | 8 | from django.core.management import execute_from_command_line 9 | 10 | execute_from_command_line(sys.argv) 11 | -------------------------------------------------------------------------------- /regcore/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eregs/regulations-core/0b2a2034baacfa1cc5ff87f14db7d1aaa8d260c3/regcore/__init__.py -------------------------------------------------------------------------------- /regcore/db/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eregs/regulations-core/0b2a2034baacfa1cc5ff87f14db7d1aaa8d260c3/regcore/db/__init__.py -------------------------------------------------------------------------------- /regcore/db/django_models.py: -------------------------------------------------------------------------------- 1 | """Each of the data structures relevant to the API (regulations, notices, 2 | etc.), implemented using Django models""" 3 | import collections 4 | 5 | from django.conf import settings 6 | from django.core.exceptions import ObjectDoesNotExist 7 | 8 | from regcore.db import interface 9 | from regcore.models import Diff, Document, Layer, Notice 10 | 11 | 12 | def treeify(node, tree_id, pos=1, level=0): 13 | """Set tree properties in memory. 14 | """ 15 | node['tree_id'] = tree_id 16 | node['level'] = level 17 | node['left'] = pos 18 | for child in node.get('children', []): 19 | pos = treeify(child, tree_id, pos=pos + 1, level=level + 1) 20 | pos = pos + 1 21 | node['right'] = pos 22 | return pos 23 | 24 | 25 | def build_adjacency_map(regs): 26 | """Build mapping from node IDs to child records 27 | :param regs: List of `Document` records 28 | """ 29 | ret = collections.defaultdict(list) 30 | for reg in regs: 31 | if reg.parent_id is not None: 32 | ret[reg.parent_id].append(reg) 33 | return ret 34 | 35 | 36 | def build_id(reg, version=None): 37 | if version is not None: 38 | return '{0}:{1}'.format(version, '-'.join(reg['label'])) 39 | return '-'.join(reg['label']) 40 | 41 | 42 | class DMDocuments(interface.Documents): 43 | """Implementation of Django-models as regulations backend""" 44 | def get(self, doc_type, label, version=None): 45 | """Find the regulation label + version""" 46 | regs = Document.objects.filter( 47 | doc_type=doc_type, 48 | label_string=label, 49 | version=version, 50 | ).get_descendants( 51 | include_self=True, 52 | ) 53 | regs = list(regs.all()) 54 | if not regs: 55 | return None 56 | adjacency_map = build_adjacency_map(regs) 57 | return self._serialize(regs[0], adjacency_map) 58 | 59 | def _serialize(self, reg, adjacency_map): 60 | ret = { 61 | 'label': reg.label_string.split('-'), 62 | 'text': reg.text, 63 | 'node_type': reg.node_type, 64 | 'children': [ 65 | self._serialize(child, adjacency_map) 66 | for child in adjacency_map.get(reg.id, []) 67 | ], 68 | } 69 | ret['lft'] = getattr(reg, 'lft', None) 70 | if reg.title: 71 | ret['title'] = reg.title 72 | return ret 73 | 74 | def _transform(self, reg, doc_type, version=None): 75 | """Create the Django object""" 76 | return Document( 77 | id=build_id(reg, version), 78 | doc_type=doc_type, 79 | version=version, 80 | parent_id=( 81 | build_id(reg['parent'], version) 82 | if reg.get('parent') 83 | else None 84 | ), 85 | tree_id=reg['tree_id'], 86 | level=reg['level'], 87 | lft=reg['left'], 88 | rght=reg['right'], 89 | label_string='-'.join(reg['label']), 90 | text=reg['text'], 91 | title=reg.get('title', ''), 92 | node_type=reg['node_type'], 93 | root=(len(reg['label']) == 1), 94 | ) 95 | 96 | def bulk_delete(self, doc_type, root_label, version): 97 | """Delete all documents that match these params""" 98 | # This does not handle subparts. Ignoring that for now 99 | Document.objects.filter( 100 | version=version, 101 | doc_type=doc_type, 102 | label_string__startswith=root_label, 103 | ).delete() 104 | 105 | def bulk_insert(self, regs, doc_type, version): 106 | """Store all document objects""" 107 | treeify(regs[0], Document.objects._get_next_tree_id()) 108 | Document.objects.bulk_create( 109 | [self._transform(r, doc_type, version) for r in regs], 110 | batch_size=settings.BATCH_SIZE) 111 | 112 | def listing(self, doc_type, label=None): 113 | """List regulation version-label pairs that match this label (or are 114 | root, if label is None)""" 115 | if label is None: 116 | query = Document.objects.filter(doc_type=doc_type, root=True) 117 | else: 118 | query = Document.objects.filter( 119 | doc_type=doc_type, label_string=label) 120 | 121 | query = query.only('version', 'label_string').order_by('version') 122 | # Flattens 123 | versions = [v for v in query.values_list('version', 'label_string')] 124 | return versions 125 | 126 | 127 | class DMLayers(interface.Layers): 128 | """Implementation of Django-models as layers backend""" 129 | def _transform(self, layer, layer_name, doc_type): 130 | """Create a Django object""" 131 | layer = dict(layer) # copy 132 | doc_id = layer.pop('doc_id') 133 | return Layer(name=layer_name, layer=layer, doc_type=doc_type, 134 | doc_id=doc_id) 135 | 136 | def bulk_delete(self, layer_name, doc_type, root_doc_id): 137 | """Delete all layer data matching the parameters""" 138 | # This does not handle subparts; Ignoring that for now 139 | # @todo - use regex to avoid deleting 222-11 when replacing 22 140 | Layer.objects.filter(name=layer_name, doc_type=doc_type, 141 | doc_id__startswith=root_doc_id).delete() 142 | 143 | def bulk_insert(self, layers, layer_name, doc_type): 144 | """Store all layer objects""" 145 | Layer.objects.bulk_create( 146 | [self._transform(l, layer_name, doc_type) for l in layers], 147 | batch_size=settings.BATCH_SIZE) 148 | 149 | def get(self, name, doc_type, doc_id): 150 | """Find the layer that matches these parameters""" 151 | try: 152 | layer = Layer.objects.get(name=name, doc_type=doc_type, 153 | doc_id=doc_id) 154 | return layer.layer 155 | except ObjectDoesNotExist: 156 | return None 157 | 158 | 159 | class DMNotices(interface.Notices): 160 | """Implementation of Django-models as notice backend""" 161 | def delete(self, doc_number): 162 | Notice.objects.filter(document_number=doc_number).delete() 163 | 164 | def insert(self, doc_number, notice): 165 | """Store a single notice""" 166 | model = Notice(document_number=doc_number, 167 | fr_url=notice['fr_url'], 168 | publication_date=notice['publication_date'], 169 | notice=notice) 170 | if 'effective_on' in notice: 171 | model.effective_on = notice['effective_on'] 172 | model.save() 173 | for cfr_part in notice.get('cfr_parts', []): 174 | model.noticecfrpart_set.create(cfr_part=cfr_part) 175 | 176 | def get(self, doc_number): 177 | """Find the associated notice""" 178 | try: 179 | return Notice.objects.get( 180 | document_number=doc_number).notice 181 | except ObjectDoesNotExist: 182 | return None 183 | 184 | def listing(self, part=None): 185 | """All notices or filtered by cfr_part""" 186 | query = Notice.objects 187 | if part: 188 | query = query.filter(noticecfrpart__cfr_part=part) 189 | results = query.values('document_number', 'effective_on', 'fr_url', 190 | 'publication_date') 191 | for result in results: 192 | for key in ('effective_on', 'publication_date'): 193 | if result[key]: 194 | result[key] = result[key].isoformat() 195 | else: 196 | del result[key] 197 | return list(results) # maintain compatibility with other backends 198 | 199 | 200 | class DMDiffs(interface.Diffs): 201 | """Implementation of Django-models as diff backend""" 202 | def insert(self, label, old_version, new_version, diff): 203 | """Store a diff between two versions of a regulation node""" 204 | Diff(label=label, old_version=old_version, new_version=new_version, 205 | diff=diff).save() 206 | 207 | def delete(self, label, old_version, new_version): 208 | Diff.objects.filter(label=label, old_version=old_version, 209 | new_version=new_version).delete() 210 | 211 | def get(self, label, old_version, new_version): 212 | """Find the associated diff""" 213 | try: 214 | diff = Diff.objects.get(label=label, old_version=old_version, 215 | new_version=new_version) 216 | return diff.diff 217 | except ObjectDoesNotExist: 218 | return None 219 | -------------------------------------------------------------------------------- /regcore/db/es.py: -------------------------------------------------------------------------------- 1 | """Each of the data structures relevant to the API (regulations, notices, 2 | etc.), implemented using Elastic Search as a data store""" 3 | import logging 4 | 5 | from cached_property import cached_property 6 | from django.conf import settings 7 | from pyelasticsearch import ElasticSearch 8 | from pyelasticsearch.exceptions import ElasticHttpNotFoundError 9 | 10 | from regcore.db import interface 11 | 12 | logger = logging.getLogger(__name__) 13 | 14 | 15 | def sanitize_doc_id(doc_id): 16 | """Not strictly required, but remove slashes from Elastic Search ids""" 17 | return ':'.join(doc_id.split('/')) 18 | 19 | 20 | class ESBase(object): 21 | """Shared code for Elastic Search storage models""" 22 | 23 | @cached_property 24 | def es(self): 25 | return ElasticSearch(settings.ELASTIC_SEARCH_URLS) 26 | 27 | def safe_fetch(self, doc_type, es_id): 28 | """Attempt to retrieve a document from Elastic Search. 29 | :return: Found document, if it exists, otherwise None""" 30 | try: 31 | result = self.es.get(settings.ELASTIC_SEARCH_INDEX, doc_type, 32 | es_id) 33 | return result['_source'] 34 | except ElasticHttpNotFoundError: 35 | return None 36 | 37 | def bulk_delete(self, *args, **kwarg): 38 | logger.warning("Elastic Search backend doesn't handle deletes") 39 | 40 | def delete(self, *args, **kwarg): 41 | logger.warning("Elastic Search backend doesn't handle deletes") 42 | 43 | 44 | class ESDocuments(ESBase, interface.Documents): 45 | """Implementation of Elastic Search as regulations backend""" 46 | def get(self, doc_type, label, version): 47 | """Find the regulation label + version""" 48 | reg_node = self.safe_fetch('reg_tree', version + '/' + label) 49 | if reg_node is not None: 50 | del reg_node['regulation'] 51 | del reg_node['version'] 52 | del reg_node['label_string'] 53 | del reg_node['id'] 54 | return reg_node 55 | 56 | def _transform(self, reg, doc_type, version): 57 | """Add some meta data fields which are ES specific""" 58 | node = dict(reg) # copy 59 | node['doc_type'] = doc_type 60 | node['version'] = version 61 | node['label_string'] = '-'.join(node['label']) 62 | node['regulation'] = node['label'][0] 63 | node['id'] = version + '/' + node['label_string'] 64 | node['root'] = len(node['label']) == 1 65 | node['is_subpart'] = ( 66 | 'Subpart' in node['label'] or 67 | 'Subjgrp' in node['label'] 68 | ) 69 | return node 70 | 71 | def bulk_insert(self, regs, doc_type, version): 72 | """Store all reg objects""" 73 | self.es.bulk_index( 74 | settings.ELASTIC_SEARCH_INDEX, 'reg_tree', 75 | [self._transform(r, doc_type, version) for r in regs], 76 | ) 77 | 78 | def listing(self, doc_type, label=None): 79 | """List regulation version-label pairs that match this label (or are 80 | root, if label is None)""" 81 | if label is None: 82 | query = {'match': {'root': True, 'doc_type': doc_type}} 83 | else: 84 | query = {'match': {'label_string': label, 'doc_type': doc_type}} 85 | query = {'fields': ['label_string', 'version'], 'query': query} 86 | result = self.es.search(query, index=settings.ELASTIC_SEARCH_INDEX, 87 | doc_type='reg_tree', size=100) 88 | return sorted((res['fields']['version'], res['fields']['label_string']) 89 | for res in result['hits']['hits']) 90 | 91 | 92 | class ESLayers(ESBase, interface.Layers): 93 | """Implementation of Elastic Search as layers backend""" 94 | def _transform(self, layer, layer_name, doc_type): 95 | """Add some meta data fields which are ES specific""" 96 | layer = dict(layer) # copy 97 | doc_id = sanitize_doc_id(layer.pop('doc_id')) 98 | return {'id': ':'.join([layer_name, doc_type, doc_id]), 'layer': layer} 99 | 100 | def bulk_insert(self, layers, layer_name, doc_type): 101 | """Store all layer objects.""" 102 | self.es.bulk_index( 103 | settings.ELASTIC_SEARCH_INDEX, 'layer', 104 | [self._transform(l, layer_name, doc_type) for l in layers]) 105 | 106 | def get(self, name, doc_type, doc_id): 107 | """Find the layer that matches these parameters""" 108 | reference = ':'.join([name, doc_type, sanitize_doc_id(doc_id)]) 109 | layer = self.safe_fetch('layer', reference) 110 | if layer is not None: 111 | return layer['layer'] 112 | 113 | 114 | class ESNotices(ESBase, interface.Notices): 115 | """Implementation of Elastic Search as notice backend""" 116 | def insert(self, doc_number, notice): 117 | """Store a single notice""" 118 | self.es.index(settings.ELASTIC_SEARCH_INDEX, 'notice', notice, 119 | id=doc_number) 120 | 121 | def get(self, doc_number): 122 | """Find the associated notice""" 123 | return self.safe_fetch('notice', doc_number) 124 | 125 | def listing(self, part=None): 126 | """All notices or filtered by cfr_part""" 127 | if part: 128 | query = {'match': {'cfr_parts': part}} 129 | else: 130 | query = {'match_all': {}} 131 | query = {'fields': ['effective_on', 'fr_url', 'publication_date'], 132 | 'query': query} 133 | notices = [] 134 | results = self.es.search(query, doc_type='notice', size=100, 135 | index=settings.ELASTIC_SEARCH_INDEX) 136 | for notice in results['hits']['hits']: 137 | notice['fields']['document_number'] = notice['_id'] 138 | notices.append(notice['fields']) 139 | return notices 140 | 141 | 142 | class ESDiffs(ESBase, interface.Diffs): 143 | """Implementation of Elastic Search as diff backend""" 144 | @staticmethod 145 | def to_id(label, old, new): 146 | return '/'.join([label, old, new]) 147 | 148 | def insert(self, label, old_version, new_version, diff): 149 | """Store a diff between two versions of a regulation node""" 150 | struct = { 151 | 'label': label, 152 | 'old_version': old_version, 153 | 'new_version': new_version, 154 | 'diff': diff 155 | } 156 | self.es.index(settings.ELASTIC_SEARCH_INDEX, 'diff', struct, 157 | id=self.to_id(label, old_version, new_version)) 158 | 159 | def get(self, label, old_version, new_version): 160 | """Find the associated diff""" 161 | diff = self.safe_fetch('diff', 162 | self.to_id(label, old_version, new_version)) 163 | if diff is not None: 164 | return diff['diff'] 165 | -------------------------------------------------------------------------------- /regcore/db/interface.py: -------------------------------------------------------------------------------- 1 | """Interfaces for each of the storage systems.""" 2 | import abc 3 | 4 | import six 5 | 6 | 7 | @six.add_metaclass(abc.ABCMeta) 8 | class Documents(object): 9 | @abc.abstractmethod 10 | def get(self, doc_type, label, version): 11 | """Returns a regulation node or None""" 12 | raise NotImplementedError 13 | 14 | @abc.abstractmethod 15 | def bulk_delete(self, doc_type, root_label, version): 16 | """Delete all documents that match these parameters""" 17 | raise NotImplementedError 18 | 19 | @abc.abstractmethod 20 | def bulk_insert(self, regs, doc_type, version): 21 | """Add many entries, each with the provided version""" 22 | raise NotImplementedError 23 | 24 | @abc.abstractmethod 25 | def listing(self, doc_type, label=None): 26 | """Return a list of (version, label) pairs for regulation objects that 27 | match the provided label (or all root regs), sorted by version""" 28 | raise NotImplementedError 29 | 30 | 31 | @six.add_metaclass(abc.ABCMeta) 32 | class Layers(object): 33 | def bulk_delete(self, layer_name, doc_type, root_doc_id): 34 | """Deletes multiple entries with the same layer_name. 35 | :param str layer_name: Identifier for this layer, e.g. "toc", 36 | "internal-citations", etc. 37 | :param str doc_type: layers are keyed by doc_type 38 | :param str root_doc_id: the doc id of the "root" layer.""" 39 | raise NotImplementedError 40 | 41 | def bulk_insert(self, layers, layer_name, doc_type): 42 | """Add multiple entries with the same layer_name. 43 | :param list[dict] layers: Each dictionary represents a layer; each 44 | should have a distinct "doc_id", which will be used during insertion. 45 | :param str layer_name: Identifier for this layer, e.g. "toc", 46 | "internal-citations", etc. 47 | :param str doc_type: layers are keyed by doc_type""" 48 | raise NotImplementedError 49 | 50 | def get(self, name, doc_type, doc_id): 51 | """Return a single layer (no meta data) or None""" 52 | raise NotImplementedError 53 | 54 | 55 | @six.add_metaclass(abc.ABCMeta) 56 | class Notices(object): 57 | def delete(self, doc_number): 58 | """:param str doc_number:""" 59 | raise NotImplementedError 60 | 61 | def insert(self, doc_number, notice): 62 | """:param str doc_number: 63 | :param dict notice:""" 64 | raise NotImplementedError 65 | 66 | def get(self, doc_number): 67 | """Return matching notice or None""" 68 | raise NotImplementedError 69 | 70 | def listing(self, part=None): 71 | """Return all notices or notices by part""" 72 | raise NotImplementedError 73 | 74 | 75 | @six.add_metaclass(abc.ABCMeta) 76 | class Diffs(object): 77 | def delete(self, label, old_version, new_version): 78 | """:param str label: 79 | :param str old_version: 80 | :param str new_version:""" 81 | raise NotImplementedError 82 | 83 | def insert(self, label, old_version, new_version, diff): 84 | """:param str label: 85 | :param str old_version: 86 | :param str new_version: 87 | :param dict diff:""" 88 | raise NotImplementedError 89 | 90 | def get(self, label, old_version, new_version): 91 | """Return matching diff or None""" 92 | raise NotImplementedError 93 | -------------------------------------------------------------------------------- /regcore/db/storage.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from django.utils.module_loading import import_string 3 | 4 | 5 | def select_for(data_type): 6 | """The storage class for each datatype is defined in a settings file. This 7 | will look up the appropriate storage backend and instantiate it. If none 8 | is found, this will default to the Django ORM versions""" 9 | class_str = settings.BACKENDS.get( 10 | data_type, 11 | 'regcore.db.django_models.DM' + data_type.capitalize()) 12 | return import_string(class_str)() 13 | 14 | 15 | for_documents = select_for('documents') 16 | for_layers = select_for('layers') 17 | for_notices = select_for('notices') 18 | for_diffs = select_for('diffs') 19 | -------------------------------------------------------------------------------- /regcore/fields.py: -------------------------------------------------------------------------------- 1 | import base64 2 | import bz2 3 | import json 4 | import logging 5 | 6 | import six 7 | from django.db import models 8 | 9 | 10 | class CompressedJSONField(models.TextField): 11 | """We store a lot of data redundantly. This field type makes each copy 12 | much smaller. We need this when inserting hundreds of regtext nodes and 13 | layer nodes into relational databases, lest we blow the packet size limit 14 | """ 15 | def to_python(self, value): 16 | """Convert the string (from the database) into a JSON dictionary""" 17 | if not isinstance(value, six.text_type): 18 | return value 19 | 20 | encoding, content = value.split('$', 1) 21 | if encoding == 'j': 22 | content = json.loads(content) 23 | elif encoding == 'jb6': 24 | content = base64.decodestring(content.encode('utf-8')) 25 | content = bz2.decompress(content) 26 | content = json.loads(content.decode('utf-8')) 27 | else: 28 | logging.warning("Unknown encoding: %s", encoding) 29 | content = {} 30 | return content 31 | 32 | def from_db_value(self, value, expression, connection, context): 33 | """Satisfies Django 1.8's custom field types requirements.""" 34 | return self.to_python(value) 35 | 36 | def get_prep_value(self, value): 37 | """Convert from a JSON dictionary to a database string""" 38 | value = json.dumps(value) 39 | encoding = 'j' 40 | 41 | if len(value) > 1000: # somewhat arbitrary length to start compression 42 | # check if compressing is smaller 43 | compressed = base64.encodestring(bz2.compress( 44 | value.encode('utf-8'))).decode('utf-8') 45 | if len(compressed) < len(value): 46 | encoding += 'b6' 47 | value = compressed 48 | 49 | return encoding + u'$' + value 50 | -------------------------------------------------------------------------------- /regcore/index.py: -------------------------------------------------------------------------------- 1 | """Schemas used by Elastic Search as well as an initialization function, 2 | which sends the schemas over to the Elastic Search instance.""" 3 | 4 | 5 | from django.conf import settings 6 | from pyelasticsearch import ElasticSearch 7 | from pyelasticsearch.exceptions import IndexAlreadyExistsError 8 | 9 | NODE_SEARCH_SCHEMA = { 10 | 'text': {'type': 'string'}, # Full text search 11 | # Do not search children, but make them available 12 | 'children': {'type': 'object', 'enabled': False}, 13 | 'label': {'type': 'string'}, # An array of strings 14 | 'label_string': {'type': 'string', 'index': 'not_analyzed'}, 15 | 'regulation': {'type': 'string', 'index': 'not_analyzed'}, 16 | 'title': {'type': 'string'}, 17 | 'node_type': {'type': 'string', 'index': 'not_analyzed'}, 18 | 'id': {'type': 'string', 'index': 'not_analyzed'}, 19 | 'version': {'type': 'string', 'index': 'not_analyzed'} 20 | } 21 | 22 | LAYER_SCHEMA = { 23 | 'id': {'type': 'string', 'index': 'not_analyzed'}, 24 | # i.e. doc_number 25 | 'version': {'type': 'string', 'index': 'not_analyzed'}, 26 | # layer's name, e.g. "internal-citations" 27 | 'name': {'type': 'string', 'index': 'not_analyzed'}, 28 | # label at which this layer applies (1005, 1005-12, 1005-12-b, etc.) 29 | 'label': {'type': 'string', 'index': 'not_analyzed'}, 30 | # Will never search on the layer components 31 | 'layer': {'type': 'object', 'enabled': False} 32 | } 33 | 34 | NOTICE_SCHEMA = { 35 | 'id': {'type': 'string', 'index': 'not_analyzed'}, 36 | # Eventually, these fields will be searchable 37 | 'notice': {'type': 'object', 'enabled': False} 38 | } 39 | 40 | DIFF_SCHEMA = { 41 | 'id': {'type': 'string', 'index': 'not_analyzed'}, 42 | 'label': {'type': 'string', 'index': 'not_analyzed'}, 43 | 'old_version': {'type': 'string', 'index': 'not_analyzed'}, 44 | 'new_version': {'type': 'string', 'index': 'not_analyzed'}, 45 | # No need to index this 46 | 'diff': {'type': 'object', 'enabled': False} 47 | } 48 | 49 | 50 | def init_schema(): 51 | """Should be called at application startup. Makes sure the mappings and 52 | index exist.""" 53 | es = ElasticSearch(settings.ELASTIC_SEARCH_URLS) 54 | try: 55 | es.create_index(settings.ELASTIC_SEARCH_INDEX) 56 | except IndexAlreadyExistsError: 57 | pass 58 | 59 | # Does not replace if exact mapping already exists 60 | es.put_mapping(settings.ELASTIC_SEARCH_INDEX, 'reg_tree', { 61 | 'reg_tree': {'properties': NODE_SEARCH_SCHEMA} 62 | }) 63 | es.put_mapping(settings.ELASTIC_SEARCH_INDEX, 'layer', { 64 | 'layer': {'properties': LAYER_SCHEMA} 65 | }) 66 | es.put_mapping(settings.ELASTIC_SEARCH_INDEX, 'notice', { 67 | 'notice': {'properties': LAYER_SCHEMA} 68 | }) 69 | es.put_mapping(settings.ELASTIC_SEARCH_INDEX, 'diff', { 70 | 'diff': {'properties': DIFF_SCHEMA} 71 | }) 72 | -------------------------------------------------------------------------------- /regcore/layer.py: -------------------------------------------------------------------------------- 1 | from collections import namedtuple 2 | 3 | LayerParams = namedtuple('LayerParams', ['doc_type', 'doc_id', 'tree_id']) 4 | 5 | 6 | # @todo Remove in a future release 7 | def standardize_params(doc_type, doc_id): 8 | """We need to convert between an older form of url params, 9 | /layer/{layer type}/{reg label id}/{version id} 10 | and a new form: 11 | /layer/{layer type}/{doc type}/{doc id which may contain more slashes} 12 | This class handles that conversion and provides a single interface for 13 | both forms""" 14 | # old format - we can remove once all data/libs are migrated 15 | if doc_type not in ('preamble', 'cfr'): 16 | # this looks backwards, but it's the format we assumed before 17 | label, version = doc_type, doc_id 18 | doc_type = 'cfr' 19 | doc_id = '/'.join([version, label]) 20 | 21 | # e.g. "111_22" in both doc_ids, "111_22" and "version/111_22" 22 | tree_id = doc_id.split('/')[-1] 23 | return LayerParams(doc_type, doc_id, tree_id) 24 | -------------------------------------------------------------------------------- /regcore/management/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'vinokurovy' 2 | -------------------------------------------------------------------------------- /regcore/management/commands/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'vinokurovy' 2 | -------------------------------------------------------------------------------- /regcore/management/commands/import_docs.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | 4 | from django.core.management.base import BaseCommand 5 | from django.test import Client, override_settings 6 | 7 | logger = logging.getLogger(__name__) 8 | 9 | 10 | def scoped_files(root): 11 | """Find all of the files which will need to be "uploaded"; trim them down 12 | to their suffix and separate their path components. We'll assume that 13 | `root` has no trailing slash""" 14 | for path, _, file_names in os.walk(root): 15 | for file_name in file_names: 16 | file_path = os.path.join(path, file_name) 17 | trimmed = file_path[len(root):] 18 | yield trimmed.split(os.sep) 19 | 20 | 21 | def save_file(root, file_parts): 22 | """Given a file (indicated by a root file path and a set of file path 23 | components), read the file from disk and write it to the database. Log 24 | results.""" 25 | file_path = os.path.join(root, *file_parts) 26 | with open(file_path, 'rb') as f: 27 | content = f.read() 28 | result = Client().put('/'.join(file_parts), data=content, 29 | content_type='application/json') 30 | if result.status_code == 204: 31 | logger.info('Saved %s', file_path) 32 | else: 33 | logger.error('Failed to save %s: (%s), %s', 34 | file_path, result.status_code, result.content[:100]) 35 | 36 | 37 | class Command(BaseCommand): 38 | help = "Import a collection of JSON files into the database." # noqa 39 | 40 | def add_arguments(self, parser): 41 | parser.add_argument( 42 | 'base_dir', default=os.getcwd(), nargs='?', 43 | help='the base filesystem path for importing JSON files' 44 | ) 45 | 46 | @override_settings(ROOT_URLCONF='regcore.urls', ALLOWED_HOSTS=['*']) 47 | def handle(self, *args, **options): 48 | root = options['base_dir'].rstrip(os.sep) 49 | 50 | for file_parts in scoped_files(root): 51 | save_file(root, file_parts) 52 | -------------------------------------------------------------------------------- /regcore/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import models, migrations 5 | import regcore.fields 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ] 12 | 13 | operations = [ 14 | migrations.CreateModel( 15 | name='Diff', 16 | fields=[ 17 | ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), 18 | ('label', models.SlugField(max_length=200)), 19 | ('old_version', models.SlugField(max_length=20)), 20 | ('new_version', models.SlugField(max_length=20)), 21 | ('diff', regcore.fields.CompressedJSONField()), 22 | ], 23 | ), 24 | migrations.CreateModel( 25 | name='Layer', 26 | fields=[ 27 | ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), 28 | ('version', models.SlugField(max_length=20)), 29 | ('name', models.SlugField(max_length=20)), 30 | ('label', models.SlugField(max_length=200)), 31 | ('layer', regcore.fields.CompressedJSONField()), 32 | ], 33 | ), 34 | migrations.CreateModel( 35 | name='Notice', 36 | fields=[ 37 | ('document_number', models.SlugField(max_length=20, serialize=False, primary_key=True)), 38 | ('effective_on', models.DateField(null=True)), 39 | ('fr_url', models.CharField(max_length=200, null=True)), 40 | ('publication_date', models.DateField()), 41 | ('notice', regcore.fields.CompressedJSONField()), 42 | ], 43 | ), 44 | migrations.CreateModel( 45 | name='NoticeCFRPart', 46 | fields=[ 47 | ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), 48 | ('cfr_part', models.SlugField(max_length=10)), 49 | ('notice', models.ForeignKey(to='regcore.Notice')), 50 | ], 51 | ), 52 | migrations.CreateModel( 53 | name='Regulation', 54 | fields=[ 55 | ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), 56 | ('version', models.SlugField(max_length=20)), 57 | ('label_string', models.SlugField(max_length=200)), 58 | ('text', models.TextField()), 59 | ('title', models.TextField(blank=True)), 60 | ('node_type', models.SlugField(max_length=10)), 61 | ('children', regcore.fields.CompressedJSONField()), 62 | ('root', models.BooleanField(default=False, db_index=True)), 63 | ], 64 | ), 65 | migrations.AlterUniqueTogether( 66 | name='regulation', 67 | unique_together=set([('version', 'label_string')]), 68 | ), 69 | migrations.AlterIndexTogether( 70 | name='regulation', 71 | index_together=set([('version', 'label_string')]), 72 | ), 73 | migrations.AlterUniqueTogether( 74 | name='layer', 75 | unique_together=set([('version', 'name', 'label')]), 76 | ), 77 | migrations.AlterIndexTogether( 78 | name='layer', 79 | index_together=set([('version', 'name', 'label')]), 80 | ), 81 | migrations.AlterUniqueTogether( 82 | name='diff', 83 | unique_together=set([('label', 'old_version', 'new_version')]), 84 | ), 85 | migrations.AlterIndexTogether( 86 | name='diff', 87 | index_together=set([('label', 'old_version', 'new_version')]), 88 | ), 89 | migrations.AlterUniqueTogether( 90 | name='noticecfrpart', 91 | unique_together=set([('notice', 'cfr_part')]), 92 | ), 93 | migrations.AlterIndexTogether( 94 | name='noticecfrpart', 95 | index_together=set([('notice', 'cfr_part')]), 96 | ), 97 | ] 98 | -------------------------------------------------------------------------------- /regcore/migrations/0002_mptt_add_fields.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import migrations, models 5 | import mptt.fields 6 | 7 | from regcore.fields import CompressedJSONField 8 | 9 | 10 | class Migration(migrations.Migration): 11 | 12 | dependencies = [ 13 | ('regcore', '0001_initial'), 14 | ] 15 | 16 | operations = [ 17 | migrations.AddField( 18 | model_name='regulation', 19 | name='level', 20 | field=models.PositiveIntegerField(default=1, editable=False, db_index=True), 21 | preserve_default=False, 22 | ), 23 | migrations.AddField( 24 | model_name='regulation', 25 | name='lft', 26 | field=models.PositiveIntegerField(default=1, editable=False, db_index=True), 27 | preserve_default=False, 28 | ), 29 | migrations.AddField( 30 | model_name='regulation', 31 | name='parent', 32 | field=mptt.fields.TreeForeignKey(blank=True, to='regcore.Regulation', null=True), 33 | ), 34 | migrations.AddField( 35 | model_name='regulation', 36 | name='rght', 37 | field=models.PositiveIntegerField(default=1, editable=False, db_index=True), 38 | preserve_default=False, 39 | ), 40 | migrations.AddField( 41 | model_name='regulation', 42 | name='tree_id', 43 | field=models.PositiveIntegerField(default=1, editable=False, db_index=True), 44 | preserve_default=False, 45 | ), 46 | migrations.AlterField( 47 | model_name='regulation', 48 | name='id', 49 | field=models.TextField(serialize=False, primary_key=True), 50 | ), 51 | migrations.AlterField( 52 | model_name='regulation', 53 | name='children', 54 | field=CompressedJSONField(null=True, blank=True), 55 | ), 56 | ] 57 | -------------------------------------------------------------------------------- /regcore/migrations/0002_preamble.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import migrations, models 5 | import regcore.fields 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('regcore', '0001_initial'), 12 | ] 13 | 14 | operations = [ 15 | migrations.CreateModel( 16 | name='Preamble', 17 | fields=[ 18 | ('document_number', models.SlugField(max_length=20, serialize=False, primary_key=True)), 19 | ('data', regcore.fields.CompressedJSONField()), 20 | ], 21 | ), 22 | ] 23 | -------------------------------------------------------------------------------- /regcore/migrations/0003_mptt_copy_children.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import migrations 5 | 6 | import mptt 7 | import mptt.managers 8 | 9 | 10 | def rebuild(apps, schema_editor): 11 | Regulation = apps.get_model('regcore', 'Regulation') 12 | Regulation.objects.filter(root=False).delete() 13 | 14 | # Bind manager 15 | manager = mptt.managers.TreeManager() 16 | manager.model = Regulation 17 | mptt.register(Regulation) 18 | manager.contribute_to_class(Regulation, 'objects') 19 | 20 | for root in Regulation.objects.all(): 21 | serialized = { 22 | 'text': root.text, 23 | 'title': root.title, 24 | 'label': root.label_string.split('-'), 25 | 'node_type': root.node_type, 26 | 'children': root.children, 27 | } 28 | root.delete() 29 | write_node(Regulation, serialized, root.version, root.label_string) 30 | 31 | 32 | def write_node(Regulation, node, version, label_id): 33 | 34 | to_save = [] 35 | labels_seen = set() 36 | 37 | def add_node(node, parent=None): 38 | label_tuple = tuple(node['label']) 39 | labels_seen.add(label_tuple) 40 | 41 | node['parent'] = parent 42 | to_save.append(node) 43 | for child in node['children']: 44 | add_node(child, parent=node) 45 | add_node(node) 46 | 47 | DMRegulations(Regulation).bulk_put(to_save, version, label_id) 48 | 49 | 50 | def treeify(node, tree_id, pos=1, level=0): 51 | """Set tree properties in memory. 52 | """ 53 | node['tree_id'] = tree_id 54 | node['level'] = level 55 | node['left'] = pos 56 | for child in node.get('children', []): 57 | pos = treeify(child, tree_id, pos=pos + 1, level=level + 1) 58 | pos = pos + 1 59 | node['right'] = pos 60 | return pos 61 | 62 | 63 | def build_id(reg, version): 64 | return '{}:{}'.format(version, '-'.join(reg['label'])) 65 | 66 | 67 | class DMRegulations(object): 68 | def __init__(self, Regulation): 69 | self.Regulation = Regulation 70 | 71 | def _transform(self, reg, version): 72 | """Create the Django object""" 73 | return self.Regulation( 74 | id=build_id(reg, version), 75 | parent_id=( 76 | build_id(reg['parent'], version) 77 | if reg.get('parent') 78 | else None 79 | ), 80 | tree_id=reg['tree_id'], 81 | level=reg['level'], 82 | lft=reg['left'], 83 | rght=reg['right'], 84 | version=version, 85 | label_string='-'.join(reg['label']), 86 | text=reg['text'], 87 | title=reg.get('title', ''), 88 | node_type=reg['node_type'], 89 | root=(len(reg['label']) == 1), 90 | ) 91 | 92 | def bulk_put(self, regs, version, root_label): 93 | """Store all reg objects""" 94 | # This does not handle subparts. Ignoring that for now 95 | self.Regulation.objects.filter( 96 | version=version, 97 | label_string__startswith=root_label).delete() 98 | treeify(regs[0], self.Regulation.objects._get_next_tree_id()) 99 | self.Regulation.objects.bulk_create( 100 | [self._transform(r, version) for r in regs], 101 | batch_size=25) 102 | 103 | 104 | class Migration(migrations.Migration): 105 | 106 | dependencies = [ 107 | ('regcore', '0002_mptt_add_fields'), 108 | ] 109 | 110 | operations = [ 111 | migrations.RunPython(rebuild), 112 | ] 113 | -------------------------------------------------------------------------------- /regcore/migrations/0004_mptt_delete_fields.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import migrations 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('regcore', '0003_mptt_copy_children'), 11 | ] 12 | 13 | operations = [ 14 | migrations.RemoveField( 15 | model_name='regulation', 16 | name='children', 17 | ), 18 | ] 19 | -------------------------------------------------------------------------------- /regcore/migrations/0005_merge.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import migrations, models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('regcore', '0002_preamble'), 11 | ('regcore', '0004_mptt_delete_fields'), 12 | ] 13 | 14 | operations = [ 15 | ] 16 | -------------------------------------------------------------------------------- /regcore/migrations/0006_auto_20160314_1126.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import migrations, models 5 | import mptt.fields 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('regcore', '0005_merge'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AddField( 16 | model_name='layer', 17 | name='reference', 18 | field=models.SlugField(default='', max_length=250), 19 | preserve_default=False, 20 | ), 21 | migrations.AlterField( 22 | model_name='regulation', 23 | name='parent', 24 | field=mptt.fields.TreeForeignKey(related_name='children', blank=True, to='regcore.Regulation', null=True), 25 | ), 26 | migrations.AlterUniqueTogether( 27 | name='layer', 28 | unique_together=set() 29 | ), 30 | migrations.AlterIndexTogether( 31 | name='layer', 32 | index_together=set() 33 | ), 34 | ] 35 | -------------------------------------------------------------------------------- /regcore/migrations/0007_auto_20160314_1135.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import migrations, models 5 | 6 | 7 | def gen_layer_reference(apps, schema_editor): 8 | """We'll combine two previously distinct fields to generalize the layers 9 | interface""" 10 | schema_editor.execute( 11 | "UPDATE regcore_layer SET reference=version||':'||label") 12 | 13 | 14 | def split_layer_reference(apps, schema_editor): 15 | """Reverse of above""" 16 | Layer = apps.get_model("regcore", "Layer") 17 | for layer in Layer.objects.filter(reference__contains=':').iterator(): 18 | layer.version, layer.label = layer.reference.split(':') 19 | layer.reference = '' 20 | layer.save() 21 | 22 | 23 | class Migration(migrations.Migration): 24 | 25 | dependencies = [ 26 | ('regcore', '0006_auto_20160314_1126'), 27 | ] 28 | 29 | operations = [ 30 | migrations.RunPython(gen_layer_reference, split_layer_reference) 31 | ] 32 | -------------------------------------------------------------------------------- /regcore/migrations/0008_auto_20160314_1144.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import migrations, models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('regcore', '0007_auto_20160314_1135'), 11 | ] 12 | 13 | operations = [ 14 | migrations.AlterUniqueTogether( 15 | name='layer', 16 | unique_together=set([('name', 'reference')]), 17 | ), 18 | migrations.AlterIndexTogether( 19 | name='layer', 20 | index_together=set([('name', 'reference')]), 21 | ), 22 | migrations.AlterField( 23 | model_name='layer', 24 | name='label', 25 | field=models.SlugField(max_length=200, default='')), 26 | migrations.RemoveField( 27 | model_name='layer', 28 | name='label', 29 | ), 30 | migrations.AlterField( 31 | model_name='layer', 32 | name='version', 33 | field=models.SlugField(max_length=20, default='')), 34 | migrations.RemoveField( 35 | model_name='layer', 36 | name='version', 37 | ), 38 | ] 39 | -------------------------------------------------------------------------------- /regcore/migrations/0009_auto_20160322_1646.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import migrations, models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('regcore', '0008_auto_20160314_1144'), 11 | ] 12 | 13 | operations = [ 14 | migrations.RenameField( 15 | model_name='layer', 16 | old_name='reference', 17 | new_name='doc_id', 18 | ), 19 | migrations.AddField( 20 | model_name='layer', 21 | name='doc_type', 22 | field=models.SlugField(default='cfr', max_length=20), 23 | preserve_default=False, 24 | ), 25 | migrations.AlterUniqueTogether( 26 | name='layer', 27 | unique_together=set([('name', 'doc_type', 'doc_id')]), 28 | ), 29 | migrations.AlterIndexTogether( 30 | name='layer', 31 | index_together=set([('name', 'doc_type', 'doc_id')]), 32 | ), 33 | ] 34 | -------------------------------------------------------------------------------- /regcore/migrations/0010_auto_20160322_1704.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import migrations, models 5 | 6 | 7 | def forward(apps, schema_editor): 8 | schema_editor.execute(""" 9 | UPDATE regcore_layer SET doc_id=REPLACE(doc_id, ':', '/') 10 | WHERE doc_type='cfr'; 11 | """) 12 | 13 | 14 | def backward(apps, schema_editor): 15 | schema_editor.execute(""" 16 | UPDATE regcore_layer SET doc_id=REPLACE(doc_id, '/', ':') 17 | WHERE doc_type='cfr'; 18 | """) 19 | 20 | 21 | class Migration(migrations.Migration): 22 | """Convert doc_ids like 11_22:33-44-aa into 11_22/33-44-aa""" 23 | 24 | dependencies = [ 25 | ('regcore', '0009_auto_20160322_1646'), 26 | ] 27 | 28 | operations = [ 29 | migrations.RunPython(forward, backward) 30 | ] 31 | -------------------------------------------------------------------------------- /regcore/migrations/0011_create_document.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import migrations, models 5 | import mptt.fields 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('regcore', '0010_auto_20160322_1704'), 12 | ] 13 | 14 | operations = [ 15 | migrations.CreateModel( 16 | name='Document', 17 | fields=[ 18 | ('id', models.TextField(serialize=False, primary_key=True)), 19 | ('doc_type', models.SlugField(max_length=20)), 20 | ('version', models.SlugField(max_length=20, null=True, blank=True)), 21 | ('label_string', models.SlugField(max_length=200)), 22 | ('text', models.TextField()), 23 | ('title', models.TextField(blank=True)), 24 | ('node_type', models.SlugField(max_length=10)), 25 | ('root', models.BooleanField(default=False, db_index=True)), 26 | ('lft', models.PositiveIntegerField(editable=False, db_index=True)), 27 | ('rght', models.PositiveIntegerField(editable=False, db_index=True)), 28 | ('tree_id', models.PositiveIntegerField(editable=False, db_index=True)), 29 | ('level', models.PositiveIntegerField(editable=False, db_index=True)), 30 | ('parent', mptt.fields.TreeForeignKey(related_name='children', blank=True, to='regcore.Document', null=True)), 31 | ], 32 | ), 33 | migrations.AlterUniqueTogether( 34 | name='document', 35 | unique_together=set([('doc_type', 'version', 'label_string')]), 36 | ), 37 | migrations.AlterIndexTogether( 38 | name='document', 39 | index_together=set([('doc_type', 'version', 'label_string')]), 40 | ), 41 | ] 42 | -------------------------------------------------------------------------------- /regcore/migrations/0012_migrate_documents.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | import collections 5 | 6 | from django.db import migrations 7 | 8 | import mptt 9 | import mptt.managers 10 | 11 | 12 | def copy_regulations(apps, schema_editor): 13 | Regulation = apps.get_model('regcore', 'Regulation') 14 | Document = apps.get_model('regcore', 'Document') 15 | for reg in Regulation.objects.all(): 16 | data = { 17 | field.name: getattr(reg, field.name) 18 | for field in Regulation._meta.fields 19 | if field.name not in {'parent'} 20 | } 21 | doc = Document(doc_type='cfr', **data) 22 | doc.parent_id = reg.parent_id 23 | doc.save() 24 | 25 | 26 | def uncopy_regulations(apps, schema_editor): 27 | Regulation = apps.get_model('regcore', 'Regulation') 28 | Document = apps.get_model('regcore', 'Document') 29 | for doc in Document.objects.filter(doc_type='cfr'): 30 | data = { 31 | field.name: getattr(doc, field.name) 32 | for field in Regulation._meta.fields 33 | if field.name not in {'parent'} 34 | } 35 | reg = Regulation(**data) 36 | reg.parent_id = doc.parent_id 37 | reg.save() 38 | 39 | 40 | def copy_preambles(apps, schema_editor): 41 | Preamble = apps.get_model('regcore', 'Preamble') 42 | Document = apps.get_model('regcore', 'Document') 43 | 44 | # Bind manager 45 | manager = mptt.managers.TreeManager() 46 | manager.model = Document 47 | mptt.register(Document) 48 | manager.contribute_to_class(Document, 'objects') 49 | 50 | for pre in Preamble.objects.all(): 51 | write_node(Document, pre.data, 'preamble', pre.data['label']) 52 | 53 | 54 | def uncopy_preambles(apps, schema_editor): 55 | Preamble = apps.get_model('regcore', 'Preamble') 56 | Document = apps.get_model('regcore', 'Document') 57 | 58 | # Bind manager 59 | manager = mptt.managers.TreeManager() 60 | manager.model = Document 61 | mptt.register(Document) 62 | manager.contribute_to_class(Document, 'objects') 63 | 64 | for doc in Document.objects.filter(doc_type='preamble', root=True): 65 | nodes = doc.get_descendants(include_self=True) 66 | data = serialize(nodes[0], build_adjacency_map(nodes)) 67 | pre = Preamble(document_number=doc.label_string, data=data) 68 | pre.save() 69 | 70 | 71 | # Copy lightly modified import helpers 72 | 73 | def serialize(pre, adjacency_map): 74 | return { 75 | 'label': pre.label_string.split('-'), 76 | 'text': pre.text, 77 | 'node_type': pre.node_type, 78 | 'children': [ 79 | serialize(child, adjacency_map) 80 | for child in adjacency_map.get(pre.id, []) 81 | ], 82 | } 83 | 84 | 85 | def build_adjacency_map(regs): 86 | """Build mapping from node IDs to child records 87 | :param regs: List of `Regulation` records 88 | """ 89 | ret = collections.defaultdict(list) 90 | for reg in regs: 91 | if reg.parent_id is not None: 92 | ret[reg.parent_id].append(reg) 93 | return ret 94 | 95 | 96 | def write_node(Document, node, doc_type, label_id, version=None): 97 | 98 | to_save = [] 99 | labels_seen = set() 100 | 101 | def add_node(node, parent=None): 102 | label_tuple = tuple(node['label']) 103 | labels_seen.add(label_tuple) 104 | 105 | node['parent'] = parent 106 | to_save.append(node) 107 | for child in node['children']: 108 | add_node(child, parent=node) 109 | add_node(node) 110 | 111 | DMDocuments(Document).bulk_put(to_save, doc_type, label_id, version) 112 | 113 | 114 | def treeify(node, tree_id, pos=1, level=0): 115 | """Set tree properties in memory. 116 | """ 117 | node['tree_id'] = tree_id 118 | node['level'] = level 119 | node['left'] = pos 120 | for child in node.get('children', []): 121 | pos = treeify(child, tree_id, pos=pos + 1, level=level + 1) 122 | pos = pos + 1 123 | node['right'] = pos 124 | return pos 125 | 126 | 127 | def build_id(reg, version=None): 128 | if version is not None: 129 | return '{}:{}'.format(version, '-'.join(reg['label'])) 130 | return '-'.join(reg['label']) 131 | 132 | 133 | class DMDocuments(object): 134 | 135 | def __init__(self, Document): 136 | self.Document = Document 137 | 138 | def _transform(self, reg, doc_type, version=None): 139 | """Create the Django object""" 140 | return self.Document( 141 | id=build_id(reg, version), 142 | doc_type=doc_type, 143 | version=version, 144 | parent_id=( 145 | build_id(reg['parent'], version) 146 | if reg.get('parent') 147 | else None 148 | ), 149 | tree_id=reg['tree_id'], 150 | level=reg['level'], 151 | lft=reg['left'], 152 | rght=reg['right'], 153 | label_string='-'.join(reg['label']), 154 | text=reg['text'], 155 | title=reg.get('title', ''), 156 | node_type=reg['node_type'], 157 | root=(len(reg['label']) == 1), 158 | ) 159 | 160 | def bulk_put(self, regs, doc_type, root_label, version): 161 | self.Document.objects.filter( 162 | version=version, 163 | doc_type=doc_type, 164 | label_string__startswith=root_label, 165 | ).delete() 166 | treeify(regs[0], self.Document.objects._get_next_tree_id()) 167 | self.Document.objects.bulk_create( 168 | [self._transform(r, doc_type, version) for r in regs], 169 | batch_size=25) 170 | 171 | 172 | class Migration(migrations.Migration): 173 | 174 | dependencies = [ 175 | ('regcore', '0011_create_document'), 176 | ] 177 | 178 | operations = [ 179 | migrations.RunPython(copy_regulations, uncopy_regulations), 180 | migrations.RunPython(copy_preambles, uncopy_preambles), 181 | ] 182 | -------------------------------------------------------------------------------- /regcore/migrations/0013_remove_models.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import migrations 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('regcore', '0012_migrate_documents'), 11 | ] 12 | 13 | operations = [ 14 | migrations.DeleteModel( 15 | name='Preamble', 16 | ), 17 | migrations.AlterUniqueTogether( 18 | name='regulation', 19 | unique_together=set([]), 20 | ), 21 | migrations.AlterIndexTogether( 22 | name='regulation', 23 | index_together=set([]), 24 | ), 25 | migrations.RemoveField( 26 | model_name='regulation', 27 | name='parent', 28 | ), 29 | migrations.DeleteModel( 30 | name='Regulation', 31 | ), 32 | ] 33 | -------------------------------------------------------------------------------- /regcore/migrations/0014_auto_20160504_0101.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import unicode_literals 3 | 4 | from django.db import migrations, models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('regcore', '0013_remove_models'), 11 | ] 12 | 13 | operations = [ 14 | migrations.AlterField( 15 | model_name='document', 16 | name='node_type', 17 | field=models.SlugField(max_length=30), 18 | ), 19 | ] 20 | -------------------------------------------------------------------------------- /regcore/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eregs/regulations-core/0b2a2034baacfa1cc5ff87f14db7d1aaa8d260c3/regcore/migrations/__init__.py -------------------------------------------------------------------------------- /regcore/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | from mptt.models import MPTTModel, TreeForeignKey 3 | 4 | from regcore.fields import CompressedJSONField 5 | 6 | 7 | class Document(MPTTModel): 8 | id = models.TextField(primary_key=True) # noqa 9 | doc_type = models.SlugField(max_length=20) 10 | parent = TreeForeignKey('self', null=True, blank=True, 11 | related_name='children', db_index=True) 12 | version = models.SlugField(max_length=20, null=True, blank=True) 13 | label_string = models.SlugField(max_length=200) 14 | text = models.TextField() 15 | title = models.TextField(blank=True) 16 | node_type = models.SlugField(max_length=30) 17 | root = models.BooleanField(default=False, db_index=True) 18 | 19 | class Meta: 20 | index_together = (('doc_type', 'version', 'label_string'),) 21 | unique_together = (('doc_type', 'version', 'label_string'),) 22 | 23 | 24 | class Layer(models.Model): 25 | name = models.SlugField(max_length=20) 26 | layer = CompressedJSONField() 27 | doc_type = models.SlugField(max_length=20) 28 | # We allow doc_ids to contain slashes, which are particularly important 29 | # for CFR docs, which use the [version_id]/[reg_label_id] format. It might 30 | # make sense to split off a version identifier into a separate field in 31 | # the future, if we can't treat that doc_id as an opaque string 32 | doc_id = models.SlugField(max_length=250) 33 | 34 | class Meta: 35 | index_together = (('name', 'doc_type', 'doc_id'),) 36 | unique_together = index_together 37 | 38 | 39 | class Notice(models.Model): 40 | document_number = models.SlugField(max_length=20, primary_key=True) 41 | effective_on = models.DateField(null=True) 42 | fr_url = models.CharField(max_length=200, null=True) 43 | publication_date = models.DateField() 44 | notice = CompressedJSONField() 45 | 46 | 47 | class NoticeCFRPart(models.Model): 48 | """Represents the one-to-many relationship between notices and CFR parts""" 49 | cfr_part = models.SlugField(max_length=10, db_index=True) 50 | notice = models.ForeignKey(Notice) 51 | 52 | class Meta: 53 | index_together = (('notice', 'cfr_part'),) 54 | unique_together = (('notice', 'cfr_part'),) 55 | 56 | 57 | class Diff(models.Model): 58 | label = models.SlugField(max_length=200) 59 | old_version = models.SlugField(max_length=20) 60 | new_version = models.SlugField(max_length=20) 61 | diff = CompressedJSONField() 62 | 63 | class Meta: 64 | index_together = (('label', 'old_version', 'new_version'),) 65 | unique_together = (('label', 'old_version', 'new_version'),) 66 | -------------------------------------------------------------------------------- /regcore/responses.py: -------------------------------------------------------------------------------- 1 | """Helper functions for creating Django HTTP responses""" 2 | import json 3 | 4 | from django.http import Http404, HttpResponse 5 | 6 | 7 | def user_error(reason): 8 | """Silly user, you get a 400""" 9 | obj = json.dumps({'reason': reason}) 10 | return HttpResponse(obj, 'application/json', 400) 11 | 12 | 13 | def success(ret_value=None): 14 | """Respond with either a JSON message or empty body""" 15 | if ret_value is not None: 16 | return HttpResponse(json.dumps(ret_value), 'application/json') 17 | else: 18 | return HttpResponse('', status=204) 19 | 20 | 21 | def four_oh_four(): 22 | """Layer of indirection for 404s. Allows easier migration between web 23 | frameworks""" 24 | raise Http404 25 | -------------------------------------------------------------------------------- /regcore/search_indexes.py: -------------------------------------------------------------------------------- 1 | from haystack import indexes 2 | 3 | from regcore import models 4 | 5 | 6 | class DocumentIndex(indexes.Indexable, indexes.SearchIndex): 7 | """Search index used by Haystack""" 8 | doc_type = indexes.CharField(model_attr='doc_type') 9 | version = indexes.CharField(model_attr='version', null=True) 10 | label_string = indexes.CharField(model_attr='label_string') 11 | text = indexes.CharField(model_attr='text') 12 | is_root = indexes.BooleanField(model_attr='root') 13 | is_subpart = indexes.BooleanField() 14 | title = indexes.MultiValueField() 15 | 16 | regulation = indexes.CharField(model_attr='label_string') 17 | text = indexes.CharField(document=True, use_template=True) 18 | 19 | def prepare_regulation(self, obj): 20 | return obj.label_string.split('-')[0] 21 | 22 | def prepare_is_subpart(self, obj): 23 | return ( 24 | 'Subpart' in obj.label_string or 25 | 'Subjgrp' in obj.label_string 26 | ) 27 | 28 | def prepare_title(self, obj): 29 | """For compatibility reasons, we make this a singleton list""" 30 | if obj.title: 31 | return [obj.title] 32 | else: 33 | return [] 34 | 35 | def get_model(self): 36 | return models.Document 37 | -------------------------------------------------------------------------------- /regcore/settings/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eregs/regulations-core/0b2a2034baacfa1cc5ff87f14db7d1aaa8d260c3/regcore/settings/__init__.py -------------------------------------------------------------------------------- /regcore/settings/base.py: -------------------------------------------------------------------------------- 1 | """Base settings file; used by manage.py. All settings can be overridden via 2 | local_settings.py""" 3 | import os 4 | 5 | from django.utils.crypto import get_random_string 6 | 7 | ALLOWED_HOSTS = [ 8 | value for key, value in os.environ.items() 9 | if key.startswith('ALLOWED_HOST') 10 | ] 11 | 12 | INSTALLED_APPS = [ 13 | 'django.contrib.contenttypes', 14 | 'mptt', 15 | 'haystack', 16 | 'regcore', 17 | 'regcore_read', 18 | 'regcore_write', 19 | ] 20 | MIDDLEWARE_CLASSES = [] 21 | 22 | SECRET_KEY = os.environ.get('DJANGO_SECRET_KEY', get_random_string(50)) 23 | 24 | 25 | DATABASES = { 26 | 'default': { 27 | 'ENGINE': 'django.db.backends.sqlite3', 28 | 'NAME': 'eregs.db' 29 | } 30 | } 31 | 32 | TEMPLATES = [{ 33 | 'BACKEND': 'django.template.backends.django.DjangoTemplates', 34 | 'APP_DIRS': True, 35 | }] 36 | 37 | ROOT_URLCONF = 'regcore.urls' 38 | 39 | DEBUG = True 40 | 41 | # Configurable storage backends, keyed by data_type (e.g. regulations, diffs) 42 | # If a key is not set, defaults to regcore.db.django_models versions 43 | BACKENDS = {} 44 | 45 | ELASTIC_SEARCH_URLS = [] 46 | ELASTIC_SEARCH_INDEX = 'eregs' 47 | 48 | HAYSTACK_CONNECTIONS = { 49 | 'default': { 50 | 'ENGINE': 'haystack.backends.simple_backend.SimpleEngine', 51 | } 52 | } 53 | 54 | LOGGING = { 55 | 'version': 1, 56 | 'disable_existing_loggers': False, 57 | 'handlers': { 58 | 'console': { 59 | 'level': 'INFO', 60 | 'class': 'logging.StreamHandler', 61 | } 62 | }, 63 | 'loggers': { 64 | '': { 65 | 'handlers': ['console'], 66 | 'level': 'INFO', 67 | }, 68 | 'django.request': { 69 | 'handlers': ['console'], 70 | 'propagate': False, 71 | 'level': 'ERROR' 72 | } 73 | } 74 | } 75 | 76 | SEARCH_HANDLER = 'regcore_read.views.haystack_search.search' 77 | 78 | # Batch size used in `bulk_create`; defaults to a conservative value to avoid 79 | # hitting SQLite limits 80 | BATCH_SIZE = 50 81 | 82 | # Lower bound for search results to appear when using pgsql search 83 | PG_SEARCH_RANK_CUTOFF = 0.15 84 | 85 | _envvars = ('HTTP_AUTH_USER', 'HTTP_AUTH_PASSWORD') 86 | for var in _envvars: 87 | globals()[var] = os.environ.get(var) 88 | 89 | try: 90 | from local_settings import * 91 | except ImportError: 92 | pass 93 | -------------------------------------------------------------------------------- /regcore/settings/elastic.py: -------------------------------------------------------------------------------- 1 | from regcore.settings.base import * # noqa 2 | 3 | INSTALLED_APPS.remove('haystack') 4 | BACKENDS = { 5 | 'regulations': 'regcore.db.es.ESRegulations', 6 | 'layers': 'regcore.db.es.ESLayers', 7 | 'notices': 'regcore.db.es.ESNotices', 8 | 'diffs': 'regcore.db.es.ESDiffs' 9 | } 10 | SEARCH_HANDLER = 'regcore_read.views.es_search.search' 11 | -------------------------------------------------------------------------------- /regcore/settings/pgsql.py: -------------------------------------------------------------------------------- 1 | from regcore.settings.base import * # noqa 2 | 3 | INSTALLED_APPS.remove('haystack') 4 | INSTALLED_APPS.extend(['regcore_pgsql', 'django.contrib.postgres']) 5 | SEARCH_HANDLER = 'regcore_pgsql.views.search' 6 | -------------------------------------------------------------------------------- /regcore/templates/search/indexes/regcore/document_text.txt: -------------------------------------------------------------------------------- 1 | {{ object.title|safe }} 2 | {{ object.text|safe }} 3 | -------------------------------------------------------------------------------- /regcore/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eregs/regulations-core/0b2a2034baacfa1cc5ff87f14db7d1aaa8d260c3/regcore/tests/__init__.py -------------------------------------------------------------------------------- /regcore/tests/db_django_models_tests.py: -------------------------------------------------------------------------------- 1 | import copy 2 | from datetime import date 3 | 4 | import pytest 5 | 6 | from regcore.db.django_models import DMDiffs, DMDocuments, DMLayers, DMNotices 7 | from regcore.models import Diff, Document, Layer, Notice 8 | 9 | 10 | @pytest.mark.django_db 11 | def test_get_404(): 12 | assert DMDocuments().get('lablab', 'verver') is None 13 | 14 | 15 | @pytest.mark.django_db 16 | def test_doc_get_cfr(): 17 | Document.objects.create(doc_type='cfr', version='verver', 18 | label_string='a-b', text='ttt', node_type='tyty') 19 | assert DMDocuments().get('cfr', 'a-b', 'verver') == { 20 | 'text': 'ttt', 21 | 'label': ['a', 'b'], 22 | 'lft': 1, 23 | 'children': [], 24 | 'node_type': 'tyty' 25 | } 26 | 27 | 28 | @pytest.mark.django_db 29 | def test_doc_get_preamble(): 30 | Document.objects.create(doc_type='preamble', version='verver', 31 | label_string='a-b', text='ttt', node_type='tyty') 32 | assert DMDocuments().get('preamble', 'a-b', 'verver') == { 33 | 'text': 'ttt', 34 | 'label': ['a', 'b'], 35 | 'lft': 1, 36 | 'children': [], 37 | 'node_type': 'tyty' 38 | } 39 | 40 | 41 | @pytest.mark.django_db 42 | def test_doc_listing(): 43 | dmr = DMDocuments() 44 | Document.objects.create(id='ver1-a-b', doc_type='cfr', version='ver1', 45 | label_string='a-b', text='textex', node_type='ty') 46 | Document.objects.create(id='aaa-a-b', doc_type='cfr', version='aaa', 47 | label_string='a-b', text='textex', node_type='ty') 48 | Document.objects.create(id='333-a-b', doc_type='cfr', version='333', 49 | label_string='a-b', text='textex', node_type='ty') 50 | Document.objects.create(id='four-a-b', doc_type='cfr', version='four', 51 | label_string='a-b', text='textex', node_type='ty') 52 | 53 | assert dmr.listing('cfr', 'a-b') == [ 54 | ('333', 'a-b'), ('aaa', 'a-b'), ('four', 'a-b'), ('ver1', 'a-b')] 55 | 56 | Document.objects.create(id='ver1-1111', doc_type='cfr', version='ver1', 57 | label_string='1111', text='aaaa', node_type='ty', 58 | root=True) 59 | Document.objects.create(id='ver2-1111', doc_type='cfr', version='ver2', 60 | label_string='1111', text='bbbb', node_type='ty', 61 | root=True) 62 | Document.objects.create(id='ver3-1111', doc_type='cfr', version='ver3', 63 | label_string='1111', text='cccc', node_type='ty', 64 | root=False) 65 | 66 | assert dmr.listing('cfr') == [('ver1', '1111'), ('ver2', '1111')] 67 | 68 | 69 | @pytest.mark.django_db 70 | def test_doc_bulk_insert(): 71 | """Writing multiple documents should save correctly. They can be 72 | modified. The lft and rght ids assigned by the Modified Preorder Tree 73 | Traversal algorithm are shown below: 74 | 75 | (1)root(6) 76 | / \ 77 | / \ 78 | (2)n2(3) (4)n3(5) 79 | """ 80 | dmr = DMDocuments() 81 | n2 = {'text': 'some text', 'label': ['111', '2'], 'lft': 2, 82 | 'children': [], 'node_type': 'tyty'} 83 | n3 = {'text': 'other', 'label': ['111', '3'], 'children': [], 'lft': 4, 84 | 'node_type': 'tyty2'} 85 | root = {'text': 'root', 'label': ['111'], 'lft': 1, 86 | 'node_type': 'tyty3', 'children': [n2, n3]} 87 | original = copy.deepcopy(root) 88 | n2['parent'] = root 89 | n3['parent'] = root 90 | nodes = [root, n2, n3] 91 | dmr.bulk_insert(nodes, 'cfr', 'verver') 92 | 93 | assert dmr.get('cfr', '111', 'verver') == original 94 | 95 | root['title'] = original['title'] = 'New Title' 96 | dmr.bulk_delete('cfr', '111', 'verver') 97 | dmr.bulk_insert(nodes, 'cfr', 'verver') 98 | 99 | assert dmr.get('cfr', '111', 'verver') == original 100 | 101 | 102 | @pytest.mark.django_db 103 | def test_layer_get_404(): 104 | assert DMLayers().get('namnam', 'cfr', 'verver/lablab') is None 105 | 106 | 107 | @pytest.mark.django_db 108 | def test_layer_get_success(): 109 | Layer.objects.create(name='namnam', doc_type='cfr', doc_id='verver/lablab', 110 | layer={"some": "body"}) 111 | assert DMLayers().get('namnam', 'cfr', 'verver/lablab') == { 112 | 'some': 'body'} 113 | 114 | 115 | @pytest.mark.django_db 116 | def test_layer_bulk_insert(): 117 | """Writing multiple documents should save correctly. They can be 118 | modified""" 119 | dml = DMLayers() 120 | layers = [{'111-22': [], '111-22-a': [], 'doc_id': 'verver/111-22'}, 121 | {'111-23': [], 'doc_id': 'verver/111-23'}] 122 | dml.bulk_insert(layers, 'name', 'cfr') 123 | 124 | assert Layer.objects.count() == 2 125 | assert dml.get('name', 'cfr', 'verver/111-22') == {'111-22': [], 126 | '111-22-a': []} 127 | assert dml.get('name', 'cfr', 'verver/111-23') == {'111-23': []} 128 | 129 | layers[1] = {'111-23': [1], 'doc_id': 'verver/111-23'} 130 | dml.bulk_delete('name', 'cfr', 'verver/111') 131 | dml.bulk_insert(layers, 'name', 'cfr') 132 | 133 | assert Layer.objects.count() == 2 134 | assert dml.get('name', 'cfr', 'verver/111-23') == {'111-23': [1]} 135 | 136 | 137 | @pytest.mark.django_db 138 | def test_notice_get_404(): 139 | assert DMNotices().get('docdoc') is None 140 | 141 | 142 | @pytest.mark.django_db 143 | def test_notice_get_success(): 144 | Notice.objects.create(document_number='docdoc', fr_url='frfr', 145 | publication_date=date.today(), 146 | notice={"some": "body"}) 147 | assert DMNotices().get('docdoc') == {'some': 'body'} 148 | 149 | 150 | @pytest.mark.django_db 151 | def test_notice_listing(): 152 | dmn = DMNotices() 153 | n = Notice.objects.create(document_number='22', fr_url='fr1', notice={}, 154 | effective_on=date(2005, 5, 5), 155 | publication_date=date(2001, 3, 3)) 156 | n.noticecfrpart_set.create(cfr_part='876') 157 | n = Notice.objects.create(document_number='9', fr_url='fr2', notice={}, 158 | publication_date=date(1999, 1, 1)) 159 | n.noticecfrpart_set.create(cfr_part='876') 160 | n.noticecfrpart_set.create(cfr_part='111') 161 | 162 | assert dmn.listing() == [ 163 | {'document_number': '22', 'fr_url': 'fr1', 164 | 'publication_date': '2001-03-03', 'effective_on': '2005-05-05'}, 165 | {'document_number': '9', 'fr_url': 'fr2', 166 | 'publication_date': '1999-01-01'} 167 | ] 168 | 169 | assert dmn.listing() == dmn.listing('876') 170 | assert dmn.listing('888') == [] 171 | 172 | 173 | @pytest.mark.django_db 174 | def test_notice_insert(): 175 | """We can insert and replace a notice""" 176 | dmn = DMNotices() 177 | doc = {"some": "structure", 178 | 'effective_on': '2011-01-01', 179 | 'fr_url': 'http://example.com', 180 | 'publication_date': '2010-02-02', 181 | 'cfr_parts': ['222']} 182 | dmn.insert('docdoc', doc) 183 | 184 | expected = {"document_number": "docdoc", 185 | "effective_on": date(2011, 1, 1), 186 | "fr_url": "http://example.com", 187 | "publication_date": date(2010, 2, 2), 188 | "noticecfrpart__cfr_part": '222', 189 | "notice": doc} 190 | assert list(Notice.objects.all().values(*expected.keys())) == [expected] 191 | 192 | doc['fr_url'] = 'url2' 193 | dmn.delete('docdoc') 194 | dmn.insert('docdoc', doc) 195 | 196 | expected['fr_url'] = 'url2' 197 | assert list(Notice.objects.all().values(*expected.keys())) == [expected] 198 | 199 | 200 | @pytest.mark.django_db 201 | def test_diff_get_404(): 202 | assert DMDiffs().get('lablab', 'oldold', 'newnew') is None 203 | 204 | 205 | @pytest.mark.django_db 206 | def test_diff_get_success(): 207 | Diff.objects.create(label='lablab', old_version='oldold', 208 | new_version='newnew', diff={"some": "body"}) 209 | 210 | assert DMDiffs().get('lablab', 'oldold', 'newnew') == {'some': 'body'} 211 | 212 | 213 | @pytest.mark.django_db 214 | def test_diff_insert_delete(): 215 | """We can insert and replace a diff""" 216 | dmd = DMDiffs() 217 | dmd.insert('lablab', 'oldold', 'newnew', {"some": "structure"}) 218 | 219 | expected = {"label": "lablab", "old_version": "oldold", 220 | "new_version": "newnew", "diff": {"some": "structure"}} 221 | assert list(Diff.objects.all().values(*expected.keys())) == [expected] 222 | 223 | dmd.delete('lablab', 'oldold', 'newnew') 224 | dmd.insert('lablab', 'oldold', 'newnew', {"other": "structure"}) 225 | expected['diff'] = {'other': 'structure'} 226 | assert list(Diff.objects.all().values(*expected.keys())) == [expected] 227 | -------------------------------------------------------------------------------- /regcore/tests/fields_tests.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | from regcore.fields import CompressedJSONField 4 | 5 | 6 | class CompressesJSONFieldTest(TestCase): 7 | def test_short_json(self): 8 | """Short JSON strings shouldn't get compressed, but should be 9 | invertible""" 10 | field = CompressedJSONField() 11 | to_store = field.get_prep_value({'a': 'dictionary'}) 12 | self.assertEqual(to_store[:2], 'j$') 13 | self.assertIn('dictionary', to_store) 14 | 15 | from_store = field.to_python(to_store) 16 | self.assertEqual(from_store, {'a': 'dictionary'}) 17 | 18 | def test_long_json(self): 19 | """Long JSON objects _do_ get compressed, in addition to being 20 | invertible""" 21 | field = CompressedJSONField() 22 | value = {'key': 'value'*1000} 23 | to_store = field.get_prep_value(value) 24 | self.assertEqual(to_store[:4], 'jb6$') 25 | self.assertNotIn('value', to_store) # because it's been compressed 26 | self.assertTrue(len(to_store) < 1000) 27 | 28 | from_store = field.to_python(to_store) 29 | self.assertEqual(from_store, value) 30 | -------------------------------------------------------------------------------- /regcore/tests/index_tests.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | import pytest 4 | from mock import patch 5 | pytest.importorskip('pyelasticsearch') # noqa 6 | from pyelasticsearch.exceptions import IndexAlreadyExistsError 7 | 8 | from regcore.index import init_schema 9 | 10 | 11 | class IndexTest(TestCase): 12 | 13 | @patch('regcore.index.ElasticSearch') 14 | def test_init_schema(self, es): 15 | init_schema() 16 | self.assertTrue(es.called) 17 | self.assertTrue(es.return_value.create_index.called) 18 | self.assertTrue(es.return_value.put_mapping.called) 19 | 20 | @patch('regcore.index.ElasticSearch') 21 | def test_init_schema_index_exists(self, es): 22 | es.return_value.create_index.side_effect = IndexAlreadyExistsError() 23 | init_schema() 24 | self.assertTrue(es.return_value.put_mapping.called) 25 | -------------------------------------------------------------------------------- /regcore/tests/layer_tests.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | from regcore.layer import standardize_params 4 | 5 | 6 | class LayerParamsTests(TestCase): 7 | def test_old_format(self): 8 | lp = standardize_params('label', 'version') 9 | self.assertEqual(lp.doc_type, 'cfr') 10 | self.assertEqual(lp.doc_id, 'version/label') 11 | self.assertEqual(lp.tree_id, 'label') 12 | 13 | def test_new_format(self): 14 | lp = standardize_params('cfr', 'version/label') 15 | self.assertEqual(lp.doc_type, 'cfr') 16 | self.assertEqual(lp.doc_id, 'version/label') 17 | self.assertEqual(lp.tree_id, 'label') 18 | 19 | lp = standardize_params('preamble', 'docid') 20 | self.assertEqual(lp.doc_type, 'preamble') 21 | self.assertEqual(lp.doc_id, 'docid') 22 | self.assertEqual(lp.tree_id, 'docid') 23 | -------------------------------------------------------------------------------- /regcore/tests/management/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eregs/regulations-core/0b2a2034baacfa1cc5ff87f14db7d1aaa8d260c3/regcore/tests/management/__init__.py -------------------------------------------------------------------------------- /regcore/tests/management/commands/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eregs/regulations-core/0b2a2034baacfa1cc5ff87f14db7d1aaa8d260c3/regcore/tests/management/commands/__init__.py -------------------------------------------------------------------------------- /regcore/tests/management/commands/import_docs_tests.py: -------------------------------------------------------------------------------- 1 | from mock import Mock 2 | 3 | from regcore.management.commands import import_docs 4 | 5 | 6 | def test_scoped_files(tmpdir): 7 | """We get the file path components for all created files, regardless of 8 | how deep in the directory structure""" 9 | a = tmpdir.mkdir('a') 10 | b = tmpdir.mkdir('b') 11 | a.mkdir('1') 12 | a.mkdir('2') 13 | b.mkdir('1').mkdir('i') 14 | 15 | tmpdir.ensure('a', '1', 'i') 16 | tmpdir.ensure('a', '1', 'ii') 17 | tmpdir.ensure('a', '2', 'i') 18 | tmpdir.ensure('b', '1', 'i', 'A') 19 | tmpdir.ensure('b', '1', 'i', 'B') 20 | tmpdir.ensure('b', '1', 'i', 'C') 21 | 22 | result = {tuple(file_parts) 23 | for file_parts in import_docs.scoped_files(str(tmpdir))} 24 | # These should always begin with an empty string due to leaving in the 25 | # trailing slash 26 | assert result == { 27 | ('', 'a', '1', 'i'), ('', 'a', '1', 'ii'), 28 | ('', 'a', '2', 'i'), 29 | ('', 'b', '1', 'i', 'A'), ('', 'b', '1', 'i', 'B'), 30 | ('', 'b', '1', 'i', 'C'), 31 | } 32 | 33 | 34 | def test_save_file(monkeypatch, tmpdir): 35 | """Saving a file should send it to a corresponding url""" 36 | monkeypatch.setattr(import_docs, 'Client', Mock()) 37 | monkeypatch.setattr(import_docs, 'logger', Mock()) 38 | # Client().put 39 | put = import_docs.Client.return_value.put 40 | put.return_value.status_code = 204 41 | 42 | tmpdir.mkdir('a').mkdir('1').join('i').write(b'content') 43 | import_docs.save_file(str(tmpdir), ['', 'a', '1', 'i']) 44 | assert put.call_args == ( 45 | ('/a/1/i',), {'data': b'content', 'content_type': 'application/json'}) 46 | assert import_docs.logger.info.called 47 | 48 | put.reset_mock() 49 | put.return_value.status_code = 404 50 | put.return_value.content = 'a'*1000 51 | import_docs.save_file(str(tmpdir), ['', 'a', '1', 'i']) 52 | assert import_docs.logger.error.called 53 | assert import_docs.logger.error.call_args[0][3] == 'a'*100 # trimmed 54 | -------------------------------------------------------------------------------- /regcore/tests/recipes.py: -------------------------------------------------------------------------------- 1 | from model_mommy.recipe import Recipe 2 | 3 | from regcore.models import Document 4 | 5 | 6 | # Account for mptt-related edge case 7 | doc_recipe = Recipe(Document, lft=None, rght=None, tree_id=None, 8 | _fill_optional=['title']) 9 | -------------------------------------------------------------------------------- /regcore/tests/responses_tests.py: -------------------------------------------------------------------------------- 1 | import json 2 | from unittest import TestCase 3 | 4 | from regcore.responses import success, user_error 5 | 6 | 7 | class ResponsesTest(TestCase): 8 | 9 | def test_user_error(self): 10 | response = user_error("my reason") 11 | self.assertEqual(400, response.status_code) 12 | self.assertEqual('application/json', response['Content-type']) 13 | response_text = response.content.decode('utf-8') 14 | self.assertIn('my reason', response_text) 15 | json.loads(response_text) # valid json 16 | 17 | def test_success_empty(self): 18 | response = success() 19 | self.assertEqual(204, response.status_code) 20 | self.assertEqual(0, len(response.content)) 21 | 22 | def test_success_full(self): 23 | structure = {'some': {'structure': 1}} 24 | response = success(structure) 25 | self.assertEqual(200, response.status_code) 26 | self.assertEqual('application/json', response['Content-type']) 27 | self.assertEqual(structure, 28 | json.loads(response.content.decode('utf-8'))) 29 | -------------------------------------------------------------------------------- /regcore/urls.py: -------------------------------------------------------------------------------- 1 | """URLs file for Django. This will inspect the installed apps and only 2 | include the read/write end points that are associated with the regcore_read 3 | and regcore_write apps""" 4 | 5 | 6 | from collections import defaultdict 7 | 8 | from django.conf import settings 9 | from django.utils.module_loading import import_string 10 | 11 | from regcore.urls_utils import by_verb_url 12 | from regcore_read.views import diff as rdiff 13 | from regcore_read.views import document as rdocument 14 | from regcore_read.views import layer as rlayer 15 | from regcore_read.views import notice as rnotice 16 | from regcore_write.views import diff as wdiff 17 | from regcore_write.views import document as wdocument 18 | from regcore_write.views import layer as wlayer 19 | from regcore_write.views import notice as wnotice 20 | 21 | mapping = defaultdict(dict) 22 | 23 | 24 | if 'regcore_read' in settings.INSTALLED_APPS: 25 | mapping['diff']['GET'] = rdiff.get 26 | mapping['layer']['GET'] = rlayer.get 27 | mapping['notice']['GET'] = rnotice.get 28 | mapping['notices']['GET'] = rnotice.listing 29 | mapping['preamble']['GET'] = rdocument.get 30 | mapping['regulation']['GET'] = rdocument.get 31 | mapping['reg-versions']['GET'] = rdocument.listing 32 | mapping['search']['GET'] = import_string(settings.SEARCH_HANDLER) 33 | 34 | 35 | if 'regcore_write' in settings.INSTALLED_APPS: 36 | # Allow both PUT and POST 37 | for verb in ('PUT', 'POST'): 38 | mapping['diff'][verb] = wdiff.add 39 | mapping['layer'][verb] = wlayer.add 40 | mapping['notice'][verb] = wnotice.add 41 | mapping['preamble'][verb] = wdocument.add 42 | mapping['regulation'][verb] = wdocument.add 43 | mapping['diff']['DELETE'] = wdiff.delete 44 | mapping['layer']['DELETE'] = wlayer.delete 45 | mapping['notice']['DELETE'] = wnotice.delete 46 | mapping['preamble']['DELETE'] = wdocument.delete 47 | mapping['regulation']['DELETE'] = wdocument.delete 48 | 49 | 50 | # Re-usable URL patterns. 51 | def seg(label): 52 | return r'(?P<{0}>[-\w]+)'.format(label) 53 | 54 | 55 | urlpatterns = [ 56 | by_verb_url( 57 | r'^diff/{0}/{1}/{2}$'.format( 58 | seg('label_id'), seg('old_version'), seg('new_version')), 59 | 'diff', mapping['diff']), 60 | by_verb_url( 61 | r'^layer/{0}/{1}/{2}$'.format( 62 | seg('name'), seg('doc_type'), r'(?P[-\w]+(/[-\w]+)*)'), 63 | 'layer', mapping['layer']), 64 | by_verb_url(r'^notice/{0}$'.format(seg('docnum')), 65 | 'notice', mapping['notice']), 66 | by_verb_url( 67 | r'^regulation/{0}/{1}$'.format(seg('label_id'), seg('version')), 68 | 'regulation', mapping['regulation'], kwargs={'doc_type': 'cfr'}), 69 | by_verb_url(r'^notice$', 'notices', mapping['notices']), 70 | by_verb_url(r'^regulation$', 'all-reg-versions', mapping['reg-versions'], 71 | kwargs={'doc_type': 'cfr'}), 72 | by_verb_url(r'^regulation/{0}$'.format(seg('label_id')), 73 | 'reg-versions', mapping['reg-versions'], 74 | kwargs={'doc_type': 'cfr'}), 75 | by_verb_url(r'^preamble/{0}$'.format(seg('label_id')), 'preamble', 76 | mapping['preamble'], kwargs={'doc_type': 'preamble'}), 77 | by_verb_url(r'^search(?:/cfr)?$', 'search', mapping['search'], 78 | kwargs={'doc_type': 'cfr'}), 79 | by_verb_url(r'^search/preamble$', 'search', mapping['search'], 80 | kwargs={'doc_type': 'preamble'}), 81 | ] 82 | -------------------------------------------------------------------------------- /regcore/urls_utils.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls import url 2 | from django.http import Http404, HttpResponseNotAllowed 3 | from django.views.decorators.csrf import csrf_exempt 4 | 5 | 6 | def by_verb_url(regex, name, by_verb, **kwargs): 7 | """Relatively clean way to segment url end points by HTTP Verb. 8 | @regex is the url regex as would be found in urls.py 9 | @name is also as would be found in urls.py 10 | @by_verb is a dictionary from verb (uppercase) to handler""" 11 | def wrapper(request, *args, **kwargs): 12 | verb = request.method.upper() 13 | if not by_verb: 14 | raise Http404 15 | elif verb in by_verb: 16 | return by_verb[verb](request, *args, **kwargs) 17 | else: 18 | return HttpResponseNotAllowed(by_verb.keys()) 19 | if any(getattr(fn, 'csrf_exempt', False) for fn in by_verb.values()): 20 | wrapper = csrf_exempt(wrapper) 21 | 22 | return url(regex, wrapper, name=name, **kwargs) 23 | -------------------------------------------------------------------------------- /regcore_pgsql/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eregs/regulations-core/0b2a2034baacfa1cc5ff87f14db7d1aaa8d260c3/regcore_pgsql/__init__.py -------------------------------------------------------------------------------- /regcore_pgsql/management/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eregs/regulations-core/0b2a2034baacfa1cc5ff87f14db7d1aaa8d260c3/regcore_pgsql/management/__init__.py -------------------------------------------------------------------------------- /regcore_pgsql/management/commands/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eregs/regulations-core/0b2a2034baacfa1cc5ff87f14db7d1aaa8d260c3/regcore_pgsql/management/commands/__init__.py -------------------------------------------------------------------------------- /regcore_pgsql/management/commands/rebuild_pgsql_index.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from django.core.management.base import BaseCommand 4 | from django.db import transaction 5 | 6 | from regcore.models import Document 7 | from regcore_pgsql.models import DocumentIndex 8 | 9 | 10 | logger = logging.getLogger(__name__) 11 | 12 | 13 | def section_documents(): 14 | return Document.objects\ 15 | .filter(label_string__contains='-')\ 16 | .exclude(label_string__regex=r'.*-.*-.*') 17 | 18 | 19 | class Command(BaseCommand): 20 | help = "Rebuild document indexes for searching sections within Postgres" 21 | 22 | @transaction.atomic 23 | def handle(self, *args, **options): 24 | DocumentIndex.objects.all().delete() 25 | count = section_documents().count() 26 | for idx, document in enumerate(section_documents().iterator()): 27 | if idx % 100 == 0: 28 | logger.info('Inserted DocumentIndex %s / %s', idx, count) 29 | DocumentIndex.from_document(document).save() 30 | DocumentIndex.rebuild_search_vectors() 31 | -------------------------------------------------------------------------------- /regcore_pgsql/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.3 on 2017-08-01 18:22 3 | from __future__ import unicode_literals 4 | 5 | import django.contrib.postgres.search 6 | from django.db import migrations, models 7 | import django.db.models.deletion 8 | 9 | 10 | class Migration(migrations.Migration): 11 | 12 | initial = True 13 | 14 | dependencies = [ 15 | ('regcore', '0014_auto_20160504_0101'), 16 | ] 17 | 18 | operations = [ 19 | migrations.CreateModel( 20 | name='DocumentIndex', 21 | fields=[ 22 | ('id', models.AutoField( 23 | auto_created=True, primary_key=True, serialize=False, 24 | verbose_name='ID')), 25 | ('combined_text', models.TextField()), 26 | ('combined_titles', models.TextField()), 27 | ('root_title', models.TextField()), 28 | ('search_vector', 29 | django.contrib.postgres.search.SearchVectorField()), 30 | ('document', models.ForeignKey( 31 | on_delete=django.db.models.deletion.CASCADE, 32 | to='regcore.Document')), 33 | ], 34 | ), 35 | ] 36 | -------------------------------------------------------------------------------- /regcore_pgsql/migrations/0002_documentindex_doc_root.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.3 on 2017-08-04 15:27 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('regcore_pgsql', '0001_initial'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AddField( 16 | model_name='documentindex', 17 | name='doc_root', 18 | field=models.SlugField(default='', max_length=200), 19 | preserve_default=False, 20 | ), 21 | ] 22 | -------------------------------------------------------------------------------- /regcore_pgsql/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eregs/regulations-core/0b2a2034baacfa1cc5ff87f14db7d1aaa8d260c3/regcore_pgsql/migrations/__init__.py -------------------------------------------------------------------------------- /regcore_pgsql/models.py: -------------------------------------------------------------------------------- 1 | from django.contrib.postgres.search import SearchVector, SearchVectorField 2 | from django.db import models 3 | 4 | from regcore.models import Document 5 | 6 | 7 | class DocumentIndex(models.Model): 8 | document = models.ForeignKey(Document, on_delete=models.CASCADE) 9 | combined_text = models.TextField() 10 | combined_titles = models.TextField() 11 | root_title = models.TextField() 12 | doc_root = models.SlugField(max_length=200) # denormalized 13 | 14 | search_vector = SearchVectorField() 15 | 16 | @classmethod 17 | def from_document(cls, document): 18 | doc_and_children = document.get_descendants(include_self=True) 19 | return cls( 20 | document=document, 21 | combined_text='\n'.join( 22 | d.text for d in doc_and_children if d.text), 23 | combined_titles='\n'.join( 24 | d.title for d in doc_and_children if d.title), 25 | root_title=document.title or '', 26 | doc_root=document.label_string.split('-')[0], 27 | ) 28 | 29 | @classmethod 30 | def rebuild_search_vectors(cls): 31 | cls.objects.update(search_vector=( 32 | # note that the root title gets double-counted, as it's also in 33 | # combined_titles 34 | SearchVector('root_title', weight='B') + 35 | SearchVector('combined_titles', weight='A') + 36 | SearchVector('combined_text', weight='B') 37 | )) 38 | -------------------------------------------------------------------------------- /regcore_pgsql/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eregs/regulations-core/0b2a2034baacfa1cc5ff87f14db7d1aaa8d260c3/regcore_pgsql/tests/__init__.py -------------------------------------------------------------------------------- /regcore_pgsql/tests/rebuild_pgsql_index_tests.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from django.core.management import call_command 3 | from mock import Mock 4 | 5 | pytest.importorskip('django', minversion='1.10') # noqa 6 | from regcore.tests.recipes import doc_recipe 7 | from regcore_pgsql.management.commands import rebuild_pgsql_index 8 | from regcore_pgsql.models import DocumentIndex 9 | 10 | 11 | @pytest.mark.django_db 12 | def test_section_documents(): 13 | """Should only grab sections, not the root or paragraphs""" 14 | root = doc_recipe.make(label_string='root') 15 | section1 = doc_recipe.make(label_string='root-1', parent=root) 16 | p1a = doc_recipe.make(label_string='root-1-a', parent=section1) 17 | doc_recipe.make(label_string='root-1-a-1', parent=p1a) 18 | doc_recipe.make(label_string='root-1-b', parent=section1) 19 | doc_recipe.make(label_string='root-2', parent=root) 20 | 21 | results = rebuild_pgsql_index.section_documents() 22 | 23 | assert {d.label_string for d in results} == {'root-1', 'root-2'} 24 | 25 | 26 | @pytest.mark.django_db 27 | def test_creates_index(monkeypatch): 28 | """We should see a new DocumentIndex per section, containing the text of 29 | all children""" 30 | monkeypatch.setattr(rebuild_pgsql_index.DocumentIndex, 31 | 'rebuild_search_vectors', Mock()) 32 | root = doc_recipe.make(label_string='root') 33 | section1 = doc_recipe.make(label_string='root-1', parent=root) 34 | p1a = doc_recipe.make(label_string='root-1-a', parent=section1) 35 | p1a1 = doc_recipe.make(label_string='root-1-a-1', parent=p1a) 36 | p1b = doc_recipe.make(label_string='root-1-b', parent=section1) 37 | section2 = doc_recipe.make(label_string='root-2', parent=root) 38 | p2c = doc_recipe.make(label_string='root-2-c', parent=section2) 39 | 40 | call_command('rebuild_pgsql_index') 41 | rebuild_fn = rebuild_pgsql_index.DocumentIndex.rebuild_search_vectors 42 | assert rebuild_fn.call_count == 1 43 | 44 | indexes = DocumentIndex.objects.order_by('document__label_string') 45 | assert indexes.count() == 2 46 | index1, index2 = indexes 47 | 48 | for d in (root, section2, p2c): 49 | assert d.text not in index1.combined_text 50 | assert d.title not in index1.combined_titles 51 | for d in (section1, p1a, p1a1, p1b): 52 | assert d.text in index1.combined_text 53 | assert d.title in index1.combined_titles 54 | assert section1.title == index1.root_title 55 | 56 | for d in (root, section1, p1a, p1a1, p1b): 57 | assert d.text not in index2.combined_text 58 | assert d.title not in index2.combined_titles 59 | for d in (section2, p2c): 60 | assert d.text in index2.combined_text 61 | assert d.title in index2.combined_titles 62 | assert section2.title == index2.root_title 63 | -------------------------------------------------------------------------------- /regcore_pgsql/tests/views_tests.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from django.db.models import Q 3 | from mock import call, Mock 4 | 5 | pytest.importorskip('django', minversion='1.10') # noqa 6 | from regcore.tests.recipes import doc_recipe 7 | from regcore_pgsql import views 8 | from regcore_read.views.search_utils import SearchArgs 9 | 10 | 11 | def make_queryset_mock(): 12 | """Mocked queryset which returns itself for most manipulations.""" 13 | queryset_mock = Mock() 14 | for transform in ('annotate', 'filter', 'order_by'): 15 | getattr(queryset_mock, transform).return_value = queryset_mock 16 | return queryset_mock 17 | 18 | 19 | def test_matching_sections(monkeypatch, settings): 20 | """Search arguments should be converted to the correct arguments.""" 21 | queryset_mock = make_queryset_mock() 22 | monkeypatch.setattr(views.Document, 'objects', queryset_mock) 23 | settings.PG_SEARCH_RANK_CUTOFF = 0.1234 24 | 25 | result = views.matching_sections(SearchArgs( 26 | q='some terms', version='vvv', regulation='rrr', 27 | is_root=None, is_subpart=None, page=0, page_size=10)) 28 | assert result == queryset_mock 29 | # no point in repeating the exact calls here; test the general flow 30 | assert 'some terms' in str(queryset_mock.annotate.call_args) 31 | assert 'search_vector' in str(queryset_mock.annotate.call_args) 32 | filters = queryset_mock.filter.call_args_list 33 | assert call(rank__gt=0.1234) in filters 34 | assert call(version='vvv') in filters 35 | assert call(documentindex__doc_root='rrr') in filters 36 | 37 | 38 | @pytest.mark.django_db 39 | def test_transform_results(monkeypatch): 40 | """If there's a text match inside a section, we should convert it to a 41 | dictionary.""" 42 | monkeypatch.setattr( # __search isn't supported by sqlite 43 | views, 'Q', Mock(return_value=Q(text__contains='matching'))) 44 | sect = doc_recipe.make(label_string='root-11', title='Sect 111', 45 | version='vvv') 46 | par_a = doc_recipe.make(label_string='root-11-a', parent=sect) 47 | doc_recipe.make(text='matching text', label_string='root-11-a-3', 48 | parent=par_a, title="Match's title") 49 | 50 | results = views.transform_results([sect], 'this is a query') 51 | assert results == [{ 52 | 'text': 'matching text', 53 | 'label': ['root', '11', 'a', '3'], 54 | 'version': 'vvv', 55 | 'regulation': 'root', 56 | 'label_string': 'root-11-a-3', 57 | 'match_title': "Match's title", 58 | 'paragraph_title': "Match's title", 59 | 'section_title': 'Sect 111', 60 | 'title': 'Sect 111', 61 | }] 62 | 63 | 64 | @pytest.mark.django_db 65 | def test_transform_title_match(monkeypatch): 66 | """If there's a title match with no text, we should conver to the correct 67 | dictionary.""" 68 | monkeypatch.setattr( # __search isn't supported by sqlite 69 | views, 'Q', Mock(return_value=Q(title__contains='matching'))) 70 | sect = doc_recipe.make(label_string='root-11', title='Sect 111', 71 | version='vvv') 72 | par_a = doc_recipe.make(label_string='root-11-a', parent=sect, text='', 73 | title='matching title') 74 | doc_recipe.make(label_string='root-11-a-3', parent=par_a, 75 | text='inner text', title='inner title') 76 | 77 | results = views.transform_results([sect], 'this is a query') 78 | assert results == [{ 79 | 'text': 'inner text', 80 | 'label': ['root', '11', 'a'], 81 | 'version': 'vvv', 82 | 'regulation': 'root', 83 | 'label_string': 'root-11-a', 84 | 'match_title': 'matching title', 85 | 'paragraph_title': 'inner title', 86 | 'section_title': 'Sect 111', 87 | 'title': 'Sect 111', 88 | }] 89 | 90 | 91 | @pytest.mark.django_db 92 | def test_transform_no_exact_match(monkeypatch): 93 | """If text is searched text is broken across multiple paragraphs, we 94 | should just graph the first text node we can find.""" 95 | monkeypatch.setattr( # __search isn't supported by sqlite 96 | views, 'Q', Mock(return_value=Q(text=None))) # will have no results 97 | sect = doc_recipe.make(label_string='root-11', text='', title='Sect 111', 98 | version='vvv') 99 | par_a = doc_recipe.make(label_string='root-11-a', parent=sect, 100 | text='has some text', title='nonmatching title') 101 | doc_recipe.make(label_string='root-11-a-3', parent=par_a) 102 | 103 | results = views.transform_results([sect], 'this is a query') 104 | assert results == [{ 105 | 'text': 'has some text', 106 | 'label': ['root', '11'], 107 | 'version': 'vvv', 108 | 'regulation': 'root', 109 | 'label_string': 'root-11', 110 | 'match_title': 'Sect 111', 111 | 'paragraph_title': 'nonmatching title', 112 | 'section_title': 'Sect 111', 113 | 'title': 'Sect 111', 114 | }] 115 | -------------------------------------------------------------------------------- /regcore_pgsql/views.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from django.contrib.postgres.search import SearchRank, SearchQuery 3 | from django.db.models import F, Q 4 | 5 | from regcore.models import Document 6 | from regcore.responses import success 7 | from regcore_read.views.search_utils import requires_search_args 8 | 9 | 10 | def matching_sections(search_args): 11 | """Retrieve all Document sections that match the parsed search args.""" 12 | sections_query = Document.objects\ 13 | .annotate(rank=SearchRank( 14 | F('documentindex__search_vector'), SearchQuery(search_args.q)))\ 15 | .filter(rank__gt=settings.PG_SEARCH_RANK_CUTOFF)\ 16 | .order_by('-rank') 17 | 18 | if search_args.version: 19 | sections_query = sections_query.filter(version=search_args.version) 20 | if search_args.regulation: 21 | sections_query = sections_query.filter( 22 | documentindex__doc_root=search_args.regulation) 23 | # can't filter regulation yet 24 | return sections_query 25 | 26 | 27 | @requires_search_args 28 | def search(request, doc_type, search_args): 29 | sections = matching_sections(search_args) 30 | start = search_args.page * search_args.page_size 31 | end = start + search_args.page_size 32 | 33 | return success({ 34 | 'total_hits': sections.count(), 35 | 'results': transform_results(sections[start:end], search_args.q), 36 | }) 37 | 38 | 39 | def transform_results(sections, search_terms): 40 | """Convert matching Section objects into the corresponding dict for 41 | serialization.""" 42 | final_results = [] 43 | for section in sections: 44 | # TODO: n+1 problem; hypothetically these could all be performed via 45 | # subqueries and annotated on the sections queryset 46 | match_node = section.get_descendants(include_self=True)\ 47 | .filter(Q(text__search=search_terms) | 48 | Q(title__search=search_terms))\ 49 | .first() or section 50 | text_node = match_node.get_descendants(include_self=True)\ 51 | .exclude(text='')\ 52 | .first() 53 | 54 | final_results.append({ 55 | 'text': text_node.text if text_node else '', 56 | 'label': match_node.label_string.split('-'), 57 | 'version': section.version, 58 | 'regulation': section.label_string.split('-')[0], 59 | 'label_string': match_node.label_string, 60 | 'match_title': match_node.title, 61 | 'paragraph_title': text_node.title if text_node else '', 62 | 'section_title': section.title, 63 | 'title': section.title, 64 | }) 65 | return final_results 66 | -------------------------------------------------------------------------------- /regcore_read/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eregs/regulations-core/0b2a2034baacfa1cc5ff87f14db7d1aaa8d260c3/regcore_read/__init__.py -------------------------------------------------------------------------------- /regcore_read/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eregs/regulations-core/0b2a2034baacfa1cc5ff87f14db7d1aaa8d260c3/regcore_read/tests/__init__.py -------------------------------------------------------------------------------- /regcore_read/tests/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls import url 2 | 3 | from regcore_read.views import es_search, haystack_search 4 | 5 | urlpatterns = [ 6 | url(r'^es_search$', es_search.search, kwargs={'doc_type': 'cfr'}), 7 | url( 8 | r'^haystack_search$', 9 | haystack_search.search, 10 | kwargs={'doc_type': 'cfr'}, 11 | ), 12 | ] 13 | -------------------------------------------------------------------------------- /regcore_read/tests/views_diff_tests.py: -------------------------------------------------------------------------------- 1 | import json 2 | from unittest import TestCase 3 | 4 | from django.test.client import Client 5 | from mock import patch 6 | 7 | 8 | class ViewsDiffTest(TestCase): 9 | @patch('regcore_read.views.diff.storage') 10 | def test_get_none(self, storage): 11 | storage.for_diffs.get.return_value = None 12 | response = Client().get('/diff/lablab/oldold/newnew') 13 | self.assertEqual(404, response.status_code) 14 | 15 | @patch('regcore_read.views.diff.storage') 16 | def test_get_empty(self, storage): 17 | storage.for_diffs.get.return_value = {} 18 | response = Client().get('/diff/lablab/oldold/newnew') 19 | self.assertEqual(200, response.status_code) 20 | self.assertEqual({}, json.loads(response.content.decode('utf-8'))) 21 | 22 | @patch('regcore_read.views.diff.storage') 23 | def test_get_results(self, storage): 24 | storage.for_diffs.get.return_value = {'example': 'response'} 25 | response = Client().get('/diff/lablab/oldold/newnew') 26 | self.assertEqual(200, response.status_code) 27 | self.assertEqual({'example': 'response'}, 28 | json.loads(response.content.decode('utf-8'))) 29 | -------------------------------------------------------------------------------- /regcore_read/tests/views_es_search_tests.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from django.test import TestCase, override_settings 3 | from django.test.client import Client 4 | from mock import patch 5 | 6 | pytest.importorskip('pyelasticsearch') # noqa 7 | from regcore_read.views.es_search import transform_results 8 | 9 | 10 | @override_settings(SEARCH_HANDLER='regcore_read.views.es_search.search') 11 | class ViewsESSearchTest(TestCase): 12 | def test_search_missing_q(self): 13 | response = Client().get('/search?non_q=test') 14 | self.assertEqual(400, response.status_code) 15 | 16 | @patch('regcore_read.views.es_search.ElasticSearch') 17 | def test_search_success(self, es): 18 | es.return_value.search.return_value = {'hits': {'hits': [], 19 | 'total': 0}} 20 | response = Client().get('/search?q=test') 21 | self.assertEqual(200, response.status_code) 22 | self.assertTrue(es.called) 23 | self.assertTrue(es.return_value.search.called) 24 | 25 | @patch('regcore_read.views.es_search.ElasticSearch') 26 | def test_search_version(self, es): 27 | es.return_value.search.return_value = {'hits': {'hits': [], 28 | 'total': 0}} 29 | response = Client().get('/search?q=test&version=12345678') 30 | self.assertEqual(200, response.status_code) 31 | self.assertTrue(es.called) 32 | self.assertTrue(es.return_value.search.called) 33 | self.assertIn('12345678', str(es.return_value.search.call_args)) 34 | 35 | @patch('regcore_read.views.es_search.ElasticSearch') 36 | def test_search_version_regulation(self, es): 37 | es.return_value.search.return_value = {'hits': {'hits': [], 38 | 'total': 0}} 39 | response = Client().get('/search?q=test&version=678®ulation=123') 40 | self.assertEqual(200, response.status_code) 41 | self.assertTrue(es.called) 42 | self.assertTrue(es.return_value.search.called) 43 | self.assertIn('678', str(es.return_value.search.call_args)) 44 | self.assertIn('123', str(es.return_value.search.call_args)) 45 | 46 | @patch('regcore_read.views.es_search.ElasticSearch') 47 | def test_search_paging(self, es): 48 | es.return_value.search.return_value = {'hits': {'hits': [], 49 | 'total': 0}} 50 | response = Client().get('/search?q=test&page=5') 51 | self.assertEqual(200, response.status_code) 52 | self.assertTrue(es.called) 53 | self.assertTrue(es.return_value.search.called) 54 | query = es.return_value.search.call_args[0][0] 55 | self.assertEqual(50, query['size']) 56 | self.assertEqual(250, query['from']) 57 | 58 | @patch('regcore_read.views.es_search.ESLayers') 59 | def test_transform_results(self, eslayers): 60 | # combine keyterms and terms into a single layer 61 | eslayers.return_value.get.return_value = { 62 | '2': [{'key_term': 'k2'}], '3': [{'key_term': 'k3'}], 63 | '6': [{'key_term': 'k6'}], '7': [{'key_term': 'k7'}], 64 | 'referenced': { 65 | 'lab1': {'reference': '1', 'term': 'd1'}, 66 | 'lab2': {'reference': '3', 'term': 'd3'}, 67 | 'lab3': {'reference': '5', 'term': 'd5'}, 68 | 'lab4': {'reference': '7', 'term': 'd7'} 69 | } 70 | } 71 | results = transform_results([ 72 | {'regulation': 'r', 'version': 'v', 'label_string': '0'}, 73 | {'regulation': 'rr', 'version': 'v', 'label_string': '1'}, 74 | {'regulation': 'r', 'version': 'vv', 'label_string': '2'}, 75 | {'regulation': 'r', 'version': 'v', 'label_string': '3'}, 76 | {'regulation': 'rr', 'version': 'vv', 'label_string': '4', 77 | 'title': 't4'}, 78 | {'regulation': 'r', 'version': 'vv', 'label_string': '5', 79 | 'title': 't5'}, 80 | {'regulation': 'rr', 'version': 'v', 'label_string': '6', 81 | 'title': 't6'}, 82 | {'regulation': 'r', 'version': 'v', 'label_string': '7', 83 | 'title': 't7'}]) 84 | 85 | self.assertNotIn('title', results[0]) 86 | self.assertEqual('d1', results[1]['title']) 87 | self.assertEqual('k2', results[2]['title']) 88 | self.assertEqual('k3', results[3]['title']) 89 | self.assertEqual('t4', results[4]['title']) 90 | self.assertEqual('t5', results[5]['title']) 91 | self.assertEqual('t6', results[6]['title']) 92 | self.assertEqual('t7', results[7]['title']) 93 | -------------------------------------------------------------------------------- /regcore_read/tests/views_haystack_search_tests.py: -------------------------------------------------------------------------------- 1 | from collections import namedtuple 2 | 3 | import pytest 4 | from django.test import TestCase, override_settings 5 | from django.test.client import Client 6 | from mock import patch 7 | 8 | pytest.importorskip('haystack') # noqa 9 | from regcore_read.views.haystack_search import transform_results 10 | 11 | 12 | @override_settings(SEARCH_HANDLER='regcore_read.views.haystack_search.search') 13 | class ViewsHaystackSearchTest(TestCase): 14 | def test_search_missing_q(self): 15 | response = Client().get('/search?non_q=test') 16 | self.assertEqual(400, response.status_code) 17 | 18 | @patch('regcore_read.views.haystack_search.SearchQuerySet') 19 | def test_search_success(self, sqs): 20 | results = sqs.return_value.models.return_value.filter 21 | results.return_value = [] 22 | response = Client().get('/search?q=test') 23 | self.assertEqual(200, response.status_code) 24 | results.assert_called_with(content='test', doc_type='cfr') 25 | 26 | @patch('regcore_read.views.haystack_search.SearchQuerySet') 27 | def test_search_version(self, sqs): 28 | results = sqs.return_value.models.return_value.filter 29 | version_filter = results.return_value.filter 30 | version_filter.return_value = [] 31 | response = Client().get('/search?q=test&version=12345678') 32 | self.assertEqual(200, response.status_code) 33 | self.assertTrue(version_filter.called) 34 | self.assertEqual('12345678', version_filter.call_args[1]['version']) 35 | 36 | @patch('regcore_read.views.haystack_search.SearchQuerySet') 37 | def test_search_root(self, sqs): 38 | results = sqs.return_value.models.return_value.filter 39 | version_filter = results.return_value.filter 40 | version_filter.return_value = [] 41 | response = Client().get('/search?q=test&is_root=false') 42 | self.assertEqual(200, response.status_code) 43 | version_filter.assert_called_with(is_root=False) 44 | 45 | @patch('regcore_read.views.haystack_search.SearchQuerySet') 46 | def test_search_subpart(self, sqs): 47 | results = sqs.return_value.models.return_value.filter 48 | version_filter = results.return_value.filter 49 | version_filter.return_value = [] 50 | response = Client().get('/search?q=test&is_subpart=true') 51 | self.assertEqual(200, response.status_code) 52 | version_filter.assert_called_with(is_subpart=True) 53 | 54 | @patch('regcore_read.views.haystack_search.SearchQuerySet') 55 | def test_search_subpart_invalid(self, sqs): 56 | results = sqs.return_value.models.return_value.filter 57 | version_filter = results.return_value.filter 58 | version_filter.return_value = [] 59 | response = Client().get('/search?q=test&is_subpart=truetrue') 60 | self.assertEqual(400, response.status_code) 61 | 62 | @patch('regcore_read.views.haystack_search.SearchQuerySet') 63 | def test_search_version_regulation(self, sqs): 64 | results = sqs.return_value.models.return_value.filter 65 | version_filter = results.return_value.filter 66 | regulation_filter = version_filter.return_value.filter 67 | 68 | regulation_filter.return_value = [] 69 | response = Client().get('/search?q=test&version=678®ulation=123') 70 | self.assertEqual(200, response.status_code) 71 | self.assertTrue(regulation_filter.called) 72 | self.assertEqual('678', version_filter.call_args[1]['version']) 73 | self.assertEqual('123', regulation_filter.call_args[1]['regulation']) 74 | 75 | @patch('regcore_read.views.haystack_search.SearchQuerySet') 76 | @patch('regcore_read.views.haystack_search.transform_results') 77 | def test_search_paging(self, transform_results, sqs): 78 | results = sqs.return_value.models.return_value.filter 79 | results.return_value = list(range(500)) 80 | transform_results.return_value = {} 81 | response = Client().get('/search?q=test&page=5') 82 | self.assertEqual(200, response.status_code) 83 | self.assertTrue(results.called) 84 | self.assertTrue(transform_results.called) 85 | self.assertEqual(list(range(250, 300)), 86 | transform_results.call_args[0][0]) 87 | 88 | @patch('regcore_read.views.haystack_search.DMLayers') 89 | def test_transform_results(self, dmlayers): 90 | # combine keyterms and terms into a single layer 91 | dmlayers.return_value.get.return_value = { 92 | '2': [{'key_term': 'k2'}], '3': [{'key_term': 'k3'}], 93 | '6': [{'key_term': 'k6'}], '7': [{'key_term': 'k7'}], 94 | 'referenced': { 95 | 'lab1': {'reference': '1', 'term': 'd1'}, 96 | 'lab2': {'reference': '3', 'term': 'd3'}, 97 | 'lab3': {'reference': '5', 'term': 'd5'}, 98 | 'lab4': {'reference': '7', 'term': 'd7'} 99 | } 100 | } 101 | 102 | Result = namedtuple('Result', ('regulation', 'version', 103 | 'label_string', 'text', 'title')) 104 | results = transform_results([ 105 | Result('r', 'v', '0', '', []), 106 | Result('rr', 'v', '1', '', []), 107 | Result('r', 'vv', '2', '', []), 108 | Result('r', 'v', '3', '', []), 109 | Result('rr', 'vv', '4', '', ['t4']), 110 | Result('r', 'vv', '5', '', ['t5']), 111 | Result('rr', 'v', '6', '', ['t6']), 112 | Result('r', 'v', '7', '', ['t7']), 113 | ]) 114 | 115 | self.assertNotIn('title', results[0]) 116 | self.assertEqual('d1', results[1]['title']) 117 | self.assertEqual('k2', results[2]['title']) 118 | self.assertEqual('k3', results[3]['title']) 119 | self.assertEqual('t4', results[4]['title']) 120 | self.assertEqual('t5', results[5]['title']) 121 | self.assertEqual('t6', results[6]['title']) 122 | self.assertEqual('t7', results[7]['title']) 123 | -------------------------------------------------------------------------------- /regcore_read/tests/views_layer_tests.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from django.test import TestCase 4 | from mock import patch 5 | 6 | 7 | class ViewsLayerTest(TestCase): 8 | 9 | @patch('regcore_read.views.layer.storage') 10 | def test_get_none(self, storage): 11 | url = '/layer/layname/cfr/verver/lablab' 12 | 13 | storage.for_layers.get.return_value = None 14 | response = self.client.get(url) 15 | self.assertEqual(404, response.status_code) 16 | 17 | @patch('regcore_read.views.layer.storage') 18 | def test_get_results(self, storage): 19 | """Verify that a request to GET a specific layer hits the backend with 20 | appropriate version info""" 21 | storage.for_layers.get.return_value = {'example': 'response'} 22 | response = self.client.get('/layer/nnn/cfr/vvv/lll') 23 | self.assertEqual(200, response.status_code) 24 | self.assertEqual(storage.for_layers.get.call_args[0], 25 | ('nnn', 'cfr', 'vvv/lll')) 26 | self.assertEqual({'example': 'response'}, 27 | json.loads(response.content.decode('utf-8'))) 28 | 29 | response = self.client.get('/layer/nnn/preamble/lll') 30 | self.assertEqual(storage.for_layers.get.call_args[0], 31 | ('nnn', 'preamble', 'lll')) 32 | self.assertEqual(200, response.status_code) 33 | self.assertEqual({'example': 'response'}, 34 | json.loads(response.content.decode('utf-8'))) 35 | 36 | @patch('regcore_read.views.layer.storage') 37 | def test_get_results_empty_layer(self, storage): 38 | storage.for_layers.get.return_value = {} 39 | response = self.client.get('/layer/nnn/cfr/vvv/lll') 40 | self.assertEqual(200, response.status_code) 41 | self.assertEqual({}, json.loads(response.content.decode('utf-8'))) 42 | -------------------------------------------------------------------------------- /regcore_read/tests/views_notice_tests.py: -------------------------------------------------------------------------------- 1 | import json 2 | from unittest import TestCase 3 | 4 | from django.test.client import Client 5 | from mock import patch 6 | 7 | 8 | class ViewsNoticeTest(TestCase): 9 | @patch('regcore_read.views.notice.storage') 10 | def test_get_none(self, storage): 11 | storage.for_notices.get.return_value = None 12 | response = Client().get('/notice/docdoc') 13 | self.assertEqual(404, response.status_code) 14 | 15 | @patch('regcore_read.views.notice.storage') 16 | def test_get_empty(self, storage): 17 | storage.for_notices.get.return_value = {} 18 | response = Client().get('/notice/docdoc') 19 | self.assertEqual(200, response.status_code) 20 | self.assertEqual({}, json.loads(response.content.decode('utf-8'))) 21 | 22 | @patch('regcore_read.views.notice.storage') 23 | def test_get_results(self, storage): 24 | storage.for_notices.get.return_value = {'example': 'response'} 25 | response = Client().get('/notice/docdoc') 26 | self.assertEqual(200, response.status_code) 27 | self.assertEqual({'example': 'response'}, 28 | json.loads(response.content.decode('utf-8'))) 29 | 30 | @patch('regcore_read.views.notice.storage') 31 | def test_listing(self, storage): 32 | storage.for_notices.listing.return_value = [1, 2, 3] 33 | response = Client().get('/notice') 34 | self.assertEqual(200, response.status_code) 35 | self.assertEqual({'results': [1, 2, 3]}, 36 | json.loads(response.content.decode('utf-8'))) 37 | -------------------------------------------------------------------------------- /regcore_read/tests/views_preamble_tests.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from django.test import TestCase 4 | from mock import patch 5 | 6 | 7 | class ViewsPreambleTests(TestCase): 8 | @patch('regcore_read.views.document.storage') 9 | def test_get(self, storage): 10 | """We should only give a 404 when we have *no* result. Otherwise, 11 | return the retrieved (possible empty) doc""" 12 | storage.for_documents.get.return_value = None 13 | self.assertEqual(404, self.client.get('/preamble/docdoc').status_code) 14 | 15 | storage.for_documents.get.return_value = {} 16 | response = self.client.get('/preamble/docdoc') 17 | self.assertEqual(200, response.status_code) 18 | self.assertEqual({}, json.loads(response.content.decode('utf-8'))) 19 | 20 | storage.for_documents.get.return_value = {'slightly': 'complex'} 21 | response = self.client.get('/preamble/docdoc') 22 | self.assertEqual(200, response.status_code) 23 | self.assertEqual(json.loads(response.content.decode('utf-8')), 24 | {'slightly': 'complex'}) 25 | -------------------------------------------------------------------------------- /regcore_read/tests/views_regulation_tests.py: -------------------------------------------------------------------------------- 1 | import json 2 | from unittest import TestCase 3 | 4 | from django.test.client import Client 5 | from mock import patch 6 | 7 | 8 | class ViewsRegulationTest(TestCase): 9 | @patch('regcore_read.views.document.storage') 10 | def test_get_good(self, storage): 11 | url = '/regulation/lab/ver' 12 | storage.for_documents.get.return_value = {"some": "thing"} 13 | response = Client().get(url) 14 | self.assertTrue(storage.for_documents.get.called) 15 | args = storage.for_documents.get.call_args[0] 16 | self.assertIn('lab', args) 17 | self.assertIn('ver', args) 18 | self.assertEqual(200, response.status_code) 19 | self.assertEqual({'some': 'thing'}, 20 | json.loads(response.content.decode('utf-8'))) 21 | 22 | @patch('regcore_read.views.document.storage') 23 | def test_get_empty(self, storage): 24 | url = '/regulation/lab/ver' 25 | storage.for_documents.get.return_value = {} 26 | response = Client().get(url) 27 | self.assertTrue(storage.for_documents.get.called) 28 | args = storage.for_documents.get.call_args[0] 29 | self.assertIn('lab', args) 30 | self.assertIn('ver', args) 31 | self.assertEqual(200, response.status_code) 32 | self.assertEqual({}, json.loads(response.content.decode('utf-8'))) 33 | 34 | @patch('regcore_read.views.document.storage') 35 | def test_get_404(self, storage): 36 | url = '/regulation/lab/ver' 37 | storage.for_documents.get.return_value = None 38 | response = Client().get(url) 39 | self.assertTrue(storage.for_documents.get.called) 40 | args = storage.for_documents.get.call_args[0] 41 | self.assertIn('lab', args) 42 | self.assertIn('ver', args) 43 | self.assertEqual(404, response.status_code) 44 | 45 | @patch('regcore_read.views.document.storage') 46 | def test_listing(self, storage): 47 | url = '/regulation/lablab' 48 | storage.for_notices.listing.return_value = [ 49 | {'document_number': '10', 'effective_on': '2010-10-10'}, 50 | {'document_number': '15', 'effective_on': '2010-10-10'}, 51 | {'document_number': '12'}, 52 | {'document_number': '20', 'effective_on': '2011-11-11'}, 53 | {'document_number': '25', 'effective_on': '2011-11-11'} 54 | ] 55 | storage.for_documents.listing.return_value = [ 56 | ('10', 'lablab'), ('15', 'lablab'), ('20', 'lablab')] 57 | 58 | response = Client().get(url) 59 | self.assertEqual(200, response.status_code) 60 | found = [False, False, False] 61 | for ver in json.loads(response.content.decode('utf-8'))['versions']: 62 | if ver['version'] == '10' and 'by_date' not in ver: 63 | found[0] = True 64 | if ver['version'] == '15' and ver['by_date'] == '2010-10-10': 65 | found[1] = True 66 | if ver['version'] == '20' and ver['by_date'] == '2011-11-11': 67 | found[2] = True 68 | self.assertEqual(found, [True, True, True]) 69 | 70 | @patch('regcore_read.views.document.storage') 71 | def test_listing_all(self, storage): 72 | url = '/regulation' 73 | storage.for_notices.listing.return_value = [ 74 | {'document_number': '10', 'effective_on': '2010-10-10'}, 75 | {'document_number': '15', 'effective_on': '2010-10-10'}, 76 | {'document_number': '12'}, 77 | {'document_number': '20', 'effective_on': '2011-11-11'}, 78 | {'document_number': '25', 'effective_on': '2011-11-11'} 79 | ] 80 | storage.for_documents.listing.return_value = [ 81 | ('10', '1111'), ('15', '1111'), ('20', '1212')] 82 | 83 | response = Client().get(url) 84 | self.assertEqual(200, response.status_code) 85 | found = [False, False, False] 86 | for ver in json.loads(response.content.decode('utf-8'))['versions']: 87 | if (ver['version'] == '10' and 'by_date' not in ver and 88 | ver['regulation'] == '1111'): 89 | found[0] = True 90 | if (ver['version'] == '15' and ver['by_date'] == '2010-10-10' and 91 | ver['regulation'] == '1111'): 92 | found[1] = True 93 | if (ver['version'] == '20' and ver['by_date'] == '2011-11-11' and 94 | ver['regulation'] == '1212'): 95 | found[2] = True 96 | self.assertEqual(found, [True, True, True]) 97 | -------------------------------------------------------------------------------- /regcore_read/tests/views_seach_utils_tests.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from regcore_read.views import search_utils 4 | 5 | 6 | def inner_fn(request, search_args): 7 | # We'd generally return a Response here, but we're mocking 8 | return search_args 9 | 10 | 11 | @pytest.mark.parametrize('page_size', ('-10', '0', '200', 'abcd', '---')) 12 | def test_invalid_page_size(page_size, rf): 13 | """Invalid page sizes should not be admitted.""" 14 | view = search_utils.requires_search_args(inner_fn) 15 | result = view(rf.get('?q=term&page_size={0}'.format(page_size))) 16 | assert result.status_code == 400 17 | 18 | 19 | def test_valid_page_size(rf): 20 | """Valid page sizes should pass through.""" 21 | view = search_utils.requires_search_args(inner_fn) 22 | result = view(rf.get('?q=term')) 23 | 24 | result = view(rf.get('?q=term&page_size=10')) 25 | assert result.page_size == 10 26 | -------------------------------------------------------------------------------- /regcore_read/views/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eregs/regulations-core/0b2a2034baacfa1cc5ff87f14db7d1aaa8d260c3/regcore_read/views/__init__.py -------------------------------------------------------------------------------- /regcore_read/views/diff.py: -------------------------------------------------------------------------------- 1 | from regcore.db import storage 2 | from regcore.responses import four_oh_four, success 3 | 4 | 5 | def get(request, label_id, old_version, new_version): 6 | """Find and return the diff with the provided label / versions""" 7 | diff = storage.for_diffs.get(label_id, old_version, new_version) 8 | if diff is not None: 9 | return success(diff) 10 | else: 11 | return four_oh_four() 12 | -------------------------------------------------------------------------------- /regcore_read/views/document.py: -------------------------------------------------------------------------------- 1 | from collections import defaultdict 2 | 3 | from regcore.db import storage 4 | from regcore.responses import four_oh_four, success 5 | 6 | 7 | def listing(request, doc_type, label_id=None): 8 | """List versions of the requested (label_id) regulation; or all regulations 9 | if label_id is None""" 10 | if label_id: 11 | reg_versions = storage.for_documents.listing(doc_type, label_id) 12 | notices = storage.for_notices.listing(label_id.split('-')[0]) 13 | else: 14 | reg_versions = storage.for_documents.listing(doc_type) 15 | notices = storage.for_notices.listing() 16 | 17 | by_date = defaultdict(list) 18 | for notice in (n for n in notices if 'effective_on' in n): 19 | by_date[notice['effective_on']].append(notice) 20 | 21 | regs = [] 22 | for effective_date in sorted(by_date.keys(), reverse=True): 23 | notices = [(n['document_number'], n['effective_on']) 24 | for n in by_date[effective_date]] 25 | notices = sorted(notices, reverse=True) 26 | found_latest = set() 27 | for doc_number, date in notices: 28 | for version, reg_part in reg_versions: 29 | if doc_number == version and reg_part in found_latest: 30 | regs.append({'version': version, 'regulation': reg_part}) 31 | elif doc_number == version: 32 | found_latest.add(reg_part) 33 | regs.append({'version': version, 'by_date': date, 34 | 'regulation': reg_part}) 35 | 36 | if regs: 37 | return success({'versions': regs}) 38 | else: 39 | return four_oh_four() 40 | 41 | 42 | def get(request, doc_type, label_id, version=None): 43 | """Find and return the regulation with this version and label""" 44 | regulation = storage.for_documents.get(doc_type, label_id, version) 45 | if regulation is not None: 46 | return success(regulation) 47 | else: 48 | return four_oh_four() 49 | -------------------------------------------------------------------------------- /regcore_read/views/es_search.py: -------------------------------------------------------------------------------- 1 | """If using the Elastic Search backend, this endpoint provides search 2 | results. If using haystack, see haystack_search.py""" 3 | 4 | from django.conf import settings 5 | from pyelasticsearch import ElasticSearch 6 | 7 | from regcore.db.es import ESLayers 8 | from regcore.responses import success 9 | from regcore_read.views.search_utils import requires_search_args 10 | 11 | 12 | @requires_search_args 13 | def search(request, doc_type, search_args): 14 | """Search elastic search for any matches in the node's text""" 15 | query = { 16 | 'fields': ['text', 'label', 'version', 'regulation', 'title', 17 | 'label_string'], 18 | 'from': search_args.page * search_args.page_size, 19 | 'size': search_args.page_size, 20 | } 21 | text_match = {'match': {'text': search_args.q, 'doc_type': doc_type}} 22 | if search_args.version or search_args.regulation: 23 | term = {} 24 | if search_args.version: 25 | term['version'] = search_args.version 26 | if search_args.regulation: 27 | term['regulation'] = search_args.regulation 28 | if search_args.is_root is not None: 29 | term['is_root'] = search_args.is_root 30 | if search_args.is_subpart is not None: 31 | term['is_subpart'] = search_args.is_subpart 32 | query['query'] = {'filtered': { 33 | 'query': text_match, 34 | 'filter': {'term': term} 35 | }} 36 | else: 37 | query['query'] = text_match 38 | es = ElasticSearch(settings.ELASTIC_SEARCH_URLS) 39 | results = es.search(query, index=settings.ELASTIC_SEARCH_INDEX) 40 | 41 | return success({ 42 | 'total_hits': results['hits']['total'], 43 | 'results': transform_results([h['fields'] for h in 44 | results['hits']['hits']]) 45 | }) 46 | 47 | 48 | def transform_results(results): 49 | """Pull out unused fields, add title field from layers if possible""" 50 | regulations = {(r['regulation'], r['version']) for r in results} 51 | 52 | layers = {} 53 | for regulation, version in regulations: 54 | terms = ESLayers().get('terms', regulation, version) 55 | # We need the references, not the locations of defined terms 56 | if terms: 57 | defined = {} 58 | for term_struct in terms['referenced'].values(): 59 | defined[term_struct['reference']] = term_struct['term'] 60 | terms = defined 61 | layers[(regulation, version)] = { 62 | 'keyterms': ESLayers().get('keyterms', regulation, version), 63 | 'terms': terms 64 | } 65 | 66 | for result in results: 67 | title = result.get('title', '') 68 | ident = (result['regulation'], result['version']) 69 | keyterms = layers[ident]['keyterms'] 70 | terms = layers[ident]['terms'] 71 | if not title and keyterms and result['label_string'] in keyterms: 72 | title = keyterms[result['label_string']][0]['key_term'] 73 | if not title and terms and result['label_string'] in terms: 74 | title = terms[result['label_string']] 75 | 76 | if title: 77 | result['title'] = title 78 | 79 | return results 80 | -------------------------------------------------------------------------------- /regcore_read/views/haystack_search.py: -------------------------------------------------------------------------------- 1 | """If using the haystack backend, this endpoing provides search results. If 2 | using Elastic Search, see es_search.py""" 3 | 4 | from haystack.query import SearchQuerySet 5 | 6 | from regcore.db.django_models import DMLayers 7 | from regcore.models import Document 8 | from regcore.responses import success 9 | from regcore_read.views.search_utils import requires_search_args 10 | 11 | 12 | @requires_search_args 13 | def search(request, doc_type, search_args): 14 | """Use haystack to find search results""" 15 | query = SearchQuerySet().models(Document).filter( 16 | content=search_args.q, doc_type=doc_type) 17 | if search_args.version: 18 | query = query.filter(version=search_args.version) 19 | if search_args.regulation: 20 | query = query.filter(regulation=search_args.regulation) 21 | if search_args.is_root is not None: 22 | query = query.filter(is_root=search_args.is_root) 23 | if search_args.is_subpart is not None: 24 | query = query.filter(is_subpart=search_args.is_subpart) 25 | 26 | start = search_args.page * search_args.page_size 27 | end = start + search_args.page_size 28 | 29 | return success({ 30 | 'total_hits': len(query), 31 | 'results': transform_results(query[start:end]), 32 | }) 33 | 34 | 35 | def transform_results(results): 36 | """Add title field from layers if possible""" 37 | regulations = {(r.regulation, r.version) for r in results} 38 | 39 | layers = {} 40 | for regulation, version in regulations: 41 | terms = DMLayers().get('terms', regulation, version) 42 | # We need the references, not the locations of defined terms 43 | if terms: 44 | defined = {} 45 | for term_struct in terms['referenced'].values(): 46 | defined[term_struct['reference']] = term_struct['term'] 47 | terms = defined 48 | layers[(regulation, version)] = { 49 | 'keyterms': DMLayers().get('keyterms', regulation, version), 50 | 'terms': terms 51 | } 52 | 53 | final_results = [] 54 | for result in results: 55 | transformed = { 56 | 'text': result.text, 57 | 'label': result.label_string.split('-'), 58 | 'version': result.version, 59 | 'regulation': result.regulation, 60 | 'label_string': result.label_string 61 | } 62 | 63 | if result.title: 64 | title = result.title[0] 65 | else: 66 | title = None 67 | ident = (result.regulation, result.version) 68 | keyterms = layers[ident]['keyterms'] 69 | terms = layers[ident]['terms'] 70 | if not title and keyterms and result.label_string in keyterms: 71 | title = keyterms[result.label_string][0]['key_term'] 72 | if not title and terms and result.label_string in terms: 73 | title = terms[result.label_string] 74 | 75 | if title: 76 | transformed['title'] = title 77 | 78 | final_results.append(transformed) 79 | 80 | return final_results 81 | -------------------------------------------------------------------------------- /regcore_read/views/layer.py: -------------------------------------------------------------------------------- 1 | from regcore.db import storage 2 | from regcore.layer import standardize_params 3 | from regcore.responses import four_oh_four, success 4 | 5 | 6 | def get(request, name, doc_type, doc_id): 7 | """Find and return the layer with this name, referring to this doc_id""" 8 | params = standardize_params(doc_type, doc_id) 9 | layer = storage.for_layers.get(name, params.doc_type, params.doc_id) 10 | if layer is not None: 11 | return success(layer) 12 | else: 13 | return four_oh_four() 14 | -------------------------------------------------------------------------------- /regcore_read/views/notice.py: -------------------------------------------------------------------------------- 1 | from regcore.db import storage 2 | from regcore.responses import four_oh_four, success 3 | 4 | 5 | def get(request, docnum): 6 | """Find and return the notice with this docnum""" 7 | notice = storage.for_notices.get(docnum) 8 | if notice is not None: 9 | return success(notice) 10 | else: 11 | return four_oh_four() 12 | 13 | 14 | def listing(request): 15 | """Find and return all notices""" 16 | return success({ 17 | 'results': storage.for_notices.listing( 18 | request.GET.get('part', None)) 19 | }) 20 | -------------------------------------------------------------------------------- /regcore_read/views/search_utils.py: -------------------------------------------------------------------------------- 1 | from collections import namedtuple 2 | from functools import wraps 3 | 4 | from webargs import fields, validate, ValidationError 5 | from webargs.djangoparser import parser 6 | 7 | from regcore.responses import user_error 8 | 9 | MAX_PAGE_SIZE = 50 10 | 11 | search_args = { 12 | 'q': fields.Str(required=True), 13 | 'version': fields.Str(missing=None), 14 | 'regulation': fields.Str(missing=None), 15 | 'is_root': fields.Bool(missing=None), 16 | 'is_subpart': fields.Bool(missing=None), 17 | 'page': fields.Int(missing=0), 18 | 'page_size': fields.Int(missing=MAX_PAGE_SIZE, 19 | validate=validate.Range(1, MAX_PAGE_SIZE)), 20 | } 21 | SearchArgs = namedtuple( 22 | 'SearchArgs', 23 | ['q', 'version', 'regulation', 'is_root', 'is_subpart', 'page', 24 | 'page_size']) 25 | 26 | 27 | def requires_search_args(view): 28 | """Wraps a view in a validation test for search arguments. Passes the 29 | correctly-parsed SearchArgs through if there's no problem""" 30 | @wraps(view) 31 | def wrapper(request, *args, **kwargs): 32 | try: 33 | user_args = parser.parse(search_args, request) 34 | except ValidationError as err: 35 | return user_error(err.messages) 36 | return view(request, *args, search_args=SearchArgs(**user_args), 37 | **kwargs) 38 | return wrapper 39 | -------------------------------------------------------------------------------- /regcore_write/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eregs/regulations-core/0b2a2034baacfa1cc5ff87f14db7d1aaa8d260c3/regcore_write/__init__.py -------------------------------------------------------------------------------- /regcore_write/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eregs/regulations-core/0b2a2034baacfa1cc5ff87f14db7d1aaa8d260c3/regcore_write/tests/__init__.py -------------------------------------------------------------------------------- /regcore_write/tests/views_diff_tests.py: -------------------------------------------------------------------------------- 1 | import json 2 | from unittest import TestCase 3 | 4 | from django.test.client import Client 5 | from mock import patch 6 | 7 | 8 | class ViewsDiffTest(TestCase): 9 | 10 | def test_add_not_json(self): 11 | url = '/diff/lablab/oldold/newnew' 12 | 13 | response = Client().put(url, content_type='application/json', 14 | data='{Invalid}') 15 | self.assertEqual(400, response.status_code) 16 | 17 | @patch('regcore_write.views.diff.storage') 18 | def test_add_label_success(self, storage): 19 | url = '/diff/lablab/oldold/newnew' 20 | 21 | Client().put(url, content_type='application/json', 22 | data=json.dumps({'some': 'struct'})) 23 | args = storage.for_diffs.insert.call_args[0] 24 | self.assertEqual(('lablab', 'oldold', 'newnew', {'some': 'struct'}), 25 | args) 26 | -------------------------------------------------------------------------------- /regcore_write/tests/views_layer_tests.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from django.test import TestCase 4 | from mock import patch 5 | 6 | from regcore.layer import standardize_params 7 | from regcore_write.views import layer 8 | 9 | 10 | class ViewsLayerTest(TestCase): 11 | def put(self, data, name='layname', doc_type='cfr', 12 | doc_id='verver/lablab'): 13 | """Shorthand function for PUTing data to a view layer""" 14 | url = '/layer/{0}/{1}/{2}'.format(name, doc_type, doc_id) 15 | if isinstance(data, dict): 16 | data = json.dumps(data) 17 | return self.client.put(url, content_type='application/json', 18 | data=data) 19 | 20 | def test_add_not_json(self): 21 | """Non-JSON is invalid""" 22 | response = self.put('{Invalid}') 23 | self.assertEqual(400, response.status_code) 24 | 25 | def test_add_label_mismatch(self): 26 | """Root label must match that found in the url""" 27 | response = self.put({'nonlab': []}, doc_id='verver/lablab') 28 | self.assertEqual(400, response.status_code) 29 | 30 | def put_with_mock_data(self, message, *labels, **kwargs): 31 | """Helper function to mock out the value returned from fetching the 32 | regulation from the database, then put the provided message, and 33 | return the response.""" 34 | root = [] 35 | children = root 36 | for label in labels: 37 | node = {'label': label.split('-'), 'children': []} 38 | children.append(node) 39 | children = node['children'] 40 | 41 | with patch('regcore_write.views.layer.storage') as storage: 42 | storage.for_documents.get.return_value = root[0] 43 | self.put(message, **kwargs) 44 | self.assertTrue(storage.for_layers.bulk_insert.called) 45 | layers_saved = storage.for_layers.bulk_insert.call_args[0][0] 46 | return list(reversed(layers_saved)) # switch to outside in 47 | 48 | def test_add_success(self): 49 | """Can correctly add layer data when node is present in the db""" 50 | message = { 51 | 'lablab': [1, 2], 52 | 'lablab-b': [2, 3], 53 | 'lablab-b-4': [3, 4], 54 | } 55 | l1, l2, l3 = self.put_with_mock_data( 56 | message, 'lablab', 'lablab-b', 'lablab-b-4', doc_type='cfr', 57 | doc_id='verver/lablab') 58 | 59 | message['doc_id'] = 'verver/lablab' 60 | self.assertEqual(message, l1) 61 | 62 | # Sub layers have fewer elements 63 | del message['lablab'] 64 | message['doc_id'] = 'verver/lablab-b' 65 | self.assertEqual(message, l2) 66 | del message['lablab-b'] 67 | message['doc_id'] = 'verver/lablab-b-4' 68 | self.assertEqual(message, l3) 69 | 70 | def test_add_skip_level(self): 71 | """Can correctly add layer data, even when skipping a node""" 72 | message = { 73 | 'lablab': [1, 2], 74 | 'lablab-b-4': [3, 4], 75 | } 76 | l1, l2, l3 = self.put_with_mock_data( 77 | message, 'lablab', 'lablab-b', 'lablab-b-4') 78 | 79 | message['doc_id'] = 'verver/lablab' 80 | self.assertEqual(message, l1) 81 | # Sub layers have fewer elements 82 | del message['lablab'] 83 | message['doc_id'] = 'verver/lablab-b' 84 | self.assertEqual(message, l2) 85 | message['doc_id'] = 'verver/lablab-b-4' 86 | self.assertEqual(message, l3) 87 | 88 | def test_add_interp_children(self): 89 | """Can correctly add layer data to interpretations""" 90 | message = {'99-5-Interp': [1, 2], '99-5-a-Interp': [3, 4]} 91 | l1, l2, l3, l4 = self.put_with_mock_data( 92 | message, '99', '99-Interp', '99-5-Interp', '99-5-a-Interp', 93 | doc_id='verver/99') 94 | for saved in (l1, l2, l3): 95 | self.assertIn('99-5-Interp', saved) 96 | self.assertIn('99-5-a-Interp', saved) 97 | self.assertNotIn('99-5-Interp', l4) 98 | self.assertIn('99-5-a-Interp', l4) 99 | 100 | def test_add_subpart_children(self): 101 | """Can correctly add layer data to subparts""" 102 | message = {'99-1': [1, 2], '99-1-a': [3, 4]} 103 | l1, l2, l3, l4 = self.put_with_mock_data( 104 | message, '99', '99-Subpart-A', '99-1', '99-1-a', 105 | doc_id='verver/99') 106 | for saved in (l1, l2, l3): 107 | self.assertIn('99-1', saved) 108 | self.assertIn('99-1-a', saved) 109 | self.assertNotIn('99-1', l4) 110 | self.assertIn('99-1-a', l4) 111 | 112 | def test_add_referenced(self): 113 | """The 'referenced' key is special; it should get added""" 114 | message = {'99-1': [1, 2], '99-1-a': [3, 4], 'referenced': [5, 6]} 115 | layers_saved = self.put_with_mock_data( 116 | message, '99', '99-Subpart-A', '99-1', '99-1-a', 117 | doc_id='verver/99') 118 | self.assertEqual(4, len(layers_saved)) 119 | self.assertTrue(all('referenced' in saved for saved in layers_saved)) 120 | 121 | @patch('regcore_write.views.layer.storage') 122 | def test_add_preamble_layer(self, storage): 123 | """If adding layer data to a preamble, we should see layers saved for 124 | each level of the preamble tree. This requires we construct a fake 125 | preamble.""" 126 | storage.for_documents.get.return_value = None 127 | storage.for_documents.get.return_value = dict( 128 | label=['111_22'], children=[ 129 | dict(label=['111_22', '1'], children=[]), 130 | dict(label=['111_22', '2'], children=[ 131 | dict(label=['111_22', '2', 'a'], children=[])]), 132 | dict(label=['111_22', '3'], children=[ 133 | dict(label=['111_22', '3', 'a'], children=[ 134 | dict(label=['111_22', '3', 'a', 'i'], children=[])]), 135 | dict(label=['111_22', '3', 'b'], children=[ 136 | dict(label=['111_22', '3', 'b', 'i'], children=[])]) 137 | ])]) 138 | message = {'111_22': 'layer1', '111_22-3': 'layer2', 139 | '111_22-3-b': 'layer3'} 140 | self.client.put('/layer/aname/preamble/111_22', 141 | data=json.dumps(message)) 142 | stored = storage.for_layers.bulk_insert.call_args[0][0] 143 | self.assertEqual(len(stored), 9) 144 | for label in ('111_22-1', '111_22-2', '111_22-2-a', '111_22-3-a', 145 | '111_22-3-a-i', '111_22-3-b-i'): 146 | self.assertIn({'doc_id': label}, stored) # i.e. empty 147 | self.assertIn({'doc_id': '111_22', '111_22': 'layer1', 148 | '111_22-3': 'layer2', '111_22-3-b': 'layer3'}, stored) 149 | self.assertIn({'doc_id': '111_22-3', '111_22-3': 'layer2', 150 | '111_22-3-b': 'layer3'}, stored) 151 | self.assertIn({'doc_id': '111_22-3-b', '111_22-3-b': 'layer3'}, 152 | stored) 153 | 154 | @patch('regcore_write.views.layer.storage') 155 | def test_child_layers_no_results(self, storage): 156 | """If the db returns no regulation data, nothing should get saved""" 157 | storage.for_documents.get.return_value = None 158 | layer_params = standardize_params('cfr', 'vvv/lll') 159 | self.assertEqual([], layer.child_layers(layer_params, {})) 160 | self.assertTrue(storage.for_documents.get.called) 161 | storage.for_documents.get.assert_called_with('cfr', 'lll', 'vvv') 162 | 163 | storage.for_documents.get.return_value = None 164 | layer_params = standardize_params('preamble', 'docdoc') 165 | self.assertEqual([], layer.child_layers(layer_params, {})) 166 | storage.for_documents.get.assert_called_with('preamble', 'docdoc') 167 | 168 | def test_child_label_of(self): 169 | """Correctly determine relationships between labels""" 170 | self.assertTrue(layer.child_label_of('1005-5-a-1-Interp-1', 171 | '1005-5-Interp')) 172 | -------------------------------------------------------------------------------- /regcore_write/tests/views_notice_tests.py: -------------------------------------------------------------------------------- 1 | import json 2 | from unittest import TestCase 3 | 4 | from django.test.client import Client 5 | from mock import patch 6 | 7 | 8 | class ViewsNoticeTest(TestCase): 9 | 10 | def test_add_not_json(self): 11 | url = '/notice/docdoc' 12 | 13 | response = Client().put(url, content_type='application/json', 14 | data='{Invalid}') 15 | self.assertEqual(400, response.status_code) 16 | 17 | @patch('regcore_write.views.notice.storage') 18 | def test_add_label_success(self, storage): 19 | url = '/notice/docdoc' 20 | 21 | Client().put(url, content_type='application/json', 22 | data=json.dumps({'some': 'struct'})) 23 | self.assertTrue(storage.for_notices.insert.called) 24 | args = storage.for_notices.insert.call_args[0] 25 | self.assertEqual('docdoc', args[0]) 26 | self.assertEqual({'some': 'struct', 'cfr_parts': []}, args[1]) 27 | 28 | Client().put(url, content_type='application/json', 29 | data=json.dumps({'some': 'struct', 'cfr_part': '1111'})) 30 | self.assertTrue(storage.for_notices.insert.called) 31 | args = storage.for_notices.insert.call_args[0] 32 | self.assertEqual('docdoc', args[0]) 33 | self.assertEqual({'some': 'struct', 'cfr_parts': ['1111']}, args[1]) 34 | 35 | Client().put( 36 | url, content_type='application/json', 37 | data=json.dumps({'some': 'struct', 'cfr_parts': ['111', '222']})) 38 | self.assertTrue(storage.for_notices.insert.called) 39 | args = storage.for_notices.insert.call_args[0] 40 | self.assertEqual('docdoc', args[0]) 41 | self.assertEqual({'some': 'struct', 'cfr_parts': ['111', '222']}, 42 | args[1]) 43 | -------------------------------------------------------------------------------- /regcore_write/tests/views_preamble_tests.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from django.test import TestCase 4 | from mock import patch 5 | 6 | 7 | class ViewsPreambleTests(TestCase): 8 | def test_invalid_json(self): 9 | """Only accepts valid JSON""" 10 | response = self.client.put( 11 | '/preamble/id_here', content_type='application/json', 12 | data='{Invalid}') 13 | self.assertEqual(400, response.status_code) 14 | 15 | @patch('regcore_write.views.document.storage') 16 | def test_stores(self, storage): 17 | """Stores any JSON it is given""" 18 | data = { 19 | 'text': 'text', 20 | 'label': ['label'], 21 | 'children': [], 22 | } 23 | self.client.put( 24 | '/preamble/label', 25 | content_type='application/json', 26 | data=json.dumps(data), 27 | ) 28 | bulk_data = dict(data) 29 | bulk_data['parent'] = None 30 | storage.for_documents.bulk_delete.assert_called_with( 31 | 'preamble', 'label', None, 32 | ) 33 | storage.for_documents.bulk_insert.assert_called_with( 34 | [bulk_data], 'preamble', None, 35 | ) 36 | -------------------------------------------------------------------------------- /regcore_write/tests/views_regulation_tests.py: -------------------------------------------------------------------------------- 1 | import json 2 | from unittest import TestCase 3 | 4 | from django.test.client import Client 5 | from mock import patch 6 | 7 | 8 | class ViewsRegulationTest(TestCase): 9 | 10 | def test_add_not_json(self): 11 | url = '/regulation/lablab/verver' 12 | 13 | response = Client().put(url, data=json.dumps( 14 | {'text': '', 'child': [], 'label': []})) 15 | self.assertEqual(400, response.status_code) 16 | 17 | response = Client().put(url, content_type='application/json', 18 | data='{Invalid}') 19 | self.assertEqual(400, response.status_code) 20 | 21 | def test_add_invalid_json(self): 22 | url = '/regulation/lablab/verver' 23 | 24 | response = Client().put(url, content_type='application/json', 25 | data=json.dumps({'incorrect': 'schema'})) 26 | self.assertEqual(400, response.status_code) 27 | 28 | message = {'text': '', 'label': []} 29 | response = Client().put(url, content_type='application/json', 30 | data=json.dumps(message)) 31 | self.assertEqual(400, response.status_code) 32 | 33 | def test_add_label_mismatch(self): 34 | url = '/regulation/lablab/verver' 35 | 36 | message = {'text': '', 'children': [], 37 | 'label': ['notlablab']} 38 | response = Client().put(url, content_type='application/json', 39 | data=json.dumps(message)) 40 | self.assertEqual(400, response.status_code) 41 | 42 | @patch('regcore_write.views.document.storage') 43 | def test_add_label_success(self, storage): 44 | url = '/regulation/p/verver' 45 | 46 | message = { 47 | 'text': 'parent text', 48 | 'label': ['p'], 49 | 'node_type': 'reg_text', 50 | 'children': [{ 51 | 'text': 'child1', 52 | 'label': ['p', 'c1'], 53 | 'node_type': 'reg_text', 54 | 'children': [] 55 | }, { 56 | 'text': 'child2', 57 | 'label': ['p', 'c2'], 58 | 'title': 'My Title', 59 | 'node_type': 'reg_text', 60 | 'children': [] 61 | }] 62 | } 63 | 64 | Client().put(url, content_type='application/json', 65 | data=json.dumps(message)) 66 | self.assertTrue(storage.for_documents.bulk_insert.called) 67 | bulk_insert_args = storage.for_documents.bulk_insert.call_args[0] 68 | self.assertEqual(3, len(bulk_insert_args[0])) 69 | found = [False, False, False] 70 | for arg in bulk_insert_args[0]: 71 | if arg['label'] == ['p']: 72 | found[0] = True 73 | if arg['label'] == ['p', 'c1']: 74 | found[1] = True 75 | if arg['label'] == ['p', 'c2']: 76 | found[2] = True 77 | self.assertEqual(found, [True, True, True]) 78 | 79 | storage.for_documents.bulk_insert.reset_mock() 80 | Client().post(url, content_type='application/json', 81 | data=json.dumps(message)) 82 | self.assertTrue(storage.for_documents.bulk_insert.called) 83 | bulk_insert_args = storage.for_documents.bulk_insert.call_args[0] 84 | self.assertEqual(3, len(bulk_insert_args[0])) 85 | 86 | @patch('regcore_write.views.document.storage') 87 | def test_add_empty_children(self, storage): 88 | url = '/regulation/p/verver' 89 | 90 | message = { 91 | 'text': 'parent text', 92 | 'label': ['p'], 93 | 'children': [] 94 | } 95 | Client().put(url, content_type='application/json', 96 | data=json.dumps(message)) 97 | self.assertTrue(storage.for_documents.bulk_insert.called) 98 | bulk_insert_args = storage.for_documents.bulk_insert.call_args[0] 99 | self.assertEqual(1, len(bulk_insert_args[0])) 100 | -------------------------------------------------------------------------------- /regcore_write/tests/views_security_tests.py: -------------------------------------------------------------------------------- 1 | import base64 2 | 3 | from django.http import HttpResponse 4 | from django.test import TestCase 5 | from django.test.client import RequestFactory 6 | from django.test.utils import override_settings 7 | 8 | from regcore_write.views import security 9 | 10 | 11 | def _wrapped_fn(request): 12 | return HttpResponse(status=204) 13 | 14 | 15 | def _encode(username, password): 16 | as_unicode = '{0}:{1}'.format(username, password).encode() 17 | encoded = base64.b64encode(as_unicode).decode('utf-8') 18 | return 'Basic ' + encoded 19 | 20 | 21 | class SecurityTest(TestCase): 22 | @override_settings(HTTP_AUTH_USER="a_user", HTTP_AUTH_PASSWORD="a_pass") 23 | def test_secure_write(self): 24 | """Basic Auth must match the configuration""" 25 | fn = security.secure_write(_wrapped_fn) 26 | 27 | request = RequestFactory().get('/') 28 | self.assertEqual(fn(request).status_code, 401) 29 | 30 | request = RequestFactory().get( 31 | '/', HTTP_AUTHORIZATION=_encode('wrong', 'pass')) 32 | self.assertEqual(fn(request).status_code, 401) 33 | 34 | request = RequestFactory().get( 35 | '/', HTTP_AUTHORIZATION=_encode('a_user', 'pass')) 36 | self.assertEqual(fn(request).status_code, 401) 37 | 38 | request = RequestFactory().get( 39 | '/', HTTP_AUTHORIZATION=_encode('wrong', 'a_pass')) 40 | self.assertEqual(fn(request).status_code, 401) 41 | 42 | request = RequestFactory().get( 43 | '/', HTTP_AUTHORIZATION=_encode('a_user', 'a_pass')) 44 | self.assertEqual(fn(request).status_code, 204) 45 | 46 | @override_settings(HTTP_AUTH_USER=None, HTTP_AUTH_PASSWORD=None) 47 | def test_secure_write_unset(self): 48 | """Basic Auth should not be required when the environment isn't set""" 49 | fn = security.secure_write(_wrapped_fn) 50 | request = RequestFactory().get('/') 51 | self.assertEqual(fn(request).status_code, 204) 52 | 53 | @override_settings(HTTP_AUTH_USER="", HTTP_AUTH_PASSWORD="") 54 | def test_secure_write_empty(self): 55 | """Basic Auth should not be required when the environment is empty""" 56 | fn = security.secure_write(_wrapped_fn) 57 | request = RequestFactory().get('/') 58 | self.assertEqual(fn(request).status_code, 204) 59 | -------------------------------------------------------------------------------- /regcore_write/views/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eregs/regulations-core/0b2a2034baacfa1cc5ff87f14db7d1aaa8d260c3/regcore_write/views/__init__.py -------------------------------------------------------------------------------- /regcore_write/views/diff.py: -------------------------------------------------------------------------------- 1 | from regcore.db import storage 2 | from regcore.responses import success 3 | from regcore_write.views.security import json_body, secure_write 4 | 5 | 6 | @secure_write 7 | @json_body 8 | def add(request, label_id, old_version, new_version): 9 | """Add the diff to the db, indexed by the label and versions""" 10 | # @todo: write a schema that verifies the diff's structure 11 | storage.for_diffs.delete(label_id, old_version, new_version) 12 | storage.for_diffs.insert( 13 | label_id, old_version, new_version, request.json_body) 14 | return success() 15 | 16 | 17 | @secure_write 18 | def delete(request, label_id, old_version, new_version): 19 | """Delete the diff from the db""" 20 | storage.for_diffs.delete(label_id, old_version, new_version) 21 | return success() 22 | -------------------------------------------------------------------------------- /regcore_write/views/document.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import jsonschema 4 | 5 | from regcore.db import storage 6 | from regcore.responses import success, user_error 7 | from regcore_write.views.security import json_body, secure_write 8 | 9 | # This JSON schema is used to validate the regulation data provided 10 | REGULATION_SCHEMA = { 11 | 'type': 'object', 12 | 'id': 'reg_tree_node', 13 | 'additionalProperties': False, 14 | 'required': ['text', 'children', 'label'], 15 | 'properties': { 16 | 'text': {'type': 'string'}, 17 | 'children': { 18 | 'type': 'array', 19 | 'additionalItems': False, 20 | 'items': {'$ref': 'reg_tree_node'} 21 | }, 22 | 'label': { 23 | 'type': 'array', 24 | 'additionalItems': False, 25 | 'items': {'type': 'string'} 26 | }, 27 | 'title': {'type': 'string'}, 28 | 'node_type': {'type': 'string'} 29 | } 30 | } 31 | 32 | 33 | @secure_write 34 | @json_body 35 | def add(request, doc_type, label_id, version=None): 36 | """Add this document node and all of its children to the db""" 37 | try: 38 | node = request.json_body 39 | jsonschema.validate(node, REGULATION_SCHEMA) 40 | except jsonschema.ValidationError: 41 | return user_error("JSON is invalid") 42 | 43 | if label_id != '-'.join(node['label']): 44 | return user_error('label mismatch') 45 | 46 | write_node(node, doc_type, label_id, version) 47 | return success() 48 | 49 | 50 | def write_node(node, doc_type, label_id, version): 51 | 52 | to_save = [] 53 | labels_seen = set() 54 | 55 | def add_node(node, parent=None): 56 | label_tuple = tuple(node['label']) 57 | if label_tuple in labels_seen: 58 | logging.warning("Repeat label: %s", label_tuple) 59 | labels_seen.add(label_tuple) 60 | 61 | node['parent'] = parent 62 | to_save.append(node) 63 | for child in node['children']: 64 | add_node(child, parent=node) 65 | add_node(node) 66 | 67 | storage.for_documents.bulk_delete(doc_type, label_id, version) 68 | storage.for_documents.bulk_insert(to_save, doc_type, version) 69 | 70 | 71 | @secure_write 72 | def delete(request, doc_type, label_id, version=None): 73 | """Delete this document node and all of its children from the db""" 74 | storage.for_documents.bulk_delete(doc_type, label_id, version) 75 | return success() 76 | -------------------------------------------------------------------------------- /regcore_write/views/layer.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from regcore.db import storage 4 | from regcore.layer import standardize_params 5 | from regcore.responses import success, user_error 6 | from regcore_write.views.security import json_body, secure_write 7 | 8 | logger = logging.getLogger(__name__) 9 | 10 | 11 | def child_label_of(lhs, rhs): 12 | """Is the lhs label a child of the rhs label""" 13 | # Interpretations have a slightly different hierarchy 14 | if 'Interp' in lhs and 'Interp' in rhs: 15 | lhs_reg, lhs_comment = lhs.split('Interp') 16 | rhs_reg, rhs_comment = rhs.split('Interp') 17 | if lhs_reg.startswith(rhs_reg): 18 | return True 19 | 20 | # Handle Interps with shared prefix as well as non-interps 21 | if lhs.startswith(rhs): 22 | return True 23 | 24 | return False 25 | 26 | 27 | @secure_write 28 | @json_body 29 | def add(request, name, doc_type, doc_id): 30 | """Add the layer node and all of its children to the db""" 31 | layer = request.json_body 32 | if not isinstance(layer, dict): 33 | return user_error('invalid format') 34 | 35 | params = standardize_params(doc_type, doc_id) 36 | if params.doc_type not in ('preamble', 'cfr'): 37 | return user_error('invalid doc type') 38 | 39 | for key in layer.keys(): 40 | # terms layer has a special attribute 41 | if not child_label_of(key, params.tree_id) and key != 'referenced': 42 | return user_error('label mismatch: {0}, {1}'.format( 43 | params.tree_id, key)) 44 | 45 | storage.for_layers.bulk_delete(name, params.doc_type, params.doc_id) 46 | storage.for_layers.bulk_insert(child_layers(params, layer), name, 47 | params.doc_type) 48 | return success() 49 | 50 | 51 | @secure_write 52 | def delete(request, name, doc_type, doc_id): 53 | """Delete the layer node and all of its children from the db""" 54 | params = standardize_params(doc_type, doc_id) 55 | if params.doc_type not in ('preamble', 'cfr'): 56 | return user_error('invalid doc type') 57 | 58 | storage.for_layers.bulk_delete(name, params.doc_type, params.doc_id) 59 | return success() 60 | 61 | 62 | def child_layers(layer_params, layer_data): 63 | """We are generally given a layer corresponding to an entire regulation. 64 | We need to split that layer up and store it per node within the 65 | regulation. If a reg has 100 nodes, but the layer only has 3 entries, it 66 | will still store 100 layer models -- many may be empty""" 67 | doc_id_components = layer_params.doc_id.split('/') 68 | if layer_params.doc_type == 'preamble': 69 | doc_tree = storage.for_documents.get('preamble', layer_params.doc_id) 70 | elif layer_params.doc_type == 'cfr': 71 | version, label = doc_id_components 72 | doc_tree = storage.for_documents.get('cfr', label, version) 73 | else: 74 | doc_tree = None 75 | logger.error("Invalid doc type: %s", layer_params.doc_type) 76 | if not doc_tree: 77 | return [] 78 | 79 | to_save = [] 80 | 81 | def find_labels(node): 82 | child_labels = [] 83 | for child in node['children']: 84 | child_labels.extend(find_labels(child)) 85 | 86 | label_id = '-'.join(node['label']) 87 | 88 | # Account for "{version}/{cfr_part}" the same as "{preamble id}" 89 | doc_id = '/'.join(doc_id_components[:-1] + [label_id]) 90 | sub_layer = {'doc_id': doc_id} 91 | for key in layer_data: 92 | # 'referenced' is a special case of the definitions layer 93 | if key == label_id or key in child_labels or key == 'referenced': 94 | sub_layer[key] = layer_data[key] 95 | 96 | to_save.append(sub_layer) 97 | 98 | return child_labels + [label_id] 99 | 100 | find_labels(doc_tree) 101 | return to_save 102 | -------------------------------------------------------------------------------- /regcore_write/views/notice.py: -------------------------------------------------------------------------------- 1 | from regcore.db import storage 2 | from regcore.responses import success 3 | from regcore_write.views.security import json_body, secure_write 4 | 5 | 6 | @secure_write 7 | @json_body 8 | def add(request, docnum): 9 | """Add the notice to the db""" 10 | notice = request.json_body 11 | 12 | # @todo: write a schema that verifies the notice's structure 13 | cfr_parts = notice.get('cfr_parts', []) 14 | if 'cfr_part' in notice: 15 | cfr_parts.append(notice['cfr_part']) 16 | del notice['cfr_part'] 17 | notice['cfr_parts'] = cfr_parts 18 | 19 | storage.for_notices.delete(docnum) 20 | storage.for_notices.insert(docnum, notice) 21 | return success() 22 | 23 | 24 | @secure_write 25 | def delete(request, docnum): 26 | """Delete the notice from the db""" 27 | storage.for_notices.delete(docnum) 28 | return success() 29 | -------------------------------------------------------------------------------- /regcore_write/views/security.py: -------------------------------------------------------------------------------- 1 | import base64 2 | import json 3 | from functools import wraps 4 | 5 | from django.conf import settings 6 | from django.http import HttpResponse 7 | from django.utils.crypto import constant_time_compare 8 | from django.views.decorators.csrf import csrf_exempt 9 | 10 | from regcore.responses import user_error 11 | 12 | 13 | def _not_authorized(): 14 | """User failed authorization""" 15 | response = HttpResponse('Bad Authorization', status=401) 16 | response['WWW-Authenticate'] = 'Basic realm="write access"' 17 | return response 18 | 19 | 20 | def _is_correct_auth(guess): 21 | """Encode the configured auth username/password as a base64 string, then 22 | safely compare `guess` against it""" 23 | user, password = settings.HTTP_AUTH_USER, settings.HTTP_AUTH_PASSWORD 24 | combined = '{0}:{1}'.format(user, password) 25 | left = base64.b64encode(combined.encode()).decode('utf-8') 26 | # Django's built in constant_time_compare short circuits if the length of 27 | # the strings is not equal. We avoid that by padding both strings out to 28 | # 1000 characters before running the constant time comparison. Note, that 29 | # an auth string of more than 1000 characters will not be as secure as 30 | # intended 31 | left = ''.join(left[i] if i < len(left) else ' ' for i in range(1000)) 32 | right = ''.join(guess[i] if i < len(guess) else ' ' for i in range(1000)) 33 | return not constant_time_compare(left, right) 34 | 35 | 36 | def basic_auth(func): 37 | """Require HTTP basic authentication""" 38 | @wraps(func) 39 | def wrapped(request, *args, **kwargs): 40 | auth_str = request.META.get('HTTP_AUTHORIZATION', '') 41 | auth_parts = auth_str.split() 42 | if len(auth_parts) != 2 or auth_parts[0].upper() != 'BASIC': 43 | return _not_authorized() 44 | elif _is_correct_auth(auth_parts[1]): 45 | return _not_authorized() 46 | else: 47 | return func(request, *args, **kwargs) 48 | return wrapped 49 | 50 | 51 | def secure_write(func): 52 | """Depending on configuration, wrap each request in the appropriate 53 | security checks""" 54 | func = csrf_exempt(func) 55 | if settings.HTTP_AUTH_USER and settings.HTTP_AUTH_PASSWORD: 56 | func = basic_auth(func) 57 | 58 | return func 59 | 60 | 61 | def json_body(func): 62 | """Return a user error if the request's body doesn't contain valid JSON. 63 | Not embedding in `secure_write` as we may want to add schema checking to 64 | this in the future""" 65 | @wraps(func) 66 | def wrapped(request, *args, **kwargs): 67 | try: 68 | request.json_body = json.loads(request.body.decode('utf-8')) 69 | return func(request, *args, **kwargs) 70 | except (ValueError, UnicodeError): 71 | return user_error('invalid format') 72 | return wrapped 73 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages, setup 2 | 3 | setup( 4 | name="regcore", 5 | version="4.2.0", 6 | license="public domain", 7 | packages=find_packages(), 8 | include_package_data=True, 9 | install_requires=[ 10 | 'cached_property', 11 | 'django>=1.10,<1.12', 12 | 'django-mptt~=0.8.6', 13 | 'jsonschema', 14 | 'six', 15 | 'webargs', 16 | ], 17 | extras_require={ 18 | 'backend-elastic': ['pyelasticsearch'], 19 | 'backend-haystack': ['django-haystack'], 20 | 'backend-pgsql': ['django>=1.10', 'psycopg2'], 21 | }, 22 | ) 23 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = clean,py{27,34,35,36}-django{18,19,110,111}-{elastic,haystack},py{27,34,35,36}-django{110,111}-pgsql,lint,docs 3 | 4 | [testenv] 5 | deps = 6 | django18: django~=1.8.0 7 | django19: django~=1.9.0 8 | django110: django~=1.10.0 9 | django111: django~=1.11.0 10 | 11 | # Test requirements 12 | coverage~=4.0 13 | mock~=1.3 14 | model_mommy~=1.3 15 | pytest~=3.0 16 | pytest_cov~=2.4 17 | pytest_django~=3.1 18 | extras = 19 | elastic: backend-elastic 20 | haystack: backend-haystack 21 | pgsql: backend-pgsql 22 | commands = 23 | pytest --cov --cov-append regcore regcore_write regcore_read 24 | pgsql: pytest --cov --cov-append regcore regcore_write regcore_read regcore_pgsql 25 | setenv = 26 | elastic: DJANGO_SETTINGS_MODULE = regcore.settings.elastic 27 | pgsql: DJANGO_SETTINGS_MODULE = regcore.settings.pgsql 28 | 29 | [testenv:clean] 30 | deps: 31 | coverage~=4.0 32 | commands = coverage erase 33 | skip_install = True 34 | skipsdist = True 35 | 36 | [testenv:lint] 37 | deps = 38 | # Dependencies are pinned so that linting is consistent 39 | bandit==1.4.0 40 | flake8==2.5.4 41 | commands = 42 | flake8 regcore regcore_pgsql regcore_read regcore_write manage.py setup.py 43 | bandit -r --ini tox.ini regcore regcore_pgsql regcore_read regcore_write manage.py setup.py 44 | 45 | [testenv:docs] 46 | deps = sphinx 47 | commands = 48 | # Tox won't do wildcard expansion, so run in a shell 49 | sh -c "rm docs/regcore*.rst" 50 | sphinx-apidoc -F -o docs regcore 51 | sphinx-apidoc -F -o docs regcore_pgsql 52 | sphinx-apidoc -F -o docs regcore_read 53 | sphinx-apidoc -F -o docs regcore_write 54 | sphinx-build -b dirhtml -d docs/_build/doctrees/ docs/ docs/_build/dirhtml/ 55 | whitelist_externals = sh 56 | 57 | 58 | [bandit] 59 | exclude = regcore/tests,regcore_pgsql/tests,regcore_read/tests,regcore_write/tests 60 | 61 | [coverage] 62 | source = regcore,regcore_pgsql,regcore_read,regcore_write 63 | 64 | [flake8] 65 | exclude = regcore/migrations/*.py,regcore/settings/*.py 66 | 67 | [isort] 68 | known_third_party=six 69 | 70 | [pytest] 71 | python_files = tests_*.py *_test.py *_tests.py 72 | DJANGO_SETTINGS_MODULE = regcore.settings.base 73 | 74 | [travis] 75 | python = 76 | 3.6: py36, lint, docs 77 | --------------------------------------------------------------------------------