├── .dockerignore ├── .github ├── ISSUE_TEMPLATE │ ├── config.yml │ ├── new-feature.yml │ └── report-a-bug.yml └── workflows │ ├── get_logs.yml │ ├── main.yml │ ├── publish.yml │ └── tests.yml ├── .gitignore ├── Dockerfile ├── Gemfile ├── LICENSE ├── README.rst ├── RELATON_VERSION.txt ├── babel.config.json ├── bib_models ├── __init__.py ├── merger.py ├── serializers.py ├── tests │ ├── __init__.py │ └── test_util.py └── util.py ├── bibxml ├── __init__.py ├── asgi.py ├── context_processors.py ├── env_checker.py ├── error_views.py ├── settings.py ├── urls.py ├── views.py ├── wsgi.py └── xml2rfc_adapters.py ├── common ├── __init__.py ├── git.py ├── pydantic.py ├── query_profiler.py └── util.py ├── datatracker ├── __init__.py ├── auth.py ├── exceptions.py ├── internet_drafts.py ├── oauth.py └── request.py ├── docker-compose.dev.yml ├── docker-compose.monitor.yml ├── docker-compose.prod.yml ├── docker-compose.test-local.yml ├── docker-compose.test.yml ├── docker-compose.yml ├── docs ├── _ext │ └── rfp.py ├── _static │ └── custom-haiku.css ├── conf.py ├── docker-compose.yml ├── evolution.rst ├── howto │ ├── add-new-output-format.rst │ ├── add-new-source.rst │ ├── adjust-xml2rfc-paths.rst │ ├── auto-reindex-sources.rst │ ├── develop-locally.rst │ ├── index.rst │ ├── mark-releases.rst │ ├── run-in-production.rst │ ├── run-tests.rst │ ├── style-web-pages.rst │ └── update-container-build.rst ├── index.rst ├── ref │ ├── containers.rst │ ├── env.rst │ ├── glossary.rst │ ├── grafana-dashboard-gui.png │ ├── index.rst │ └── modules │ │ ├── bib_models.rst │ │ ├── bibxml.rst │ │ ├── common.rst │ │ ├── datatracker.rst │ │ ├── doi.rst │ │ ├── index.rst │ │ ├── main.rst │ │ ├── management.rst │ │ ├── prometheus.rst │ │ ├── sources.rst │ │ └── xml2rfc_compat.rst ├── rfp-requirements.rst └── topics │ ├── architecture.rst │ ├── auth.rst │ ├── dependency-diagram.svg │ ├── index.rst │ ├── production-setup.rst │ ├── relaton-format.rst │ ├── sourcing.rst │ ├── validation.rst │ └── xml2rfc-compat.rst ├── doi ├── __init__.py └── crossref.py ├── k8s ├── bib.yaml ├── kustomization.yaml └── secrets.yaml ├── main ├── __init__.py ├── api.py ├── app.py ├── exceptions.py ├── external_sources.py ├── migrations │ ├── 0001_initial.py │ ├── 0002_refdata_body_gin.py │ ├── 0003_refdata_body_astext_gin.py │ ├── 0004_refdata_body_ts_gin.py │ ├── 0005_refdata_body_docid_gin.py │ ├── 0006_auto_20220123_1622.py │ ├── 0007_alter_refdata_dataset_alter_refdata_ref_and_more.py │ ├── 0008_refdata_latest_date.py │ └── __init__.py ├── models.py ├── query.py ├── query_utils.py ├── search.py ├── sources.py ├── templates │ ├── browse │ │ ├── base.html │ │ ├── citation_details.html │ │ ├── dataset.html │ │ ├── home.html │ │ ├── indexed_bibitem_details.html │ │ ├── paginator.html │ │ ├── search_citations.html │ │ ├── search_form_quick.html │ │ └── search_forms.html │ ├── citation │ │ ├── bibitem_source.html │ │ ├── docid_search_link.html │ │ ├── export.html │ │ ├── in_list.html │ │ ├── series_search_link.html │ │ ├── sourced_bibitem_raw_table.html │ │ ├── sources.html │ │ ├── srp_link_href_get_query.html │ │ ├── summary.html │ │ └── validation_errors.html │ ├── deflist │ │ ├── entry.html │ │ ├── entry_list.html │ │ └── entry_recursivedict.html │ └── relaton │ │ ├── contributor.html │ │ ├── copyright.html │ │ ├── date.html │ │ ├── doc_ids.html │ │ ├── formatted_string.html │ │ ├── keyword.html │ │ ├── link.html │ │ ├── org.html │ │ ├── person_name.html │ │ ├── relation.html │ │ ├── series.html │ │ └── smart_title.html ├── templatetags │ ├── __init__.py │ ├── common.py │ ├── pydantic.py │ └── relaton.py ├── tests │ ├── __init__.py │ ├── test_apis.py │ ├── test_query.py │ └── test_query_utils.py ├── types.py └── views.py ├── manage.py ├── management ├── __init__.py ├── api.py ├── app.py ├── auth.py ├── management │ ├── __init__.py │ └── commands │ │ ├── __init__.py │ │ └── clear_cache.py ├── models.py ├── templates │ ├── api_button.html │ ├── indexing_task.html │ ├── management │ │ ├── base.html │ │ ├── dataset.html │ │ ├── datasets.html │ │ ├── home.html │ │ ├── task.html │ │ ├── xml2rfc.html │ │ └── xml2rfc_directory.html │ └── source_header.html ├── tests.py └── views.py ├── mypy.ini ├── ops ├── db.Dockerfile ├── grafana-dashboard-api-usage.json ├── grafana-dashboard-gui-usage.json ├── grafana-dashboards.yml ├── grafana-datasource.yml ├── load-postgres-extensions.sh ├── prometheus-config.yml └── prometheus.Dockerfile ├── package-lock.json ├── package.json ├── postcss.config.js ├── prometheus ├── __init__.py ├── metrics.py └── views.py ├── pyright.lsp.Dockerfile ├── requirements.txt ├── sources ├── __init__.py ├── app.py ├── celery.py ├── indexable.py ├── migrations │ ├── 0001_initial.py │ └── __init__.py ├── models.py ├── signals.py ├── task_status.py └── tasks.py ├── static ├── css │ └── main.css ├── img │ └── about │ │ ├── bibitem-details-2.png │ │ └── bibitem-details.png ├── js │ ├── api-button.js │ ├── format-xml.js │ ├── image-annotation.js │ ├── matomo.js │ ├── simple-cache.js │ ├── vendor │ │ ├── async.js │ │ ├── axios.js │ │ └── axios.map │ ├── windowed-listing.js │ ├── xml2rfc-browser.js │ ├── xml2rfc-path-widget.js │ └── xml2rfc-resolver.js └── keep ├── tailwind.config.js ├── templates ├── _about_links.html ├── _datatracker_user_block.html ├── _datatracker_user_link.html ├── _error_icon.html ├── _list_item_classes.html ├── _list_item_inner_classes.html ├── _message.html ├── _messages.html ├── _profiling_block.html ├── _search_icon.html ├── _side_block_classes.html ├── about.html ├── base.html ├── error.html ├── human_readable_openapi_spec.html ├── openapi-legacy.yaml ├── openapi.yaml └── robots.txt ├── test.Dockerfile ├── wait-for-migrations.sh └── xml2rfc_compat ├── __init__.py ├── adapters.py ├── aliases.py ├── app.py ├── fixtures ├── __init__.py └── test_refdata.json ├── management_views.py ├── migrations ├── 0001_initial.py ├── 0002_manualpathmap.py ├── 0003_manualpathmap_docid.py ├── 0004_remove_manualpathmap_query_and_more.py ├── 0005_alter_manualpathmap_xml2rfc_subpath.py ├── 0006_delete_manualpathmap_xml2rfcitem_sidecar_meta.py └── __init__.py ├── models.py ├── serializer.py ├── serializers ├── __init__.py ├── abstracts.py ├── anchor.py ├── authors.py ├── reference.py ├── series.py └── target.py ├── source.py ├── tests ├── __init__.py ├── static │ └── schemas │ │ ├── SVG-1.2-RFC.xsd │ │ ├── ns1.xsd │ │ ├── v3.xsd │ │ ├── xlink.xsd │ │ └── xml.xsd ├── test_adapters.py ├── test_serializers.py └── validate_remote_data_against_models.py ├── types.py ├── urls.py └── views.py /.dockerignore: -------------------------------------------------------------------------------- 1 | env 2 | data 3 | node_modules 4 | test-artifacts 5 | __pycache__ 6 | *.sqlite 7 | *.pyc 8 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: false 2 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/new-feature.yml: -------------------------------------------------------------------------------- 1 | name: New Feature / Enhancement 2 | description: Propose a new idea to be implemented 3 | labels: ["enhancement"] 4 | body: 5 | - type: markdown 6 | attributes: 7 | value: | 8 | Thanks for taking the time to propose a new feature / enhancement idea. 9 | - type: textarea 10 | id: description 11 | attributes: 12 | label: Description 13 | description: Include as much info as possible, including mockups / screenshots if available. 14 | placeholder: Description 15 | validations: 16 | required: true 17 | - type: checkboxes 18 | id: terms 19 | attributes: 20 | label: Code of Conduct 21 | description: By submitting this request, you agree to follow our [Code of Conduct](https://github.com/ietf-tools/.github/blob/main/CODE_OF_CONDUCT.md). 22 | options: 23 | - label: I agree to follow the [IETF's Code of Conduct](https://github.com/ietf-tools/.github/blob/main/CODE_OF_CONDUCT.md) 24 | required: true 25 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/report-a-bug.yml: -------------------------------------------------------------------------------- 1 | name: Report a Bug 2 | description: Something isn't right? File a bug report 3 | labels: ["bug"] 4 | body: 5 | - type: markdown 6 | attributes: 7 | value: | 8 | Thanks for taking the time to fill out this bug report. 9 | - type: textarea 10 | id: description 11 | attributes: 12 | label: Describe the issue 13 | description: Include as much info as possible, including the current behavior, expected behavior, screenshots, etc. If this is a display / UX issue, make sure to list the browser(s) you're experiencing the issue on. 14 | placeholder: Description 15 | validations: 16 | required: true 17 | - type: checkboxes 18 | id: terms 19 | attributes: 20 | label: Code of Conduct 21 | description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/ietf-tools/.github/blob/main/CODE_OF_CONDUCT.md) 22 | options: 23 | - label: I agree to follow the [IETF's Code of Conduct](https://github.com/ietf-tools/.github/blob/main/CODE_OF_CONDUCT.md) 24 | required: true 25 | -------------------------------------------------------------------------------- /.github/workflows/get_logs.yml: -------------------------------------------------------------------------------- 1 | name: Get_logs 2 | 3 | on: 4 | workflow_dispatch: 5 | 6 | jobs: 7 | deploy: 8 | runs-on: ubuntu-latest 9 | 10 | env: 11 | AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} 12 | AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} 13 | AWS_REGION: us-east-1 14 | 15 | steps: 16 | - name: Configure AWS credentials 17 | uses: aws-actions/configure-aws-credentials@v1 18 | with: 19 | aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} 20 | aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} 21 | aws-region: us-east-1 22 | 23 | - name: Login to Amazon ECR 24 | id: login-ecr 25 | uses: aws-actions/amazon-ecr-login@v1 26 | 27 | - name: Setup kubeconfig 28 | env: 29 | EKS_CLUSTER_NAME: bibxml-test-cluster 30 | run: | 31 | aws eks --region $AWS_REGION update-kubeconfig --name $EKS_CLUSTER_NAME 32 | 33 | - name: Install kubectl 34 | run: | 35 | curl -o kubectl https://s3.us-west-2.amazonaws.com/amazon-eks/1.22.6/2022-03-09/bin/linux/amd64/kubectl 36 | sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl 37 | kubectl version --client 38 | 39 | - name: Describe deployments 40 | env: 41 | KUBECONFIG: /home/runner/.kube/config 42 | run: | 43 | kubectl describe deployment web 44 | kubectl describe deployment celery 45 | 46 | - name: Get logs 47 | env: 48 | KUBECONFIG: /home/runner/.kube/config 49 | run: | 50 | kubectl logs $(kubectl get po | tr " " "\n" | grep web) 51 | kubectl logs $(kubectl get po | tr " " "\n" | grep celery) 52 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: main 2 | 3 | on: 4 | push: 5 | tags: 6 | - '*' 7 | workflow_dispatch: 8 | 9 | jobs: 10 | deploy: 11 | runs-on: ubuntu-latest 12 | 13 | env: 14 | AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} 15 | AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} 16 | AWS_REGION: us-east-1 17 | EKS_CLUSTER_NAME: bibxml-test-cluster 18 | KUBECONFIG: /home/runner/.kube/config 19 | 20 | steps: 21 | - name: Configure AWS credentials 22 | uses: aws-actions/configure-aws-credentials@v1 23 | with: 24 | aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} 25 | aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} 26 | aws-region: us-east-1 27 | 28 | - name: Login to Amazon ECR 29 | id: login-ecr 30 | uses: aws-actions/amazon-ecr-login@v1 31 | 32 | - name: Checkout repo 33 | uses: actions/checkout@v2 34 | with: 35 | ref: main 36 | 37 | - name: Build images 38 | run: | 39 | docker build --no-cache --build-arg SNAPSHOT="${GITHUB_REF#refs/*/}" --platform=linux/amd64 -t bibxml . 40 | 41 | docker tag bibxml:latest 916411375649.dkr.ecr.us-east-1.amazonaws.com/bibxml:latest 42 | 43 | docker push 916411375649.dkr.ecr.us-east-1.amazonaws.com/bibxml:latest 44 | 45 | - name: Trigger db migration 46 | env: 47 | GH_USERNAME: ${{ secrets.GH_USERNAME }} 48 | GH_PAT: ${{ secrets.GH_PAT }} 49 | run: | 50 | curl -X POST \ 51 | https://api.github.com/repos/ietf-ribose/bibxml-infrastructure/actions/workflows/db-migration.yml/dispatches \ 52 | -H "Accept: application/vnd.github.v3+json" \ 53 | -u ${GH_USERNAME}:${GH_PAT} \ 54 | --data '{ 55 | "ref": "main" 56 | }' 57 | 58 | - name: Setup kubeconfig 59 | run: | 60 | aws eks --region $AWS_REGION update-kubeconfig --name $EKS_CLUSTER_NAME 61 | 62 | - name: Install kubectl 63 | run: | 64 | curl -o kubectl https://s3.us-west-2.amazonaws.com/amazon-eks/1.22.6/2022-03-09/bin/linux/amd64/kubectl 65 | sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl 66 | kubectl version --client 67 | 68 | - name: Wait for db migration job to complete or timeout after 300s 69 | run: | 70 | kubectl wait --for=condition=complete --timeout=300s job/db-migration 71 | 72 | - name: Rollout updates 73 | if: always() 74 | run: | 75 | kubectl rollout restart deployment web 76 | kubectl rollout restart deployment celery 77 | kubectl rollout restart deployment flower 78 | 79 | - name: Check rollout statuses 80 | run: | 81 | kubectl rollout status deployment web 82 | kubectl rollout status deployment celery 83 | kubectl rollout status deployment flower 84 | kubectl get po -o wide 85 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: Publish 2 | run-name: Publish build ${{ github.run_number }} of branch ${{ github.ref_name }} by @${{ github.actor }} 3 | 4 | on: 5 | workflow_dispatch: 6 | 7 | jobs: 8 | build: 9 | runs-on: ubuntu-latest 10 | permissions: 11 | contents: write 12 | packages: write 13 | id-token: write 14 | 15 | steps: 16 | - name: Checkout repository 17 | uses: actions/checkout@v4 18 | with: 19 | fetch-depth: 0 20 | 21 | - name: Get Next Version 22 | id: semver 23 | uses: ietf-tools/semver-action@v1 24 | with: 25 | token: ${{ github.token }} 26 | patchList: fix, bugfix, perf, refactor, test, tests, chore, ci, build 27 | branch: main 28 | skipInvalidTags: true 29 | 30 | - name: Create Draft Release 31 | uses: ncipollo/release-action@v1.13.0 32 | with: 33 | prerelease: true 34 | draft: false 35 | commit: ${{ github.sha }} 36 | tag: ${{ steps.semver.outputs.nextStrict }} 37 | name: ${{ steps.semver.outputs.nextStrict }} 38 | body: '*pending*' 39 | token: ${{ secrets.GITHUB_TOKEN }} 40 | 41 | - name: Set up QEMU 42 | uses: docker/setup-qemu-action@v3 43 | 44 | - name: Setup Docker buildx 45 | uses: docker/setup-buildx-action@v3 46 | 47 | - name: Login to GitHub Container Registry 48 | uses: docker/login-action@v3 49 | with: 50 | registry: ghcr.io 51 | username: ${{ github.actor }} 52 | password: ${{ secrets.GITHUB_TOKEN }} 53 | 54 | - name: Build and push Docker image 55 | uses: docker/build-push-action@v5 56 | with: 57 | context: . 58 | file: Dockerfile 59 | push: true 60 | build-args: | 61 | SNAPSHOT=${{ steps.semver.outputs.nextStrict }} 62 | platforms: linux/amd64,linux/arm64 63 | tags: ghcr.io/${{ github.repository }}:${{ steps.semver.outputs.nextStrict }}, ghcr.io/${{ github.repository }}:latest 64 | 65 | - name: Update CHANGELOG 66 | id: changelog 67 | uses: requarks/changelog-action@v1 68 | with: 69 | token: ${{ github.token }} 70 | fromTag: ${{ steps.semver.outputs.nextStrict }} 71 | toTag: ${{ steps.semver.outputs.current }} 72 | excludeTypes: '' 73 | writeToFile: false 74 | 75 | - name: Create Release 76 | uses: ncipollo/release-action@v1.13.0 77 | with: 78 | allowUpdates: true 79 | makeLatest: true 80 | draft: false 81 | tag: ${{ steps.semver.outputs.nextStrict }} 82 | name: ${{ steps.semver.outputs.nextStrict }} 83 | body: ${{ steps.changelog.outputs.changes }} 84 | token: ${{ secrets.GITHUB_TOKEN }} 85 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | workflow_dispatch: 8 | 9 | permissions: 10 | contents: read 11 | 12 | jobs: 13 | build: 14 | 15 | runs-on: ubuntu-latest 16 | 17 | env: 18 | PORT: 8000 19 | DB_NAME: indexer 20 | DB_SECRET: qwert 21 | DJANGO_SECRET: "FDJDSLJFHUDHJCTKLLLCMNII(****#TEFF" 22 | HOST: localhost 23 | API_SECRET: "test" 24 | SERVICE_NAME: "IETF BibXML service" 25 | CONTACT_EMAIL: "test@email.com" 26 | DEBUG: 1 27 | SNAPSHOT: "test" 28 | steps: 29 | - name: Set up Python 3.10 30 | uses: actions/setup-python@v4 31 | with: 32 | python-version: "3.10" 33 | - uses: actions/checkout@v3 34 | - name: Install mypy 35 | run: pip install "mypy==1.4.1" && pip install -r requirements.txt 36 | - name: Run mypy 37 | run: mkdir .mypy_cache && mypy --ignore-missing-imports --install-types --non-interactive . 38 | - name: Build the Docker image 39 | run: docker compose -f docker-compose.test.yml build 40 | - name: Run tests 41 | env: 42 | CODECOV_TOKEN: ${{secrets.CODECOV_TOKEN}} 43 | run: docker compose -f docker-compose.test.yml up --exit-code-from test 44 | - name: Dump docker logs on failure 45 | if: failure() 46 | uses: jwalton/gh-docker-logs@v1 47 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /build 2 | /docs/build 3 | /env 4 | /node_modules 5 | /test-artifacts 6 | __pycache__ 7 | .env 8 | *.pyc 9 | http_cache.sqlite 10 | .DS_Store 11 | .vscode 12 | .idea -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # syntax=docker/dockerfile:1 2 | FROM python:3.10-slim@sha256:502f6626d909ab4ce182d85fcaeb5f73cf93fcb4e4a5456e83380f7b146b12d3 3 | # FROM python:3.11-rc-slim -- no lxml wheel yet 4 | 5 | ENV PYTHONUNBUFFERED=1 6 | 7 | ARG SNAPSHOT 8 | ENV SNAPSHOT=$SNAPSHOT 9 | 10 | RUN ["python", "-m", "pip", "install", "--upgrade", "pip"] 11 | 12 | # Could probably be removed for non-slim Python image 13 | RUN apt-get update && apt-get install -yq curl libpq-dev build-essential git 14 | 15 | # To build lxml from source, need at least this (but maybe better stick to wheels): 16 | # RUN apt-get install -yq libxml2-dev zlib1g-dev libxslt-dev 17 | 18 | # Install Node. 19 | # We need to have both Python (for backend Django code) 20 | # and Node (for Babel-based cross-browser frontend build). 21 | # Starting with Python image and adding Node is simpler. 22 | RUN curl -fsSL https://deb.nodesource.com/setup_current.x | bash - 23 | RUN apt-get update 24 | RUN apt-get install -yq nodejs 25 | 26 | RUN apt-get clean && rm -rf /var/lib/apt/lists/ 27 | 28 | # Install requirements for building docs 29 | RUN pip install sphinx 30 | 31 | # Copy and install requirements separately to let Docker cache layers 32 | COPY requirements.txt /code/requirements.txt 33 | COPY package.json /code/package.json 34 | COPY package-lock.json /code/package-lock.json 35 | 36 | WORKDIR /code 37 | 38 | RUN ["pip", "install", "-r", "requirements.txt"] 39 | RUN ["npm", "install"] 40 | 41 | # Copy the rest of the codebase 42 | COPY . /code 43 | 44 | RUN mkdir -p /code/build/static 45 | 46 | RUN ["python", "manage.py", "collectstatic", "--noinput"] 47 | RUN ["python", "manage.py", "compress"] 48 | 49 | # Build docs 50 | RUN sphinx-build -a -E -n -v -b html /code/docs/ /code/build/static/docs 51 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | 3 | gem 'relaton', "~>1.15.3" 4 | gem 'relaton-3gpp', "~>1.14.6" 5 | gem 'relaton-bib', "~>1.14.11" 6 | gem 'relaton-bipm', "~>1.14.11" 7 | gem 'relaton-bsi', "~>1.14.6" 8 | gem 'relaton-calconnect', "~>1.14.2" 9 | gem 'relaton-cen', "~>1.14.1" 10 | gem 'relaton-cie', "~>1.14.1" 11 | gem 'relaton-cli', "~>1.15.1" 12 | gem 'relaton-doi', "~>1.14.4" 13 | gem 'relaton-ecma', "~>1.14.6" 14 | gem 'relaton-gb', "~>1.14.0" 15 | gem 'relaton-iana', "~>1.14.3" 16 | gem 'relaton-iec', "~>1.14.4" 17 | gem 'relaton-ieee', "~>1.14.8" 18 | gem 'relaton-ietf', "~>1.14.4" 19 | gem 'relaton-iho', "~>1.14.3" 20 | gem 'relaton-iso', "~>1.15.4" 21 | gem 'relaton-iso-bib', "~>1.14.0" 22 | gem 'relaton-itu', "~>1.14.3" 23 | gem 'relaton-jis', "~>1.14.4" 24 | gem 'relaton-nist', "~>1.14.6" 25 | gem 'relaton-oasis', "~>1.14.4" 26 | gem 'relaton-ogc', "~>1.14.3" 27 | gem 'relaton-omg', "~>1.14.0" 28 | gem 'relaton-un', "~>1.14.1" 29 | gem 'relaton-w3c', "~>1.14.3" -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2022, The IETF Trust 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | 1. Redistributions of source code must retain the above copyright notice, 8 | this list of conditions and the following disclaimer. 9 | 10 | 2. Redistributions in binary form must reproduce the above copyright 11 | notice, this list of conditions and the following disclaimer in the 12 | documentation and/or other materials provided with the distribution. 13 | 14 | 3. Neither the name of the copyright holder nor the names of its contributors 15 | may be used to endorse or promote products derived from this software 16 | without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 22 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ============== 2 | BibXML Service 3 | ============== 4 | 5 | .. image:: https://codecov.io/gh/ietf-tools/bibxml-service/branch/main/graph/badge.svg?token=P963U4A2LL 6 | :target: https://codecov.io/gh/ietf-tools/bibxml-service 7 | 8 | For an overview, see https://github.com/ietf-ribose/bibxml-project. 9 | 10 | This project uses Docker, Django and PostgreSQL. 11 | 12 | 13 | Quick start 14 | ----------- 15 | 16 | Please refer to the `local development environment setup `_ 17 | section of the documentation. 18 | 19 | You can browse documentation `in this repository `_, 20 | but as it makes use of Sphinx-specific directives unsupported by GitHub 21 | it is recommended to `build documentation into HTML `_ first. 22 | This can be done using Docker without any other dependencies. 23 | 24 | Credits 25 | ------- 26 | 27 | Authored by Ribose as produced under the IETF BibXML SOW. 28 | -------------------------------------------------------------------------------- /RELATON_VERSION.txt: -------------------------------------------------------------------------------- 1 | 1.13.0 2 | -------------------------------------------------------------------------------- /babel.config.json: -------------------------------------------------------------------------------- 1 | { 2 | "presets": [ 3 | [ 4 | "@babel/preset-env", 5 | { 6 | "targets": "> 0.25%, not dead", 7 | "useBuiltIns": false, 8 | "exclude": [ 9 | "babel-plugin-transform-async-to-generator", 10 | "babel-plugin-transform-regenerator"], 11 | "modules": false 12 | } 13 | ] 14 | ] 15 | } 16 | -------------------------------------------------------------------------------- /bib_models/__init__.py: -------------------------------------------------------------------------------- 1 | """Utilities for working with Relaton bibliographic data. 2 | 3 | Data models are re-exported from :mod:`relaton.models`. 4 | """ 5 | 6 | from . import merger 7 | from . import serializers 8 | from .util import construct_bibitem 9 | 10 | from relaton.models import * 11 | 12 | 13 | __all__ = ( 14 | 'merger', 15 | 'construct_bibitem', 16 | 'serializers', 17 | 'BibliographicItem', 18 | 'DocID', 19 | 'Series', 20 | 'BiblioNote', 21 | 'Contributor', 22 | 'Relation', 23 | 'Person', 24 | 'PersonName', 25 | 'PersonAffiliation', 26 | 'Organization', 27 | 'Contact', 28 | 'Date', 29 | 'Link', 30 | 'Title', 31 | 'GenericStringValue', 32 | 'FormattedContent', 33 | ) 34 | -------------------------------------------------------------------------------- /bib_models/merger.py: -------------------------------------------------------------------------------- 1 | """ 2 | Helpers for merging dictionary representations 3 | of :class:`relaton.models.bibdata.BibliographicItem` instances. 4 | """ 5 | 6 | from typing import Any 7 | from deepmerge import Merger, STRATEGY_END 8 | from common.util import as_list 9 | 10 | 11 | def deduplicate_and_coerce_to_list( 12 | config, path, 13 | base: Any, nxt: Any): 14 | """ 15 | Takes two values, any or both of which could be lists, 16 | and merges as a combined list unless values are identical. 17 | 18 | ``None`` values are omitted. 19 | 20 | Doesn’t flatten recursively. 21 | 22 | For example: 23 | 24 | - If neither ``base`` nor ``nxt`` is a list, produces ``[base, nxt]``. 25 | - If ``base`` == ``nxt``, produces ``base``. 26 | - If only ``base`` is a list, produces effectively ``[*base, nxt]``. 27 | 28 | Suitable as list, fallback and type-conflict deepmerge strategy. 29 | """ 30 | 31 | if base == nxt or None in [base, nxt]: 32 | return base or nxt 33 | 34 | else: 35 | list1, list2 = as_list(base), as_list(nxt) 36 | for item in list2: 37 | if item not in list1: 38 | list1.append(item) 39 | return [item for item in list1 if item is not None] 40 | 41 | return STRATEGY_END 42 | 43 | 44 | bibitem_merger = Merger( 45 | [ 46 | (list, [deduplicate_and_coerce_to_list]), 47 | (dict, ['merge']), 48 | (set, ['union']), 49 | ], 50 | [deduplicate_and_coerce_to_list], 51 | [deduplicate_and_coerce_to_list], 52 | ) 53 | """A ``deepmerge`` merger 54 | for :class:`~relaton.models.bibdata.BibliographicItem` 55 | dictionary representations. 56 | """ 57 | -------------------------------------------------------------------------------- /bib_models/serializers.py: -------------------------------------------------------------------------------- 1 | """Pluggable serializer registry 2 | for :class:`relaton.models.bibdata.BibliographicItem` instances. 3 | 4 | Currently, only serialization 5 | into various utf-8 strings is supported. 6 | """ 7 | 8 | from typing import Callable, Dict 9 | from dataclasses import dataclass 10 | 11 | 12 | def register(id: str, content_type: str): 13 | """Parametrized decorator that, given ID and content_type, 14 | returns a function that will register a serializer function. 15 | 16 | Serializer function must take 17 | a :class:`relaton.models.bibdata.BibliographicItem` instance 18 | and return an utf-8-encoded string. 19 | """ 20 | def wrapper(func: Callable[..., bytes]): 21 | registry[id] = Serializer( 22 | serialize=func, 23 | content_type=content_type, 24 | ) 25 | return func 26 | return wrapper 27 | 28 | 29 | @dataclass 30 | class Serializer: 31 | """A registered serializer. 32 | Instantiated automatically by the :func:`~bib_models.serializers.register` 33 | function. 34 | """ 35 | serialize: Callable[..., bytes] 36 | """Serializer function. Returns a string.""" 37 | 38 | content_type: str 39 | """Content type to be used with this serializer, e.g. in HTTP responses.""" 40 | 41 | 42 | def get(id: str) -> Serializer: 43 | """Get previously registered serializer by ID. 44 | 45 | :raises SerializerNotFound:""" 46 | 47 | try: 48 | return registry[id] 49 | except KeyError: 50 | raise SerializerNotFound(id) 51 | 52 | 53 | class SerializerNotFound(RuntimeError): 54 | """No registered serializer with given ID.""" 55 | pass 56 | 57 | 58 | registry: Dict[str, Serializer] = {} 59 | """Registry of serializers.""" 60 | -------------------------------------------------------------------------------- /bib_models/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ietf-tools/bibxml-service/830eb8a3f03ed3885ed08b5d7092516ccb711139/bib_models/tests/__init__.py -------------------------------------------------------------------------------- /bib_models/tests/test_util.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | from bib_models import DocID 4 | 5 | from ..util import get_primary_docid 6 | 7 | 8 | class UtilTestcase(TestCase): 9 | """ 10 | Test cases for units in util.py 11 | """ 12 | 13 | def test_get_primary_docid(self): 14 | primary_id_value = "primary_id_value" 15 | raw_ids = [ 16 | DocID(id=primary_id_value, type="type", primary= True), 17 | DocID(id="id2", type="type2"), 18 | DocID(id="id3", type="type3", scope="scope"), 19 | ] 20 | primary_id = get_primary_docid(raw_ids) 21 | self.assertIsNotNone(primary_id) 22 | self.assertIsInstance(primary_id, DocID) 23 | self.assertEqual(primary_id.id, primary_id_value) # type: ignore 24 | 25 | def test_fail_get_primary_docid_if_no_primary_id(self): 26 | """ 27 | get_primary_docid should return None if no entry has primary == True 28 | """ 29 | primary_id_value = "primary_id_value" 30 | raw_ids = [ 31 | DocID(id=primary_id_value, type="type", primary=False), 32 | DocID(id="id2", type="type2"), 33 | DocID(id="id3", type="type3", scope="scope"), 34 | ] 35 | self.assertIsNone(get_primary_docid(raw_ids)) 36 | -------------------------------------------------------------------------------- /bibxml/__init__.py: -------------------------------------------------------------------------------- 1 | """Django project module.""" 2 | 3 | from django.core import checks 4 | from .env_checker import env_checker 5 | 6 | checks.register(env_checker) 7 | -------------------------------------------------------------------------------- /bibxml/asgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | ASGI config for bibxml project. 3 | 4 | It exposes the ASGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/3.2/howto/deployment/asgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.asgi import get_asgi_application 13 | 14 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'bibxml.settings') 15 | 16 | application = get_asgi_application() 17 | -------------------------------------------------------------------------------- /bibxml/context_processors.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from sources.indexable import registry as indexable_sources 3 | 4 | 5 | def service_meta(request): 6 | return dict( 7 | snapshot=settings.SNAPSHOT, 8 | service_name=settings.SERVICE_NAME, 9 | repo_url=settings.SOURCE_REPOSITORY_URL, 10 | ) 11 | 12 | 13 | def matomo(request): 14 | """Makes the :data:`.settings.MATOMO` available to templates.""" 15 | return dict( 16 | matomo=getattr(settings, 'MATOMO', None), 17 | ) 18 | 19 | 20 | def sources(request): 21 | return dict( 22 | indexable_sources=indexable_sources.keys(), 23 | relaton_datasets=settings.RELATON_DATASETS, 24 | ) 25 | -------------------------------------------------------------------------------- /bibxml/env_checker.py: -------------------------------------------------------------------------------- 1 | from typing import List, Tuple, Callable, Any 2 | from os import environ 3 | 4 | from django.core import checks 5 | from django.conf import settings 6 | 7 | 8 | def env_checker(**kwargs): 9 | """Checks that environment contains requisite variables.""" 10 | 11 | env_checks: List[Tuple[ 12 | str, 13 | Callable[[Any], bool], 14 | str 15 | ]] = [( 16 | 'CONTACT_EMAIL', 17 | lambda val: val.strip() != '', 18 | "contact email must be specified", 19 | ), ( 20 | 'SERVICE_NAME', 21 | lambda val: val.strip() != '', 22 | "service name must be specified", 23 | ), ( 24 | 'PRIMARY_HOSTNAME', 25 | lambda val: all([ 26 | val.strip() != '', 27 | val.strip() != '*' or settings.DEBUG, 28 | ]), 29 | "primary hostname must be specified", 30 | ), ( 31 | 'DB_NAME', 32 | lambda val: val.strip() != '', 33 | "default PostgreSQL database name must be specified", 34 | ), ( 35 | 'DB_USER', 36 | lambda val: val.strip() != '', 37 | "database username must be specified", 38 | ), ( 39 | 'DB_SECRET', 40 | lambda val: val.strip() != '', 41 | "database user credential must be specified", 42 | ), ( 43 | 'DB_HOST', 44 | lambda val: val.strip() != '', 45 | "default database server hostname must be specified", 46 | ), ( 47 | 'DJANGO_SECRET', 48 | lambda val: val.strip() != '', 49 | "Django secret must be set", 50 | ), ( 51 | 'REDIS_HOST', 52 | lambda val: val.strip() != '', 53 | "Redis server host must be specified", 54 | ), ( 55 | 'REDIS_PORT', 56 | lambda val: val.strip() != '', 57 | "Redis server port must be specified", 58 | ), ( 59 | 'SNAPSHOT', 60 | lambda val: val.strip() != '', 61 | "snapshot must be specified (use git describe --abbrev=0)", 62 | )] 63 | 64 | failed_env_checks: List[Tuple[str, str]] = [ 65 | (name, err) 66 | for name, check, err in env_checks 67 | if check(str(environ.get(name, ''))) is False 68 | ] 69 | 70 | return [ 71 | checks.Error( 72 | f'{failed_check[1]}', 73 | hint=f'(variable {failed_check[0]}', 74 | ) 75 | for failed_check in failed_env_checks 76 | ] 77 | -------------------------------------------------------------------------------- /bibxml/error_views.py: -------------------------------------------------------------------------------- 1 | """Error handlers for Django project’s root URL configuration.""" 2 | 3 | from typing import Union, Optional 4 | from django.shortcuts import render 5 | 6 | 7 | def server_error(request, *args, **kwargs): 8 | """Handler for HTTP 500 errors.""" 9 | 10 | exc = kwargs.get('exception', None) 11 | return _render_error(request, "Server error (500)", 500, exc) 12 | 13 | 14 | def not_authorized(request, *args, **kwargs): 15 | """Handler for HTTP 403 errors.""" 16 | 17 | exc = kwargs.get('exception', None) 18 | return _render_error(request, "Not authorized (403)", 403, exc) 19 | 20 | 21 | def not_found(request, *args, **kwargs): 22 | """Handler for HTTP 404 errors.""" 23 | 24 | exc = kwargs.get('exception', None) 25 | 26 | exc_repr: Optional[str] = None 27 | 28 | if exc: 29 | resolver_404_path = _get_resolver_404_path(exc) 30 | if resolver_404_path: 31 | # Let’s sanitize Django’s resolver 404 32 | exc_repr = "Path “{}” could not be resolved".format( 33 | resolver_404_path) 34 | else: 35 | exc_repr = str(exc) 36 | 37 | return _render_error(request, "Not found (404)", 404, exc_repr) 38 | 39 | 40 | _get_resolver_404_path = ( 41 | lambda exc: 42 | exc.args[0].get('path', None) 43 | if len(exc.args) > 0 and hasattr(exc.args[0], 'get') 44 | else None) 45 | """A helper that takes any exception 46 | and if it looks like Django’s resolver 404 then returns a path, 47 | otherwise ``None``.""" 48 | 49 | 50 | def _render_error( 51 | request, 52 | title: str, 53 | status_code: int, 54 | exc: Union[Exception, None, str], 55 | ): 56 | """Generic error view. 57 | 58 | Renders the ``error.html`` template with given title and error description. 59 | 60 | If error description is an instance of :class:`Exception`, 61 | casts it to string.""" 62 | 63 | exc_repr = ( 64 | exc 65 | if exc is None or isinstance(exc, str) 66 | else str(exc)) 67 | return render( 68 | request, 69 | 'error.html', 70 | dict( 71 | error_description=exc_repr, 72 | error_title=title), 73 | status=status_code) 74 | -------------------------------------------------------------------------------- /bibxml/views.py: -------------------------------------------------------------------------------- 1 | import textwrap 2 | 3 | import yaml 4 | 5 | from django.http import HttpResponseBadRequest 6 | from django.urls import reverse, NoReverseMatch 7 | from django.shortcuts import render 8 | 9 | from bib_models import BibliographicItem 10 | from main.query import list_doctypes 11 | from main.api import CitationSearchResultListView 12 | 13 | 14 | def about(request): 15 | """Serves an about/help page.""" 16 | 17 | return render(request, 'about.html', dict( 18 | )) 19 | 20 | 21 | def openapi_spec(request): 22 | """Serves machine-readable spec for main API.""" 23 | 24 | schemas = BibliographicItem.schema( 25 | ref_template='#/components/schemas/{model}', 26 | )['definitions'] 27 | bibitem_objects = textwrap.indent( 28 | yaml.dump(schemas), 29 | ' ') 30 | 31 | search_formats = CitationSearchResultListView.supported_query_formats 32 | 33 | return render(request, 'openapi.yaml', dict( 34 | known_doctypes=list_doctypes(), 35 | pre_indented_bibliographic_item_definitions=bibitem_objects, 36 | supported_search_query_formats=search_formats, 37 | ), content_type='text/x-yaml') 38 | 39 | 40 | def legacy_openapi_spec(request): 41 | """Serves machine-readable spec for compatibility/legacy API.""" 42 | return render( 43 | request, 44 | 'openapi-legacy.yaml', 45 | dict(), 46 | content_type='text/x-yaml') 47 | 48 | 49 | def readable_openapi_spec(request, spec: str): 50 | """Serves human-readable page for given OpenAPI spec 51 | (provided as a valid arg-free urlpattern name).""" 52 | 53 | try: 54 | path = reverse(spec) 55 | except NoReverseMatch: 56 | return HttpResponseBadRequest("Invalid spec") 57 | else: 58 | return render(request, 'human_readable_openapi_spec.html', dict( 59 | spec_path=path, 60 | )) 61 | 62 | 63 | def readable_openapi_spec_main(request): 64 | """A shortut for :func:`readable_openapi_spec` 65 | with pre-filled spec ID referencing main OpenAPI spec.""" 66 | return readable_openapi_spec(request, 'openapi_spec_main') 67 | -------------------------------------------------------------------------------- /bibxml/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for bibxml project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/3.2/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.wsgi import get_wsgi_application 13 | 14 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'bibxml.settings') 15 | 16 | application = get_wsgi_application() 17 | -------------------------------------------------------------------------------- /common/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ietf-tools/bibxml-service/830eb8a3f03ed3885ed08b5d7092516ccb711139/common/__init__.py -------------------------------------------------------------------------------- /common/query_profiler.py: -------------------------------------------------------------------------------- 1 | import time 2 | import logging 3 | from django.conf import settings 4 | from django.db import connection 5 | 6 | 7 | log = logging.getLogger(__name__) 8 | 9 | 10 | def context_processor(request): 11 | """Adds ``query_times`` template variable, 12 | using either ``request._queries`` if provided by middleware 13 | or ``connection.queries`` (is empty without DEBUG) if not. 14 | """ 15 | 16 | if hasattr(request, '_queries'): 17 | # Without DEBUG on, we may have middleware populate queries 18 | queries = request._queries 19 | else: 20 | queries = connection.queries 21 | return dict( 22 | profiling=dict( 23 | query_times=[str(p['time'])[:5] for p in queries], 24 | ), 25 | ) 26 | 27 | 28 | def middleware(get_response): 29 | """Unless DEBUG is on, runs query profiler against each request 30 | and places results under ``request._queries``. 31 | """ 32 | 33 | def middleware(request): 34 | if not settings.DEBUG: 35 | profiler = QueryProfiler() 36 | request._queries = profiler.queries 37 | 38 | with connection.execute_wrapper(profiler): 39 | return get_response(request) 40 | 41 | else: 42 | return get_response(request) 43 | 44 | return middleware 45 | 46 | 47 | class QueryProfiler: 48 | """ 49 | Use as follows:: 50 | 51 | profiler = QueryProfiler() 52 | 53 | with connection.execute_wrapper(ql): 54 | do_queries() 55 | 56 | # Here, profiler.queries is populated. 57 | 58 | """ 59 | def __init__(self, warning_threshold_sec=4): 60 | self.queries = [] 61 | self.warning_threshold_sec = warning_threshold_sec 62 | 63 | def __call__(self, execute, sql, params, many, context): 64 | start = time.monotonic() 65 | 66 | result = execute(sql, params, many, context) 67 | 68 | elapsed_seconds = time.monotonic() - start 69 | self.queries.append(dict( 70 | sql=sql, 71 | params=params, 72 | time=elapsed_seconds, 73 | )) 74 | 75 | if elapsed_seconds > self.warning_threshold_sec: 76 | log.warning( 77 | "Query took %s seconds (too long): %s (params %s)", 78 | str(elapsed_seconds)[:5], 79 | sql, repr(params)) 80 | 81 | return result 82 | -------------------------------------------------------------------------------- /common/util.py: -------------------------------------------------------------------------------- 1 | """List-related utilities.""" 2 | 3 | from typing import Any, Union, TypeVar, Iterable, List, Optional 4 | import re 5 | 6 | 7 | T = TypeVar('T') 8 | 9 | 10 | def flatten(items: Iterable[Any]) -> Iterable[Any]: 11 | """Flattens an iterable of possibly nested iterables 12 | (preserving strings, byte arrays and dictionaries).""" 13 | 14 | for x in items: 15 | if isinstance(x, Iterable) and not isinstance(x, (str, bytes, dict)): 16 | for sub_x in flatten(x): 17 | yield sub_x 18 | else: 19 | yield x 20 | 21 | 22 | def as_list(value: Union[T, List[T]]) -> List[T]: 23 | """Coerces given value to list if needed. 24 | 25 | :param value: any value. 26 | :returns: the value itself if it’s a list, 27 | a single-item list with the value 28 | if the value is neither a list nor ``None``, 29 | or an empty list if value is ``None``. 30 | :rtype: list""" 31 | 32 | if isinstance(value, list): 33 | return value 34 | elif value is None: 35 | return [] 36 | else: 37 | return [value] 38 | 39 | 40 | def get_fuzzy_match_regex( 41 | val: str, 42 | split_sep: str = r'[^a-zA-Z0-9]', 43 | match_sep: Optional[str] = None, 44 | deduplicate=False, 45 | ) -> str: 46 | """ 47 | Helper for fuzzy-matching a string using regular expressions. 48 | 49 | Splits a string by given separator regexp, 50 | and returns a string that can be used to fuzzy-match the string 51 | by its parts, ignoring the exact separators used. 52 | (The obtained string can be given in regular expressions.) 53 | 54 | Each string parts inbetween the separators 55 | are passed through ``re.escape()``. 56 | 57 | :param str val: 58 | A string like ``foo-bar/foo-bar-1+rr`` 59 | 60 | :returns: 61 | A string like 62 | ``foo[^a-zA-Z0-9]bar[^a-zA-Z0-9]foo[^a-zA-Z0-9]bar[^a-zA-Z0-9]rr`` 63 | 64 | :param str split_sep: 65 | Used to split the string, by default ``[^a-zA-Z0-9]`` 66 | 67 | :param str match_sep: 68 | Used to concatenate the final regular expression string, 69 | by default same as ``sep`` 70 | 71 | :param bool deduplicate: 72 | If explicitly set to ``True``, 73 | repeated parts of the string are omitted. 74 | 75 | .. note:: This may not work with default ``split_sep``/``match_sep``. 76 | """ 77 | match_sep = match_sep or split_sep 78 | _parts: List[str] = [ 79 | re.escape(part) 80 | for part in re.split(split_sep, val) 81 | ] 82 | if deduplicate: 83 | parts = list(dict.fromkeys(_parts)) 84 | else: 85 | parts = _parts 86 | return match_sep.join(parts) 87 | -------------------------------------------------------------------------------- /datatracker/__init__.py: -------------------------------------------------------------------------------- 1 | """Datatracker API integration. 2 | """ 3 | 4 | 5 | from . import request 6 | from .exceptions import UnexpectedDatatrackerResponse 7 | 8 | from . import internet_drafts 9 | # Make external source register. 10 | 11 | 12 | __all__ = ( 13 | 'request', 14 | 'UnexpectedDatatrackerResponse', 15 | ) 16 | -------------------------------------------------------------------------------- /datatracker/exceptions.py: -------------------------------------------------------------------------------- 1 | class UnexpectedDatatrackerResponse(ValueError): 2 | pass 3 | -------------------------------------------------------------------------------- /datatracker/request.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | 4 | BASE_AUTH_DOMAIN = 'https://auth.ietf.org/' 5 | """Base domain for Datatracker OIDC.""" 6 | 7 | BASE_DOMAIN = 'https://datatracker.ietf.org' 8 | """Base domain for Datatracker API.""" 9 | 10 | 11 | def post(endpoint: str, api_key: str) -> requests.Response: 12 | """Handles Datatracker authenticated POST request. 13 | 14 | :param str endpoint: Endpoint URL relative to :data:`.BASE_DOMAIN`, 15 | with leading slash. 16 | """ 17 | return requests.post( 18 | f'{BASE_DOMAIN}{endpoint}', 19 | files={'apikey': (None, api_key)}, 20 | ) 21 | 22 | 23 | def get(endpoint: str, format='json') -> requests.Response: 24 | """Handles Datatracker GET request, specifies JSON format. 25 | 26 | :param str endpoint: Endpoint URL relative to :data:`.BASE_DOMAIN`, 27 | with leading slash. 28 | """ 29 | return requests.get( 30 | f'{BASE_DOMAIN}{endpoint}', 31 | params={'format': format} if format else None, 32 | ) 33 | -------------------------------------------------------------------------------- /docker-compose.dev.yml: -------------------------------------------------------------------------------- 1 | # Meant to be used in conjunction (after) the main docker-compose.yml 2 | # in development environment: 3 | # 4 | # docker compose -f docker-compose.yml -f docker-compose.dev.yml up 5 | 6 | services: 7 | web: 8 | command: 9 | # This command overrides the one from the “main” Compose file 10 | # in order to run the extra `npm install` required 11 | # after the mounted `code` directory overwrites `node_modules` 12 | # created during Dockerfile build process. 13 | - /bin/sh 14 | - -c 15 | - | 16 | git config --global --add safe.directory /code && 17 | export SNAPSHOT=$$(git describe --abbrev=0) && 18 | npm install && 19 | ./wait-for-migrations.sh && 20 | python manage.py runserver 0.0.0.0:8000 21 | volumes: 22 | # If working on relaton-py, it could be mounted as follows 23 | # (given Docker can mount parent directory) 24 | # - ../relaton-py/relaton:/usr/local/lib/python3.10/site-packages/relaton:ro 25 | - .:/code 26 | celery: 27 | volumes: 28 | # If working on relaton-py, it could be mounted as follows 29 | # (given Docker can mount parent directory) 30 | # - ../relaton-py/relaton:/usr/local/lib/python3.10/site-packages/relaton:ro 31 | - .:/code 32 | -------------------------------------------------------------------------------- /docker-compose.monitor.yml: -------------------------------------------------------------------------------- 1 | # Meant to be used in conjunction (after) the main docker-compose.yml: 2 | # 3 | # docker compose -f docker-compose.yml -f docker-compose.prod.yml -f docker-compose.monitor.yml up 4 | 5 | services: 6 | flower: 7 | image: mher/flower 8 | hostname: flower 9 | environment: 10 | CELERY_BROKER_URL: "redis://redis:6379" 11 | CELERY_RESULT_BACKEND: "redis://redis:6379" 12 | depends_on: 13 | - celery 14 | ports: 15 | - "5555:5555" 16 | 17 | prometheus: 18 | image: prom/prometheus 19 | hostname: prometheus 20 | build: 21 | context: ops 22 | dockerfile: prometheus.Dockerfile 23 | args: 24 | # These hostnames match service names in main docker-compose.yml 25 | TARGET_HOSTS: "['web:8000', 'celery:9080', 'celery-exporter:9808']" 26 | HTTP_BASIC_USERNAME: "ietf" 27 | HTTP_BASIC_PASSWORD: "${API_SECRET:?missing API secret}" 28 | JOB_NAME: "bibxml-service" 29 | depends_on: 30 | - web 31 | - celery 32 | - celery-exporter 33 | ports: 34 | - "9090:9090" 35 | 36 | grafana: 37 | image: grafana/grafana-oss 38 | hostname: grafana 39 | environment: 40 | GF_SECURITY_ADMIN_USER: "ietf" 41 | GF_SECURITY_ADMIN_PASSWORD: ${API_SECRET} 42 | SOURCE_NAME: BibXML’s Prometheus 43 | SOURCE: "http://prometheus:9090/" 44 | SOURCE_VERSION: 1 45 | volumes: 46 | - ./ops/grafana-datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml 47 | - ./ops/grafana-dashboards.yml:/etc/grafana/provisioning/dashboards/dashboards.yml 48 | - ./ops/grafana-dashboard-gui-usage.json:/etc/dashboards/bibxml/gui_usage.json 49 | - ./ops/grafana-dashboard-api-usage.json:/etc/dashboards/bibxml/api_usage.json 50 | depends_on: 51 | - prometheus 52 | ports: 53 | - "3000:3000" 54 | -------------------------------------------------------------------------------- /docker-compose.prod.yml: -------------------------------------------------------------------------------- 1 | version: "3.3" 2 | 3 | services: 4 | redis: 5 | logging: 6 | driver: "syslog" 7 | options: 8 | syslog-address: "${SYSLOG}" 9 | 10 | db: 11 | logging: 12 | driver: "syslog" 13 | options: 14 | syslog-address: "${SYSLOG}" 15 | volumes: 16 | - ./bibdb:/var/lib/postgresql/data 17 | 18 | web: 19 | logging: 20 | driver: "syslog" 21 | options: 22 | syslog-address: "${SYSLOG}" 23 | 24 | celery: 25 | logging: 26 | driver: "syslog" 27 | options: 28 | syslog-address: "${SYSLOG}" 29 | -------------------------------------------------------------------------------- /docker-compose.test-local.yml: -------------------------------------------------------------------------------- 1 | version: "3.9" 2 | 3 | services: 4 | 5 | test: 6 | command: sh -c "/wait && python manage.py test" 7 | -------------------------------------------------------------------------------- /docker-compose.test.yml: -------------------------------------------------------------------------------- 1 | version: "3.9" 2 | 3 | services: 4 | test-redis: 5 | image: redis 6 | 7 | test-db: 8 | build: 9 | context: ops 10 | dockerfile: db.Dockerfile 11 | args: 12 | POSTGRES_DB: test 13 | POSTGRES_USER: test 14 | POSTGRES_PASSWORD: test 15 | 16 | test: 17 | image: local/test:latest 18 | build: 19 | context: . 20 | dockerfile: test.Dockerfile 21 | args: 22 | SNAPSHOT: ${SNAPSHOT:?err} 23 | depends_on: 24 | - test-db 25 | - test-redis 26 | command: sh -c "/wait && python -m coverage run manage.py test && ./codecov -t ${CODECOV_TOKEN}" 27 | environment: 28 | - CODECOV_TOKEN 29 | - WAIT_HOSTS=test-db:5432 30 | - WAIT_HOSTS_TIMEOUT=300 31 | - WAIT_SLEEP_INTERVAL=30 32 | - WAIT_HOST_CONNECT_TIMEOUT=30 33 | - WAIT_BEFORE_HOSTS=15 34 | # If working on relaton-py, it could be mounted as follows 35 | # volumes: 36 | # - ../relaton-py/relaton:/usr/local/lib/python3.10/site-packages/relaton:ro 37 | -------------------------------------------------------------------------------- /docs/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.9" 2 | 3 | services: 4 | watch: 5 | image: local/web-precheck:latest 6 | command: 7 | - /bin/sh 8 | - -c 9 | - | 10 | cd /code && 11 | rm -rf /build/html/* && 12 | pip install watchdog && 13 | export SNAPSHOT=$$(git describe --abbrev=0) && 14 | sphinx-build -a -E -n -v -b dirhtml /code/docs/ /build/html && 15 | watchmedo shell-command \ 16 | --patterns="*.rst;*.py;*.css" \ 17 | --recursive \ 18 | --drop \ 19 | --command='rm -rf /build/html/* && sphinx-build -a -E -n -v -b dirhtml /code/docs/ /build/html' 20 | volumes: 21 | - ..:/code 22 | - ./build:/build 23 | serve: 24 | image: python:3-slim 25 | command: 26 | - /bin/sh 27 | - -c 28 | - | 29 | cd /build/html && python -m http.server 8080 30 | volumes: 31 | - ./build:/build 32 | depends_on: 33 | - watch 34 | ports: 35 | - "8001:8080" 36 | -------------------------------------------------------------------------------- /docs/howto/add-new-output-format.rst: -------------------------------------------------------------------------------- 1 | =========================================== 2 | How to add new output format support to API 3 | =========================================== 4 | 5 | Default serialization method is Relaton JSON. 6 | 7 | There is support for pluggable additional serialization methods. 8 | 9 | 1. Define a serializer function. It must accept 10 | a :class:`relaton.models.bibdata.BibliographicItem` instance 11 | as a positional argument, arbitrary keyword arguments 12 | (which it may use or not), and return an utf8-encoded string. 13 | 14 | 2. Register your serializer like this:: 15 | 16 | from typing import Tuple 17 | from bib_models import serializers 18 | from relaton.models import BibliographicItem 19 | 20 | @serializers.register('foobar', 'application/x-foobar') 21 | def to_foobar(item: BibliographicItem, **kwargs) -> str: 22 | serialized: str = some_magic(item) 23 | return serialized 24 | 25 | .. note:: Make sure the module that does the registration is imported at startup 26 | (use ``app.py`` if it’s a Django app, or project-wide ``bibxml.__init__``). 27 | 28 | In the above example, we associate serializer with ID “foobar”, 29 | meaning API callers will be able to specify ``format=foobar`` in GET parameters, 30 | and content type ``application/json``, meaning that will be the MIME type of response they receive. 31 | 32 | .. seealso:: :rfp:req:`16` 33 | -------------------------------------------------------------------------------- /docs/howto/add-new-source.rst: -------------------------------------------------------------------------------- 1 | ======================= 2 | How to add a new source 3 | ======================= 4 | 5 | New Relaton source 6 | ================== 7 | 8 | For a Relaton source that conforms to the same layout 9 | as existing ``ietf-tools/relaton-data-*`` 10 | repositories on GitHub, it should be sufficient to add 11 | new source ID under :data:`bibxml.settings.RELATON_DATASETS`. 12 | 13 | New source ID must not clash with any of the other IDs 14 | of an internal or external source, 15 | and requires that the 16 | ``ietf-tools/relaton-data-`` repository 17 | exists and uses the exact same layout as the preexisting data repositories. 18 | 19 | Other sources 20 | ============= 21 | 22 | Currently, only sources backed by Git are supported. 23 | 24 | If your source uses a Git repository (or multiple), 25 | define your custom indexing and cleanup functions 26 | and register them using :func:`sources.indexable.register_git_source`. 27 | 28 | Registered source will be possible to reindex from management GUI and API. 29 | 30 | If your source provides bibliographic data in Relaton format, 31 | your source should create a :class:`main.models.RefData` instance for each 32 | indexed item (make sure to use your source ID as ``dataset`` value). 33 | This will make indexed bibliographic data available via service 34 | retrieval API and GUI. 35 | 36 | .. note:: If your source does **not** provide bibliographic data in Relaton format, 37 | you should make conversion to Relaton part of your indexing logic. 38 | 39 | Not doing so is only suitable for certain special scenarios such as data migration fallback: 40 | for an example of that, see :mod:`xml2rfc_compat`, 41 | which registers custom indexing logic (:func:`xml2rfc_compat.source.index_xml2rfc_source`) 42 | that caches data for xml2rfc-style path fallback. 43 | -------------------------------------------------------------------------------- /docs/howto/adjust-xml2rfc-paths.rst: -------------------------------------------------------------------------------- 1 | ======================================== 2 | How to adjust xml2rfc-style path parsing 3 | ======================================== 4 | 5 | Do you have xml2rfc-style paths which result in fallback behavior? 6 | 7 | You may need to make it resolve to a Relaton bibliographic item obtained 8 | from up-to-date sources. 9 | 10 | - If it’s a general case 11 | and the :term:`xml2rfc anchor` 12 | is enough to deduce Relaton bibliographic item lookup, 13 | edit the adapter subclass (in particular, the ``resolve()`` method): 14 | adjust the query e.g. by adding an OR condition. 15 | 16 | .. seealso:: :mod:`bibxml.xml2rfc_adapters` 17 | 18 | - Otherwise, you can create a manual mapping 19 | by updating sidecar metadata YAML file within :term:`xml2rfc mirror source` 20 | repository (and reindexing the source in BibXML service). 21 | 22 | 1. Locate an item using service’s general-purpose search functionality, 23 | and copy the first citeable identifier 24 | from bibliographic item details page. 25 | 26 | 2. Use that identifier when creating a YAML file with contents like this:: 27 | 28 | primary_docid: "" 29 | 30 | .. seealso:: 31 | 32 | - :doc:`/topics/xml2rfc-compat` 33 | - :mod:`xml2rfc_compat` 34 | -------------------------------------------------------------------------------- /docs/howto/auto-reindex-sources.rst: -------------------------------------------------------------------------------- 1 | ==================================== 2 | How to reindex sources automatically 3 | ==================================== 4 | 5 | Service’s database serves as more of a cache, 6 | and all data is indexed from Git repositories 7 | (termed :term:`indexable sources `). 8 | Thus, to ensure up-to-date information output in API and GUI, 9 | you may want to reindex those sources periodically. 10 | 11 | There are two possible approaches to this: 12 | 13 | - Set up an integration that periodically invokes service’s 14 | reindex API endpoint. 15 | 16 | This method is more explicit, and allows you to catch 17 | an error returned by reindex API endpoint if service experiences issues. 18 | 19 | - Set ``AUTO_REINDEX_INTERVAL`` to a positive integer 20 | in production environment. 21 | 22 | This method uses ``celerybeat`` scheduler. It’s easier to set up, 23 | but puts more importance on monitoring the status of task worker. 24 | 25 | .. seealso:: :data:`bibxml.settings.AUTO_REINDEX_INTERVAL` 26 | 27 | How frequently to reindex? 28 | -------------------------- 29 | 30 | In short: it’s recommended to check how frequently the sources change 31 | and how long it takes to index the largest source, 32 | and pick a value higher than the maximum between those two. 33 | 34 | Long answer: you may want to ensure that reindex interval 35 | is larger than the time it takes for a source to be reindexed 36 | (which could be multiple minutes for sources with many documents, 37 | such as Internet Drafts)—otherwise, task queue may grow indefinitely. 38 | 39 | Furthermore, it’s currently recommended to not reindex too often, 40 | as it’d overwhelm task outcome listings in management GUI 41 | (making it harder to track down issues) and cause unnecessary requests 42 | to Git server APIs. 43 | 44 | Other than that, it’s OK to reindex sources as often as you want. 45 | 46 | .. note:: 47 | 48 | If no changes were detected in source’s underlying Git repo 49 | (the HEAD commit is the same), then indexing task will skip reindexing. 50 | 51 | This ensures the service does not reindex unnecessarily, 52 | but it also means that if you change indexing implementation 53 | you’d need to ensure each source 54 | has at least one commit since then for changes to have effect. 55 | 56 | .. note:: Regardless of the method you choose, make sure to monitor 57 | task execution as part of overall 58 | :doc:`service monitoring in production `. 59 | -------------------------------------------------------------------------------- /docs/howto/index.rst: -------------------------------------------------------------------------------- 1 | ============= 2 | How-to guides 3 | ============= 4 | 5 | .. toctree:: 6 | :maxdepth: 1 7 | 8 | develop-locally 9 | run-tests 10 | style-web-pages 11 | adjust-citation-rendering 12 | adjust-xml2rfc-paths 13 | add-new-source 14 | add-new-output-format 15 | mark-releases 16 | update-container-build 17 | run-in-production 18 | auto-reindex-sources 19 | -------------------------------------------------------------------------------- /docs/howto/mark-releases.rst: -------------------------------------------------------------------------------- 1 | ======================== 2 | How to mark new releases 3 | ======================== 4 | 5 | To mark a new release, tag current commit with new version. 6 | 7 | This repository uses the following versioning scheme:: 8 | 9 | yyyy.m.d_i 10 | 11 | Where ``i`` starts with 1 12 | and increments with each release made this day. 13 | 14 | Example tagging command:: 15 | 16 | git tag -s "2022.1.29_1" -m "Tag today’s release" 17 | -------------------------------------------------------------------------------- /docs/howto/run-tests.rst: -------------------------------------------------------------------------------- 1 | ================ 2 | How to run tests 3 | ================ 4 | 5 | CI 6 | -- 7 | 8 | :: 9 | 10 | docker compose -f docker-compose.test.yml up --build --exit-code-from test 11 | 12 | The ``--exit-code-from`` flag will automatically stop 13 | all the containers after the tests are completed. 14 | 15 | Tests are integrated as part as a Github Action workflow (CI). 16 | Successfully running the tests will produce a coverage report. 17 | This report will be available as a comment in every PR pointing 18 | at the main branch. 19 | 20 | .. note:: 21 | 22 | This will build a different Docker image than the one used 23 | for live project. See ``test.Dockerfile`` for details. 24 | 25 | .. note:: 26 | 27 | If a valid CODECOV_TOKEN is provided as environment variable, the 28 | report will be uploaded to CodeCov. CODECOV_TOKEN should only be 29 | provided in the CI environment. 30 | 31 | Locally 32 | ------- 33 | 34 | Tests can be run locally without the need of having a Codecov 35 | configuration. To do so, override docker-compose.test.yml 36 | with docker-compose.test-local.yml. 37 | 38 | :: 39 | 40 | docker compose -f docker-compose.test.yml -f docker-compose.test-local.yml up --build --exit-code-from test 41 | 42 | 43 | -------------------------------------------------------------------------------- /docs/howto/update-container-build.rst: -------------------------------------------------------------------------------- 1 | ============================= 2 | How to update container build 3 | ============================= 4 | 5 | If you need to make changes that require changing some aspects 6 | of container build, please follow this section. 7 | 8 | Updating Dockerfile 9 | =================== 10 | 11 | If you, for example, need to add some system requirements 12 | or alter runtime environment, your changes may affect the Dockerfile. 13 | 14 | Key points to remember if you update Dockerfile: 15 | 16 | - Make equivalent changes to test.Dockerfile, 17 | and note if steps are expected to be different for Ubuntu. 18 | - Update docker-compose file(s) used in development/testing 19 | if changes in Dockerfile affect how the image should be used by Compose. 20 | -------------------------------------------------------------------------------- /docs/ref/grafana-dashboard-gui.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ietf-tools/bibxml-service/830eb8a3f03ed3885ed08b5d7092516ccb711139/docs/ref/grafana-dashboard-gui.png -------------------------------------------------------------------------------- /docs/ref/index.rst: -------------------------------------------------------------------------------- 1 | ========== 2 | References 3 | ========== 4 | 5 | .. toctree:: 6 | :maxdepth: 2 7 | :caption: Contents: 8 | 9 | glossary 10 | modules/index 11 | env 12 | containers 13 | -------------------------------------------------------------------------------- /docs/ref/modules/bib_models.rst: -------------------------------------------------------------------------------- 1 | ========================================== 2 | ``bib_models``: Bibliographic data classes 3 | ========================================== 4 | 5 | .. automodule:: bib_models 6 | 7 | .. contents:: 8 | :local: 9 | 10 | ``serializers``: Output formats 11 | =============================== 12 | 13 | .. automodule:: bib_models.serializers 14 | :members: 15 | 16 | 17 | ``util``: Utilities for instantiating bibliographic items 18 | ========================================================= 19 | 20 | .. automodule:: bib_models.util 21 | :members: 22 | 23 | 24 | ``merger``: Merging bibliographic items representing the same resource 25 | ====================================================================== 26 | 27 | .. automodule:: bib_models.merger 28 | :members: 29 | -------------------------------------------------------------------------------- /docs/ref/modules/bibxml.rst: -------------------------------------------------------------------------------- 1 | =============================== 2 | ``bibxml``: Django project root 3 | =============================== 4 | 5 | .. automodule:: bibxml 6 | 7 | .. contents:: 8 | :local: 9 | 10 | xml2rfc adapters 11 | ================ 12 | 13 | .. automodule:: bibxml.xml2rfc_adapters 14 | :members: 15 | 16 | URLs 17 | ==== 18 | 19 | .. automodule:: bibxml.urls 20 | :members: 21 | 22 | URL configuration tries to be self-explanatory, 23 | see :github:`bibxml/urls.py`. 24 | 25 | Includes aren’t preferred, instead we have a single hierarchy 26 | that exposes all service URLs at a glance. 27 | 28 | Django settings 29 | =============== 30 | 31 | .. seealso:: 32 | 33 | - For settings recognized by Django core and contrib apps, 34 | by Django Compressor, Whitenoise, Celery, etc.: 35 | their respective docs online 36 | - For environment variables that influence settings, 37 | look for those marked as accepted or required by Django in :doc:`/ref/env` 38 | - :github:`bibxml/settings.py` for undocumented members 39 | 40 | .. automodule:: bibxml.settings 41 | :members: 42 | 43 | Views 44 | ===== 45 | 46 | .. automodule:: bibxml.views 47 | :members: 48 | 49 | 50 | Error views 51 | ----------- 52 | 53 | .. automodule:: bibxml.error_views 54 | :members: 55 | 56 | 57 | Context processors 58 | ------------------ 59 | 60 | .. automodule:: bibxml.context_processors 61 | :members: 62 | :undoc-members: 63 | 64 | 65 | Deployment helpers 66 | ================== 67 | 68 | ``env_checker`` 69 | --------------- 70 | 71 | .. automodule:: bibxml.env_checker 72 | :members: 73 | 74 | ``asgi`` 75 | -------- 76 | 77 | .. automodule:: bibxml.asgi 78 | :members: 79 | 80 | 81 | ``wsgi`` 82 | -------- 83 | 84 | .. automodule:: bibxml.wsgi 85 | :members: 86 | -------------------------------------------------------------------------------- /docs/ref/modules/common.rst: -------------------------------------------------------------------------------- 1 | ============================ 2 | ``common``: Shared utilities 3 | ============================ 4 | 5 | .. automodule:: common 6 | 7 | DB query profiling 8 | ================== 9 | 10 | .. automodule:: common.query_profiler 11 | :members: 12 | 13 | 14 | Working with Git repositories 15 | ============================= 16 | 17 | .. automodule:: common.git 18 | :members: 19 | 20 | 21 | Pydantic utilities 22 | ================== 23 | 24 | .. automodule:: common.pydantic 25 | :members: 26 | 27 | 28 | Utilities 29 | ========= 30 | 31 | .. automodule:: common.util 32 | :members: 33 | -------------------------------------------------------------------------------- /docs/ref/modules/datatracker.rst: -------------------------------------------------------------------------------- 1 | ============================================= 2 | ``datatracker`` — Datatracker API integration 3 | ============================================= 4 | 5 | .. automodule:: datatracker 6 | 7 | .. contents:: 8 | :local: 9 | 10 | 11 | Internet Draft retrieval 12 | ======================== 13 | 14 | .. automodule:: datatracker.internet_drafts 15 | :members: 16 | 17 | 18 | Authentication 19 | ============== 20 | 21 | .. seealso:: :rfp:req:`15` 22 | 23 | Developer token-based 24 | --------------------- 25 | 26 | .. automodule:: datatracker.auth 27 | :members: 28 | 29 | 30 | OAuth2/OIDC 31 | ----------- 32 | 33 | .. automodule:: datatracker.oauth 34 | :members: 35 | :undoc-members: 36 | 37 | Requests 38 | ======== 39 | 40 | .. automodule:: datatracker.request 41 | :members: 42 | 43 | 44 | Exceptions 45 | ========== 46 | 47 | .. automodule:: datatracker.exceptions 48 | :members: 49 | -------------------------------------------------------------------------------- /docs/ref/modules/doi.rst: -------------------------------------------------------------------------------- 1 | ================================ 2 | ``doi``: Retrieving data via DOI 3 | ================================ 4 | 5 | .. automodule:: doi 6 | :members: 7 | 8 | Crossref interoperation 9 | ======================= 10 | 11 | .. automodule:: doi.crossref 12 | :members: 13 | -------------------------------------------------------------------------------- /docs/ref/modules/index.rst: -------------------------------------------------------------------------------- 1 | ======================= 2 | Python module reference 3 | ======================= 4 | 5 | .. toctree:: 6 | :maxdepth: 2 7 | 8 | bib_models 9 | bibxml 10 | sources 11 | main 12 | management 13 | doi 14 | datatracker 15 | xml2rfc_compat 16 | prometheus 17 | common 18 | -------------------------------------------------------------------------------- /docs/ref/modules/main.rst: -------------------------------------------------------------------------------- 1 | ======================================= 2 | ``main`` — Bibliographic data retrieval 3 | ======================================= 4 | 5 | .. automodule:: main 6 | 7 | .. contents:: 8 | :local: 9 | 10 | Retrieval interfaces 11 | ==================== 12 | 13 | GUI 14 | --- 15 | 16 | .. automodule:: main.views 17 | :members: 18 | :show-inheritance: 19 | 20 | API 21 | --- 22 | 23 | .. automodule:: main.api 24 | :members: 25 | :show-inheritance: 26 | 27 | Base search view & utilities 28 | ---------------------------- 29 | 30 | .. automodule:: main.search 31 | :members: 32 | :show-inheritance: 33 | 34 | Template tags 35 | ------------- 36 | 37 | .. automodule:: main.templatetags.relaton 38 | :members: 39 | 40 | Sourcing data 41 | ============= 42 | 43 | Relaton source implementation 44 | ----------------------------- 45 | 46 | .. automodule:: main.sources 47 | :members: 48 | 49 | Querying indexed sources 50 | ------------------------ 51 | 52 | .. automodule:: main.query 53 | :members: 54 | 55 | 56 | .. automodule:: main.query_utils 57 | :members: 58 | 59 | External source registry 60 | ------------------------ 61 | 62 | .. automodule:: main.external_sources 63 | :members: 64 | 65 | Data models 66 | =========== 67 | 68 | .. autoclass:: main.models.RefData 69 | :members: 70 | 71 | Types 72 | ----- 73 | 74 | .. automodule:: main.types 75 | :members: 76 | :show-inheritance: 77 | :exclude-members: __init__ 78 | 79 | Exceptions 80 | ---------- 81 | 82 | .. automodule:: main.exceptions 83 | :members: 84 | :show-inheritance: 85 | -------------------------------------------------------------------------------- /docs/ref/modules/management.rst: -------------------------------------------------------------------------------- 1 | ===================================================== 2 | ``management`` — interfaces for operating the service 3 | ===================================================== 4 | 5 | .. automodule:: management 6 | 7 | .. contents:: 8 | :local: 9 | 10 | xml2rfc resolution checker tool 11 | =============================== 12 | 13 | This tool is part of management GUI, but relevant view functions 14 | are defined in :mod:`xml2rfc_compat.views`. 15 | 16 | Management GUI 17 | ============== 18 | 19 | .. automodule:: management.views 20 | :members: 21 | 22 | Management API 23 | ============== 24 | 25 | .. automodule:: management.api 26 | :members: 27 | 28 | Authentication 29 | ============== 30 | 31 | .. automodule:: management.auth 32 | :members: 33 | -------------------------------------------------------------------------------- /docs/ref/modules/prometheus.rst: -------------------------------------------------------------------------------- 1 | ======================================= 2 | ``prometheus`` — Prometheus integration 3 | ======================================= 4 | 5 | .. automodule:: prometheus 6 | 7 | .. automodule:: prometheus.views 8 | :members: 9 | 10 | Metrics 11 | ======= 12 | 13 | .. automodule:: prometheus.metrics 14 | :members: 15 | :undoc-members: 16 | :private-members: 17 | -------------------------------------------------------------------------------- /docs/ref/modules/sources.rst: -------------------------------------------------------------------------------- 1 | ============================================ 2 | ``sources``: Managing indexable data sources 3 | ============================================ 4 | 5 | .. automodule:: sources 6 | 7 | .. contents:: 8 | :local: 9 | 10 | ``indexable``: Indexable sources 11 | ================================ 12 | 13 | .. automodule:: sources.indexable 14 | :members: 15 | :show-inheritance: 16 | :exclude-members: registry, __init__ 17 | 18 | .. autodata:: sources.indexable.registry 19 | :no-value: 20 | 21 | Managing indexing tasks 22 | ======================= 23 | 24 | ``celery``: Celery app 25 | ---------------------- 26 | 27 | .. automodule:: sources.celery 28 | :members: 29 | 30 | ``tasks``: Celery task definitions 31 | ---------------------------------- 32 | 33 | .. automodule:: sources.tasks 34 | :members: 35 | 36 | ``task_status``: Task status helper 37 | ----------------------------------- 38 | 39 | .. automodule:: sources.task_status 40 | :members: 41 | -------------------------------------------------------------------------------- /docs/ref/modules/xml2rfc_compat.rst: -------------------------------------------------------------------------------- 1 | ======================================================== 2 | ``xml2rfc_compat``: xml2rfc compatibility implementation 3 | ======================================================== 4 | 5 | .. automodule:: xml2rfc_compat 6 | 7 | .. seealso:: :doc:`/topics/xml2rfc-compat` topic 8 | 9 | .. contents:: 10 | :local: 11 | 12 | Directory name aliases 13 | ====================== 14 | 15 | .. automodule:: xml2rfc_compat.aliases 16 | :members: 17 | 18 | URL patterns 19 | ============ 20 | 21 | .. automodule:: xml2rfc_compat.urls 22 | :members: 23 | 24 | xml2rfc API views 25 | ================= 26 | 27 | .. automodule:: xml2rfc_compat.views 28 | :members: 29 | :show-inheritance: 30 | 31 | Management views 32 | ================ 33 | 34 | .. automodule:: xml2rfc_compat.management_views 35 | :members: 36 | :show-inheritance: 37 | 38 | Adapters for xml2rfc paths 39 | ========================== 40 | 41 | .. automodule:: xml2rfc_compat.adapters 42 | :members: 43 | 44 | Serializing per RFC 7991 45 | ======================== 46 | 47 | .. automodule:: xml2rfc_compat.serializer 48 | :members: 49 | 50 | Indexable source 51 | ================ 52 | 53 | .. automodule:: xml2rfc_compat.source 54 | :members: 55 | 56 | Data models (ORM) 57 | ================= 58 | 59 | .. automodule:: xml2rfc_compat.models 60 | :show-inheritance: 61 | :members: 62 | 63 | Data models (dataclasses) 64 | ========================= 65 | 66 | .. automodule:: xml2rfc_compat.types 67 | :show-inheritance: 68 | :members: 69 | :exclude-members: __init__ 70 | -------------------------------------------------------------------------------- /docs/topics/architecture.rst: -------------------------------------------------------------------------------- 1 | ============ 2 | Architecture 3 | ============ 4 | 5 | Dependency overview 6 | =================== 7 | 8 | .. image:: ./dependency-diagram.svg 9 | :alt: 10 | A diagram describing dependencies between 11 | services and data sources, under and outside of IETF control. 12 | 13 | Basic entities 14 | ============== 15 | 16 | The two most important entities operated upon by the service are: 17 | 18 | 1. Indexed :term:`reference`. 19 | 20 | Contains partial (e.g., without fully resolved relations) 21 | or full representation of a bibliographic item. 22 | 23 | Uniquely identified by source and own ID within the source. 24 | 25 | 2. :term:`Bibliographic item`. 26 | 27 | A “hydrated” bibliographic item displayed by the service, 28 | constructed from (possibly multiple) indexed references. 29 | 30 | These are fuzzily identified by a :term:`primary resource identifier`. 31 | 32 | .. note:: 33 | 34 | Currently bibliographic items are constructed from indexed references 35 | on the fly, but as a further optimization measure the service may 36 | start constructing bibliographic items and storing them in the database 37 | for faster access. 38 | 39 | Data sources 40 | ============ 41 | 42 | BibXML service relies on indexable 43 | :term:`bibliographic data sources ` 44 | configured (registered). 45 | From those sources, BibXML service obtains bibliographic data 46 | in the Relaton YAML format and indexes it for search purposes. 47 | Indexing can be triggered either via management GUI or via API. 48 | 49 | Those sources themselves are Git repositories, 50 | the contents of which are in some cases static snapshots 51 | but typically are generated periodically using respective 52 | Github Actions workflows, which retrieve and parse bibliographic data 53 | from authoritative sources and output it formatted consistently 54 | per the Relaton data model. 55 | 56 | .. seealso:: 57 | 58 | * For a full list of Relaton datasets and how they are generated: 59 | https://github.com/ietf-tools/bibxml-service/wiki/Data-sources 60 | 61 | * For where Relaton sources are registered, :mod:`main.sources` 62 | 63 | For :doc:`/topics/xml2rfc-compat` functionality, 64 | BibXML service requires an additional indexable source (not pictured above) 65 | that contains xml2rfc paths and associated XML data. 66 | This source is used for xml2rfc path resolution fallback functionality. 67 | Indexing can be triggered the same way as for bibliographic data sources. 68 | 69 | .. seealso:: 70 | 71 | * For where the xml2rfc archive source is registered, 72 | :mod:`xml2rfc_compat.source` 73 | 74 | The role of the database 75 | ======================== 76 | 77 | The PostgreSQL database that is part of the service does not contain 78 | critical data, losing all data would only require reindexing 79 | the above bibliographic data sources. 80 | -------------------------------------------------------------------------------- /docs/topics/index.rst: -------------------------------------------------------------------------------- 1 | ============ 2 | Topic guides 3 | ============ 4 | 5 | .. toctree:: 6 | :maxdepth: 2 7 | 8 | architecture 9 | relaton-format 10 | validation 11 | sourcing 12 | xml2rfc-compat 13 | auth 14 | production-setup 15 | -------------------------------------------------------------------------------- /docs/topics/relaton-format.rst: -------------------------------------------------------------------------------- 1 | =================== 2 | Relaton data format 3 | =================== 4 | 5 | The canonical representation of a bibliographic item used by this service internally 6 | is in Relaton format. 7 | 8 | The exact specification is in development, but a rough outline is available as: 9 | 10 | - `RelaxNG grammar `_ 11 | - `LutaML models `_ 12 | 13 | Also, see the :class:`BibliographicItem ` class 14 | definition. 15 | -------------------------------------------------------------------------------- /docs/topics/validation.rst: -------------------------------------------------------------------------------- 1 | ============================= 2 | Bibliographic data validation 3 | ============================= 4 | 5 | This service deals with bibliographic data obtained 6 | from different sources compiled by external tools. 7 | 8 | The data models defined (see :mod:`bib_models`) 9 | handle data validation automatically, meaning constructed 10 | bibliographic items can be assumed to have the expected properties 11 | of correct types. 12 | 13 | In some scenarios, source data may not validate. 14 | This could be due to an error in the code (external or internal) 15 | that constructs source data, or due to backwards-incompatible changes 16 | in Relaton model (in case an external source conforms to a newer version of the spec, 17 | while this service expects a previous version). 18 | 19 | .. _strict-validation: 20 | 21 | Strict validation with the “``strict``” parameter 22 | ================================================= 23 | 24 | To make such situations less problematic, 25 | some functions responsible for constructing bibliographic item instances 26 | support a ``strict`` boolean keyword argument. 27 | 28 | By default it is ``True``, and any item that fails validation 29 | will make the function raise a :class:`pydantic.ValidationError`. 30 | The caller should never receive a malformed item. 31 | 32 | If explicitly set to ``False``, the item will be constructed anyway, 33 | but it may contain unexpected data types. For example, it may 34 | have dictionaries instead of objects, or timestamps as strings 35 | instead of appropriate datetime objects. 36 | 37 | .. note:: 38 | 39 | Setting ``strict=False`` is intended for cases like forgiving template rendering, 40 | and is not recommended if returned item is to be used programmatically. 41 | 42 | .. seealso:: 43 | 44 | Some functions that use ``strict``: 45 | 46 | - :func:`main.query.build_citation_for_docid` 47 | - :func:`bib_models.util.construct_bibitem`, with callers: 48 | 49 | - :func:`main.query.get_indexed_item` 50 | - :func:`main.query_utils.compose_bibitem` 51 | - :func:`datatracker.internet_drafts.get_internet_draft` 52 | - :func:`doi.get_doi_ref` 53 | -------------------------------------------------------------------------------- /doi/__init__.py: -------------------------------------------------------------------------------- 1 | """Obtaining bibliographic items by DOI. 2 | 3 | Registers an external source. 4 | """ 5 | 6 | import logging 7 | 8 | import requests 9 | import requests_cache 10 | from simplejson import JSONDecodeError 11 | 12 | from bib_models import DocID 13 | from main.types import ExternalBibliographicItem 14 | from main import external_sources 15 | 16 | from .crossref import get_bibitem 17 | 18 | 19 | log = logging.getLogger(__name__) 20 | 21 | 22 | @external_sources.register_for_types('doi', {'DOI': False}) 23 | def get_doi_ref( 24 | doi: str, 25 | strict: bool = True, 26 | ) -> ExternalBibliographicItem: 27 | """ 28 | Obtains an item by DOI using Crossref. 29 | 30 | :param bool strict: see :ref:`strict-validation`. 31 | :rtype: main.types.ExternalBibliographicItem 32 | :raises main.exceptions.RefNotFoundError: reference not found 33 | """ 34 | 35 | with requests_cache.enabled(): 36 | try: 37 | sourced_item: ExternalBibliographicItem = \ 38 | get_bibitem(DocID( 39 | type='DOI', 40 | id=doi, 41 | ), strict=strict) 42 | except requests.exceptions.ConnectionError: 43 | raise RuntimeError("Error connecting to external source") 44 | except JSONDecodeError: 45 | raise RuntimeError("Could not decode external source response") 46 | except RuntimeError: 47 | raise 48 | else: 49 | return sourced_item 50 | -------------------------------------------------------------------------------- /k8s/kustomization.yaml: -------------------------------------------------------------------------------- 1 | namespace: bib 2 | resources: 3 | - bib.yaml 4 | -------------------------------------------------------------------------------- /k8s/secrets.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | metadata: 4 | name: secrets-env 5 | type: Opaque 6 | stringData: 7 | AUTO_REINDEX_INTERVAL: "5400" 8 | CELERY_BROKER_URL: "redis://localhost:6379" 9 | CELERY_RESULT_BACKEND: "redis://localhost:6379" 10 | CONTACT_EMAIL: "tools-help@ietf.org" 11 | DATASET_TMP_ROOT: "/data/datasets" 12 | DEBUG: "0" 13 | INTERNAL_HOSTNAMES: "localhost,bib.bib.svc.cluster.local,127.0.0.1" 14 | 15 | # DATATRACKER_CLIENT_ID: null 16 | 17 | # MATOMO_SITE_ID: null 18 | # MATOMO_TAG_MANAGER_CONTAINER: null 19 | # MATOMO_URL: "analytics.ietf.org" 20 | 21 | PORT: "8000" 22 | PRIMARY_HOSTNAME: "bib.ietf.org" 23 | PYTHONUNBUFFERED: "1" 24 | REDIS_HOST: "localhost" 25 | REDIS_PORT: "6379" 26 | SERVER_EMAIL: "support@ietf.org" 27 | SERVICE_NAME: "IETF BibXML Service" 28 | SOURCE_REPO_URL: "https://github.com/ietf-tools/bibxml-service" 29 | 30 | # Secrets from Vault: 31 | # DB_HOST: "" 32 | # DB_NAME: "" 33 | # DB_PORT: "" 34 | # DB_SECRET: "" 35 | # DB_USER: "" 36 | # DJANGO_SECRET: "" 37 | # EXTRA_API_SECRETS: "" 38 | # SENTRY_DSN: "" 39 | -------------------------------------------------------------------------------- /main/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ietf-tools/bibxml-service/830eb8a3f03ed3885ed08b5d7092516ccb711139/main/__init__.py -------------------------------------------------------------------------------- /main/app.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | from django.apps import AppConfig 3 | 4 | 5 | class Config(AppConfig): 6 | default_auto_field = 'django.db.models.BigAutoField' 7 | name = 'main' 8 | 9 | def ready(self): 10 | importlib.import_module('main.sources') 11 | -------------------------------------------------------------------------------- /main/exceptions.py: -------------------------------------------------------------------------------- 1 | class RefNotFoundError(RuntimeError): 2 | """Bibliographic item could not be found in source. 3 | 4 | :param str query: Reference or query that did not yield a match.""" 5 | 6 | def __init__(self, message="Item not found", query="(unknown query)"): 7 | super().__init__(message) 8 | self.query = query 9 | -------------------------------------------------------------------------------- /main/external_sources.py: -------------------------------------------------------------------------------- 1 | """Provides an external source registry.""" 2 | 3 | from typing import Dict, Callable, Optional, Union 4 | import logging 5 | 6 | from pydantic.dataclasses import dataclass 7 | 8 | from bib_models import DocID 9 | 10 | from .types import ExternalBibliographicItem 11 | 12 | 13 | log = logging.getLogger(__name__) 14 | 15 | 16 | @dataclass 17 | class ExternalSource: 18 | """Represents a registered external source.""" 19 | 20 | get_item: Callable[[str, Optional[bool]], ExternalBibliographicItem] 21 | """Returns an item given docid.id. The ``strict`` argument 22 | is True by default and means the method must throw 23 | if received item did not pass validation. 24 | """ 25 | 26 | applies_to: Callable[[DocID], bool] 27 | """Returns True if this source applies to given identifier. 28 | Can be used to automatically fetch data from sources. 29 | """ 30 | 31 | primary_for: Callable[[DocID], bool] 32 | """Returns True if this source is primary to given identifier. 33 | This means it contains complete authoritative data 34 | relative to other sources. 35 | 36 | Can be used to automatically fetch data from sources. 37 | """ 38 | 39 | 40 | def get(id: str) -> ExternalSource: 41 | """Returns a registered external source by ID.""" 42 | return registry[id] 43 | 44 | 45 | def register_for_types(id: str, doc_types: Dict[str, bool]): 46 | """ 47 | Registers external source with given ID for specified document types 48 | (``docid.type`` values in Relaton model). 49 | """ 50 | def applies_to(docid: DocID) -> bool: 51 | return doc_types.get(docid.type, None) is not None 52 | 53 | def primary_for(docid: DocID) -> bool: 54 | return doc_types.get(docid.type, None) == True 55 | 56 | def register_external_source(item_getter: Callable[ 57 | [str, Optional[bool]], ExternalBibliographicItem 58 | ]): 59 | registry[id] = ExternalSource( 60 | applies_to=applies_to, 61 | primary_for=primary_for, 62 | get_item=item_getter, 63 | ) 64 | return item_getter 65 | 66 | return register_external_source 67 | 68 | 69 | registry: Dict[str, ExternalSource] = {} 70 | """Registry of external sources.""" 71 | -------------------------------------------------------------------------------- /main/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.9 on 2021-11-18 21:12 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | initial = True 9 | 10 | dependencies = [ 11 | ] 12 | 13 | operations = [ 14 | migrations.CreateModel( 15 | name='RefData', 16 | fields=[ 17 | ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 18 | ('ref', models.CharField(help_text='Reference (or ID). Corresponds to source dataset filename without extension.', max_length=128)), 19 | ('ref_id', models.CharField(max_length=64)), 20 | ('ref_type', models.CharField(max_length=24)), 21 | ('dataset', models.CharField(help_text='Internal dataset ID.', max_length=24)), 22 | ('body', models.JSONField()), 23 | ('representations', models.JSONField()), 24 | ], 25 | options={ 26 | 'db_table': 'api_ref_data', 27 | 'unique_together': {('ref', 'dataset')}, 28 | }, 29 | ), 30 | ] 31 | -------------------------------------------------------------------------------- /main/migrations/0002_refdata_body_gin.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.11 on 2022-01-09 12:28 2 | 3 | import django.contrib.postgres.indexes 4 | from django.db import migrations 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('main', '0001_initial'), 11 | ] 12 | 13 | operations = [ 14 | migrations.AddIndex( 15 | model_name='refdata', 16 | index=django.contrib.postgres.indexes.GinIndex(fields=['body'], name='body_gin'), 17 | ), 18 | ] 19 | -------------------------------------------------------------------------------- /main/migrations/0003_refdata_body_astext_gin.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.11 on 2022-01-09 12:44 2 | 3 | import django.contrib.postgres.indexes 4 | import django.contrib.postgres.search 5 | from django.db import migrations, models 6 | import django.db.models.functions.comparison 7 | 8 | 9 | class Migration(migrations.Migration): 10 | 11 | dependencies = [ 12 | ('main', '0002_refdata_body_gin'), 13 | ] 14 | 15 | operations = [ 16 | migrations.AddIndex( 17 | model_name='refdata', 18 | index=django.contrib.postgres.indexes.GinIndex(django.contrib.postgres.search.SearchVector(django.db.models.functions.comparison.Cast('body', models.TextField()), config='english'), name='body_astext_gin'), 19 | ), 20 | ] 21 | -------------------------------------------------------------------------------- /main/migrations/0004_refdata_body_ts_gin.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.11 on 2022-01-09 12:55 2 | 3 | import django.contrib.postgres.indexes 4 | import django.contrib.postgres.search 5 | from django.db import migrations 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('main', '0003_refdata_body_astext_gin'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AddIndex( 16 | model_name='refdata', 17 | index=django.contrib.postgres.indexes.GinIndex(django.contrib.postgres.search.SearchVector('body', config='english'), name='body_ts_gin'), 18 | ), 19 | ] 20 | -------------------------------------------------------------------------------- /main/migrations/0005_refdata_body_docid_gin.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.11 on 2022-01-11 15:41 2 | 3 | import django.contrib.postgres.indexes 4 | import django.contrib.postgres.search 5 | from django.db import migrations 6 | import django.db.models.fields.json 7 | 8 | 9 | class Migration(migrations.Migration): 10 | 11 | dependencies = [ 12 | ('main', '0004_refdata_body_ts_gin'), 13 | ] 14 | 15 | operations = [ 16 | migrations.AddIndex( 17 | model_name='refdata', 18 | index=django.contrib.postgres.indexes.GinIndex(django.contrib.postgres.search.SearchVector(django.db.models.fields.json.KeyTransform('docid', 'body'), config='english'), name='body_docid_gin'), 19 | ), 20 | ] 21 | -------------------------------------------------------------------------------- /main/migrations/0006_auto_20220123_1622.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.0.1 on 2022-01-23 16:22 2 | 3 | from django.db import migrations 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('main', '0005_refdata_body_docid_gin'), 10 | ] 11 | 12 | operations = [ 13 | migrations.RunSQL(''' 14 | CREATE INDEX "body_json_ts_gin" ON "api_ref_data" 15 | USING gin (to_tsvector('english'::regconfig, body)); 16 | '''), 17 | ] 18 | -------------------------------------------------------------------------------- /main/migrations/0007_alter_refdata_dataset_alter_refdata_ref_and_more.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.0.2 on 2022-02-14 22:17 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('main', '0006_auto_20220123_1622'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AlterField( 14 | model_name='refdata', 15 | name='dataset', 16 | field=models.CharField(db_index=True, help_text='Internal dataset ID.', max_length=24), 17 | ), 18 | migrations.AlterField( 19 | model_name='refdata', 20 | name='ref', 21 | field=models.CharField(db_index=True, help_text='Reference (or ID). Corresponds to source dataset filename without extension.', max_length=128), 22 | ), 23 | migrations.AlterField( 24 | model_name='refdata', 25 | name='representations', 26 | field=models.JSONField(default=dict), 27 | ), 28 | ] 29 | -------------------------------------------------------------------------------- /main/migrations/0008_refdata_latest_date.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.0.2 on 2022-02-14 22:18 2 | 3 | import datetime 4 | from django.db import migrations, models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('main', '0007_alter_refdata_dataset_alter_refdata_ref_and_more'), 11 | ] 12 | 13 | operations = [ 14 | migrations.AddField( 15 | model_name='refdata', 16 | name='latest_date', 17 | field=models.DateField(default=datetime.date(2022, 2, 14)), 18 | preserve_default=False, 19 | ), 20 | ] 21 | -------------------------------------------------------------------------------- /main/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ietf-tools/bibxml-service/830eb8a3f03ed3885ed08b5d7092516ccb711139/main/migrations/__init__.py -------------------------------------------------------------------------------- /main/templates/browse/base.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | -------------------------------------------------------------------------------- /main/templates/browse/dataset.html: -------------------------------------------------------------------------------- 1 | {% extends "browse/base.html" %} 2 | 3 | {% load relaton %} 4 | 5 | {% block title %}{{ block.super }} — {{ dataset_id }}{% endblock %} 6 | 7 | {% block header_extras %} 8 |
15 |

Indexed references from source {{ dataset_id }}

16 |
17 | 18 | 24 | 25 | 31 | 32 | {{ block.super }} 33 | {% endblock %} 34 | 35 | {% block content %} 36 | {{ block.super }} 37 | 38 | {% for ref in object_list %} 39 | {% include "citation/in_list.html" with data=ref.bibitem|default:ref.body dataset_id=dataset_id ref=ref.ref dataset_id=ref.dataset %} 40 | {% endfor %} 41 | {% endblock %} 42 | -------------------------------------------------------------------------------- /main/templates/browse/home.html: -------------------------------------------------------------------------------- 1 | {% extends "browse/base.html" %} 2 | 3 | {% block container_grid_classes %} 4 | xl:thegrid-mini 5 | xl:place-content-center 6 | {% endblock %} 7 | 8 | {% block content_grid_classes %} 9 | xl:thegrid-mini 10 | md:grid-cols-6 11 | lg:grid-cols-10 12 | xl:grid-cols-12 13 | 14 | md:row-[span_6_/_span_6] 15 | {% endblock %} 16 | 17 | {% block footer %}{% endblock %} 18 | 19 | {% block header_extras %} 20 | 21 |
29 |

30 | Bibliographic database 31 | for people 32 | who work 33 | on IETF standards. 34 | {% include "_about_links.html" with htmlclass="font-semibold mt-1 block" %} 35 |

36 |
37 | 38 | {{ block.super }} 39 | 40 |
41 |

42 | {% if browsable_datasets %} 43 | Bibliographic item sources: 44 | {% for ds_id in browsable_datasets %} 45 | {{ ds_id }}{% if not forloop.last %}, {% endif %} 47 | {% endfor %} 48 | {% else %} 49 | There are no indexed items at this time. 50 | {% endif %} 51 |

52 |
53 | {% endblock %} 54 | 55 | 56 | {% block content %} 57 |
68 | {% include "browse/search_forms.html" with expanded=1 query=request.GET.query query_format=request.GET.query_format|default:"websearch" %} 69 |
70 | 71 |
72 |
81 | 87 | 90 |
91 |
92 | {% endblock %} 93 | 94 | {% block before_container %} 95 | {% include "_messages.html" with wrapper_htmlclass="bg-dark-700 xl:bg-transparent" %} 96 | {% endblock %} 97 | -------------------------------------------------------------------------------- /main/templates/browse/paginator.html: -------------------------------------------------------------------------------- 1 |
2 | « 9 | prev. 15 | 16 | 20 | {{ page_obj.number }}/{{ page_obj.paginator.num_pages }} 21 | 22 | 23 | next 29 | » 36 |
37 | 38 | 47 | 48 | 49 |
50 | {{ page_obj.paginator.count }}{% if result_cap and page_obj.paginator.count >= result_cap %}+{% endif %} 51 | to show. 52 | {% if query %} 53 | Refine query 57 | {% endif %} 58 |
59 | -------------------------------------------------------------------------------- /main/templates/browse/search_citations.html: -------------------------------------------------------------------------------- 1 | {% extends "browse/base.html" %} 2 | 3 | {% block title %}{{ block.super }} — Search: {{ query }}{% endblock %} 4 | 5 | {% block header_extras %} 6 |

16 |
17 | Search results 18 | {% if query_format_label %} 19 | for {{ query_format_label }}: 20 | {% endif %} 21 |
22 | 
{{ query|default:request.GET.query|default:"N/A" }}
28 |

29 | 30 | 36 | 37 | {{ block.super }} 38 | {% endblock %} 39 | 40 | {% block content %} 41 | {{ block.super }} 42 | 43 | {% for citation in object_list %} 44 | {% include "citation/in_list.html" with data=citation %} 45 | {% empty %} 46 |

No items to show.

47 | {% endfor %} 48 | {% endblock %} 49 | 50 | {% block after_content %} 51 | {% include "_profiling_block.html" %} 52 | {% endblock %} 53 | -------------------------------------------------------------------------------- /main/templates/browse/search_form_quick.html: -------------------------------------------------------------------------------- 1 | {# Accepts optional detailed, htmlclass, control_htmlclass, tabindex, autofocus, placeholder #} 2 | 3 |
13 | 23 | 26 | 29 |
30 | -------------------------------------------------------------------------------- /main/templates/citation/bibitem_source.html: -------------------------------------------------------------------------------- 1 | {# Accepts sourced_item, indexed (true if item is not external), htmlclass, show_error_icon, show_internal_links. #} 2 | 14 | {% if sourced_item.validation_errors|length > 0 or sourced_item.indexed_object %} 15 | {% if show_error_icon and sourced_item.validation_errors|length > 0 %} 16 | {% include "_error_icon.html" with title="Source did not validate" htmlclass="h-5 w-5 mr-1 inline text-rose-600" %} 17 | {% endif %} 18 | {% if sourced_item.indexed_object %} 19 | {% if show_internal_links %} 20 | 24 | {{ sourced_item.indexed_object.name }} 25 | 26 | {% else %} 27 | 28 | {{ sourced_item.indexed_object.name }} 29 | 30 | {% endif %} 31 | {% endif %} 32 | {% endif %} 33 | 34 | {% if indexed %} 35 | indexed from 36 | {% else %} 37 | requested via 38 | {% endif %} 39 | {% if show_internal_links and indexed %} 40 | {{ sourced_item.source.id }} 44 | {% else %} 45 | {{ sourced_item.source.id }} 46 | {% endif %} 47 | 48 | 49 | -------------------------------------------------------------------------------- /main/templates/citation/docid_search_link.html: -------------------------------------------------------------------------------- 1 | {% load relaton %} 2 | {% with link=val|substruct_search_link:'{"docid": [{"id": %s}]}' %} 3 | {{ val }} 5 | {% endwith %} 6 | -------------------------------------------------------------------------------- /main/templates/citation/in_list.html: -------------------------------------------------------------------------------- 1 | {# Expects context: bibliographic item as data, forloop, optional ref, dataset_id #} 2 | 3 | {% load relaton common %} 4 | 5 |
6 |
7 |

8 | {% with link=data|bibitem_link %} 9 | {% if ref or link %} 10 | 22 | {% include "relaton/smart_title.html" with bibitem=data %} 23 | 24 | {% else %} 25 | {# TODO: #196, this occurs when bibitem_link fails due to missing docids #} 26 | {% include "relaton/smart_title.html" with bibitem=data %} 27 | {% endif %} 28 | {% endwith %} 29 |

30 | 31 |
39 | {% include "relaton/doc_ids.html" with ids=data.docid|as_list query=query tabindex="0" %} 40 |
41 | 42 |
43 | {% if data.headline %} 44 | … {{ data.headline|safe }} … 45 | {% else %} 46 | 55 | {% endif %} 56 |
57 |
58 |
59 | -------------------------------------------------------------------------------- /main/templates/citation/series_search_link.html: -------------------------------------------------------------------------------- 1 | {% load relaton %} 2 | {% with link=val|substruct_search_link:'{"series": [{"title": {"content": %s}}]}' %} 3 | {{ val }} 5 | {% endwith %} 6 | -------------------------------------------------------------------------------- /main/templates/citation/sourced_bibitem_raw_table.html: -------------------------------------------------------------------------------- 1 | {# Outputs a generic grid of bibitem properties. #} 2 | {# Expects bibitem (a BibliographicItem instance), sourced_item (SourcedBibliographicItem instance) #} 3 | 4 | {% load pydantic %} 5 | 6 | 15 | {# Browsers insert a tbody anyway, so we might as well have it here to not forget #} 16 | 17 | {% for field in bibitem|flatten_and_annotate:sourced_item.validation_errors %} 18 | {% with field.pydantic_loc|with_parents|get_validation_errors:sourced_item.validation_errors as field_errors %} 19 | {% if field.value or field.validation_errors %} {# Omit null/None fields, unless they have errors #} 20 | 25 | 40 | 48 | 49 | {% endif %} 50 | {% endwith %} 51 | {% endfor %} 52 | 53 |
38 | {{ field.pydantic_loc|pretty_print_loc }} 39 | 46 | {{ field.value|default:"N/A" }} 47 |
54 | -------------------------------------------------------------------------------- /main/templates/citation/sources.html: -------------------------------------------------------------------------------- 1 | {% for sourced_item in items %} 2 | {{ sourced_item.source.id }} 3 | {% endfor %} 4 | -------------------------------------------------------------------------------- /main/templates/citation/srp_link_href_get_query.html: -------------------------------------------------------------------------------- 1 | {# Accepts query, query_format, page (Paginator page object) #} 2 | {% if query %}query={{ query|urlencode:"" }}{% if query_format %}&query_format={{ query_format }}{% endif %}{% if page %}&page={{ page.number }}{% endif %}{% endif %} 3 | -------------------------------------------------------------------------------- /main/templates/citation/validation_errors.html: -------------------------------------------------------------------------------- 1 | {% spaceless %}{# Whitespace/indentation is important here, this may be displayed in a tooltip. #} 2 | {% load pydantic %} 3 | {% for err in validation_errors %}{% if in_page_links %}{{ err.loc|pretty_print_loc }}{% else %}{{ err.loc|pretty_print_loc }}{% endif %}: {{ err.msg }}{% if not forloop.last %}, {% endif %} 4 | {% endfor %} 5 | {% endspaceless %} 6 | -------------------------------------------------------------------------------- /main/templates/deflist/entry.html: -------------------------------------------------------------------------------- 1 | {# Expects key, val, tmpl, idx, wide, break_row, htmlclass, key_sr_only, tooltip #} 2 | 3 |
6 |
7 | {{ key }}{% if idx %} ({{ idx }}){% endif %} 8 |
9 |
10 | 11 | {% if tmpl %} 12 | {% include tmpl with val=val %} 13 | {% else %} 14 | {{ val|default:"—" }} 15 | {% endif %} 16 | 17 |
18 |
19 | -------------------------------------------------------------------------------- /main/templates/deflist/entry_list.html: -------------------------------------------------------------------------------- 1 | {# Expects key, val, tmpl, idx, wide, break_row, htmlclass, item_htmlclass, key_sr_only, inline, comma_sep #} 2 | 3 |
6 |
7 | {{ key|default:"N/A" }}{% if idx %} ({{ idx }}){% endif %} 8 |
9 |
10 |
    18 | {% for val in items %} 19 |
  • 27 | {% if tmpl %} 28 | {% include tmpl with key="item "|add:forloop.counter key_sr_only=True val=val %} 29 | {% else %} 30 | {{ val|default:"—" }} 31 | {% endif %} 32 |
    • 33 | {% endfor %} 34 |
35 |
36 |
37 | -------------------------------------------------------------------------------- /main/templates/deflist/entry_recursivedict.html: -------------------------------------------------------------------------------- 1 | {# Expects key, val, tmpl, idx, inline (?), wide, break_row, htmlclass, key_sr_only #} 2 | 3 |
9 |
10 | {{ key|default:"N/A" }}{% if idx %} {{ idx }}{% endif %}: 11 |
12 |
13 | {% if val.items %} 14 | {# Recurse into a dictionary #} 15 | { 16 |
17 | {% for key, val in val.items %} 18 | {% if val %} 19 | {% include "deflist/entry_recursivedict.html" with key=key val=val inline=1 %} 20 | {% endif %} 21 | {% endfor %} 22 |
23 | } 24 | {% elif val|stringformat:"r"|first == "[" %} 25 | {# Recurse into a list #} 26 | [ 27 |
    28 | {% for item in val %} 29 |
  • 30 | {% if item.items or val|stringformat:"r"|first == "[" %} 31 | {% include "deflist/entry_recursivedict.html" with key="item "|add:forloop.counter key_sr_only=True val=item inline=1 %} 32 | {% else %} 33 | {{ val|default:"—" }} 34 | {% endif %} 35 |
    • 36 | {% endfor %} 37 |
38 | ] 39 | {% else %} 40 | {# Show simple value #} 41 | {{ val|default:"—" }} 42 | {% endif %} 43 |
44 |
45 | -------------------------------------------------------------------------------- /main/templates/relaton/contributor.html: -------------------------------------------------------------------------------- 1 | {% load relaton common %} 2 | 3 | {% if val.organization.abbreviation or val.organization.name or val.person.name %} 4 | 5 | {% if val.role %} 6 | 7 | {% for role in val.role|as_list %} 8 | {# NB: roles should be sorted by type for ifchanged to make sense #} 9 | {% ifchanged role.type %} 10 | {# We just ignore roles without type for now #} 11 | {% if role.type %} 12 | {{ role.type }} 16 | {% endif %} 17 | {% endifchanged %} 18 | {% endfor %} 19 | 20 | {% endif %} 21 | 22 | {% if val.organization.abbreviation or val.organization.name %} 23 | {% with link=val|substruct_search_link:'{"contributor": [%s]};as_list=yes;only=organization.abbreviation,organization.name[*].content' %} 24 | 25 | {% include "relaton/org.html" with val=val.organization %} 26 | 27 | {% endwith %} 28 | {% elif val.person.name %} 29 | {% with link=val|substruct_search_link:'{"contributor": [%s]};as_list=yes;only=person.name.completename.content,person.name.surname.content,person.name.given.forename[*].content' %} 30 | 31 | {% include "relaton/person_name.html" with val=val.person.name %} 32 | 33 | {% endwith %} 34 | {% endif %} 35 | 36 | {% endif %} 37 | -------------------------------------------------------------------------------- /main/templates/relaton/copyright.html: -------------------------------------------------------------------------------- 1 | © {{ val.from }} 2 | 3 | {% for owner in val.owner %} 4 | {% with owner.name|first as owner_name %} 5 | {% if owner.url %}{{ owner_name.content }}{% else %}{{ owner_name }}{% endif %}{% if not forloop.last %}, {% endif %} 6 | {% endwith %} 7 | {% endfor %} 8 | 9 | -------------------------------------------------------------------------------- /main/templates/relaton/date.html: -------------------------------------------------------------------------------- 1 | 2 | {{ val.type|default:"date" }} 3 | 4 | {{ val.value }} 5 | -------------------------------------------------------------------------------- /main/templates/relaton/doc_ids.html: -------------------------------------------------------------------------------- 1 | {% for docid in ids %} 2 | {% if not docid.scope %} {# TODO: Remove condition when scope is removed #} 3 | 8 | {% if as_links %} 9 | {% if show_type and not docid.primary %} 10 | 11 | {{ docid.type }} 12 | 13 | {% endif %} 14 | 18 | {{ docid.id }} 19 | {% else %} 20 | {% if show_type and not docid.primary and docid.type %} 21 | 22 | {{ docid.type }} 23 | 24 | {% endif %} 25 | 26 | {{ docid.id }} 27 | 28 | {% endif %} 29 | 30 | {% endif %} 31 | {% endfor %} 32 | -------------------------------------------------------------------------------- /main/templates/relaton/formatted_string.html: -------------------------------------------------------------------------------- 1 | {% load relaton %} 2 | 3 | {% if val.content %} 4 | {% if val.format == "text/html" %} 5 | {{ val.content|safe }} 6 | {% else %} 7 | {{ val|to_html|safe }} 8 | {% endif %} 9 | {% else %} 10 | N/A 11 | {% endif %} 12 | -------------------------------------------------------------------------------- /main/templates/relaton/keyword.html: -------------------------------------------------------------------------------- 1 | {% load relaton %} 2 | {% with link=val|substruct_search_link:'{"keyword": [%s]}' %} 3 | {{ val.content }} 5 | {% endwith %} 6 | -------------------------------------------------------------------------------- /main/templates/relaton/link.html: -------------------------------------------------------------------------------- 1 | {% if "http" in val.content %} 2 | external link 3 | {{ val.type|default:val.content }} 4 | {% else %} 5 | {{ val.type|default:"external link" }} 6 | {{ val.content }} 7 | {% endif %} 8 | -------------------------------------------------------------------------------- /main/templates/relaton/org.html: -------------------------------------------------------------------------------- 1 | {% load common %} 2 | {# Expects `val` to be an Organization instance #} 3 | 4 | {% with org_name=val.name|as_list|first %} 5 | {% if val.abbreviation.content %} 6 | {{ val.abbreviation.content }} 7 | {% elif org_name.content %} 8 | {{ org_name.content }} 9 | {% endif %} 10 | {% endwith %} 11 | -------------------------------------------------------------------------------- /main/templates/relaton/person_name.html: -------------------------------------------------------------------------------- 1 | {% load common %} 2 | 3 | {% with val.surname.content as surname and val.completename.content as fullname and val.given.forename as given_names and val.initial|as_list as initials %} 4 | {% if fullname %} 5 | {{ fullname }} 6 | {% else %} 7 | {% for name in given_names %} 8 | {{ name.content }}{% if not forloop.last %}, {% endif %} 9 | {% endfor %} 10 | {% for initial in initials %} 11 | {{ initial.content }}. 12 | {% endfor %} 13 | {{ surname }} 14 | {% endif %} 15 | {% endwith %} 16 | -------------------------------------------------------------------------------- /main/templates/relaton/relation.html: -------------------------------------------------------------------------------- 1 | {% load common %} 2 | 3 | {% with rel_type=val.description.content %} 4 | 5 | {{ val.type|split_camel_case|join:" " }} 6 | {% if rel_type %} 7 | ({{ rel_type }}) 8 | {% endif %} 9 | 10 | {% endwith %} 11 | 12 | {% with formattedref_content=val.bibitem.formattedref.content formattedref=val.bibitem.formattedref rel_docid=val.bibitem.docid|first rel_title=val.bibitem.title|as_list|first %} 13 | {% with linked_id=rel_docid.id|default:formattedref_content|default:formattedref %} 14 | {% if linked_id %} 15 | {% include "citation/docid_search_link.html" with val=linked_id htmlclass="whitespace-nowrap" %} 16 | {% if rel_title.content and rel_title.content != bibitem.title.0.content %} 17 | {{ rel_title.content }} 18 | {% endif %} 19 | {% endif %} 20 | {% endwith %} 21 | {% endwith %} 22 | -------------------------------------------------------------------------------- /main/templates/relaton/series.html: -------------------------------------------------------------------------------- 1 | 2 | {% if val.formattedref %} 3 | {% include "citation/series_search_link.html" with val=val.formattedref %} 4 | {% else %} 5 | in series 6 | {% include "citation/series_search_link.html" with val=val.title.content %} 7 | as {{ val.number }} 8 | {% endif %} 9 | 10 | -------------------------------------------------------------------------------- /main/templates/relaton/smart_title.html: -------------------------------------------------------------------------------- 1 | {% load common %} 2 | 3 | {# TODO: (#196) below accounts for bibitems without docid but with formattedref, which should be addressed after #196 #} 4 | {% with own_title=bibitem.title|as_list|first relations=bibitem.relation|as_list %} 5 | {% if own_title.content or bibitem.formattedref.content %} 6 | {% with fallback_docid=bibitem.docid.0.id|default:"" fallback_formattedref=bibitem.formattedref.content %} 7 | {{ own_title.content|default:fallback_formattedref|default:fallback_docid|default:"(no title)" }} 8 | {% endwith %} 9 | {% endif %} 10 | {% if not plain_only and relations|length > 0 and not own_title.content %} 11 | 12 | {% for relation in relations %} 13 | {% ifchanged relation.type %}{{ relation.type|default:"relates to" }}{% endifchanged %} 14 | {% with title=relation.bibitem.title|as_list|first docid=relation.bibitem.docid|as_list|first formattedref=relation.bibitem.formattedref.content %} 15 | {% if docid or formattedref %}{{ docid.id|default:formattedref }}{% endif %}{% if not forloop.last %}, {% endif %} 16 | {% if relations|length < 2 and title.content %} 17 | {{ title.content }} 18 | {% endif %} 19 | {% endwith %} 20 | {% endfor %} 21 | 22 | {% endif %} 23 | {% endwith %} 24 | -------------------------------------------------------------------------------- /main/templatetags/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ietf-tools/bibxml-service/830eb8a3f03ed3885ed08b5d7092516ccb711139/main/templatetags/__init__.py -------------------------------------------------------------------------------- /main/templatetags/common.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | import re 3 | 4 | from django import template 5 | 6 | from common.util import as_list as base_as_list 7 | 8 | 9 | register = template.Library() 10 | 11 | 12 | @register.filter 13 | def as_list(value): 14 | """Returns the value as a list (see :func:`common.util.as_list`), 15 | omitting any ``None`` values.""" 16 | 17 | result: Any = base_as_list(value) 18 | 19 | return [val for val in result if val is not None and val != ''] 20 | 21 | 22 | @register.filter 23 | def split_camel_case(value: str): 24 | """Converts a camelCased string to its list of components in lowercase. 25 | Doesn’t do anything if string contains spaces. 26 | """ 27 | if isinstance(value, str) and ' ' not in value: 28 | 29 | # Separate components by space 30 | space_separated = re.sub( 31 | r''' 32 | ( 33 | (?<=[a-z]) [A-Z] # lowecase followed by uppercase 34 | | # or 35 | (? 15 | {{ label }}: 16 | {{ method }} {{ endpoint }} 17 | {% if openapi_op_id and openapi_spec_root %} 18 | (spec) 19 | {% endif %} 20 | 21 | -------------------------------------------------------------------------------- /management/templates/indexing_task.html: -------------------------------------------------------------------------------- 1 |
2 | 3 | {% if do_link %} 4 | 5 | {{ task.task_id }} 6 | 7 | {% else %} 8 | {{ task.task_id }} 9 | {% endif %} 10 | 11 | {% if task.status != "PENDING" %} 12 | 18 | {{ task.status }} 19 | 20 | {% endif %} 21 | {% if task.progress %} 22 | 27 | 28 | {% url "api_stop_task" task.task_id as stop_task_url %} 29 | {% include "api_button.html" with endpoint=stop_task_url method="POST" label="Revoke task" small=True openapi_op_id="stopTask" openapi_spec_root="/api/v1/" htmlclass="text-xs uppercase" %} 30 | {% endif %} 31 |
32 | 33 |
34 | {% if task.error %} 35 |

{{ task.error.type }}

36 |

{{ task.error.message }}

37 | {% endif %} 38 | {% if task.progress %} 39 |

40 | {{ task.action }} 41 |
42 | {% if task.progress %} 43 | at {{ task.progress.current }} of {{ task.progress.total }} 44 | {% endif %} 45 |

46 | {% endif %} 47 |
48 | 49 | {% if task.status != "PENDING" %} 50 |

51 | {% if task.dataset_id %} 52 | Requested for 53 | 54 | {{ task.dataset_id }}: 55 | {% endif %} 56 | {% if task.requested_refs %} 57 |

    58 | {% for ref in task.requested_refs %} 59 |
  • {{ ref }}
    • 60 | {% endfor %} 61 |
62 | {% else %} 63 | entire dataset 64 | {% endif %} 65 |

66 | 67 |

68 | Completed at: 69 | {% if task.completed_at %} 70 | {{ task.completed_at }} 71 | {% else %} 72 | — 73 | {% endif %} 74 |

75 | {% if show_summary and task.outcome_summary %} 76 | 
{{ task.outcome_summary }}
77 | {% endif %} 78 | {% else %} 79 |

No reliable task information. 80 | {% endif %} 81 | -------------------------------------------------------------------------------- /management/templates/management/base.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | {% load compress static %} 3 | 4 | {% block title %} 5 | {{ block.super }} 6 | — 7 | Management 8 | {% endblock %} 9 | 10 | {% block extra_scripts %} 11 | {% compress js %} 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | {% endcompress %} 23 | {% endblock %} 24 | 25 | {% block root_html_attrs %}data-api-secret="{{ api_secret }}"{% endblock %} 26 | 27 | {% block root_html_class %}management{% endblock %} 28 | 29 | {% block home_link %}{% url "manage" %}{% endblock %} 30 | 31 | {% block after_site_header_content %} 32 | Management dashboard 33 | {% endblock %} 34 | 35 | {% block after_content %} 36 |

42 | 43 | Indexable sources 44 | 45 |
46 | 47 | xml2rfc compatibility 48 | 49 |
50 | 51 | Public-facing service 52 | 53 |
54 | 55 | API reference 56 | 57 |
58 | 59 | Docs 60 | 61 |
62 | 63 | {% if monitoring.flower %} 64 | 69 | Worker dashboard at {{ task_monitor_host }} → 70 | 71 | {% endif %} 72 | {% endblock %} 73 | -------------------------------------------------------------------------------- /management/templates/management/dataset.html: -------------------------------------------------------------------------------- 1 | {% extends "management/datasets.html" %} 2 | 3 | {% block title %} 4 | {{ block.super }} 5 | — 6 | {{ dataset_id }} 7 | {% endblock %} 8 | 9 | {% block content %} 10 | {% include "source_header.html" %} 11 | {% url "api_run_indexer" dataset_id as reindex_url %} 12 | {% url "api_reset_index" dataset_id as reset_url %} 13 | 14 |
15 | {% include "api_button.html" with label="Queue reindex task" endpoint=reindex_url get_query='{"force": true}' method="POST" openapi_op_id="indexDataset" openapi_spec_root="/api/v1/" htmlclass="" %} 16 | {% include "api_button.html" with label="Reset index" endpoint=reset_url method="POST" openapi_op_id="resetDatasetIndex" openapi_spec_root="/api/v1/" htmlclass="" %} 17 |
18 | 19 | {% for task in history %} 20 |
21 |
22 | {% include "indexing_task.html" with task=task show_summary=forloop.first do_link=1 %} 23 |
24 |
25 | {% endfor %} 26 | {% endblock %} 27 | -------------------------------------------------------------------------------- /management/templates/management/datasets.html: -------------------------------------------------------------------------------- 1 | {% extends "management/base.html" %} 2 | 3 | {% block title %} 4 | {{ block.super }} 5 | — 6 | Indexable sources 7 | {% endblock %} 8 | 9 | {% block content %} 10 | {{ block.super }} 11 | 12 | {% for dataset in datasets %} 13 |
14 |
15 | {{ dataset.name }} 17 |   18 | {% if dataset.task_progress %} 19 | 24 | 25 | {% else %} 26 | {{ dataset.item_count }} currently indexed 27 | {% endif %} 28 |
29 | 30 | {% if dataset.task_id %} 31 | 32 | {{ dataset.status }} 33 | {% else %} 34 | {{ dataset.status }} 35 | {% endif %} 36 | 37 | 38 | {# Workaround for insufficient height causing weird shadow. #} 39 |

40 | 41 |
42 |
43 | {% endfor %} 44 | {% endblock %} 45 | -------------------------------------------------------------------------------- /management/templates/management/home.html: -------------------------------------------------------------------------------- 1 | {% extends "management/base.html" %} 2 | 3 | {% block before_container %} 4 | {% include "_messages.html" %} 5 | {% endblock %} 6 | 7 | {% block content %} 8 |
9 | {% if running_tasks %} 10 |
    11 | {% for task in running_tasks %} 12 |
  • 13 | {% include "indexing_task.html" with task=task %} 14 | {% endfor %} 15 |
16 | {% endif %} 17 |
18 | {% endblock %} 19 | -------------------------------------------------------------------------------- /management/templates/management/task.html: -------------------------------------------------------------------------------- 1 | {% extends "management/datasets.html" %} 2 | 3 | {% block title %} 4 | {{ block.super }} 5 | — 6 | indexing task {{ task.task_id }} 7 | {% endblock %} 8 | 9 | {% block content %} 10 | {% if source %} 11 | {% include "source_header.html" %} 12 | {% endif %} 13 | 14 |
22 | {% include "indexing_task.html" with task=task show_summary=1 %} 23 |
24 | {% endblock %} 25 | 26 | -------------------------------------------------------------------------------- /management/templates/management/xml2rfc.html: -------------------------------------------------------------------------------- 1 | {% extends "management/base.html" %} 2 | 3 | {% block title %} 4 | {{ block.super }} 5 | — 6 | xml2rfc path overview 7 | {% endblock %} 8 | 9 | {% block content %} 10 | {% include "_messages.html" with htmlclass="md:col-span-2" %} 11 | 12 | {% for dir in available_paths %} 13 |
14 |
15 | {{ dir.name }} 17 | {% if dir.aliases %} 18 |   19 | 20 | aliased as 21 | {% for alias in dir.aliases %} 22 | {{ alias }}{% if not forloop.last %}, {% endif %} 23 | {% endfor %} 24 | 25 | {% endif %} 26 |
27 | 28 | {{ dir.total_count }} indexed, 29 | {{ dir.manually_mapped_count }} manually mapped 30 | 31 |
32 | 33 | {% if dir.fetcher_name %} 34 | Auto resolver: {{ dir.fetcher_name }}() 35 | {% else %} 36 | Not automatically resolved. 37 | {% endif %} 38 | 39 |
40 |
41 | {% empty %} 42 |
43 |

44 | No xml2rfc paths available. 45 | Most likely, 46 | the xml2rfc source 47 | has not been indexed yet. 48 |

49 |
50 | {% endfor %} 51 | {% endblock %} 52 | -------------------------------------------------------------------------------- /management/templates/source_header.html: -------------------------------------------------------------------------------- 1 | {# Expects source. #} 2 |
3 |

4 | 5 | {{ source.id }}: 6 | source indexing task status & history 7 |

8 |
9 | {% for repo_url in source.list_repository_urls %} 10 | {{ repo_url }}{% if not forloop.last %}, {% endif %} 11 | {% endfor %} 12 |
13 |
14 | -------------------------------------------------------------------------------- /management/tests.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | 3 | from django.test import TestCase, Client 4 | from django.db.utils import IntegrityError 5 | from django.conf import settings 6 | from django.urls import reverse 7 | 8 | from main.models import RefData 9 | 10 | 11 | class RefDataModelTests(TestCase): 12 | def setUp(self): 13 | self.dataset1_name = "test_dataset_01" 14 | self.dataset2_name = "test_dataset_02" 15 | self.ref_body = {} 16 | 17 | def test_fail_duplicate_ref(self): 18 | self.ref1 = RefData.objects.create( 19 | ref="ref_01", dataset=self.dataset1_name, body=self.ref_body, representations={}, 20 | latest_date=datetime.datetime.now().date() 21 | ) 22 | 23 | self.assertRaises( 24 | IntegrityError, 25 | RefData.objects.create, 26 | ref="ref_01", 27 | dataset=self.dataset1_name, 28 | body=self.ref_body, 29 | representations={}, 30 | latest_date=datetime.datetime.now().date() 31 | ) 32 | 33 | def test_same_ref_diff_datasets(self): 34 | self.ref1 = RefData.objects.create( 35 | ref="ref_01", dataset=self.dataset1_name, body=self.ref_body, representations={}, 36 | latest_date=datetime.datetime.now().date() 37 | ) 38 | self.ref2 = RefData.objects.create( 39 | ref="ref_01", dataset=self.dataset2_name, body=self.ref_body, representations={}, 40 | latest_date=datetime.datetime.now().date() 41 | ) 42 | 43 | 44 | class IndexerTest(TestCase): 45 | def setUp(self): 46 | self.client = Client() 47 | self.real_dataset = list(settings.RELATON_DATASETS)[0] 48 | self.auth = { 49 | 'HTTP_X_IETF_TOKEN': 'test', 50 | } 51 | 52 | def test_run_indexer(self): 53 | """ 54 | TODO: 55 | We need to create test dataset at some remote repo 56 | and make more complicated integration test. 57 | Or make it with self.real_dataset? 58 | But it can be unnecessary a waste of resources. 59 | """ 60 | pass 61 | 62 | # NOTE: To test index process abortion, we need to test new stop_task() API; 63 | # but let’s be mindful about not testing Celery itself and perhaps mock things instead. 64 | # def test_stop_indexer(self): 65 | # url = reverse("api_stop_indexer", args=[self.real_dataset]) 66 | # response = self.client.get(url) 67 | 68 | # # Should get error when we trying to stop not running indexer: 69 | # self.assertEqual(response.status_code, 500) 70 | # self.assertTrue(len(response.json()["error"]) > 0) 71 | 72 | def test_indexer_status(self): 73 | url = reverse("api_indexer_status", args=[self.real_dataset]) 74 | response = self.client.get(url) 75 | self.assertEqual(response.status_code, 200) 76 | self.assertTrue(len(response.json()["tasks"]) == 0) 77 | 78 | def test_reset_indexer(self): 79 | url = reverse("api_reset_index", args=[self.real_dataset]) 80 | response = self.client.post(url, **self.auth) 81 | self.assertEqual(response.status_code, 200) 82 | self.assertTrue(RefData.objects.count() == 0) 83 | -------------------------------------------------------------------------------- /mypy.ini: -------------------------------------------------------------------------------- 1 | [mypy] 2 | plugins = pydantic.mypy 3 | 4 | warn_redundant_casts = True 5 | warn_unused_ignores = True 6 | disallow_any_generics = True 7 | check_untyped_defs = True 8 | no_implicit_reexport = True 9 | 10 | show_error_codes = true 11 | show_error_context = true 12 | strict_equality = true 13 | strict_optional = true 14 | 15 | # Too strict 16 | # disallow_untyped_defs = True 17 | 18 | # Too relaxed? 19 | # follow_imports = silent 20 | 21 | [pydantic-mypy] 22 | init_forbid_extra = True 23 | init_typed = True 24 | warn_required_dynamic_aliases = True 25 | warn_untyped_fields = True 26 | 27 | [mypy-crossref.*] 28 | ignore_missing_imports = True 29 | 30 | [mypy-requests_oauthlib.*] 31 | ignore_missing_imports = True 32 | 33 | [mypy-django.*] 34 | ignore_missing_imports = True 35 | 36 | [mypy-celery.*] 37 | ignore_missing_imports = True 38 | 39 | [mypy-lxml.*] 40 | ignore_missing_imports = True 41 | 42 | [mypy-deepmerge.*] 43 | ignore_missing_imports = True 44 | 45 | [mypy-*.migrations.*] 46 | ignore_errors = True 47 | -------------------------------------------------------------------------------- /ops/db.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM postgres:17.2 2 | 3 | ARG POSTGRES_DB 4 | ARG POSTGRES_USER 5 | ARG POSTGRES_PASSWORD 6 | 7 | ENV POSTGRES_DB=$POSTGRES_DB 8 | ENV POSTGRES_USER=$POSTGRES_USER 9 | ENV POSTGRES_PASSWORD=$POSTGRES_PASSWORD 10 | 11 | COPY load-postgres-extensions.sh /docker-entrypoint-initdb.d/ 12 | RUN chmod 755 /docker-entrypoint-initdb.d/load-postgres-extensions.sh 13 | -------------------------------------------------------------------------------- /ops/grafana-dashboards.yml: -------------------------------------------------------------------------------- 1 | apiVersion: 1 2 | 3 | providers: 4 | - name: dashboards 5 | type: file 6 | updateIntervalSeconds: 30 7 | options: 8 | path: /etc/dashboards 9 | foldersFromFilesStructure: true 10 | -------------------------------------------------------------------------------- /ops/grafana-datasource.yml: -------------------------------------------------------------------------------- 1 | apiVersion: 1 2 | version: ${SOURCE_VERSION} 3 | datasources: 4 | - name: ${SOURCE_NAME} 5 | type: prometheus 6 | url: ${SOURCE} 7 | -------------------------------------------------------------------------------- /ops/load-postgres-extensions.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | 4 | psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL 5 | create extension if not exists "btree_gin"; 6 | select * FROM pg_extension; 7 | EOSQL 8 | -------------------------------------------------------------------------------- /ops/prometheus-config.yml: -------------------------------------------------------------------------------- 1 | global: 2 | scrape_interval: 10s 3 | evaluation_interval: 10s 4 | 5 | scrape_configs: 6 | - job_name: ${JOB_NAME} 7 | basic_auth: 8 | username: ${HTTP_BASIC_USERNAME} 9 | password: ${HTTP_BASIC_PASSWORD} 10 | static_configs: 11 | - targets: ${TARGET_HOSTS} 12 | -------------------------------------------------------------------------------- /ops/prometheus.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:latest 2 | 3 | RUN apk add --update --no-cache gettext 4 | 5 | COPY prometheus-config.yml /tmp/prometheus-template.yml 6 | 7 | ARG TARGET_HOSTS 8 | ARG HTTP_BASIC_USERNAME 9 | ARG HTTP_BASIC_PASSWORD 10 | ARG JOB_NAME 11 | 12 | ENV TARGET_HOSTS=$TARGET_HOSTS 13 | ENV HTTP_BASIC_USERNAME=$HTTP_BASIC_USERNAME 14 | ENV HTTP_BASIC_PASSWORD=$HTTP_BASIC_PASSWORD 15 | ENV JOB_NAME=$JOB_NAME 16 | 17 | RUN envsubst < /tmp/prometheus-template.yml > /prometheus.yml 18 | 19 | RUN echo /prometheus.yml 20 | 21 | FROM prom/prometheus:latest 22 | COPY --from=0 /prometheus.yml /etc/prometheus/prometheus.yml 23 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bibxml-service-frontend", 3 | "version": "1.0.0", 4 | "private": true, 5 | "devDependencies": { 6 | "@babel/cli": "^7.16.7", 7 | "@babel/preset-env": "^7.16.7", 8 | "@tailwindcss/forms": "^0.4.0", 9 | "@tailwindcss/line-clamp": "^0.3.1", 10 | "autoprefixer": "^10.4.2", 11 | "postcss": "^8.4.31", 12 | "postcss-cli": "^9.1.0", 13 | "postcss-preset-env": "^7.2.3", 14 | "tailwindcss": "^3.1.4" 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /postcss.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | plugins: [ 3 | require('tailwindcss/nesting'), 4 | require('tailwindcss'), 5 | require('postcss-preset-env')({ 6 | features: { 'nesting-rules': false }, 7 | browsers: 'last 2 versions', 8 | }), 9 | ] 10 | } 11 | -------------------------------------------------------------------------------- /prometheus/__init__.py: -------------------------------------------------------------------------------- 1 | """Basic Prometheus integration.""" 2 | -------------------------------------------------------------------------------- /prometheus/metrics.py: -------------------------------------------------------------------------------- 1 | """Prometheus recommends to be thoughtful about which metrics are tracked, 2 | so we instantiate all of them in a single module 3 | rather than in corresponding modules. 4 | """ 5 | 6 | # Empty docstrings are workarounds 7 | # to include these self-explanatory metrics in Sphinx autodoc. 8 | 9 | from prometheus_client import Counter 10 | 11 | 12 | _prefix_ = 'bibxml_service_' 13 | 14 | 15 | gui_home_page_hits = Counter( 16 | f'{_prefix_}gui_home_page_hits_total', 17 | "Home page hits", 18 | ) 19 | """""" 20 | 21 | 22 | gui_search_hits = Counter( 23 | f'{_prefix_}gui_search_hits_total', 24 | "Searches via GUI", 25 | ['query_format', 'got_results'], 26 | # got_results should be 'no', 'yes', 'too_many' 27 | ) 28 | """""" 29 | 30 | 31 | gui_bibitem_hits = Counter( 32 | f'{_prefix_}gui_bibitem_hits_total', 33 | "Bibitem accesses via GUI", 34 | ['document_id', 'outcome'], 35 | # outcome should be either success or not_found 36 | ) 37 | """""" 38 | 39 | 40 | api_search_hits = Counter( 41 | f'{_prefix_}api_search_hits_total', 42 | "Searches via API", 43 | ['query_format', 'got_results'], 44 | # got_results should be 'no', 'yes', 'too_many' 45 | ) 46 | """""" 47 | 48 | 49 | api_bibitem_hits = Counter( 50 | f'{_prefix_}api_bibitem_hits_total', 51 | "Bibitem accesses via API", 52 | ['document_id', 'outcome', 'format'], 53 | # outcome should be a limited enum, 54 | # e.g. success/not_found/validation_error/serialization_error 55 | ) 56 | """""" 57 | 58 | 59 | xml2rfc_api_bibitem_hits = Counter( 60 | f'{_prefix_}xml2rfc_api_bibitem_hits_total', 61 | "Bibitem accesses via xml2rfc tools style API", 62 | ['path', 'outcome'], 63 | # outcome should be either success, fallback or not_found 64 | ) 65 | """""" 66 | -------------------------------------------------------------------------------- /prometheus/views.py: -------------------------------------------------------------------------------- 1 | import prometheus_client 2 | from django.http import HttpResponse 3 | 4 | 5 | def metrics(request): 6 | """A simple view that exports Prometheus metrics. 7 | 8 | .. important:: The use of the global registry assumes 9 | no multiprocessing 10 | (i.e., run only one ASGI/WSGI/Celery worker per machine.) 11 | """ 12 | 13 | registry = prometheus_client.REGISTRY 14 | metrics_page = prometheus_client.generate_latest(registry) 15 | return HttpResponse( 16 | metrics_page, 17 | content_type=prometheus_client.CONTENT_TYPE_LATEST, 18 | ) 19 | -------------------------------------------------------------------------------- /pyright.lsp.Dockerfile: -------------------------------------------------------------------------------- 1 | # syntax=docker/dockerfile:1 2 | FROM python:3.10-slim@sha256:502f6626d909ab4ce182d85fcaeb5f73cf93fcb4e4a5456e83380f7b146b12d3 3 | 4 | # FROM python:3.11-rc-slim -- no lxml wheel yet 5 | # FROM lspcontainers/pyright-langserver 6 | 7 | ENV PYTHONUNBUFFERED=1 8 | 9 | ARG SNAPSHOT 10 | ENV SNAPSHOT=$SNAPSHOT 11 | 12 | RUN ["python", "-m", "pip", "install", "--upgrade", "pip"] 13 | 14 | # Could probably be removed for non-slim Python image 15 | RUN apt-get update && apt-get install -yq curl libpq-dev build-essential git 16 | 17 | # To build lxml from source, need at least this (but maybe better stick to wheels): 18 | # RUN apt-get install -yq libxml2-dev zlib1g-dev libxslt-dev 19 | 20 | # Install Node. 21 | # We need to have both Python (for backend Django code) 22 | # and Node (for Babel-based cross-browser frontend build). 23 | # Starting with Python image and adding Node is simpler. 24 | RUN curl -fsSL https://deb.nodesource.com/setup_current.x | bash - 25 | RUN apt-get update 26 | RUN apt-get install -yq nodejs 27 | 28 | # Install requirements for building docs 29 | RUN pip install sphinx 30 | 31 | # Copy and install requirements separately to let Docker cache layers 32 | COPY requirements.txt /code/requirements.txt 33 | COPY package.json /code/package.json 34 | COPY package-lock.json /code/package-lock.json 35 | 36 | WORKDIR /code 37 | 38 | RUN ["pip", "install", "-r", "requirements.txt"] 39 | 40 | RUN npm install -g pyright 41 | 42 | CMD ["pyright-langserver", "--stdio"] 43 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | hypercorn>=0.13.2,<0.18 2 | 3 | Django>=4.0,<5.0 4 | 5 | django-cors-headers>=3.11.0,<4.5 6 | django_debug_toolbar 7 | django_compressor>4.1,<4.6 8 | prometheus_client>=0.13.1,<0.21 9 | deepmerge>=1.0,<2.0 10 | dnspython>=2.2.0,<2.7 11 | psycopg2 12 | celery>=4.0,<6.0 13 | redis>=3.0,<6.0 14 | types-redis 15 | pyyaml>=5.4 16 | types-PyYAML 17 | gitpython>=3.1,<4.0 18 | lxml==4.9.1 19 | requests 20 | requests_oauthlib>=1.3.1,<2.1 21 | requests_cache>=0.7.4,<1.3 22 | flake8 23 | whitenoise>=5.3.0,<7.0 24 | pydantic>=1.10,<2.0 25 | crossrefapi>=1.5,<2 26 | simplejson 27 | sentry-sdk 28 | relaton==0.2.32 29 | -------------------------------------------------------------------------------- /sources/__init__.py: -------------------------------------------------------------------------------- 1 | """This module implements pluggable data sources. 2 | 3 | Currently, it only takes care of Git-based indexable sources 4 | (see :func:`sources.indexable.register_git_source`). 5 | 6 | It provides types that focus on bibliographic data sourcing, 7 | but generally a registered indexable source can index any data. 8 | """ 9 | 10 | import redis 11 | from .celery import app as celery_app 12 | 13 | from django.conf import settings 14 | 15 | 16 | cache = redis.Redis( 17 | host=settings.REDIS_HOST, 18 | port=int(settings.REDIS_PORT), 19 | charset="utf-8", 20 | decode_responses=True, 21 | ) 22 | 23 | __all__ = ( 24 | 'celery_app', 25 | 'cache', 26 | ) 27 | -------------------------------------------------------------------------------- /sources/app.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class Config(AppConfig): 5 | default_auto_field = 'django.db.models.BigAutoField' 6 | name = 'sources' 7 | 8 | def ready(self): 9 | import sources.signals -------------------------------------------------------------------------------- /sources/celery.py: -------------------------------------------------------------------------------- 1 | """When run as a web service, 2 | this module provides an API for adding tasks to the queue. 3 | 4 | When run as Celery worker, this module sets up 5 | Celery to discover Django settings and task queue, 6 | and a signal listener that runs a simple HTTP server in a thread 7 | to export Celery-level Prometheus metrics. 8 | """ 9 | 10 | from __future__ import absolute_import 11 | 12 | import os 13 | from celery import Celery 14 | from celery.signals import worker_process_init 15 | from prometheus_client import start_http_server 16 | 17 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'bibxml.settings') 18 | 19 | app = Celery('bibxml-indexable-sources') 20 | app.config_from_object('django.conf:settings', namespace='CELERY') 21 | app.conf.task_track_started = True 22 | app.autodiscover_tasks() 23 | 24 | 25 | @worker_process_init.connect 26 | def start_prometheus_exporter(*args, **kwargs): 27 | """Starts Prometheus exporter when worker process initializes. 28 | 29 | .. important:: 30 | 31 | **No accommodations are made for multiprocessing mode.** 32 | 33 | Prometheus support for multi-process export is finicky, 34 | not so great, and not enabled. 35 | 36 | This handler should break your setup if you run more than 37 | one worker process 38 | (since the first process should occupy the port below), 39 | and this is left intentional. 40 | 41 | Key takeaway is: 42 | 43 | - Don’t run Celery in default worker mode (prefork) 44 | with more than one worker. 45 | It’ll be a problem for other reasons than Prometheus export, 46 | unless tasks are properly parallelized. 47 | - Once parallelized into smaller I/O bound tasks, 48 | switch to eventlet/gevent pooling, 49 | and use the appropriate signal (possibly ``celeryd_init``) 50 | for Prometheus exporter service startup. 51 | 52 | See :doc:`/howto/run-in-production` for more regarding production setup. 53 | """ 54 | start_http_server(9080) 55 | -------------------------------------------------------------------------------- /sources/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.0.5 on 2022-06-26 16:45 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | initial = True 9 | 10 | dependencies = [ 11 | ] 12 | 13 | operations = [ 14 | migrations.CreateModel( 15 | name='SourceIndexationOutcome', 16 | fields=[ 17 | ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 18 | ('source_id', models.CharField(db_index=True, max_length=250)), 19 | ('task_id', models.CharField(db_index=True, max_length=250, unique=True)), 20 | ('timestamp', models.DateTimeField(auto_now_add=True, db_index=True)), 21 | ('successful', models.BooleanField(default=False)), 22 | ('notes', models.TextField(default='')), 23 | ], 24 | ), 25 | ] 26 | -------------------------------------------------------------------------------- /sources/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ietf-tools/bibxml-service/830eb8a3f03ed3885ed08b5d7092516ccb711139/sources/migrations/__init__.py -------------------------------------------------------------------------------- /sources/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | 4 | class SourceIndexationOutcome(models.Model): 5 | """ 6 | Serves for capturing indexing outcomes for management GUI and API. 7 | """ 8 | 9 | source_id = models.CharField(max_length=250, db_index=True) 10 | """Identifier of the :term:`indexable source`.""" 11 | 12 | task_id = models.CharField(max_length=250, db_index=True, unique=True) 13 | """Celery task ID.""" 14 | 15 | timestamp = models.DateTimeField(auto_now_add=True, db_index=True) 16 | """When this run concluded.""" 17 | 18 | successful = models.BooleanField(default=False) 19 | """Whether this run was successful.""" 20 | 21 | notes = models.TextField(default='') 22 | """Any notes, e.g. warnings or stats.""" 23 | -------------------------------------------------------------------------------- /sources/signals.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime, timedelta 2 | 3 | from django.db.models.signals import post_save 4 | from django.dispatch import receiver 5 | 6 | from sources.models import SourceIndexationOutcome 7 | 8 | 9 | @receiver(post_save, sender=SourceIndexationOutcome) 10 | def delete_old_source_indexation_outcome_entries(sender, **kwargs): 11 | days = 9 12 | print(f"Deleting SourceIndexationOutcome entries older than {days} days.") 13 | SourceIndexationOutcome.objects.filter(timestamp__lte=datetime.now()-timedelta(days=days)).delete() 14 | -------------------------------------------------------------------------------- /static/img/about/bibitem-details-2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ietf-tools/bibxml-service/830eb8a3f03ed3885ed08b5d7092516ccb711139/static/img/about/bibitem-details-2.png -------------------------------------------------------------------------------- /static/img/about/bibitem-details.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ietf-tools/bibxml-service/830eb8a3f03ed3885ed08b5d7092516ccb711139/static/img/about/bibitem-details.png -------------------------------------------------------------------------------- /static/js/api-button.js: -------------------------------------------------------------------------------- 1 | (function () { 2 | 3 | const apiSecret = document.documentElement.dataset['api-secret']; 4 | 5 | function callIndexerAPI (url, method, urlParams) { 6 | if (urlParams) { 7 | url = `${url}?${urlParams}` 8 | } 9 | fetch(url, { 10 | method: method, 11 | headers: { 12 | 'X-IETF-Token': apiSecret, 13 | }, 14 | }).then(function () { document.location.reload() }); 15 | } 16 | 17 | function makeClickableButton (el) { 18 | const label = el.dataset['api-action-label']; 19 | const url = el.dataset['api-endpoint']; 20 | const meth = el.dataset['api-method']; 21 | 22 | let getQuery; 23 | const getQueryJSON = el.dataset['api-get-query']; 24 | if (getQueryJSON) { 25 | try { 26 | getQuery = new URLSearchParams(JSON.parse(getQueryJSON)); 27 | } catch (e) { 28 | console.error("Failed to parse GET parameters", e, getQueryJSON); 29 | throw e; 30 | } 31 | } else { 32 | getQuery = undefined; 33 | } 34 | 35 | if (url && meth) { 36 | el.classList.add('button'); 37 | el.style.cursor = 'pointer'; 38 | el.innerHTML = label; 39 | el.setAttribute('role', 'button'); 40 | el.addEventListener('click', function () { 41 | callIndexerAPI(url, meth, getQuery); 42 | }); 43 | } else { 44 | console.warn("Invalid data API endpoint", el); 45 | } 46 | } 47 | 48 | if (apiSecret) { 49 | document.querySelectorAll('[data-api-endpoint]').forEach(makeClickableButton); 50 | } 51 | 52 | })(); 53 | -------------------------------------------------------------------------------- /static/js/format-xml.js: -------------------------------------------------------------------------------- 1 | (function () { 2 | 3 | const xsltDoc = new DOMParser().parseFromString([ 4 | '', 5 | ' ', 6 | ' ', 7 | ' ', 8 | ' ', 9 | ' ', 10 | ' ', 11 | ' ', 12 | ' ', 13 | '', 14 | ].join('\n'), 'application/xml'); 15 | 16 | const xsltProcessor = new XSLTProcessor(); 17 | xsltProcessor.importStylesheet(xsltDoc); 18 | 19 | function formatXML (rawString) { 20 | const xmlDoc = new DOMParser().parseFromString(rawString, 'application/xml'); 21 | const resultDoc = xsltProcessor.transformToDocument(xmlDoc); 22 | return new XMLSerializer().serializeToString(resultDoc); 23 | }; 24 | 25 | window.formatXML = formatXML; 26 | })(); 27 | -------------------------------------------------------------------------------- /static/js/matomo.js: -------------------------------------------------------------------------------- 1 | // Below variables are set on root in base template, if relevant settings exist. 2 | 3 | var MATOMO_URL = document.documentElement.getAttribute('data-matomo-url'); 4 | // Required for integration to work 5 | 6 | // One of these is required: 7 | var MTM_CONTAINER = document.documentElement.getAttribute('data-matomo-tag-manager-container'); 8 | var MATOMO_SITE_ID = document.documentElement.getAttribute('data-matomo-site-id'); 9 | 10 | if (MATOMO_URL && MTM_CONTAINER) { 11 | // Tag manager integration 12 | var _mtm = window._mtm = window._mtm || []; 13 | _mtm.push({'mtm.startTime': (new Date().getTime()), 'event': 'mtm.Start'}); 14 | 15 | var d = document, g = d.createElement('script'), s = d.getElementsByTagName('script')[0]; 16 | g.type = 'text/javascript'; 17 | g.async = true; 18 | g.src = `https://${MATOMO_URL}/js/container_${MTM_CONTAINER}.js`; 19 | s.parentNode.insertBefore(g, s); 20 | 21 | } else if (MATOMO_URL && MATOMO_SITE_ID) { 22 | // Basic tracker integration 23 | var _paq = window._paq = window._paq || []; 24 | _paq.push(['trackPageView']); 25 | _paq.push(['enableLinkTracking']); 26 | 27 | (function() { 28 | var u = `//${MATOMO_URL}/`; 29 | _paq.push(['setTrackerUrl', u + 'matomo.php']); 30 | _paq.push(['setSiteId', MATOMO_SITE_ID]); 31 | 32 | var d = document, g = d.createElement('script'), s = d.getElementsByTagName('script')[0]; 33 | g.type = 'text/javascript'; 34 | g.async = true; 35 | g.src = u + 'matomo.js'; 36 | s.parentNode.insertBefore(g, s); 37 | })(); 38 | } 39 | -------------------------------------------------------------------------------- /static/js/simple-cache.js: -------------------------------------------------------------------------------- 1 | (function () { 2 | 3 | /** 4 | * Instantiates a simple localStorage-backed storage. 5 | * 6 | * Since localStorage is slow and blocking, 7 | * under the hood loads data from storage at instantiation 8 | * and stores it at specified `debounceStoreMs` 9 | * only if items were set (state is dirty). 10 | */ 11 | function getCache(cacheKey, initial, options = { ttlMs: undefined, debounceStoreMs: undefined }) { 12 | const DEFAULT_CACHE_TTL_MS = 3600000; // 1 hour 13 | const DEFAULT_DEBOUNCE_MS = 2000; // 2 seconds 14 | 15 | const ttl = options?.ttlMs ?? DEFAULT_CACHE_TTL_MS; 16 | const debounceMs = options?.debounceStoreMs ?? DEFAULT_DEBOUNCE_MS; 17 | 18 | const data = loadCache() ?? initial; 19 | 20 | let dirty = false; 21 | const cacheStoreInterval = window.setInterval(function cacheTick() { 22 | if (dirty) { 23 | storeCache(data); 24 | dirty = false; 25 | } 26 | }, debounceMs); 27 | 28 | function storeCache(data) { 29 | localStorage.setItem(cacheKey, JSON.stringify({ 30 | ts: Date.now(), 31 | data, 32 | })); 33 | } 34 | 35 | function loadCache() { 36 | const cached = localStorage.getItem(cacheKey); 37 | if (cached !== null) { 38 | let deserialized; 39 | try { 40 | deserialized = JSON.parse(cached); 41 | } catch (e) { 42 | console.error("Failed to deserialize cached data", cached, e); 43 | deserialized = null; 44 | } 45 | 46 | if (deserialized?.ts && deserialized?.data) { 47 | const freshEnough = deserialized.ts > (Date.now() - ttl); 48 | 49 | if (freshEnough) { 50 | return deserialized.data; 51 | 52 | } else { 53 | console.debug("Cached data has expired", deserialized.ts); 54 | localStorage.removeItem(cacheKey); 55 | } 56 | } else { 57 | console.warn("Unexpected cached data format", deserialized || cached); 58 | localStorage.removeItem(cacheKey); 59 | } 60 | } 61 | return null; 62 | } 63 | 64 | return { 65 | 66 | set: function setItem (itemKey, value) { 67 | data[itemKey] = value; 68 | dirty = true; 69 | }, 70 | 71 | /** Returns stored item or null. */ 72 | get: function getItem (itemKey) { 73 | return data[itemKey] ?? null; 74 | }, 75 | 76 | destroy: function destroyCache() { 77 | window.clearInterval(cacheStoreInterval); 78 | }, 79 | 80 | }; 81 | 82 | } 83 | 84 | window.getCache = getCache; 85 | 86 | })(); 87 | -------------------------------------------------------------------------------- /static/keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ietf-tools/bibxml-service/830eb8a3f03ed3885ed08b5d7092516ccb711139/static/keep -------------------------------------------------------------------------------- /tailwind.config.js: -------------------------------------------------------------------------------- 1 | const colors = require('tailwindcss/colors') 2 | 3 | module.exports = { 4 | content: [ 5 | "./templates/**/*.html", 6 | "./main/templates/**/*.html", 7 | "./management/templates/**/*.html", 8 | "./static/js/*.js", 9 | "./**/views.py", 10 | ], 11 | // Currently it is required to re-run Django compressor after template changes, 12 | // and since it won’t pick up styling change 13 | // you must touch the main.css file for compressor to update mtime. 14 | // TODO: Figure out how to speed up Tailwind style iteration 15 | // safelist: [ 16 | // { pattern: /.*/, variants: ['hover', 'focus'] } 17 | // ], 18 | theme: { 19 | extend: { 20 | colors: { 21 | dark: colors.slate, 22 | // Necessary for gradient workaround 23 | 'dark700transparentfix': 'rgb(30 41 59 / 0)', 24 | }, 25 | screens: { 26 | 'xl': '1368px', 27 | }, 28 | }, 29 | }, 30 | plugins: [ 31 | require('@tailwindcss/forms'), 32 | require('@tailwindcss/line-clamp'), 33 | ], 34 | } 35 | -------------------------------------------------------------------------------- /templates/_about_links.html: -------------------------------------------------------------------------------- 1 | 2 | {% if repo_url %} 3 | v{{ snapshot }} 4 | {% else %} 5 | v{{ snapshot }} 6 | {% endif %} 7 |  •  8 | About 9 |  •  10 | API 11 | 12 | -------------------------------------------------------------------------------- /templates/_datatracker_user_block.html: -------------------------------------------------------------------------------- 1 | {% if datatracker_oauth_enabled %} 2 | {% if datatracker_user %} 3 | 13 | {% else %} 14 | 24 | Log in via Datatracker 25 | 26 | {% endif %} 27 | {% endif %} 28 | -------------------------------------------------------------------------------- /templates/_datatracker_user_link.html: -------------------------------------------------------------------------------- 1 | {% if datatracker_oauth_enabled %} 2 | {% if datatracker_user %} 3 | 4 | Log out 5 | ({{ datatracker_user.name|default:"name N/A" }}) 6 | 7 | {% else %} 8 | 11 | Log in via Datatracker 12 | 13 | {% endif %} 14 | {% endif %} 15 | -------------------------------------------------------------------------------- /templates/_error_icon.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /templates/_list_item_classes.html: -------------------------------------------------------------------------------- 1 | {# Classes for item in listings within
element, together with snap and dark styles. #} 2 | {# Expects forloop #} 3 | 4 | {% if forloop and not forloop.last %} 5 | snap-start 6 | {% endif %} 7 | overflow-hidden 8 | 9 | outline-2 10 | outline-dark-300 11 | outline-offset-[-4px] 12 | outline-sky-300 13 | dark:outline-sky-500 14 | 15 | hover:overflow-visible 16 | focus-within:overflow-visible 17 | hover:z-20 18 | focus-within:z-10 19 | 20 | relative 21 | 22 | {# Below is a little buggy for items that end up shorter than grid division #} 23 | {# (often happens on small viewport heights when there are not enough search results, and may be indicative of grid rules issue) #} 24 | 25 | {# This implements slight shadow serving as a border between adjacent items. #} 26 | {# May be redundant if there is enough vertical whitespace. #} 27 | shadow 28 | shadow-dark-500 29 | dark:shadow-dark-900 30 | hover:shadow-none 31 | 32 | {% comment %} 33 | {# This implements fadeout-to-white at the bottom of the item. #} 34 | after:absolute 35 | after:bottom-0 36 | after:w-full 37 | after:h-8 38 | after:bg-gradient-to-t 39 | after:from-[white] 40 | after:[--tw-gradient-stops:var(--tw-gradient-from),rgb(255_255_255/0)] 41 | dark:after:from-dark-800 42 | dark:after:[--tw-gradient-stops:var(--tw-gradient-from),theme('colors.dark700transparentfix')] 43 | hover:after:!hidden 44 | focus-within:after:!hidden 45 | {% endcomment %} 46 | -------------------------------------------------------------------------------- /templates/_list_item_inner_classes.html: -------------------------------------------------------------------------------- 1 | {# Classes for a block element which is a direct child of item in list #} 2 | {# (See _list_item_classes.html) #} 3 | 4 | {% if forloop.last %} 5 | snap-end 6 | {% endif %} 7 | py-4 8 | place-self-stretch 9 | 10 | bg-dark-200 11 | dark:bg-dark-800 dark:text-inherit 12 | 13 | md:min-h-[max(5rem,12.5vh)] 14 | xl:min-h-[max(5rem,6.25vh)] 15 | ring-0 16 | ring-sky-300 17 | dark:ring-sky-700 18 | ring-inset 19 | 20 | hover:ring-4 21 | focus-within:ring-4 22 | -------------------------------------------------------------------------------- /templates/_message.html: -------------------------------------------------------------------------------- 1 |
21 | 22 | info icon 23 | 24 | 25 | {{ message|safe }} 26 |
27 | -------------------------------------------------------------------------------- /templates/_messages.html: -------------------------------------------------------------------------------- 1 | {% if messages %} 2 | {% for message in messages %} 3 | {% include "_message.html" with tags=message.tags message=message htmlclass=htmlclass %} 4 | {% endfor %} 5 | {% endif %} 6 | -------------------------------------------------------------------------------- /templates/_profiling_block.html: -------------------------------------------------------------------------------- 1 | {% if profiling.query_times %} 2 | 12 | {% endif %} 13 | -------------------------------------------------------------------------------- /templates/_search_icon.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /templates/_side_block_classes.html: -------------------------------------------------------------------------------- 1 | {% if not small %} 2 | md:col-span-2 3 | lg:col-span-3 4 | {% endif %} 5 | {% if not stack %} 6 | {# Only makes sense if small. #} 7 | md:!col-start-1 8 | {% else %} 9 | {# Only can stack up to two. #} 10 | md:!col-start-2 11 | {% endif %} 12 | 13 | tracking-tight 14 | leading-tight 15 | overflow-hidden 16 | -------------------------------------------------------------------------------- /templates/error.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | 3 | {% block title %} 4 | {{ block.super }} 5 | — 6 | {{ error_title|default:"Server error" }} 7 | {% endblock %} 8 | 9 | 10 | {% block content %} 11 |

{{ error_title|default:"Server error" }}

12 | 13 |

14 | {{ error_description|default:"Something went wrong on our side." }} 15 | 16 |

17 |

Go to

18 | 32 |
33 | {% endblock %} 34 | -------------------------------------------------------------------------------- /templates/human_readable_openapi_spec.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | {{ title|default:"API documentation" }} 5 | 6 | 7 | 10 | 11 | 12 | 13 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /templates/openapi-legacy.yaml: -------------------------------------------------------------------------------- 1 | openapi: 3.1.0 2 | 3 | info: 4 | title: IETF BibXML service — compatibility API 5 | description: | 6 | These are APIs for compatibility with existing third-party systems during the migration period. 7 | contact: 8 | email: ietf-ribose@ribose.com 9 | license: 10 | name: BSD 3-Clause 11 | url: https://github.com/ietf-ribose/bibxml-service/blob/main/LICENSE 12 | version: {{ snapshot }} 13 | 14 | servers: 15 | - url: / 16 | 17 | paths: 18 | 19 | /public/rfc/{directory_name}/{filename}.xml: 20 | parameters: 21 | - name: directory_name 22 | in: path 23 | description: Directory name, as per xml2rfc web server’s content. 24 | required: true 25 | schema: 26 | type: string 27 | example: bibxml4 28 | - name: filename 29 | in: path 30 | description: Filename, as per xml2rfc web server’s content. Typically starts with “reference” or “_reference”. 31 | required: true 32 | schema: 33 | example: reference.W3C.NOTE-XML-FRAG-REQ-19981123 34 | type: string 35 | - name: anchor 36 | in: query 37 | description: | 38 | Replace the `anchor` in *top-level* `` or `` 39 | to specified string. 40 | 41 | NOTE: Does not affect any of the nested `` elements. 42 | required: true 43 | schema: 44 | type: string 45 | 46 | get: 47 | summary: Get bibliographic item by xml2rfc tools path 48 | description: | 49 | This endpoint returns raw XML response. 50 | Returned bibliographic item is either obtained from a Relaton source, 51 | or (if impossible to locate given anchor, and xml2rfc data is indexed) 52 | from xml2rfc snapshot. 53 | responses: 54 | 55 | 200: 56 | description: successful operation 57 | content: 58 | application/xml: 59 | schema: 60 | type: string 61 | 62 | 404: 63 | description: reference not found. (At this time, there may be false negatives) 64 | content: 65 | application/json: 66 | schema: 67 | $ref: '#/components/schemas/ErrorResponse' 68 | 69 | 500: 70 | description: error retrieving or serializing item 71 | content: 72 | application/json: 73 | schema: 74 | $ref: '#/components/schemas/ErrorResponse' 75 | 76 | components: 77 | 78 | schemas: 79 | 80 | ErrorResponse: 81 | type: object 82 | properties: 83 | error: 84 | type: object 85 | properties: 86 | code: 87 | type: integer 88 | description: Code for automatic error processing 89 | message: 90 | type: string 91 | description: Human readable error message 92 | -------------------------------------------------------------------------------- /templates/robots.txt: -------------------------------------------------------------------------------- 1 | User-agent: * 2 | Disallow: / 3 | -------------------------------------------------------------------------------- /test.Dockerfile: -------------------------------------------------------------------------------- 1 | # syntax=docker/dockerfile:1 2 | FROM python:3.10-slim@sha256:502f6626d909ab4ce182d85fcaeb5f73cf93fcb4e4a5456e83380f7b146b12d3 3 | # FROM python:3.11-rc-slim -- no lxml wheel yet 4 | 5 | RUN ["python", "-m", "pip", "install", "--upgrade", "pip"] 6 | 7 | # Could probably be removed for non-slim Python image 8 | RUN apt-get update && apt-get install -yq curl libpq-dev build-essential git 9 | 10 | # Copy and install requirements separately to let Docker cache layers 11 | COPY requirements.txt /code/requirements.txt 12 | 13 | # Install Coverage (required for tests only) 14 | RUN ["pip", "install", "coverage"] 15 | 16 | WORKDIR /code 17 | 18 | RUN ["pip", "install", "-r", "requirements.txt"] 19 | 20 | # Provide default test environment 21 | ENV SNAPSHOT=test 22 | ENV SERVICE_NAME="BibXML test" 23 | ENV DB_NAME=test 24 | ENV DB_USER=test 25 | ENV DB_SECRET=test 26 | ENV DB_HOST=test-db 27 | ENV DJANGO_SECRET="IRE()dA(*EFW&*RHIUWEJFOUHSVSJO(ER*#U#(JRIAELFKJfLAJFIFJ(JOSIERJO$IFUHOI()#*JRIU" 28 | ENV PRIMARY_HOSTNAME=test.local 29 | ENV REDIS_HOST=test-redis 30 | ENV REDIS_PORT=6379 31 | ENV CONTACT_EMAIL="example@ribose.com" 32 | ENV API_SECRET=test 33 | 34 | # Copy the rest of the codebase 35 | COPY . /code 36 | 37 | ENV WAIT_VERSION 2.7.2 38 | ADD https://github.com/ufoscout/docker-compose-wait/releases/download/$WAIT_VERSION/wait /wait 39 | RUN chmod +x /wait 40 | 41 | RUN curl -Os https://uploader.codecov.io/latest/linux/codecov 42 | RUN chmod +x codecov 43 | 44 | CMD python -m coverage run manage.py test && ./codecov -t ${CODECOV_TOKEN} 45 | -------------------------------------------------------------------------------- /wait-for-migrations.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # wait-for-migrations.sh 3 | 4 | set -e 5 | 6 | python manage.py check --deploy || exit 1 7 | 8 | until python manage.py migrate --check; do 9 | >&2 echo "Unapplied migrations in default DB might still exist: sleeping… $?" 10 | sleep 2 11 | done 12 | 13 | >&2 echo "Migrations in default DB applied: proceeding…" 14 | 15 | exec "$@" 16 | -------------------------------------------------------------------------------- /xml2rfc_compat/__init__.py: -------------------------------------------------------------------------------- 1 | """Module providing logic for retrieving bibitems 2 | via xml2rfc-style paths, and serializing them to 3 | xml2rfc-style RFC 7991 XML. 4 | """ 5 | -------------------------------------------------------------------------------- /xml2rfc_compat/aliases.py: -------------------------------------------------------------------------------- 1 | """Helpers for working with xml2rfc directory aliases 2 | (symbolic links). 3 | 4 | “Actual” directory names are considered those defined in archive source 5 | (see :mod:`xml2rfc_compat.source`). All others are considered aliases. 6 | 7 | Aliases are defined via :data:`bibxml.settings.XML2RFC_COMPAT_DIR_ALIASES` 8 | setting. 9 | """ 10 | from typing import List 11 | 12 | from django.conf import settings 13 | 14 | 15 | __all__ = ('ALIASES', 'get_aliases', 'unalias') 16 | 17 | 18 | ALIASES = getattr(settings, 'XML2RFC_COMPAT_DIR_ALIASES', {}) 19 | 20 | 21 | def get_aliases(dirname: str) -> List[str]: 22 | """Get aliases for given directory.""" 23 | 24 | return ALIASES.get(dirname, []) 25 | 26 | 27 | def unalias(alias: str) -> str: 28 | """Resolve provided alias to actual directory.""" 29 | 30 | for dirname, aliases in ALIASES.items(): 31 | if alias == dirname or alias in aliases: 32 | return dirname 33 | raise ValueError("Not a directory name or alias") 34 | -------------------------------------------------------------------------------- /xml2rfc_compat/app.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | from django.apps import AppConfig 3 | 4 | 5 | class Config(AppConfig): 6 | default_auto_field = 'django.db.models.BigAutoField' 7 | name = 'xml2rfc_compat' 8 | 9 | def ready(self): 10 | # Import modules to make things register as a side effect 11 | importlib.import_module('xml2rfc_compat.source') 12 | importlib.import_module('xml2rfc_compat.serializer') 13 | -------------------------------------------------------------------------------- /xml2rfc_compat/fixtures/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ietf-tools/bibxml-service/830eb8a3f03ed3885ed08b5d7092516ccb711139/xml2rfc_compat/fixtures/__init__.py -------------------------------------------------------------------------------- /xml2rfc_compat/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.0.2 on 2022-02-03 18:59 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | initial = True 9 | 10 | dependencies = [ 11 | ] 12 | 13 | operations = [ 14 | migrations.CreateModel( 15 | name='Xml2rfcItem', 16 | fields=[ 17 | ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 18 | ('subpath', models.CharField(db_index=True, max_length=255)), 19 | ('xml_repr', models.TextField()), 20 | ], 21 | ), 22 | ] 23 | -------------------------------------------------------------------------------- /xml2rfc_compat/migrations/0002_manualpathmap.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.0.2 on 2022-02-05 14:18 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('xml2rfc_compat', '0001_initial'), 10 | ] 11 | 12 | operations = [ 13 | migrations.CreateModel( 14 | name='ManualPathMap', 15 | fields=[ 16 | ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 17 | ('xml2rfc_subpath', models.CharField(db_index=True, max_length=255)), 18 | ('query', models.TextField()), 19 | ('query_format', models.CharField(choices=[('json_struct', 'JSON containment'), ('docid_regex', 'docid substring'), ('json_path', 'JSON path'), ('websearch', 'web search')], max_length=50)), 20 | ], 21 | ), 22 | ] 23 | -------------------------------------------------------------------------------- /xml2rfc_compat/migrations/0003_manualpathmap_docid.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.0.2 on 2022-02-08 14:36 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('xml2rfc_compat', '0002_manualpathmap'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AddField( 14 | model_name='manualpathmap', 15 | name='docid', 16 | field=models.CharField(db_index=True, default='', max_length=255), 17 | preserve_default=False, 18 | ), 19 | ] 20 | -------------------------------------------------------------------------------- /xml2rfc_compat/migrations/0004_remove_manualpathmap_query_and_more.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.0.2 on 2022-02-08 15:51 2 | 3 | from django.db import migrations 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('xml2rfc_compat', '0003_manualpathmap_docid'), 10 | ] 11 | 12 | operations = [ 13 | migrations.RemoveField( 14 | model_name='manualpathmap', 15 | name='query', 16 | ), 17 | migrations.RemoveField( 18 | model_name='manualpathmap', 19 | name='query_format', 20 | ), 21 | ] 22 | -------------------------------------------------------------------------------- /xml2rfc_compat/migrations/0005_alter_manualpathmap_xml2rfc_subpath.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.0.2 on 2022-02-09 13:20 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('xml2rfc_compat', '0004_remove_manualpathmap_query_and_more'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AlterField( 14 | model_name='manualpathmap', 15 | name='xml2rfc_subpath', 16 | field=models.CharField(db_index=True, max_length=255, unique=True), 17 | ), 18 | ] 19 | -------------------------------------------------------------------------------- /xml2rfc_compat/migrations/0006_delete_manualpathmap_xml2rfcitem_sidecar_meta.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.0.5 on 2022-06-18 07:29 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('xml2rfc_compat', '0005_alter_manualpathmap_xml2rfc_subpath'), 10 | ] 11 | 12 | operations = [ 13 | migrations.DeleteModel( 14 | name='ManualPathMap', 15 | ), 16 | migrations.AddField( 17 | model_name='xml2rfcitem', 18 | name='sidecar_meta', 19 | field=models.JSONField(default=dict), 20 | preserve_default=False, 21 | ), 22 | ] 23 | -------------------------------------------------------------------------------- /xml2rfc_compat/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ietf-tools/bibxml-service/830eb8a3f03ed3885ed08b5d7092516ccb711139/xml2rfc_compat/migrations/__init__.py -------------------------------------------------------------------------------- /xml2rfc_compat/serializer.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module registers :func:`~.to_xml_string` 3 | in this project’s serializer registry (:mod:`bib_models.serializers`). 4 | """ 5 | from lxml import etree 6 | 7 | from bib_models import serializers, BibliographicItem 8 | 9 | 10 | __all__ = ( 11 | 'to_xml_string', 12 | ) 13 | 14 | from xml2rfc_compat.serializers import serialize 15 | 16 | 17 | @serializers.register('bibxml', 'application/xml') 18 | def to_xml_string(item: BibliographicItem, **kwargs) -> bytes: 19 | """ 20 | A wrapper around :func:`xml2rfc_compat.serializer.serialize`. 21 | """ 22 | # get a tree 23 | canonicalized_tree = etree.fromstring( 24 | # obtained from a canonicalized string representation 25 | etree.tostring( 26 | # of the original bibxml tree 27 | serialize(item, **kwargs), 28 | method='c14n2', 29 | ) 30 | # ^ this returns a unicode string 31 | ) 32 | 33 | # pretty-print that tree in utf-8 without declaration and doctype 34 | return etree.tostring( 35 | canonicalized_tree, 36 | encoding='utf-8', 37 | xml_declaration=False, 38 | doctype=None, 39 | pretty_print=True, 40 | ) 41 | -------------------------------------------------------------------------------- /xml2rfc_compat/serializers/__init__.py: -------------------------------------------------------------------------------- 1 | """Serialization of :class:`relaton.models.bibdata.BibliographicItem` 2 | into BibXML (xml2rfc) format roughly per RFC 7991, 3 | with bias towards existing xml2rfc documents where differs. 4 | 5 | Primary API is :func:`.serialize()`. 6 | 7 | .. seealso:: :mod:`~relaton.serializers.bibxml_string` 8 | """ 9 | 10 | from typing import List, Optional 11 | 12 | from lxml import objectify, etree 13 | from lxml.etree import _Element 14 | from relaton.models import Relation 15 | 16 | from common.util import as_list 17 | from .anchor import get_suitable_anchor 18 | from .reference import create_reference, create_referencegroup 19 | from .target import get_suitable_target 20 | from bib_models import BibliographicItem 21 | 22 | __all__ = ( 23 | 'serialize', 24 | ) 25 | 26 | 27 | def serialize( 28 | item: BibliographicItem, 29 | anchor: Optional[str] = None 30 | ) -> _Element: 31 | """Converts a BibliographicItem to XML, 32 | trying to follow RFC 7991. 33 | 34 | Returned root element is either a ```` 35 | or a ````. 36 | 37 | :param str anchor: resulting root element ``anchor`` property. 38 | 39 | :raises ValueError: if there are different issues 40 | with given item’s structure 41 | that make it unrenderable per RFC 7991. 42 | """ 43 | 44 | relations: List[Relation] = as_list(item.relation or []) 45 | 46 | constituents = [rel for rel in relations if rel.type == 'includes'] 47 | 48 | is_referencegroup = len(constituents) > 0 49 | 50 | if is_referencegroup: 51 | root = create_referencegroup([ 52 | ref.bibitem 53 | for ref in constituents 54 | ]) 55 | else: 56 | root = create_reference(item) 57 | 58 | # Fill in default root element anchor, unless specified 59 | if anchor is None: 60 | try: 61 | anchor = get_suitable_anchor(item) 62 | except ValueError: 63 | pass 64 | if anchor: 65 | root.set('anchor', anchor) 66 | 67 | if is_referencegroup: 68 | # Fill in appropriate target 69 | try: 70 | target = get_suitable_target(as_list(item.link or [])) 71 | except ValueError: 72 | pass 73 | else: 74 | root.set('target', target) 75 | 76 | objectify.deannotate(root) 77 | etree.cleanup_namespaces(root) 78 | 79 | return root 80 | -------------------------------------------------------------------------------- /xml2rfc_compat/serializers/abstracts.py: -------------------------------------------------------------------------------- 1 | from typing import List, cast 2 | 3 | from lxml import etree, objectify 4 | from lxml.etree import _Element 5 | from relaton.models import GenericStringValue 6 | 7 | __all__ = ( 8 | 'create_abstract', 9 | ) 10 | 11 | E = objectify.E 12 | 13 | 14 | JATS_XMLNS = "http://www.ncbi.nlm.nih.gov/JATS1" 15 | 16 | 17 | def create_abstract(abstracts: List[GenericStringValue]) -> _Element: 18 | """ 19 | Formats an ```` element. 20 | """ 21 | if len(abstracts) < 1: 22 | raise ValueError("No abstracts are available") 23 | 24 | # Try to pick an English abstract, or the first one available 25 | abstract = ( 26 | [a for a in abstracts if a.language in ('en', 'eng')] or 27 | abstracts 28 | )[0] 29 | 30 | return E.abstract(*( 31 | E.t(p) 32 | for p in get_paragraphs(abstract) 33 | )) 34 | 35 | 36 | def get_paragraphs(val: GenericStringValue) -> List[str]: 37 | """Converts HTML or JATS to a list of strings representing 38 | paragraphs. 39 | """ 40 | try: 41 | match val.format: 42 | case 'text/html': 43 | return get_paragraphs_html(val.content) 44 | case 'application/x-jats+xml': 45 | return get_paragraphs_jats(val.content) 46 | case _: 47 | raise ValueError("Unknown format for paragraph extraction") 48 | 49 | except (etree.XMLSyntaxError, ValueError): 50 | return get_paragraphs_plain(val.content) 51 | 52 | 53 | def get_paragraphs_html(val: str) -> List[str]: 54 | tree = etree.fromstring(f'
{val}
') 55 | ps = cast(List[str], [ 56 | p.text for p in tree.findall('p') 57 | if (getattr(p, 'text', '') or '').strip() != '' 58 | ]) 59 | if len(ps) > 0: 60 | return ps 61 | else: 62 | raise ValueError("No HTML paragraphs detected") 63 | 64 | 65 | def get_paragraphs_jats(val: str) -> List[str]: 66 | tree = etree.fromstring(f'
{val}
') 67 | ps = cast(List[str], [ 68 | p.text for p in tree.findall('jats:p', {'jats': JATS_XMLNS}) 69 | if (getattr(p, 'text', '') or '').strip() != '' 70 | ]) 71 | if len(ps) > 0: 72 | return ps 73 | else: 74 | raise ValueError("No JATS paragraphs detected") 75 | 76 | 77 | def get_paragraphs_plain(val: str) -> List[str]: 78 | return [ 79 | p.strip() 80 | for p in val.split('\n\n') 81 | if p.strip() != '' 82 | ] 83 | -------------------------------------------------------------------------------- /xml2rfc_compat/serializers/series.py: -------------------------------------------------------------------------------- 1 | from typing import Union, Tuple, Callable, List 2 | 3 | from relaton.models import DocID 4 | 5 | 6 | __all__ = ( 7 | 'DOCID_SERIES_EXTRACTORS', 8 | ) 9 | 10 | 11 | def extract_doi_series(docid: DocID) -> Union[Tuple[str, str], None]: 12 | if docid.type.lower() == 'doi': 13 | return 'DOI', docid.id 14 | return None 15 | 16 | 17 | def extract_rfc_series(docid: DocID) -> Union[Tuple[str, str], None]: 18 | if docid.type.lower() == 'ietf' and docid.id.lower().startswith('rfc '): 19 | return 'RFC', docid.id.replace('.', ' ').split(' ')[-1] 20 | return None 21 | 22 | 23 | def extract_id_series(docid: DocID) -> Union[Tuple[str, str], None]: 24 | if docid.type.lower() == 'internet-draft': 25 | return 'Internet-Draft', docid.id 26 | return None 27 | 28 | 29 | def extract_w3c_series(docid: DocID) -> Union[Tuple[str, str], None]: 30 | if docid.type.lower() == 'w3c': 31 | return 'W3C', docid.id.replace('.', ' ').split('W3C ')[-1] 32 | return None 33 | 34 | 35 | def extract_3gpp_tr_series(docid: DocID) -> Union[Tuple[str, str], None]: 36 | if docid.type.lower() == '3gpp': 37 | ver = docid.id.split('/')[-1] 38 | # TODO: This is insufficient 39 | try: 40 | id = docid.id.split('3GPP TR ')[1].split(':')[0] 41 | except IndexError: 42 | return None 43 | return '3GPP TR', f'{id} {ver}' 44 | return None 45 | 46 | 47 | def extract_ieee_series(docid: DocID) -> Union[Tuple[str, str], None]: 48 | if docid.type.lower() == 'ieee': 49 | try: 50 | id, year, *_ = docid.id.split(' ')[-1].lower().strip().split('.') 51 | except ValueError: 52 | return 'IEEE', docid.id 53 | else: 54 | return 'IEEE', '%s-%s' % (id.replace('-', '.'), year) 55 | return None 56 | 57 | 58 | DOCID_SERIES_EXTRACTORS: List[ 59 | Callable[[DocID], Union[Tuple[str, str], None]] 60 | ] = [ 61 | extract_rfc_series, 62 | extract_id_series, 63 | extract_w3c_series, 64 | extract_3gpp_tr_series, 65 | extract_ieee_series, 66 | extract_doi_series, 67 | ] 68 | """A list of functions capable of extracting series information 69 | as 2-tuple (series name, document number) 70 | from a :class:`~relaton.models.bibdata.DocID`. 71 | Each function is expected to either return a tuple or ``None``, 72 | and not throw.""" 73 | -------------------------------------------------------------------------------- /xml2rfc_compat/serializers/target.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | 4 | from relaton.models import Link 5 | 6 | 7 | __all__ = ( 8 | 'get_suitable_target', 9 | ) 10 | 11 | 12 | def get_suitable_target(links: List[Link]): 13 | """From a list of :class:`~relaton.models.links.Link` instances, 14 | return a string suitable to be used as value of ``target`` attribute 15 | on root XML element. 16 | 17 | It prefers a link with ``type`` set to “src”, 18 | if not present then first available link. 19 | """ 20 | try: 21 | target: Link = ( 22 | [l for l in links if l.type in ('src', 'pdf')] 23 | or links)[0] 24 | except IndexError: 25 | raise ValueError("Unable to find a suitable target (no links given)") 26 | else: 27 | return target.content 28 | -------------------------------------------------------------------------------- /xml2rfc_compat/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ietf-tools/bibxml-service/830eb8a3f03ed3885ed08b5d7092516ccb711139/xml2rfc_compat/tests/__init__.py -------------------------------------------------------------------------------- /xml2rfc_compat/tests/static/schemas/ns1.xsd: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | -------------------------------------------------------------------------------- /xml2rfc_compat/tests/static/schemas/xlink.xsd: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | -------------------------------------------------------------------------------- /xml2rfc_compat/tests/static/schemas/xml.xsd: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | -------------------------------------------------------------------------------- /xml2rfc_compat/tests/validate_remote_data_against_models.py: -------------------------------------------------------------------------------- 1 | # type: ignore 2 | import os 3 | from unittest import TestCase 4 | 5 | import yaml 6 | from lxml import etree 7 | 8 | from relaton.models import BibliographicItem 9 | from ..serializers import serialize 10 | 11 | 12 | class DataSourceValidationTestCase(TestCase): 13 | 14 | def setUp(self): 15 | module_dir = os.path.dirname(__file__) 16 | file_path = os.path.join(module_dir, "static/schemas/v3.xsd") 17 | self.xmlschema = etree.XMLSchema(file=file_path) 18 | 19 | def _validate_yaml_data(self, url): 20 | import requests 21 | r = requests.get(url) 22 | yaml_object = yaml.safe_load(r.content) 23 | bibitem = BibliographicItem(**yaml_object) 24 | serialized_data = serialize(bibitem) 25 | 26 | self.xmlschema.assertValid(serialized_data) 27 | 28 | def test_validate_rfcs_data(self): 29 | url = "https://raw.githubusercontent.com/ietf-tools/relaton-data-rfcs/main/data/RFC0001.yaml" 30 | self._validate_yaml_data(url) 31 | 32 | def test_validate_misc_data(self): 33 | url = "https://raw.githubusercontent.com/ietf-tools/relaton-data-misc/main/data/reference.ANSI.T1-102.1987.yaml" 34 | self._validate_yaml_data(url) 35 | 36 | def test_validate_internet_drafts_data(self): 37 | url = "https://raw.githubusercontent.com/ietf-tools/relaton-data-ids/main/data/draft--pale-email-00.yaml" 38 | self._validate_yaml_data(url) 39 | 40 | def test_validate_w3c_data(self): 41 | url = "https://raw.githubusercontent.com/ietf-tools/relaton-data-w3c/main/data/2dcontext.yaml" 42 | self._validate_yaml_data(url) 43 | 44 | def test_validate_threegpp_data(self): 45 | url = "https://raw.githubusercontent.com/ietf-tools/relaton-data-3gpp/main/data/TR_00.01U_UMTS_3.0.0.yaml" 46 | self._validate_yaml_data(url) 47 | 48 | def test_validate_ieee_data(self): 49 | url = "https://raw.githubusercontent.com/ietf-tools/relaton-data-ieee/main/data/AIEE_11-1943.yaml" 50 | self._validate_yaml_data(url) 51 | 52 | def test_validate_iana_data(self): 53 | url = "https://raw.githubusercontent.com/ietf-tools/relaton-data-iana/main/data/_6lowpan-parameters.yaml" 54 | self._validate_yaml_data(url) 55 | 56 | def test_validate_rfcsubseries_data(self): 57 | url = "https://raw.githubusercontent.com/ietf-tools/relaton-data-rfcsubseries/main/data/BCP0003.yaml" 58 | self._validate_yaml_data(url) 59 | 60 | def test_validate_nist_data(self): 61 | url = "https://raw.githubusercontent.com/ietf-tools/relaton-data-nist/main/data/NBS_BH_1.yaml" 62 | self._validate_yaml_data(url) 63 | -------------------------------------------------------------------------------- /xml2rfc_compat/types.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | from pydantic.dataclasses import dataclass 3 | 4 | 5 | @dataclass 6 | class Xml2rfcPathMetadata: 7 | """Describes xml2rfc filename sidecar YAML data.""" 8 | 9 | primary_docid: Optional[str] = None 10 | """Well-formed :term:`primary document identifier` 11 | that represents a bibliographic item corresponding 12 | to this filename. 13 | 14 | References :term:`docid.id` string value 15 | of the primary identifier of the target bibliographic item. 16 | """ 17 | 18 | # primary_docid_type: Optional[str] = None 19 | # """Only has effect if primary_docid is supplied. 20 | # 21 | # Can be used if ``primary_docid`` field is ambiguous 22 | # and differentiation by docid type is necessary. 23 | # """ 24 | 25 | invalid: Optional[bool] = False 26 | """Indicates that bibliographic item served by this 27 | xml2rfc path was created in error and not supposed to exist 28 | (does not map to a published resource, not represented 29 | by authoritative bibliographic data sources, and so on). 30 | """ 31 | -------------------------------------------------------------------------------- /xml2rfc_compat/urls.py: -------------------------------------------------------------------------------- 1 | """ 2 | Provides utilities for constructing patterns 3 | fit for inclusion in site’s root URL configuration. 4 | """ 5 | from django.urls import re_path 6 | from django.views.decorators.cache import never_cache 7 | from django.views.decorators.http import require_safe 8 | 9 | from .aliases import get_aliases 10 | from .models import dir_subpath_regex 11 | from .adapters import adapters 12 | from .views import handle_xml2rfc_path 13 | 14 | 15 | def get_urls(): 16 | """Returns a list of URL patterns suitable for inclusion 17 | in site’s root URL configuration, based on registered adapters. 18 | 19 | The directory name for each registered adapter is automatically 20 | expanded to include aliases 21 | (e.g., “bibxml-ids” is included in addition to canonical “bibxml3”). 22 | 23 | Adapters should have been all registered prior 24 | to calling this function. 25 | 26 | Each generated URL pattern is in the shape of 27 | ``/[_]reference..xml``, 28 | and constructed view handles bibliographic item resolution 29 | according to :ref:`xml2rfc-path-resolution-algorithm`. 30 | 31 | .. seealso:: :func:`.handle_xml2rfc_path` for how requests are handled. 32 | """ 33 | dirnames_with_aliases = [ 34 | d 35 | for dirname in adapters.keys() 36 | for d in [dirname, *get_aliases(dirname)] 37 | ] 38 | return [ 39 | re_path( 40 | dir_subpath_regex % dirname, 41 | never_cache(require_safe(handle_xml2rfc_path)), 42 | name='xml2rfc_%s' % dirname) 43 | for dirname in dirnames_with_aliases 44 | ] 45 | --------------------------------------------------------------------------------