├── .gitattributes ├── .github └── workflows │ └── actions.yml ├── .gitignore ├── .pre-commit-config.yaml ├── CONTRIBUTING.rst ├── LICENSE ├── MANIFEST.in ├── README.rst ├── build.sh ├── build_fe.sh ├── build_whl.sh ├── clean.sh ├── data_browser ├── __init__.py ├── admin.py ├── api.py ├── apps.py ├── common.py ├── fe_build │ ├── asset-manifest.json │ ├── index.html │ └── static │ │ ├── css │ │ ├── main.c698af7a.css │ │ └── main.c698af7a.css.map │ │ └── js │ │ ├── main.c4ca9cb3.js │ │ ├── main.c4ca9cb3.js.LICENSE.txt │ │ └── main.c4ca9cb3.js.map ├── format_csv.py ├── helpers.py ├── migration_helpers.py ├── migrations │ ├── 0001_initial.py │ ├── 0002_auto_20200331_1842.py │ ├── 0003_remove_view_app.py │ ├── 0004_auto_20200501_0903.py │ ├── 0005_auto_20200516_1726.py │ ├── 0006_auto_20200531_1450.py │ ├── 0007_view_public_slug.py │ ├── 0008_view_limit.py │ ├── 0009_migrate_saved_views.py │ ├── 0010_shared.py │ ├── 0011_folder.py │ ├── 0012_can_share.py │ └── __init__.py ├── model.txt ├── models.py ├── orm_admin.py ├── orm_aggregates.py ├── orm_debug.py ├── orm_fields.py ├── orm_functions.py ├── orm_lookups.py ├── orm_results.py ├── orm_types.py ├── query.py ├── templates │ └── data_browser │ │ └── index.html ├── types.py ├── urls.py ├── util.py ├── views.py └── web_root │ ├── android-chrome-192x192.png │ ├── android-chrome-512x512.png │ ├── apple-touch-icon.png │ ├── favicon-16x16.png │ ├── favicon-32x32.png │ ├── favicon.ico │ └── site.webmanifest ├── frontend ├── .gitignore ├── README.md ├── package-lock.json ├── package.json ├── public │ └── index.html └── src │ ├── App.js │ ├── App.scss │ ├── App.test.js │ ├── Config.js │ ├── ContextMenu.js │ ├── CurrentSavedView.js │ ├── FieldList.js │ ├── FilterList.js │ ├── HomePage.js │ ├── Network.js │ ├── Query.js │ ├── Query.test.js │ ├── QueryPage.js │ ├── Results.js │ ├── SavedViewPage.js │ ├── Tooltip.js │ ├── Util.js │ ├── WindowDimensions.js │ ├── index.js │ ├── index.scss │ ├── logo.svg │ └── setupTests.js ├── pyproject.toml ├── requirements.txt ├── screenshot.png ├── setup.py ├── structure.svg ├── tests ├── __init__.py ├── array │ ├── __init__.py │ └── models.py ├── conftest.py ├── core │ ├── __init__.py │ ├── admin.py │ ├── migrations │ │ ├── 0001_initial.py │ │ └── __init__.py │ └── models.py ├── json │ ├── __init__.py │ └── models.py ├── snapshots │ ├── __init__.py │ └── snap_test_views.py ├── test_admin.py ├── test_api.py ├── test_array_field.py ├── test_common.py ├── test_helpers.py ├── test_json_field.py ├── test_migrations.py ├── test_models.py ├── test_orm.py ├── test_orm_debug.py ├── test_query.py ├── test_tests.py ├── test_views.py ├── urls.py └── util.py └── tox.ini /.gitattributes: -------------------------------------------------------------------------------- 1 | data_browser/fe_build/** linguist-generated=true 2 | data_browser/fe_build/** -diff -merge 3 | data_browser/templates/data_browser/index.html linguist-generated=true 4 | data_browser/templates/data_browser/index.html -diff -merge 5 | -------------------------------------------------------------------------------- /.github/workflows/actions.yml: -------------------------------------------------------------------------------- 1 | on: [push, pull_request] 2 | 3 | jobs: 4 | lint: 5 | runs-on: ubuntu-latest 6 | steps: 7 | - name: Checkout 8 | uses: actions/checkout@v2 9 | - name: Set up Python 10 | uses: actions/setup-python@v2 11 | with: 12 | python-version: "3.11" 13 | - name: Install dependencies 14 | run: | 15 | python -m pip install --upgrade pip 16 | python -m pip install pre-commit 17 | - name: Lint with pre-commit 18 | run: pre-commit run --all-files 19 | 20 | test: 21 | runs-on: ubuntu-latest 22 | strategy: 23 | fail-fast: false 24 | matrix: 25 | python: ["3.8", "3.9", "3.10", "3.11", "3.12"] 26 | database_url: 27 | - sqlite:///db.sqlite3 28 | - mysql://root:test@127.0.0.1:3306/data_browser 29 | - postgres://postgres:test@127.0.0.1:5432/data_browser, 30 | services: 31 | mysql: 32 | image: mysql 33 | env: 34 | MYSQL_ROOT_PASSWORD: test 35 | ports: 36 | - 3306:3306 37 | postgres: 38 | image: postgres 39 | env: 40 | POSTGRES_PASSWORD: test 41 | ports: 42 | - 5432:5432 43 | env: 44 | DATABASE_URL: ${{ matrix.database_url }} 45 | steps: 46 | - name: Checkout 47 | uses: actions/checkout@v2 48 | - name: Set up Python 49 | uses: actions/setup-python@v2 50 | with: 51 | python-version: ${{ matrix.python }} 52 | - name: Install dependencies 53 | run: | 54 | python -m pip install --upgrade pip 55 | python -m pip install tox tox-gh-actions 56 | - name: Test with tox 57 | run: tox run 58 | - name: Upload coverage data 59 | uses: actions/upload-artifact@v2 60 | with: 61 | name: coverage-data 62 | path: ".coverage.*" 63 | 64 | coverage: 65 | runs-on: ubuntu-latest 66 | needs: test 67 | steps: 68 | - name: Checkout 69 | uses: actions/checkout@v2 70 | - name: Set up Python 71 | uses: actions/setup-python@v2 72 | with: 73 | python-version: "3.11" 74 | - name: Install dependencies 75 | run: | 76 | python -m pip install --upgrade pip 77 | python -m pip install coverage[toml] 78 | - name: Download coverage data. 79 | uses: actions/download-artifact@v2 80 | with: 81 | name: coverage-data 82 | - name: Combine coverage & fail if it's <100%. 83 | run: | 84 | python -m coverage combine 85 | python -m coverage html 86 | python -m coverage report --fail-under=100 87 | - name: Upload HTML report if check failed. 88 | if: ${{ failure() }} 89 | uses: actions/upload-artifact@v2 90 | with: 91 | name: html-report 92 | path: htmlcov 93 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | # See https://pre-commit.com for more information 2 | # See https://pre-commit.com/hooks.html for more hooks 3 | default_language_version: 4 | python: python3.10 5 | default_stages: [commit] 6 | repos: 7 | - repo: https://github.com/pre-commit/pre-commit-hooks 8 | rev: v4.5.0 9 | hooks: 10 | - id: trailing-whitespace 11 | - id: end-of-file-fixer 12 | - id: check-yaml 13 | - id: check-added-large-files 14 | - repo: https://github.com/asottile/pyupgrade 15 | rev: v3.15.0 16 | hooks: 17 | - id: pyupgrade 18 | args: [--py37-plus, --keep-runtime-typing] 19 | - repo: https://github.com/adamchainz/django-upgrade 20 | rev: 1.15.0 21 | hooks: 22 | - id: django-upgrade 23 | args: [--target-version, "3.2"] 24 | - repo: https://github.com/hakancelik96/unimport 25 | rev: 1.2.1 26 | hooks: 27 | - id: unimport 28 | args: [--remove, --include-star-import, --ignore-init, --gitignore] 29 | - repo: https://github.com/pre-commit/mirrors-isort 30 | rev: v5.10.1 31 | hooks: 32 | - id: isort 33 | - repo: https://github.com/psf/black 34 | rev: 24.1.1 35 | hooks: 36 | - id: black 37 | - repo: https://github.com/pycqa/flake8 38 | rev: 6.1.0 39 | hooks: 40 | - id: flake8 41 | entry: pflake8 42 | additional_dependencies: 43 | - pyproject-flake8 44 | - flake8-use-fstring 45 | - flake8-tidy-imports 46 | - flake8-comprehensions 47 | - flake8-bugbear 48 | - flake8-print 49 | - flake8-debugger 50 | - flake8-simplify 51 | - flake8-return 52 | - flake8-no-pep420 53 | - flake8-tuple 54 | - repo: https://github.com/Lucas-C/pre-commit-hooks-markup 55 | rev: v1.0.1 56 | hooks: 57 | - id: rst-linter 58 | - repo: local 59 | hooks: 60 | - id: no-prints 61 | name: Check for python prints 62 | entry: ' *print\(' 63 | language: pygrep 64 | types: [python] 65 | exclude: tests/ 66 | exclude: 67 | data_browser/fe_build/|data_browser/templates/data_browser/index.html 68 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2020, Gordon Wrigley 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | 1. Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | 2. Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | 3. Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include *.png 2 | include *.rst 3 | include *.sh 4 | include *.svg 5 | include *.txt 6 | include *.yaml 7 | include *.cfg 8 | include *.ini 9 | include .coveragerc 10 | include .flake8 11 | include LICENSE 12 | 13 | recursive-include tests *.py 14 | 15 | include frontend/package.json 16 | include frontend/package-lock.json 17 | include frontend/README.md 18 | recursive-include frontend/src *.scss 19 | recursive-include frontend/src *.html 20 | recursive-include frontend/src *.js 21 | recursive-include frontend/src *.json 22 | recursive-include frontend/src *.md 23 | recursive-include frontend/src *.svg 24 | recursive-include frontend/public *.html 25 | 26 | exclude data_browser/*.txt 27 | -------------------------------------------------------------------------------- /build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -ex 3 | 4 | git add README.rst data_browser/__init__.py 5 | git update-index --refresh # will error if there are unstaged changes 6 | 7 | ./build_fe.sh 8 | 9 | git add data_browser/fe_build data_browser/templates/data_browser/index.html 10 | 11 | ./build_whl.sh 12 | 13 | version=$(python -c "import data_browser; print(data_browser.version)") 14 | set +x 15 | echo SUCCESS 16 | echo 17 | echo "To release run the following:" 18 | echo " git commit -m $version && git tag -a -m $version $version && git push --follow-tags && python -m twine upload dist/*" 19 | -------------------------------------------------------------------------------- /build_fe.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -ex 3 | 4 | (cd frontend; npm run build) 5 | rm -Rf data_browser/fe_build 6 | cp -a frontend/build data_browser/fe_build 7 | mkdir -p data_browser/templates/data_browser 8 | cp frontend/build/index.html data_browser/templates/data_browser/index.html 9 | -------------------------------------------------------------------------------- /build_whl.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -ex 3 | 4 | rm -Rf dist build 5 | python -m pip install -U pip 6 | python -m pip install --upgrade setuptools wheel twine check-manifest 7 | check-manifest -v 8 | python setup.py sdist bdist_wheel 9 | twine check dist/* 10 | -------------------------------------------------------------------------------- /clean.sh: -------------------------------------------------------------------------------- 1 | rm -Rf data_browser/fe_build/* frontend/build/ data_browser/templates/data_browser/index.html dist build 2 | -------------------------------------------------------------------------------- /data_browser/__init__.py: -------------------------------------------------------------------------------- 1 | version = "4.2.10" 2 | -------------------------------------------------------------------------------- /data_browser/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | from django.utils.html import format_html 3 | 4 | from data_browser import models 5 | from data_browser.common import PUBLIC_PERM 6 | from data_browser.common import has_permission 7 | from data_browser.common import set_global_state 8 | from data_browser.helpers import AdminMixin 9 | from data_browser.helpers import attributes 10 | 11 | 12 | @admin.register(models.View) 13 | class ViewAdmin(AdminMixin, admin.ModelAdmin): 14 | fieldsets = [ 15 | ( 16 | None, 17 | { 18 | "fields": [ 19 | "name", 20 | "owner", 21 | "valid", 22 | "open_view", 23 | "folder", 24 | "description", 25 | ] 26 | }, 27 | ), 28 | ( 29 | "Public", 30 | { 31 | "fields": [ 32 | "public", 33 | "public_slug", 34 | "public_link", 35 | "google_sheets_formula", 36 | ] 37 | }, 38 | ), 39 | ("Query", {"fields": ["model_name", "fields", "query", "limit"]}), 40 | ("Internal", {"fields": ["id", "created_time", "shared"]}), 41 | ] 42 | list_display = ["__str__", "owner", "public"] 43 | 44 | def has_add_permission(self, request): 45 | return False 46 | 47 | def has_change_permission(self, request, obj=None): 48 | return False 49 | 50 | def get_fieldsets(self, request, obj=None): 51 | res = super().get_fieldsets(request, obj) 52 | if not has_permission(request.user, PUBLIC_PERM): 53 | res = [fs for fs in res if fs[0] != "Public"] 54 | return res 55 | 56 | def changeform_view(self, request, *args, **kwargs): 57 | with set_global_state(request=request, set_ddb=False): 58 | res = super().changeform_view(request, *args, **kwargs) 59 | res.render() 60 | return res 61 | 62 | @staticmethod 63 | def open_view(obj): 64 | if not obj.model_name: 65 | return "N/A" # pragma: no cover 66 | return format_html(f'view') 67 | 68 | @attributes(boolean=True) 69 | def valid(self, obj): 70 | with set_global_state(user=obj.owner, public_view=False): 71 | return obj.is_valid() 72 | -------------------------------------------------------------------------------- /data_browser/api.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | import django.contrib.admin.views.decorators as admin_decorators 4 | from django.shortcuts import get_object_or_404 5 | from django.views.decorators import csrf 6 | 7 | from data_browser.common import SHARE_PERM 8 | from data_browser.common import HttpResponse 9 | from data_browser.common import JsonResponse 10 | from data_browser.common import global_state 11 | from data_browser.common import set_global_state 12 | from data_browser.common import str_user 13 | from data_browser.common import users_with_permission 14 | from data_browser.models import View 15 | from data_browser.util import group_by 16 | 17 | 18 | def clean_str(field, value): 19 | return value.strip()[: View._meta.get_field(field).max_length] 20 | 21 | 22 | def clean_uint(field, value): 23 | try: 24 | value = int(value) 25 | except Exception: # noqa: E722 input sanitization 26 | value = 1 27 | return max(value, 1) 28 | 29 | 30 | def clean_noop(field, value): 31 | return value 32 | 33 | 34 | WRITABLE_FIELDS = [ # model_field_name, api_field_name, clean 35 | ("name", "name", clean_str), 36 | ("description", "description", clean_noop), 37 | ("public", "public", clean_noop), 38 | ("model_name", "model", clean_noop), 39 | ("fields", "fields", clean_noop), 40 | ("query", "query", clean_noop), 41 | ("limit", "limit", clean_uint), 42 | ("folder", "folder", clean_str), 43 | ("shared", "shared", clean_noop), 44 | ] 45 | 46 | 47 | def deserialize(data): 48 | res = { 49 | model_field_name: clean(model_field_name, data[api_field_name]) 50 | for model_field_name, api_field_name, clean in WRITABLE_FIELDS 51 | if api_field_name in data 52 | } 53 | 54 | return res 55 | 56 | 57 | def name_sort(entries): 58 | return sorted(entries, key=lambda entry: entry["name"].lower()) 59 | 60 | 61 | def serialize(view): 62 | return { 63 | **{ 64 | api_field_name: getattr(view, model_field_name) 65 | for model_field_name, api_field_name, clean in WRITABLE_FIELDS 66 | }, 67 | "publicLink": view.public_link(), 68 | "googleSheetsFormula": view.google_sheets_formula(), 69 | "link": view.get_query().get_url("html"), 70 | "createdTime": f"{view.created_time:%Y-%m-%d %H:%M:%S}", 71 | "pk": view.pk, 72 | "shared": bool(view.shared and view.name), 73 | "valid": view.is_valid(), 74 | "can_edit": global_state.request.user == view.owner, 75 | "type": "view", 76 | } 77 | 78 | 79 | def serialize_list(views, *, include_invalid=False): 80 | res = [serialize(view) for view in views] 81 | if not include_invalid: 82 | res = [row for row in res if row["valid"]] 83 | return name_sort(res) 84 | 85 | 86 | def serialize_folders(views, *, include_invalid=False): 87 | grouped_views = group_by(views, key=lambda v: v.folder.strip()) 88 | flat_views = grouped_views.pop("", []) 89 | 90 | res = serialize_list(flat_views, include_invalid=include_invalid) 91 | for folder_name, views in sorted(grouped_views.items()): 92 | entries = serialize_list(views, include_invalid=include_invalid) 93 | if entries: 94 | res.append({"name": folder_name, "type": "folder", "entries": entries}) 95 | return name_sort(res) 96 | 97 | 98 | def get_queryset(user): 99 | return View.objects.filter(owner=user) 100 | 101 | 102 | @csrf.csrf_protect 103 | @admin_decorators.staff_member_required 104 | @set_global_state(public_view=False) 105 | def view_list(request): 106 | if request.method == "GET": 107 | # saved 108 | saved_views = get_queryset(request.user) 109 | saved_views_serialized = serialize_folders(saved_views, include_invalid=True) 110 | 111 | # shared 112 | shared_views = ( 113 | View.objects.exclude(owner=request.user) 114 | .filter(owner__in=users_with_permission(SHARE_PERM), shared=True) 115 | .exclude(name="") 116 | .prefetch_related("owner") 117 | ) 118 | shared_views_by_user = group_by(shared_views, lambda v: str_user(v.owner)) 119 | shared_views_serialized = [] 120 | for owner_name, shared_views in shared_views_by_user.items(): 121 | entries = serialize_folders(shared_views) 122 | if entries: 123 | shared_views_serialized.append( 124 | {"name": owner_name, "type": "folder", "entries": entries} 125 | ) 126 | 127 | # response 128 | return JsonResponse( 129 | {"saved": saved_views_serialized, "shared": shared_views_serialized} 130 | ) 131 | elif request.method == "POST": 132 | data = json.loads(request.body) 133 | view = View.objects.create(owner=request.user, **deserialize(data)) 134 | return JsonResponse(serialize(view)) 135 | else: 136 | return HttpResponse(status=400) 137 | 138 | 139 | @csrf.csrf_protect 140 | @admin_decorators.staff_member_required 141 | @set_global_state(public_view=False) 142 | def view_detail(request, pk): 143 | view = get_object_or_404(get_queryset(request.user), pk=pk) 144 | 145 | if request.method == "GET": 146 | return JsonResponse(serialize(view)) 147 | elif request.method == "PATCH": 148 | data = json.loads(request.body) 149 | for k, v in deserialize(data).items(): 150 | setattr(view, k, v) 151 | view.save() 152 | return JsonResponse(serialize(view)) 153 | elif request.method == "DELETE": 154 | view.delete() 155 | return HttpResponse(status=204) 156 | else: 157 | return HttpResponse(status=400) 158 | -------------------------------------------------------------------------------- /data_browser/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | from django.db.models import AutoField 3 | 4 | 5 | class DataBrowserConfig(AppConfig): 6 | name = "data_browser" 7 | verbose_name = "Data Browser" 8 | default_auto_field = AutoField 9 | -------------------------------------------------------------------------------- /data_browser/common.py: -------------------------------------------------------------------------------- 1 | import functools 2 | import logging 3 | import math 4 | import threading 5 | import traceback 6 | from copy import copy 7 | 8 | from django import http 9 | from django.contrib.auth import get_user_model 10 | from django.contrib.auth.models import Permission 11 | from django.contrib.contenttypes.models import ContentType 12 | from django.utils.functional import cached_property 13 | 14 | from data_browser import version 15 | 16 | 17 | class Settings: 18 | _defaults = { 19 | "DATA_BROWSER_ALLOW_PUBLIC": False, 20 | "DATA_BROWSER_AUTH_USER_COMPAT": True, 21 | "DATA_BROWSER_DEFAULT_ROW_LIMIT": 1000, 22 | "DATA_BROWSER_DEV": False, 23 | "DATA_BROWSER_FE_DSN": None, 24 | "DATA_BROWSER_ADMIN_FIELD_NAME": "admin", 25 | "DATA_BROWSER_USING_DB": "default", 26 | "DATA_BROWSER_ADMIN_OPTIONS": {}, 27 | "DATA_BROWSER_APPS_EXPANDED": True, 28 | "DATA_BROWSER_ADMIN_SITE": None, 29 | } 30 | 31 | def __getattr__(self, name): 32 | from django.conf import settings 33 | 34 | if hasattr(settings, name): 35 | return getattr(settings, name) 36 | return self._defaults[name] 37 | 38 | 39 | settings = Settings() 40 | 41 | 42 | PUBLIC_PERM = "make_view_public" 43 | SHARE_PERM = "share_view" 44 | 45 | 46 | def has_permission(user, permission): 47 | if user is None: 48 | return False 49 | 50 | return user.has_perm(f"data_browser.{permission}") 51 | 52 | 53 | def users_with_permission(permission): 54 | from data_browser.models import View 55 | 56 | ct = ContentType.objects.get_for_model(View) 57 | perm = Permission.objects.get(codename=permission, content_type=ct) 58 | User = get_user_model() 59 | 60 | qs = User.objects.none() 61 | for backend_path in settings.AUTHENTICATION_BACKENDS: 62 | qs |= User.objects.with_perm(perm, backend=backend_path) 63 | 64 | return qs 65 | 66 | 67 | def str_user(user): 68 | return ( 69 | str(user) or user.get_username() or getattr(user, user.get_email_field_name()) 70 | ) 71 | 72 | 73 | def JsonResponse(data): 74 | res = http.JsonResponse(data, safe=False) 75 | res["X-Version"] = version 76 | res["Access-Control-Expose-Headers"] = "X-Version" 77 | return res 78 | 79 | 80 | def HttpResponse(*args, **kwargs): 81 | res = http.HttpResponse(*args, **kwargs) 82 | res["X-Version"] = version 83 | res["Access-Control-Expose-Headers"] = "X-Version" 84 | return res 85 | 86 | 87 | def debug_log(msg, exc=None): # pragma: no cover 88 | if exc: 89 | if isinstance(exc, AssertionError): 90 | raise 91 | msg = f"{msg}:\n{traceback.format_exc()}" 92 | 93 | if settings.DEBUG: 94 | logging.getLogger(__name__).warning(f"DDB: {msg}") 95 | 96 | 97 | def all_subclasses(cls): 98 | res = set() 99 | queue = {cls} 100 | while queue: 101 | cls = queue.pop() 102 | subs = set(cls.__subclasses__()) 103 | queue.update(subs - res) 104 | res.update(subs) 105 | return res 106 | 107 | 108 | def get_optimal_decimal_places(nums, sf=3, max_dp=6): 109 | actual_dps = set() 110 | filtered = set() 111 | for num in nums: 112 | if num: 113 | s = f"{num:g}" 114 | filtered.add(num) 115 | if "e-" in s: 116 | actual_dps.add(float("inf")) 117 | elif "." in s: 118 | actual_dps.add(len(s.split(".")[1])) 119 | else: 120 | actual_dps.add(0) 121 | 122 | if not filtered: 123 | return 0 124 | 125 | max_actual_dp = max(actual_dps) 126 | if max_actual_dp <= 2: 127 | return max_actual_dp 128 | 129 | min_value = min(filtered) 130 | min_magnitude = math.floor(math.log(min_value, 10)) 131 | dp_for_sf = sf - min_magnitude - 1 132 | 133 | return max(0, min(dp_for_sf, max_actual_dp, max_dp)) 134 | 135 | 136 | class GlobalState(threading.local): 137 | def __init__(self): 138 | self._state = None 139 | 140 | @property 141 | def request(self): 142 | return self._state.request 143 | 144 | @property 145 | def models(self): 146 | return self._state.models 147 | 148 | 149 | global_state = GlobalState() 150 | 151 | 152 | class _UNSPECIFIED: 153 | pass 154 | 155 | 156 | class _State: 157 | def __init__( 158 | self, 159 | prev, 160 | *, 161 | request=_UNSPECIFIED, 162 | user=_UNSPECIFIED, 163 | public_view=_UNSPECIFIED, 164 | set_ddb=True, 165 | ): 166 | if request is _UNSPECIFIED: 167 | request = prev.request 168 | 169 | new_request = copy(request) 170 | 171 | if user is not _UNSPECIFIED: 172 | new_request.user = user 173 | 174 | if set_ddb: 175 | assert public_view is not _UNSPECIFIED 176 | 177 | new_request.data_browser = { 178 | "public_view": public_view, 179 | "fields": set(), 180 | "calculated_fields": set(), 181 | } 182 | 183 | self.request = new_request 184 | self._children = {} 185 | 186 | @cached_property 187 | def models(self): 188 | from data_browser.orm_admin import get_models 189 | 190 | old = global_state._state 191 | global_state._state = None 192 | try: 193 | return get_models(self.request) 194 | finally: 195 | global_state._state = old 196 | 197 | 198 | class set_global_state: 199 | def __init__(self, request=_UNSPECIFIED, **kwargs): 200 | self.request = request 201 | self.kwargs = kwargs 202 | 203 | def __call__(self, func): 204 | @functools.wraps(func) 205 | def wrapper(request, *args, **kwargs): 206 | assert self.request is _UNSPECIFIED 207 | self.request = request 208 | try: 209 | with self: 210 | return func(request, *args, **kwargs) 211 | finally: 212 | self.request = _UNSPECIFIED 213 | 214 | return wrapper 215 | 216 | def __enter__(self): 217 | self.old = global_state._state 218 | 219 | cachable = self.old and self.request is _UNSPECIFIED 220 | key = tuple(self.kwargs.items()) 221 | 222 | if cachable and key in self.old._children: 223 | state = self.old._children[key] 224 | else: 225 | state = _State(self.old, request=self.request, **self.kwargs) 226 | if cachable: 227 | self.old._children[key] = state 228 | 229 | global_state._state = state 230 | 231 | def __exit__(self, exc_type, exc_value, traceback): 232 | global_state._state = self.old 233 | -------------------------------------------------------------------------------- /data_browser/fe_build/asset-manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "files": { 3 | "main.css": "./static/css/main.c698af7a.css", 4 | "main.js": "./static/js/main.c4ca9cb3.js", 5 | "index.html": "./index.html", 6 | "main.c698af7a.css.map": "./static/css/main.c698af7a.css.map", 7 | "main.c4ca9cb3.js.map": "./static/js/main.c4ca9cb3.js.map" 8 | }, 9 | "entrypoints": [ 10 | "static/css/main.c698af7a.css", 11 | "static/js/main.c4ca9cb3.js" 12 | ] 13 | } -------------------------------------------------------------------------------- /data_browser/fe_build/index.html: -------------------------------------------------------------------------------- 1 | {% load static %}View{% block extrahead %} {% endblock %}{% block messages %} {% if messages %}{% endif %} {% endblock messages %}{% csrf_token %}
-------------------------------------------------------------------------------- /data_browser/fe_build/static/css/main.c698af7a.css: -------------------------------------------------------------------------------- 1 | body{-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale;font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Roboto,Oxygen,Ubuntu,Cantarell,Fira Sans,Droid Sans,Helvetica Neue,sans-serif}code{font-family:source-code-pro,Menlo,Monaco,Consolas,Courier New,monospace}*,:after,:before{box-sizing:border-box}:root{--dark-color:#333e48;--link-color:#000be6;--dark-link-color:#000899;--border-color:#ccc;--shade-color:#ededed;--null-color:#dedede;--bad-color:#ff3c00;--good-color:#188123;--real-field-color:#d2eed5;--calculated-field-color:#e0e0e0;--aggregate-field-color:#d2d4ee;--function-field-color:#d4ede9;--annotated-field-color:#d4ede9;--related-field-color:#fff}.RealField{background-color:#d2eed5;background-color:var(--real-field-color)}.CalculatedField{background-color:#e0e0e0;background-color:var(--calculated-field-color)}.AggregateField{background-color:#d2d4ee;background-color:var(--aggregate-field-color)}.FunctionField{background-color:#d4ede9;background-color:var(--function-field-color)}.RelatedField{background-color:#fff;background-color:var(--related-field-color)}.AnnotatedField{background-color:#d4ede9;background-color:var(--annotated-field-color)}button,input,select,textarea,th{font-family:inherit;font-size:inherit;font-weight:inherit}body{color:#333e48;color:var(--dark-color);font-family:Roboto,Noto Sans,sans-serif;font-size:16px;margin:0;padding:0}a{color:#000be6;color:var(--link-color);text-decoration:none}a:visited{color:#000899;color:var(--dark-link-color)}a:hover{text-decoration:underline}h1{font-size:28px}h1,h2{font-weight:400}h2{font-size:20px;margin:0;padding:0}p{margin:5px auto}input,select{border:1px solid #ccc;border:1px solid var(--border-color);border-radius:3px}input{padding:3px 6px}select{display:inline;padding:2px;width:auto}.TLink{background:none!important;border:none;color:#000899;color:var(--dark-link-color);cursor:pointer;padding:0!important;text-align:left}.TLink:hover{text-decoration:underline}.SLink,.Symbol{font-size:inherit;font-weight:inherit;vertical-align:-20%}.SLink{background:none!important;border:none;color:#000899;color:var(--dark-link-color);cursor:pointer;padding:0!important}.SLink:hover{text-decoration:underline}input::-webkit-input-placeholder,textarea::-webkit-input-placeholder{color:#ccc;color:var(--border-color)}input:-ms-input-placeholder,textarea:-ms-input-placeholder{color:#ccc;color:var(--border-color)}input::placeholder,textarea::placeholder{color:#ccc;color:var(--border-color)}#root{position:relative}.QueryPage{display:flex;flex-direction:column;max-height:100vh;padding:10px 0 0}.QueryPage>*{flex:0 0 auto}.QueryPage>.MainSpace{flex:0 1 auto}.MainSpace{display:flex;flex-flow:row;justify-content:space-between;max-width:100vw;overflow-y:hidden;padding:0}.MainSpace>*{flex:0 0 auto}.MainSpace>.Results{flex:0 1 auto;margin:10px;overflow:hidden}.MainSpace>.FieldsList{margin:10px 0 10px 10px}.MainSpace>.FieldsList>.FieldsFilter{display:flex;flex-direction:row}.MainSpace>.FieldsList>.FieldsFilter>input{flex:1 1}.Scroller{max-height:100%;max-width:100%;overflow:auto}.ModelSelector{border:none;color:#333e48;color:var(--dark-color);font-size:28px;margin:0 auto;padding:1px;text-align:center;text-align-last:center}.ModelSelector:hover{border:1px solid #ccc;border:1px solid var(--border-color);border-radius:3px;padding:0}.ModelSelector optgroup,.ModelSelector option{text-align:left;text-align-last:left}.Filters,.InvalidFields{align-items:center;display:flex;flex-direction:column;margin:0 auto;min-height:19px;position:relative}.Filters .FiltersToggle,.InvalidFields .FiltersToggle{position:absolute;right:-18px}.Filters p,.InvalidFields p{margin:5px}.Filters table,.Filters table td,.Filters table th,.InvalidFields table,.InvalidFields table td,.InvalidFields table th{border:none;padding:3px 1px}input.FilterValue,select.FilterValue{width:330px}.FilterValue.Half{width:50%}select.Lookup{width:100%}.FieldsList{display:flex;flex-flow:column nowrap}.FieldsList .Scroller{border:1px solid #ccc;border:1px solid var(--border-color);border-radius:3px}.FieldsList .FieldRow{background:#fff;display:flex;flex-flow:row nowrap;padding:2px}.FieldsList .FieldRowExpanded{padding-bottom:0;position:-webkit-sticky;position:sticky}.FieldsList .FieldRowExpanded .FieldExpand,.FieldsList .FieldRowExpanded .FieldName{border-bottom:1px solid #ccc;border-bottom:1px solid var(--border-color);border-radius:4px 4px 0 0}.FieldsList .FieldExpand,.FieldsList .FieldFilter{min-width:20px;padding:4px 0 0}.FieldsList .FieldName{border-radius:4px;padding:4px;width:100%}.FieldsList .FieldSubFields{border:1px solid #ccc;border:1px solid var(--border-color);border-radius:0;border-top:none;margin:0 2px 2px 22px}.Results{position:relative}.Results .Freeze{position:-webkit-sticky;position:sticky;top:2px}.Results .Scroller,.Results td,.Results th{border:1px solid #ccc;border:1px solid var(--border-color);border-radius:2px}.Results tr{line-height:120%}.Results td,.Results th{padding:5px}.Results td.number,.Results td.time{text-align:right}.Results td.Empty{border:none}.Results td.HoriBorder,.Results th.HoriBorder{border-top:2px solid #000}.Results td.LeftBorder,.Results th.LeftBorder{border-left:2px solid #000}.Results th{text-align:center}a.Logo{color:#333e48;color:var(--dark-color);cursor:pointer;font-family:Anton,sans-serif;font-size:40px;margin:0 15px;position:absolute}a.Logo>span{display:inline-block}a.Logo>span.Version{font-size:20px}.HomePage{align-items:flex-start;display:flex;margin:0;padding:40px 30px 30px}.HomePage>div{border:1px solid #ccc;border:1px solid var(--border-color);border-radius:3px;color:#333e48;color:var(--dark-color);flex:1 1;margin:30px 2% 0}.HomePage>div>div{margin:20px}.HomePage h1{margin:20px 0 10px}.HomePage p{margin:0}.HomePage .ModelList .AppModels{padding-left:30px}.HomePage .SavedAndSharedViews .SavedView,.HomePage .SavedAndSharedViews .SavedViewsFolder{margin:5px 0 10px}.HomePage .SavedAndSharedViews .SavedView div,.HomePage .SavedAndSharedViews .SavedViewsFolder div{padding-left:30px}.HomePage .SavedAndSharedViews .SavedView .SavedViewDetail,.HomePage .SavedAndSharedViews .SavedViewsFolder .SavedViewDetail{padding:0 0 0 20px}.EditSavedView{display:flex;flex-flow:column nowrap;margin:0 auto;padding:50px 10px 30px}.EditSavedView>div{margin:0 auto}.EditSavedView .SavedViewTitle{font-size:28px}.EditSavedView form{border:1px solid #ccc;border:1px solid var(--border-color);border-radius:3px;padding:12px}.EditSavedView .SavedViewName{font-size:20px;width:100%}.EditSavedView textarea{border:1px solid #ccc;border:1px solid var(--border-color);border-radius:3px;min-height:100px;min-width:500px;padding:10px;width:100%}.EditSavedView th{font-weight:700;min-width:115px;padding:2px 2px 2px 6px;text-align:left;white-space:nowrap}.EditSavedView td{padding:2px}.EditSavedView td p{margin:0;padding:0 0 0 4px}.EditSavedView table{padding:5px 2px}.EditSavedView table input[type=number],.EditSavedView table input[type=text]{width:100%}.SavedViewActions{align-items:flex-end;display:flex;justify-content:space-between;padding:10px 20px}input.RowLimit{padding:1px 3px;width:100px}.CopyToClipboard{font-size:13px}.Overlay{align-items:center;bottom:0;display:flex;justify-content:center;left:0;pointer-events:none;position:absolute;right:0;top:0;z-index:99}.Fade{opacity:.3}.Error{color:#ff3c00;color:var(--bad-color)}.Success{color:#188123;color:var(--good-color)}.ContextMenu,.Tooltip{background:#fff;border:1px solid #ccc;border:1px solid var(--border-color);box-shadow:3px 3px 10px #333e48;box-shadow:3px 3px 10px var(--dark-color);padding:5px;position:fixed;z-index:100}.ContextMenu p,.Tooltip p{margin:0;padding:5px}.Tooltip{margin:10px}.DataCell .Null{color:#dedede;color:var(--null-color)}.ContextCursor{cursor:context-menu} 2 | /*# sourceMappingURL=main.c698af7a.css.map*/ -------------------------------------------------------------------------------- /data_browser/fe_build/static/js/main.c4ca9cb3.js.LICENSE.txt: -------------------------------------------------------------------------------- 1 | /* 2 | object-assign 3 | (c) Sindre Sorhus 4 | @license MIT 5 | */ 6 | 7 | /*! 8 | * The buffer module from node.js, for the browser. 9 | * 10 | * @author Feross Aboukhadijeh 11 | * @license MIT 12 | */ 13 | 14 | /** @license React v0.19.1 15 | * scheduler.production.min.js 16 | * 17 | * Copyright (c) Facebook, Inc. and its affiliates. 18 | * 19 | * This source code is licensed under the MIT license found in the 20 | * LICENSE file in the root directory of this source tree. 21 | */ 22 | 23 | /** @license React v16.13.1 24 | * react-is.production.min.js 25 | * 26 | * Copyright (c) Facebook, Inc. and its affiliates. 27 | * 28 | * This source code is licensed under the MIT license found in the 29 | * LICENSE file in the root directory of this source tree. 30 | */ 31 | 32 | /** @license React v16.14.0 33 | * react-dom.production.min.js 34 | * 35 | * Copyright (c) Facebook, Inc. and its affiliates. 36 | * 37 | * This source code is licensed under the MIT license found in the 38 | * LICENSE file in the root directory of this source tree. 39 | */ 40 | 41 | /** @license React v16.14.0 42 | * react-jsx-runtime.production.min.js 43 | * 44 | * Copyright (c) Facebook, Inc. and its affiliates. 45 | * 46 | * This source code is licensed under the MIT license found in the 47 | * LICENSE file in the root directory of this source tree. 48 | */ 49 | 50 | /** @license React v16.14.0 51 | * react.production.min.js 52 | * 53 | * Copyright (c) Facebook, Inc. and its affiliates. 54 | * 55 | * This source code is licensed under the MIT license found in the 56 | * LICENSE file in the root directory of this source tree. 57 | */ 58 | -------------------------------------------------------------------------------- /data_browser/format_csv.py: -------------------------------------------------------------------------------- 1 | def head_cell(field): 2 | return [" ".join(field.verbose_path)] 3 | 4 | 5 | def data_cell(value, span=1): 6 | return [value] + [""] * (span - 1) 7 | 8 | 9 | def v_table_head_row(fields): 10 | res = [] 11 | for field in fields: 12 | res += head_cell(field) 13 | return res 14 | 15 | 16 | def v_table_body_row(fields, row): 17 | res = [] 18 | for field in fields: 19 | if row: 20 | res += data_cell(row[field.path_str]) 21 | else: 22 | res += [""] 23 | return res 24 | 25 | 26 | def h_table_row(field, data, span): 27 | res = head_cell(field) 28 | for col in data: 29 | res += data_cell(col[field.path_str], span) 30 | return res 31 | 32 | 33 | def get_csv_rows(bound_query, results): 34 | col_fields = bound_query.col_fields 35 | top_title_space = len(bound_query.row_fields) - 1 36 | side_title_space = ( 37 | 1 - len(bound_query.row_fields) if len(bound_query.col_fields) else 0 38 | ) 39 | has_body = bound_query.row_fields or bound_query.body_fields 40 | 41 | # col headers and data aka pivots 42 | for field in col_fields: 43 | yield [""] * top_title_space + h_table_row( 44 | field=field, data=results["cols"], span=len(bound_query.body_fields) 45 | ) 46 | 47 | if has_body: 48 | # body/aggregate headers 49 | row = [""] * side_title_space 50 | row += v_table_head_row(bound_query.row_fields) 51 | for _ in results["cols"] or [None]: 52 | row += v_table_head_row(bound_query.body_fields) 53 | yield row 54 | 55 | # row headers and body 56 | for row_index, row_data in enumerate(results["rows"]): 57 | row = [""] * side_title_space 58 | row += v_table_body_row(bound_query.row_fields, row_data) 59 | for table in results["body"]: 60 | row += v_table_body_row(bound_query.body_fields, table[row_index]) 61 | yield row 62 | -------------------------------------------------------------------------------- /data_browser/helpers.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | from urllib.parse import urlencode 4 | 5 | from django.contrib.admin.options import BaseModelAdmin 6 | from django.core.serializers.json import DjangoJSONEncoder 7 | from django.db.models import BooleanField 8 | from django.urls import reverse 9 | 10 | from data_browser.common import settings 11 | 12 | 13 | def attributes(**kwargs): 14 | def inner(func): 15 | for k, v in kwargs.items(): 16 | setattr(func, k, v) 17 | return func 18 | 19 | return inner 20 | 21 | 22 | class _Everything: 23 | def __contains__(self, item): 24 | return True 25 | 26 | 27 | class _AdminOptions: 28 | ddb_ignore = False 29 | ddb_extra_fields = [] 30 | ddb_hide_fields = [] 31 | ddb_json_fields = {} 32 | ddb_default_filters = [] 33 | ddb_action_url = None 34 | 35 | def get_ddb_ignore(self, request): 36 | return self.ddb_ignore 37 | 38 | def get_ddb_extra_fields(self, request): 39 | return list(self.ddb_extra_fields) 40 | 41 | def get_ddb_hide_fields(self, request): 42 | return list(self.ddb_hide_fields) 43 | 44 | def get_ddb_json_fields(self, request): 45 | return dict(self.ddb_json_fields) 46 | 47 | def get_ddb_default_filters(self): 48 | return list(self.ddb_default_filters) 49 | 50 | def get_ddb_action_url(self, request): 51 | if self.ddb_action_url: 52 | return self.ddb_action_url # pragma: no cover 53 | 54 | meta = self.model._meta 55 | return reverse(f"admin:{meta.app_label}_{meta.model_name}_changelist") + "?" 56 | 57 | 58 | def _get_option(admin, name, *args): 59 | admin_name = f"{admin.__class__.__module__}.{admin.__class__.__qualname__}" 60 | defaults = settings.DATA_BROWSER_ADMIN_OPTIONS.get(admin_name, {}) 61 | 62 | field = f"ddb_{name}" 63 | func = f"get_ddb_{name}" 64 | 65 | if hasattr(admin, func): 66 | return getattr(admin, func)(*args) 67 | elif hasattr(admin, field): 68 | return getattr(admin, field) 69 | elif name in defaults: 70 | return defaults[name] 71 | else: 72 | options = _AdminOptions() 73 | options.model = getattr(admin, "model", None) 74 | return getattr(options, func)(*args) 75 | 76 | 77 | class _AdminAnnotations: 78 | def _get_fields_for_request(self, request): 79 | if hasattr(request, "data_browser"): 80 | return request.data_browser["fields"] 81 | elif ( 82 | request.resolver_match 83 | and request.resolver_match.func.__name__ == "changelist_view" 84 | ): 85 | return set(self.get_list_display(request)) 86 | else: 87 | return _Everything() 88 | 89 | def get_queryset(self, request): 90 | qs = super().get_queryset(request) 91 | fields = self._get_fields_for_request(request) 92 | 93 | for name, descriptor in self._ddb_annotations().items(): 94 | if name not in fields: 95 | continue 96 | 97 | qs = descriptor.get_queryset(self, request, qs) 98 | 99 | annotation = qs.query.annotations.get(descriptor.name) 100 | if not annotation: # pragma: no cover 101 | raise Exception( 102 | f"Can't find annotation '{descriptor.name}' for" 103 | f" {self}.{descriptor.name}" 104 | ) 105 | 106 | field_type = getattr(annotation, "output_field", None) 107 | if not field_type: # pragma: no cover 108 | raise Exception( 109 | f"Annotation '{descriptor.name}' for" 110 | f" {self}.{descriptor.name} doesn't specify 'output_field'" 111 | ) 112 | 113 | descriptor.boolean = isinstance(field_type, BooleanField) 114 | return qs 115 | 116 | def get_readonly_fields(self, request, obj=None): 117 | res = super().get_readonly_fields(request, obj) 118 | return list(res) + list(self._ddb_annotations()) 119 | 120 | @classmethod 121 | def _ddb_annotations(cls): 122 | res = {} 123 | for name in dir(cls): 124 | value = getattr(cls, name) 125 | if isinstance(value, _AnnotationDescriptor): 126 | res[name] = value 127 | return res 128 | 129 | 130 | # we need BaseModelAdmin in the inheritnce chain to ensure our mixins end up before it 131 | # in the MRO of the final class 132 | class AdminMixin(_AdminOptions, _AdminAnnotations, BaseModelAdmin): 133 | def changelist_view(self, request, extra_context=None): 134 | """Inject ddb_url""" 135 | extra_context = extra_context or {} 136 | 137 | if not self.get_ddb_ignore(request): 138 | url = reverse( 139 | "data_browser:query_html", 140 | args=[f"{self.model._meta.app_label}.{self.model.__name__}", ""], 141 | ) 142 | args = self.get_ddb_default_filters() 143 | params = urlencode([ 144 | ( 145 | f"{field}__{lookup}", 146 | ( 147 | value 148 | if isinstance(value, str) 149 | else json.dumps(value, cls=DjangoJSONEncoder) 150 | ), 151 | ) 152 | for field, lookup, value in args 153 | ]) 154 | extra_context["ddb_url"] = f"{url}?{params}" 155 | 156 | return super().changelist_view(request, extra_context) 157 | 158 | 159 | class _AnnotationDescriptor: 160 | def __init__(self, get_queryset): 161 | self.get_queryset = get_queryset 162 | 163 | def __set_name__(self, owner, name): 164 | self.name = name 165 | self.__name__ = name 166 | self.admin_order_field = name 167 | 168 | def __get__(self, instance, owner=None): 169 | if not issubclass(owner, AdminMixin): # pragma: no cover 170 | raise Exception( 171 | "Django Data Browser 'annotation' decorator used without 'AdminMixin'" 172 | ) 173 | return self 174 | 175 | def __call__(self, obj): 176 | return getattr(obj, self.name) 177 | 178 | def __getattr__(self, name): 179 | return getattr(self.get_queryset, name) 180 | 181 | 182 | def ddb_hide(func): # pragma: no cover 183 | logging.getLogger(__name__).warning( 184 | "ddb_hide is deprecated in favor of @attributes(ddb_hide=True)" 185 | ) 186 | func.ddb_hide = True 187 | return func 188 | 189 | 190 | annotation = _AnnotationDescriptor 191 | -------------------------------------------------------------------------------- /data_browser/migration_helpers.py: -------------------------------------------------------------------------------- 1 | from urllib.parse import parse_qsl 2 | from urllib.parse import urlencode 3 | 4 | from django.contrib.auth import get_user_model 5 | from django.test import RequestFactory 6 | from django.urls import reverse 7 | 8 | from data_browser.common import global_state 9 | from data_browser.common import set_global_state 10 | from data_browser.types import IsNullType 11 | from data_browser.types import NumberChoiceArrayType 12 | from data_browser.types import NumberChoiceType 13 | from data_browser.types import StringChoiceArrayType 14 | from data_browser.types import StringChoiceType 15 | 16 | 17 | def _fix_filter(models, field, parts, lookup, value): 18 | if lookup == "is_null": 19 | value = {"true": "IsNull", "false": "NotNull"}.get(value.lower(), value) 20 | elif field.type_ == IsNullType and lookup == "equals": 21 | value = {"true": "IsNull", "false": "NotNull"}.get(value.lower(), value) 22 | elif field.type_ in [StringChoiceType, NumberChoiceType]: 23 | if lookup in ["equals", "not_equals"]: 24 | value, err = field.type_.raw_type.parse_lookup(lookup, value, None) 25 | choices = dict(field.choices) 26 | if err is None and value in choices: 27 | value = choices[value] 28 | else: 29 | parts.append("raw") 30 | elif lookup == "is_null": 31 | assert False # should have been caught above 32 | else: 33 | parts.append("raw") 34 | elif field.type_ in [StringChoiceArrayType, NumberChoiceArrayType]: 35 | if lookup in ["contains", "not_contains"]: 36 | value, err = field.type_.raw_type.parse_lookup(lookup, value, None) 37 | choices = dict(field.choices) 38 | if err is None and value in choices: 39 | value = choices[value] 40 | else: 41 | parts.append("raw") 42 | else: 43 | pass 44 | 45 | return parts, lookup, value 46 | 47 | 48 | def forwards_0009(View): 49 | User = get_user_model() 50 | user = User(is_superuser=True) 51 | request = RequestFactory().get(reverse("admin:index")) 52 | with set_global_state(request=request, user=user, public_view=False): 53 | models = global_state.models 54 | 55 | for view in View.objects.all(): 56 | filters = [] 57 | for key, value in parse_qsl(view.query): 58 | *parts, lookup = key.split("__") 59 | 60 | model_name = view.model_name 61 | for part in parts: 62 | model = models[model_name] 63 | if part not in model.fields: 64 | break 65 | field = model.fields[part] 66 | model_name = field.rel_name 67 | else: 68 | parts, lookup, value = _fix_filter(models, field, parts, lookup, value) 69 | 70 | key = "__".join(parts + [lookup]) 71 | filters.append((key, value)) 72 | view.query = urlencode(filters) 73 | view.save() 74 | -------------------------------------------------------------------------------- /data_browser/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.0.13 on 2020-03-31 16:35 2 | 3 | import django.db.models.deletion 4 | import django.utils.timezone 5 | from django.conf import settings 6 | from django.db import migrations 7 | from django.db import models 8 | 9 | 10 | class Migration(migrations.Migration): 11 | initial = True 12 | 13 | dependencies = [migrations.swappable_dependency(settings.AUTH_USER_MODEL)] 14 | 15 | operations = [ 16 | migrations.CreateModel( 17 | name="View", 18 | fields=[ 19 | ( 20 | "id", 21 | models.AutoField( 22 | auto_created=True, 23 | primary_key=True, 24 | serialize=False, 25 | verbose_name="ID", 26 | ), 27 | ), 28 | ( 29 | "created_time", 30 | models.DateTimeField(default=django.utils.timezone.now), 31 | ), 32 | ("name", models.CharField(max_length=64)), 33 | ("description", models.TextField(blank=True)), 34 | ("public", models.BooleanField(default=False)), 35 | ("app", models.CharField(max_length=16)), 36 | ("model", models.CharField(max_length=32)), 37 | ("fields", models.TextField(blank=True)), 38 | ("query", models.TextField()), 39 | ( 40 | "owner", 41 | models.ForeignKey( 42 | blank=True, 43 | null=True, 44 | on_delete=django.db.models.deletion.SET_NULL, 45 | to=settings.AUTH_USER_MODEL, 46 | ), 47 | ), 48 | ], 49 | ) 50 | ] 51 | -------------------------------------------------------------------------------- /data_browser/migrations/0002_auto_20200331_1842.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.0.13 on 2020-03-31 17:42 2 | 3 | from django.db import migrations 4 | from django.db import models 5 | 6 | import data_browser.models 7 | 8 | 9 | class Migration(migrations.Migration): 10 | dependencies = [("data_browser", "0001_initial")] 11 | 12 | operations = [ 13 | migrations.AlterField( 14 | model_name="view", 15 | name="id", 16 | field=models.CharField( 17 | default=data_browser.models.get_id, 18 | max_length=12, 19 | primary_key=True, 20 | serialize=False, 21 | ), 22 | ) 23 | ] 24 | -------------------------------------------------------------------------------- /data_browser/migrations/0003_remove_view_app.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.0.13 on 2020-04-29 19:40 2 | 3 | from django.db import migrations 4 | 5 | 6 | class Migration(migrations.Migration): 7 | dependencies = [("data_browser", "0002_auto_20200331_1842")] 8 | 9 | operations = [migrations.RemoveField(model_name="view", name="app")] 10 | -------------------------------------------------------------------------------- /data_browser/migrations/0004_auto_20200501_0903.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.0.13 on 2020-05-01 08:03 2 | 3 | from django.db import migrations 4 | 5 | 6 | class Migration(migrations.Migration): 7 | dependencies = [("data_browser", "0003_remove_view_app")] 8 | 9 | operations = [ 10 | migrations.RenameField( 11 | model_name="view", old_name="model", new_name="model_name" 12 | ) 13 | ] 14 | -------------------------------------------------------------------------------- /data_browser/migrations/0005_auto_20200516_1726.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.0.13 on 2020-05-16 16:26 2 | 3 | from django.db import migrations 4 | 5 | 6 | class Migration(migrations.Migration): 7 | dependencies = [("data_browser", "0004_auto_20200501_0903")] 8 | 9 | operations = [ 10 | migrations.AlterModelOptions( 11 | name="view", 12 | options={ 13 | "permissions": [ 14 | ("make_view_public", "Can make a saved view publically available") 15 | ] 16 | }, 17 | ) 18 | ] 19 | -------------------------------------------------------------------------------- /data_browser/migrations/0006_auto_20200531_1450.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.0.13 on 2020-05-31 13:50 2 | 3 | from django.db import migrations 4 | from django.db import models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | dependencies = [("data_browser", "0005_auto_20200516_1726")] 9 | 10 | operations = [ 11 | migrations.AlterModelOptions( 12 | name="view", 13 | options={ 14 | "permissions": [ 15 | ("make_view_public", "Can make a saved view publicly available") 16 | ] 17 | }, 18 | ), 19 | migrations.AlterField( 20 | model_name="view", name="query", field=models.TextField(blank=True) 21 | ), 22 | ] 23 | -------------------------------------------------------------------------------- /data_browser/migrations/0007_view_public_slug.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.0.13 on 2020-06-14 10:37 2 | 3 | from django.db import migrations 4 | from django.db import models 5 | 6 | import data_browser.models 7 | 8 | 9 | class Migration(migrations.Migration): 10 | dependencies = [("data_browser", "0006_auto_20200531_1450")] 11 | 12 | operations = [ 13 | migrations.AddField( 14 | model_name="view", 15 | name="public_slug", 16 | field=models.CharField(default=data_browser.models.get_id, max_length=12), 17 | ) 18 | ] 19 | -------------------------------------------------------------------------------- /data_browser/migrations/0008_view_limit.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.0.7 on 2020-07-02 07:43 2 | 3 | from django.db import migrations 4 | from django.db import models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | dependencies = [("data_browser", "0007_view_public_slug")] 9 | 10 | operations = [ 11 | migrations.AddField( 12 | model_name="view", name="limit", field=models.IntegerField(default=1000) 13 | ) 14 | ] 15 | -------------------------------------------------------------------------------- /data_browser/migrations/0009_migrate_saved_views.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.2.16 on 2020-10-26 20:25 2 | 3 | from django.db import migrations 4 | 5 | 6 | def forwards(apps, schema_editor): 7 | View = apps.get_model("data_browser", "View") 8 | if View.objects.exists(): 9 | from data_browser.migration_helpers import forwards_0009 10 | 11 | forwards_0009(View) 12 | 13 | 14 | class Migration(migrations.Migration): 15 | dependencies = [("data_browser", "0008_view_limit")] 16 | operations = [migrations.RunPython(forwards, migrations.RunPython.noop)] 17 | -------------------------------------------------------------------------------- /data_browser/migrations/0010_shared.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.18 on 2023-04-11 07:09 2 | 3 | from django.db import migrations 4 | from django.db import models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | dependencies = [("data_browser", "0009_migrate_saved_views")] 9 | 10 | operations = [ 11 | migrations.AddField( 12 | model_name="view", name="shared", field=models.BooleanField(default=False) 13 | ) 14 | ] 15 | -------------------------------------------------------------------------------- /data_browser/migrations/0011_folder.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.18 on 2023-04-16 11:00 2 | 3 | from django.db import migrations 4 | from django.db import models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | dependencies = [("data_browser", "0010_shared")] 9 | 10 | operations = [ 11 | migrations.AddField( 12 | model_name="view", 13 | name="folder", 14 | field=models.CharField(blank=True, max_length=64), 15 | ), 16 | migrations.AlterField( 17 | model_name="view", name="model_name", field=models.CharField(max_length=64) 18 | ), 19 | ] 20 | -------------------------------------------------------------------------------- /data_browser/migrations/0012_can_share.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.18 on 2023-04-17 18:20 2 | 3 | from django.db import migrations 4 | 5 | 6 | class Migration(migrations.Migration): 7 | dependencies = [("data_browser", "0011_folder")] 8 | 9 | operations = [ 10 | migrations.AlterModelOptions( 11 | name="view", 12 | options={ 13 | "permissions": [ 14 | ("make_view_public", "Can make a saved view publicly available"), 15 | ("share_view", "Can share a saved view with other users"), 16 | ] 17 | }, 18 | ) 19 | ] 20 | -------------------------------------------------------------------------------- /data_browser/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tolomea/django-data-browser/317d7f0b8c366c6281c973d87e792c6322469625/data_browser/migrations/__init__.py -------------------------------------------------------------------------------- /data_browser/model.txt: -------------------------------------------------------------------------------- 1 | url path characters 2 | unreserved 0-9a-zA-Z -._~ 3 | sub delims !$&'()*+,;= 4 | : @ %XX 5 | used ,+- 6 | 7 | 8 | sql 9 | select 10 | from 11 | where 12 | group 13 | having 14 | order 15 | 16 | django 17 | lookup annotate 18 | regular filters 19 | values (optional) 20 | distinct 21 | aggregate annotate 22 | aggregate filters 23 | sort 24 | get results (optional) 25 | preload 26 | get results 27 | 28 | 29 | rel_name -> expand (model_name on the api and FE) 30 | type_ -> add 31 | concrete -> filter, sort 32 | pivot -> pivot 33 | 34 | 35 | 36 | expand add filter sort pivot color color is based on pivot type, brightness on concrete/calc 37 | fk rel - - - - Blue 38 | 2m rel/agg - - - - Blue not implemented yet, agg only the fields 39 | normal agg + + + yes Green 40 | function -? + + + yes Green we could aggregate functions 41 | calculated - + - - yes Dark Green does this prevent aggregation? 42 | aggregate - + + + data Red 43 | 44 | 45 | 46 | asserts 47 | 1, !type -> rel 48 | 2, concrete -> type 49 | 3, pivot -> concrete 50 | 51 | r t c p 52 | 0 0 0 0 1 53 | 0 0 0 1 3 1 54 | 0 0 1 0 2 1 55 | 0 0 1 1 2 1 56 | 0 1 0 0 red calc 57 | 0 1 0 1 3 58 | 0 1 1 0 yellow agg 59 | 0 1 1 1 green norm/func 60 | 61 | 1 0 0 0 blue fk 62 | 1 0 0 1 3 63 | 1 0 1 0 2 64 | 1 0 1 1 2 65 | 1 1 0 0 red calc 66 | 1 1 0 1 3 67 | 1 1 1 0 yellow agg 68 | 1 1 1 1 green norm / func 69 | -------------------------------------------------------------------------------- /data_browser/models.py: -------------------------------------------------------------------------------- 1 | import hyperlink 2 | from django.db import models 3 | from django.urls import reverse 4 | from django.utils import crypto 5 | from django.utils import timezone 6 | 7 | from data_browser.common import PUBLIC_PERM 8 | from data_browser.common import SHARE_PERM 9 | from data_browser.common import global_state 10 | from data_browser.common import has_permission 11 | from data_browser.common import set_global_state 12 | from data_browser.common import settings 13 | 14 | 15 | def get_id(): 16 | return crypto.get_random_string(length=12) 17 | 18 | 19 | class View(models.Model): 20 | class Meta: 21 | permissions = [ 22 | (PUBLIC_PERM, "Can make a saved view publicly available"), 23 | (SHARE_PERM, "Can share a saved view with other users"), 24 | ] 25 | 26 | id = models.CharField(primary_key=True, max_length=12, default=get_id) 27 | created_time = models.DateTimeField(default=timezone.now) 28 | 29 | name = models.CharField(max_length=64, blank=False) 30 | description = models.TextField(blank=True) 31 | folder = models.CharField(max_length=64, blank=True) 32 | owner = models.ForeignKey( 33 | settings.AUTH_USER_MODEL, null=True, blank=True, on_delete=models.SET_NULL 34 | ) 35 | 36 | public = models.BooleanField(default=False) 37 | public_slug = models.CharField(max_length=12, default=get_id, blank=False) 38 | shared = models.BooleanField(default=False) 39 | 40 | model_name = models.CharField(max_length=64, blank=False) 41 | fields = models.TextField(blank=True) 42 | query = models.TextField(blank=True) 43 | limit = models.IntegerField(blank=False, null=False, default=1000) 44 | 45 | def get_query(self): 46 | from data_browser.query import Query 47 | 48 | params = list(hyperlink.parse(f"?{self.query}").query) 49 | params.append(("limit", str(self.limit))) 50 | return Query.from_request(self.model_name, self.fields, params) 51 | 52 | @property 53 | def url(self): 54 | return self.get_query().get_full_url("html") 55 | 56 | def _public_url(self, fmt): 57 | with set_global_state(user=self.owner, public_view=True): 58 | if not (has_permission(self.owner, PUBLIC_PERM) and self.public): 59 | return "N/A" 60 | 61 | if not settings.DATA_BROWSER_ALLOW_PUBLIC: 62 | return "Public Views are disabled in Django settings." 63 | 64 | if not self.is_valid(): 65 | return "View is invalid" 66 | 67 | url = reverse( 68 | "data_browser:view", kwargs={"pk": self.public_slug, "media": "csv"} 69 | ) 70 | url = global_state.request.build_absolute_uri(url) 71 | 72 | return fmt.format(url=url) 73 | 74 | def public_link(self): 75 | return self._public_url("{url}") 76 | 77 | def google_sheets_formula(self): 78 | return self._public_url('=importdata("{url}")') 79 | 80 | def __str__(self): 81 | return f"{self.model_name} view: {self.name}" 82 | 83 | def is_valid(self): 84 | return self.get_query().is_valid(global_state.models) 85 | -------------------------------------------------------------------------------- /data_browser/orm_aggregates.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | 3 | from django.conf import settings 4 | from django.db import models 5 | from django.db.models import DurationField 6 | from django.db.models import IntegerField 7 | from django.db.models import Value 8 | from django.db.models.functions import Cast 9 | 10 | from data_browser.orm_fields import OrmBaseField 11 | from data_browser.orm_fields import OrmBoundField 12 | from data_browser.types import ARRAY_TYPES 13 | from data_browser.types import TYPES 14 | from data_browser.types import BaseType 15 | from data_browser.types import BooleanType 16 | from data_browser.types import DateTimeType 17 | from data_browser.types import DateType 18 | from data_browser.types import DurationType 19 | from data_browser.types import NumberType 20 | from data_browser.util import annotation_path 21 | 22 | try: 23 | from django.contrib.postgres.aggregates import ArrayAgg 24 | except ModuleNotFoundError: # pragma: no cover 25 | ArrayAgg = None 26 | 27 | 28 | class OrmAggregateField(OrmBaseField): 29 | def __init__(self, base_type, name, agg): 30 | super().__init__( 31 | base_type.name, name, name.replace("_", " "), type_=agg.type_, concrete=True 32 | ) 33 | self.base_type = base_type 34 | self.func = agg.func 35 | 36 | def bind(self, previous): 37 | assert previous 38 | assert previous.type_ == self.base_type 39 | full_path = previous.full_path + [self.name] 40 | return OrmBoundField( 41 | field=self, 42 | previous=previous, 43 | full_path=full_path, 44 | verbose_path=previous.verbose_path + [self.verbose_name], 45 | queryset_path=[annotation_path(full_path)], 46 | aggregate_clause=self.func(previous.queryset_path_str), 47 | having=True, 48 | ) 49 | 50 | 51 | class _CastDuration(Cast): 52 | def __init__(self, expression): 53 | super().__init__(expression, output_field=DurationField()) 54 | 55 | def as_mysql(self, compiler, connection, **extra_context): 56 | # https://github.com/django/django/pull/13398 57 | template = "%(function)s(%(expressions)s AS signed integer)" 58 | return self.as_sql(compiler, connection, template=template, **extra_context) 59 | 60 | 61 | @dataclass 62 | class Agg: 63 | func: callable 64 | type_: BaseType 65 | 66 | 67 | TYPE_AGGREGATES = {type_: {} for type_ in TYPES.values()} 68 | 69 | for type_ in TYPES.values(): 70 | if type_ != BooleanType: 71 | TYPE_AGGREGATES[type_]["count"] = Agg( 72 | lambda x: models.Count(x, distinct=True), NumberType 73 | ) 74 | 75 | for type_ in [DateTimeType, DateType, DurationType, NumberType]: 76 | TYPE_AGGREGATES[type_]["max"] = Agg(models.Max, type_) 77 | TYPE_AGGREGATES[type_]["min"] = Agg(models.Min, type_) 78 | 79 | TYPE_AGGREGATES[NumberType]["average"] = Agg(models.Avg, NumberType) 80 | TYPE_AGGREGATES[NumberType]["std_dev"] = Agg(models.StdDev, NumberType) 81 | TYPE_AGGREGATES[NumberType]["sum"] = Agg(models.Sum, NumberType) 82 | TYPE_AGGREGATES[NumberType]["variance"] = Agg(models.Variance, NumberType) 83 | 84 | TYPE_AGGREGATES[DurationType]["average"] = Agg( 85 | lambda x: models.Avg(_CastDuration(x)), DurationType 86 | ) 87 | TYPE_AGGREGATES[DurationType]["sum"] = Agg( 88 | lambda x: models.Sum(_CastDuration(x)), DurationType 89 | ) 90 | 91 | TYPE_AGGREGATES[BooleanType]["average"] = Agg( 92 | lambda x: models.Avg(Cast(x, output_field=IntegerField())), NumberType 93 | ) 94 | TYPE_AGGREGATES[BooleanType]["sum"] = Agg( 95 | lambda x: models.Sum(Cast(x, output_field=IntegerField())), NumberType 96 | ) 97 | 98 | if "postgresql" in settings.DATABASES["default"]["ENGINE"]: 99 | for array_type in ARRAY_TYPES.values(): 100 | if array_type.raw_type is None: 101 | TYPE_AGGREGATES[array_type.element_type]["all"] = Agg( 102 | lambda x: ArrayAgg(x, default=Value([]), distinct=True, ordering=x), 103 | array_type, 104 | ) 105 | 106 | 107 | def get_aggregates_for_type(type_): 108 | return { 109 | name: OrmAggregateField(type_, name, agg) 110 | for name, agg in TYPE_AGGREGATES[type_].items() 111 | } 112 | -------------------------------------------------------------------------------- /data_browser/orm_debug.py: -------------------------------------------------------------------------------- 1 | import contextlib 2 | from textwrap import dedent 3 | from textwrap import indent 4 | 5 | from django.db.models import ExpressionWrapper 6 | from django.db.models import F 7 | from django.db.models import Field 8 | from django.db.models import Q 9 | 10 | SPACES = " " 11 | 12 | 13 | def _format_value(value): 14 | if isinstance(value, Q): 15 | 16 | def format_child(child): 17 | if isinstance(child, tuple): 18 | return f"{child[0]}={_format_value(child[1])}" 19 | else: 20 | return _format_value(child) 21 | 22 | if len(value.children) == 1 and isinstance(value.children[0], Q): 23 | child = _format_value(value.children[0]) 24 | return f"~Q({child})" if value.negated else child 25 | 26 | if value.connector == Q.AND: 27 | children = ", ".join(format_child(c) for c in value.children) 28 | return f"~Q({children})" if value.negated else f"Q({children})" 29 | else: 30 | children = " | ".join(f"Q({format_child(c)})" for c in value.children) 31 | return f"~({children})" if value.negated else children 32 | elif isinstance(value, Field): 33 | return f"{value.__class__.__name__}()" 34 | elif isinstance(value, ExpressionWrapper): 35 | expression = _format_value(value.expression) 36 | output_field = _format_value(value.output_field) 37 | return dedent( 38 | "ExpressionWrapper(\n" 39 | f"{indent(expression, SPACES)},\n" 40 | f"{SPACES}output_field={output_field},\n)" 41 | ) 42 | else: 43 | return repr(value) 44 | 45 | 46 | @contextlib.contextmanager 47 | def _better_reprs(): 48 | F_orig = F.__repr__ 49 | F.__repr__ = lambda s: f"F({s.name!r})" 50 | 51 | yield 52 | 53 | F.__repr__ = F_orig 54 | 55 | 56 | class DebugQS: 57 | def __init__(self, s): 58 | self.s = s 59 | 60 | def __getattr__(self, name): 61 | return DebugQS(f"{self.s}.{name}") 62 | 63 | def __call__(self, *args, **kwargs): 64 | if args or kwargs: 65 | with _better_reprs(): 66 | flat_args = ",\n".join( 67 | [_format_value(a) for a in args] 68 | + [f"{n}={_format_value(v)}" for n, v in kwargs.items()] 69 | ) 70 | return DebugQS(f"{self.s}(\n{indent(flat_args, SPACES)},\n)") 71 | else: 72 | return DebugQS(f"{self.s}()") 73 | 74 | def __getitem__(self, index): 75 | if isinstance(index, slice): 76 | assert index.start is None 77 | assert index.step is None 78 | return DebugQS(f"{self.s}[: {index.stop}]") 79 | else: # pragma: no cover 80 | return DebugQS(f"{self.s}[index]") 81 | 82 | def __str__(self): 83 | return self.s 84 | 85 | def __repr__(self): 86 | return self.s 87 | -------------------------------------------------------------------------------- /data_browser/orm_fields.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import Sequence 3 | from typing import Tuple 4 | 5 | from django.db import models 6 | from django.db.models import OuterRef 7 | from django.db.models import Subquery 8 | 9 | from data_browser.common import global_state 10 | from data_browser.orm_debug import DebugQS 11 | from data_browser.types import ASC 12 | from data_browser.types import BaseType 13 | from data_browser.types import BooleanType 14 | from data_browser.types import DateTimeType 15 | from data_browser.types import DateType 16 | from data_browser.types import HTMLType 17 | from data_browser.types import UnknownType 18 | from data_browser.types import URLType 19 | from data_browser.util import annotation_path 20 | 21 | 22 | @dataclass 23 | class OrmBoundField: 24 | field: "OrmBaseField" 25 | previous: "OrmBoundField" 26 | full_path: Sequence[str] 27 | verbose_path: Sequence[str] 28 | queryset_path: Sequence[str] 29 | aggregate_clause: Tuple[str, models.Func] = None 30 | filter_: bool = False 31 | having: bool = False 32 | model_name: str = None 33 | 34 | @property 35 | def path_str(self): 36 | return "__".join(self.full_path) 37 | 38 | @property 39 | def queryset_path_str(self): 40 | return "__".join(self.queryset_path) 41 | 42 | @property 43 | def group_by(self): 44 | return self.field.can_pivot 45 | 46 | def _lineage(self): 47 | if self.previous: 48 | return self.previous._lineage() + [self] 49 | return [self] 50 | 51 | def annotate(self, qs, debug=False): 52 | for field in self._lineage(): 53 | qs = field._annotate(qs, debug=debug) 54 | return qs 55 | 56 | def _annotate(self, qs, debug=False): 57 | return qs 58 | 59 | def _annotate_qs(self, qs, expression): 60 | assert "__" not in self.queryset_path_str 61 | return qs.annotate(**{self.queryset_path_str: expression}) 62 | 63 | def __getattr__(self, name): 64 | return getattr(self.field, name) 65 | 66 | def parse_lookup(self, lookup, value): 67 | return self.type_.parse_lookup(lookup, value, self.choices) 68 | 69 | def format_lookup(self, lookup, value): 70 | return self.type_.format_lookup(lookup, value, self.choices) 71 | 72 | @classmethod 73 | def blank(cls): 74 | return cls( 75 | field=None, previous=None, full_path=[], verbose_path=[], queryset_path=[] 76 | ) 77 | 78 | def get_format_hints(self, data): 79 | hints = self.type_.get_format_hints(self.path_str, data) 80 | return {**hints, **(self.format_hints or {})} 81 | 82 | 83 | @dataclass 84 | class OrmBaseField: 85 | model_name: str 86 | name: str 87 | verbose_name: str 88 | type_: BaseType = None 89 | concrete: bool = False 90 | rel_name: str = None 91 | can_pivot: bool = False 92 | choices: Sequence[Tuple[str, str]] = () 93 | default_sort: str = None 94 | format_hints: dict = None 95 | actions: dict = None 96 | to_many: bool = False 97 | real: bool = False 98 | 99 | def __post_init__(self): 100 | if not self.type_: 101 | assert self.rel_name 102 | if self.concrete: 103 | assert self.type_ 104 | # ideally all concrete fields would be equals filterable 105 | assert "equals" in self.type_.lookups or self.type_ in { 106 | UnknownType, 107 | HTMLType, 108 | }, (self.model_name, self.name, self.type_) 109 | if self.can_pivot: 110 | assert self.type_ 111 | 112 | def get_formatter(self): 113 | return self.type_.get_formatter(self.choices) 114 | 115 | 116 | class OrmFkField(OrmBaseField): 117 | def __init__(self, model_name, name, verbose_name, rel_name, to_many): 118 | super().__init__( 119 | model_name, name, verbose_name, rel_name=rel_name, to_many=to_many 120 | ) 121 | 122 | def bind(self, previous): 123 | previous = previous or OrmBoundField.blank() 124 | return OrmBoundField( 125 | field=self, 126 | previous=previous, 127 | full_path=previous.full_path + [self.name], 128 | verbose_path=previous.verbose_path + [self.verbose_name], 129 | queryset_path=previous.queryset_path + [self.name], 130 | ) 131 | 132 | 133 | class OrmRealField(OrmBaseField): 134 | def __init__( 135 | self, model_name, name, verbose_name, type_, rel_name, choices, actions=None 136 | ): 137 | super().__init__( 138 | model_name, 139 | name, 140 | verbose_name, 141 | concrete=True, 142 | type_=type_, 143 | rel_name=rel_name, 144 | can_pivot=True, 145 | choices=choices or (), 146 | default_sort=ASC if type_ in [DateType, DateTimeType] else None, 147 | actions=actions, 148 | real=True, 149 | ) 150 | 151 | def bind(self, previous): 152 | previous = previous or OrmBoundField.blank() 153 | return OrmBoundField( 154 | field=self, 155 | previous=previous, 156 | full_path=previous.full_path + [self.name], 157 | verbose_path=previous.verbose_path + [self.verbose_name], 158 | queryset_path=previous.queryset_path + [self.name], 159 | filter_=True, 160 | ) 161 | 162 | 163 | class OrmRawField(OrmRealField): 164 | def bind(self, previous): 165 | return OrmBoundField( 166 | field=self, 167 | previous=previous, 168 | full_path=previous.full_path + [self.name], 169 | verbose_path=previous.verbose_path + [self.verbose_name], 170 | queryset_path=previous.queryset_path, 171 | filter_=True, 172 | ) 173 | 174 | 175 | class OrmCalculatedField(OrmBaseField): 176 | def __init__(self, model_name, name, verbose_name, func, actions=None): 177 | if getattr(func, "boolean", False): 178 | type_ = BooleanType 179 | else: 180 | type_ = HTMLType 181 | 182 | super().__init__( 183 | model_name, name, verbose_name, type_=type_, can_pivot=True, actions=actions 184 | ) 185 | self.func = func 186 | 187 | def bind(self, previous): 188 | previous = previous or OrmBoundField.blank() 189 | return OrmBoundField( 190 | field=self, 191 | previous=previous, 192 | full_path=previous.full_path + [self.name], 193 | verbose_path=previous.verbose_path + [self.verbose_name], 194 | queryset_path=previous.queryset_path + ["pk"], 195 | model_name=self.model_name, 196 | ) 197 | 198 | def get_formatter(self): 199 | base_formatter = super().get_formatter() 200 | 201 | def format(obj): 202 | if obj is None: 203 | return None 204 | 205 | try: 206 | value = self.func(obj) 207 | except Exception as e: 208 | return str(e) 209 | 210 | return base_formatter(value) 211 | 212 | return format 213 | 214 | 215 | class OrmBoundAnnotatedField(OrmBoundField): 216 | def _annotate(self, qs, debug=False): 217 | from data_browser.orm_admin import admin_get_queryset 218 | 219 | if debug: 220 | subquery = DebugQS("Subquery") 221 | outer_ref = DebugQS("OuterRef") 222 | else: 223 | subquery = Subquery 224 | outer_ref = OuterRef 225 | 226 | return self._annotate_qs( 227 | qs, 228 | subquery( 229 | admin_get_queryset( 230 | global_state.request, self.admin, [self.name], debug=debug 231 | ) 232 | .filter(pk=outer_ref("__".join(self.previous.queryset_path + ["pk"]))) 233 | .values(self.name)[:1], 234 | output_field=self.django_field, 235 | ), 236 | ) 237 | 238 | 239 | class OrmAnnotatedField(OrmBaseField): 240 | def __init__( 241 | self, model_name, name, verbose_name, type_, django_field, admin, choices 242 | ): 243 | super().__init__( 244 | model_name, 245 | name, 246 | verbose_name, 247 | type_=type_, 248 | rel_name=type_.name, 249 | can_pivot=True, 250 | concrete=True, 251 | choices=choices or (), 252 | ) 253 | self.django_field = django_field 254 | self.admin = admin 255 | 256 | def bind(self, previous): 257 | previous = previous or OrmBoundField.blank() 258 | 259 | full_path = previous.full_path + [self.name] 260 | return OrmBoundAnnotatedField( 261 | field=self, 262 | previous=previous, 263 | full_path=full_path, 264 | verbose_path=previous.verbose_path + [self.verbose_name], 265 | queryset_path=[annotation_path(full_path)], 266 | filter_=True, 267 | ) 268 | 269 | 270 | class OrmFileField(OrmRealField): 271 | def __init__(self, model_name, name, verbose_name, django_field): 272 | super().__init__( 273 | model_name, 274 | name, 275 | verbose_name, 276 | type_=URLType, 277 | rel_name=URLType.name, 278 | choices=None, 279 | ) 280 | self.django_field = django_field 281 | 282 | def get_formatter(self): 283 | def format(value): 284 | if not value: 285 | return value 286 | 287 | try: 288 | # some storage backends will hard fail if their underlying storage isn't 289 | # setup right https://github.com/tolomea/django-data-browser/issues/11 290 | return self.django_field.storage.url(value) 291 | except Exception as e: 292 | return str(e) 293 | 294 | return format 295 | -------------------------------------------------------------------------------- /data_browser/orm_functions.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from dataclasses import field 3 | from typing import Optional 4 | 5 | from django.db.models import BooleanField 6 | from django.db.models import DateField 7 | from django.db.models import ExpressionWrapper 8 | from django.db.models import Q 9 | from django.db.models import functions 10 | 11 | from data_browser.orm_fields import OrmBaseField 12 | from data_browser.orm_fields import OrmBoundField 13 | from data_browser.types import ARRAY_TYPES 14 | from data_browser.types import ASC 15 | from data_browser.types import TYPES 16 | from data_browser.types import BaseType 17 | from data_browser.types import DateTimeType 18 | from data_browser.types import DateType 19 | from data_browser.types import IsNullType 20 | from data_browser.types import NumberChoiceType 21 | from data_browser.types import NumberType 22 | from data_browser.types import StringType 23 | from data_browser.util import annotation_path 24 | 25 | try: 26 | from django.contrib.postgres.fields.array import ArrayLenTransform 27 | except ModuleNotFoundError: # pragma: no cover 28 | ArrayLenTransform = None 29 | 30 | 31 | class OrmBoundFunctionField(OrmBoundField): 32 | def __init__(self, *args, func, **kwargs): 33 | super().__init__(*args, **kwargs) 34 | self.func = func 35 | 36 | def _annotate(self, qs, debug=False): 37 | return self._annotate_qs(qs, self.func(self.previous.queryset_path_str)) 38 | 39 | def parse_lookup(self, lookup, value): 40 | parsed, error_message = super().parse_lookup(lookup, value) 41 | if ( 42 | self.name in ["year", "iso_year"] 43 | and parsed is not None 44 | and lookup != "is_null" 45 | ): 46 | if parsed < 2: 47 | error_message = "Can't filter to years less than 2" 48 | if parsed > 9998: 49 | error_message = "Can't filter to years greater than 9998" 50 | return parsed, error_message 51 | 52 | 53 | class OrmFunctionField(OrmBaseField): 54 | def __init__(self, base_type, name, func): 55 | super().__init__( 56 | base_type.name, 57 | name, 58 | name.replace("_", " "), 59 | type_=func.type_, 60 | concrete=True, 61 | can_pivot=True, 62 | choices=func.choices, 63 | default_sort=func.default_sort, 64 | format_hints=func.format_hints, 65 | ) 66 | self.base_type = base_type 67 | self.func = func.func 68 | 69 | def bind(self, previous): 70 | assert previous 71 | assert previous.type_ == self.base_type 72 | full_path = previous.full_path + [self.name] 73 | return OrmBoundFunctionField( 74 | field=self, 75 | previous=previous, 76 | full_path=full_path, 77 | verbose_path=previous.verbose_path + [self.verbose_name], 78 | queryset_path=[annotation_path(full_path)], 79 | filter_=True, 80 | func=self.func, 81 | ) 82 | 83 | 84 | def IsNull(field_name): 85 | return ExpressionWrapper(Q(**{field_name: None}), output_field=BooleanField()) 86 | 87 | 88 | _month_choices = [ 89 | (1, "January"), 90 | (2, "February"), 91 | (3, "March"), 92 | (4, "April"), 93 | (5, "May"), 94 | (6, "June"), 95 | (7, "July"), 96 | (8, "August"), 97 | (9, "September"), 98 | (10, "October"), 99 | (11, "November"), 100 | (12, "December"), 101 | ] 102 | 103 | 104 | _weekday_choices = [ 105 | (1, "Sunday"), 106 | (2, "Monday"), 107 | (3, "Tuesday"), 108 | (4, "Wednesday"), 109 | (5, "Thursday"), 110 | (6, "Friday"), 111 | (7, "Saturday"), 112 | ] 113 | 114 | 115 | @dataclass 116 | class Func: 117 | func: callable 118 | type_: BaseType 119 | choices: tuple = () 120 | default_sort: Optional[str] = ASC 121 | format_hints: dict = field(default_factory=dict) 122 | 123 | 124 | TYPE_FUNCTIONS = {type_: {} for type_ in TYPES.values()} 125 | 126 | for type_ in TYPES.values(): 127 | TYPE_FUNCTIONS[type_]["is_null"] = Func(IsNull, IsNullType, default_sort=None) 128 | 129 | for array_type in ARRAY_TYPES.values(): 130 | TYPE_FUNCTIONS[array_type]["length"] = Func( 131 | ArrayLenTransform, NumberType, default_sort=None 132 | ) 133 | 134 | for type_ in [DateType, DateTimeType]: 135 | TYPE_FUNCTIONS[type_]["year"] = Func( 136 | functions.ExtractYear, NumberType, format_hints={"useGrouping": False} 137 | ) 138 | TYPE_FUNCTIONS[type_]["quarter"] = Func(functions.ExtractQuarter, NumberType) 139 | TYPE_FUNCTIONS[type_]["month"] = Func( 140 | functions.ExtractMonth, NumberChoiceType, choices=_month_choices 141 | ) 142 | TYPE_FUNCTIONS[type_]["day"] = Func(functions.ExtractDay, NumberType) 143 | TYPE_FUNCTIONS[type_]["week_day"] = Func( 144 | functions.ExtractWeekDay, NumberChoiceType, choices=_weekday_choices 145 | ) 146 | TYPE_FUNCTIONS[type_]["month_start"] = Func( 147 | lambda x: functions.TruncMonth(x, DateField()), DateType 148 | ) 149 | TYPE_FUNCTIONS[type_]["iso_year"] = Func(functions.ExtractIsoYear, NumberType) 150 | TYPE_FUNCTIONS[type_]["iso_week"] = Func(functions.ExtractWeek, NumberType) 151 | TYPE_FUNCTIONS[type_]["week_start"] = Func( 152 | lambda x: functions.TruncWeek(x, DateField()), DateType 153 | ) 154 | 155 | TYPE_FUNCTIONS[DateTimeType]["hour"] = Func(functions.ExtractHour, NumberType) 156 | TYPE_FUNCTIONS[DateTimeType]["minute"] = Func(functions.ExtractMinute, NumberType) 157 | TYPE_FUNCTIONS[DateTimeType]["second"] = Func(functions.ExtractSecond, NumberType) 158 | TYPE_FUNCTIONS[DateTimeType]["date"] = Func(functions.TruncDate, DateType) 159 | 160 | TYPE_FUNCTIONS[StringType]["length"] = Func( 161 | functions.Length, NumberType, default_sort=None 162 | ) 163 | 164 | 165 | def get_functions_for_type(type_): 166 | return { 167 | name: OrmFunctionField(type_, name, func) 168 | for name, func in TYPE_FUNCTIONS[type_].items() 169 | } 170 | -------------------------------------------------------------------------------- /data_browser/orm_lookups.py: -------------------------------------------------------------------------------- 1 | from django.db.models import Q 2 | 3 | from data_browser.types import ArrayTypeMixin 4 | from data_browser.types import IsNullType 5 | from data_browser.types import StringType 6 | 7 | 8 | def get_django_filter(field_type, path_str, lookup, filter_value): 9 | # because on JsonFields Q(field=None) != Q(field__isnull=True) 10 | if lookup == "is_null": 11 | if filter_value is True: 12 | return Q(**{path_str: None}) | Q(**{f"{path_str}__isnull": True}) 13 | elif filter_value is False: 14 | return ~Q(**{path_str: None}) 15 | else: 16 | assert False 17 | 18 | if field_type == IsNullType: 19 | assert lookup == "equals", lookup 20 | if filter_value is True: 21 | return Q(**{path_str: None}) | Q(**{path_str: True}) 22 | elif filter_value is False: 23 | return Q(**{path_str: False}) 24 | else: 25 | assert False 26 | 27 | if lookup == "field_equals": 28 | lookup, filter_value = filter_value 29 | elif issubclass(field_type, StringType): 30 | lookup = { 31 | "equals": "iexact", 32 | "regex": "iregex", 33 | "contains": "icontains", 34 | "starts_with": "istartswith", 35 | "ends_with": "iendswith", 36 | "is_null": "isnull", 37 | }[lookup] 38 | elif issubclass(field_type, ArrayTypeMixin) and lookup == "contains": 39 | # django expects the contains value for a list field to be a list 40 | filter_value = [filter_value] 41 | else: 42 | lookup = { 43 | "equals": "exact", 44 | "is_null": "isnull", 45 | "gt": "gt", 46 | "gte": "gte", 47 | "lt": "lt", 48 | "lte": "lte", 49 | "contains": "contains", 50 | "length": "len", 51 | "has_key": "has_key", 52 | }[lookup] 53 | 54 | return Q(**{f"{path_str}__{lookup}": filter_value, f"{path_str}__isnull": False}) 55 | -------------------------------------------------------------------------------- /data_browser/orm_types.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | from django.db.models import JSONField 3 | 4 | from data_browser.common import debug_log 5 | from data_browser.types import BooleanType 6 | from data_browser.types import DateTimeType 7 | from data_browser.types import DateType 8 | from data_browser.types import DurationType 9 | from data_browser.types import JSONType 10 | from data_browser.types import NumberArrayType 11 | from data_browser.types import NumberChoiceArrayType 12 | from data_browser.types import NumberChoiceType 13 | from data_browser.types import NumberType 14 | from data_browser.types import StringableType 15 | from data_browser.types import StringArrayType 16 | from data_browser.types import StringChoiceArrayType 17 | from data_browser.types import StringChoiceType 18 | from data_browser.types import StringType 19 | from data_browser.types import UnknownType 20 | from data_browser.types import URLType 21 | from data_browser.types import UUIDType 22 | 23 | try: 24 | from django.contrib.postgres.fields import ArrayField 25 | except ModuleNotFoundError: # pragma: no cover 26 | ArrayField = None.__class__ 27 | 28 | 29 | _STRING_FIELDS = (models.CharField, models.TextField, models.GenericIPAddressField) 30 | _NUMBER_FIELDS = ( 31 | models.DecimalField, 32 | models.FloatField, 33 | models.IntegerField, 34 | models.AutoField, 35 | ) 36 | _FIELD_TYPE_MAP = { 37 | models.BooleanField: BooleanType, 38 | models.DurationField: DurationType, 39 | models.NullBooleanField: BooleanType, 40 | models.DateTimeField: DateTimeType, 41 | models.DateField: DateType, 42 | models.UUIDField: UUIDType, 43 | models.URLField: URLType, 44 | **{f: StringType for f in _STRING_FIELDS}, 45 | **{f: NumberType for f in _NUMBER_FIELDS}, 46 | } 47 | 48 | 49 | # hashid support 50 | try: # pragma: no cover 51 | from hashid_field import BigHashidAutoField 52 | from hashid_field import BigHashidField 53 | from hashid_field import HashidAutoField 54 | from hashid_field import HashidField 55 | except ModuleNotFoundError: 56 | pass 57 | else: # pragma: no cover 58 | _FIELD_TYPE_MAP[BigHashidAutoField] = StringableType 59 | _FIELD_TYPE_MAP[BigHashidField] = StringableType 60 | _FIELD_TYPE_MAP[HashidAutoField] = StringableType 61 | _FIELD_TYPE_MAP[HashidField] = StringableType 62 | 63 | 64 | def _fmt_choices(choices): 65 | return [(value, str(label)) for value, label in choices or []] 66 | 67 | 68 | def get_field_type(field_name, field): 69 | if isinstance(field, ArrayField) and isinstance(field.base_field, _STRING_FIELDS): 70 | base_field, choices = get_field_type(field_name, field.base_field) 71 | array_types = { 72 | StringType: StringArrayType, 73 | NumberType: NumberArrayType, 74 | StringChoiceType: StringChoiceArrayType, 75 | NumberChoiceType: NumberChoiceArrayType, 76 | } 77 | if base_field in array_types: 78 | return array_types[base_field], choices 79 | else: # pragma: no cover 80 | debug_log( 81 | f"{field.model.__name__}.{field_name} unsupported subarray type" 82 | f" {type(field.base_field).__name__}" 83 | ) 84 | return UnknownType, None 85 | 86 | elif isinstance(field, ArrayField) and isinstance(field.base_field, _NUMBER_FIELDS): 87 | if field.base_field.choices: 88 | return NumberChoiceArrayType, _fmt_choices(field.base_field.choices) 89 | else: 90 | return NumberArrayType, None 91 | elif isinstance(field, JSONField): 92 | res = JSONType 93 | elif field.__class__ in _FIELD_TYPE_MAP: 94 | res = _FIELD_TYPE_MAP[field.__class__] 95 | else: 96 | for django_type, field_type in _FIELD_TYPE_MAP.items(): 97 | if isinstance(field, django_type): 98 | res = field_type 99 | break 100 | else: 101 | debug_log( 102 | f"{field.model.__name__}.{field_name} unsupported type" 103 | f" {type(field).__name__}" 104 | ) 105 | res = UnknownType 106 | 107 | # Choice fields have different lookups 108 | if res is StringType and field.choices: 109 | return StringChoiceType, _fmt_choices(field.choices) 110 | elif res is NumberType and field.choices: 111 | return NumberChoiceType, _fmt_choices(field.choices) 112 | else: 113 | return res, None 114 | -------------------------------------------------------------------------------- /data_browser/templates/data_browser/index.html: -------------------------------------------------------------------------------- 1 | {% load static %}View{% block extrahead %} {% endblock %}{% block messages %} {% if messages %}{% endif %} {% endblock messages %}{% csrf_token %}
-------------------------------------------------------------------------------- /data_browser/urls.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from django.urls import path 4 | from django.urls import re_path 5 | from django.urls import register_converter 6 | from django.urls.converters import StringConverter 7 | from django.views.generic.base import RedirectView 8 | from django.views.static import serve 9 | 10 | from data_browser.api import view_detail 11 | from data_browser.api import view_list 12 | from data_browser.common import settings 13 | from data_browser.views import proxy_js_dev_server 14 | from data_browser.views import query 15 | from data_browser.views import query_ctx 16 | from data_browser.views import query_html 17 | from data_browser.views import view 18 | 19 | FE_BUILD_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fe_build") 20 | WEB_ROOT_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "web_root") 21 | 22 | 23 | class OptionalString(StringConverter): 24 | regex = "[^/]*" 25 | 26 | 27 | register_converter(OptionalString, "opt_str") 28 | 29 | 30 | app_name = "data_browser" 31 | 32 | QUERY_PATH = "query//" 33 | 34 | if settings.DATA_BROWSER_DEV: # pragma: no cover 35 | static_view = (proxy_js_dev_server,) 36 | else: 37 | static_view = (serve, {"document_root": FE_BUILD_DIR}) 38 | 39 | urlpatterns = [ 40 | # queries 41 | path(f"{QUERY_PATH}.html", query_html, name="query_html"), 42 | path(f"{QUERY_PATH}.ctx", query_ctx), 43 | path(f"{QUERY_PATH}.", query, name="query"), 44 | # views 45 | path("view/.", view, name="view"), 46 | # api 47 | path("api/views/", view_list, name="view_list"), 48 | path("api/views//", view_detail, name="view_detail"), 49 | # other html pages 50 | re_path(r".*\.html", query_html), 51 | re_path(r".*\.ctx", query_ctx), 52 | path("", query_html, name="home"), 53 | # static files 54 | re_path(r"^(?Pstatic/.*)$", *static_view, name="static"), 55 | re_path( 56 | r"^.*/(?Pstatic/.*)$", 57 | RedirectView.as_view(pattern_name="data_browser:static", permanent=True), 58 | ), 59 | re_path(r"^(?P.*)$", serve, {"document_root": WEB_ROOT_DIR}), 60 | ] 61 | -------------------------------------------------------------------------------- /data_browser/util.py: -------------------------------------------------------------------------------- 1 | from django.utils.text import slugify 2 | 3 | 4 | def annotation_path(path): 5 | res = "_".join(["ddb"] + path) 6 | res = res.replace("__", "_0") 7 | return res 8 | 9 | 10 | def str_to_field(s): 11 | return slugify(s).replace("-", "_") 12 | 13 | 14 | def group_by(things, key): 15 | res = {} 16 | for thing in things: 17 | res.setdefault(key(thing), []).append(thing) 18 | return res 19 | 20 | 21 | def title_case(s): 22 | return " ".join(w[0].upper() + w[1:] for w in s.split()) 23 | -------------------------------------------------------------------------------- /data_browser/web_root/android-chrome-192x192.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tolomea/django-data-browser/317d7f0b8c366c6281c973d87e792c6322469625/data_browser/web_root/android-chrome-192x192.png -------------------------------------------------------------------------------- /data_browser/web_root/android-chrome-512x512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tolomea/django-data-browser/317d7f0b8c366c6281c973d87e792c6322469625/data_browser/web_root/android-chrome-512x512.png -------------------------------------------------------------------------------- /data_browser/web_root/apple-touch-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tolomea/django-data-browser/317d7f0b8c366c6281c973d87e792c6322469625/data_browser/web_root/apple-touch-icon.png -------------------------------------------------------------------------------- /data_browser/web_root/favicon-16x16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tolomea/django-data-browser/317d7f0b8c366c6281c973d87e792c6322469625/data_browser/web_root/favicon-16x16.png -------------------------------------------------------------------------------- /data_browser/web_root/favicon-32x32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tolomea/django-data-browser/317d7f0b8c366c6281c973d87e792c6322469625/data_browser/web_root/favicon-32x32.png -------------------------------------------------------------------------------- /data_browser/web_root/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tolomea/django-data-browser/317d7f0b8c366c6281c973d87e792c6322469625/data_browser/web_root/favicon.ico -------------------------------------------------------------------------------- /data_browser/web_root/site.webmanifest: -------------------------------------------------------------------------------- 1 | { 2 | "name": "", 3 | "short_name": "", 4 | "icons": [ 5 | { 6 | "src": "android-chrome-192x192.png", 7 | "sizes": "192x192", 8 | "type": "image/png" 9 | }, 10 | { 11 | "src": "android-chrome-512x512.png", 12 | "sizes": "512x512", 13 | "type": "image/png" 14 | } 15 | ], 16 | "theme_color": "#ffffff", 17 | "background_color": "#ffffff", 18 | "display": "standalone" 19 | } 20 | -------------------------------------------------------------------------------- /frontend/.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | /node_modules 5 | /.pnp 6 | .pnp.js 7 | 8 | # testing 9 | /coverage 10 | 11 | # production 12 | /build 13 | 14 | # misc 15 | .DS_Store 16 | .env.local 17 | .env.development.local 18 | .env.test.local 19 | .env.production.local 20 | 21 | npm-debug.log* 22 | yarn-debug.log* 23 | yarn-error.log* 24 | -------------------------------------------------------------------------------- /frontend/README.md: -------------------------------------------------------------------------------- 1 | This project was bootstrapped with [Create React App](https://github.com/facebook/create-react-app). 2 | 3 | ## Available Scripts 4 | 5 | In the project directory, you can run: 6 | 7 | ### `npm start` 8 | 9 | Runs the app in the development mode.
10 | Open [http://localhost:3000](http://localhost:3000) to view it in the browser. 11 | 12 | The page will reload if you make edits.
13 | You will also see any lint errors in the console. 14 | 15 | ### `npm test` 16 | 17 | Launches the test runner in the interactive watch mode.
18 | See the section about [running tests](https://facebook.github.io/create-react-app/docs/running-tests) for more information. 19 | 20 | ### `npm run build` 21 | 22 | Builds the app for production to the `build` folder.
23 | It correctly bundles React in production mode and optimizes the build for the best performance. 24 | 25 | The build is minified and the filenames include the hashes.
26 | Your app is ready to be deployed! 27 | 28 | See the section about [deployment](https://facebook.github.io/create-react-app/docs/deployment) for more information. 29 | 30 | ### `npm run eject` 31 | 32 | **Note: this is a one-way operation. Once you `eject`, you can’t go back!** 33 | 34 | If you aren’t satisfied with the build tool and configuration choices, you can `eject` at any time. This command will remove the single build dependency from your project. 35 | 36 | Instead, it will copy all the configuration files and the transitive dependencies (webpack, Babel, ESLint, etc) right into your project so you have full control over them. All of the commands except `eject` will still work, but they will point to the copied scripts so you can tweak them. At this point you’re on your own. 37 | 38 | You don’t have to ever use `eject`. The curated feature set is suitable for small and middle deployments, and you shouldn’t feel obligated to use this feature. However we understand that this tool wouldn’t be useful if you couldn’t customize it when you are ready for it. 39 | 40 | ## Learn More 41 | 42 | You can learn more in the [Create React App documentation](https://facebook.github.io/create-react-app/docs/getting-started). 43 | 44 | To learn React, check out the [React documentation](https://reactjs.org/). 45 | 46 | ### Code Splitting 47 | 48 | This section has moved here: https://facebook.github.io/create-react-app/docs/code-splitting 49 | 50 | ### Analyzing the Bundle Size 51 | 52 | This section has moved here: https://facebook.github.io/create-react-app/docs/analyzing-the-bundle-size 53 | 54 | ### Making a Progressive Web App 55 | 56 | This section has moved here: https://facebook.github.io/create-react-app/docs/making-a-progressive-web-app 57 | 58 | ### Advanced Configuration 59 | 60 | This section has moved here: https://facebook.github.io/create-react-app/docs/advanced-configuration 61 | 62 | ### Deployment 63 | 64 | This section has moved here: https://facebook.github.io/create-react-app/docs/deployment 65 | 66 | ### `npm run build` fails to minify 67 | 68 | This section has moved here: https://facebook.github.io/create-react-app/docs/troubleshooting#npm-run-build-fails-to-minify 69 | -------------------------------------------------------------------------------- /frontend/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "frontend", 3 | "version": "0.1.0", 4 | "private": true, 5 | "homepage": ".", 6 | "dependencies": { 7 | "@sentry/browser": "^6.17.5", 8 | "@testing-library/jest-dom": "^5.16.2", 9 | "@testing-library/react": "^12.1.2", 10 | "@testing-library/user-event": "^13.5.0", 11 | "assert": "^2.0.0", 12 | "js-cookie": "^3.0.1", 13 | "lodash": "^4.17.21", 14 | "react": "^16", 15 | "react-dom": "^16", 16 | "react-router": "^5", 17 | "react-router-dom": "^5", 18 | "react-scripts": "^5.0.0", 19 | "sass": "^1.49.7" 20 | }, 21 | "scripts": { 22 | "start": "react-scripts start", 23 | "build": "react-scripts build", 24 | "test": "react-scripts test", 25 | "eject": "react-scripts eject" 26 | }, 27 | "eslintConfig": { 28 | "extends": "react-app" 29 | }, 30 | "browserslist": { 31 | "production": [ 32 | ">0.2%", 33 | "not dead", 34 | "not op_mini all" 35 | ], 36 | "development": [ 37 | "last 1 chrome version", 38 | "last 1 firefox version", 39 | "last 1 safari version" 40 | ] 41 | }, 42 | "devDependencies": { 43 | "prettier": "^2.5.1" 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /frontend/public/index.html: -------------------------------------------------------------------------------- 1 | {% load static %} 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 14 | 18 | 22 | 26 | 30 | 35 | 41 | 47 | 48 | View 49 | 50 | 78 | 79 | {% block extrahead %} {% endblock %} 80 | 81 | 82 | {% block messages %} 83 | {% if messages %} 84 |
    {% for message in messages %} 85 | {{ message|capfirst }} 86 | {% endfor %}
87 | {% endif %} 88 | {% endblock messages %} 89 | 90 | 91 | 92 | {% csrf_token %} 93 | 96 | 99 | 100 |
101 | 111 | 112 | 113 | -------------------------------------------------------------------------------- /frontend/src/App.js: -------------------------------------------------------------------------------- 1 | import React, { useContext } from "react"; 2 | import { BrowserRouter, Switch, Route, Link } from "react-router-dom"; 3 | 4 | import { ContextMenu } from "./ContextMenu"; 5 | import { Tooltip } from "./Tooltip"; 6 | import { CurrentSavedView } from "./CurrentSavedView"; 7 | import { HomePage } from "./HomePage"; 8 | import { QueryPage } from "./QueryPage"; 9 | import { SavedViewPage } from "./SavedViewPage"; 10 | import { version } from "./Network"; 11 | import { Config } from "./Config"; 12 | 13 | import "./App.scss"; 14 | 15 | function Logo(props) { 16 | return ( 17 | 18 | DDB 19 | v{version} 20 | 21 | ); 22 | } 23 | 24 | function App(props) { 25 | const config = useContext(Config); 26 | return ( 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | ); 48 | } 49 | 50 | export default App; 51 | -------------------------------------------------------------------------------- /frontend/src/App.test.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { render } from '@testing-library/react'; 3 | import App from './App'; 4 | 5 | test('renders learn react link', () => { 6 | const { getByText } = render(); 7 | const linkElement = getByText(/learn react/i); 8 | expect(linkElement).toBeInTheDocument(); 9 | }); 10 | -------------------------------------------------------------------------------- /frontend/src/Config.js: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | const Config = React.createContext(); 3 | 4 | export { Config }; 5 | -------------------------------------------------------------------------------- /frontend/src/ContextMenu.js: -------------------------------------------------------------------------------- 1 | import React, { useState, useRef, useEffect } from "react"; 2 | import "./App.scss"; 3 | import useWindowDimensions from "./WindowDimensions"; 4 | import { TLink } from "./Util"; 5 | 6 | const ShowContextMenu = React.createContext(); 7 | 8 | function ContextMenu(props) { 9 | const hidden = { x: 0, y: 0, top: 0, left: 0, entries: [] }; 10 | const pad = 10; 11 | const node = useRef(); 12 | const [state, setState] = useState(hidden); 13 | const { width, height } = useWindowDimensions(); 14 | 15 | function handleClick(e) { 16 | if (node.current && node.current.contains(e.target)) return; 17 | setState(hidden); 18 | } 19 | 20 | useEffect(() => { 21 | document.addEventListener("mousedown", handleClick); 22 | return () => { 23 | document.removeEventListener("mousedown", handleClick); 24 | }; 25 | }); 26 | 27 | useEffect(() => { 28 | if (node.current) { 29 | const w = node.current.offsetWidth; 30 | const h = node.current.offsetHeight; 31 | setState({ 32 | x: state.x, 33 | y: state.y, 34 | top: state.y + h + pad > height ? height - h - pad : state.y, 35 | left: state.x + w + pad > width ? width - w - pad : state.x, 36 | entries: state.entries, 37 | }); 38 | } 39 | }, [width, height, state.entries, state.x, state.y]); 40 | 41 | function showContextMenu(event, entries) { 42 | entries = entries.filter((x) => x); 43 | if (entries.length && window.getSelection().toString().length === 0) { 44 | setState({ 45 | entries, 46 | y: event.clientY, 47 | x: event.clientX, 48 | top: 0, 49 | left: 0, 50 | }); 51 | event.preventDefault(); 52 | } 53 | } 54 | 55 | const divStyle = { 56 | left: state.left, 57 | top: state.top, 58 | visibility: state.left + state.top === 0 ? "hidden" : "visible", 59 | }; 60 | 61 | return ( 62 | 63 | {props.children} 64 | {state.entries.length ? ( 65 |
66 | {state.entries.map((entry) => ( 67 |

68 | { 70 | entry.fn(); 71 | setState(hidden); 72 | }} 73 | > 74 | {entry.name} 75 | 76 |

77 | ))} 78 |
79 | ) : null} 80 | 81 | ); 82 | } 83 | 84 | export { ContextMenu, ShowContextMenu }; 85 | -------------------------------------------------------------------------------- /frontend/src/CurrentSavedView.js: -------------------------------------------------------------------------------- 1 | import React, { useState } from "react"; 2 | 3 | const GetCurrentSavedView = React.createContext(); 4 | const SetCurrentSavedView = React.createContext(); 5 | 6 | function CurrentSavedView(props) { 7 | const [state, setState] = useState(null); 8 | 9 | return ( 10 | 11 | 12 | {props.children} 13 | 14 | 15 | ); 16 | } 17 | 18 | export { CurrentSavedView, GetCurrentSavedView, SetCurrentSavedView }; 19 | -------------------------------------------------------------------------------- /frontend/src/FieldList.js: -------------------------------------------------------------------------------- 1 | import { 2 | TLink, 3 | SLink, 4 | HasActionIcon, 5 | HasToManyIcon, 6 | useToggle, 7 | strMatch, 8 | } from "./Util"; 9 | 10 | import "./App.scss"; 11 | 12 | function Field(props) { 13 | const { query, path, modelField, filterParts, nesting } = props; 14 | const type = query.getType(modelField); 15 | const [toggled, toggleLink] = useToggle(); 16 | const expanded = modelField.model && (toggled || filterParts.length || ""); 17 | const stickyOffsetStyle = { 18 | top: -2 + nesting * 30, 19 | zIndex: 99 - nesting, 20 | }; 21 | const zStyle = { 22 | zIndex: 99 - nesting - 1, 23 | }; 24 | 25 | return ( 26 | <> 27 |
28 |
32 | {/* filter */} 33 |
34 | {modelField.concrete && type.defaultLookup && ( 35 | query.addFilter(path.join("__"))}> 36 | filter_alt 37 | 38 | )} 39 |
40 | 41 | {/* expand */} 42 |
{modelField.model && toggleLink}
43 | 44 | {/* name */} 45 |
46 | {modelField.type ? ( 47 | 49 | query.addField(path.join("__"), modelField.defaultSort) 50 | } 51 | > 52 | {modelField.verboseName} 53 | 57 | 58 | ) : ( 59 | <> 60 | {modelField.verboseName} 61 | 65 | 66 | )} 67 |
68 |
69 | 70 | {/* sub fields */} 71 | {expanded && ( 72 |
73 | 78 |
79 | )} 80 |
81 | 82 | ); 83 | } 84 | 85 | function FieldGroup(props) { 86 | const { query, model, path, filterParts, nesting } = props; 87 | const modelFields = query.getModelFields(model); 88 | 89 | return ( 90 | <> 91 | {modelFields.sortedFields.map((fieldName) => { 92 | const modelField = modelFields.fields[fieldName]; 93 | if (!strMatch(filterParts[0], fieldName, modelField.verboseName)) 94 | return null; 95 | return ( 96 | 102 | ); 103 | })} 104 | 105 | ); 106 | } 107 | 108 | function FieldList(props) { 109 | const { query, model, filterParts } = props; 110 | 111 | return ( 112 |
113 | 114 |
115 | ); 116 | } 117 | 118 | export { FieldList }; 119 | -------------------------------------------------------------------------------- /frontend/src/FilterList.js: -------------------------------------------------------------------------------- 1 | import React, { useContext } from "react"; 2 | 3 | import { TLink, SLink, useToggle } from "./Util"; 4 | import { ShowTooltip, HideTooltip } from "./Tooltip"; 5 | 6 | import "./App.scss"; 7 | 8 | function FilterValue(props) { 9 | const { lookupType, onChange, value, field } = props; 10 | const onChangeEvent = (e) => onChange(e.target.value); 11 | const showTooltip = useContext(ShowTooltip); 12 | const hideTooltip = useContext(HideTooltip); 13 | const helpText = { 14 | date: [ 15 | "Date filter values consist of a series of clauses applied in order left to right starting with a value of `today`.", 16 | "e.g. 'day=1 month+1 tuesday+2' which means move to the 1st of this month, then move forward a month, then move forward to the second Tuesday.", 17 | "Possible clauses include 'today', 'now' and literal date values in a variety of formats e.g. '2020-12-21'.", 18 | "Or you can use 'year', 'month', 'week' or 'day' with '+', '-', or '=' to add remove or replace the given quantity.", 19 | "Or you can use a weekday name with '+' or '-' to get the n-th next or previous (including today) instance of that day.", 20 | "Bear in mind that 'day=1 month+1' may produce a different result from 'month+1 day=1', for example on Jan 31st.", 21 | ], 22 | datetime: [ 23 | "Datetime filter consist of a series of clauses applied in order left to right starting with a value of `now`.", 24 | "e.g. 'day=1 month+1 tuesday+2' which means move to the 1st of this month, then move forward a month, then move forward to the second Tuesday.", 25 | "Possible clauses include 'today', 'now' and literal date and time values in a variety of formats e.g. '2020-12-21 14:56'.", 26 | "Or you can use 'year', 'month', 'week', 'day', 'hour', 'minute' or 'second' with '+', '-', or '=' to add remove or replace the given quantity.", 27 | "Or you can use a weekday name with '+' or '-' to get the n-th next or previous (including today) instance of that day.", 28 | "Bear in mind that 'day=1 month+1' may produce a different result from 'month+1 day=1', for example on Jan 31st.", 29 | ], 30 | }; 31 | 32 | if (lookupType === "boolean") { 33 | return ( 34 | 38 | ); 39 | } else if (lookupType === "isnull") { 40 | return ( 41 | 45 | ); 46 | } else if (lookupType.endsWith("choice")) { 47 | return ( 48 | 55 | ); 56 | } else if (lookupType === "number") { 57 | return ( 58 | 65 | ); 66 | } else if (lookupType === "jsonfield") { 67 | const parts = value.split(/\|(.*)/); 68 | return ( 69 | <> 70 | onChange(`${e.target.value}|${parts[1]}`)} 73 | className="FilterValue Half" 74 | type="text" 75 | /> 76 | onChange(`${parts[0]}|${e.target.value}`)} 79 | className="FilterValue Half" 80 | type="text" 81 | /> 82 | 83 | ); 84 | } else { 85 | return ( 86 | showTooltip(e, helpText[lookupType])} 92 | onMouseLeave={(e) => hideTooltip(e)} 93 | /> 94 | ); 95 | } 96 | } 97 | 98 | function Filter(props) { 99 | const { pathStr, index, lookup, query, value, errorMessage, parsed } = props; 100 | const field = query.getField(pathStr); 101 | var type = null; 102 | var lookupType = null; 103 | if (field !== null) { 104 | type = query.getType(field); 105 | if (type.lookups.hasOwnProperty(lookup)) 106 | lookupType = type.lookups[lookup].type; 107 | } 108 | 109 | if (lookupType === null) 110 | return ( 111 | 112 | 113 | {" "} 114 | query.removeFilter(index)}>close{" "} 115 | {pathStr} 116 | 117 | {lookup} 118 | = 119 | 120 | {value} 121 |

{errorMessage}

122 | 123 | 124 | ); 125 | 126 | return ( 127 | 128 | 129 | query.removeFilter(index)}>close{" "} 130 | query.addField(pathStr, field.defaultSort)}> 131 | {query.verbosePathStr(pathStr)} 132 | {" "} 133 | 134 | 135 | 146 | 147 | = 148 | 149 | query.setFilterValue(index, val)} 152 | /> 153 | {errorMessage &&

{errorMessage}

} 154 | {parsed !== null && 155 | (lookupType === "date" || lookupType === "datetime") && ( 156 |

{parsed}

157 | )} 158 | 159 | 160 | ); 161 | } 162 | 163 | function FilterList(props) { 164 | const { query, filters } = props; 165 | const [toggled, toggleLink] = useToggle(true); 166 | if (!filters.length) return ""; 167 | return ( 168 |
e.preventDefault()}> 169 |
{toggleLink}
170 | {toggled && ( 171 | 172 | 173 | {filters.map((filter, index) => ( 174 | 175 | ))} 176 | 177 |
178 | )} 179 |
180 | ); 181 | } 182 | 183 | export { FilterList }; 184 | -------------------------------------------------------------------------------- /frontend/src/HomePage.js: -------------------------------------------------------------------------------- 1 | import React, { useContext } from "react"; 2 | import { Link } from "react-router-dom"; 3 | 4 | import { useData } from "./Network"; 5 | import { usePersistentToggle } from "./Util"; 6 | import { getRelUrlForQuery } from "./Query"; 7 | import { SetCurrentSavedView } from "./CurrentSavedView"; 8 | import { Config } from "./Config"; 9 | 10 | import "./App.scss"; 11 | 12 | function View(props) { 13 | const { view } = props; 14 | const setCurrentSavedView = useContext(SetCurrentSavedView); 15 | return ( 16 |
17 |

18 | view.can_edit && setCurrentSavedView(view)} 22 | > 23 | {view.name || ""} 24 | {" "} 25 | {view.can_edit && (edit)} 26 |

27 |
28 |

29 | on {view.model} 30 |

31 |

32 | {view.can_edit && view.shared && Shared } 33 | {view.can_edit && view.public && Public } 34 | {view.can_edit && !view.valid && ( 35 | Invalid 36 | )} 37 |

38 | {view.description &&

{view.description}

} 39 |
40 |
41 | ); 42 | } 43 | 44 | function Folder(props) { 45 | const { parentName, folder, foldersExpanded } = props; 46 | const fullName = `${parentName}.${folder.name}`; 47 | const [toggled, toggleLink] = usePersistentToggle( 48 | `${fullName}.toggle`, 49 | foldersExpanded, 50 | ); 51 | 52 | return ( 53 |
54 |

55 | {toggleLink} 56 | {folder.name} 57 |

58 | {toggled && } 59 |
60 | ); 61 | } 62 | 63 | function Entries(props) { 64 | const { entries, parentName, foldersExpanded } = props; 65 | return entries.map((entry, index) => 66 | entry.type === "view" ? ( 67 | 68 | ) : ( 69 | 70 | ), 71 | ); 72 | } 73 | 74 | function SavedAndSharedViews(props) { 75 | const config = useContext(Config); 76 | const [savedViews] = useData(`${config.baseUrl}api/views/`); 77 | 78 | if (!savedViews) return ""; // loading state 79 | 80 | return ( 81 |
82 |
83 |

Your Saved Views

84 | 89 | {!!savedViews.shared.length &&

Views Shared by Others

} 90 | 95 |
96 |
97 | ); 98 | } 99 | 100 | function AppEntry(props) { 101 | const config = useContext(Config); 102 | const { appVerboseName, models } = props; 103 | const [toggled, toggleLink] = usePersistentToggle( 104 | `model.${appVerboseName}.toggle`, 105 | config.appsExpanded, 106 | ); 107 | return ( 108 | <> 109 |

110 | {toggleLink} 111 | {appVerboseName} 112 |

113 | {toggled && ( 114 |
115 | {models.map((modelEntry) => { 116 | return ( 117 |

118 | 132 | {modelEntry.verboseName} 133 | 134 |

135 | ); 136 | })} 137 |
138 | )} 139 | 140 | ); 141 | } 142 | 143 | function ModelList(props) { 144 | const config = useContext(Config); 145 | return ( 146 |
147 |
148 |

Models

149 | {config.modelIndex.map(({ appVerboseName, models }) => ( 150 | 151 | ))} 152 |
153 |
154 | ); 155 | } 156 | 157 | function HomePage(props) { 158 | const setCurrentSavedView = useContext(SetCurrentSavedView); 159 | setCurrentSavedView(null); 160 | 161 | return ( 162 |
163 | 164 | 165 |
166 | ); 167 | } 168 | 169 | export { HomePage }; 170 | -------------------------------------------------------------------------------- /frontend/src/Network.js: -------------------------------------------------------------------------------- 1 | import { useState, useEffect } from "react"; 2 | 3 | const assert = require("assert"); 4 | let fetchInProgress = false; 5 | let nextFetch = undefined; 6 | 7 | const version = document.getElementById("backend-version").textContent.trim(); 8 | const csrf_token = document.querySelector("[name=csrfmiddlewaretoken]").value; 9 | 10 | class AbortError extends Error { 11 | name = "AbortError"; 12 | } 13 | 14 | function doFetch(url, options, process) { 15 | if (fetchInProgress) { 16 | if (nextFetch) { 17 | nextFetch.reject(new AbortError("skipped")); 18 | } 19 | return new Promise((resolve, reject) => { 20 | nextFetch = { resolve, reject, url, options, process }; 21 | }); 22 | } 23 | 24 | fetchInProgress = true; 25 | 26 | return fetch(url, options) 27 | .then((response) => { 28 | // do we have a next fetch we need to trigger 29 | const next = nextFetch; 30 | nextFetch = undefined; 31 | fetchInProgress = false; 32 | 33 | if (next) { 34 | doFetch(next.url, next.options, next.process).then( 35 | (res) => next.resolve(res), 36 | (err) => next.reject(err), 37 | ); 38 | throw new AbortError("superceeded"); 39 | } else { 40 | return response; 41 | } 42 | }) 43 | .then((response) => { 44 | // check status 45 | assert.ok(response.status >= 200); 46 | assert.ok(response.status < 300); 47 | return response; 48 | }) 49 | .then((response) => { 50 | // check server version 51 | const response_version = response.headers.get("x-version"); 52 | if (response_version !== version) { 53 | console.log( 54 | "Version mismatch, hard reload", 55 | version, 56 | response_version, 57 | ); 58 | window.location.reload(true); 59 | } 60 | return response; 61 | }) 62 | .then((response) => process(response)); // process data 63 | } 64 | 65 | function doGet(url) { 66 | return doFetch(url, { method: "GET" }, (response) => response.json()); 67 | } 68 | 69 | function doDelete(url) { 70 | return doFetch( 71 | url, 72 | { 73 | method: "DELETE", 74 | headers: { "X-CSRFToken": csrf_token }, 75 | }, 76 | (response) => response, 77 | ); 78 | } 79 | 80 | function doPatch(url, data) { 81 | return doFetch( 82 | url, 83 | { 84 | method: "PATCH", 85 | headers: { 86 | "Content-Type": "application/json", 87 | "X-CSRFToken": csrf_token, 88 | }, 89 | body: JSON.stringify(data), 90 | }, 91 | (response) => response.json(), 92 | ); 93 | } 94 | 95 | function doPost(url, data) { 96 | return doFetch( 97 | url, 98 | { 99 | method: "POST", 100 | headers: { 101 | "Content-Type": "application/json", 102 | "X-CSRFToken": csrf_token, 103 | }, 104 | body: JSON.stringify(data), 105 | }, 106 | (response) => response.json(), 107 | ); 108 | } 109 | 110 | function syncPost(url, data) { 111 | const form = document.createElement("form"); 112 | form.method = "post"; 113 | form.action = url; 114 | 115 | data.push(["csrfmiddlewaretoken", csrf_token]); 116 | 117 | for (const [key, value] of data) { 118 | const hiddenField = document.createElement("input"); 119 | hiddenField.type = "hidden"; 120 | hiddenField.name = key; 121 | hiddenField.value = value; 122 | 123 | form.appendChild(hiddenField); 124 | } 125 | 126 | document.body.appendChild(form); 127 | form.submit(); 128 | } 129 | 130 | function useData(url) { 131 | const [data, setData] = useState(); 132 | useEffect(() => { 133 | doGet(url).then((response) => setData(response)); 134 | }, [url]); 135 | return [ 136 | data, 137 | (updates) => { 138 | setData((prev) => ({ ...prev, ...updates })); 139 | doPatch(url, updates) 140 | .then((response) => 141 | setData((prev) => ({ ...prev, ...response })), 142 | ) 143 | .catch((e) => { 144 | if (e.name !== "AbortError") throw e; 145 | }); 146 | }, 147 | ]; 148 | } 149 | 150 | export { 151 | doPatch, 152 | doGet, 153 | doDelete, 154 | doPost, 155 | useData, 156 | version, 157 | fetchInProgress, 158 | syncPost, 159 | }; 160 | -------------------------------------------------------------------------------- /frontend/src/Query.test.js: -------------------------------------------------------------------------------- 1 | import { Query, getUrlForQuery } from "./Query"; 2 | import context from "./context_fixture.json"; 3 | 4 | it("bob", () => { 5 | expect(1).toEqual(1); 6 | }); 7 | -------------------------------------------------------------------------------- /frontend/src/QueryPage.js: -------------------------------------------------------------------------------- 1 | import * as Sentry from "@sentry/browser"; 2 | import React, { useState, useEffect, useContext } from "react"; 3 | import { useParams, useLocation } from "react-router-dom"; 4 | 5 | import { doGet, fetchInProgress } from "./Network"; 6 | import { SLink, Save, Update, useToggle, matchPrepare } from "./Util"; 7 | import { Results } from "./Results"; 8 | import { getPartsForQuery, Query, getUrlForQuery, empty } from "./Query"; 9 | import { ShowTooltip, HideTooltip } from "./Tooltip"; 10 | import { GetCurrentSavedView } from "./CurrentSavedView"; 11 | import { Config } from "./Config"; 12 | import { FieldList } from "./FieldList"; 13 | import { FilterList } from "./FilterList"; 14 | 15 | import "./App.scss"; 16 | 17 | const BOOTING = "Booting..."; 18 | const LOADING = "Loading..."; 19 | const ERROR = "Error"; 20 | 21 | function InvalidField(props) { 22 | const { query, field } = props; 23 | 24 | return ( 25 | 26 | 27 | {" "} 28 | query.removeField(field)}>close{" "} 29 | {field.pathStr} 30 | 31 | 32 |

{field.errorMessage}

33 | 34 | 35 | ); 36 | } 37 | 38 | function InvalidFields(props) { 39 | const { query } = props; 40 | const invalidFields = query.invalidFields(); 41 | 42 | if (!invalidFields.length) return ""; 43 | 44 | return ( 45 |
46 | 47 | 48 | {invalidFields.map((field, index) => ( 49 | 50 | ))} 51 | 52 |
53 |
54 | ); 55 | } 56 | 57 | function ModelSelector(props) { 58 | const config = useContext(Config); 59 | const { query, model } = props; 60 | 61 | return ( 62 | 81 | ); 82 | } 83 | 84 | function FieldsFilter(props) { 85 | const { fieldFilter, setFieldFilter } = props; 86 | const showTooltip = useContext(ShowTooltip); 87 | const hideTooltip = useContext(HideTooltip); 88 | 89 | return ( 90 |
91 | { 95 | setFieldFilter(event.target.value); 96 | }} 97 | onMouseEnter={(e) => 98 | showTooltip(e, [ 99 | "Use ' ' to seperate search terms.", 100 | "Use '.' to filter inside related models.", 101 | ]) 102 | } 103 | onMouseLeave={(e) => hideTooltip(e)} 104 | /> 105 | 113 |
114 | ); 115 | } 116 | 117 | function FilterSideBar(props) { 118 | const { query, model } = props; 119 | const [fieldsToggled, fieldsToggleLink] = useToggle(true); 120 | const [fieldFilter, setFieldFilter] = useState(""); 121 | 122 | return ( 123 |
124 |
{fieldsToggleLink}
125 | {fieldsToggled && } 126 | {fieldsToggled && ( 127 | 131 | )} 132 |
133 | ); 134 | } 135 | 136 | function ResultsPane(props) { 137 | const { query, rows, cols, body, overlay, formatHints } = props; 138 | 139 | if (!query.validFields().length) return

No fields selected

; 140 | 141 | return ; 142 | } 143 | 144 | function UpdateSavedView(props) { 145 | const config = useContext(Config); 146 | const { query } = props; 147 | const currentSavedView = useContext(GetCurrentSavedView); 148 | 149 | if (!currentSavedView) return null; 150 | 151 | return ( 152 |

153 | "}`} 155 | apiUrl={`${config.baseUrl}api/views/${currentSavedView.pk}/`} 156 | data={{ ...currentSavedView, ...getPartsForQuery(query.query) }} 157 | redirectUrl={`/views/${currentSavedView.pk}.html`} 158 | /> 159 |

160 | ); 161 | } 162 | 163 | function Header(props) { 164 | const config = useContext(Config); 165 | const { query, length, model, filters, limit } = props; 166 | 167 | return ( 168 | <> 169 | 170 | 171 |

172 | = limit ? "Error" : ""}> 173 | Limit:{" "} 174 | { 179 | query.setLimit(event.target.value); 180 | }} 181 | min="1" 182 | />{" "} 183 | - Showing {length} results -{" "} 184 | 185 | Download as CSV -{" "} 186 | View as JSON -{" "} 187 | View SQL Query -{" "} 188 | `/views/${view.pk}.html`} 193 | /> 194 |

195 | 196 | 197 | 198 | ); 199 | } 200 | 201 | function QueryPageContent(props) { 202 | const { 203 | query, 204 | rows, 205 | cols, 206 | body, 207 | length, 208 | model, 209 | filters, 210 | overlay, 211 | formatHints, 212 | limit, 213 | } = props; 214 | 215 | return ( 216 |
217 |
226 |
227 | 228 | 229 |
230 |
231 |
232 | ); 233 | } 234 | 235 | function QueryPage(props) { 236 | const config = useContext(Config); 237 | const { model, fieldStr } = useParams(); 238 | const [status, setStatus] = useState(BOOTING); 239 | const [query, setQuery] = useState({ 240 | model: "", 241 | fields: [], 242 | filters: [], 243 | limit: config.defaultRowLimit, 244 | ...empty, 245 | }); 246 | const queryStr = useLocation().search; 247 | 248 | const handleError = (e) => { 249 | if (e.name !== "AbortError") { 250 | setStatus(ERROR); 251 | console.log(e); 252 | Sentry.captureException(e); 253 | } 254 | }; 255 | 256 | const fetchResults = (state) => { 257 | setStatus(LOADING); 258 | const url = getUrlForQuery(config.baseUrl, state, "json"); 259 | 260 | return doGet(url).then((response) => { 261 | setQuery((query) => ({ 262 | ...response, 263 | })); 264 | setStatus(fetchInProgress ? LOADING : undefined); 265 | return response; 266 | }); 267 | }; 268 | 269 | useEffect(() => { 270 | const popstate = (e) => { 271 | setQuery(e.state); 272 | fetchResults(e.state).catch(handleError); 273 | }; 274 | 275 | const url = `${config.baseUrl}query/${model}/${ 276 | fieldStr || "" 277 | }.query${queryStr}`; 278 | 279 | doGet(url).then((response) => { 280 | const reqState = { 281 | model: response.model, 282 | fields: response.fields, 283 | filters: response.filters, 284 | limit: response.limit, 285 | ...empty, 286 | }; 287 | setQuery(reqState); 288 | setStatus(LOADING); 289 | window.history.replaceState( 290 | reqState, 291 | null, 292 | getUrlForQuery(config.baseUrl, reqState, "html"), 293 | ); 294 | window.addEventListener("popstate", popstate); 295 | fetchResults(reqState).catch(handleError); 296 | }); 297 | 298 | return () => { 299 | window.removeEventListener("popstate", popstate); 300 | }; 301 | // eslint-disable-next-line 302 | }, []); 303 | 304 | const handleQueryChange = (queryChange, reload = true) => { 305 | const newState = { ...query, ...queryChange }; 306 | 307 | setQuery(newState); 308 | 309 | const request = { 310 | model: newState.model, 311 | fields: newState.fields, 312 | filters: newState.filters, 313 | limit: newState.limit, 314 | ...empty, 315 | }; 316 | window.history.pushState( 317 | request, 318 | null, 319 | getUrlForQuery(config.baseUrl, newState, "html"), 320 | ); 321 | 322 | if (!reload) return; 323 | 324 | fetchResults(newState).catch(handleError); 325 | }; 326 | 327 | if (status === BOOTING) return ""; 328 | const queryObj = new Query(config, query, handleQueryChange); 329 | return ; 330 | } 331 | 332 | export { QueryPage }; 333 | -------------------------------------------------------------------------------- /frontend/src/SavedViewPage.js: -------------------------------------------------------------------------------- 1 | import React, { useContext } from "react"; 2 | import { Link, useParams } from "react-router-dom"; 3 | 4 | import { useData } from "./Network"; 5 | import { Delete, CopyText } from "./Util"; 6 | import { SetCurrentSavedView } from "./CurrentSavedView"; 7 | import { Config } from "./Config"; 8 | import { ShowTooltip, HideTooltip } from "./Tooltip"; 9 | 10 | import "./App.scss"; 11 | 12 | function SavedViewPage(props) { 13 | const config = useContext(Config); 14 | const { pk } = useParams(); 15 | const url = `${config.baseUrl}api/views/${pk}/`; 16 | const [view, setView] = useData(url); 17 | const setCurrentSavedView = useContext(SetCurrentSavedView); 18 | const showTooltip = useContext(ShowTooltip); 19 | const hideTooltip = useContext(HideTooltip); 20 | 21 | setCurrentSavedView(null); 22 | if (!view) return ""; 23 | return ( 24 |
25 |
26 |
27 | Saved View 28 | setCurrentSavedView(view)}> 29 | Open 30 | 31 |
32 |
33 | 34 | 35 | 36 | 47 | 48 | 49 | 50 | 51 | 61 | 62 | 63 | 64 | 65 | 68 | 69 | 70 | 71 | 72 | 75 | 76 | 77 | 78 | 79 | 82 | 83 | 84 | 85 | 86 | 96 | 97 | 98 | 99 | 100 | 103 | 104 | 105 | 106 |
37 | { 41 | setView({ name: event.target.value }); 42 | }} 43 | className="SavedViewName" 44 | placeholder="enter a name" 45 | /> 46 |
Folder: 52 | { 56 | setView({ folder: event.target.value }); 57 | }} 58 | placeholder="enter a folder name" 59 | /> 60 |
Model: 66 |

{view.model}

67 |
Fields: 73 |

{view.fields.replace(/,/g, "\u200b,")}

74 |
Filters: 80 |

{view.query.replace(/&/g, "\u200b&")}

81 |
Limit: 87 | { 92 | setView({ limit: event.target.value }); 93 | }} 94 | /> 95 |
Created Time: 101 |

{view.createdTime}

102 |
107 |