├── .codecov.yml ├── .coveragerc ├── .devcontainer.json ├── .editorconfig ├── .env.sample ├── .flake8 ├── .github └── workflows │ ├── build-push-image.yaml │ ├── checkov.yaml │ ├── codeql.yaml │ ├── deploy-to-k8s.yaml │ ├── helm_lint.yaml │ ├── lint.yaml │ ├── no_debug_allowed.yaml │ ├── no_forgoten_migrations.yaml │ ├── publish-and-deploy-gamma.yaml │ ├── publish-and-deploy.yaml │ ├── run-integ-tests.yaml │ ├── run_django_tests.yaml │ └── scorecard.yml ├── .gitignore ├── Dockerfile ├── LICENSE ├── README.md ├── docker-compose.yaml ├── infra ├── README.md └── helm │ └── meshdb │ ├── .gitignore │ ├── Chart.yaml │ ├── README.md │ ├── templates │ ├── _helpers.tpl │ ├── celery.yaml │ ├── configmap.yaml │ ├── ingress.yaml │ ├── meshweb.yaml │ ├── meshweb_static_pvc.yaml │ ├── nginx.yaml │ ├── nginx_configmap.yaml │ ├── pelias.yaml │ ├── postgres.yaml │ ├── postgres_pvc.yaml │ ├── pull_secret.yaml │ ├── redis.yaml │ ├── secrets.yaml │ └── service.yaml │ └── values.yaml ├── integ-tests └── meshapi_integ_test │ ├── __init__.py │ ├── integ_test_case.py │ └── test_api_200.py ├── nginx └── app.conf ├── pyproject.toml ├── sampledata ├── map_json ├── meshdb.kml └── meshdb_local.kml ├── scripts ├── celery │ ├── celery_beat.sh │ ├── celery_worker.sh │ └── probes │ │ ├── celery_beat_liveness.py │ │ ├── celery_beat_readiness.py │ │ ├── celery_liveness.py │ │ └── celery_readiness.py ├── clear_history_tables.sh ├── create_importable_datadump.sh ├── entrypoint.sh ├── import_datadump.sh └── reconcile_stripe_subscriptions.py ├── src ├── manage.py ├── meshapi │ ├── __init__.py │ ├── admin │ │ ├── __init__.py │ │ ├── admin.py │ │ ├── inlines.py │ │ ├── models │ │ │ ├── __init__.py │ │ │ ├── access_point.py │ │ │ ├── billing.py │ │ │ ├── building.py │ │ │ ├── device.py │ │ │ ├── install.py │ │ │ ├── link.py │ │ │ ├── los.py │ │ │ ├── member.py │ │ │ ├── node.py │ │ │ └── sector.py │ │ ├── password_reset.py │ │ ├── ranked_search.py │ │ ├── urls.py │ │ └── utils.py │ ├── apps.py │ ├── docs.py │ ├── exceptions.py │ ├── management │ │ ├── __init__.py │ │ └── commands │ │ │ ├── __init__.py │ │ │ ├── create_groups.py │ │ │ ├── replay_join_records.py │ │ │ ├── scramble_members.py │ │ │ └── sync_panoramas.py │ ├── migrations │ │ ├── 0001_initial.py │ │ ├── 0002_create_meshdb_ro.py │ │ ├── 0003_alter_historicalinstall_request_date_and_more.py │ │ ├── 0004_alter_historicalinstall_status_and_more.py │ │ ├── 0005_alter_install_options.py │ │ ├── 0006_install_additional_members.py │ │ ├── 0007_installfeebillingdatum_and_more.py │ │ ├── 0008_historicalinstall_stripe_subscription_id_and_more.py │ │ ├── 0009_alter_historicalnode_network_number_and_more.py │ │ ├── 0010_alter_historicallink_type_alter_link_type.py │ │ └── __init__.py │ ├── models │ │ ├── __init__.py │ │ ├── billing.py │ │ ├── building.py │ │ ├── devices │ │ │ ├── __init__.py │ │ │ ├── access_point.py │ │ │ ├── device.py │ │ │ └── sector.py │ │ ├── install.py │ │ ├── link.py │ │ ├── los.py │ │ ├── member.py │ │ ├── node.py │ │ ├── permission.py │ │ └── util │ │ │ └── auto_incrementing_integer_field.py │ ├── pelias.py │ ├── permissions.py │ ├── serializers │ │ ├── __init__.py │ │ ├── javascript_date_field.py │ │ ├── map.py │ │ ├── model_api.py │ │ ├── nested_key_object_related_field.py │ │ ├── nested_object_references.py │ │ └── query_api.py │ ├── static │ │ ├── admin │ │ │ ├── install_tabular.css │ │ │ ├── magic_wand.png │ │ │ └── map │ │ │ │ └── img │ │ │ │ ├── cross.png │ │ │ │ ├── handlebar.svg │ │ │ │ ├── map.png │ │ │ │ ├── map │ │ │ │ ├── active.svg │ │ │ │ ├── ap.svg │ │ │ │ ├── dead.svg │ │ │ │ ├── hub.svg │ │ │ │ ├── kiosk-5g.svg │ │ │ │ ├── kiosk.svg │ │ │ │ ├── omni.svg │ │ │ │ ├── pop.svg │ │ │ │ ├── potential-hub.svg │ │ │ │ ├── potential-supernode.svg │ │ │ │ ├── potential.svg │ │ │ │ ├── supernode.svg │ │ │ │ └── vpn.svg │ │ │ │ └── recenter.png │ │ └── widgets │ │ │ ├── auto_populate_location.css │ │ │ ├── auto_populate_location.js │ │ │ ├── flickity.min.css │ │ │ ├── flickity.pkgd.min.js │ │ │ ├── panorama_viewer.css │ │ │ ├── warn_about_date.css │ │ │ └── warn_about_date.js │ ├── tasks.py │ ├── templates │ │ ├── admin │ │ │ ├── install_tabular.html │ │ │ └── node_panorama_viewer.html │ │ └── widgets │ │ │ ├── auto_populate_location.html │ │ │ ├── dob_identifier.html │ │ │ ├── external_link.html │ │ │ ├── install_status.html │ │ │ ├── ip_address.html │ │ │ ├── panorama_viewer.html │ │ │ ├── uisp_link.html │ │ │ └── warn_about_date.html │ ├── templatetags │ │ ├── __init__.py │ │ └── env_extras.py │ ├── tests │ │ ├── __init__.py │ │ ├── group_helpers.py │ │ ├── sample_data.py │ │ ├── sample_geocode_response.py │ │ ├── sample_join_form_data.py │ │ ├── sample_join_records.py │ │ ├── sample_kiosk_data.py │ │ ├── test_accesspoint.py │ │ ├── test_admin_change_view.py │ │ ├── test_admin_list_view.py │ │ ├── test_admin_panel.py │ │ ├── test_admin_search_view.py │ │ ├── test_billing_datum.py │ │ ├── test_building.py │ │ ├── test_captchas.py │ │ ├── test_crawl_uisp.py │ │ ├── test_device.py │ │ ├── test_docs.py │ │ ├── test_foreign_key_views_and_perms.py │ │ ├── test_geocode_api.py │ │ ├── test_helpers.py │ │ ├── test_install.py │ │ ├── test_install_create_signals.py │ │ ├── test_join_form.py │ │ ├── test_join_record_processor_bad_env_vars.py │ │ ├── test_join_record_viewer.py │ │ ├── test_kml_endpoint.py │ │ ├── test_link.py │ │ ├── test_lookups.py │ │ ├── test_los.py │ │ ├── test_maintenance_mode.py │ │ ├── test_map_endpoints.py │ │ ├── test_member.py │ │ ├── test_meshweb.py │ │ ├── test_nn.py │ │ ├── test_node.py │ │ ├── test_pagination.py │ │ ├── test_query_form.py │ │ ├── test_replay_join_records.py │ │ ├── test_sector.py │ │ ├── test_serializers.py │ │ ├── test_settings.py │ │ ├── test_slack_notification.py │ │ ├── test_tasks.py │ │ ├── test_uisp_import.py │ │ ├── test_update_panos_github.py │ │ ├── test_validation.py │ │ ├── test_view_autocomplete.py │ │ ├── test_views_explorer.py │ │ ├── test_views_get.py │ │ ├── test_views_post_delete.py │ │ ├── test_website_stats.py │ │ ├── test_widgets.py │ │ └── util.py │ ├── types │ │ ├── __init__.py │ │ └── uisp_api │ │ │ ├── __init__.py │ │ │ ├── data_links.py │ │ │ ├── devices.py │ │ │ └── literals.py │ ├── urls.py │ ├── util │ │ ├── __init__.py │ │ ├── admin_notifications.py │ │ ├── constants.py │ │ ├── django_flag_decorator.py │ │ ├── django_pglocks.py │ │ ├── drf_renderer.py │ │ ├── drf_utils.py │ │ ├── events │ │ │ ├── __init__.py │ │ │ ├── join_requests_slack_channel.py │ │ │ └── osticket_creation.py │ │ ├── join_records.py │ │ ├── network_number.py │ │ ├── panoramas.py │ │ └── uisp_import │ │ │ ├── __init__.py │ │ │ ├── constants.py │ │ │ ├── fetch_uisp.py │ │ │ ├── sync_handlers.py │ │ │ ├── uisp.mesh.nycmesh.net.crt │ │ │ ├── update_objects.py │ │ │ └── utils.py │ ├── validation.py │ ├── views │ │ ├── __init__.py │ │ ├── autocomplete.py │ │ ├── forms.py │ │ ├── geography.py │ │ ├── helpers.py │ │ ├── lookups.py │ │ ├── map.py │ │ ├── model_api.py │ │ ├── query_api.py │ │ └── uisp_import.py │ ├── widgets.py │ └── zips.py ├── meshapi_hooks │ ├── __init__.py │ ├── admin.py │ ├── apps.py │ ├── hooks.py │ ├── migrations │ │ ├── 0001_initial.py │ │ ├── 0002_celeryserializerhook_and_more.py │ │ └── __init__.py │ ├── models.py │ ├── tasks.py │ └── tests │ │ ├── __init__.py │ │ └── test_webhooks.py ├── meshdb │ ├── __init__.py │ ├── admin.py │ ├── apps.py │ ├── asgi.py │ ├── celery.py │ ├── settings.py │ ├── templates │ │ ├── admin │ │ │ ├── base.html │ │ │ ├── base_site.html │ │ │ ├── iframed.html │ │ │ ├── login.html │ │ │ └── password_reset │ │ │ │ ├── password_reset_complete.html │ │ │ │ ├── password_reset_confirm.html │ │ │ │ ├── password_reset_done.html │ │ │ │ ├── password_reset_email.html │ │ │ │ ├── password_reset_email_subject.txt │ │ │ │ └── password_reset_form.html │ │ ├── drf_spectacular │ │ │ └── swagger_ui.html │ │ └── rest_framework │ │ │ └── login_base.html │ ├── urls.py │ ├── utils │ │ └── __init__.py │ ├── views.py │ └── wsgi.py └── meshweb │ ├── __init__.py │ ├── middleware.py │ ├── static │ ├── admin │ │ ├── admin_ext.css │ │ ├── iframe_check.js │ │ ├── iframed.css │ │ ├── intercept.js │ │ ├── map.js │ │ ├── mobile_check.js │ │ └── panel_url_check.js │ └── meshweb │ │ ├── developer.png │ │ ├── favicon.png │ │ ├── join_record_viewer.css │ │ ├── logo.svg │ │ ├── member.png │ │ ├── meshdb.png │ │ ├── styles.css │ │ ├── uisp_on_demand_form.css │ │ ├── uisp_on_demand_form.js │ │ ├── uispondemand.png │ │ └── volunteer.png │ ├── templates │ └── meshweb │ │ ├── footer.html │ │ ├── head.html │ │ ├── index.html │ │ ├── join_record_viewer.html │ │ ├── maintenance.html │ │ ├── nav.html │ │ └── uisp_on_demand_form.html │ ├── tests.py │ ├── urls.py │ └── views │ ├── __init__.py │ ├── explorer_redirect.py │ ├── index.py │ ├── join_record_viewer.py │ ├── maintenance.py │ ├── uisp_on_demand.py │ └── website_stats.py ├── tasks.py └── test-results └── .last-run.json /.codecov.yml: -------------------------------------------------------------------------------- 1 | coverage: 2 | status: 3 | project: 4 | default: 5 | target: 90% 6 | 7 | patch: 8 | default: 9 | target: 90% -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | omit = 3 | *apps.py, 4 | */migrations/*, 5 | settings.py, 6 | */tests/*, 7 | */types/*, 8 | src/meshdb/utils/spreadsheet_import/*, 9 | *wsgi.py, 10 | manage.py, 11 | venv/* 12 | .venv/* 13 | integ-tests/* 14 | -------------------------------------------------------------------------------- /.devcontainer.json: -------------------------------------------------------------------------------- 1 | { 2 | "image": "docker.io/python:3.11-bookworm", 3 | "runArgs": [ 4 | "--network=host", 5 | ], 6 | "postStartCommand": "pip install -e '.[dev]'" 7 | } -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # top-most EditorConfig file 2 | root = true 3 | 4 | # Unix-style newlines with a newline ending every file 5 | [*] 6 | end_of_line = lf 7 | insert_final_newline = true 8 | trim_trailing_whitespace = true 9 | -------------------------------------------------------------------------------- /.env.sample: -------------------------------------------------------------------------------- 1 | MESHDB_ENVIRONMENT=local 2 | DB_NAME=meshdb 3 | DB_USER=meshdb 4 | DB_PASSWORD=abcd1234 5 | DB_USER_RO=meshdb_ro 6 | DB_PASSWORD_RO=secret 7 | # Change to 'postgres' when using meshdb in docker-compose. Defaults to localhost 8 | # DB_HOST= 9 | DB_PORT=5432 10 | 11 | 12 | # For password reset emails 13 | SMTP_HOST= 14 | SMTP_PORT= 15 | SMTP_USER= 16 | SMTP_PASSWORD= 17 | 18 | # For local testing with Minio. Don't include unless you have minio configured 19 | # S3_ENDPOINT="http://127.0.0.1:9000" 20 | 21 | # Backups 22 | AWS_ACCESS_KEY_ID=sampleaccesskey 23 | AWS_SECRET_ACCESS_KEY=samplesecretkey 24 | 25 | # For replaying Join Records 26 | JOIN_RECORD_BUCKET_NAME="meshdb-join-form-log" 27 | JOIN_RECORD_PREFIX="join-form-submissions-dev" 28 | 29 | # Change to 'redis' when using meshdb in docker-compose. 30 | # Defaults to redis://localhost:6379/0 31 | # CELERY_BROKER= 32 | 33 | # DO NOT USE THIS KEY IN PRODUCTION 34 | DJANGO_SECRET_KEY=sapwnffdtj@6p)ghfw249dz+@e6f2#i+5gia8*7&nup(szt9hp 35 | # Change to pelias:3000 when using full docker-compose. 36 | # Defaults to http://localhost:6800/parser/parse 37 | # PELIAS_ADDRESS_PARSER_URL= 38 | 39 | # Secrets for the legacy forms endpoints 40 | NN_ASSIGN_PSK=localdev 41 | QUERY_PSK=localdev 42 | 43 | SITE_BASE_URL=http://localhost:8000 44 | 45 | # Integ Testing Credentials 46 | INTEG_TEST_MESHDB_API_TOKEN= 47 | 48 | # Comment this out to enter prod mode 49 | DEBUG=True 50 | DISABLE_PROFILING=False # Set to True to disable profiling. Profiling also requires DEBUG=True 51 | 52 | # Comment this out to allow edits to the panoramas in the admin panel 53 | DISALLOW_PANO_EDITS=True 54 | 55 | # https://github.com/settings/tokens 56 | PANO_GITHUB_TOKEN= 57 | 58 | # Docker compose environment variables 59 | # Set this to true in prod. Use false in dev 60 | COMPOSE_EXTERNAL_NETWORK=false 61 | # Set this to traefik-net in prod. Use api in dev 62 | COMPOSE_NETWORK_NAME=api 63 | 64 | UISP_URL=https://uisp.mesh.nycmesh.net/nms 65 | UISP_USER=nycmesh_readonly 66 | UISP_PASS= 67 | 68 | ADMIN_MAP_BASE_URL=http://adminmap.devdb.nycmesh.net 69 | MAP_BASE_URL=https://map.nycmesh.net 70 | LOS_URL=https://los.devdb.nycmesh.net 71 | FORMS_URL=https://forms.devdb.nycmesh.net 72 | 73 | OSTICKET_URL=https://support.nycmesh.net 74 | OSTICKET_API_TOKEN= 75 | OSTICKET_NEW_TICKET_ENDPOINT=https://devsupport.nycmesh.net/api/http.php/tickets.json 76 | 77 | SLACK_ADMIN_NOTIFICATIONS_WEBHOOK_URL= 78 | SLACK_JOIN_REQUESTS_CHANNEL_WEBHOOK_URL= 79 | 80 | RECAPTCHA_DISABLE_VALIDATION=True # Set this to false in production! 81 | RECAPTCHA_SERVER_SECRET_KEY_V2= 82 | RECAPTCHA_SERVER_SECRET_KEY_V3= 83 | RECAPTCHA_INVISIBLE_TOKEN_SCORE_THRESHOLD=0.5 84 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 120 3 | extend-ignore = E203 4 | per-file-ignores = **/__init__.py:F401,F403,src/meshapi_hooks/models.py:F401,F403,src/*/migrations/*.py:E501,src/*/tests/*.py:E501,src/meshapi_hooks/tests/test_webhooks.py:E402 5 | exclude = src/meshdb/utils/spreadsheet_import 6 | -------------------------------------------------------------------------------- /.github/workflows/build-push-image.yaml: -------------------------------------------------------------------------------- 1 | name: Build Image 2 | permissions: read-all 3 | 4 | on: 5 | workflow_call: 6 | inputs: 7 | environment: 8 | required: true 9 | type: string 10 | image_tag: 11 | required: true 12 | type: string 13 | outputs: 14 | image_digest: 15 | description: "Digest of the built image" 16 | value: ${{ jobs.push_to_registry_env.outputs.image_digest }} 17 | 18 | jobs: 19 | push_to_registry_env: 20 | name: Push Docker Image to Docker Hub 21 | runs-on: ubuntu-latest 22 | environment: ${{ inputs.environment }} 23 | outputs: 24 | image_digest: ${{ steps.build.outputs.digest }} 25 | steps: 26 | - name: Check out the repo 27 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 28 | 29 | - name: Log in to Docker Hub 30 | uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a 31 | with: 32 | username: ${{ secrets.DOCKER_USERNAME }} 33 | password: ${{ secrets.DOCKER_PASSWORD }} 34 | 35 | - name: Extract metadata (tags, labels) for Docker 36 | id: meta 37 | uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 38 | with: 39 | images: willnilges/meshdb 40 | 41 | - name: Build and push Docker image 42 | id: build 43 | uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 44 | with: 45 | context: . 46 | file: ./Dockerfile 47 | push: true 48 | tags: ${{ inputs.image_tag }} 49 | labels: ${{ steps.meta.outputs.labels }} 50 | -------------------------------------------------------------------------------- /.github/workflows/checkov.yaml: -------------------------------------------------------------------------------- 1 | name: Checkov 2 | 3 | on: 4 | push: 5 | branches: [ "main" ] 6 | pull_request: 7 | branches: [ "main" ] 8 | workflow_dispatch: 9 | 10 | permissions: read-all 11 | 12 | jobs: 13 | checkov-job: 14 | permissions: 15 | actions: read 16 | contents: read 17 | security-events: write 18 | statuses: none 19 | runs-on: ubuntu-latest 20 | name: checkov-action 21 | steps: 22 | - name: Checkout repo 23 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 24 | 25 | - name: Run Checkov action 26 | id: checkov 27 | uses: bridgecrewio/checkov-action@0549dc60bddd4c55cb85c6c3a07072e3cf2ca48e 28 | with: 29 | skip_check: CKV_DOCKER_2,CKV_DOCKER_3,CKV_SECRET_6 30 | quiet: true 31 | output_format: cli,sarif 32 | output_file_path: console,results.sarif 33 | download_external_modules: true 34 | skip_path: infra/helm/meshdb/charts 35 | 36 | - name: Upload SARIF file 37 | uses: github/codeql-action/upload-sarif@d39d31e687223d841ef683f52467bd88e9b21c14 # v3 38 | if: success() || failure() 39 | with: 40 | sarif_file: results.sarif 41 | -------------------------------------------------------------------------------- /.github/workflows/codeql.yaml: -------------------------------------------------------------------------------- 1 | name: "CodeQL" 2 | 3 | on: 4 | push: 5 | branches: [ "main" ] 6 | pull_request: 7 | branches: [ "main" ] 8 | schedule: 9 | - cron: '32 9 * * 6' 10 | 11 | permissions: read-all 12 | 13 | jobs: 14 | analyze: 15 | name: Analyze (${{ matrix.language }}) 16 | runs-on: 'ubuntu-latest' 17 | timeout-minutes: 360 18 | permissions: 19 | # required for all workflows 20 | security-events: write 21 | 22 | # required to fetch internal or private CodeQL packs 23 | packages: read 24 | 25 | # only required for workflows in private repositories 26 | actions: read 27 | contents: read 28 | 29 | strategy: 30 | fail-fast: false 31 | matrix: 32 | include: 33 | - language: javascript-typescript 34 | build-mode: none 35 | - language: python 36 | build-mode: none 37 | steps: 38 | - name: Checkout repository 39 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 40 | 41 | # Initializes the CodeQL tools for scanning. 42 | - name: Initialize CodeQL 43 | uses: github/codeql-action/init@d39d31e687223d841ef683f52467bd88e9b21c14 # @v3 44 | with: 45 | languages: ${{ matrix.language }} 46 | build-mode: ${{ matrix.build-mode }} 47 | 48 | - if: matrix.build-mode == 'manual' 49 | run: | 50 | echo 'If you are using a "manual" build mode for one or more of the' \ 51 | 'languages you are analyzing, replace this with the commands to build' \ 52 | 'your code, for example:' 53 | echo ' make bootstrap' 54 | echo ' make release' 55 | exit 1 56 | 57 | - name: Perform CodeQL Analysis 58 | uses: github/codeql-action/analyze@d39d31e687223d841ef683f52467bd88e9b21c14 # @v3 59 | with: 60 | category: "/language:${{matrix.language}}" 61 | -------------------------------------------------------------------------------- /.github/workflows/helm_lint.yaml: -------------------------------------------------------------------------------- 1 | name: Lint and Test Chart 2 | 3 | on: pull_request 4 | 5 | permissions: read-all 6 | 7 | jobs: 8 | lint-test: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - name: Checkout 12 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 13 | with: 14 | fetch-depth: 0 15 | 16 | - name: Set up Helm 17 | uses: azure/setup-helm@20d2b4f98d41febe2bbca46408499dbb535b6258 # v3 18 | with: 19 | version: v3.14.0 20 | 21 | - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4 22 | with: 23 | python-version: '3.12' 24 | check-latest: true 25 | 26 | - name: Set up chart-testing 27 | uses: helm/chart-testing-action@e6669bcd63d7cb57cb4380c33043eebe5d111992 # v2.6.1 28 | 29 | - name: Run chart-testing (list-changed) 30 | id: list-changed 31 | run: | 32 | changed=$(ct list-changed --target-branch ${{ github.event.repository.default_branch }}) 33 | if [[ -n "$changed" ]]; then 34 | echo "changed=true" >> "$GITHUB_OUTPUT" 35 | fi 36 | 37 | - name: Run chart-testing (lint) 38 | if: steps.list-changed.outputs.changed == 'true' 39 | run: ct lint --target-branch ${{ github.event.repository.default_branch }} 40 | 41 | - name: Create kind cluster 42 | if: steps.list-changed.outputs.changed == 'true' 43 | uses: helm/kind-action@dda0770415bac9fc20092cacbc54aa298604d140 # v1.8.0 44 | 45 | - name: Run chart-testing (install) 46 | if: steps.list-changed.outputs.changed == 'true' 47 | run: ct install --target-branch ${{ github.event.repository.default_branch }} 48 | -------------------------------------------------------------------------------- /.github/workflows/lint.yaml: -------------------------------------------------------------------------------- 1 | name: Lint 2 | 3 | on: [pull_request] 4 | 5 | permissions: read-all 6 | 7 | jobs: 8 | invoke_lint: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 12 | - name: Set up Python 13 | uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4 14 | with: 15 | python-version: '3.11' 16 | - name: "Upgrade pip" 17 | run: "pip install --upgrade pip" 18 | - name: "Print python version" 19 | run: "python --version" 20 | - name: "Install package" 21 | run: pip install ".[dev]" 22 | - name: "Run lint checks" 23 | run: invoke lint 24 | env: 25 | DJANGO_SECRET_KEY: SECRET_KEY 26 | -------------------------------------------------------------------------------- /.github/workflows/no_debug_allowed.yaml: -------------------------------------------------------------------------------- 1 | name: Make sure Debug mode is OFF! 2 | 3 | on: [pull_request] 4 | 5 | permissions: read-all 6 | 7 | jobs: 8 | is-debug-off: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 12 | - name: Set up Python 13 | uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4 14 | with: 15 | python-version: '3.11' 16 | - name: "Upgrade pip" 17 | run: "pip install --upgrade pip" 18 | - name: "Install package" 19 | run: pip install ".[dev]" 20 | - name: "Your PR is not running in debug mode" 21 | run: cd src && python3 -c "import meshdb.settings; assert meshdb.settings.DEBUG == False" 22 | -------------------------------------------------------------------------------- /.github/workflows/no_forgoten_migrations.yaml: -------------------------------------------------------------------------------- 1 | name: Make sure to run manage.py makemigrations if you change models 2 | 3 | on: [pull_request] 4 | 5 | permissions: read-all 6 | 7 | jobs: 8 | is-migration-diff-clean: 9 | runs-on: ubuntu-latest 10 | services: 11 | postgres: 12 | image: docker.io/postgres:15-bookworm 13 | env: 14 | POSTGRES_DB: nycmesh-dev 15 | POSTGRES_USER: nycmesh 16 | POSTGRES_PASSWORD: abcd1234 17 | POSTGRES_PORT: 5432 18 | ports: 19 | - 5432:5432 20 | options: >- 21 | --health-cmd pg_isready 22 | --health-interval 10s 23 | --health-timeout 5s 24 | --health-retries 5 25 | steps: 26 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 27 | - name: Set up Python 28 | uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4 29 | with: 30 | python-version: '3.11' 31 | - name: "Upgrade pip" 32 | run: "pip install --upgrade pip" 33 | - name: "Install package" 34 | run: pip install ".[dev]" 35 | - name: "You forgot to run manage.py makemigrations for model changes" 36 | env: 37 | DB_NAME: nycmesh-dev 38 | DB_USER: nycmesh 39 | DB_PASSWORD: abcd1234 40 | DB_HOST: localhost 41 | DB_PORT: 5432 42 | DJANGO_SECRET_KEY: k7j&!u07c%%97s!^a_6%mh_wbzo*$hl4lj_6c2ee6dk)y9!k88 43 | run: | 44 | python src/manage.py makemigrations meshapi meshapi_hooks --dry-run # Run extra time for debug output 45 | python src/manage.py makemigrations meshapi meshapi_hooks --dry-run | grep "No changes detected in apps" 46 | 47 | -------------------------------------------------------------------------------- /.github/workflows/publish-and-deploy-gamma.yaml: -------------------------------------------------------------------------------- 1 | name: Publish and Deploy Gamma 2 | 3 | on: 4 | push: 5 | branches: [ dev ] 6 | workflow_dispatch: 7 | branches: 8 | - dev 9 | 10 | permissions: read-all 11 | 12 | concurrency: 13 | group: ${{ github.workflow }}-${{ github.ref }} 14 | cancel-in-progress: true 15 | 16 | jobs: 17 | push_to_registry_gamma: 18 | name: Push to gamma1 19 | uses: ./.github/workflows/build-push-image.yaml 20 | with: 21 | environment: gamma1 22 | image_tag: willnilges/meshdb:gamma1 23 | secrets: inherit 24 | if: github.ref == 'refs/heads/dev' 25 | 26 | deploy_to_gamma1: 27 | name: Deploy to gamma1 28 | uses: ./.github/workflows/deploy-to-k8s.yaml 29 | with: 30 | environment: gamma1 31 | useTag: gamma1 32 | secrets: inherit 33 | needs: push_to_registry_gamma 34 | if: github.ref == 'refs/heads/dev' 35 | 36 | 37 | -------------------------------------------------------------------------------- /.github/workflows/publish-and-deploy.yaml: -------------------------------------------------------------------------------- 1 | name: Publish and Deploy 2 | # Combined these two workflows for visbility 3 | 4 | on: 5 | push: 6 | branches: [ main ] 7 | workflow_dispatch: 8 | branches: 9 | - main 10 | 11 | permissions: read-all 12 | 13 | concurrency: 14 | group: ${{ github.workflow }}-${{ github.ref }} 15 | cancel-in-progress: true 16 | 17 | jobs: 18 | # Dev and Prod use the same image 19 | push_to_registry_prod: 20 | name: Push to prod 21 | uses: ./.github/workflows/build-push-image.yaml 22 | with: 23 | environment: prod 24 | image_tag: willnilges/meshdb:main 25 | secrets: inherit 26 | if: github.ref == 'refs/heads/main' 27 | 28 | deploy_to_dev3: 29 | name: Deploy to dev 3 30 | uses: ./.github/workflows/deploy-to-k8s.yaml 31 | with: 32 | environment: dev3 33 | image_digest: ${{ needs.push_to_registry_prod.outputs.image_digest }} 34 | secrets: inherit 35 | needs: push_to_registry_prod 36 | if: github.ref == 'refs/heads/main' 37 | 38 | integration_test_dev3: 39 | name: Integration test dev 3 40 | uses: ./.github/workflows/run-integ-tests.yaml 41 | with: 42 | environment: dev3 43 | secrets: inherit 44 | needs: deploy_to_dev3 45 | if: github.ref == 'refs/heads/main' 46 | 47 | deploy_to_prod1: 48 | name: Deploy to prod1 49 | uses: ./.github/workflows/deploy-to-k8s.yaml 50 | with: 51 | environment: prod 52 | image_digest: ${{ needs.push_to_registry_prod.outputs.image_digest }} 53 | secrets: inherit 54 | needs: [push_to_registry_prod, integration_test_dev3] 55 | if: github.ref == 'refs/heads/main' 56 | 57 | deploy_to_prod2: 58 | name: Deploy to prod2 59 | uses: ./.github/workflows/deploy-to-k8s.yaml 60 | with: 61 | environment: prod2 62 | image_digest: ${{ needs.push_to_registry_prod.outputs.image_digest }} 63 | secrets: inherit 64 | needs: [push_to_registry_prod, deploy_to_prod1] 65 | if: github.ref == 'refs/heads/main' 66 | -------------------------------------------------------------------------------- /.github/workflows/run-integ-tests.yaml: -------------------------------------------------------------------------------- 1 | name: Run Integration Tests 2 | 3 | on: 4 | workflow_call: 5 | inputs: 6 | environment: 7 | required: true 8 | type: string 9 | 10 | permissions: read-all 11 | 12 | jobs: 13 | run-integ-tests: 14 | name: Run Integration Tests 15 | runs-on: ubuntu-latest 16 | environment: ${{ inputs.environment }} 17 | steps: 18 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 19 | - name: Set up Python 20 | uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4 21 | with: 22 | python-version: '3.11' 23 | - name: "Upgrade pip" 24 | run: "pip install --upgrade pip" 25 | - name: "Install package" 26 | run: pip install ".[dev]" 27 | - name: Run Integration Tests 28 | env: 29 | SITE_BASE_URL: ${{ vars.SITE_BASE_URL }} 30 | INTEG_TEST_MESHDB_API_TOKEN: ${{ secrets.INTEG_TEST_MESHDB_API_TOKEN }} 31 | run: | 32 | # Run integ tests (only if we are not deploying to prod, since these tests can write data) 33 | if ! [[ "${{ inputs.environment }}" =~ "prod" ]]; then 34 | pytest integ-tests 35 | else 36 | echo "This action should not be run against prod, is something wrong with the workflow config?" 37 | exit 1 38 | fi 39 | 40 | -------------------------------------------------------------------------------- /.github/workflows/run_django_tests.yaml: -------------------------------------------------------------------------------- 1 | name: Run Django Tests 2 | 3 | on: 4 | pull_request: 5 | push: 6 | branches: [ main ] 7 | 8 | permissions: read-all 9 | 10 | jobs: 11 | run-django-tests: 12 | runs-on: ubuntu-latest 13 | services: 14 | postgres: 15 | image: docker.io/postgres:15-bookworm 16 | env: 17 | POSTGRES_DB: nycmesh-dev 18 | POSTGRES_USER: nycmesh 19 | POSTGRES_PASSWORD: abcd1234 20 | POSTGRES_PORT: 5432 21 | ports: 22 | - 5432:5432 23 | options: >- 24 | --health-cmd pg_isready 25 | --health-interval 10s 26 | --health-timeout 5s 27 | --health-retries 5 28 | pelias: 29 | image: pelias/parser:latest 30 | ports: 31 | - 6800:3000 32 | redis: 33 | image: redis 34 | ports: 35 | - 6379:6379 36 | steps: 37 | - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 38 | - name: Set up Python 39 | uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4 40 | with: 41 | python-version: '3.11' 42 | - name: "Upgrade pip" 43 | run: "pip install --upgrade pip" 44 | - name: "Install package" 45 | run: pip install ".[dev]" 46 | - name: Run Django Tests 47 | env: 48 | DB_NAME: nycmesh-dev 49 | DB_USER: nycmesh 50 | DB_PASSWORD: abcd1234 51 | DB_HOST: localhost 52 | DB_PORT: 5432 53 | DJANGO_SECRET_KEY: k7j&!u07c%%97s!^a_6%mh_wbzo*$hl4lj_6c2ee6dk)y9!k88 54 | PELIAS_ADDRESS_PARSER_URL: http://localhost:6800/parser/parse 55 | QUERY_PSK: localdev 56 | NN_ASSIGN_PSK: localdev 57 | PANO_GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} 58 | DB_USER_RO: meshdb_ro 59 | DB_PASSWORD_RO: readonly 60 | JOIN_RECORD_BUCKET_NAME: meshdb-join-form-log 61 | JOIN_RECORD_PREFIX: dev-join-form-submissions 62 | AWS_ACCESS_KEY_ID: sampleaccesskey 63 | AWS_SECRET_ACCESS_KEY: samplesecretkey 64 | AWS_REGION: us-east-1 65 | run: coverage run src/manage.py test meshapi meshapi_hooks 66 | - name: Write coverage report to disk 67 | run: coverage html 68 | - name: Upload coverage reports to Codecov 69 | uses: codecov/codecov-action@e0b68c6749509c5f83f984dd99a76a1c1a231044 # v4.0.1 70 | with: 71 | token: ${{ secrets.CODECOV_TOKEN }} 72 | -------------------------------------------------------------------------------- /.github/workflows/scorecard.yml: -------------------------------------------------------------------------------- 1 | name: Scorecard supply-chain security 2 | on: 3 | # For Branch-Protection check. Only the default branch is supported. See 4 | # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection 5 | branch_protection_rule: 6 | # To guarantee Maintained check is occasionally updated. See 7 | # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained 8 | schedule: 9 | - cron: '41 17 * * 3' 10 | push: 11 | branches: [ "main" ] 12 | 13 | # Declare default permissions as read only. 14 | permissions: read-all 15 | 16 | jobs: 17 | analysis: 18 | name: Scorecard analysis 19 | runs-on: ubuntu-latest 20 | # `publish_results: true` only works when run from the default branch. conditional can be removed if disabled. 21 | if: github.event.repository.default_branch == github.ref_name || github.event_name == 'pull_request' 22 | permissions: 23 | # Needed to upload the results to code-scanning dashboard. 24 | security-events: write 25 | # Needed to publish results and get a badge (see publish_results below). 26 | id-token: write 27 | 28 | steps: 29 | - name: "Checkout code" 30 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 31 | with: 32 | persist-credentials: false 33 | 34 | - name: "Run analysis" 35 | uses: ossf/scorecard-action@f49aabe0b5af0936a0987cfb85d86b75731b0186 # v2.4.1 36 | with: 37 | results_file: results.sarif 38 | results_format: sarif 39 | # (Optional) "write" PAT token. Uncomment the `repo_token` line below if: 40 | # - you want to enable the Branch-Protection check on a *public* repository, or 41 | # - you are installing Scorecard on a *private* repository 42 | # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action?tab=readme-ov-file#authentication-with-fine-grained-pat-optional. 43 | # repo_token: ${{ secrets.SCORECARD_TOKEN }} 44 | 45 | # Public repositories: 46 | # - Publish results to OpenSSF REST API for easy access by consumers 47 | # - Allows the repository to include the Scorecard badge. 48 | # - See https://github.com/ossf/scorecard-action#publishing-results. 49 | publish_results: true 50 | 51 | # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF 52 | # format to the repository Actions tab. 53 | - name: "Upload artifact" 54 | uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 55 | with: 56 | name: SARIF file 57 | path: results.sarif 58 | retention-days: 5 59 | 60 | # Upload the results to GitHub's code scanning dashboard (optional). 61 | - name: "Upload to code-scanning" 62 | uses: github/codeql-action/upload-sarif@d39d31e687223d841ef683f52467bd88e9b21c14 # v3 63 | with: 64 | sarif_file: results.sarif 65 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.11-bookworm 2 | 3 | # For healthcheck 4 | RUN apt-get -y update; apt-get -y install netcat-openbsd postgresql-client-15 5 | 6 | WORKDIR /opt/meshdb 7 | 8 | COPY pyproject.toml . 9 | RUN mkdir src 10 | RUN pip install . 11 | 12 | RUN useradd -ms /bin/bash celery # Celery does not recommend running as root 13 | 14 | COPY ./scripts ./scripts 15 | 16 | # Doing it like this should enable both dev and prod to work fine 17 | COPY ./src/meshweb/static . 18 | 19 | COPY ./src . 20 | 21 | ENTRYPOINT ./scripts/entrypoint.sh && exec ddtrace-run gunicorn 'meshdb.wsgi' --workers 4 --timeout 120 --graceful-timeout 2 --bind=0.0.0.0:8081 22 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Andy Baumgartner 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /infra/README.md: -------------------------------------------------------------------------------- 1 | # Meshdb Environment Setup 2 | 3 | These instructions will set up a 4 node k3s cluster on proxmox. 4 | - 1 "manager" node for control plane and to be used for deployments. 5 | - 3 "agent" nodes to run services. 6 | 7 | 1. Setup a new cluster via [nycmeshnet/k8s-infra](https://github.com/nycmeshnet/k8s-infra). Get the ssh key of the mgr node via ssh-keyscan. 8 | 9 | 2. Create a new "environment" in this repo and add the required secrets to the "environment": 10 | 11 | | Name | Description | 12 | | -------- | ------- | 13 | | `ACCESS_KEY_ID` | Access key ID for s3 backups | 14 | | `SECRET_ACCESS_KEY` | Secret access key for s3 backups | 15 | | `BACKUP_S3_BUCKET_NAME` | Name of the s3 bucket to store backups | 16 | | `DJANGO_SECRET_KEY` | Django secret key | 17 | | `GH_TOKEN` | Github token for pulling down panoramas | 18 | | `NN_ASSIGN_PSK` | Legacy node number assign password | 19 | | `PG_PASSWORD` | meshdb postgres database password | 20 | | `QUERY_PSK` | Legacy query password | 21 | | `SSH_KNOWN_HOSTS` | Copy paste from `ssh-keyscan ` | 22 | | `SSH_PRIVATE_KEY` | SSH key for the mgr node. | 23 | | `SSH_TARGET_IP` | Mgr node IP | 24 | | `SSH_USER` | Mgr username for ssh | 25 | | `UISP_PSK` | UISP readonly password | 26 | | `UISP_USER` | UISP readonly username | 27 | | `WIREGUARD_ENDPOINT` | IP and port of the wireguard server for deployment in the format `:` | 28 | | `WIREGUARD_OVERLAY_NETWORK_IP` | Overlay network IP for wireguard server used for deployment | 29 | | `WIREGUARD_PEER_PUBLIC_KEY` | Public key of the wireguard server for deployment | 30 | | `WIREGUARD_PRIVATE_KEY` | Private key for connecting to wireguard for deployment | 31 | 32 | 3. Create a new environment in `.github/workflows/publish-and-deploy.yaml` 33 | 34 | 4. Run the deployment. 35 | 36 | 5. If you need a superuser, ssh into the mgr node and: `kubectl exec -it -n meshdb service/meshdb-meshweb python manage.py createsuperuser` 37 | 38 | 6. If you need to import data: `cat meshdb_export.sql | kubectl exec -it --tty -n meshdb pod/meshdb-postgres-.... -- psql -U meshdb -d meshdb` 39 | -------------------------------------------------------------------------------- /infra/helm/meshdb/.gitignore: -------------------------------------------------------------------------------- 1 | meshdb.yaml 2 | secret.values.yaml 3 | -------------------------------------------------------------------------------- /infra/helm/meshdb/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v2 2 | name: meshdb 3 | description: A Helm chart for Kubernetes 4 | 5 | type: application 6 | 7 | version: 0.1.0 8 | 9 | appVersion: "1.16.0" 10 | -------------------------------------------------------------------------------- /infra/helm/meshdb/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* 2 | Expand the name of the chart. 3 | */}} 4 | {{- define "meshdb.name" -}} 5 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} 6 | {{- end }} 7 | 8 | {{/* 9 | Create a default fully qualified app name. 10 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 11 | If release name contains chart name it will be used as a full name. 12 | */}} 13 | {{- define "meshdb.fullname" -}} 14 | {{- if .Values.fullnameOverride }} 15 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} 16 | {{- else }} 17 | {{- $name := default .Chart.Name .Values.nameOverride }} 18 | {{- if contains $name .Release.Name }} 19 | {{- .Release.Name | trunc 63 | trimSuffix "-" }} 20 | {{- else }} 21 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} 22 | {{- end }} 23 | {{- end }} 24 | {{- end }} 25 | 26 | {{/* 27 | Create chart name and version as used by the chart label. 28 | */}} 29 | {{- define "meshdb.chart" -}} 30 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} 31 | {{- end }} 32 | 33 | {{/* 34 | Common labels 35 | */}} 36 | {{- define "meshdb.labels" -}} 37 | helm.sh/chart: {{ include "meshdb.chart" . }} 38 | {{ include "meshdb.selectorLabels" . }} 39 | {{- if .Chart.AppVersion }} 40 | app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} 41 | {{- end }} 42 | app.kubernetes.io/managed-by: {{ .Release.Service }} 43 | {{- end }} 44 | 45 | {{/* 46 | Selector labels 47 | */}} 48 | {{- define "meshdb.selectorLabels" -}} 49 | app.kubernetes.io/name: {{ include "meshdb.name" . }} 50 | app.kubernetes.io/instance: {{ .Release.Name }} 51 | {{- end }} 52 | 53 | {{/* 54 | Create the name of the service account to use 55 | */}} 56 | {{- define "meshdb.serviceAccountName" -}} 57 | {{- if .Values.serviceAccount.create }} 58 | {{- default (include "meshdb.fullname" .) .Values.serviceAccount.name }} 59 | {{- else }} 60 | {{- default "default" .Values.serviceAccount.name }} 61 | {{- end }} 62 | {{- end }} 63 | 64 | {{- define "imagePullSecret" }} 65 | {{- with .Values.imageCredentials }} 66 | {{- printf "{\"auths\":{\"%s\":{\"username\":\"%s\",\"password\":\"%s\",\"email\":\"%s\",\"auth\":\"%s\"}}}" .registry .username .password .email (printf "%s:%s" .username .password | b64enc) | b64enc }} 67 | {{- end }} 68 | {{- end }} 69 | -------------------------------------------------------------------------------- /infra/helm/meshdb/templates/celery.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: StatefulSet 3 | metadata: 4 | name: {{ include "meshdb.fullname" . }}-celery-worker 5 | namespace: meshdb 6 | labels: 7 | {{- include "meshdb.labels" . | nindent 4 }} 8 | spec: 9 | replicas: {{ .Values.celery.replicaCount }} 10 | selector: 11 | matchLabels: 12 | {{- include "meshdb.selectorLabels" . | nindent 6 }} 13 | template: 14 | metadata: 15 | annotations: 16 | admission.datadoghq.com/python-lib.version: v2.17 17 | {{- with .Values.podAnnotations }} 18 | {{- toYaml . | nindent 8 }} 19 | {{- end }} 20 | labels: 21 | {{- include "meshdb.labels" . | nindent 8 }} 22 | {{- with .Values.podLabels }} 23 | {{- toYaml . | nindent 8 }} 24 | {{- end }} 25 | spec: 26 | 27 | affinity: 28 | nodeAffinity: 29 | requiredDuringSchedulingIgnoredDuringExecution: 30 | nodeSelectorTerms: 31 | - matchExpressions: 32 | - key: node-role.kubernetes.io/control-plane 33 | operator: NotIn 34 | values: 35 | - "true" 36 | {{- if .Values.imageCredentials }} 37 | imagePullSecrets: 38 | - name: pull-secret 39 | {{- end }} 40 | securityContext: 41 | {{- toYaml .Values.celery.podSecurityContext | nindent 8 }} 42 | containers: 43 | {{- range .Values.celery.containers }} 44 | - name: {{ .name }} 45 | securityContext: 46 | {{- toYaml $.Values.celery.securityContext | nindent 12 }} 47 | {{- if $.Values.celery.image.digest }} 48 | image: "{{ $.Values.celery.image.repository }}@{{ $.Values.celery.image.digest }}" 49 | {{- else }} 50 | image: "{{ $.Values.celery.image.repository }}:{{ $.Values.celery.image.tag | default $.Chart.AppVersion }}" 51 | {{- end }} 52 | imagePullPolicy: {{ $.Values.celery.image.pullPolicy }} 53 | resources: 54 | {{- toYaml .resources | nindent 12 }} 55 | command: {{ toJson .command }} 56 | env: 57 | - name: DD_SERVICE 58 | value: celery 59 | # TODO (willnilges): Fix this in a later PR. 60 | # https://github.com/nycmeshnet/meshdb/issues/519 61 | envFrom: 62 | - configMapRef: 63 | name: meshdbconfig 64 | - secretRef: 65 | name: meshdb-secrets 66 | {{- if .livenessProbe }} 67 | livenessProbe: 68 | {{- toYaml .livenessProbe | nindent 12 }} 69 | {{- end }} 70 | {{- if .readinessProbe }} 71 | readinessProbe: 72 | {{- toYaml .readinessProbe | nindent 12 }} 73 | {{- end }} 74 | {{- end }} 75 | -------------------------------------------------------------------------------- /infra/helm/meshdb/templates/configmap.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: meshdbconfig 5 | namespace: {{ .Values.meshdb_app_namespace }} 6 | data: 7 | MESHDB_ENVIRONMENT: {{ .Values.meshweb.environment }} 8 | DD_ENV: {{ .Values.meshweb.environment }} 9 | DD_TRACE_AGENT_URL: http://datadog-agent.datadog.svc.cluster.local:8126 10 | DB_NAME: {{ .Values.pg.dbname }} 11 | DB_USER: {{ .Values.pg.user | quote }} 12 | DB_USER_RO: {{ .Values.pg.user_ro | quote }} 13 | DB_HOST: {{ include "meshdb.fullname" . }}-postgres.{{ .Values.meshdb_app_namespace }}.svc.cluster.local 14 | DB_PORT: {{ .Values.pg.port | quote }} 15 | # Backups 16 | BACKUP_S3_BUCKET_NAME: {{ .Values.meshweb.backup_s3_bucket_name | quote }} 17 | BACKUP_S3_BASE_FOLDER: {{ .Values.meshweb.backup_s3_base_folder | quote }} 18 | 19 | SMTP_HOST: {{ .Values.email.smtp_host | quote }} 20 | SMTP_PORT: {{ .Values.email.smtp_port | quote }} 21 | SMTP_USER: {{ .Values.email.smtp_user | quote }} 22 | 23 | CELERY_BROKER: "redis://{{ include "meshdb.fullname" . }}-redis.{{ .Values.meshdb_app_namespace }}.svc.cluster.local:{{ .Values.redis.port }}/0" 24 | 25 | # Change to pelias:3000 when using full docker-compose 26 | PELIAS_ADDRESS_PARSER_URL: http://{{ include "meshdb.fullname" . }}-pelias.{{ .Values.meshdb_app_namespace }}.svc.cluster.local:{{ .Values.pelias.port }}/parser/parse 27 | 28 | # Comment this out to enter prod mode 29 | DEBUG: {{ .Values.meshweb.enable_debug | quote }} 30 | DISABLE_PROFILING: {{ .Values.meshweb.disable_profiling | quote }} 31 | 32 | UISP_URL: {{ .Values.uisp.url | quote }} 33 | UISP_USER: {{ .Values.uisp.user | quote }} 34 | 35 | ADMIN_MAP_BASE_URL: {{ .Values.adminmap.base_url | quote }} 36 | MAP_BASE_URL: {{ .Values.map.base_url | quote }} 37 | LOS_URL: {{ .Values.meshweb.los_url | quote }} 38 | FORMS_URL: {{ .Values.meshweb.forms_url | quote }} 39 | 40 | SITE_BASE_URL: {{ .Values.meshdb.site_base_url | quote }} 41 | 42 | RECAPTCHA_DISABLE_VALIDATION: {{ .Values.meshweb.recaptcha_disable | quote }} 43 | RECAPTCHA_INVISIBLE_TOKEN_SCORE_THRESHOLD: {{ .Values.meshweb.recaptcha_score_threshold | quote }} 44 | 45 | OSTICKET_NEW_TICKET_ENDPOINT: {{ .Values.meshweb.osticket_new_ticket_endpoint | quote }} 46 | -------------------------------------------------------------------------------- /infra/helm/meshdb/templates/ingress.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.ingress.enabled -}} 2 | {{- $fullName := include "meshdb.fullname" . -}} 3 | {{- $svcPort := .Values.nginx.port -}} 4 | {{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} 5 | {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} 6 | {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} 7 | {{- end }} 8 | {{- end }} 9 | {{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} 10 | apiVersion: networking.k8s.io/v1 11 | {{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} 12 | apiVersion: networking.k8s.io/v1beta1 13 | {{- else -}} 14 | apiVersion: extensions/v1beta1 15 | {{- end }} 16 | kind: Ingress 17 | metadata: 18 | name: {{ $fullName }} 19 | labels: 20 | {{- include "meshdb.labels" . | nindent 4 }} 21 | {{- with .Values.ingress.annotations }} 22 | annotations: 23 | {{- toYaml . | nindent 4 }} 24 | {{- end }} 25 | spec: 26 | {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} 27 | ingressClassName: {{ .Values.ingress.className }} 28 | {{- end }} 29 | {{- if .Values.ingress.tls }} 30 | tls: 31 | {{- range .Values.ingress.tls }} 32 | - hosts: 33 | {{- range .hosts }} 34 | - {{ . | quote }} 35 | {{- end }} 36 | secretName: {{ .secretName }} 37 | {{- end }} 38 | {{- end }} 39 | rules: 40 | {{- range .Values.ingress.hosts }} 41 | - host: {{ .host | quote }} 42 | http: 43 | paths: 44 | {{- range .paths }} 45 | - path: {{ .path }} 46 | {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} 47 | pathType: {{ .pathType }} 48 | {{- end }} 49 | backend: 50 | {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} 51 | service: 52 | name: {{ $.Values.ingress.targetService }} 53 | port: 54 | number: {{ $svcPort }} 55 | {{- else }} 56 | serviceName: {{ $fullName }} 57 | servicePort: {{ $svcPort }} 58 | {{- end }} 59 | {{- end }} 60 | {{- end }} 61 | {{- end }} 62 | -------------------------------------------------------------------------------- /infra/helm/meshdb/templates/meshweb_static_pvc.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: PersistentVolumeClaim 3 | metadata: 4 | name: {{ .Values.meshweb.static_pvc_name }} 5 | namespace: {{ .Values.meshdb_app_namespace }} 6 | spec: 7 | accessModes: 8 | - ReadWriteMany 9 | storageClassName: longhorn 10 | resources: 11 | requests: 12 | storage: {{ .Values.meshweb.static_pvc_size }} -------------------------------------------------------------------------------- /infra/helm/meshdb/templates/nginx.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: {{ include "meshdb.fullname" . }}-nginx 5 | namespace: {{ .Values.meshdb_app_namespace }} 6 | labels: 7 | name: meshdb-nginx 8 | {{- include "meshdb.labels" . | nindent 4 }} 9 | spec: 10 | replicas: 1 11 | selector: 12 | matchLabels: 13 | {{- include "meshdb.selectorLabels" . | nindent 6 }} 14 | revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} 15 | template: 16 | metadata: 17 | {{- with .Values.podAnnotations }} 18 | annotations: 19 | {{- toYaml . | nindent 8 }} 20 | {{- end }} 21 | labels: 22 | app: meshdb-nginx-app 23 | admission.datadoghq.com/enabled: "false" 24 | {{- include "meshdb.labels" . | nindent 8 }} 25 | {{- with .Values.podLabels }} 26 | {{- toYaml . | nindent 8 }} 27 | {{- end }} 28 | spec: 29 | securityContext: 30 | {{- toYaml .Values.nginx.podSecurityContext | nindent 8 }} 31 | {{- if .Values.imageCredentials }} 32 | imagePullSecrets: 33 | - name: pull-secret 34 | {{- end }} 35 | containers: 36 | - name: {{ .Chart.Name }}-nginx 37 | securityContext: 38 | {{- toYaml .Values.nginx.securityContext | nindent 12 }} 39 | {{- if .Values.nginx.image.digest }} 40 | image: "{{ .Values.nginx.image.repository }}@{{ .Values.nginx.image.digest }}" 41 | {{- else }} 42 | image: "{{ .Values.nginx.image.repository }}:{{ .Values.nginx.image.tag }}" 43 | {{- end }} 44 | imagePullPolicy: {{ .Values.nginx.image.pullPolicy }} 45 | ports: 46 | - name: nginx 47 | containerPort: {{ .Values.nginx.port }} 48 | protocol: TCP 49 | volumeMounts: 50 | - name: nginx-conf 51 | mountPath: /etc/nginx/conf.d/nginx.conf 52 | subPath: nginx.conf 53 | readOnly: true 54 | - name: static-data-vol 55 | mountPath: /var/www/html/static 56 | readOnly: true 57 | resources: 58 | {{- toYaml .Values.nginx.resources | nindent 12 }} 59 | volumes: 60 | - name: nginx-conf 61 | configMap: 62 | name: nginx-conf 63 | items: 64 | - key: nginx.conf 65 | path: nginx.conf 66 | - name: static-data-vol 67 | persistentVolumeClaim: 68 | claimName: {{ .Values.meshweb.static_pvc_name }} 69 | {{- with .Values.nginx.nodeSelector }} 70 | nodeSelector: 71 | {{- toYaml . | nindent 8 }} 72 | {{- end }} 73 | {{- with .Values.nginx.affinity }} 74 | affinity: 75 | {{- toYaml . | nindent 8 }} 76 | {{- end }} 77 | {{- with .Values.nginx.tolerations }} 78 | tolerations: 79 | {{- toYaml . | nindent 8 }} 80 | {{- end }} 81 | -------------------------------------------------------------------------------- /infra/helm/meshdb/templates/nginx_configmap.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: ConfigMap 3 | metadata: 4 | name: nginx-conf 5 | namespace: {{ .Values.meshdb_app_namespace }} 6 | data: 7 | nginx.conf: | 8 | server { 9 | listen {{ .Values.nginx.port }}; 10 | server_name {{ .Values.nginx.server_name }}; 11 | 12 | access_log /var/log/nginx/access.log; 13 | error_log /var/log/nginx/error.log debug; 14 | 15 | location = /favicon.ico { access_log off; log_not_found off; } 16 | location /static/ { 17 | root /var/www/html; 18 | } 19 | 20 | location / { 21 | proxy_pass http://{{ include "meshdb.fullname" . }}-meshweb.{{ .Values.meshdb_app_namespace }}.svc.cluster.local:{{ .Values.meshweb.port }}/; 22 | proxy_set_header Host $host; 23 | proxy_set_header X-Forwarded-Proto https; 24 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 25 | } 26 | } 27 | 28 | -------------------------------------------------------------------------------- /infra/helm/meshdb/templates/pelias.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: {{ include "meshdb.fullname" . }}-pelias 5 | namespace: {{ .Values.meshdb_app_namespace }} 6 | labels: 7 | {{- include "meshdb.labels" . | nindent 4 }} 8 | spec: 9 | replicas: 1 10 | selector: 11 | matchLabels: 12 | {{- include "meshdb.selectorLabels" . | nindent 6 }} 13 | revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} 14 | template: 15 | metadata: 16 | {{- with .Values.podAnnotations }} 17 | annotations: 18 | {{- toYaml . | nindent 8 }} 19 | {{- end }} 20 | labels: 21 | app: meshdb-pelias-app 22 | admission.datadoghq.com/enabled: "false" 23 | {{- include "meshdb.labels" . | nindent 8 }} 24 | {{- with .Values.podLabels }} 25 | {{- toYaml . | nindent 8 }} 26 | {{- end }} 27 | spec: 28 | securityContext: 29 | {{- toYaml .Values.pelias.podSecurityContext | nindent 8 }} 30 | {{- if .Values.imageCredentials }} 31 | imagePullSecrets: 32 | - name: pull-secret 33 | {{- end }} 34 | containers: 35 | - name: {{ .Chart.Name }}-pelias 36 | securityContext: 37 | {{- toYaml .Values.pelias.securityContext | nindent 12 }} 38 | {{- if .Values.pelias.image.digest }} 39 | image: "{{ .Values.pelias.image.repository }}@{{ .Values.pelias.image.digest }}" 40 | {{- else }} 41 | image: "{{ .Values.pelias.image.repository }}:{{ .Values.pelias.image.tag }}" 42 | {{- end }} 43 | imagePullPolicy: {{ .Values.pelias.image.pullPolicy }} 44 | ports: 45 | - name: pelias 46 | containerPort: {{ .Values.pelias.port }} 47 | protocol: TCP 48 | resources: 49 | {{- toYaml .Values.pelias.resources | nindent 12 }} 50 | {{- with .Values.pelias.nodeSelector }} 51 | nodeSelector: 52 | {{- toYaml . | nindent 8 }} 53 | {{- end }} 54 | {{- with .Values.pelias.affinity }} 55 | affinity: 56 | {{- toYaml . | nindent 8 }} 57 | {{- end }} 58 | {{- with .Values.pelias.tolerations }} 59 | tolerations: 60 | {{- toYaml . | nindent 8 }} 61 | {{- end }} 62 | -------------------------------------------------------------------------------- /infra/helm/meshdb/templates/postgres_pvc.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: PersistentVolumeClaim 3 | metadata: 4 | name: {{ .Values.pg.pvc_name }} 5 | namespace: {{ .Values.meshdb_app_namespace }} 6 | spec: 7 | accessModes: 8 | - ReadWriteOnce 9 | storageClassName: longhorn-encrypted 10 | resources: 11 | requests: 12 | storage: {{ .Values.pg.pvc_size }} -------------------------------------------------------------------------------- /infra/helm/meshdb/templates/pull_secret.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | metadata: 4 | name: pull-secret 5 | namespace: {{ .Values.meshdb_app_namespace }} 6 | type: kubernetes.io/dockerconfigjson 7 | data: 8 | .dockerconfigjson: {{ template "imagePullSecret" . }} 9 | -------------------------------------------------------------------------------- /infra/helm/meshdb/templates/redis.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: {{ include "meshdb.fullname" . }}-redis 5 | namespace: {{ .Values.meshdb_app_namespace }} 6 | labels: 7 | {{- include "meshdb.labels" . | nindent 4 }} 8 | spec: 9 | replicas: 1 10 | selector: 11 | matchLabels: 12 | {{- include "meshdb.selectorLabels" . | nindent 6 }} 13 | revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} 14 | template: 15 | metadata: 16 | annotations: 17 | ad.datadoghq.com/redis.checks: | 18 | { 19 | "redisdb": { 20 | "init_config": {}, 21 | "instances": [ 22 | { 23 | "host": "%%host%%", 24 | "port":"6379", 25 | "password":"%%env_REDIS_PASSWORD%%" 26 | } 27 | ] 28 | } 29 | } 30 | {{- with .Values.podAnnotations }} 31 | {{- toYaml . | nindent 8 }} 32 | {{- end }} 33 | labels: 34 | admission.datadoghq.com/enabled: "false" 35 | app: meshdb-redis-app 36 | {{- include "meshdb.labels" . | nindent 8 }} 37 | {{- with .Values.podLabels }} 38 | {{- toYaml . | nindent 8 }} 39 | {{- end }} 40 | spec: 41 | securityContext: 42 | {{- toYaml .Values.redis.podSecurityContext | nindent 8 }} 43 | {{- if .Values.imageCredentials }} 44 | imagePullSecrets: 45 | - name: pull-secret 46 | {{- end }} 47 | containers: 48 | - name: {{ .Chart.Name }}-redis 49 | securityContext: 50 | {{- toYaml .Values.redis.securityContext | nindent 12 }} 51 | {{- if .Values.redis.image.digest }} 52 | image: "{{ .Values.redis.image.repository }}@{{ .Values.redis.image.digest }}" 53 | {{- else }} 54 | image: "{{ .Values.redis.image.repository }}:{{ .Values.redis.image.tag }}" 55 | {{- end }} 56 | imagePullPolicy: {{ .Values.redis.image.pullPolicy }} 57 | ports: 58 | - name: redis 59 | containerPort: {{ .Values.redis.port }} 60 | protocol: TCP 61 | resources: 62 | {{- toYaml .Values.redis.resources | nindent 12 }} 63 | {{ if eq .Values.redis.liveness_probe "true" }} 64 | livenessProbe: 65 | exec: 66 | command: 67 | - "redis-cli" 68 | - "--raw" 69 | - "incr" 70 | - "ping" 71 | periodSeconds: 3 72 | initialDelaySeconds: 2 73 | timeoutSeconds: 3 74 | {{ end }} 75 | {{- with .Values.redis.nodeSelector }} 76 | nodeSelector: 77 | {{- toYaml . | nindent 8 }} 78 | {{- end }} 79 | {{- with .Values.redis.affinity }} 80 | affinity: 81 | {{- toYaml . | nindent 8 }} 82 | {{- end }} 83 | {{- with .Values.redis.tolerations }} 84 | tolerations: 85 | {{- toYaml . | nindent 8 }} 86 | {{- end }} 87 | -------------------------------------------------------------------------------- /infra/helm/meshdb/templates/secrets.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | metadata: 4 | name: meshdb-secrets 5 | namespace: {{ .Values.meshdb_app_namespace }} 6 | type: Opaque 7 | data: 8 | DB_PASSWORD: {{ .Values.pg.password | b64enc | quote }} 9 | DB_PASSWORD_RO: {{ .Values.pg.password_ro | b64enc | quote }} 10 | AWS_ACCESS_KEY_ID: {{ .Values.aws.access_key_id | b64enc | quote }} 11 | AWS_SECRET_ACCESS_KEY: {{ .Values.aws.secret_access_key | b64enc | quote }} 12 | SMTP_PASSWORD: {{ .Values.email.smtp_password | b64enc | quote }} 13 | DJANGO_SECRET_KEY: {{ .Values.meshweb.django_secret_key | b64enc | quote }} 14 | NN_ASSIGN_PSK: {{ .Values.meshweb.nn_assign_psk | b64enc | quote }} 15 | QUERY_PSK: {{ .Values.meshweb.query_psk | b64enc | quote }} 16 | PANO_GITHUB_TOKEN: {{ .Values.meshweb.pano_github_token | b64enc | quote }} 17 | UISP_PASS: {{ .Values.uisp.psk | b64enc | quote }} 18 | SLACK_ADMIN_NOTIFICATIONS_WEBHOOK_URL: {{ .Values.meshweb.slack_webhook | b64enc | quote }} 19 | SLACK_JOIN_REQUESTS_CHANNEL_WEBHOOK_URL: {{ .Values.meshweb.slack_join_webhook | b64enc | quote }} 20 | OSTICKET_API_TOKEN: {{ .Values.meshweb.osticket_api_token | b64enc | quote }} 21 | RECAPTCHA_SERVER_SECRET_KEY_V2: {{ .Values.meshweb.recaptcha_v2_secret | b64enc | quote }} 22 | RECAPTCHA_SERVER_SECRET_KEY_V3: {{ .Values.meshweb.recaptcha_v3_secret | b64enc | quote }} 23 | JOIN_RECORD_BUCKET_NAME: {{ .Values.meshweb.join_record_bucket_name | b64enc | quote }} 24 | JOIN_RECORD_PREFIX: {{ .Values.meshweb.join_record_prefix | b64enc | quote }} 25 | -------------------------------------------------------------------------------- /infra/helm/meshdb/templates/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | name: {{ include "meshdb.fullname" . }}-nginx 5 | namespace: {{ .Values.meshdb_app_namespace }} 6 | labels: 7 | {{- include "meshdb.labels" . | nindent 4 }} 8 | spec: 9 | ports: 10 | - port: {{ .Values.nginx.port }} 11 | targetPort: {{ .Values.nginx.port }} 12 | protocol: TCP 13 | name: nginx 14 | selector: 15 | app: meshdb-nginx-app 16 | --- 17 | apiVersion: v1 18 | kind: Service 19 | metadata: 20 | name: {{ include "meshdb.fullname" . }}-postgres 21 | namespace: {{ .Values.meshdb_app_namespace }} 22 | labels: 23 | {{- include "meshdb.labels" . | nindent 4 }} 24 | spec: 25 | ports: 26 | - port: {{ .Values.pg.port }} 27 | targetPort: {{ .Values.pg.port }} 28 | protocol: TCP 29 | name: postgres 30 | selector: 31 | app: meshdb-postgres-app 32 | --- 33 | apiVersion: v1 34 | kind: Service 35 | metadata: 36 | name: {{ include "meshdb.fullname" . }}-redis 37 | namespace: {{ .Values.meshdb_app_namespace }} 38 | labels: 39 | {{- include "meshdb.labels" . | nindent 4 }} 40 | spec: 41 | ports: 42 | - port: {{ .Values.redis.port }} 43 | targetPort: {{ .Values.redis.port }} 44 | protocol: TCP 45 | name: redis 46 | selector: 47 | app: meshdb-redis-app 48 | --- 49 | apiVersion: v1 50 | kind: Service 51 | metadata: 52 | name: {{ include "meshdb.fullname" . }}-pelias 53 | namespace: {{ .Values.meshdb_app_namespace }} 54 | labels: 55 | {{- include "meshdb.labels" . | nindent 4 }} 56 | spec: 57 | ports: 58 | - port: {{ .Values.pelias.port }} 59 | targetPort: {{ .Values.pelias.port }} 60 | protocol: TCP 61 | name: pelias 62 | selector: 63 | app: meshdb-pelias-app 64 | --- 65 | apiVersion: v1 66 | kind: Service 67 | metadata: 68 | name: {{ include "meshdb.fullname" . }}-meshweb 69 | namespace: {{ .Values.meshdb_app_namespace }} 70 | labels: 71 | {{- include "meshdb.labels" . | nindent 4 }} 72 | spec: 73 | ports: 74 | - port: {{ .Values.meshweb.port }} 75 | targetPort: {{ .Values.meshweb.port }} 76 | protocol: TCP 77 | name: meshweb-service 78 | selector: 79 | app: meshdb-meshweb-app 80 | -------------------------------------------------------------------------------- /integ-tests/meshapi_integ_test/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nycmeshnet/meshdb/a2f79331699ff26480e0d1123a387b6677df2fb1/integ-tests/meshapi_integ_test/__init__.py -------------------------------------------------------------------------------- /integ-tests/meshapi_integ_test/integ_test_case.py: -------------------------------------------------------------------------------- 1 | import os 2 | import unittest 3 | 4 | import requests 5 | from dotenv import load_dotenv 6 | 7 | load_dotenv() 8 | 9 | SITE_BASE_URL = os.environ["SITE_BASE_URL"] 10 | INTEG_TEST_MESHDB_API_TOKEN = os.environ["INTEG_TEST_MESHDB_API_TOKEN"] 11 | 12 | 13 | class IntegTestCase(unittest.TestCase): 14 | def __init__(self, *args, **kwargs): 15 | super().__init__(*args, **kwargs) 16 | self.authed_session = requests.Session() 17 | self.authed_session.headers = {"Authorization": f"Bearer {INTEG_TEST_MESHDB_API_TOKEN}"} 18 | 19 | def get_url(self, url_path): 20 | return SITE_BASE_URL + url_path 21 | -------------------------------------------------------------------------------- /integ-tests/meshapi_integ_test/test_api_200.py: -------------------------------------------------------------------------------- 1 | from .integ_test_case import IntegTestCase 2 | 3 | 4 | class TestAPI200(IntegTestCase): 5 | def test_api_200(self): 6 | response = self.authed_session.get(self.get_url("/api/v1/")) 7 | self.assertEqual(response.status_code, 200) 8 | -------------------------------------------------------------------------------- /nginx/app.conf: -------------------------------------------------------------------------------- 1 | server { 2 | listen 80; 3 | server_name db.grandsvc.mesh db.grandsvc.mesh.nycmesh.net; 4 | 5 | access_log /var/log/nginx/access.log; 6 | error_log /var/log/nginx/error.log debug; 7 | 8 | location = /favicon.ico { access_log off; log_not_found off; } 9 | location /static/ { 10 | root /var/www/html; 11 | } 12 | 13 | location / { 14 | proxy_pass http://meshdb:8081/; 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /sampledata/meshdb.kml: -------------------------------------------------------------------------------- 1 | 2 | NYCMesh (from MeshDB) 3 | 0 4 | 5 | https://db.nycmesh.net/api/v1/geography/whole-mesh.kml 6 | onRequest 7 | 8 | -------------------------------------------------------------------------------- /sampledata/meshdb_local.kml: -------------------------------------------------------------------------------- 1 | 2 | NYCMesh (from MeshDB) 3 | 0 4 | 5 | http://127.0.0.1:8000/api/v1/geography/whole-mesh.kml 6 | onRequest 7 | 8 | -------------------------------------------------------------------------------- /scripts/celery/celery_beat.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ddtrace-run celery -A meshdb beat -l DEBUG -s /tmp/celerybeat-schedule --pidfile=/tmp/celery-beat.pid 4 | -------------------------------------------------------------------------------- /scripts/celery/celery_worker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ddtrace-run celery -A meshdb worker -l DEBUG --uid $(id -u celery) 4 | -------------------------------------------------------------------------------- /scripts/celery/probes/celery_beat_liveness.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from pathlib import Path 3 | 4 | PID_FILE = Path("/tmp/celery-beat.pid") 5 | 6 | if not PID_FILE.is_file(): 7 | print("Celery beat PID file NOT found.") 8 | sys.exit(1) 9 | 10 | print("Celery beat PID file found.") 11 | sys.exit(0) 12 | -------------------------------------------------------------------------------- /scripts/celery/probes/celery_beat_readiness.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from pathlib import Path 3 | 4 | READINESS_FILE = Path("/tmp/celery_beat_ready") 5 | 6 | if not READINESS_FILE.is_file(): 7 | sys.exit(1) 8 | 9 | sys.exit(0) 10 | -------------------------------------------------------------------------------- /scripts/celery/probes/celery_liveness.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import time 3 | from pathlib import Path 4 | 5 | LIVENESS_FILE = Path("/tmp/celery_worker_heartbeat") 6 | 7 | if not LIVENESS_FILE.is_file(): 8 | print("Celery liveness file NOT found.") 9 | sys.exit(1) 10 | 11 | stats = LIVENESS_FILE.stat() 12 | heartbeat_timestamp = stats.st_mtime 13 | current_timestamp = time.time() 14 | time_diff = current_timestamp - heartbeat_timestamp 15 | 16 | if time_diff > 60: 17 | print("Celery Worker liveness file timestamp DOES NOT matches the given constraint.") 18 | sys.exit(1) 19 | 20 | print("Celery Worker liveness file found and timestamp matches the given constraint.") 21 | sys.exit(0) 22 | -------------------------------------------------------------------------------- /scripts/celery/probes/celery_readiness.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from pathlib import Path 3 | 4 | READINESS_FILE = Path("/tmp/celery_worker_ready") 5 | 6 | if not READINESS_FILE.is_file(): 7 | sys.exit(1) 8 | 9 | sys.exit(0) 10 | -------------------------------------------------------------------------------- /scripts/clear_history_tables.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | DOCKER_PG_COMMAND="docker exec -i meshdb-postgres-1 psql -U meshdb -d meshdb" 4 | tables=( 5 | "meshapi_historicalaccesspoint" 6 | "meshapi_historicalbuilding" 7 | "meshapi_historicalbuilding_nodes" 8 | "meshapi_historicaldevice" 9 | "meshapi_historicalinstall" 10 | "meshapi_historicallink" 11 | "meshapi_historicallos" 12 | "meshapi_historicalmember" 13 | "meshapi_historicalnode" 14 | "meshapi_historicalsector" 15 | ) 16 | 17 | set -ex 18 | 19 | for table_name in "${tables[@]}" 20 | do 21 | echo "TRUNCATE ${table_name} CASCADE;" | $DOCKER_PG_COMMAND 22 | done 23 | 24 | -------------------------------------------------------------------------------- /scripts/create_importable_datadump.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | DOCKER_PG_COMMAND="docker exec -i meshdb-postgres-1 pg_dump -U meshdb" 4 | DATA_DIR="./data/" 5 | tables=( 6 | "meshapi_accesspoint" 7 | "meshapi_building" 8 | "meshapi_building_nodes" 9 | "meshapi_device" 10 | "meshapi_historicalaccesspoint" 11 | "meshapi_historicalbuilding" 12 | "meshapi_historicalbuilding_nodes" 13 | "meshapi_historicaldevice" 14 | "meshapi_historicalinstall" 15 | "meshapi_historicallink" 16 | "meshapi_historicallos" 17 | "meshapi_historicalmember" 18 | "meshapi_historicalnode" 19 | "meshapi_historicalsector" 20 | "meshapi_install" 21 | "meshapi_link" 22 | "meshapi_los" 23 | "meshapi_member" 24 | "meshapi_node" 25 | "meshapi_sector" 26 | ) 27 | set -ex 28 | 29 | # Make sure our files exist. 30 | if [ ! -d "$DATA_DIR" ]; then 31 | echo "$DATA_DIR missing!" 32 | exit 1 33 | fi 34 | 35 | docker exec -i meshdb-postgres-1 pg_dump -U meshdb -d meshdb ${tables[@]/#/-t } > "$DATA_DIR/full_dump.sql" 36 | 37 | -------------------------------------------------------------------------------- /scripts/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo 'Waiting for Postgres' 4 | 5 | echo "DB_HOST: $DB_HOST" 6 | echo "DB_PORT: $DB_PORT" 7 | 8 | while ! nc -z $DB_HOST $DB_PORT; do 9 | sleep 0.1 10 | done 11 | 12 | echo 'DB started' 13 | 14 | echo 'Running Migrations...' 15 | python manage.py makemigrations 16 | python manage.py migrate 17 | 18 | echo 'Collecting Static Files...' 19 | python manage.py collectstatic --no-input 20 | 21 | echo 'Creating Groups...' 22 | python manage.py create_groups 23 | -------------------------------------------------------------------------------- /scripts/import_datadump.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | DOCKER_PG_COMMAND="docker exec -i meshdb-postgres-1 psql -U meshdb -d meshdb" 4 | DATA_DIR="./data/" 5 | tables=( 6 | "meshapi_los" 7 | "meshapi_link" 8 | "meshapi_accesspoint" 9 | "meshapi_sector" 10 | "meshapi_device" 11 | "meshapi_building_nodes" 12 | "meshapi_node" 13 | "meshapi_install" 14 | "meshapi_building" 15 | "meshapi_member" 16 | "meshapi_historicallos" 17 | "meshapi_historicallink" 18 | "meshapi_historicalaccesspoint" 19 | "meshapi_historicalsector" 20 | "meshapi_historicaldevice" 21 | "meshapi_historicalbuilding_nodes" 22 | "meshapi_historicalnode" 23 | "meshapi_historicalinstall" 24 | "meshapi_historicalbuilding" 25 | "meshapi_historicalmember" 26 | ) 27 | set -ex 28 | 29 | # Make sure our files exist. 30 | if [ ! -d "$DATA_DIR" ]; then 31 | echo "$DATA_DIR missing!" 32 | exit 1 33 | fi 34 | 35 | if [ ! -e "$DATA_DIR/full_dump.sql" ]; then 36 | echo "full_dump.sql is missing!" 37 | exit 1 38 | fi 39 | 40 | num_tables=${#tables[@]} 41 | for ((i = num_tables - 1; i >= 0; i--)); 42 | do 43 | $DOCKER_PG_COMMAND -c "DROP TABLE IF EXISTS ${tables[i]} CASCADE" 44 | done 45 | 46 | 47 | # Import the new data 48 | cat "$DATA_DIR/full_dump.sql" | $DOCKER_PG_COMMAND 49 | 50 | 51 | # Fix the auto numbering sequence for installs 52 | max_install_number=$(($(${DOCKER_PG_COMMAND} -c "SELECT MAX(install_number) FROM meshapi_install" -At) + 1)) 53 | ${DOCKER_PG_COMMAND} -c "ALTER SEQUENCE meshapi_install_install_number_seq RESTART WITH ${max_install_number}" 54 | -------------------------------------------------------------------------------- /scripts/reconcile_stripe_subscriptions.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import os 3 | import sys 4 | from csv import DictReader 5 | from typing import List, Optional 6 | 7 | import requests 8 | 9 | MESHDB_API_TOKEN = os.environ["MESHDB_API_TOKEN"] 10 | 11 | 12 | def find_install_numbers_by_stripe_email(stripe_email: str) -> List[str]: 13 | member_response = requests.get( 14 | f"https://db.nycmesh.net/api/v1/members/lookup/?email_address={stripe_email}", 15 | headers={"Authorization": f"Token {MESHDB_API_TOKEN}"}, 16 | ) 17 | member_response.raise_for_status() 18 | 19 | member_data = member_response.json() 20 | 21 | install_numbers = [] 22 | for member in member_data["results"]: 23 | for install in member["installs"]: 24 | install_detail_response = requests.get( 25 | f"https://db.nycmesh.net/api/v1/installs/{install['id']}", 26 | headers={"Authorization": f"Token {MESHDB_API_TOKEN}"}, 27 | ) 28 | install_detail_response.raise_for_status() 29 | install_detail = install_detail_response.json() 30 | 31 | if install_detail["status"] == "Active": 32 | install_numbers.append(str(install["install_number"])) 33 | 34 | return install_numbers 35 | 36 | 37 | def main(): 38 | if len(sys.argv) != 2: 39 | print(f"Usage: {sys.argv[0]} ") 40 | sys.exit(1) 41 | 42 | stripe_csv_filename = sys.argv[1] 43 | with open(stripe_csv_filename, "r") as csv_file: 44 | reader = DictReader(csv_file) 45 | 46 | with open("output.csv", "w") as output_file: 47 | writer = csv.DictWriter(output_file, fieldnames=list(reader.fieldnames) + ["install_nums"]) 48 | 49 | for row in reader: 50 | stripe_email = row["Customer Email"] 51 | if stripe_email: # Some rows are blank 52 | print(row["Customer Email"]) 53 | install_numbers = find_install_numbers_by_stripe_email(stripe_email) 54 | row["install_nums"] = ",".join(install_numbers) 55 | writer.writerow(row) 56 | 57 | 58 | if __name__ == "__main__": 59 | main() 60 | -------------------------------------------------------------------------------- /src/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """Django's command-line utility for administrative tasks.""" 3 | import os 4 | import sys 5 | 6 | from dotenv import load_dotenv 7 | 8 | load_dotenv() 9 | 10 | 11 | def main() -> None: 12 | """Run administrative tasks.""" 13 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "meshdb.settings") 14 | try: 15 | from django.core.management import execute_from_command_line 16 | except ImportError as exc: 17 | raise ImportError( 18 | "Couldn't import Django. Are you sure it's installed and " 19 | "available on your PYTHONPATH environment variable? Did you " 20 | "forget to activate a virtual environment?" 21 | ) from exc 22 | execute_from_command_line(sys.argv) 23 | 24 | 25 | if __name__ == "__main__": 26 | main() 27 | -------------------------------------------------------------------------------- /src/meshapi/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nycmeshnet/meshdb/a2f79331699ff26480e0d1123a387b6677df2fb1/src/meshapi/__init__.py -------------------------------------------------------------------------------- /src/meshapi/admin/__init__.py: -------------------------------------------------------------------------------- 1 | from .admin import * 2 | from .inlines import * 3 | from .models import * 4 | -------------------------------------------------------------------------------- /src/meshapi/admin/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | 3 | admin.site.site_header = "MeshDB Admin" 4 | admin.site.site_title = "MeshDB Admin Portal" 5 | admin.site.index_title = "Welcome to MeshDB Admin Portal" 6 | -------------------------------------------------------------------------------- /src/meshapi/admin/models/__init__.py: -------------------------------------------------------------------------------- 1 | from .access_point import * 2 | from .billing import * 3 | from .building import * 4 | from .device import * 5 | from .install import * 6 | from .link import * 7 | from .los import * 8 | from .member import * 9 | from .node import * 10 | from .sector import * 11 | -------------------------------------------------------------------------------- /src/meshapi/admin/models/access_point.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Optional, Type 2 | 3 | from django.contrib import admin 4 | from django.contrib.postgres.search import SearchVector 5 | from django.forms import Field, ModelForm 6 | from django.http import HttpRequest 7 | from import_export.admin import ExportActionMixin, ImportExportMixin 8 | from simple_history.admin import SimpleHistoryAdmin 9 | 10 | from meshapi.models import AccessPoint 11 | from meshapi.widgets import AutoPopulateLocationWidget, DeviceIPAddressWidget, ExternalHyperlinkWidget 12 | 13 | from ..ranked_search import RankedSearchMixin 14 | from .device import UISP_URL, DeviceAdmin, DeviceAdminForm, DeviceLinkInline 15 | 16 | 17 | class AccessPointAdminForm(DeviceAdminForm): 18 | auto_populate_location_field = Field( 19 | required=False, 20 | widget=AutoPopulateLocationWidget("Node"), 21 | ) 22 | 23 | class Meta: 24 | model = AccessPoint 25 | fields = "__all__" 26 | widgets = { 27 | "ip_address": DeviceIPAddressWidget(), 28 | "uisp_id": ExternalHyperlinkWidget( 29 | lambda uisp_id: f"{UISP_URL}/devices#id={uisp_id}&panelType=device-panel", 30 | title="View in UISP", 31 | ), 32 | } 33 | 34 | 35 | @admin.register(AccessPoint) 36 | class AccessPointAdmin(RankedSearchMixin, ImportExportMixin, ExportActionMixin, SimpleHistoryAdmin): 37 | form = AccessPointAdminForm 38 | search_fields = ["name__icontains", "@notes"] 39 | search_vector = SearchVector("name", weight="A") + SearchVector("notes", weight="D") 40 | list_display = [ 41 | "__str__", 42 | "name", 43 | "node", 44 | ] 45 | list_filter = [ 46 | "status", 47 | "install_date", 48 | ] 49 | inlines = [DeviceLinkInline] 50 | fieldsets = DeviceAdmin.fieldsets + [ 51 | ( 52 | "Location Attributes", 53 | { 54 | "fields": [ 55 | "auto_populate_location_field", 56 | "latitude", 57 | "longitude", 58 | "altitude", 59 | ] 60 | }, 61 | ), 62 | ] # type: ignore[assignment] 63 | 64 | def get_form( 65 | self, request: HttpRequest, obj: Optional[Any] = None, change: bool = False, **kwargs: Any 66 | ) -> Type[ModelForm]: 67 | form = super().get_form(request, obj, change, **kwargs) 68 | form.base_fields["auto_populate_location_field"].label = "" 69 | return form 70 | -------------------------------------------------------------------------------- /src/meshapi/admin/models/billing.py: -------------------------------------------------------------------------------- 1 | from django import forms 2 | from django.contrib import admin 3 | from django.contrib.postgres.search import SearchVector 4 | from import_export.admin import ExportActionMixin, ImportExportMixin 5 | from simple_history.admin import SimpleHistoryAdmin 6 | 7 | from meshapi.models.billing import InstallFeeBillingDatum 8 | 9 | from ..ranked_search import RankedSearchMixin 10 | 11 | 12 | class InstallFeeBillingDatumAdminForm(forms.ModelForm): 13 | class Meta: 14 | model = InstallFeeBillingDatum 15 | fields = "__all__" 16 | 17 | 18 | @admin.register(InstallFeeBillingDatum) 19 | class InstallFeeBillingDatumAdmin(RankedSearchMixin, ImportExportMixin, ExportActionMixin, SimpleHistoryAdmin): 20 | form = InstallFeeBillingDatumAdminForm 21 | search_fields = [ 22 | "invoice_number__iexact", 23 | # Also allow search by install, network number, street address, etc 24 | "install__node__network_number__iexact", 25 | "install__install_number__iexact", 26 | "install__building__street_address__icontains", 27 | "@notes", 28 | ] 29 | search_vector = ( 30 | SearchVector("invoice_number", weight="A") 31 | + SearchVector("install__node__network_number", weight="A") 32 | + SearchVector("install__install_number", weight="A") 33 | + SearchVector("install__building__street_address", weight="B") 34 | + SearchVector("notes", weight="D") 35 | ) 36 | list_display = ["__str__", "status", "billing_date", "invoice_number", "notes"] 37 | list_filter = ["status", "billing_date"] 38 | 39 | autocomplete_fields = ["install"] 40 | -------------------------------------------------------------------------------- /src/meshapi/admin/models/link.py: -------------------------------------------------------------------------------- 1 | from django import forms 2 | from django.contrib import admin 3 | from django.contrib.postgres.search import SearchVector 4 | from import_export.admin import ExportActionMixin, ImportExportMixin 5 | from simple_history.admin import SimpleHistoryAdmin 6 | 7 | from meshapi.models import Link 8 | 9 | from ..ranked_search import RankedSearchMixin 10 | 11 | 12 | class LinkAdminForm(forms.ModelForm): 13 | class Meta: 14 | model = Link 15 | fields = "__all__" 16 | widgets = { 17 | "description": forms.TextInput(), 18 | } 19 | 20 | 21 | @admin.register(Link) 22 | class LinkAdmin(RankedSearchMixin, ImportExportMixin, ExportActionMixin, SimpleHistoryAdmin): 23 | form = LinkAdminForm 24 | search_fields = [ 25 | "from_device__node__name__icontains", 26 | "to_device__node__name__icontains", 27 | "from_device__node__buildings__street_address__icontains", 28 | "to_device__node__buildings__street_address__icontains", 29 | "from_device__node__network_number__iexact", 30 | "to_device__node__network_number__iexact", 31 | "@notes", 32 | ] 33 | search_vector = ( 34 | SearchVector("from_device__node__network_number", weight="A") 35 | + SearchVector("to_device__node__network_number", weight="A") 36 | + SearchVector("from_device__node__name", weight="B") 37 | + SearchVector("to_device__node__name", weight="B") 38 | + SearchVector("from_device__node__buildings__street_address", weight="C") 39 | + SearchVector("to_device__node__buildings__street_address", weight="C") 40 | + SearchVector("notes", weight="D") 41 | ) 42 | list_display = ["__str__", "status", "from_device", "to_device", "description"] 43 | list_filter = ["status", "type"] 44 | 45 | autocomplete_fields = ["from_device", "to_device"] 46 | -------------------------------------------------------------------------------- /src/meshapi/admin/models/los.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | from typing import Optional 3 | 4 | from django import forms 5 | from django.contrib import admin 6 | from django.contrib.postgres.search import SearchVector 7 | from django.forms import ModelForm 8 | from django.http import HttpRequest 9 | from import_export.admin import ExportActionMixin, ImportExportMixin 10 | from simple_history.admin import SimpleHistoryAdmin 11 | 12 | from meshapi.models import LOS 13 | 14 | from ..ranked_search import RankedSearchMixin 15 | 16 | 17 | class LOSAdminForm(forms.ModelForm): 18 | class Meta: 19 | model = LOS 20 | fields = "__all__" 21 | 22 | 23 | @admin.register(LOS) 24 | class LOSAdmin(RankedSearchMixin, ImportExportMixin, ExportActionMixin, SimpleHistoryAdmin): 25 | form = LOSAdminForm 26 | search_fields = [ 27 | "from_building__nodes__network_number__iexact", 28 | "to_building__nodes__network_number__iexact", 29 | "from_building__installs__install_number__iexact", 30 | "to_building__installs__install_number__iexact", 31 | "from_building__nodes__name__icontains", 32 | "to_building__nodes__name__icontains", 33 | "from_building__street_address__icontains", 34 | "to_building__street_address__icontains", 35 | "@notes", 36 | ] 37 | search_vector = ( 38 | SearchVector("from_building__nodes__network_number", weight="A") 39 | + SearchVector("to_building__nodes__network_number", weight="A") 40 | + SearchVector("from_building__installs__install_number", weight="A") 41 | + SearchVector("to_building__installs__install_number", weight="A") 42 | + SearchVector("from_building__nodes__name", weight="B") 43 | + SearchVector("to_building__nodes__name", weight="B") 44 | + SearchVector("from_building__street_address", weight="B") 45 | + SearchVector("to_building__street_address", weight="B") 46 | + SearchVector("notes", weight="D") 47 | ) 48 | list_display = ["__str__", "source", "from_building", "to_building", "analysis_date"] 49 | list_filter = ["source"] 50 | 51 | autocomplete_fields = ["from_building", "to_building"] 52 | 53 | def get_form( 54 | self, 55 | request: HttpRequest, 56 | obj: Optional[LOS] = None, 57 | change: bool = False, 58 | **kwargs: dict, 59 | ) -> type[ModelForm]: 60 | form = super().get_form(request, obj, change, **kwargs) 61 | if not obj: 62 | # Autofill the form with today's date, unless we're editing 63 | # an existing object (so we don't accidentally mutate something) 64 | form.base_fields["analysis_date"].initial = datetime.date.today().isoformat() 65 | 66 | return form 67 | -------------------------------------------------------------------------------- /src/meshapi/admin/models/sector.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | from django.contrib.postgres.search import SearchVector 3 | from import_export.admin import ExportActionMixin, ImportExportMixin 4 | from simple_history.admin import SimpleHistoryAdmin 5 | 6 | from meshapi.models import Sector 7 | from meshapi.widgets import ExternalHyperlinkWidget 8 | 9 | from ..ranked_search import RankedSearchMixin 10 | from .device import UISP_URL, DeviceAdmin, DeviceAdminForm, DeviceLinkInline 11 | 12 | 13 | class SectorAdminForm(DeviceAdminForm): 14 | class Meta: 15 | model = Sector 16 | fields = "__all__" 17 | widgets = { 18 | "uisp_id": ExternalHyperlinkWidget( 19 | lambda uisp_id: f"{UISP_URL}/devices#id={uisp_id}&panelType=device-panel", 20 | title="View in UISP", 21 | ), 22 | } 23 | 24 | 25 | @admin.register(Sector) 26 | class SectorAdmin(RankedSearchMixin, ImportExportMixin, ExportActionMixin, SimpleHistoryAdmin): 27 | form = SectorAdminForm 28 | search_fields = ["name__icontains", "@notes"] 29 | search_vector = SearchVector("name", weight="A") + SearchVector("notes", weight="D") 30 | list_display = [ 31 | "__str__", 32 | "name", 33 | ] 34 | list_filter = [ 35 | "status", 36 | "install_date", 37 | ] 38 | inlines = [DeviceLinkInline] 39 | fieldsets = DeviceAdmin.fieldsets + [ 40 | ( 41 | "Sector Attributes", 42 | { 43 | "fields": [ 44 | "radius", 45 | "azimuth", 46 | "width", 47 | ] 48 | }, 49 | ), 50 | ] # type: ignore[assignment] 51 | -------------------------------------------------------------------------------- /src/meshapi/admin/password_reset.py: -------------------------------------------------------------------------------- 1 | from django.contrib.auth.views import ( 2 | PasswordResetCompleteView, 3 | PasswordResetConfirmView, 4 | PasswordResetDoneView, 5 | PasswordResetView, 6 | ) 7 | 8 | 9 | class AdminPasswordResetView(PasswordResetView): 10 | subject_template_name = "admin/password_reset/password_reset_email_subject.txt" 11 | html_email_template_name = "admin/password_reset/password_reset_email.html" 12 | template_name = "admin/password_reset/password_reset_form.html" 13 | 14 | 15 | class AdminPasswordResetDoneView(PasswordResetDoneView): 16 | template_name = "admin/password_reset/password_reset_done.html" 17 | 18 | 19 | class AdminPasswordResetConfirmView(PasswordResetConfirmView): 20 | template_name = "admin/password_reset/password_reset_confirm.html" 21 | 22 | 23 | class AdminPasswordResetCompleteView(PasswordResetCompleteView): 24 | template_name = "admin/password_reset/password_reset_complete.html" 25 | -------------------------------------------------------------------------------- /src/meshapi/admin/urls.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | from django.urls import path 3 | 4 | from meshapi.admin.password_reset import ( 5 | AdminPasswordResetCompleteView, 6 | AdminPasswordResetConfirmView, 7 | AdminPasswordResetDoneView, 8 | AdminPasswordResetView, 9 | ) 10 | from meshdb.views import admin_iframe_view 11 | 12 | urlpatterns = [ 13 | path("password_reset/", AdminPasswordResetView.as_view(), name="admin_password_reset"), 14 | path("password_reset/done/", AdminPasswordResetDoneView.as_view(), name="password_reset_done"), 15 | path("password_reset///", AdminPasswordResetConfirmView.as_view(), name="password_reset_confirm"), 16 | path("password_reset/done/", AdminPasswordResetCompleteView.as_view(), name="password_reset_complete"), 17 | path("iframe_wrapper/", admin_iframe_view), 18 | path("", admin.site.urls), 19 | ] 20 | -------------------------------------------------------------------------------- /src/meshapi/admin/utils.py: -------------------------------------------------------------------------------- 1 | from typing import Union 2 | from urllib.parse import urljoin 3 | 4 | from django.contrib.contenttypes.models import ContentType 5 | from django.db.models import Model 6 | from django.urls import reverse 7 | 8 | from meshapi.models import AccessPoint, Device, Sector 9 | 10 | 11 | def get_admin_url(model: Model, site_base_url: str) -> str: 12 | """ 13 | Get the admin URL corresponding to the given model 14 | """ 15 | content_type = ContentType.objects.get_for_model(model.__class__) 16 | return urljoin( 17 | site_base_url, 18 | reverse( 19 | f"admin:{content_type.app_label}_{content_type.model}_change", 20 | args=(model.pk,), 21 | ), 22 | ) 23 | 24 | 25 | def downclass_device(device: Device) -> Union[Device, Sector, AccessPoint]: 26 | """ 27 | Every sector and AP is also a device because we are using model inheritance. This function takes 28 | a device object and returns the leaf object that this device corresponds to. 29 | 30 | That is, it looks up the device ID in the sector and AP tables, and returns the appropriate 31 | object type if it exists. Returns the input device if no such leaf exists 32 | :param device: the device to use to search for leaf objects 33 | :return: the leaf object corresponding to this device 34 | """ 35 | sector = Sector.objects.filter(device_ptr=device).first() 36 | access_point = AccessPoint.objects.filter(device_ptr=device).first() 37 | 38 | db_object = device 39 | if sector: 40 | db_object = sector 41 | elif access_point: 42 | db_object = access_point 43 | 44 | return db_object 45 | -------------------------------------------------------------------------------- /src/meshapi/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class MeshapiConfig(AppConfig): 5 | default_auto_field = "django.db.models.BigAutoField" 6 | name = "meshapi" 7 | 8 | def ready(self) -> None: 9 | # Implicitly connect signal handlers decorated with @receiver. 10 | from meshapi.util import events # noqa: F401 11 | -------------------------------------------------------------------------------- /src/meshapi/exceptions.py: -------------------------------------------------------------------------------- 1 | class MeshDBError(Exception): 2 | pass 3 | 4 | 5 | # Used in Validation to warn of an invalid address 6 | class AddressError(MeshDBError): 7 | pass 8 | 9 | 10 | # Used in Validation to warn that one of the APIs we depend on might be 11 | # borked. 12 | class AddressAPIError(MeshDBError): 13 | pass 14 | 15 | 16 | # Used when something goes wrong with NYC Open Data 17 | class OpenDataAPIError(MeshDBError): 18 | pass 19 | -------------------------------------------------------------------------------- /src/meshapi/management/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nycmeshnet/meshdb/a2f79331699ff26480e0d1123a387b6677df2fb1/src/meshapi/management/__init__.py -------------------------------------------------------------------------------- /src/meshapi/management/commands/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nycmeshnet/meshdb/a2f79331699ff26480e0d1123a387b6677df2fb1/src/meshapi/management/commands/__init__.py -------------------------------------------------------------------------------- /src/meshapi/management/commands/create_groups.py: -------------------------------------------------------------------------------- 1 | from argparse import ArgumentParser 2 | from typing import Any 3 | 4 | from django.contrib.auth.models import Group, Permission 5 | from django.core.management.base import BaseCommand 6 | 7 | 8 | class Command(BaseCommand): 9 | help = "Creates basic MeshDB groups" 10 | 11 | def add_arguments(self, parser: ArgumentParser) -> None: 12 | pass 13 | 14 | def handle(self, *args: Any, **options: Any) -> None: 15 | self.create_base_groups() 16 | self.create_extra_groups() 17 | 18 | def create_base_groups(self) -> None: 19 | models = [ 20 | "building", 21 | "member", 22 | "install", 23 | "node", 24 | "device", 25 | "link", 26 | "los", 27 | "sector", 28 | "accesspoint", 29 | ] 30 | all_permissions = Permission.objects.all() 31 | 32 | billing_models = ["installfeebillingdatum"] 33 | 34 | admin, _ = Group.objects.get_or_create(name="Admin") 35 | installer, _ = Group.objects.get_or_create(name="Installer") 36 | read_only, _ = Group.objects.get_or_create(name="Read Only") 37 | 38 | billing_editor, _ = Group.objects.get_or_create(name="Billing Editor") 39 | 40 | for p in all_permissions: 41 | code = p.codename 42 | 43 | act, obj = code.split("_", maxsplit=1) 44 | 45 | # read_only 46 | if act == "view" and (obj in models or obj in billing_models): 47 | read_only.permissions.add(p) 48 | installer.permissions.add(p) 49 | 50 | # installer 51 | if (act == "change" and obj in models) or code == "assign_nn": 52 | installer.permissions.add(p) 53 | 54 | if act == "add" and obj in ["accesspoint", "link"]: 55 | installer.permissions.add(p) 56 | 57 | # billing editor 58 | if obj in billing_models: 59 | billing_editor.permissions.add(p) 60 | 61 | # admin 62 | if ( 63 | obj in models 64 | or act == "view" 65 | or obj in ["user", "token", "tokenproxy", "celeryserializerhook"] 66 | or code == "assign_nn" 67 | or code == "update_panoramas" 68 | ): 69 | admin.permissions.add(p) 70 | 71 | def create_extra_groups(self) -> None: 72 | all_permissions = Permission.objects.all() 73 | 74 | # Loop again to make groups for specific actions 75 | explorer, _ = Group.objects.get_or_create(name="Explorer Access") 76 | 77 | for p in all_permissions: 78 | code = p.codename 79 | if "explorer_access" in code: 80 | explorer.permissions.add(p) 81 | -------------------------------------------------------------------------------- /src/meshapi/management/commands/sync_panoramas.py: -------------------------------------------------------------------------------- 1 | from argparse import ArgumentParser 2 | from typing import Any 3 | 4 | from datadog import statsd 5 | from django.core.management.base import BaseCommand 6 | 7 | from meshapi.util.panoramas import sync_github_panoramas 8 | 9 | 10 | class Command(BaseCommand): 11 | help = "Syncs panoramas to MeshDB" 12 | 13 | def add_arguments(self, parser: ArgumentParser) -> None: 14 | pass 15 | 16 | def handle(self, *args: Any, **options: Any) -> None: 17 | statsd.increment("meshdb.commands.sync_panoramas", tags=[]) 18 | print("Syncing panoramas from GitHub...") 19 | panoramas_saved, warnings = sync_github_panoramas() 20 | print(f"Saved {panoramas_saved} panoramas. Got {len(warnings)} warnings.") 21 | print(f"warnings:\n{warnings}") 22 | -------------------------------------------------------------------------------- /src/meshapi/migrations/0003_alter_historicalinstall_request_date_and_more.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.2.16 on 2024-11-22 19:14 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | dependencies = [ 8 | ("meshapi", "0002_create_meshdb_ro"), 9 | ] 10 | 11 | operations = [ 12 | migrations.AlterField( 13 | model_name="historicalinstall", 14 | name="request_date", 15 | field=models.DateTimeField(help_text="The date and time that this install request was received"), 16 | ), 17 | migrations.AlterField( 18 | model_name="install", 19 | name="request_date", 20 | field=models.DateTimeField(help_text="The date and time that this install request was received"), 21 | ), 22 | ] 23 | -------------------------------------------------------------------------------- /src/meshapi/migrations/0005_alter_install_options.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.2.17 on 2025-01-08 01:59 2 | 3 | from django.db import migrations 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ("meshapi", "0004_alter_historicalinstall_status_and_more"), 10 | ] 11 | 12 | operations = [ 13 | migrations.AlterModelOptions( 14 | name="install", 15 | options={ 16 | "ordering": ["-install_number"], 17 | "permissions": [ 18 | ("assign_nn", "Can assign an NN to install"), 19 | ("disambiguate_number", "Can disambiguate an install number from an NN"), 20 | ("update_panoramas", "Can update panoramas"), 21 | ], 22 | }, 23 | ), 24 | ] 25 | -------------------------------------------------------------------------------- /src/meshapi/migrations/0006_install_additional_members.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.2.19 on 2025-03-11 04:38 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ("meshapi", "0005_alter_install_options"), 10 | ] 11 | 12 | operations = [ 13 | migrations.AddField( 14 | model_name="install", 15 | name="additional_members", 16 | field=models.ManyToManyField( 17 | blank=True, 18 | help_text="Any additional members associated with this install. E.g. roommates, parents, caretakers etc. Anyone that might contact us on behalf of this install belongs here", 19 | related_name="additional_installs", 20 | to="meshapi.member", 21 | ), 22 | ), 23 | ] 24 | -------------------------------------------------------------------------------- /src/meshapi/migrations/0008_historicalinstall_stripe_subscription_id_and_more.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.2.17 on 2025-04-23 01:37 2 | 3 | import django.core.validators 4 | from django.db import migrations, models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ("meshapi", "0007_installfeebillingdatum_and_more"), 11 | ] 12 | 13 | operations = [ 14 | migrations.AddField( 15 | model_name="historicalinstall", 16 | name="stripe_subscription_id", 17 | field=models.CharField( 18 | blank=True, 19 | help_text="The Stripe.com Subscription ID for the monthly contributions associated with this install. The presence of a subscription id here does not imply an active subscription, to determine if a subscription is active, one must contact Stripe directly with this identifier", 20 | null=True, 21 | validators=[ 22 | django.core.validators.RegexValidator("^sub_\\w{24,254}$", "Invalid Stripe subscription ID") 23 | ], 24 | ), 25 | ), 26 | migrations.AddField( 27 | model_name="install", 28 | name="stripe_subscription_id", 29 | field=models.CharField( 30 | blank=True, 31 | help_text="The Stripe.com Subscription ID for the monthly contributions associated with this install. The presence of a subscription id here does not imply an active subscription, to determine if a subscription is active, one must contact Stripe directly with this identifier", 32 | null=True, 33 | validators=[ 34 | django.core.validators.RegexValidator("^sub_\\w{24,254}$", "Invalid Stripe subscription ID") 35 | ], 36 | ), 37 | ), 38 | ] 39 | -------------------------------------------------------------------------------- /src/meshapi/migrations/0009_alter_historicalnode_network_number_and_more.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.2.17 on 2025-01-07 04:05 2 | 3 | import django.core.validators 4 | from django.db import migrations, models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ("meshapi", "0008_historicalinstall_stripe_subscription_id_and_more"), 11 | ] 12 | 13 | operations = [ 14 | migrations.AlterField( 15 | model_name="historicalnode", 16 | name="network_number", 17 | field=models.IntegerField( 18 | blank=True, db_index=True, null=True, validators=[django.core.validators.MaxValueValidator(8000)] 19 | ), 20 | ), 21 | migrations.AlterField( 22 | model_name="node", 23 | name="network_number", 24 | field=models.IntegerField( 25 | blank=True, null=True, unique=True, validators=[django.core.validators.MaxValueValidator(8000)] 26 | ), 27 | ), 28 | ] 29 | -------------------------------------------------------------------------------- /src/meshapi/migrations/0010_alter_historicallink_type_alter_link_type.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.2.19 on 2025-05-30 00:32 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ("meshapi", "0009_alter_historicalnode_network_number_and_more"), 10 | ] 11 | 12 | operations = [ 13 | migrations.AlterField( 14 | model_name="historicallink", 15 | name="type", 16 | field=models.CharField( 17 | blank=True, 18 | choices=[ 19 | ("5 GHz", "5 GHz"), 20 | ("6 GHz", "6 GHz"), 21 | ("24 GHz", "24 GHz"), 22 | ("60 GHz", "60 GHz"), 23 | ("70-80 GHz", "70-80 GHz"), 24 | ("VPN", "VPN"), 25 | ("Fiber", "Fiber"), 26 | ("Ethernet", "Ethernet"), 27 | ], 28 | default=None, 29 | help_text="The technology used for this link 5Ghz, 60Ghz, fiber, etc.", 30 | null=True, 31 | ), 32 | ), 33 | migrations.AlterField( 34 | model_name="link", 35 | name="type", 36 | field=models.CharField( 37 | blank=True, 38 | choices=[ 39 | ("5 GHz", "5 GHz"), 40 | ("6 GHz", "6 GHz"), 41 | ("24 GHz", "24 GHz"), 42 | ("60 GHz", "60 GHz"), 43 | ("70-80 GHz", "70-80 GHz"), 44 | ("VPN", "VPN"), 45 | ("Fiber", "Fiber"), 46 | ("Ethernet", "Ethernet"), 47 | ], 48 | default=None, 49 | help_text="The technology used for this link 5Ghz, 60Ghz, fiber, etc.", 50 | null=True, 51 | ), 52 | ), 53 | ] 54 | -------------------------------------------------------------------------------- /src/meshapi/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nycmeshnet/meshdb/a2f79331699ff26480e0d1123a387b6677df2fb1/src/meshapi/migrations/__init__.py -------------------------------------------------------------------------------- /src/meshapi/models/__init__.py: -------------------------------------------------------------------------------- 1 | from .billing import * 2 | from .building import * 3 | from .devices import * 4 | from .install import * 5 | from .link import * 6 | from .los import * 7 | from .member import * 8 | from .node import * 9 | from .permission import * 10 | -------------------------------------------------------------------------------- /src/meshapi/models/billing.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | 3 | from django.db import models 4 | from simple_history.models import HistoricalRecords 5 | 6 | 7 | class InstallFeeBillingDatum(models.Model): 8 | """ 9 | For some installs, another organization is responsible for paying install fees, rather than the 10 | resident of the apartment we have installed. This object tracks additional data relevant to that 11 | situation, such as when/if the invoice has been sent, what invoice a given install was 12 | included on, etc. 13 | """ 14 | 15 | class Meta: 16 | verbose_name = "Install Fee Billing Datum" 17 | verbose_name_plural = "Install Fee Billing Data" 18 | 19 | history = HistoricalRecords() 20 | 21 | class BillingStatus(models.TextChoices): 22 | TO_BE_BILLED = "ToBeBilled", "To Be Billed" 23 | BILLED = "Billed", "Billed" 24 | NOT_BILLING_DUPLICATE = "NotBillingDuplicate", "Not Billing - Duplicate" 25 | NOT_BILLING_OTHER = "NotBillingOther", "Not Billing - Other" 26 | 27 | id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) 28 | 29 | install = models.OneToOneField( 30 | "Install", 31 | related_name="install_fee_billing_datum", 32 | on_delete=models.PROTECT, 33 | help_text="Which Install object does this billing data refer to", 34 | ) 35 | 36 | status = models.CharField( 37 | choices=BillingStatus.choices, 38 | help_text="The billing status of the associated install", 39 | default=BillingStatus.TO_BE_BILLED, 40 | ) 41 | 42 | billing_date = models.DateField( 43 | default=None, 44 | blank=True, 45 | null=True, 46 | help_text="The date that the associated install was billed to the responsible organization", 47 | ) 48 | 49 | invoice_number = models.CharField( 50 | default=None, 51 | blank=True, 52 | null=True, 53 | help_text="The invoice number that the associated install was billed via", 54 | ) 55 | 56 | notes = models.TextField( 57 | blank=True, 58 | null=True, 59 | help_text="A free-form text description, to track any additional information.", 60 | ) 61 | 62 | def __str__(self) -> str: 63 | return f"Billing Datum for {self.install}" 64 | -------------------------------------------------------------------------------- /src/meshapi/models/devices/__init__.py: -------------------------------------------------------------------------------- 1 | from .access_point import * 2 | from .device import * 3 | from .sector import * 4 | -------------------------------------------------------------------------------- /src/meshapi/models/devices/access_point.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | from simple_history.models import HistoricalRecords 3 | 4 | from .device import Device 5 | 6 | 7 | class AccessPoint(Device): 8 | history = HistoricalRecords() 9 | 10 | latitude = models.FloatField( 11 | help_text="Approximate AP latitude in decimal degrees (this will match the attached " 12 | "Node object in most cases, but has been manually moved around in some cases to " 13 | "more accurately reflect the device location)" 14 | ) 15 | longitude = models.FloatField( 16 | help_text="Approximate AP longitude in decimal degrees (this will match the attached " 17 | "Node object in most cases, but has been manually moved around in some cases to " 18 | "more accurately reflect the device location)" 19 | ) 20 | altitude = models.FloatField( 21 | blank=True, 22 | null=True, 23 | help_text='Approximate AP altitude in "absolute" meters above mean sea level (this ' 24 | "will match the attached Node object in most cases, but has been manually moved around in " 25 | "some cases to more accurately reflect the device location)", 26 | ) 27 | 28 | def __str__(self) -> str: 29 | if self.name: 30 | return self.name 31 | return f"MeshDB AP ID {self.id}" 32 | -------------------------------------------------------------------------------- /src/meshapi/models/devices/sector.py: -------------------------------------------------------------------------------- 1 | from django.core.validators import MaxValueValidator, MinValueValidator 2 | from django.db import models 3 | from simple_history.models import HistoricalRecords 4 | 5 | from .device import Device 6 | 7 | 8 | class Sector(Device): 9 | history = HistoricalRecords() 10 | 11 | radius = models.FloatField( 12 | help_text="The radius to display this sector on the map (in km)", 13 | validators=[MinValueValidator(0)], 14 | ) 15 | azimuth = models.IntegerField( 16 | help_text="The compass heading that this sector is pointed towards", 17 | validators=[ 18 | MinValueValidator(0), 19 | MaxValueValidator(360), 20 | ], 21 | ) 22 | width = models.IntegerField( 23 | help_text="The approximate width of the beam this sector produces", 24 | validators=[ 25 | MinValueValidator(0), 26 | MaxValueValidator(360), 27 | ], 28 | ) 29 | 30 | def __str__(self) -> str: 31 | if self.name: 32 | return self.name 33 | return f"MeshDB Sector ID {self.id}" 34 | -------------------------------------------------------------------------------- /src/meshapi/models/permission.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | 4 | class Permission(models.Model): 5 | class Meta: 6 | managed = False # No database table creation or deletion \ 7 | # operations will be performed for this model. 8 | 9 | default_permissions = () # disable "add", "change", "delete" 10 | # and "view" default permissions 11 | 12 | permissions = ( 13 | ("maintenance_mode", "Can toggle maintenance mode"), 14 | ("explorer_access", "Can access SQL Explorer"), 15 | ) 16 | -------------------------------------------------------------------------------- /src/meshapi/models/util/auto_incrementing_integer_field.py: -------------------------------------------------------------------------------- 1 | # Stolen basically verbatim from here 2 | # https://forum.djangoproject.com/t/django-4-2-is-a-2nd-autofield-like-field-on-a-model-possible/28278 3 | from typing import List, Tuple 4 | 5 | from django.db.backends.base.base import BaseDatabaseWrapper 6 | from django.db.models import Expression, IntegerField 7 | from django.db.models.sql.compiler import SQLCompiler 8 | 9 | 10 | class Default(Expression): 11 | def as_sql(self, compiler: SQLCompiler, connection: BaseDatabaseWrapper) -> Tuple[str, List[str | int]]: 12 | return "DEFAULT", [] 13 | 14 | def __str__(self) -> str: 15 | return "-" 16 | 17 | 18 | class AutoIncrementingIntegerField(IntegerField): 19 | @property 20 | def db_returning(self) -> bool: 21 | return True 22 | 23 | def get_default(self) -> Expression: 24 | return Default() 25 | 26 | def db_type_suffix(self, connection: BaseDatabaseWrapper) -> str: 27 | return "GENERATED BY DEFAULT AS IDENTITY" 28 | -------------------------------------------------------------------------------- /src/meshapi/permissions.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | from typing import Any, Optional 4 | 5 | from django.contrib.auth.models import User 6 | from django.db.models import Model 7 | from django.views.generic import View 8 | from rest_framework import permissions 9 | from rest_framework.exceptions import PermissionDenied 10 | from rest_framework.permissions import BasePermission 11 | from rest_framework.request import Request 12 | 13 | 14 | class IsSuperUser(BasePermission): 15 | def has_permission(self, request: Request, view: View) -> bool: 16 | return bool(request.user.is_superuser) 17 | 18 | 19 | class IsReadOnly(BasePermission): 20 | """ 21 | The request is a read-only request. Add this to any View that needs to be accessible to 22 | unauthenticated users. Be sure to keep the Django authenticator also (e.g.): 23 | permission_classes = [permissions.DjangoModelPermissions | IsReadOnly] 24 | """ 25 | 26 | def has_permission(self, request: Request, view: View) -> bool: 27 | return bool(request.method in permissions.SAFE_METHODS) 28 | 29 | 30 | class HasDjangoPermission(BasePermission): 31 | django_permission: str | None = None 32 | 33 | def has_permission(self, request: Request, view: View) -> bool: 34 | if not self.django_permission: 35 | raise NotImplementedError( 36 | "You must subclass HasDjangoPermission and specify the django_permission attribute" 37 | ) 38 | return bool(request.user) and request.user.has_perm(self.django_permission) 39 | 40 | 41 | class HasNNAssignPermission(HasDjangoPermission): 42 | django_permission = "meshapi.assign_nn" 43 | 44 | 45 | class HasDisambiguateNumberPermission(HasDjangoPermission): 46 | django_permission = "meshapi.disambiguate_number" 47 | 48 | 49 | class HasMaintenanceModePermission(HasDjangoPermission): 50 | django_permission = "meshapi.maintenance_mode" 51 | 52 | 53 | class HasExplorerAccessPermission(HasDjangoPermission): 54 | django_permission = "meshapi.explorer_access" 55 | 56 | 57 | # Janky 58 | class LegacyMeshQueryPassword(permissions.BasePermission): 59 | def has_permission(self, request: Request, view: View) -> bool: 60 | if ( 61 | request.headers["Authorization"] 62 | and request.headers["Authorization"] == f"Bearer {os.environ.get('QUERY_PSK')}" 63 | ): 64 | return True 65 | 66 | raise PermissionDenied("Authentication Failed.") 67 | 68 | 69 | class LegacyNNAssignmentPassword(permissions.BasePermission): 70 | def has_permission(self, request: Request, view: Any) -> bool: 71 | request_json = json.loads(request.body) 72 | if "password" in request_json and request_json["password"] == os.environ.get("NN_ASSIGN_PSK"): 73 | return True 74 | 75 | raise PermissionDenied("Authentication Failed.") 76 | 77 | 78 | def check_has_model_view_permission(user: Optional[User], model: Model) -> bool: 79 | if not user: 80 | # Unauthenticated requests do not have permission by default 81 | return False 82 | 83 | return user.has_perm(f"{__package__}.view_{model._meta.model_name}") 84 | -------------------------------------------------------------------------------- /src/meshapi/serializers/__init__.py: -------------------------------------------------------------------------------- 1 | from .map import * 2 | from .model_api import * 3 | -------------------------------------------------------------------------------- /src/meshapi/serializers/javascript_date_field.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | from typing import Optional 3 | 4 | from drf_spectacular.types import OpenApiTypes 5 | from drf_spectacular.utils import extend_schema_field 6 | from rest_framework import serializers 7 | 8 | 9 | def javascript_time_to_internal_dt(datetime_int_val: Optional[int]) -> Optional[datetime.datetime]: 10 | if datetime_int_val is None: 11 | return None 12 | 13 | return datetime.datetime.fromtimestamp(datetime_int_val / 1000, tz=datetime.timezone.utc) 14 | 15 | 16 | def dt_to_javascript_time(datetime_val: Optional[datetime.datetime]) -> Optional[int]: 17 | if datetime_val is None: 18 | return None 19 | 20 | return int(datetime_val.timestamp() * 1000) 21 | 22 | 23 | @extend_schema_field(OpenApiTypes.INT) 24 | class JavascriptDatetimeField(serializers.Field): 25 | def to_internal_value(self, date_int_val: Optional[int]) -> Optional[datetime.datetime]: 26 | return javascript_time_to_internal_dt(date_int_val) 27 | 28 | def to_representation(self, datetime_val: Optional[datetime.datetime]) -> Optional[int]: 29 | return dt_to_javascript_time(datetime_val) 30 | 31 | 32 | @extend_schema_field(OpenApiTypes.INT) 33 | class JavascriptDateField(serializers.Field): 34 | def to_internal_value(self, date_int_val: Optional[int]) -> Optional[datetime.date]: 35 | internal_dt = javascript_time_to_internal_dt(date_int_val) 36 | if not internal_dt: 37 | return None 38 | 39 | return internal_dt.date() 40 | 41 | def to_representation(self, date_val: Optional[datetime.date]) -> Optional[int]: 42 | if date_val is None: 43 | return None 44 | 45 | return dt_to_javascript_time( 46 | datetime.datetime.combine( 47 | date_val, 48 | datetime.datetime.min.time(), 49 | ).astimezone(datetime.timezone.utc) 50 | ) 51 | -------------------------------------------------------------------------------- /src/meshapi/serializers/nested_object_references.py: -------------------------------------------------------------------------------- 1 | from rest_framework import serializers 2 | 3 | from meshapi.models import Install 4 | from meshapi.serializers import NestedKeyObjectRelatedField, NestedKeyRelatedMixIn 5 | 6 | 7 | class InstallNestedRefSerializer(NestedKeyRelatedMixIn, serializers.ModelSerializer): 8 | serializer_related_field = NestedKeyObjectRelatedField 9 | 10 | class Meta: 11 | model = Install 12 | fields = ["install_number", "id", "node"] 13 | extra_kwargs = { 14 | "node": {"additional_keys": ("network_number",)}, 15 | "install_number": {"read_only": True}, 16 | } 17 | -------------------------------------------------------------------------------- /src/meshapi/serializers/query_api.py: -------------------------------------------------------------------------------- 1 | from rest_framework import serializers 2 | 3 | from meshapi.models import Install 4 | 5 | 6 | class QueryFormSerializer(serializers.ModelSerializer): 7 | """ 8 | Objects which approximate the CSV output from the legacy docs query form. These approximately 9 | correspond to the spreadsheet row format, by flattening attributes across many models into a 10 | single JSON object 11 | """ 12 | 13 | class Meta: 14 | model = Install 15 | fields = ( 16 | "install_number", 17 | "street_address", 18 | "unit", 19 | "city", 20 | "state", 21 | "zip_code", 22 | "name", 23 | "phone_number", 24 | "additional_phone_numbers", 25 | "primary_email_address", 26 | "stripe_email_address", 27 | "additional_email_addresses", 28 | "notes", 29 | "network_number", 30 | "network_number_status", 31 | "status", 32 | ) 33 | 34 | street_address = serializers.CharField(source="building.street_address") 35 | city = serializers.CharField(source="building.city") 36 | state = serializers.CharField(source="building.state") 37 | zip_code = serializers.CharField(source="building.zip_code") 38 | 39 | name = serializers.CharField(source="member.name") 40 | phone_number = serializers.CharField(source="member.phone_number") 41 | additional_phone_numbers = serializers.ListField( 42 | source="member.additional_phone_numbers", child=serializers.CharField() 43 | ) 44 | 45 | network_number = serializers.IntegerField(source="node.network_number", allow_null=True) 46 | network_number_status = serializers.CharField(source="node.status", allow_null=True) 47 | 48 | primary_email_address = serializers.CharField(source="member.primary_email_address") 49 | stripe_email_address = serializers.CharField(source="member.stripe_email_address") 50 | additional_email_addresses = serializers.ListField( 51 | source="member.additional_email_addresses", child=serializers.CharField() 52 | ) 53 | 54 | notes = serializers.SerializerMethodField("concat_all_notes") 55 | 56 | def concat_all_notes(self, install: Install) -> str: 57 | note_sources = [notes for notes in [install.notes, install.building.notes, install.member.notes] if notes] 58 | if install.node and install.node.notes: 59 | note_sources.append(install.node.notes) 60 | 61 | return "\n".join(note_sources) 62 | -------------------------------------------------------------------------------- /src/meshapi/static/admin/install_tabular.css: -------------------------------------------------------------------------------- 1 | /* Increase size of titles. Currently unused, but may be useful later. */ 2 | .inline-group .tabular td.original p { 3 | font-size: 12px; 4 | } 5 | 6 | /* Hide titles so we can hardcode them into the template */ 7 | td.original p { 8 | visibility: hidden 9 | } 10 | 11 | .inline-group .tabular tr.has_original td { 12 | padding-top: 5px; 13 | } 14 | 15 | .inline-action-row { 16 | padding: 12px 14px 12px; 17 | margin: 0 0 10px; 18 | overflow: hidden; 19 | display: flex; 20 | gap: 20px; 21 | flex-wrap: wrap; 22 | } -------------------------------------------------------------------------------- /src/meshapi/static/admin/magic_wand.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nycmeshnet/meshdb/a2f79331699ff26480e0d1123a387b6677df2fb1/src/meshapi/static/admin/magic_wand.png -------------------------------------------------------------------------------- /src/meshapi/static/admin/map/img/cross.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nycmeshnet/meshdb/a2f79331699ff26480e0d1123a387b6677df2fb1/src/meshapi/static/admin/map/img/cross.png -------------------------------------------------------------------------------- /src/meshapi/static/admin/map/img/handlebar.svg: -------------------------------------------------------------------------------- 1 | 2 | 41 | -------------------------------------------------------------------------------- /src/meshapi/static/admin/map/img/map.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nycmeshnet/meshdb/a2f79331699ff26480e0d1123a387b6677df2fb1/src/meshapi/static/admin/map/img/map.png -------------------------------------------------------------------------------- /src/meshapi/static/admin/map/img/map/active.svg: -------------------------------------------------------------------------------- 1 | 2 | 10 | -------------------------------------------------------------------------------- /src/meshapi/static/admin/map/img/map/ap.svg: -------------------------------------------------------------------------------- 1 | 2 | 10 | -------------------------------------------------------------------------------- /src/meshapi/static/admin/map/img/map/dead.svg: -------------------------------------------------------------------------------- 1 | 2 | 10 | 11 | -------------------------------------------------------------------------------- /src/meshapi/static/admin/map/img/map/hub.svg: -------------------------------------------------------------------------------- 1 | 2 | 10 | -------------------------------------------------------------------------------- /src/meshapi/static/admin/map/img/map/kiosk-5g.svg: -------------------------------------------------------------------------------- 1 | 2 | 10 | 11 | -------------------------------------------------------------------------------- /src/meshapi/static/admin/map/img/map/kiosk.svg: -------------------------------------------------------------------------------- 1 | 2 | 10 | 11 | -------------------------------------------------------------------------------- /src/meshapi/static/admin/map/img/map/omni.svg: -------------------------------------------------------------------------------- 1 | 2 | 10 | -------------------------------------------------------------------------------- /src/meshapi/static/admin/map/img/map/pop.svg: -------------------------------------------------------------------------------- 1 | 2 | 10 | 11 | -------------------------------------------------------------------------------- /src/meshapi/static/admin/map/img/map/potential-hub.svg: -------------------------------------------------------------------------------- 1 | 2 | 10 | -------------------------------------------------------------------------------- /src/meshapi/static/admin/map/img/map/potential-supernode.svg: -------------------------------------------------------------------------------- 1 | 2 | 10 | 11 | -------------------------------------------------------------------------------- /src/meshapi/static/admin/map/img/map/potential.svg: -------------------------------------------------------------------------------- 1 | 2 | 10 | 11 | -------------------------------------------------------------------------------- /src/meshapi/static/admin/map/img/map/supernode.svg: -------------------------------------------------------------------------------- 1 | 2 | 10 | 11 | -------------------------------------------------------------------------------- /src/meshapi/static/admin/map/img/map/vpn.svg: -------------------------------------------------------------------------------- 1 | 2 | 10 | 11 | -------------------------------------------------------------------------------- /src/meshapi/static/admin/map/img/recenter.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nycmeshnet/meshdb/a2f79331699ff26480e0d1123a387b6677df2fb1/src/meshapi/static/admin/map/img/recenter.png -------------------------------------------------------------------------------- /src/meshapi/static/widgets/auto_populate_location.css: -------------------------------------------------------------------------------- 1 | 2 | .disabled-button { 3 | background-color: #ddd; /* Light gray background */ 4 | color: #999; /* Dim text color */ 5 | border: 1px solid #ccc; /* Light gray border */ 6 | cursor: not-allowed; /* Change cursor to indicate not clickable */ 7 | opacity: 0.6; /* Reduce opacity to visually indicate disabled state */ 8 | pointer-events: none; /* Disable pointer events */ 9 | } 10 | 11 | .geocode-error { 12 | margin-top: 20px; 13 | padding: 10px; 14 | border: 1px red; 15 | color: red; 16 | background: pink; 17 | } -------------------------------------------------------------------------------- /src/meshapi/static/widgets/flickity.min.css: -------------------------------------------------------------------------------- 1 | /*! Flickity v2.3.0 2 | https://flickity.metafizzy.co 3 | ---------------------------------------------- */ 4 | .flickity-enabled{position:relative}.flickity-enabled:focus{outline:0}.flickity-viewport{overflow:hidden;position:relative;height:100%}.flickity-slider{position:absolute;width:100%;height:100%}.flickity-enabled.is-draggable{-webkit-tap-highlight-color:transparent;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none}.flickity-enabled.is-draggable .flickity-viewport{cursor:move;cursor:-webkit-grab;cursor:grab}.flickity-enabled.is-draggable .flickity-viewport.is-pointer-down{cursor:-webkit-grabbing;cursor:grabbing}.flickity-button{position:absolute;background:hsla(0,0%,100%,.75);border:none;color:#333}.flickity-button:hover{background:#fff;cursor:pointer}.flickity-button:focus{outline:0;box-shadow:0 0 0 5px #19f}.flickity-button:active{opacity:.6}.flickity-button:disabled{opacity:.3;cursor:auto;pointer-events:none}.flickity-button-icon{fill:currentColor}.flickity-prev-next-button{top:50%;width:44px;height:44px;border-radius:50%;transform:translateY(-50%)}.flickity-prev-next-button.previous{left:10px}.flickity-prev-next-button.next{right:10px}.flickity-rtl .flickity-prev-next-button.previous{left:auto;right:10px}.flickity-rtl .flickity-prev-next-button.next{right:auto;left:10px}.flickity-prev-next-button .flickity-button-icon{position:absolute;left:20%;top:20%;width:60%;height:60%}.flickity-page-dots{position:absolute;width:100%;bottom:-25px;padding:0;margin:0;list-style:none;text-align:center;line-height:1}.flickity-rtl .flickity-page-dots{direction:rtl}.flickity-page-dots .dot{display:inline-block;width:10px;height:10px;margin:0 8px;background:#333;border-radius:50%;opacity:.25;cursor:pointer}.flickity-page-dots .dot.is-selected{opacity:1} -------------------------------------------------------------------------------- /src/meshapi/static/widgets/panorama_viewer.css: -------------------------------------------------------------------------------- 1 | .main-carousel { 2 | width:100%; 3 | min-height:fit-content; 4 | max-height:300px; 5 | } 6 | .carousel-cell { 7 | width: 100%; /* full width */ 8 | height: 200px; 9 | /* center images in cells with flexbox */ 10 | display: flex; 11 | align-items: center; 12 | justify-content: center; 13 | margin-right: 10px; 14 | } 15 | 16 | .carousel-cell img { 17 | display: block; 18 | } 19 | 20 | .no-panos { 21 | display:flex; 22 | align-items: center; 23 | justify-content: center; 24 | border-radius: 5px; 25 | } 26 | 27 | -------------------------------------------------------------------------------- /src/meshapi/static/widgets/warn_about_date.css: -------------------------------------------------------------------------------- 1 | .field-validate_install_abandon_date_set_widget { 2 | display: none; 3 | } 4 | -------------------------------------------------------------------------------- /src/meshapi/templates/admin/node_panorama_viewer.html: -------------------------------------------------------------------------------- 1 | {{ inline_admin_formset.formset.management_form }} 2 |
3 |

Panoramas

4 |
5 | {% include 'widgets/panorama_viewer.html' with widget=inline_admin_formset.opts.all_panoramas %} 6 |
7 |
8 | 9 | 10 | {% comment %} 11 | Everything below this point exists because there are nonsense hidden fields that Django breaks the 12 | ability to submit the form without, so we include the original template blow but make everything hidden 13 | {% endcomment %} 14 | 15 | 20 | 21 | {% include "admin/edit_inline/tabular.html" %} -------------------------------------------------------------------------------- /src/meshapi/templates/widgets/auto_populate_location.html: -------------------------------------------------------------------------------- 1 | {% load i18n static %} 2 | 3 | -------------------------------------------------------------------------------- /src/meshapi/templates/widgets/dob_identifier.html: -------------------------------------------------------------------------------- 1 | {% load i18n static %} 2 |
3 | {% include "django/forms/widgets/input.html" %} 4 | {% block links %} 5 | {% spaceless %} 6 | 12 | {% translate 'Navigate' %} 13 | 14 | {% endspaceless %} 15 | {% endblock %} 16 |
17 | -------------------------------------------------------------------------------- /src/meshapi/templates/widgets/external_link.html: -------------------------------------------------------------------------------- 1 | {% load i18n static %} 2 |
3 | {% include "django/forms/widgets/input.html" %} 4 | {% block links %} 5 | {% spaceless %} 6 | 12 | {% translate 'Navigate' %} 13 | 14 | {% endspaceless %} 15 | {% endblock %} 16 |
17 | -------------------------------------------------------------------------------- /src/meshapi/templates/widgets/install_status.html: -------------------------------------------------------------------------------- 1 | {% load i18n static %} 2 |
3 | {% include "django/forms/widgets/select.html" %} 4 | {% if reassigned_node_id %} 5 | {% block links %} 6 | {% spaceless %} 7 | 10 | {% translate 'Navigate' %} 11 | 12 | {% endspaceless %} 13 | {% endblock %} 14 | {% endif %} 15 |
16 | -------------------------------------------------------------------------------- /src/meshapi/templates/widgets/ip_address.html: -------------------------------------------------------------------------------- 1 | {% load i18n static %} 2 |
3 | {% include "django/forms/widgets/input.html" %} 4 | {% block links %} 5 | {% spaceless %} 6 | 10 | {% translate 'Navigate' %} 11 | 12 | {% endspaceless %} 13 | {% endblock %} 14 |
-------------------------------------------------------------------------------- /src/meshapi/templates/widgets/panorama_viewer.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 |
5 | {% if widget.value|length == 0 %} 6 |
7 |

There are no panoramas.

8 |
9 | {% endif %} 10 | {% for panorama in widget.value %} 11 |
12 | 13 |
14 | {% endfor %} 15 |
16 | -------------------------------------------------------------------------------- /src/meshapi/templates/widgets/uisp_link.html: -------------------------------------------------------------------------------- 1 | {% load i18n static %} 2 |
3 | {% include "django/forms/widgets/input.html" %} 4 | {% block links %} 5 | {% spaceless %} 6 | 12 | {% translate 'Navigate' %} 13 | 14 | {% endspaceless %} 15 | {% endblock %} 16 |
17 | -------------------------------------------------------------------------------- /src/meshapi/templates/widgets/warn_about_date.html: -------------------------------------------------------------------------------- 1 |
2 |

3 | Warning: Status set to without setting Date 4 |

5 | 6 | Use Today's Date 7 | 8 | 9 | Continue Without Setting Date 10 | 11 |
12 | -------------------------------------------------------------------------------- /src/meshapi/templatetags/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nycmeshnet/meshdb/a2f79331699ff26480e0d1123a387b6677df2fb1/src/meshapi/templatetags/__init__.py -------------------------------------------------------------------------------- /src/meshapi/templatetags/env_extras.py: -------------------------------------------------------------------------------- 1 | # https://stackoverflow.com/a/62798100 2 | import os 3 | 4 | from django import template 5 | 6 | register = template.Library() 7 | 8 | 9 | @register.simple_tag 10 | def get_env_var(key: str) -> str: 11 | return str(os.environ.get(key) or "") 12 | -------------------------------------------------------------------------------- /src/meshapi/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nycmeshnet/meshdb/a2f79331699ff26480e0d1123a387b6677df2fb1/src/meshapi/tests/__init__.py -------------------------------------------------------------------------------- /src/meshapi/tests/group_helpers.py: -------------------------------------------------------------------------------- 1 | from django.contrib.auth.models import Group 2 | from django.core.management import call_command 3 | 4 | 5 | def create_groups(): 6 | # Call the manage.py command to create the groups as they will be created at runtime, 7 | # this also assigns expected permissions 8 | call_command("create_groups") 9 | 10 | # Fetch the newly created groups and return 11 | admin_group = Group.objects.get(name="Admin") 12 | installer_group = Group.objects.get(name="Installer") 13 | read_only_group = Group.objects.get(name="Read Only") 14 | return admin_group, installer_group, read_only_group 15 | -------------------------------------------------------------------------------- /src/meshapi/tests/test_building.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | from meshapi.models import Building 4 | 5 | 6 | class TestBuilding(TestCase): 7 | def test_building_address_single_line_str(self): 8 | full_address_building = Building( 9 | street_address="123 Chom Street", 10 | city="Brooklyn", 11 | state="NY", 12 | zip_code="12345", 13 | latitude=0, 14 | longitude=0, 15 | ) 16 | self.assertEqual(full_address_building.one_line_complete_address, "123 Chom Street, Brooklyn NY, 12345") 17 | 18 | limited_address_building = Building( 19 | street_address="123 Chom Street", 20 | latitude=0, 21 | longitude=0, 22 | ) 23 | self.assertEqual(limited_address_building.one_line_complete_address, "123 Chom Street") 24 | 25 | full_address_building = Building( 26 | street_address="123 Chom Street", 27 | state="NY", 28 | zip_code="12345", 29 | latitude=0, 30 | longitude=0, 31 | ) 32 | self.assertEqual(full_address_building.one_line_complete_address, "123 Chom Street, NY, 12345") 33 | -------------------------------------------------------------------------------- /src/meshapi/tests/test_crawl_uisp.py: -------------------------------------------------------------------------------- 1 | # @patch("meshapi.util.join_records.JOIN_RECORD_PREFIX", MOCK_JOIN_RECORD_PREFIX) 2 | from django.contrib.auth.models import User 3 | from django.test import Client, TestCase 4 | 5 | 6 | class TestUISPOnDemandImportForm(TestCase): 7 | a = Client() # Anonymous client 8 | c = Client() 9 | 10 | def setUp(self) -> None: 11 | self.admin_user = User.objects.create_superuser( 12 | username="admin", password="admin_password", email="admin@example.com" 13 | ) 14 | self.c.login(username="admin", password="admin_password") 15 | 16 | def test_uisp_on_demand_unauthenticated(self): 17 | response = self.a.get("/uisp-on-demand/") 18 | # Redirected to admin login 19 | self.assertEqual(302, response.status_code) 20 | 21 | def test_uisp_on_demand(self): 22 | response = self.c.get("/uisp-on-demand/") 23 | self.assertEqual(200, response.status_code) 24 | -------------------------------------------------------------------------------- /src/meshapi/tests/test_docs.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | 4 | class TestDevice(TestCase): 5 | def test_docs_200_unauth(self): 6 | self.assertEqual(self.client.get("/api-docs/swagger/").status_code, 200) 7 | self.assertEqual(self.client.get("/api-docs/redoc/").status_code, 200) 8 | self.assertEqual(self.client.get("/api-docs/openapi3.json").status_code, 200) 9 | -------------------------------------------------------------------------------- /src/meshapi/tests/test_join_record_processor_bad_env_vars.py: -------------------------------------------------------------------------------- 1 | from unittest.mock import patch 2 | 3 | from botocore.exceptions import ClientError 4 | from django.test import TestCase 5 | 6 | from meshapi.util.join_records import JoinRecordProcessor 7 | 8 | 9 | class TestJoinRecordProcessorBadEnvVars(TestCase): 10 | @patch("meshapi.util.join_records.JOIN_RECORD_BUCKET_NAME", "") 11 | def test_missing_bucket_name(self): 12 | with self.assertRaises(EnvironmentError): 13 | JoinRecordProcessor() 14 | 15 | @patch("meshapi.util.join_records.JOIN_RECORD_PREFIX", "") 16 | def test_missing_prefix(self): 17 | with self.assertRaises(EnvironmentError): 18 | JoinRecordProcessor() 19 | 20 | @patch("meshapi.util.join_records.JOIN_RECORD_BUCKET_NAME", "chom") 21 | def test_bad_bucket_name(self): 22 | with self.assertRaises(ClientError): 23 | JoinRecordProcessor().get_all() 24 | -------------------------------------------------------------------------------- /src/meshapi/tests/test_meshweb.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | 4 | class TestViewsGetUnauthenticated(TestCase): 5 | def test_landing_page_get_unauthenticated(self): 6 | response = self.client.get("/") 7 | self.assertEqual( 8 | 200, 9 | response.status_code, 10 | f"status code incorrect for /. Should be 200, but got {response.status_code}", 11 | ) 12 | -------------------------------------------------------------------------------- /src/meshapi/tests/test_query_form.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from django.conf import os 4 | from django.test import Client, TestCase 5 | 6 | from meshapi.models import Building, Install, Member, Node 7 | 8 | from .sample_data import sample_building, sample_install, sample_member 9 | 10 | 11 | class TestQueryForm(TestCase): 12 | c = Client() 13 | 14 | def setUp(self): 15 | sample_install_copy = sample_install.copy() 16 | building = Building(**sample_building) 17 | building.save() 18 | sample_install_copy["building"] = building 19 | 20 | member = Member(**sample_member) 21 | member.save() 22 | sample_install_copy["member"] = member 23 | 24 | node = Node(latitude=0, longitude=0, status=Node.NodeStatus.ACTIVE) 25 | node.save() 26 | 27 | self.install = Install(**sample_install_copy) 28 | self.install.node = node 29 | self.install.save() 30 | 31 | def query(self, route, field, data): 32 | code = 200 33 | password = os.environ.get("QUERY_PSK") 34 | route = f"/api/v1/query/{route}/?{field}={data}" 35 | headers = {"Authorization": f"Bearer {password}"} 36 | response = self.c.get(route, headers=headers) 37 | self.assertEqual( 38 | code, 39 | response.status_code, 40 | f"status code incorrect for {route}. Should be {code}, but got {response.status_code}", 41 | ) 42 | 43 | resp_json = json.loads(response.content.decode("utf-8")) 44 | self.assertEqual(len(resp_json["results"]), 1) 45 | 46 | def test_query_address(self): 47 | self.query("buildings", "street_address", self.install.building.street_address) 48 | 49 | def test_query_email(self): 50 | self.query("members", "email_address", self.install.member.primary_email_address) 51 | 52 | def test_query_name(self): 53 | self.query("members", "name", self.install.member.name) 54 | 55 | def test_query_nn(self): 56 | self.query("installs", "network_number", self.install.node.network_number) 57 | -------------------------------------------------------------------------------- /src/meshapi/tests/test_sector.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from django.contrib.auth.models import User 4 | from django.test import Client, TestCase 5 | 6 | from ..models import Device, Node, Sector 7 | from .sample_data import sample_node 8 | 9 | 10 | class TestSector(TestCase): 11 | c = Client() 12 | 13 | def setUp(self): 14 | self.admin_user = User.objects.create_superuser( 15 | username="admin", password="admin_password", email="admin@example.com" 16 | ) 17 | self.c.login(username="admin", password="admin_password") 18 | 19 | self.node = Node( 20 | network_number=7, 21 | name="Test Node", 22 | status=Node.NodeStatus.ACTIVE, 23 | latitude=0, 24 | longitude=0, 25 | ) 26 | self.node.save() 27 | 28 | def test_new_sector(self): 29 | response = self.c.post( 30 | "/api/v1/sectors/", 31 | { 32 | "node": {"id": str(self.node.id)}, 33 | "status": Device.DeviceStatus.ACTIVE, 34 | "azimuth": 0, 35 | "width": 120, 36 | "radius": 0.3, 37 | }, 38 | content_type="application/json", 39 | ) 40 | code = 201 41 | self.assertEqual( 42 | code, 43 | response.status_code, 44 | f"status code incorrect. Should be {code}, but got {response.status_code}", 45 | ) 46 | 47 | def test_broken_sector(self): 48 | response = self.c.post( 49 | "/api/v1/sectors/", 50 | { 51 | "name": "Vernon", 52 | "node": {"id": str(self.node.id)}, 53 | "latitude": 0, 54 | "longitude": 0, 55 | "azimuth": 0, 56 | "width": 120, 57 | "radius": 0.3, 58 | }, 59 | content_type="application/json", 60 | ) 61 | code = 400 62 | self.assertEqual( 63 | code, 64 | response.status_code, 65 | f"status code incorrect. Should be {code}, but got {response.status_code}", 66 | ) 67 | 68 | def test_get_sector(self): 69 | node = Node(**sample_node) 70 | node.save() 71 | sector = Sector( 72 | name="Vernon", 73 | status="Active", 74 | azimuth=0, 75 | width=120, 76 | radius=0.3, 77 | node=node, 78 | ) 79 | sector.save() 80 | 81 | response = self.c.get(f"/api/v1/sectors/{sector.id}/") 82 | 83 | code = 200 84 | self.assertEqual( 85 | code, 86 | response.status_code, 87 | f"status code incorrect. Should be {code}, but got {response.status_code}", 88 | ) 89 | 90 | response_obj = json.loads(response.content) 91 | self.assertEqual(response_obj["status"], "Active") 92 | self.assertEqual(response_obj["node"]["network_number"], node.network_number) 93 | -------------------------------------------------------------------------------- /src/meshapi/tests/test_settings.py: -------------------------------------------------------------------------------- 1 | SECRET_KEY = "fake-key" 2 | INSTALLED_APPS = [ 3 | "tests", 4 | ] 5 | -------------------------------------------------------------------------------- /src/meshapi/tests/test_view_autocomplete.py: -------------------------------------------------------------------------------- 1 | from django.contrib.auth.models import User 2 | from django.test import Client, TestCase 3 | 4 | 5 | class TestViewAutocomplete(TestCase): 6 | c = Client() 7 | a = Client() # anon client 8 | 9 | def setUp(self) -> None: 10 | self.admin_user = User.objects.create_superuser( 11 | username="admin", password="admin_password", email="admin@example.com" 12 | ) 13 | self.c.login(username="admin", password="admin_password") 14 | 15 | def test_views_get_autocomplete(self): 16 | r = self.c.get("/member-autocomplete/") 17 | code = 200 18 | self.assertEqual(code, r.status_code) 19 | 20 | # The view just returns empty if it's unauthenticated 21 | def test_views_get_autocomplete_unauthenticated(self): 22 | r = self.a.get("/member-autocomplete/") 23 | 24 | # We'll get a 200, but the response will be empty 25 | code = 200 26 | self.assertEqual(code, r.status_code) 27 | j = r.json() 28 | self.assertEqual([], j["results"]) 29 | -------------------------------------------------------------------------------- /src/meshapi/tests/test_views_explorer.py: -------------------------------------------------------------------------------- 1 | from django.contrib.auth.models import Group, User 2 | from django.test import TestCase 3 | 4 | from meshapi.tests.group_helpers import create_groups 5 | 6 | 7 | class TestViewsExplorer(TestCase): 8 | databases = "__all__" 9 | 10 | def setUp(self): 11 | self.explorer_user = User.objects.create_user( 12 | username="exploreruser", password="explorer_password", email="explorer@example.com" 13 | ) 14 | create_groups() 15 | self.explorer_user.groups.add(Group.objects.get(name="Explorer Access")) 16 | 17 | def test_views_get_explorer(self): 18 | self.client.login(username="exploreruser", password="explorer_password") 19 | 20 | routes = [ 21 | ("/explorer/", 200), 22 | ] 23 | 24 | for route, code in routes: 25 | response = self.client.get(route) 26 | self.assertEqual( 27 | code, 28 | response.status_code, 29 | f"status code incorrect for {route}. Should be {code}, but got {response.status_code}", 30 | ) 31 | 32 | def test_views_get_explorer_unauthenticated(self): 33 | self.client.logout() # log out just in case 34 | 35 | routes = [ 36 | ("/explorer/", 302), 37 | ] 38 | 39 | for route, code in routes: 40 | response = self.client.get(route) 41 | self.assertEqual( 42 | code, 43 | response.status_code, 44 | f"status code incorrect for {route}. Should be {code}, but got {response.status_code}", 45 | ) 46 | 47 | def test_views_post_explorer_playground(self): 48 | self.client.login(username="exploreruser", password="explorer_password") 49 | 50 | route = "/explorer/play/" 51 | code = 200 52 | post_data = {"sql": "SELECT+*+FROM+meshapi_member;"} 53 | 54 | response = self.client.post(route, data=post_data) 55 | self.assertEqual( 56 | code, 57 | response.status_code, 58 | f"status code incorrect for {route}. Should be {code}, but got {response.status_code}", 59 | ) 60 | -------------------------------------------------------------------------------- /src/meshapi/tests/test_widgets.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | from meshapi.widgets import PanoramaViewer 4 | 5 | 6 | class TestPanoramaViewer(TestCase): 7 | def setUp(self): 8 | pass 9 | 10 | def test_pano_get_context(self): 11 | PanoramaViewer.pano_get_context("test", '["blah", "blah2"]') 12 | 13 | def test_pano_get_context_bad_value(self): 14 | PanoramaViewer.pano_get_context("test", 100) # type: ignore 15 | PanoramaViewer.pano_get_context("test", None) # type: ignore 16 | -------------------------------------------------------------------------------- /src/meshapi/tests/util.py: -------------------------------------------------------------------------------- 1 | from threading import Thread 2 | 3 | from bs4 import BeautifulSoup 4 | from django.db import connection 5 | 6 | 7 | class TestThread(Thread): 8 | def run(self): 9 | try: 10 | super().run() 11 | finally: 12 | connection.close() 13 | 14 | 15 | def get_admin_results_count(html_string: str): 16 | soup = BeautifulSoup(html_string, "html.parser") 17 | result_list = soup.find(id="result_list") 18 | if not result_list: 19 | return 0 20 | 21 | return sum(1 for tr in result_list.find("tbody").find_all("tr") if tr.find_all("td")) 22 | -------------------------------------------------------------------------------- /src/meshapi/types/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nycmeshnet/meshdb/a2f79331699ff26480e0d1123a387b6677df2fb1/src/meshapi/types/__init__.py -------------------------------------------------------------------------------- /src/meshapi/types/uisp_api/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nycmeshnet/meshdb/a2f79331699ff26480e0d1123a387b6677df2fb1/src/meshapi/types/uisp_api/__init__.py -------------------------------------------------------------------------------- /src/meshapi/util/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nycmeshnet/meshdb/a2f79331699ff26480e0d1123a387b6677df2fb1/src/meshapi/util/__init__.py -------------------------------------------------------------------------------- /src/meshapi/util/admin_notifications.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | import os 4 | from typing import Optional, Sequence, Type 5 | 6 | import requests 7 | from django.db.models import Model 8 | from django.http import HttpRequest 9 | from rest_framework.serializers import Serializer 10 | 11 | from meshapi.admin.utils import get_admin_url 12 | 13 | SLACK_ADMIN_NOTIFICATIONS_WEBHOOK_URL = os.environ.get("SLACK_ADMIN_NOTIFICATIONS_WEBHOOK_URL") 14 | SITE_BASE_URL = os.environ.get("SITE_BASE_URL") 15 | 16 | 17 | def escape_slack_text(text: str) -> str: 18 | return text.replace("↔", "<->").replace("&", "&").replace("<", "<").replace(">", ">") 19 | 20 | 21 | def notify_administrators_of_data_issue( 22 | model_instances: Sequence[Model], 23 | serializer_class: Type[Serializer], 24 | message: str, 25 | request: Optional[HttpRequest] = None, 26 | raise_exception_on_failure: bool = False, 27 | ) -> None: 28 | serializer = serializer_class(model_instances, many=True) 29 | 30 | site_base_url = SITE_BASE_URL 31 | if request: 32 | site_base_url = f"{request.scheme}://{request.get_host()}" 33 | 34 | if not site_base_url: 35 | logging.error( 36 | "Env var SITE_BASE_URL is not set and a request was not available to infer this value from, " 37 | "please set this variable to prevent silenced slack notifications" 38 | ) 39 | return 40 | 41 | if "\n" not in message: 42 | message = f"*{message}*. " 43 | 44 | slack_message = { 45 | "text": f"Encountered the following data issue which may require admin attention: {escape_slack_text(message)}" 46 | f"\n\nWhen processing the following {model_instances[0]._meta.verbose_name_plural}: " 47 | + ", ".join(f"<{get_admin_url(m, site_base_url)}|{escape_slack_text(str(m))}>" for m in model_instances) 48 | + ". Please open the database admin UI using the provided links to correct this.\n\n" 49 | + "The current database state of these object(s) is: \n" 50 | + f"```\n{json.dumps(serializer.data, indent=2, default=str)}\n```", 51 | } 52 | 53 | if not SLACK_ADMIN_NOTIFICATIONS_WEBHOOK_URL: 54 | logging.error( 55 | "Env var SLACK_ADMIN_NOTIFICATIONS_WEBHOOK_URL is not set, please set this " 56 | "variable to prevent silenced notifications. Unable to notify admins of " 57 | f"the following message: {slack_message}" 58 | ) 59 | return 60 | 61 | response = requests.post(SLACK_ADMIN_NOTIFICATIONS_WEBHOOK_URL, json=slack_message) 62 | 63 | if raise_exception_on_failure: 64 | response.raise_for_status() 65 | elif response.status_code != 200: 66 | logging.error( 67 | f"Got HTTP {response.status_code} while sending slack notification to slack admin. " 68 | f"HTTP response was {response.text}. Unable to notify admins of " 69 | f"the following message: {slack_message}" 70 | ) 71 | -------------------------------------------------------------------------------- /src/meshapi/util/constants.py: -------------------------------------------------------------------------------- 1 | DEFAULT_EXTERNAL_API_TIMEOUT_SECONDS = 5 2 | INVALID_ALTITUDE = None 3 | 4 | RECAPTCHA_CHECKBOX_TOKEN_HEADER = "X-Recaptcha-V2-Token" 5 | RECAPTCHA_INVISIBLE_TOKEN_HEADER = "X-Recaptcha-V3-Token" 6 | -------------------------------------------------------------------------------- /src/meshapi/util/django_flag_decorator.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from functools import wraps 3 | from typing import Any, Callable 4 | 5 | from flags.state import flag_state 6 | 7 | 8 | def skip_if_flag_disabled(flag_name: str) -> Callable: 9 | """ 10 | Decorator that transforms the annotated function into a noop if the given flag name is disabled 11 | :param flag_name: the flag to check 12 | """ 13 | 14 | def decorator(func: Callable) -> Callable: 15 | def inner(*args: list, **kwargs: dict) -> Any: 16 | enabled = flag_state(flag_name) 17 | 18 | if enabled: 19 | return func(*args, **kwargs) 20 | else: 21 | logging.info(f"Skipping call to function {func.__name__} due to disabled flag {flag_name}") 22 | 23 | return wraps(func)(inner) 24 | 25 | return decorator 26 | -------------------------------------------------------------------------------- /src/meshapi/util/django_pglocks.py: -------------------------------------------------------------------------------- 1 | # Copied from https://github.com/Xof/django-pglocks/blob/master/django_pglocks/__init__.py 2 | # we don't just pip install, since this package is incompatible with 3.12 due to its outdated 3 | # build system, and though a patch has been waiting in PR for 3 years: 4 | # https://github.com/Xof/django-pglocks/pull/27 5 | # it does not appear the repo is maintained anymore. This code is so simple it's unlikely we'll 6 | # need a patch from upstream anyhow... 7 | 8 | from contextlib import contextmanager 9 | from zlib import crc32 10 | 11 | 12 | @contextmanager 13 | def advisory_lock(lock_id, shared=False, wait=True, using=None): 14 | import six 15 | from django.db import DEFAULT_DB_ALIAS, connections 16 | 17 | if using is None: 18 | using = DEFAULT_DB_ALIAS 19 | 20 | # Assemble the function name based on the options. 21 | 22 | function_name = "pg_" 23 | 24 | if not wait: 25 | function_name += "try_" 26 | 27 | function_name += "advisory_lock" 28 | 29 | if shared: 30 | function_name += "_shared" 31 | 32 | release_function_name = "pg_advisory_unlock" 33 | if shared: 34 | release_function_name += "_shared" 35 | 36 | # Format up the parameters. 37 | 38 | tuple_format = False 39 | 40 | if isinstance( 41 | lock_id, 42 | ( 43 | list, 44 | tuple, 45 | ), 46 | ): 47 | if len(lock_id) != 2: 48 | raise ValueError("Tuples and lists as lock IDs must have exactly two entries.") 49 | 50 | if not isinstance(lock_id[0], six.integer_types) or not isinstance(lock_id[1], six.integer_types): 51 | raise ValueError("Both members of a tuple/list lock ID must be integers") 52 | 53 | tuple_format = True 54 | elif isinstance(lock_id, six.string_types): 55 | # Generates an id within postgres integer range (-2^31 to 2^31 - 1). 56 | # crc32 generates an unsigned integer in Py3, we convert it into 57 | # a signed integer using 2's complement (this is a noop in Py2) 58 | pos = crc32(lock_id.encode("utf-8")) 59 | lock_id = (2**31 - 1) & pos 60 | if pos & 2**31: 61 | lock_id -= 2**31 62 | elif not isinstance(lock_id, six.integer_types): 63 | raise ValueError("Cannot use %s as a lock id" % lock_id) 64 | 65 | if tuple_format: 66 | base = "SELECT %s(%d, %d)" 67 | params = ( 68 | lock_id[0], 69 | lock_id[1], 70 | ) 71 | else: 72 | base = "SELECT %s(%d)" 73 | params = (lock_id,) 74 | 75 | acquire_params = (function_name,) + params 76 | 77 | command = base % acquire_params 78 | cursor = connections[using].cursor() 79 | 80 | cursor.execute(command) 81 | 82 | if not wait: 83 | acquired = cursor.fetchone()[0] 84 | else: 85 | acquired = True 86 | 87 | try: 88 | yield acquired 89 | finally: 90 | if acquired: 91 | release_params = (release_function_name,) + params 92 | 93 | command = base % release_params 94 | cursor.execute(command) 95 | 96 | cursor.close() 97 | -------------------------------------------------------------------------------- /src/meshapi/util/drf_renderer.py: -------------------------------------------------------------------------------- 1 | from rest_framework.renderers import BrowsableAPIRenderer 2 | from rest_framework.serializers import BaseSerializer 3 | 4 | 5 | class OnlyRawBrowsableAPIRenderer(BrowsableAPIRenderer): 6 | def render_form_for_serializer(self, serializer: BaseSerializer) -> str: 7 | return "" 8 | -------------------------------------------------------------------------------- /src/meshapi/util/drf_utils.py: -------------------------------------------------------------------------------- 1 | from rest_framework.pagination import PageNumberPagination 2 | 3 | 4 | class CustomSizePageNumberPagination(PageNumberPagination): 5 | page_size_query_param = "page_size" # items per page 6 | -------------------------------------------------------------------------------- /src/meshapi/util/events/__init__.py: -------------------------------------------------------------------------------- 1 | from .join_requests_slack_channel import send_join_request_slack_message 2 | from .osticket_creation import create_os_ticket_for_install 3 | -------------------------------------------------------------------------------- /src/meshapi/util/events/join_requests_slack_channel.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import time 4 | 5 | import requests 6 | from django.db.models.base import ModelBase 7 | from django.db.models.signals import post_save 8 | from django.dispatch import receiver 9 | 10 | from meshapi.models import Install 11 | from meshapi.util.django_flag_decorator import skip_if_flag_disabled 12 | 13 | SLACK_JOIN_REQUESTS_CHANNEL_WEBHOOK_URL = os.environ.get("SLACK_JOIN_REQUESTS_CHANNEL_WEBHOOK_URL") 14 | 15 | 16 | @receiver(post_save, sender=Install, dispatch_uid="join_requests_slack_channel") 17 | @skip_if_flag_disabled("INTEGRATION_ENABLED_SEND_JOIN_REQUEST_SLACK_MESSAGES") 18 | def send_join_request_slack_message(sender: ModelBase, instance: Install, created: bool, **kwargs: dict) -> None: 19 | if not created: 20 | return 21 | 22 | install: Install = instance 23 | if not SLACK_JOIN_REQUESTS_CHANNEL_WEBHOOK_URL: 24 | logging.error( 25 | f"Unable to send join request notification for install {str(install)}, did you set the " 26 | f"SLACK_JOIN_REQUESTS_CHANNEL_WEBHOOK_URL environment variable?" 27 | ) 28 | return 29 | 30 | building_height = str(int(install.building.altitude)) + "m" if install.building.altitude else "Altitude not found" 31 | roof_access = "Roof access" if install.roof_access else "No roof access" 32 | 33 | attempts = 0 34 | while attempts < 4: 35 | attempts += 1 36 | response = requests.post( 37 | SLACK_JOIN_REQUESTS_CHANNEL_WEBHOOK_URL, 38 | json={ 39 | "text": f"**\n" 41 | f"{building_height} · {roof_access} · No LoS Data Available" 42 | }, 43 | ) 44 | 45 | if response.status_code == 200: 46 | break 47 | 48 | time.sleep(1) 49 | 50 | if response.status_code != 200: 51 | logging.error( 52 | f"Got HTTP {response.status_code} while sending install create notification to " 53 | f"join-requests channel. HTTP response was {response.text}" 54 | ) 55 | -------------------------------------------------------------------------------- /src/meshapi/util/uisp_import/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nycmeshnet/meshdb/a2f79331699ff26480e0d1123a387b6677df2fb1/src/meshapi/util/uisp_import/__init__.py -------------------------------------------------------------------------------- /src/meshapi/util/uisp_import/constants.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | 3 | EXCLUDED_UISP_DEVICE_CATEGORIES = ["optical"] 4 | 5 | 6 | DEVICE_NAME_NETWORK_NUMBER_SUBSTITUTIONS = { 7 | "sn1": "227", 8 | "supernode1": "227", 9 | "375p": "227", 10 | "sn3": "713", 11 | } 12 | 13 | NETWORK_NUMBER_REGEX_FOR_DEVICE_NAME = r"\b\d{1,4}\b" 14 | 15 | DEFAULT_SECTOR_AZIMUTH = 0 # decimal degrees (compass heading) 16 | DEFAULT_SECTOR_WIDTH = 0 # decimal degrees 17 | DEFAULT_SECTOR_RADIUS = 1 # km 18 | 19 | # The guesses in this object are okay, since we will always communicate these guesses 20 | # via a slack notification, giving admins a chance to update the data 21 | DEVICE_MODEL_TO_BEAM_WIDTH = { 22 | "LAP-120": 120, 23 | "LAP-GPS": 120, 24 | "PS-5AC": 45, # In reality this is based on the antenna used, this is just a guess based on our historical use 25 | "RP-5AC-Gen2": 90, # In reality this is based on the antenna used, this is just a guess based on our historical use 26 | } 27 | 28 | # Controls how long a device can be offline in UISP for before we mark 29 | # it as "Inactive" in meshdb 30 | UISP_OFFLINE_DURATION_BEFORE_MARKING_INACTIVE = datetime.timedelta(days=30) 31 | 32 | # Controls how long a device has to be "abandoned" for us to be worried that it is a different 33 | # device if it shows back up in UISP 34 | UISP_ABANDON_DATE_AGE_BEFORE_WARNING_ABOUT_REACTIVATION = datetime.timedelta(days=30) 35 | -------------------------------------------------------------------------------- /src/meshapi/util/uisp_import/fetch_uisp.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | from typing import List, Optional 4 | 5 | import requests 6 | from dotenv import load_dotenv 7 | 8 | from meshapi.types.uisp_api.data_links import DataLink as UISPDataLink 9 | from meshapi.types.uisp_api.devices import Device as UISPDevice 10 | 11 | load_dotenv() 12 | 13 | 14 | UISP_URL = os.environ.get("UISP_URL") 15 | UISP_USER = os.environ.get("UISP_USER") 16 | UISP_PASS = os.environ.get("UISP_PASS") 17 | 18 | 19 | def get_uisp_devices() -> List[UISPDevice]: 20 | session = get_uisp_session() 21 | 22 | if not UISP_URL: 23 | raise EnvironmentError("Missing UISP_URL, please set it via an environment variable") 24 | 25 | return json.loads( 26 | session.get( 27 | os.path.join(UISP_URL, "api/v2.1/devices"), 28 | ).content.decode("UTF8") 29 | ) 30 | 31 | 32 | def get_uisp_links() -> List[UISPDataLink]: 33 | session = get_uisp_session() 34 | 35 | if not UISP_URL: 36 | raise EnvironmentError("Missing UISP_URL, please set it via an environment variable") 37 | 38 | return json.loads( 39 | session.get( 40 | os.path.join(UISP_URL, "api/v2.1/data-links"), 41 | ).content.decode("UTF8") 42 | ) 43 | 44 | 45 | def get_uisp_device_detail(device_id: str, session: Optional[requests.Session] = None) -> UISPDevice: 46 | if not session: 47 | session = get_uisp_session() 48 | 49 | if not UISP_URL: 50 | raise EnvironmentError("Missing UISP_URL, please set it via an environment variable") 51 | 52 | return json.loads( 53 | session.get( 54 | os.path.join(UISP_URL, f"api/v2.1/devices/{device_id}"), 55 | ).content.decode("UTF8") 56 | ) 57 | 58 | 59 | def get_uisp_token(session: requests.Session) -> str: 60 | if not UISP_URL: 61 | raise EnvironmentError("Missing UISP_URL, please set it via an environment variable") 62 | 63 | return session.post( 64 | os.path.join(UISP_URL, "api/v2.1/user/login"), 65 | json={ 66 | "username": UISP_USER, 67 | "password": UISP_PASS, 68 | }, 69 | ).headers["x-auth-token"] 70 | 71 | 72 | def get_uisp_session() -> requests.Session: 73 | session = requests.Session() 74 | session.verify = os.path.join(os.path.dirname(__file__), "uisp.mesh.nycmesh.net.crt") 75 | session.headers = {"x-auth-token": get_uisp_token(session)} 76 | 77 | return session 78 | -------------------------------------------------------------------------------- /src/meshapi/util/uisp_import/uisp.mesh.nycmesh.net.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDtDCCApygAwIBAgIUEG5y4t3otCXpwDVINupyHhvQLEgwDQYJKoZIhvcNAQEL 3 | BQAwYDELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAk5ZMREwDwYDVQQHDAhOZXcgWW9y 4 | azERMA8GA1UECgwITllDIE1lc2gxHjAcBgNVBAMMFXVpc3AubWVzaC5ueWNtZXNo 5 | Lm5ldDAeFw0yNDEwMTQwMDEyMTdaFw0yNjEwMTQwMDEyMTdaMGAxCzAJBgNVBAYT 6 | AlVTMQswCQYDVQQIDAJOWTERMA8GA1UEBwwITmV3IFlvcmsxETAPBgNVBAoMCE5Z 7 | QyBNZXNoMR4wHAYDVQQDDBV1aXNwLm1lc2gubnljbWVzaC5uZXQwggEiMA0GCSqG 8 | SIb3DQEBAQUAA4IBDwAwggEKAoIBAQCZvFvB32McdE8/4sfRGPRSUDe+yYmC2Blu 9 | U5PZ5JeT14xSbEUh+ZQUPOfdCtBWTRQp2XvAvmhGYALR8Ly563/xqQwxa7j9Mi89 10 | Ij018/hCSYG3tqCSXBqEn99Gfy+g0Tt0OVzfrFf1ZI2C/eN+3QNTUlXuXeXZxcxD 11 | +K0LARel2Eaifb68ASW+luk/TV9SRnVEoiRLIat3pWhLkRGmpZkSvc8PNK1DsXZ9 12 | P9+Z1WBR2COHBgksJIoQCaCkgEKQE2oQU7iMQMITXMkax72PUVSKJ/SvkcUlJzUR 13 | ypSTGEeNxtQcJKvzQk+Upaz2JBAfGdyZrXItJr7Jd6F1xZQbCg/7AgMBAAGjZjBk 14 | MAkGA1UdEwQCMAAwCwYDVR0PBAQDAgXgMCsGA1UdEQQkMCKCFXVpc3AubWVzaC5u 15 | eWNtZXNoLm5ldIIJdWlzcC5tZXNoMB0GA1UdDgQWBBTRV1I5R00yHSy7uEppuUE4 16 | dCN8qzANBgkqhkiG9w0BAQsFAAOCAQEALIdGfrXUN9E5+q424ysKBt9wGbndXFRn 17 | jy5v0AvgGq/qNNEURKsAPzzZlbzBOMzzIClUI53l3ME9W9XBehLI+20oV42yEbfl 18 | D3I/Zx1TDOV6hyiJ/E+upBlCannP87jDPs7LKdDEFTpKCWAkuxUtxFEwfpcbEH42 19 | 1HseZGk8aQcw6w/cug+PR3MBAmWMmlEKSr2mBSnwAW8lJCMbxL3cStccOVHZ9cFM 20 | aXNqp8USR/JC5Wctel/h+IFJqI818mC5LTbYIEsDapzOZU9U5GtmPhjlp8qAFzAl 21 | +gonMMCsIRCUlEg10+eESfkFsgXvnCUylxL163y/F1+JSJmDJYKCdg== 22 | -----END CERTIFICATE----- 23 | -------------------------------------------------------------------------------- /src/meshapi/views/__init__.py: -------------------------------------------------------------------------------- 1 | from .forms import * 2 | from .geography import * 3 | from .helpers import * 4 | from .lookups import * 5 | from .map import * 6 | from .model_api import * 7 | from .query_api import * 8 | from .uisp_import import * 9 | -------------------------------------------------------------------------------- /src/meshapi/views/autocomplete.py: -------------------------------------------------------------------------------- 1 | from dal_select2.views import Select2QuerySetView 2 | from django.db.models import QuerySet 3 | 4 | from meshapi.models import Member 5 | 6 | 7 | # Used in Install Member many-to-many field inline in the Admin panel 8 | class MemberAutocomplete(Select2QuerySetView): 9 | def get_queryset(self) -> QuerySet[Member]: 10 | user = self.request.user 11 | if not user.is_authenticated: 12 | return Member.objects.none() 13 | 14 | queryset = Member.objects.all() 15 | 16 | if self.q: 17 | queryset = queryset.filter(name__istartswith=self.q) 18 | 19 | return queryset 20 | -------------------------------------------------------------------------------- /src/meshapi_hooks/__init__.py: -------------------------------------------------------------------------------- 1 | # This package exists just to hold the custom Hook model, mostly so that it appears separately 2 | # from the rest of the meshapi models in the admin UI. Having them together makes it hard to 3 | # see that it's a metadata field for the functioning of the app, rather than a primary datatype 4 | # used to represent mesh org data 5 | 6 | default_app_config = "meshapi_hooks.apps.MeshAPIHooksConfig" 7 | -------------------------------------------------------------------------------- /src/meshapi_hooks/admin.py: -------------------------------------------------------------------------------- 1 | import drf_hooks.admin 2 | from django.contrib import admin 3 | 4 | from meshapi_hooks.hooks import CelerySerializerHook 5 | 6 | admin.site.unregister(CelerySerializerHook) 7 | 8 | 9 | @admin.register(CelerySerializerHook) 10 | class CelerySerializerHookAdmin(drf_hooks.admin.HookAdmin): 11 | fields = ("enabled", "user", "target", "event", "headers", "consecutive_failures") 12 | readonly_fields = ["consecutive_failures"] 13 | 14 | class Meta: 15 | model = CelerySerializerHook 16 | -------------------------------------------------------------------------------- /src/meshapi_hooks/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class MeshAPIHooksConfig(AppConfig): 5 | name = "meshapi_hooks" 6 | verbose_name = "Webhooks" 7 | -------------------------------------------------------------------------------- /src/meshapi_hooks/hooks.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Dict, Optional, Sequence 2 | 3 | from django.contrib.auth.models import User 4 | from django.db import models 5 | from drf_hooks.models import AbstractHook 6 | 7 | 8 | class CelerySerializerHook(AbstractHook): 9 | """ 10 | A drf-hooks Hook model that overrides the hook firing behavior to allow retries via Celery. 11 | Keeps track of the number of consecutive delivery failures, and disables delivery once 12 | this reaches a configurable limit 13 | """ 14 | 15 | MAX_CONSECUTIVE_FAILURES_BEFORE_DISABLE = 5 16 | 17 | enabled = models.BooleanField( 18 | default=True, 19 | help_text="Should this webhook be used? This field be automatically changed by the system " 20 | "when too many consecutive failures are detected at the recipient", 21 | ) 22 | consecutive_failures = models.IntegerField( 23 | default=0, 24 | help_text="The number of consecutive failures detected for this endpoint. " 25 | "This should not be modified by administrators", 26 | ) 27 | 28 | def __str__(self) -> str: 29 | return f'Webhook for delivery of "{self.event}" event to {self.user}' 30 | 31 | class Meta: 32 | verbose_name = "Webhook Target" 33 | verbose_name_plural = "Webhook Targets" 34 | 35 | def deliver_hook(self, serialized_hook: Dict[str, Any]) -> None: 36 | # Inline import to prevent circular import loop 37 | from meshapi_hooks.tasks import deliver_webhook_task 38 | 39 | deliver_webhook_task.apply_async([self.id, serialized_hook]) 40 | 41 | @classmethod 42 | def find_hooks(cls, event_name: str, user: Optional[User] = None) -> Sequence[AbstractHook]: 43 | hooks = super().find_hooks(event_name, user=user) 44 | return hooks.filter(enabled=True) 45 | -------------------------------------------------------------------------------- /src/meshapi_hooks/migrations/0002_celeryserializerhook_and_more.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.2.9 on 2024-02-19 00:54 2 | 3 | import django.db.models.deletion 4 | import drf_hooks.models 5 | from django.conf import settings 6 | from django.db import migrations, models 7 | 8 | 9 | class Migration(migrations.Migration): 10 | dependencies = [ 11 | migrations.swappable_dependency(settings.AUTH_USER_MODEL), 12 | ("meshapi_hooks", "0001_initial"), 13 | ] 14 | 15 | operations = [ 16 | migrations.CreateModel( 17 | name="CelerySerializerHook", 18 | fields=[ 19 | ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), 20 | ("created", models.DateTimeField(auto_now_add=True)), 21 | ("updated", models.DateTimeField(auto_now=True)), 22 | ("event", models.CharField(db_index=True, max_length=64, verbose_name="Event")), 23 | ("target", models.URLField(max_length=255, verbose_name="Target URL")), 24 | ("headers", models.JSONField(default=drf_hooks.models.get_default_headers)), 25 | ( 26 | "enabled", 27 | models.BooleanField( 28 | default=True, 29 | help_text="Should this webhook be used? This field be automatically changed by the system when too many consecutive failures are detected at the recipient", 30 | ), 31 | ), 32 | ( 33 | "consecutive_failures", 34 | models.IntegerField( 35 | default=0, 36 | help_text="The number of consecutive failures detected for this endpoint. This should not be modified by administrators", 37 | ), 38 | ), 39 | ( 40 | "user", 41 | models.ForeignKey( 42 | on_delete=django.db.models.deletion.CASCADE, 43 | related_name="%(class)ss", 44 | to=settings.AUTH_USER_MODEL, 45 | ), 46 | ), 47 | ], 48 | options={ 49 | "verbose_name": "Webhook Target", 50 | "verbose_name_plural": "Webhook Targets", 51 | }, 52 | ), 53 | migrations.RemoveField( 54 | model_name="recursiveserializerhook", 55 | name="user", 56 | ), 57 | migrations.DeleteModel( 58 | name="CeleryRecursiveSerializerHook", 59 | ), 60 | migrations.DeleteModel( 61 | name="RecursiveSerializerHook", 62 | ), 63 | ] 64 | -------------------------------------------------------------------------------- /src/meshapi_hooks/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nycmeshnet/meshdb/a2f79331699ff26480e0d1123a387b6677df2fb1/src/meshapi_hooks/migrations/__init__.py -------------------------------------------------------------------------------- /src/meshapi_hooks/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | # drf-hooks looks in this file for hook objects, but it feels weird to inline 4 | # it here, so we import it instead 5 | from .hooks import CelerySerializerHook 6 | -------------------------------------------------------------------------------- /src/meshapi_hooks/tasks.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Dict 2 | 3 | import requests 4 | from celery import Task, shared_task 5 | from celery.exceptions import MaxRetriesExceededError 6 | 7 | from meshapi_hooks.hooks import CelerySerializerHook 8 | 9 | HTTP_ATTEMPT_COUNT_PER_DELIVERY_ATTEMPT = 4 10 | 11 | 12 | @shared_task(bind=True, max_retries=HTTP_ATTEMPT_COUNT_PER_DELIVERY_ATTEMPT - 1) 13 | def deliver_webhook_task(self: Task, hook_id: int, payload: Dict[str, Any]) -> None: 14 | """Deliver the payload to the hook target""" 15 | hook = CelerySerializerHook.objects.get(id=hook_id) 16 | try: 17 | response = requests.post(url=hook.target, data=payload, headers=hook.headers) 18 | if response.status_code >= 400: 19 | response.raise_for_status() 20 | except (requests.ConnectionError, requests.HTTPError) as exc: 21 | try: 22 | self.retry(countdown=2**self.request.retries) 23 | except MaxRetriesExceededError: 24 | disable_message = ( 25 | f"we will attempt to deliver to " 26 | f"this hook up to " 27 | f"{(hook.MAX_CONSECUTIVE_FAILURES_BEFORE_DISABLE - hook.consecutive_failures)} " 28 | f"more times (with {HTTP_ATTEMPT_COUNT_PER_DELIVERY_ATTEMPT} HTTP retries " 29 | f"each attempt) before we disable it automatically" 30 | ) 31 | hook.consecutive_failures += 1 32 | if hook.consecutive_failures > hook.MAX_CONSECUTIVE_FAILURES_BEFORE_DISABLE: 33 | disable_message = ( 34 | f"we have disabled this hook due to exceeding the limit of " 35 | f"{hook.MAX_CONSECUTIVE_FAILURES_BEFORE_DISABLE} consecutive failures" 36 | ) 37 | hook.enabled = False 38 | hook.save() 39 | 40 | raise RuntimeError( 41 | f"Max retry count exceeded for target {hook.target}, {disable_message}", 42 | ) from exc 43 | 44 | if hook.consecutive_failures != 0: 45 | hook.consecutive_failures = 0 46 | hook.save() 47 | -------------------------------------------------------------------------------- /src/meshapi_hooks/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nycmeshnet/meshdb/a2f79331699ff26480e0d1123a387b6677df2fb1/src/meshapi_hooks/tests/__init__.py -------------------------------------------------------------------------------- /src/meshdb/__init__.py: -------------------------------------------------------------------------------- 1 | # This will make sure the app is always imported when 2 | # Django starts so that shared_task will use this app. 3 | from .celery import app as celery_app 4 | 5 | __all__ = ("celery_app",) 6 | -------------------------------------------------------------------------------- /src/meshdb/admin.py: -------------------------------------------------------------------------------- 1 | from typing import Any, List, Literal, Optional 2 | 3 | from admin_site_search.views import AdminSiteSearchView 4 | from django.contrib import admin 5 | from django.http import HttpRequest 6 | 7 | 8 | class MeshDBAdminSite(AdminSiteSearchView, admin.AdminSite): 9 | site_search_method: Literal["model_char_fields", "admin_search_fields"] = "admin_search_fields" 10 | 11 | def get_app_list(self, request: HttpRequest, app_label: Optional[str] = None) -> List[Any]: 12 | """Reorder the apps in the admin site. 13 | 14 | By default, django admin apps are order alphabetically. 15 | 16 | To keep things organized, we want all the auth/metadata stuff to be listed before the 17 | actual models of the meshapi app 18 | 19 | And since django does not offer a simple way to order apps, we have to tinker 20 | with the default app list, to change the sorting 21 | """ 22 | apps = super().get_app_list(request, app_label) 23 | 24 | SORT_OVERRIDES = {"auth": "0", "authtoken": "1", "meshapi_hooks": "2"} 25 | 26 | return sorted( 27 | apps, 28 | key=lambda x: SORT_OVERRIDES[x["app_label"]] if x["app_label"] in SORT_OVERRIDES else x["name"].lower(), 29 | ) 30 | -------------------------------------------------------------------------------- /src/meshdb/apps.py: -------------------------------------------------------------------------------- 1 | from django.contrib.admin.apps import AdminConfig 2 | 3 | 4 | class MeshDBAdminConfig(AdminConfig): 5 | default_site = "meshdb.admin.MeshDBAdminSite" 6 | -------------------------------------------------------------------------------- /src/meshdb/asgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | ASGI config for meshdb project. 3 | 4 | It exposes the ASGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/4.2/howto/deployment/asgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.asgi import get_asgi_application 13 | 14 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "meshdb.settings") 15 | 16 | application = get_asgi_application() 17 | -------------------------------------------------------------------------------- /src/meshdb/celery.py: -------------------------------------------------------------------------------- 1 | import os 2 | from pathlib import Path 3 | 4 | from celery import Celery, bootsteps 5 | from celery.apps.worker import Worker 6 | from celery.signals import beat_init, worker_ready, worker_shutdown 7 | from datadog import initialize, statsd 8 | from dotenv import load_dotenv 9 | 10 | HEARTBEAT_FILE = Path("/tmp/celery_worker_heartbeat") 11 | READINESS_FILE = Path("/tmp/celery_worker_ready") 12 | BEAT_READINESS_FILE = Path("/tmp/celery_beat_ready") 13 | 14 | 15 | # This is still somewhat contentious in the Celery community, but the popular 16 | # opinion seems to be this approach: 17 | # https://medium.com/ambient-innovation/health-checks-for-celery-in-kubernetes-cf3274a3e106 18 | # https://github.com/celery/celery/issues/4079#issuecomment-1270085680 19 | # https://docs.celeryq.dev/projects/pytest-celery/en/latest/userguide/signals.html#signals-py 20 | class LivenessProbe(bootsteps.StartStopStep): 21 | requires = {"celery.worker.components:Timer"} 22 | 23 | def __init__(self, parent: Worker, **kwargs: dict) -> None: 24 | self.requests: list = [] 25 | self.tref = None 26 | 27 | def start(self, parent: Worker) -> None: 28 | self.tref = parent.timer.call_repeatedly( 29 | 1.0, 30 | self.update_heartbeat_file, 31 | (parent,), 32 | priority=10, 33 | ) 34 | 35 | def stop(self, parent: Worker) -> None: 36 | HEARTBEAT_FILE.unlink(missing_ok=True) 37 | 38 | def update_heartbeat_file(self, parent: Worker) -> None: 39 | statsd.increment("meshdb.celery.heartbeat") 40 | HEARTBEAT_FILE.touch() 41 | 42 | 43 | @worker_ready.connect # type: ignore[no-redef] 44 | def worker_ready(**_: dict) -> None: 45 | READINESS_FILE.touch() 46 | 47 | 48 | @worker_shutdown.connect # type: ignore[no-redef] 49 | def worker_shutdown(**_: dict) -> None: 50 | READINESS_FILE.unlink(missing_ok=True) 51 | 52 | 53 | @beat_init.connect 54 | def beat_ready(**_: dict) -> None: 55 | BEAT_READINESS_FILE.touch() 56 | 57 | 58 | # Initialize dogstatsd 59 | initialize(statsd_host="datadog-agent.datadog.svc.cluster.local", statsd_port=8125) 60 | 61 | load_dotenv() 62 | 63 | # Set the default Django settings module for the 'celery' program. 64 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "meshdb.settings") 65 | 66 | # Use the docker-hosted Redis container as the backend for Celery 67 | app = Celery("meshdb", broker=os.environ.get("CELERY_BROKER", "redis://localhost:6379/0")) 68 | 69 | app.steps["worker"].add(LivenessProbe) 70 | 71 | # https://stackoverflow.com/questions/77479841/adding-celery-settings-what-does-it-do#78088996 72 | app.conf.broker_connection_retry_on_startup = True 73 | 74 | # Using a string here means the worker doesn't have to serialize 75 | # the configuration object to child processes. 76 | # - namespace='CELERY' means all celery-related configuration keys 77 | # should have a `CELERY_` prefix. 78 | app.config_from_object("django.conf:settings", namespace="CELERY") 79 | 80 | # Load task modules from all registered Django apps. 81 | app.autodiscover_tasks() 82 | -------------------------------------------------------------------------------- /src/meshdb/templates/admin/base_site.html: -------------------------------------------------------------------------------- 1 | {% extends "admin/base_site.html" %} 2 | 3 | {% load static %} 4 | 5 | {% block extrahead %} 6 | 7 | {% include 'admin_site_search/head.html' %} 8 | {{ block.super }} 9 | {% endblock %} 10 | 11 | {% block branding %} 12 |

{{ site_header|default:_('Django administration') }}

13 | {% if user.is_anonymous %} 14 | {% include "admin/color_theme_toggle.html" %} 15 | {% endif %} 16 | {% endblock %} 17 | 18 | {% block userlinks %} 19 | {{ block.super }} 20 | {% endblock %} 21 | 22 | {% block footer %} 23 | {{ block.super }} 24 | {% include 'admin_site_search/modal.html' %} 25 | {% endblock %} 26 | 27 | {% block usertools %} 28 | {% include 'admin_site_search/button.html' %} 29 | {{ block.super }} 30 | {% endblock %} 31 | -------------------------------------------------------------------------------- /src/meshdb/templates/admin/login.html: -------------------------------------------------------------------------------- 1 | {% extends "admin/login.html" %} 2 | 3 | {% load static %} 4 | 5 | {% block branding %} 6 |

{{ site_header|default:_('Django administration') }}

7 | {% if user.is_anonymous %} 8 | {% include "admin/color_theme_toggle.html" %} 9 | {% endif %} 10 | {% endblock %} 11 | 12 | -------------------------------------------------------------------------------- /src/meshdb/templates/admin/password_reset/password_reset_complete.html: -------------------------------------------------------------------------------- 1 | {% extends "registration/password_reset_complete.html" %} 2 | 3 | {% block map_sidebar %}{% endblock %} -------------------------------------------------------------------------------- /src/meshdb/templates/admin/password_reset/password_reset_confirm.html: -------------------------------------------------------------------------------- 1 | {% extends "registration/password_reset_confirm.html" %} 2 | 3 | {% block map_sidebar %}{% endblock %} -------------------------------------------------------------------------------- /src/meshdb/templates/admin/password_reset/password_reset_done.html: -------------------------------------------------------------------------------- 1 | {% extends "registration/password_reset_done.html" %} 2 | 3 | {% block map_sidebar %}{% endblock %} -------------------------------------------------------------------------------- /src/meshdb/templates/admin/password_reset/password_reset_email_subject.txt: -------------------------------------------------------------------------------- 1 | {% load i18n %}{% autoescape off %} 2 | {% blocktranslate %}[MeshDB] Password Reset Request{% endblocktranslate %} 3 | {% endautoescape %} -------------------------------------------------------------------------------- /src/meshdb/templates/admin/password_reset/password_reset_form.html: -------------------------------------------------------------------------------- 1 | {% extends "registration/password_reset_form.html" %} 2 | 3 | {% block map_sidebar %}{% endblock %} -------------------------------------------------------------------------------- /src/meshdb/templates/drf_spectacular/swagger_ui.html: -------------------------------------------------------------------------------- 1 | {% extends "drf_spectacular/swagger_ui.html" %} 2 | 3 | {% block head %} 4 | {{ block.super }} 5 | 41 | {% endblock head %} -------------------------------------------------------------------------------- /src/meshdb/templates/rest_framework/login_base.html: -------------------------------------------------------------------------------- 1 | {% extends "rest_framework/login_base.html" %} 2 | 3 | {% block branding %} 4 |

MeshDB Data API Login

5 |

6 | 7 | Admin UI credentials work here, but non "staff" users who don't have admin 8 | console access can also log in here to access the API and API docs 9 | 10 |

11 | {% endblock %} 12 | 13 | -------------------------------------------------------------------------------- /src/meshdb/urls.py: -------------------------------------------------------------------------------- 1 | """ 2 | URL configuration for meshdb project. 3 | 4 | The `urlpatterns` list routes URLs to views. For more information please see: 5 | https://docs.djangoproject.com/en/4.2/topics/http/urls/ 6 | Examples: 7 | Function views 8 | 1. Add an import: from my_app import views 9 | 2. Add a URL to urlpatterns: path('', views.home, name='home') 10 | Class-based views 11 | 1. Add an import: from other_app.views import Home 12 | 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') 13 | Including another URLconf 14 | 1. Import the include() function: from django.urls import include, path 15 | 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) 16 | """ 17 | 18 | from django.urls import include, path 19 | from drf_spectacular.views import SpectacularAPIView, SpectacularRedocView 20 | 21 | from meshapi.docs import SpectacularSwaggerInjectVarsView 22 | from meshapi.views.autocomplete import MemberAutocomplete 23 | from meshdb.settings import PROFILING_ENABLED 24 | 25 | urlpatterns = [ 26 | path("", include("meshweb.urls")), 27 | path("auth/", include("rest_framework.urls")), 28 | path("admin/", include("meshapi.admin.urls")), 29 | path("api/v1/", include("meshapi.urls")), 30 | path("api-docs/openapi3.json", SpectacularAPIView.as_view(), name="api-docs-schema"), 31 | path( 32 | "api-docs/swagger/", 33 | SpectacularSwaggerInjectVarsView.as_view( 34 | url_name="api-docs-schema" # , template_name="drf_spectacular/swagger_ui.html" 35 | ), 36 | name="api-docs-swagger", 37 | ), 38 | path("api-docs/redoc/", SpectacularRedocView.as_view(url_name="api-docs-schema"), name="api-docs-redoc"), 39 | path( 40 | "member-autocomplete/", 41 | MemberAutocomplete.as_view(), 42 | name="member-autocomplete", 43 | ), 44 | ] 45 | 46 | if PROFILING_ENABLED: 47 | urlpatterns.append(path("silk/", include("silk.urls", namespace="silk"))) 48 | -------------------------------------------------------------------------------- /src/meshdb/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nycmeshnet/meshdb/a2f79331699ff26480e0d1123a387b6677df2fb1/src/meshdb/utils/__init__.py -------------------------------------------------------------------------------- /src/meshdb/views.py: -------------------------------------------------------------------------------- 1 | from django.contrib.admin.views.decorators import staff_member_required 2 | from django.http import HttpRequest, HttpResponse 3 | from django.shortcuts import render 4 | 5 | 6 | @staff_member_required 7 | def admin_iframe_view(request: HttpRequest) -> HttpResponse: 8 | return render(request, "admin/iframed.html") 9 | -------------------------------------------------------------------------------- /src/meshdb/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for meshdb project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/4.2/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | 12 | # Initialize dogstatsd 13 | from datadog import initialize 14 | from django.core.wsgi import get_wsgi_application 15 | 16 | initialize(statsd_host="datadog-agent.datadog.svc.cluster.local", statsd_port=8125) 17 | 18 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "meshdb.settings") 19 | 20 | application = get_wsgi_application() 21 | -------------------------------------------------------------------------------- /src/meshweb/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nycmeshnet/meshdb/a2f79331699ff26480e0d1123a387b6677df2fb1/src/meshweb/__init__.py -------------------------------------------------------------------------------- /src/meshweb/middleware.py: -------------------------------------------------------------------------------- 1 | from typing import Callable 2 | 3 | from django.http import HttpRequest, HttpResponse, HttpResponseRedirect 4 | from django.urls import reverse 5 | from flags.state import flag_enabled 6 | 7 | 8 | class MaintenanceModeMiddleware: 9 | def __init__(self, get_response: Callable) -> None: 10 | self.get_response = get_response 11 | 12 | def __call__(self, request: HttpRequest) -> HttpResponse: 13 | path = request.META.get("PATH_INFO", "") 14 | if flag_enabled("MAINTENANCE_MODE") and path not in [ 15 | reverse("maintenance"), 16 | reverse("maintenance-enable"), 17 | reverse("maintenance-disable"), 18 | reverse("rest_framework:login"), 19 | ]: 20 | response = HttpResponseRedirect(reverse("maintenance")) 21 | return response 22 | 23 | response = self.get_response(request) 24 | 25 | return response 26 | -------------------------------------------------------------------------------- /src/meshweb/static/admin/iframe_check.js: -------------------------------------------------------------------------------- 1 | async function checkIframed() { 2 | // Check if we're in an iframe 3 | const inIframe = window.self !== window.top; 4 | 5 | // If we are, do nothing 6 | if (inIframe || mobileCheck()) { 7 | return; 8 | } 9 | 10 | // Also do nothing if we're not in the admin panel proper (eg not logged in) 11 | const escURLs = ["login", "password_reset"] 12 | const currentURL = window.location.href; 13 | 14 | var shouldEscape = escURLs.some(url => currentURL.includes(url)); 15 | if (shouldEscape) { 16 | return; 17 | } 18 | 19 | // If we're not in an iframe, then we'll want to swap the user to the iframed 20 | // view 21 | try { 22 | response = await fetch(PANEL_URL); 23 | if (!response.ok) { 24 | throw new Error( 25 | `Error loading new contents for page: ${response.status} ${response.statusText}` 26 | ); 27 | } 28 | } catch (e) { 29 | console.error(`Error during page nav to %s`, PANEL_URL, e) 30 | const mapWrapper = document.getElementById("container"); 31 | 32 | const pageLink = document.createElement("a"); 33 | pageLink.className = "capture-exclude"; 34 | pageLink.href = PANEL_URL; 35 | pageLink.textContent = PANEL_URL; 36 | 37 | const errorNotice = document.createElement("p"); 38 | errorNotice.className = "error-box"; 39 | errorNotice.innerHTML = `Error loading page: ${pageLink.outerHTML}
${e}` 40 | 41 | mapWrapper.parentNode.insertBefore( 42 | errorNotice, 43 | mapWrapper 44 | ); 45 | return; 46 | } 47 | 48 | document.open(); // Clears the screen 49 | document.write(await response.text()); 50 | document.close(); 51 | } 52 | 53 | checkIframed(); 54 | -------------------------------------------------------------------------------- /src/meshweb/static/admin/iframed.css: -------------------------------------------------------------------------------- 1 | iframe { 2 | height: 100%; 3 | width: 100%; 4 | border: none; 5 | z-index: 1; 6 | } 7 | 8 | .frameGrow { 9 | flex-grow: 1; 10 | border: none; 11 | margin: 0; 12 | padding: 0; 13 | } 14 | 15 | #page_container { 16 | display: flex; 17 | flex-direction: row; 18 | height: 100%; 19 | } 20 | -------------------------------------------------------------------------------- /src/meshweb/static/admin/mobile_check.js: -------------------------------------------------------------------------------- 1 | // Taken from: https://stackoverflow.com/a/11381730 2 | const mobileCheck = function() { 3 | let check = false; 4 | (function(a){if(/(android|bb\d+|meego).+mobile|avantgo|bada\/|blackberry|blazer|compal|elaine|fennec|hiptop|iemobile|ip(hone|od)|iris|kindle|lge |maemo|midp|mmp|mobile.+firefox|netfront|opera m(ob|in)i|palm( os)?|phone|p(ixi|re)\/|plucker|pocket|psp|series(4|6)0|symbian|treo|up\.(browser|link)|vodafone|wap|windows ce|xda|xiino/i.test(a)||/1207|6310|6590|3gso|4thp|50[1-6]i|770s|802s|a wa|abac|ac(er|oo|s\-)|ai(ko|rn)|al(av|ca|co)|amoi|an(ex|ny|yw)|aptu|ar(ch|go)|as(te|us)|attw|au(di|\-m|r |s )|avan|be(ck|ll|nq)|bi(lb|rd)|bl(ac|az)|br(e|v)w|bumb|bw\-(n|u)|c55\/|capi|ccwa|cdm\-|cell|chtm|cldc|cmd\-|co(mp|nd)|craw|da(it|ll|ng)|dbte|dc\-s|devi|dica|dmob|do(c|p)o|ds(12|\-d)|el(49|ai)|em(l2|ul)|er(ic|k0)|esl8|ez([4-7]0|os|wa|ze)|fetc|fly(\-|_)|g1 u|g560|gene|gf\-5|g\-mo|go(\.w|od)|gr(ad|un)|haie|hcit|hd\-(m|p|t)|hei\-|hi(pt|ta)|hp( i|ip)|hs\-c|ht(c(\-| |_|a|g|p|s|t)|tp)|hu(aw|tc)|i\-(20|go|ma)|i230|iac( |\-|\/)|ibro|idea|ig01|ikom|im1k|inno|ipaq|iris|ja(t|v)a|jbro|jemu|jigs|kddi|keji|kgt( |\/)|klon|kpt |kwc\-|kyo(c|k)|le(no|xi)|lg( g|\/(k|l|u)|50|54|\-[a-w])|libw|lynx|m1\-w|m3ga|m50\/|ma(te|ui|xo)|mc(01|21|ca)|m\-cr|me(rc|ri)|mi(o8|oa|ts)|mmef|mo(01|02|bi|de|do|t(\-| |o|v)|zz)|mt(50|p1|v )|mwbp|mywa|n10[0-2]|n20[2-3]|n30(0|2)|n50(0|2|5)|n7(0(0|1)|10)|ne((c|m)\-|on|tf|wf|wg|wt)|nok(6|i)|nzph|o2im|op(ti|wv)|oran|owg1|p800|pan(a|d|t)|pdxg|pg(13|\-([1-8]|c))|phil|pire|pl(ay|uc)|pn\-2|po(ck|rt|se)|prox|psio|pt\-g|qa\-a|qc(07|12|21|32|60|\-[2-7]|i\-)|qtek|r380|r600|raks|rim9|ro(ve|zo)|s55\/|sa(ge|ma|mm|ms|ny|va)|sc(01|h\-|oo|p\-)|sdk\/|se(c(\-|0|1)|47|mc|nd|ri)|sgh\-|shar|sie(\-|m)|sk\-0|sl(45|id)|sm(al|ar|b3|it|t5)|so(ft|ny)|sp(01|h\-|v\-|v )|sy(01|mb)|t2(18|50)|t6(00|10|18)|ta(gt|lk)|tcl\-|tdg\-|tel(i|m)|tim\-|t\-mo|to(pl|sh)|ts(70|m\-|m3|m5)|tx\-9|up(\.b|g1|si)|utst|v400|v750|veri|vi(rg|te)|vk(40|5[0-3]|\-v)|vm40|voda|vulc|vx(52|53|60|61|70|80|81|83|85|98)|w3c(\-| )|webc|whit|wi(g |nc|nw)|wmlb|wonu|x700|yas\-|your|zeto|zte\-/i.test(a.substr(0,4))) check = true;})(navigator.userAgent||navigator.vendor||window.opera); 5 | return check; 6 | }; 7 | -------------------------------------------------------------------------------- /src/meshweb/static/admin/panel_url_check.js: -------------------------------------------------------------------------------- 1 | // Used to check if a user has accessed the PANEL_URL directly, and adjusts the 2 | // URL for them 3 | // This file is meant to be loaded from iframe.html where that variable is defined 4 | function checkForPanelURL() { 5 | const entryPath = new URL(window.location.href).pathname; 6 | if (entryPath.includes(PANEL_URL)) { 7 | window.history.pushState("MeshDB Admin Panel", "", entryPath.replace(PANEL_URL, "/admin/")); 8 | } 9 | } 10 | 11 | checkForPanelURL(); 12 | -------------------------------------------------------------------------------- /src/meshweb/static/meshweb/developer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nycmeshnet/meshdb/a2f79331699ff26480e0d1123a387b6677df2fb1/src/meshweb/static/meshweb/developer.png -------------------------------------------------------------------------------- /src/meshweb/static/meshweb/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nycmeshnet/meshdb/a2f79331699ff26480e0d1123a387b6677df2fb1/src/meshweb/static/meshweb/favicon.png -------------------------------------------------------------------------------- /src/meshweb/static/meshweb/join_record_viewer.css: -------------------------------------------------------------------------------- 1 | 2 | h1, h3 { 3 | font-family: arial, sans-serif; 4 | } 5 | 6 | table { 7 | font-family: arial, sans-serif; 8 | border-collapse: collapse; 9 | width: 100%; 10 | } 11 | 12 | td, th { 13 | border: 1px solid #dddddd; 14 | text-align: left; 15 | padding: 8px; 16 | } 17 | 18 | tr:nth-child(even) { 19 | background-color: #dddddd; 20 | } 21 | 22 | .recordTable { 23 | display:flex; 24 | flex-direction: column; 25 | align-content: flex-start; 26 | justify-content: space-between; 27 | margin-top: 20px; 28 | overflow: scroll; 29 | margin: 20px; 30 | } 31 | 32 | .recordViewerHead { 33 | display: flex; 34 | flex-direction: row; 35 | justify-content: space-between; 36 | margin: 20px; 37 | } 38 | 39 | #paramForm { 40 | display: flex; 41 | flex-direction: row; 42 | justify-content: space-between; 43 | column-gap: 20px; 44 | } 45 | -------------------------------------------------------------------------------- /src/meshweb/static/meshweb/logo.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | background 5 | 6 | 7 | 8 | Layer 1 9 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /src/meshweb/static/meshweb/member.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nycmeshnet/meshdb/a2f79331699ff26480e0d1123a387b6677df2fb1/src/meshweb/static/meshweb/member.png -------------------------------------------------------------------------------- /src/meshweb/static/meshweb/meshdb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nycmeshnet/meshdb/a2f79331699ff26480e0d1123a387b6677df2fb1/src/meshweb/static/meshweb/meshdb.png -------------------------------------------------------------------------------- /src/meshweb/static/meshweb/styles.css: -------------------------------------------------------------------------------- 1 | /* 2 | * Not sure why bootstrap does this, but gotta hardcode the bottom margin to not 3 | * Be 0 4 | * */ 5 | .h1, .h2, .h3, .h4, .h5, .h6, h1, h2, h3, h4, h5, h6 { 6 | margin-bottom: 0; 7 | } 8 | 9 | .max-width-container { 10 | max-width: 1200px !important; 11 | padding: 0em 3em; 12 | /* margin: 0 auto; */ 13 | /* padding: 10em; */ 14 | } 15 | 16 | a { 17 | color: inherit !important; 18 | } 19 | 20 | body { 21 | min-height: 100vh; 22 | display: flex; 23 | flex-direction: column; 24 | align-items: stretch; 25 | } 26 | 27 | footer, 28 | nav { 29 | --bs-bg-opacity: 1; 30 | background-color: #f4f4f4 !important; 31 | } 32 | 33 | footer { 34 | width: 100%; 35 | left: 0; 36 | bottom: 0; 37 | flex-grow: 1; 38 | } 39 | 40 | footer a { 41 | text-decoration: none; 42 | } 43 | 44 | .logo-no-underline a { 45 | text-decoration: none; 46 | } 47 | 48 | .navbar-brand.collapse .text { 49 | display: none; 50 | } 51 | 52 | .joinButton { 53 | display: flex; 54 | justify-content: center; 55 | padding: 20px; 56 | } 57 | 58 | .meshDbHeader { 59 | display:flex; 60 | flex-direction:row; 61 | justify-content: center; 62 | padding: 40px 0 0 0; 63 | } 64 | 65 | .links { 66 | display: flex; 67 | justify-content: center !important; 68 | column-gap: 3rem; 69 | row-gap: 2rem; 70 | flex-wrap: wrap; 71 | align-content: flex-start; 72 | flex-direction: row; 73 | margin-top: 20px; 74 | } 75 | 76 | .toolList { 77 | display: flex; 78 | flex-direction: column; 79 | } 80 | 81 | .toolListHeading { 82 | display: flex; 83 | align-content: center; 84 | justify-content: space-between; 85 | color: #55585b; 86 | border-style: solid; 87 | border-color: #55585b; 88 | border-width: 0px 0px 1px 0px; 89 | padding: 10px; 90 | } 91 | 92 | .toolListItem { 93 | display:flex; 94 | padding: 16px 10px; 95 | } 96 | 97 | .toolListLink { 98 | text-decoration: none; 99 | } 100 | 101 | .toolListItem:hover { 102 | background-color: #f9e79f; 103 | border-radius: 5px; 104 | } 105 | -------------------------------------------------------------------------------- /src/meshweb/static/meshweb/uisp_on_demand_form.css: -------------------------------------------------------------------------------- 1 | .mainContent { 2 | display:flex; 3 | justify-content: center; 4 | padding: 0 20%; 5 | } 6 | 7 | .banner { 8 | display: flex; 9 | flex-direction: row; 10 | align-items: center; 11 | justify-content: center; 12 | } 13 | 14 | .banner h1 { 15 | padding: 0 20px; 16 | } 17 | -------------------------------------------------------------------------------- /src/meshweb/static/meshweb/uisp_on_demand_form.js: -------------------------------------------------------------------------------- 1 | function getCookie(name) { 2 | const value = `; ${document.cookie}`; 3 | const parts = value.split(`; ${name}=`); 4 | if (parts.length === 2) return parts.pop().split(';').shift(); 5 | } 6 | 7 | async function submitForm(event) { 8 | event.preventDefault(); 9 | 10 | loadingBanner = document.getElementById('loadingBanner'); 11 | successBanner = document.getElementById('successBanner'); 12 | errorBanner = document.getElementById('errorBanner'); 13 | submitButton = document.getElementById('submitButton'); 14 | 15 | // Hide the result banners 16 | successBanner.style.display = 'none'; 17 | errorBanner.style.display = 'none'; 18 | submitButton.disabled = true; 19 | 20 | // Show loading banner 21 | loadingBanner.style.display = 'flex'; 22 | const number = document.getElementById('numberInput').value; 23 | fetch(`/api/v1/uisp-import/nn/${number}/`, { 24 | method: 'POST', 25 | headers: { 26 | 'Content-Type': 'application/json', 27 | 'X-CSRFToken': getCookie('csrftoken'), 28 | }, 29 | }) 30 | .then(async response => { 31 | if (!response.ok) { 32 | const j = await response.json() 33 | throw new Error(`${response.status} ${j.detail}`); 34 | } 35 | return response.json(); 36 | }) 37 | .then(data => { 38 | console.log('Success:', data.status); 39 | loadingBanner.style.display = 'none'; 40 | successBanner.style.display = 'flex'; 41 | submitButton.disabled = false; 42 | }) 43 | .catch(error => { 44 | document.getElementById('errorDetail').innerHTML = `${error}`; 45 | loadingBanner.style.display = 'none'; 46 | errorBanner.style.display = 'flex'; 47 | submitButton.disabled = false; 48 | }); 49 | } 50 | -------------------------------------------------------------------------------- /src/meshweb/static/meshweb/uispondemand.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nycmeshnet/meshdb/a2f79331699ff26480e0d1123a387b6677df2fb1/src/meshweb/static/meshweb/uispondemand.png -------------------------------------------------------------------------------- /src/meshweb/static/meshweb/volunteer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/nycmeshnet/meshdb/a2f79331699ff26480e0d1123a387b6677df2fb1/src/meshweb/static/meshweb/volunteer.png -------------------------------------------------------------------------------- /src/meshweb/templates/meshweb/head.html: -------------------------------------------------------------------------------- 1 | {% load static %} 2 | 3 | 4 | 5 | Welcome to MeshDB 6 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /src/meshweb/templates/meshweb/index.html: -------------------------------------------------------------------------------- 1 | {% load static %} 2 | 3 | 4 | 5 | 6 | 7 | {% include "meshweb/head.html" %} 8 | 9 | 10 | 11 | {% include "meshweb/nav.html" %} 12 |
13 | 18 |
19 |

Welcome to MeshDB!

20 |

21 | MeshDB is used to track Installs, Members, and Buildings in the mesh.
22 | Reach out on Slack using the #meshdb channel if you need access. 23 |

24 |
25 |
26 | {% if not user.is_authenticated %} 27 |
28 | 29 |
30 | {% endif %} 31 |
32 | {% for category, tools in links.items %} 33 |
34 |
35 | 36 |
{{ category.1 }}
37 |
38 | {% for link, description in tools %} 39 | 40 |
41 |
{{ description }}
42 |
43 | 44 | {% endfor %} 45 |
46 | {% endfor %} 47 |
48 | 51 | 52 | 53 | {% include "meshweb/footer.html" %} 54 | 55 | -------------------------------------------------------------------------------- /src/meshweb/templates/meshweb/join_record_viewer.html: -------------------------------------------------------------------------------- 1 | {% load static %} 2 | 3 | 4 | 5 | {% include "meshweb/head.html" %} 6 | 7 | 8 | 9 | {% include "meshweb/nav.html" %} 10 |
11 |

Join Record Viewer

12 |
13 |
14 | 15 | 16 |
17 |
18 | 19 | 20 |
21 | 22 |
23 |
24 |
25 | {% if records and not all %} 26 |

The following records need to be replayed:

27 | {% else %} 28 |

Showing all records since selected time

29 | {% endif %} 30 |
31 |
32 | {% if records %} 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | {% for record in records %} 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | {% endfor %} 72 |
first_namelast_nameemail_addressphone_numberstreet_addresscitystatezip_codeapartmentroof_accessreferralncltrust_me_brosubmission_timecodeinstall_number
{{ record.first_name }}{{ record.last_name }}{{ record.email_address }}{{ record.phone_number }}{{ record.street_address }}{{ record.city }}{{ record.state }}{{ record.zip_code }}{{ record.apartment }}{{ record.roof_access }}{{ record.referral }}{{ record.ncl }}{{ record.trust_me_bro }}{{ record.submission_time }}{{ record.code }}{{ record.install_number }}
73 | {% else %} 74 | {% if not all %} 75 |

All good! No records need to be replayed.

76 | {% else %} 77 |

No records found. Check your query.

78 | {% endif %} 79 | {% endif %} 80 |
81 | 82 | {% include "meshweb/footer.html" %} 83 | 84 | -------------------------------------------------------------------------------- /src/meshweb/templates/meshweb/maintenance.html: -------------------------------------------------------------------------------- 1 | {% load static %} 2 | 3 | 4 | 5 | 6 | 7 | 8 | Maintenance Mode 9 | 11 | 12 | 13 | 14 | 15 |
16 |
18 | 19 |

MeshDB is in Maintenance Mode.

20 |

{{ message }}

21 |
22 |
23 | {% if redirect != "" %} 24 | 25 | {% endif %} 26 | 27 | 28 | -------------------------------------------------------------------------------- /src/meshweb/templates/meshweb/nav.html: -------------------------------------------------------------------------------- 1 | {% load static %} 2 | 3 | 42 | -------------------------------------------------------------------------------- /src/meshweb/templates/meshweb/uisp_on_demand_form.html: -------------------------------------------------------------------------------- 1 | {% load static %} 2 | 3 | 4 | 5 | {% include "meshweb/head.html" %} 6 | 7 | 8 | 9 | 10 | {% include "meshweb/nav.html" %} 11 |
12 | 13 |
14 |
15 |

Use this form to trigger an import of UISP objects for a given Network Number. This is typically an expensive operation, so we only import objects from UISP hourly. However, if you need the latest data, a single NN will take ~5 minutes.

16 |
17 |
18 |
19 | 20 | 21 |
22 | 23 |
24 |
25 |
26 | 32 | 35 | 38 |
39 | 40 | {% include "meshweb/footer.html" %} 41 | 42 | -------------------------------------------------------------------------------- /src/meshweb/tests.py: -------------------------------------------------------------------------------- 1 | # from django.test import TestCase 2 | 3 | # Create your tests here. 4 | -------------------------------------------------------------------------------- /src/meshweb/urls.py: -------------------------------------------------------------------------------- 1 | from django.urls import include, path 2 | 3 | from meshweb import views 4 | 5 | urlpatterns = [ 6 | path("", views.index, name="main"), 7 | path("maintenance/", views.maintenance, name="maintenance"), 8 | path("maintenance/enable/", views.enable_maintenance, name="maintenance-enable"), 9 | path("maintenance/disable/", views.disable_maintenance, name="maintenance-disable"), 10 | path("website-embeds/stats-graph.svg", views.website_stats_graph, name="legacy-stats-svg"), 11 | path("website-embeds/stats-graph.json", views.website_stats_json, name="legacy-stats-json"), 12 | path("explorer/", include("explorer.urls")), 13 | path("join-records/view/", views.join_record_viewer, name="join-record-viewer"), 14 | path("uisp-on-demand/", views.uisp_on_demand_form, name="uisp-on-demand"), 15 | ] 16 | -------------------------------------------------------------------------------- /src/meshweb/views/__init__.py: -------------------------------------------------------------------------------- 1 | from .explorer_redirect import * 2 | from .index import * 3 | from .join_record_viewer import * 4 | from .maintenance import * 5 | from .uisp_on_demand import * 6 | from .website_stats import * 7 | -------------------------------------------------------------------------------- /src/meshweb/views/explorer_redirect.py: -------------------------------------------------------------------------------- 1 | from django.http import HttpRequest, HttpResponse, HttpResponseRedirect 2 | 3 | 4 | def explorer_auth_redirect(_: HttpRequest) -> HttpResponse: 5 | # Auth Redirect to ensure that behavior is consistent with admin panel 6 | return HttpResponseRedirect("/admin/login/?next=/explorer/") 7 | -------------------------------------------------------------------------------- /src/meshweb/views/index.py: -------------------------------------------------------------------------------- 1 | from datadog import statsd 2 | from django.conf import settings 3 | from django.http import HttpRequest, HttpResponse 4 | from django.template import loader 5 | 6 | 7 | def index(request: HttpRequest) -> HttpResponse: 8 | statsd.increment("meshdb.views.index", tags=[]) 9 | template = loader.get_template("meshweb/index.html") 10 | links = { 11 | ("meshweb/member.png", "Member Tools"): [ 12 | (f"{settings.FORMS_URL}/join/", "Join Form"), 13 | (settings.LOS_URL, "Line of Sight Tool"), 14 | (settings.MAP_URL, "Map"), 15 | ("https://github.com/orgs/nycmeshnet/projects/6/views/1", "Feature Requests"), 16 | ], 17 | ("meshweb/volunteer.png", "Volunteer Tools"): [ 18 | ("/admin", "Admin Panel"), 19 | ("/api/v1/geography/whole-mesh.kml", "KML Download"), 20 | ("/explorer/play", "SQL Explorer"), 21 | (f"{settings.FORMS_URL}/nn-assign/", "NN Assign Form"), 22 | (f"{settings.FORMS_URL}/query/", "Query Form"), 23 | ("/join-records/view/", "Join Record Viewer"), 24 | ("/uisp-on-demand/", "UISP Import"), 25 | ], 26 | ("meshweb/developer.png", "Developer Tools"): [ 27 | ("https://github.com/nycmeshnet/meshdb", "Source Code"), 28 | ("/api/v1/", "MeshDB Data API"), 29 | ("/api-docs/swagger/", "API Docs (Swagger)"), 30 | ("/api-docs/redoc/", "API Docs (Redoc)"), 31 | ], 32 | } 33 | context = {"links": links, "logo": "meshweb/logo.svg"} 34 | return HttpResponse(template.render(context, request)) 35 | -------------------------------------------------------------------------------- /src/meshweb/views/join_record_viewer.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from datetime import datetime, timezone 3 | 4 | from botocore.exceptions import ClientError 5 | from django.contrib.admin.views.decorators import staff_member_required 6 | from django.http import HttpRequest, HttpResponse 7 | from django.template import loader 8 | 9 | from meshapi.management.commands import replay_join_records 10 | from meshapi.util.join_records import JoinRecordProcessor 11 | 12 | 13 | @staff_member_required 14 | def join_record_viewer(request: HttpRequest) -> HttpResponse: 15 | since_param = request.GET.get("since") 16 | if not since_param: 17 | since = replay_join_records.Command.past_week() 18 | else: 19 | try: 20 | since = datetime.fromisoformat(since_param + "Z") 21 | except ValueError: 22 | status = 400 23 | m = f"({status}) Bad ISO-formatted string for parameter 'since'" 24 | logging.exception(m) 25 | return HttpResponse(m, status=status) 26 | 27 | if since > datetime.now(timezone.utc): 28 | status = 400 29 | m = f"({status}) Cannot retrieve records from the future!" 30 | logging.error(m) 31 | return HttpResponse(m, status=status) 32 | 33 | all = request.GET.get("all") == "True" 34 | 35 | template = loader.get_template("meshweb/join_record_viewer.html") 36 | 37 | try: 38 | records = JoinRecordProcessor().ensure_pre_post_consistency(since) 39 | except ClientError: 40 | status = 503 41 | m = f"({status}) Could not retrieve join records. Check bucket credentials." 42 | logging.exception(m) 43 | return HttpResponse(m, status=status) 44 | 45 | relevant_records = ( 46 | [r for _, r in records.items() if not replay_join_records.Command.filter_irrelevant_record(r)] 47 | if not all 48 | else records.values() 49 | ) 50 | 51 | context = {"records": relevant_records, "all": all, "logo": "meshweb/logo.svg"} 52 | return HttpResponse(template.render(context, request)) 53 | -------------------------------------------------------------------------------- /src/meshweb/views/maintenance.py: -------------------------------------------------------------------------------- 1 | from django.http import HttpRequest, HttpResponse, HttpResponseRedirect 2 | from django.template import loader 3 | from django.urls import reverse 4 | from drf_spectacular.utils import extend_schema 5 | from flags.state import disable_flag, enable_flag, flag_enabled 6 | from rest_framework.decorators import api_view, permission_classes 7 | 8 | from meshapi.permissions import HasMaintenanceModePermission 9 | 10 | 11 | def maintenance(request: HttpRequest) -> HttpResponse: 12 | if not flag_enabled("MAINTENANCE_MODE"): 13 | return HttpResponseRedirect(reverse("main")) 14 | template = loader.get_template("meshweb/maintenance.html") 15 | context = { 16 | "message": "Please check back later.", 17 | "redirect": "", 18 | } 19 | return HttpResponse(template.render(context, request)) 20 | 21 | 22 | @extend_schema(exclude=True) # Don't show on docs page 23 | @api_view(["POST"]) 24 | @permission_classes([HasMaintenanceModePermission]) 25 | def enable_maintenance(request: HttpRequest) -> HttpResponse: 26 | enable_flag("MAINTENANCE_MODE") 27 | template = loader.get_template("meshweb/maintenance.html") 28 | context = { 29 | "message": "Enabled maintenance mode.", 30 | "redirect": "maintenance", 31 | } 32 | return HttpResponse(template.render(context, request)) 33 | 34 | 35 | @extend_schema(exclude=True) # Don't show on docs page 36 | @api_view(["POST"]) 37 | @permission_classes([HasMaintenanceModePermission]) 38 | def disable_maintenance(request: HttpRequest) -> HttpResponse: 39 | if not flag_enabled("MAINTENANCE_MODE"): 40 | return HttpResponseRedirect("main") 41 | disable_flag("MAINTENANCE_MODE") 42 | template = loader.get_template("meshweb/maintenance.html") 43 | context = { 44 | "message": "Disabled maintenance mode.", 45 | "redirect": "main", 46 | } 47 | return HttpResponse(template.render(context, request)) 48 | -------------------------------------------------------------------------------- /src/meshweb/views/uisp_on_demand.py: -------------------------------------------------------------------------------- 1 | from ddtrace import tracer 2 | from django.contrib.admin.views.decorators import staff_member_required 3 | from django.http import HttpRequest, HttpResponse 4 | from django.template import loader 5 | 6 | 7 | @tracer.wrap() 8 | @staff_member_required 9 | def uisp_on_demand_form(request: HttpRequest) -> HttpResponse: 10 | template = loader.get_template("meshweb/uisp_on_demand_form.html") 11 | context = {"logo": "meshweb/logo.svg"} 12 | return HttpResponse(template.render(context, request)) 13 | -------------------------------------------------------------------------------- /tasks.py: -------------------------------------------------------------------------------- 1 | from invoke import task 2 | 3 | 4 | @task 5 | def format(context): 6 | context.run("black .") 7 | context.run("isort .") 8 | 9 | 10 | @task 11 | def lint(context): 12 | context.run("black . --check") 13 | context.run("isort . --check") 14 | context.run("flake8 src") 15 | context.run("mypy") 16 | 17 | 18 | @task 19 | def test(context): 20 | context.run("python src/manage.py test meshapi meshapi_hooks") 21 | -------------------------------------------------------------------------------- /test-results/.last-run.json: -------------------------------------------------------------------------------- 1 | { 2 | "status": "failed", 3 | "failedTests": [] 4 | } --------------------------------------------------------------------------------