├── requirements.txt ├── .dockerignore ├── restgdf_api ├── gunicorn.conf.py ├── app.py ├── layer.py ├── models.py ├── directory.py ├── mappingsupport.py └── utils.py ├── .github ├── pull_request_template.md ├── ISSUE_TEMPLATE │ ├── feature_request.md │ └── bug_report.md ├── dependabot.yml └── workflows │ ├── docker-hub.yml │ └── bumpver.yml ├── bumpver.toml ├── Dockerfile ├── Dockerfile-lab ├── docker-compose.yml ├── LICENSE ├── .gitignore ├── kubernetes └── resources.yaml ├── .pre-commit-config.yaml └── README.md /requirements.txt: -------------------------------------------------------------------------------- 1 | aiohttp==3.9.4 2 | fastapi==0.110.1 3 | gunicorn==22.0.0 4 | numpy==1.26.4 5 | pandas==2.2.2 6 | pydantic==2.7.0 7 | restgdf==1.0.0 8 | uvicorn==0.29.0 9 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | .env 2 | .env-example 3 | .git/ 4 | .github 5 | .gitignore 6 | .idea 7 | .mypy_cache 8 | .pre-commit-config.yaml 9 | .ruff_cache 10 | Dockerfile 11 | kubernetes 12 | docker-compose.yml 13 | junk/ 14 | kubernetes/ 15 | -------------------------------------------------------------------------------- /restgdf_api/gunicorn.conf.py: -------------------------------------------------------------------------------- 1 | # Gunicorn configuration file 2 | import multiprocessing 3 | 4 | max_requests = 1000 5 | max_requests_jitter = 50 6 | 7 | log_file = "-" 8 | 9 | bind = "0.0.0.0:8080" 10 | 11 | worker_class = "uvicorn.workers.UvicornWorker" 12 | workers = (multiprocessing.cpu_count() * 2) + 1 13 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | Thank you for contributing! 2 | Before submitting this PR, please make sure: 3 | 4 | - [ ] Your code builds clean without any errors or warnings 5 | - [ ] Your code doesn't break anything we can't fix 6 | - [ ] You have added appropriate tests 7 | 8 | Please check one or more of the following to describe the nature of this PR: 9 | - [ ] New feature 10 | - [ ] Bug fix 11 | - [ ] Documentation 12 | - [ ] Other 13 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: enhancement 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the solution you'd like** 11 | A clear and concise description of what you want to happen. 12 | 13 | **Describe alternatives you've considered** 14 | A clear and concise description of any alternative solutions or features you've considered. 15 | 16 | **Additional context** 17 | Add any other context or screenshots about the feature request here. 18 | -------------------------------------------------------------------------------- /bumpver.toml: -------------------------------------------------------------------------------- 1 | [bumpver] 2 | current_version = "3.6.5" 3 | version_pattern = "MAJOR.MINOR.PATCH" 4 | commit_message = "bump version {old_version} -> {new_version}" 5 | tag_message = "{new_version}" 6 | tag_scope = "default" 7 | pre_commit_hook = "" 8 | post_commit_hook = "" 9 | commit = true 10 | tag = true 11 | push = true 12 | 13 | [bumpver.file_patterns] 14 | "bumpver.toml" = [ 15 | 'current_version = "{version}"', 16 | ] 17 | "restgdf_api/app.py" = ['__version__ = "{version}"'] 18 | "kubernetes/resources.yaml" = [' image: joshuasundance/restgdf_api:{version}'] 19 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.11-slim-bookworm 2 | 3 | RUN adduser --uid 1000 --disabled-password --gecos '' appuser 4 | USER 1000 5 | 6 | ENV PYTHONDONTWRITEBYTECODE=1 \ 7 | PYTHONUNBUFFERED=1 \ 8 | PATH="/home/appuser/.local/bin:$PATH" 9 | 10 | RUN pip install --user --no-cache-dir --upgrade pip 11 | COPY ./requirements.txt /home/appuser/requirements.txt 12 | RUN pip install --user --no-cache-dir --upgrade -r /home/appuser/requirements.txt 13 | 14 | COPY ./restgdf_api /home/appuser/restgdf_api 15 | WORKDIR /home/appuser/restgdf_api 16 | 17 | EXPOSE 8080 18 | 19 | CMD ["gunicorn", "app:app"] 20 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: "pip" # See documentation for possible values 9 | directory: "/" # Location of package manifests 10 | schedule: 11 | interval: "weekly" 12 | groups: 13 | app: 14 | patterns: 15 | - "*" 16 | -------------------------------------------------------------------------------- /Dockerfile-lab: -------------------------------------------------------------------------------- 1 | FROM python:3.11-slim-bookworm 2 | 3 | RUN adduser --uid 1000 --disabled-password --gecos '' appuser 4 | USER 1000 5 | 6 | ENV PYTHONDONTWRITEBYTECODE=1 \ 7 | PYTHONUNBUFFERED=1 \ 8 | PATH="/home/appuser/.local/bin:$PATH" 9 | 10 | RUN pip install --user --no-cache-dir --upgrade pip 11 | COPY ./requirements.txt /home/appuser/requirements.txt 12 | RUN pip install --user --no-cache-dir --upgrade -r /home/appuser/requirements.txt 13 | 14 | RUN pip install --user --no-cache-dir --upgrade ipywidgets jupyterlab 15 | 16 | EXPOSE 8081 17 | 18 | CMD ["jupyter", "lab", "--ip", "0.0.0.0", "--port", "8081", "--no-browser"] 19 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.8' 2 | 3 | services: 4 | restgdf_api: 5 | build: 6 | context: . 7 | dockerfile: Dockerfile 8 | image: restgdf_api 9 | container_name: restgdf_api 10 | env_file: 11 | - .env 12 | ports: 13 | - "${APP_PORT:-8080}:8080" 14 | working_dir: /home/appuser/restgdf_api 15 | command: ["gunicorn", "app:app"] 16 | lab: 17 | build: 18 | context: . 19 | dockerfile: Dockerfile-lab 20 | image: restgdf_api_lab 21 | container_name: restgdf_api_lab 22 | env_file: 23 | - .env 24 | ports: 25 | - "${JUPYTER_PORT:-8081}:8081" 26 | volumes: 27 | - ./lab:/home/appuser/lab 28 | working_dir: /home/appuser/lab 29 | command: ["jupyter", "lab", "--ip", "0.0.0.0", "--port", "8081", "--no-browser"] 30 | -------------------------------------------------------------------------------- /.github/workflows/docker-hub.yml: -------------------------------------------------------------------------------- 1 | name: Push to Docker Hub 2 | 3 | on: 4 | push: 5 | tags: 6 | - '*.*.*' 7 | 8 | jobs: 9 | build-and-push-docker: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v2 13 | with: 14 | fetch-depth: 0 15 | token: ${{ secrets.WORKFLOW_GIT_ACCESS_TOKEN }} 16 | 17 | - name: Log in to Docker Hub 18 | uses: docker/login-action@v1 19 | with: 20 | username: joshuasundance 21 | password: ${{ secrets.DOCKERHUB_TOKEN }} 22 | 23 | - name: Build Docker image 24 | run: | 25 | docker build \ 26 | -t joshuasundance/restgdf_api:${{ github.ref_name }} \ 27 | -t joshuasundance/restgdf_api:latest \ 28 | . 29 | 30 | - name: Push to Docker Hub 31 | run: docker push -a joshuasundance/restgdf_api 32 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: bug 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | **Desktop (please complete the following information):** 27 | - OS: [e.g. iOS] 28 | - Browser [e.g. chrome, safari] 29 | - Version [e.g. 22] 30 | 31 | **Smartphone (please complete the following information):** 32 | - Device: [e.g. iPhone6] 33 | - OS: [e.g. iOS8.1] 34 | - Browser [e.g. stock browser, safari] 35 | - Version [e.g. 22] 36 | 37 | **Additional context** 38 | Add any other context about the problem here. 39 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Joshua Sundance Bailey 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 10 | -------------------------------------------------------------------------------- /restgdf_api/app.py: -------------------------------------------------------------------------------- 1 | from fastapi import FastAPI 2 | 3 | from directory import directory_router 4 | from layer import layer_router 5 | from mappingsupport import mappingsupport_router 6 | 7 | __version__ = "3.6.5" 8 | 9 | app = FastAPI( 10 | title="restgdf_api", 11 | description="A REST API for interacting with ArcGIS FeatureLayers, powered by restgdf.", 12 | version=__version__, 13 | ) 14 | 15 | app.include_router(mappingsupport_router) 16 | app.include_router(directory_router) 17 | app.include_router(layer_router) 18 | 19 | 20 | # @app.put("/server", tags=["server"], summary="Add server") 21 | # async def add_server( 22 | # url: str, 23 | # name: str, 24 | # default_token: Optional[str] = None, 25 | # session: ClientSession = Depends(get_session), 26 | # ): 27 | # try: 28 | # router = await make_clone( 29 | # session, 30 | # url, 31 | # default_token, 32 | # f"/{name}", 33 | # [name], 34 | # ) 35 | # app.include_router(router) 36 | # return {"result": "success"} 37 | # except Exception as e: 38 | # return {"result": "failure", "error": str(e)} 39 | -------------------------------------------------------------------------------- /restgdf_api/layer.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from typing import Optional 3 | 4 | from aiohttp import ClientSession 5 | from fastapi import Depends, APIRouter 6 | 7 | from models import ( 8 | GeoDataFrameResponse, 9 | MultiGeoDataFrameRequest, 10 | MultiGeoDataFrameResponse, 11 | ) 12 | from utils import fetch_gdf, get_session 13 | 14 | layer_router = APIRouter(prefix="/layer", tags=["layer"]) 15 | 16 | 17 | @layer_router.get("/", response_model=GeoDataFrameResponse) 18 | async def layer( 19 | url: str, 20 | token: Optional[str] = None, 21 | where: str = "1=1", 22 | session: ClientSession = Depends(get_session), 23 | ): 24 | """Retrieve FeatureLayer.""" 25 | return await fetch_gdf(url, session, token, where) 26 | 27 | 28 | @layer_router.post("/multiple/", response_model=MultiGeoDataFrameResponse) 29 | async def layers( 30 | request: MultiGeoDataFrameRequest, 31 | session: ClientSession = Depends(get_session), 32 | ): 33 | """Retrieve multiple FeatureLayers.""" 34 | tasks = [fetch_gdf(url, session) for url in request.urls] 35 | results = await asyncio.gather(*tasks) 36 | return MultiGeoDataFrameResponse(gdfs=dict(zip(request.urls, results))) 37 | -------------------------------------------------------------------------------- /.github/workflows/bumpver.yml: -------------------------------------------------------------------------------- 1 | name: Bump Version 2 | 3 | on: 4 | workflow_dispatch: 5 | inputs: 6 | bump: 7 | type: choice 8 | description: 'Bump major, minor, or patch version' 9 | required: true 10 | default: 'patch' 11 | options: 12 | - 'major' 13 | - 'minor' 14 | - 'patch' 15 | 16 | jobs: 17 | bump-version: 18 | runs-on: ubuntu-latest 19 | permissions: 20 | contents: write 21 | steps: 22 | - name: Checkout code 23 | uses: actions/checkout@v4 24 | with: 25 | token: ${{ secrets.WORKFLOW_GIT_ACCESS_TOKEN }} 26 | fetch-depth: 0 27 | - name: Set up Python 28 | uses: actions/setup-python@v4 29 | with: 30 | python-version: 3.11 31 | cache: pip 32 | - name: Install Python libraries 33 | run: | 34 | pip install --user bumpver 35 | - name: git config 36 | run: | 37 | git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" 38 | git config --local user.name "github-actions[bot]" 39 | - name: Bump version 40 | run: bumpver update --commit --tag-commit --${{ github.event.inputs.bump }} --push 41 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *$py.class 2 | *.chainlit 3 | *.chroma 4 | *.cover 5 | *.egg 6 | *.egg-info/ 7 | *.env 8 | *.langchain.db 9 | *.log 10 | *.manifest 11 | *.mo 12 | *.pot 13 | *.py,cover 14 | *.py[cod] 15 | *.sage.py 16 | *.so 17 | *.spec 18 | .DS_STORE 19 | .Python 20 | .cache 21 | .coverage 22 | .coverage.* 23 | .dmypy.json 24 | .eggs/ 25 | .env 26 | .hypothesis/ 27 | .idea 28 | .installed.cfg 29 | .ipynb_checkpoints 30 | .mypy_cache/ 31 | .nox/ 32 | .pyre/ 33 | .pytest_cache/ 34 | .python-version 35 | .ropeproject 36 | .ruff_cache/ 37 | .scrapy 38 | .spyderproject 39 | .spyproject 40 | .tox/ 41 | .venv 42 | .vscode 43 | .webassets-cache 44 | /site 45 | ENV/ 46 | MANIFEST 47 | __pycache__ 48 | __pycache__/ 49 | __pypackages__/ 50 | build/ 51 | celerybeat-schedule 52 | celerybeat.pid 53 | coverage.xml 54 | credentials.json 55 | data/ 56 | db.sqlite3 57 | db.sqlite3-journal 58 | develop-eggs/ 59 | dist/ 60 | dmypy.json 61 | docs/_build/ 62 | downloads/ 63 | eggs/ 64 | env.bak/ 65 | env/ 66 | fly.toml 67 | htmlcov/ 68 | instance/ 69 | ipython_config.py 70 | junk/ 71 | lib/ 72 | lib64/ 73 | local_settings.py 74 | models/*.bin 75 | nosetests.xml 76 | lab/scratch/ 77 | lab/ 78 | parts/ 79 | pip-delete-this-directory.txt 80 | pip-log.txt 81 | pip-wheel-metadata/ 82 | profile_default/ 83 | sdist/ 84 | share/python-wheels/ 85 | storage 86 | target/ 87 | token.json 88 | var/ 89 | venv 90 | venv.bak/ 91 | venv/ 92 | wheels/ 93 | -------------------------------------------------------------------------------- /kubernetes/resources.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: restgdf_api-deployment 5 | spec: 6 | replicas: 1 7 | selector: 8 | matchLabels: 9 | app: restgdf_api 10 | template: 11 | metadata: 12 | labels: 13 | app: restgdf_api 14 | spec: 15 | containers: 16 | - name: restgdf_api 17 | image: joshuasundance/restgdf_api:3.6.5 18 | imagePullPolicy: Always 19 | resources: 20 | requests: 21 | cpu: "100m" 22 | memory: "200Mi" 23 | limits: 24 | cpu: "500m" 25 | memory: "500Mi" 26 | securityContext: 27 | runAsNonRoot: true 28 | --- 29 | apiVersion: v1 30 | kind: Service 31 | metadata: 32 | name: restgdf_api-service 33 | # configure on Azure and uncomment below to use a vnet 34 | # annotations: 35 | # service.beta.kubernetes.io/azure-load-balancer-internal: "true" 36 | # service.beta.kubernetes.io/azure-load-balancer-ipv4: vnet.ip.goes.here 37 | # service.beta.kubernetes.io/azure-dns-label-name: "restgdf_api" 38 | spec: 39 | selector: 40 | app: restgdf_api 41 | ports: 42 | - protocol: TCP 43 | port: 80 44 | targetPort: 8080 45 | type: LoadBalancer 46 | --- 47 | apiVersion: networking.k8s.io/v1 48 | kind: NetworkPolicy 49 | metadata: 50 | name: restgdf_api-network-policy 51 | spec: 52 | podSelector: 53 | matchLabels: 54 | app: restgdf_api 55 | policyTypes: 56 | - Ingress 57 | ingress: 58 | - from: [] # An empty array here means it will allow traffic from all sources. 59 | ports: 60 | - protocol: TCP 61 | port: 8080 62 | -------------------------------------------------------------------------------- /restgdf_api/models.py: -------------------------------------------------------------------------------- 1 | from typing import Optional, Any 2 | 3 | from pydantic import BaseModel, Field 4 | 5 | 6 | class GeoDataFrameResponse(BaseModel): 7 | metadata: Optional[dict[str, Any]] = Field(None, description="Layer metadata") 8 | data: Optional[str] = Field(None, description="gpd.GeoDataFrame.to_json") 9 | error: Optional[str] = Field(None, description="Error message") 10 | 11 | 12 | class MultiGeoDataFrameRequest(BaseModel): 13 | urls: list[str] = Field( 14 | ..., 15 | example=["https://maps1.vcgov.org/arcgis/rest/services/VC_Public/MapServer/0"], 16 | ) 17 | token: Optional[str] = Field( 18 | None, 19 | description="Token to send to the server with requests.", 20 | ) 21 | 22 | 23 | class MultiGeoDataFrameResponse(BaseModel): 24 | gdfs: Optional[dict[str, GeoDataFrameResponse]] = Field( 25 | None, 26 | description="List of GeoDataFrameResponses", 27 | ) 28 | error: Optional[str] = Field(None, description="Error message") 29 | 30 | 31 | class LayersResponse(BaseModel): 32 | layers: Optional[list] = Field( 33 | None, 34 | description="Layer JSON", 35 | ) 36 | error: Optional[str] = Field(None, description="Error message") 37 | 38 | 39 | class MultiLayersRequest(BaseModel): 40 | urls: list[str] = Field( 41 | ..., 42 | example=[ 43 | "https://maps1.vcgov.org/arcgis/rest/services", 44 | "https://ocgis4.ocfl.net/arcgis/rest/services", 45 | ], 46 | ) 47 | token: Optional[str] = Field( 48 | None, 49 | description="Token to send to the server with requests.", 50 | ) 51 | 52 | 53 | class MultiLayersResponse(BaseModel): 54 | layers: dict[str, LayersResponse] = Field( 55 | ..., 56 | description="List of LayersResponses", 57 | ) 58 | 59 | 60 | class UniqueValuesResponse(BaseModel): 61 | values: list = Field(..., description="List of unique values") 62 | -------------------------------------------------------------------------------- /restgdf_api/directory.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from typing import Optional 3 | 4 | from aiohttp import ClientSession 5 | from fastapi import Depends, APIRouter 6 | 7 | from models import ( 8 | LayersResponse, 9 | MultiLayersRequest, 10 | MultiLayersResponse, 11 | ) 12 | from utils import ( 13 | get_session, 14 | layers_from_directory, 15 | feature_layers_from_directory, 16 | rasters_from_directory, 17 | ) 18 | 19 | directory_router = APIRouter(prefix="/directory", tags=["directory"]) 20 | 21 | 22 | @directory_router.get("/", response_model=LayersResponse) 23 | async def directory( 24 | url: str, 25 | token: Optional[str] = None, 26 | session: ClientSession = Depends(get_session), 27 | ): 28 | """Discover content in an ArcGIS Services Directory.""" 29 | return await layers_from_directory(url, session, token) 30 | 31 | 32 | @directory_router.get("/featurelayers/", response_model=LayersResponse) 33 | async def featurelayers( 34 | url: str, 35 | token: Optional[str] = None, 36 | session: ClientSession = Depends(get_session), 37 | ): 38 | """Discover feature layers in an ArcGIS Services Directory.""" 39 | return await feature_layers_from_directory(url, session, token) 40 | 41 | 42 | @directory_router.get("/rasters/", response_model=LayersResponse) 43 | async def rasters( 44 | url: str, 45 | token: Optional[str] = None, 46 | session: ClientSession = Depends(get_session), 47 | ): 48 | """Discover rasters in an ArcGIS Services Directory.""" 49 | return await rasters_from_directory(url, session, token) 50 | 51 | 52 | @directory_router.post("/multiple/", response_model=MultiLayersResponse) 53 | async def discoveries( 54 | request: MultiLayersRequest, 55 | session: ClientSession = Depends(get_session), 56 | ): 57 | """Discover content in multiple ArcGIS Services Directories.""" 58 | tasks = [layers_from_directory(url, session) for url in request.urls] 59 | results = await asyncio.gather(*tasks) 60 | return MultiLayersResponse(layers=dict(zip(request.urls, results))) 61 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | # Don't know what this file is? See https://pre-commit.com/ 2 | # pip install pre-commit 3 | # pre-commit install 4 | # pre-commit autoupdate 5 | # Apply to all files without commiting: 6 | # pre-commit run --all-files 7 | # I recommend running this until you pass all checks, and then commit. 8 | # Fix what you need to and then let the pre-commit hooks resolve their conflicts. 9 | # You may need to git add -u between runs. 10 | exclude: "AI_CHANGELOG.md" 11 | repos: 12 | - repo: https://github.com/charliermarsh/ruff-pre-commit 13 | rev: "v0.1.5" 14 | hooks: 15 | - id: ruff 16 | args: [--fix, --exit-non-zero-on-fix, --ignore, E501] 17 | - repo: https://github.com/koalaman/shellcheck-precommit 18 | rev: v0.9.0 19 | hooks: 20 | - id: shellcheck 21 | - repo: https://github.com/pre-commit/pre-commit-hooks 22 | rev: v4.5.0 23 | hooks: 24 | - id: check-ast 25 | - id: check-builtin-literals 26 | - id: check-merge-conflict 27 | - id: check-symlinks 28 | - id: check-toml 29 | - id: check-xml 30 | - id: debug-statements 31 | - id: check-case-conflict 32 | - id: check-docstring-first 33 | - id: check-executables-have-shebangs 34 | - id: check-json 35 | # - id: check-yaml 36 | - id: debug-statements 37 | - id: fix-byte-order-marker 38 | - id: detect-private-key 39 | - id: end-of-file-fixer 40 | - id: trailing-whitespace 41 | - id: mixed-line-ending 42 | - id: requirements-txt-fixer 43 | - repo: https://github.com/pre-commit/mirrors-mypy 44 | rev: v1.7.0 45 | hooks: 46 | - id: mypy 47 | - repo: https://github.com/asottile/add-trailing-comma 48 | rev: v3.1.0 49 | hooks: 50 | - id: add-trailing-comma 51 | #- repo: https://github.com/dannysepler/rm_unneeded_f_str 52 | # rev: v0.2.0 53 | # hooks: 54 | # - id: rm-unneeded-f-str 55 | - repo: https://github.com/psf/black 56 | rev: 23.11.0 57 | hooks: 58 | - id: black 59 | - repo: https://github.com/PyCQA/bandit 60 | rev: 1.7.5 61 | hooks: 62 | - id: bandit 63 | args: ["-x", "tests/*.py"] 64 | -------------------------------------------------------------------------------- /restgdf_api/mappingsupport.py: -------------------------------------------------------------------------------- 1 | import time 2 | from typing import Union 3 | 4 | import numpy as np 5 | import pandas as pd 6 | from fastapi import Depends, APIRouter 7 | from pydantic import BaseModel, Field 8 | from pydantic.functional_validators import AfterValidator 9 | from typing_extensions import Annotated 10 | 11 | url = "https://mappingsupport.com/p/surf_gis/list-federal-state-county-city-GIS-servers.csv" 12 | names = [ 13 | "Line-number", 14 | "Type", 15 | "State", 16 | "County", 17 | "Town", 18 | "FIPS", 19 | "Server-owner", 20 | "ArcGIS-url", 21 | "https", 22 | "Show-contents", 23 | "SSL", 24 | "Open", 25 | "Comment", 26 | "Unnamed: 13", 27 | "Unnamed: 14", 28 | ] 29 | 30 | data_cache = { 31 | "last_download_time": 0.0, 32 | "data": pd.DataFrame(), 33 | } 34 | 35 | 36 | def prep_str(s: Union[str, None]) -> Union[str, None]: 37 | return s.lower().strip().replace(" ", "_") if s else s 38 | 39 | 40 | def get_df() -> pd.DataFrame: 41 | return ( 42 | pd.read_csv( 43 | url, 44 | encoding="cp1252", 45 | skip_blank_lines=True, 46 | names=names, 47 | ) 48 | .replace([np.inf, -np.inf, np.nan], None) 49 | .assign( 50 | **{ 51 | c: (lambda df, c=c: df[c].apply(prep_str)) 52 | for c in ["State", "County", "Town"] 53 | }, 54 | ) 55 | ) 56 | 57 | 58 | def get_df_depends() -> pd.DataFrame: 59 | global data_cache 60 | 61 | # Get the current time 62 | current_time = time.time() 63 | 64 | # Check if it's been more than a week (604800 seconds) 65 | if ( 66 | current_time - data_cache["last_download_time"] > 604800 67 | or len(data_cache["data"]) == 0 68 | ): 69 | # Download the data and update the time 70 | data_cache["data"] = get_df() 71 | data_cache["last_download_time"] = current_time 72 | 73 | return data_cache["data"] 74 | 75 | 76 | mappingsupport_router = APIRouter( 77 | prefix="/mappingsupport", 78 | tags=["mappingsupport"], 79 | on_startup=[get_df_depends], 80 | ) 81 | 82 | 83 | PreppedStr = Annotated[str, AfterValidator(prep_str)] 84 | 85 | 86 | class StateRequest(BaseModel): 87 | state_name: PreppedStr = Field( 88 | ..., 89 | example="Florida", 90 | description="Full name of the state", 91 | ) 92 | 93 | 94 | class CountyRequest(StateRequest): 95 | county_name: PreppedStr = Field( 96 | ..., 97 | example="Volusia", 98 | description="Full name of the county", 99 | ) 100 | 101 | 102 | class TownRequest(StateRequest): 103 | town_name: PreppedStr = Field( 104 | ..., 105 | example="Daytona Beach Shores", 106 | description="Full name of the town", 107 | ) 108 | 109 | 110 | @mappingsupport_router.get( 111 | "/", 112 | description="This endpoint uses data from https://mappingsupport.com/p/surf_gis/list-federal-state-county-city-GIS-servers.csv", 113 | ) 114 | async def mappingsupport(df: pd.DataFrame = Depends(get_df_depends)): 115 | """Return all data as json.""" 116 | return df.to_dict(orient="records") 117 | 118 | 119 | @mappingsupport_router.post( 120 | "/state/", 121 | description="This endpoint uses data from https://mappingsupport.com/p/surf_gis/list-federal-state-county-city-GIS-servers.csv", 122 | ) 123 | async def state( 124 | request: StateRequest, 125 | df: pd.DataFrame = Depends(get_df_depends), 126 | ): 127 | """Return data for a state as json.""" 128 | return df.loc[df["State"] == request.state_name].to_dict(orient="records") 129 | 130 | 131 | @mappingsupport_router.post( 132 | "/county/", 133 | description="This endpoint uses data from https://mappingsupport.com/p/surf_gis/list-federal-state-county-city-GIS-servers.csv", 134 | ) 135 | async def county( 136 | request: CountyRequest, 137 | df: pd.DataFrame = Depends(get_df_depends), 138 | ): 139 | """Return data for a county as json.""" 140 | m1 = df["State"] == request.state_name 141 | m2 = df["County"] == request.county_name 142 | return df.loc[m1 & m2].to_dict(orient="records") 143 | 144 | 145 | @mappingsupport_router.post( 146 | "/town/", 147 | description="This endpoint uses data from https://mappingsupport.com/p/surf_gis/list-federal-state-county-city-GIS-servers.csv", 148 | ) 149 | async def town( 150 | request: TownRequest, 151 | df: pd.DataFrame = Depends(get_df_depends), 152 | ): 153 | """Return data for a town as json.""" 154 | m1 = df["State"] == request.state_name 155 | m2 = df["Town"] == request.town_name 156 | return df.loc[m1 & m2].to_dict(orient="records") 157 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # restgdf_api 2 | 3 | [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) 4 | [![python](https://img.shields.io/badge/Python-3.11-3776AB.svg?style=flat&logo=python&logoColor=white)](https://www.python.org) 5 | 6 | [![Push to Docker Hub](https://github.com/joshuasundance-swca/restgdf_api/actions/workflows/docker-hub.yml/badge.svg)](https://github.com/joshuasundance-swca/restgdf_api/actions/workflows/docker-hub.yml) 7 | [![restgdf_api on Docker Hub](https://img.shields.io/docker/v/joshuasundance/restgdf_api?label=restgdf_api&logo=docker)](https://hub.docker.com/r/joshuasundance/restgdf_api) 8 | [![Docker Image Size (tag)](https://img.shields.io/docker/image-size/joshuasundance/restgdf_api/latest)](https://hub.docker.com/r/joshuasundance/restgdf_api) 9 | 10 | ![Code Climate maintainability](https://img.shields.io/codeclimate/maintainability/joshuasundance-swca/restgdf_api) 11 | ![Code Climate issues](https://img.shields.io/codeclimate/issues/joshuasundance-swca/restgdf_api) 12 | ![Code Climate technical debt](https://img.shields.io/codeclimate/tech-debt/joshuasundance-swca/restgdf_api) 13 | 14 | [![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white)](https://github.com/pre-commit/pre-commit) 15 | [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/charliermarsh/ruff/main/assets/badge/v1.json)](https://github.com/charliermarsh/ruff) 16 | [![Checked with mypy](http://www.mypy-lang.org/static/mypy_badge.svg)](http://mypy-lang.org/) 17 | [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) 18 | 19 | [![security: bandit](https://img.shields.io/badge/security-bandit-yellow.svg)](https://github.com/PyCQA/bandit) 20 | ![Known Vulnerabilities](https://snyk.io/test/github/joshuasundance-swca/restgdf_api/badge.svg) 21 | 22 | 23 | 🤖 This `README` was written by GPT-4. 🤖 24 | 25 | ## Overview 26 | `restgdf_api` is an asynchronous server powered by [FastAPI](https://github.com/tiangolo/fastapi) and the open-source [restgdf](https://github.com/joshuasundance-swca/restgdf) library. It acts as an efficient, user-friendly proxy for ArcGIS servers, offering high-speed interactions with ArcGIS FeatureLayers. With comprehensive OpenAPI documentation, this API simplifies and accelerates the process of accessing and managing GIS data, making it an ideal alternative to the traditional ArcGIS API for Python. 27 | 28 | ## Key Features 29 | - **Asynchronous Operations**: Optimized for speed and efficiency, leveraging the power of restgdf for non-blocking data operations. 30 | - **Open-Source and Community-Focused**: Contributions are welcome, fostering a collaborative environment for continuous improvement. 31 | - **Comprehensive OpenAPI Documentation**: Detailed and interactive documentation for easy integration and usage. 32 | - **Versatile GIS Data Management**: Enables discovery, fetching, and manipulation of GIS data across various ArcGIS services. 33 | 34 | ## Use Cases 35 | - **Data Discovery**: Explore content in ArcGIS Services Directory, including feature layers, rasters, and more. 36 | - **Data Retrieval**: Fetch individual or multiple FeatureLayers, with flexible querying capabilities. 37 | - **Administrative Data Access**: Access GIS data for states, counties, cities, and towns using data from [Mapping Support](https://mappingsupport.com/). 38 | - **Proxy for ArcGIS Servers**: Can serve as a documented proxy for any ArcGIS server, providing a streamlined interface for external data sources. 39 | 40 | ## System Requirements 41 | - Python 3.11 or higher 42 | - Docker (optional, for containerized deployment) 43 | 44 | ## Installation 45 | 46 | ### Option 1: Docker Hub Deployment 47 | ```bash 48 | docker run --name restgdf_api -p 8080:8080 joshuasundance/restgdf_api:latest 49 | ``` 50 | 51 | ### Option 2: Clone the Repository 52 | ```bash 53 | git clone https://github.com/joshuasundance-swca/restgdf_api.git 54 | cd restgdf_api 55 | docker compose up 56 | ``` 57 | 58 | ### Option 3: Deploy on Kubernetes 59 | ```bash 60 | git clone https://github.com/joshuasundance-swca/restgdf_api.git 61 | cd restgdf_api 62 | kubectl apply -f kubernetes/resources.yaml 63 | ``` 64 | 65 | ### Option 4: Manual Local Installation 66 | ```bash 67 | git clone https://github.com/joshuasundance-swca/restgdf_api.git 68 | cd restgdf_api 69 | pip install -r requirements.txt 70 | cd restgdf_api 71 | gunicorn app:app 72 | ``` 73 | 74 | ## API Endpoints Overview 75 | - `mappingsupport`: Leverages data from [Mapping Support](https://mappingsupport.com/) to provide GIS server listings and specific geographic area data. 76 | - `directory`: Discover content in ArcGIS directories, including feature layers and raster data. 77 | - `layer`: Retrieve specific FeatureLayers, supporting single and multiple queries. 78 | - `server`: Add and configure ArcGIS servers, enhancing flexibility and control. 79 | 80 | 81 | ## Contributing 82 | Your contributions are welcome! To contribute, please follow the standard fork and pull request workflow. 83 | 84 | ## License 85 | This project is licensed under the MIT License. 86 | -------------------------------------------------------------------------------- /restgdf_api/utils.py: -------------------------------------------------------------------------------- 1 | """Utility functions for restgdf_api.""" 2 | 3 | from typing import Optional 4 | 5 | from aiohttp import ClientSession 6 | from fastapi import APIRouter, Depends 7 | from restgdf import Directory, FeatureLayer 8 | 9 | from models import GeoDataFrameResponse, LayersResponse 10 | 11 | 12 | async def get_session(): 13 | """Return an aiohttp ClientSession.""" 14 | async with ClientSession() as session: 15 | yield session 16 | 17 | 18 | us_state_to_abbrev = { 19 | "Alabama": "AL", 20 | "Alaska": "AK", 21 | "Arizona": "AZ", 22 | "Arkansas": "AR", 23 | "California": "CA", 24 | "Colorado": "CO", 25 | "Connecticut": "CT", 26 | "Delaware": "DE", 27 | "Florida": "FL", 28 | "Georgia": "GA", 29 | "Hawaii": "HI", 30 | "Idaho": "ID", 31 | "Illinois": "IL", 32 | "Indiana": "IN", 33 | "Iowa": "IA", 34 | "Kansas": "KS", 35 | "Kentucky": "KY", 36 | "Louisiana": "LA", 37 | "Maine": "ME", 38 | "Maryland": "MD", 39 | "Massachusetts": "MA", 40 | "Michigan": "MI", 41 | "Minnesota": "MN", 42 | "Mississippi": "MS", 43 | "Missouri": "MO", 44 | "Montana": "MT", 45 | "Nebraska": "NE", 46 | "Nevada": "NV", 47 | "New Hampshire": "NH", 48 | "New Jersey": "NJ", 49 | "New Mexico": "NM", 50 | "New York": "NY", 51 | "North Carolina": "NC", 52 | "North Dakota": "ND", 53 | "Ohio": "OH", 54 | "Oklahoma": "OK", 55 | "Oregon": "OR", 56 | "Pennsylvania": "PA", 57 | "Rhode Island": "RI", 58 | "South Carolina": "SC", 59 | "South Dakota": "SD", 60 | "Tennessee": "TN", 61 | "Texas": "TX", 62 | "Utah": "UT", 63 | "Vermont": "VT", 64 | "Virginia": "VA", 65 | "Washington": "WA", 66 | "West Virginia": "WV", 67 | "Wisconsin": "WI", 68 | "Wyoming": "WY", 69 | "District of Columbia": "DC", 70 | "American Samoa": "AS", 71 | "Guam": "GU", 72 | "Northern Mariana Islands": "MP", 73 | "Puerto Rico": "PR", 74 | "United States Minor Outlying Islands": "UM", 75 | "U.S. Virgin Islands": "VI", 76 | } 77 | 78 | state_dict = { 79 | k.replace(" ", "").replace(".", "").upper(): v 80 | for k, v in us_state_to_abbrev.items() 81 | } 82 | 83 | abbrev_dict = {v: k for k, v in state_dict.items()} 84 | 85 | 86 | def state_to_abbrev(state: str) -> str: 87 | """Convert a state name to its abbreviation.""" 88 | return state_dict.get(state.replace(" ", "").replace(".", "").upper(), "NA") 89 | 90 | 91 | def abbrev_to_state(abbrev: str) -> str: 92 | """Convert a state abbreviation to its name.""" 93 | return us_state_to_abbrev.get( 94 | abbrev.replace(".", "").replace(" ", "").upper(), 95 | "NA", 96 | ) 97 | 98 | 99 | async def get_directory( 100 | url: str, 101 | session: ClientSession, 102 | token: Optional[str] = None, 103 | ) -> Directory: 104 | rest_obj = await Directory.from_url( 105 | url, 106 | session=session, 107 | token=token, 108 | ) 109 | _ = await rest_obj.crawl() 110 | return rest_obj 111 | 112 | 113 | async def layers_from_directory( 114 | url: str, 115 | session: ClientSession, 116 | token: Optional[str] = None, 117 | ): 118 | """Discover content in an ArcGIS Services Directory.""" 119 | try: 120 | rest_obj = await get_directory(url, session, token) 121 | return dict(layers=rest_obj.services) 122 | except Exception as e: 123 | return dict(error=str(e)) 124 | 125 | 126 | async def feature_layers_from_directory( 127 | url: str, 128 | session: ClientSession, 129 | token: Optional[str] = None, 130 | ) -> dict: 131 | """Discover content in an ArcGIS Services Directory.""" 132 | try: 133 | rest_obj = await get_directory(url, session, token) 134 | return dict(layers=rest_obj.feature_layers()) 135 | except Exception as e: 136 | return dict(error=str(e)) 137 | 138 | 139 | async def rasters_from_directory( 140 | url: str, 141 | session: ClientSession, 142 | token: Optional[str] = None, 143 | ): 144 | """Discover content in an ArcGIS Services Directory.""" 145 | try: 146 | rest_obj = await get_directory(url, session, token) 147 | return dict(layers=rest_obj.rasters()) 148 | except Exception as e: 149 | return dict(error=str(e)) 150 | 151 | 152 | def relative_path(remote_url: str, root_url: str) -> str: 153 | """Return a relative path from a remote url and a root url.""" 154 | _path = remote_url.strip("/ ").replace(root_url.strip("/ "), "").strip("/ ") 155 | return f"/{_path}/" 156 | 157 | 158 | async def fetch_gdf( 159 | url: str, 160 | session: ClientSession, 161 | token: Optional[str] = None, 162 | where: str = "1=1", 163 | **kwargs, 164 | ) -> GeoDataFrameResponse: 165 | try: 166 | rest_obj = await FeatureLayer.from_url( 167 | url, 168 | token=token, 169 | session=session, 170 | where=where, 171 | **kwargs, 172 | ) 173 | gdf = await rest_obj.getgdf() 174 | return GeoDataFrameResponse( 175 | metadata=rest_obj.metadata, 176 | data=gdf.to_json(), # Convert to JSON 177 | ) 178 | except Exception as e: 179 | return GeoDataFrameResponse(error=str(e)) 180 | 181 | 182 | async def make_clone( 183 | session: ClientSession, 184 | cloned_url: str, 185 | default_token: Optional[str] = None, 186 | prefix: str = "/clone", 187 | tags: list[str] = ["clone"], 188 | **kwargs, 189 | ) -> APIRouter: 190 | router = APIRouter(prefix=prefix, tags=tags, **kwargs) 191 | 192 | @router.get("/", response_model=LayersResponse) 193 | async def directory( 194 | token: Optional[str] = None, 195 | session: ClientSession = Depends(get_session), 196 | ): 197 | """Discover content in an ArcGIS Services Directory.""" 198 | return await layers_from_directory(cloned_url, session, token) 199 | 200 | @router.get("/featurelayers/", response_model=LayersResponse) 201 | async def featurelayers( 202 | token: Optional[str] = None, 203 | session: ClientSession = Depends(get_session), 204 | ): 205 | """Discover feature layers in an ArcGIS Services Directory.""" 206 | return await feature_layers_from_directory(cloned_url, session, token) 207 | 208 | @router.get("/rasters/", response_model=LayersResponse) 209 | async def rasters( 210 | token: Optional[str] = None, 211 | session: ClientSession = Depends(get_session), 212 | ): 213 | """Discover rasters in an ArcGIS Services Directory.""" 214 | return await rasters_from_directory(cloned_url, session, token) 215 | 216 | @router.get( 217 | "/{path:path}/", 218 | response_model=GeoDataFrameResponse, 219 | ) 220 | async def layer( 221 | path: str, 222 | token: Optional[str] = None, 223 | where: str = "1=1", 224 | session: ClientSession = Depends(get_session), 225 | ): 226 | """Retrieve FeatureLayer.""" 227 | return await fetch_gdf( 228 | f"{cloned_url.strip('/ ')}/{path.strip('/ ')}", 229 | session, 230 | token or default_token, 231 | where, 232 | ) 233 | 234 | def create_layer_endpoint(rel_path: str): 235 | """Create a closure that captures the relative path.""" 236 | 237 | async def endpoint_function( 238 | token: Optional[str] = None, 239 | where: str = "1=1", 240 | session: ClientSession = Depends(get_session), 241 | ): 242 | """Endpoint function using captured relative path.""" 243 | full_url = f"{cloned_url.strip('/ ')}/{rel_path.strip('/ ')}" 244 | return await fetch_gdf( 245 | full_url, 246 | session, 247 | token or default_token, 248 | where, 249 | ) 250 | 251 | return endpoint_function 252 | 253 | feature_layers = await feature_layers_from_directory( 254 | cloned_url, 255 | session, 256 | default_token, 257 | ) 258 | if "error" in feature_layers: 259 | raise ValueError(feature_layers["error"]) 260 | for _layer in feature_layers["layers"]: 261 | layer_name = _layer["metadata"]["name"] 262 | rel_path = relative_path(_layer["url"], root_url=cloned_url) 263 | layer_endpoint = create_layer_endpoint(rel_path) 264 | router.add_api_route( 265 | path=rel_path, 266 | endpoint=layer_endpoint, 267 | response_model=GeoDataFrameResponse, 268 | summary=layer_name, 269 | ) 270 | 271 | return router 272 | --------------------------------------------------------------------------------