├── web ├── __init__.py ├── update_tasks.py ├── v1_init_tables.sql ├── config.py ├── update_tasks.sql ├── crs.py ├── templates │ ├── list.html │ └── browse.html ├── dashed_draw.py ├── models.py ├── wms.py ├── main.py └── db.py ├── .gitignore ├── setup.cfg ├── requirements.txt ├── README.md ├── run.sh ├── proxy_worker.py └── LICENSE /web/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | __pycache__/ 3 | .venv/ 4 | config_local.py 5 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 99 3 | 4 | [mypy] 5 | ignore_missing_imports = true 6 | check_untyped_defs = true 7 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | fastapi 2 | uvicorn[standard] 3 | gunicorn 4 | psycopg[binary,pool] 5 | Pillow 6 | jinja2 7 | aiohttp 8 | authlib 9 | itsdangerous 10 | httpx 11 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # GeoScribble 2 | 3 | A simple server that receives and returns lines and labels inside a bounding box. 4 | No authentication, no antispam, use at your own risk. Produces GeoJSON and WMS output. 5 | 6 | # Author and License 7 | 8 | Written by Ilya Zverev, published under ISC License. 9 | -------------------------------------------------------------------------------- /web/update_tasks.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import logging 3 | from .db import init_database, update_tasks 4 | 5 | 6 | async def update(): 7 | await init_database() 8 | await update_tasks() 9 | 10 | 11 | if __name__ == '__main__': 12 | logging.basicConfig( 13 | level=logging.INFO, format='[%(levelname)s] %(message)s') 14 | asyncio.run(update()) 15 | -------------------------------------------------------------------------------- /run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e -u 3 | export PGDATABASE="${PGDATABASE:-}" 4 | export PGUSER="${PGUSER:-$(whoami)}" 5 | export PGHOST=localhost 6 | 7 | HERE="$(dirname "$0")" 8 | VENV="${VENV:-$HERE/.venv}" 9 | 10 | # Creating a virtual environment if it doesn't exist 11 | if [ ! -d "$VENV" ]; then 12 | python3 -m venv "$VENV" 13 | "$VENV"/bin/pip install -r "$HERE/web/requirements.txt" 14 | fi 15 | 16 | "$VENV/bin/uvicorn" --host 0.0.0.0 --port 8000 web.main:app 17 | -------------------------------------------------------------------------------- /proxy_worker.py: -------------------------------------------------------------------------------- 1 | from uvicorn.workers import UvicornWorker 2 | 3 | 4 | class ProxyUvicornWorker(UvicornWorker): 5 | CONFIG_KWARGS = { 6 | "loop": "auto", # Use 'auto' to automatically choose the best loop, 'uvloop' can be specified for performance. 7 | "http": "auto", # Use 'auto' to automatically choose the best HTTP protocol support, 'httptools' can be specified. 8 | "lifespan": "on", # 'on' to enable lifespan support. 9 | "proxy_headers": True, # Corresponding to `--proxy-headers` 10 | "forwarded_allow_ips": "*", # Corresponding to `--forwarded-allow-ips=*` 11 | } 12 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | ISC License 2 | 3 | Copyright 2023 Ilya Zverev 4 | 5 | Permission to use, copy, modify, and/or distribute this software 6 | for any purpose with or without fee is hereby granted, provided 7 | that the above copyright notice and this permission notice appear in all copies. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 10 | WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY 11 | AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, 12 | DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER 13 | RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, 14 | NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH 15 | THE USE OR PERFORMANCE OF THIS SOFTWARE. 16 | -------------------------------------------------------------------------------- /web/v1_init_tables.sql: -------------------------------------------------------------------------------- 1 | create table if not exists scribbles ( 2 | scribble_id serial primary key, 3 | geom geometry not null, 4 | username text not null, 5 | user_id integer not null, 6 | editor text not null, 7 | created timestamp with time zone not null default now(), 8 | 9 | style text, 10 | color text, 11 | thin boolean not null default true, 12 | dashed boolean not null default false, 13 | label text, 14 | 15 | deleted timestamp with time zone, 16 | deleted_by_id integer 17 | ); 18 | 19 | create index if not exists scribbles_idx_geom on scribbles using gist (geom); 20 | create index if not exists scribbles_idx_created on scribbles (created); 21 | create index if not exists scribbles_idx_user_id on scribbles (user_id); 22 | 23 | create table if not exists tasks ( 24 | task_id serial primary key, 25 | location geometry not null, 26 | location_str text, 27 | scribbles integer not null, 28 | username text not null, 29 | user_id integer not null, 30 | created timestamp with time zone not null, 31 | processed timestamp with time zone, 32 | processed_by_id integer 33 | ); 34 | 35 | create index if not exists tasks_idx_location on tasks (location); 36 | create index if not exists tasks_idx_created on tasks (created); 37 | create index if not exists tasks_idx_user_id on tasks (user_id); 38 | -------------------------------------------------------------------------------- /web/config.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | BASE_URL = os.getenv('BASE_URL', '') 4 | 5 | # Set to localhost or a socket path. 6 | PG_HOST = os.getenv('PGHOST', '/var/run/postgresql') 7 | 8 | # Might want to use a proper user. 9 | PG_USER = os.getenv('PGUSER', 'postgres') 10 | 11 | # These are pretty much standard. 12 | PG_PORT = os.getenv('PGPORT', '5432') 13 | PG_DATABASE = os.getenv('PGDATABASE', '') 14 | 15 | # Geoscribble-specific numbers 16 | MAX_POINTS = int(os.getenv('MAX_POINTS', '100')) 17 | MAX_LENGTH = int(os.getenv('MAX_LENGTH', '5000')) # in meters 18 | DEFAULT_AGE = int(os.getenv('DEFAULT_AGE', '91')) # three months 19 | 20 | # For bbox requests 21 | MAX_COORD_SPAN = float(os.getenv('MAX_COORD_SPAN', '0.3')) 22 | MAX_IMAGE_WIDTH = int(os.getenv('MAX_IMAGE_WIDTH', '3000')) 23 | MAX_IMAGE_HEIGHT = int(os.getenv('MAX_IMAGE_HEIGHT', '2000')) 24 | 25 | # Fonts for labels 26 | FONT = os.getenv('FONT', '/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf') 27 | 28 | # Email for geocoder 29 | EMAIL = os.getenv('EMAIL', 'geoscribble@example.com') 30 | MAX_GEOCODE = int(os.getenv('MAX_GEOCODE', '20')) 31 | 32 | # For OpenStreetMap authentication 33 | SECRET_KEY = os.getenv('SECRET_KEY', 'whatever') 34 | OAUTH_ID = os.getenv('OAUTH_ID', '') 35 | OAUTH_SECRET = os.getenv('OAUTH_SECRET', '') 36 | 37 | try: 38 | from .config_local import * 39 | except ImportError: 40 | pass 41 | -------------------------------------------------------------------------------- /web/update_tasks.sql: -------------------------------------------------------------------------------- 1 | with start_date_table as ( 2 | -- Get the starting date for new clusters. 3 | select coalesce(date_trunc('day', max(created)) - interval '1 day', timestamp '2024-01-01 00:00Z') as start_date from tasks 4 | ) 5 | 6 | , clusters as ( 7 | -- Clusterize scribbles. 8 | select ST_ClusterDBSCAN(geom, 0.02, 1) over (partition by user_id, date_trunc('day', created)) as cid, 9 | username, user_id, created, geom 10 | from scribbles, start_date_table 11 | where deleted is null and created >= start_date 12 | ) 13 | 14 | , new_tasks as ( 15 | -- Group clusters into tasks. 16 | select count(*) as cnt, max(username) username, user_id, max(created) created, ST_GeometricMedian(ST_Collect(ST_Centroid(geom))) loc 17 | from clusters group by cid, user_id, date_trunc('day', created) 18 | 19 | ), existing as ( 20 | -- Find new tasks to merge with existing tasks. 21 | select n.*, t.task_id from new_tasks n left join tasks t on n.user_id = t.user_id and date_trunc('day', n.created) = date_trunc('day', t.created) and ST_DWithin(n.loc, t.location, 0.02) 22 | 23 | ), inserting as ( 24 | -- Insert new tasks. 25 | insert into tasks (location, scribbles, username, user_id, created) 26 | select loc, cnt, username, user_id, created 27 | from existing where task_id is null 28 | order by created 29 | 30 | ), updating as ( 31 | -- Update old tasks. 32 | update tasks 33 | set location = e.loc, created = e.created, scribbles = e.cnt 34 | from existing e where tasks.task_id = e.task_id 35 | ) 36 | 37 | -- Return some debug information. 38 | select * from existing, start_date_table 39 | order by created; 40 | -------------------------------------------------------------------------------- /web/crs.py: -------------------------------------------------------------------------------- 1 | from math import radians, degrees, cos, tan, log, pi, asin, tanh 2 | 3 | 4 | __all__ = ['BaseCRS', 'CRS_LIST', 'BBox'] 5 | 6 | 7 | EARTH_RADIUS = 6378137 8 | 9 | 10 | class BaseCRS: 11 | def coords_to_pixel(self, lon: float, lat: float) -> tuple[float, float]: 12 | """Always returns (x, y).""" 13 | return lon, lat 14 | 15 | def pixel_to_coords(self, x: float, y: float) -> tuple[float, float]: 16 | """Returns either (lon, lat) or (lat, lon) depending on CRS.""" 17 | return x, y 18 | 19 | @property 20 | def flip(self): 21 | return False 22 | 23 | 24 | class CRS_4326(BaseCRS): 25 | pass 26 | 27 | 28 | class CRS_3857(BaseCRS): 29 | def coords_to_pixel(self, lon: float, lat: float) -> tuple[float, float]: 30 | rlat = radians(lat) 31 | x = radians(lon) 32 | if lat > 85: 33 | y = pi * 2 34 | elif lat < -85: 35 | y = -pi * 2 36 | else: 37 | y = log(tan(rlat) + (1/cos(rlat))) 38 | return x * EARTH_RADIUS, y * EARTH_RADIUS 39 | 40 | def pixel_to_coords(self, x: float, y: float) -> tuple[float, float]: 41 | fx = x / EARTH_RADIUS 42 | fy = asin(tanh(y / EARTH_RADIUS)) 43 | return degrees(fx), degrees(fy) 44 | 45 | @property 46 | def flip(self): 47 | return True 48 | 49 | 50 | CRS_LIST: dict[str, BaseCRS] = { 51 | 'EPSG:4326': CRS_4326(), 52 | 'EPSG:3857': CRS_3857(), 53 | } 54 | 55 | 56 | class BBox: 57 | def __init__(self, crs: BaseCRS, proj_bbox: list[float]): 58 | self.crs = crs 59 | self.x = proj_bbox[0] 60 | self.y = proj_bbox[1] 61 | self.w = proj_bbox[2] - proj_bbox[0] 62 | self.h = proj_bbox[3] - proj_bbox[1] 63 | 64 | def to_pixel(self, lonlat: tuple[float, float]) -> tuple[float, float]: 65 | proj = self.crs.coords_to_pixel(lonlat[0], lonlat[1]) 66 | x, y = (proj[0] - self.x) / self.w, (proj[1] - self.y) / self.h 67 | if self.crs.flip: 68 | y = 1 - y 69 | return x, y 70 | 71 | def to_4326(self) -> list[float]: 72 | return [ 73 | *self.crs.pixel_to_coords(self.x, self.y), 74 | *self.crs.pixel_to_coords(self.x + self.w, self.y + self.h), 75 | ] 76 | -------------------------------------------------------------------------------- /web/templates/list.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | GeoScribble edits 5 | 6 | 7 | 8 | 9 |

GeoScribble edits

10 | 11 | {% if not username %} 12 |
13 | 14 |
15 | {% else %} 16 |
17 | You are {{ username }}. 18 | {% if nofilter %}Show mine. 19 | {% else %}Show all.{% endif %} 20 | 21 |
22 | {% endif %} 23 | 24 | 25 | {% for edit in edits %} 26 | 27 | 28 | 29 | 30 | 31 | 37 | {% if username %} 38 | 48 | {% endif %} 49 | 50 | 51 | {% endfor %} 52 |
{{ '+' if edit.processed else '' }}{{ edit.username }}{{ edit.created | format_date }}({{ edit.scribbles }}) 32 | map 33 | josm 34 | id 35 | rapid 36 | 39 | {% if not edit.processed %} 40 |
41 | 42 | 43 |
44 | {% else %} 45 | Not done yet 46 | {% endif %} 47 | 		{{ edit.location_str or '' }}
53 | 54 | 55 | -------------------------------------------------------------------------------- /web/templates/browse.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | GeoScribbles 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 |
14 | 57 | 58 | 59 | -------------------------------------------------------------------------------- /web/dashed_draw.py: -------------------------------------------------------------------------------- 1 | import math 2 | from PIL import ImageDraw 3 | 4 | 5 | # Copied from https://stackoverflow.com/a/65893631/1297601 6 | class DashedImageDraw(ImageDraw.ImageDraw): 7 | 8 | def thick_line(self, xy, direction, fill=None, width=0): 9 | # xy – Sequence of 2-tuples like [(x, y), (x, y), ...] 10 | # direction – Sequence of 2-tuples like [(x, y), (x, y), ...] 11 | if xy[0] != xy[1]: 12 | self.line(xy, fill=fill, width=width) 13 | else: 14 | x1, y1 = xy[0] 15 | dx1, dy1 = direction[0] 16 | dx2, dy2 = direction[1] 17 | if dy2 - dy1 < 0: 18 | x1 -= 1 19 | if dx2 - dx1 < 0: 20 | y1 -= 1 21 | if dy2 - dy1 != 0: 22 | if dx2 - dx1 != 0: 23 | k = - (dx2 - dx1)/(dy2 - dy1) 24 | a = 1 / math.sqrt(1 + k**2) 25 | b = (width*a - 1) / 2 26 | else: 27 | k = 0 28 | b = (width - 1)/2 29 | x3 = x1 - math.floor(b) 30 | y3 = y1 - int(k*b) 31 | x4 = x1 + math.ceil(b) 32 | y4 = y1 + int(k*b) 33 | else: 34 | x3 = x1 35 | y3 = y1 - math.floor((width - 1)/2) 36 | x4 = x1 37 | y4 = y1 + math.ceil((width - 1)/2) 38 | self.line([(x3, y3), (x4, y4)], fill=fill, width=1) 39 | return 40 | 41 | def dashed_line(self, xy, dash=(2, 2), fill=None, width=0): 42 | # xy – Sequence of 2-tuples like [(x, y), (x, y), ...] 43 | for i in range(len(xy) - 1): 44 | x1, y1 = xy[i] 45 | x2, y2 = xy[i + 1] 46 | x_length = x2 - x1 47 | y_length = y2 - y1 48 | length = math.sqrt(x_length**2 + y_length**2) 49 | dash_enabled = True 50 | postion = 0 51 | while length > 0 and postion <= length: 52 | for dash_step in dash: 53 | if postion > length: 54 | break 55 | if dash_enabled: 56 | start = postion/length 57 | end = min((postion + dash_step - 1) / length, 1) 58 | self.thick_line([(round(x1 + start*x_length), 59 | round(y1 + start*y_length)), 60 | (round(x1 + end*x_length), 61 | round(y1 + end*y_length))], 62 | xy, fill, width) 63 | dash_enabled = not dash_enabled 64 | postion += dash_step 65 | return 66 | -------------------------------------------------------------------------------- /web/models.py: -------------------------------------------------------------------------------- 1 | from math import radians, cos, sqrt 2 | from typing import Annotated, Optional 3 | from annotated_types import Len 4 | from pydantic import BaseModel, Field, PastDatetime 5 | from pydantic.functional_validators import AfterValidator 6 | from . import config 7 | 8 | 9 | class Identification(BaseModel): 10 | username: Annotated[str, Field(description='OSM user name')] 11 | user_id: Annotated[int, Field( 12 | gt=0, description='OSM user id', examples=[1234])] 13 | editor: Annotated[str, Field( 14 | description='Editor that uploaded the element', examples=['Every Door'])] 15 | 16 | 17 | def distance(p1: tuple[float, float], p2: tuple[float, float]) -> float: 18 | l1 = radians(p1[1]) 19 | l2 = radians(p2[1]) 20 | f1 = radians(p1[0]) 21 | f2 = radians(p2[0]) 22 | x = (l2 - l1) * cos((f1 + f2) / 2) 23 | y = f2 - f1 24 | return sqrt(x * x + y * y) * 6371000 25 | 26 | 27 | def validate_length(points: list[tuple[float, float]]): 28 | length = 0.0 29 | for i in range(1, len(points)): 30 | length += distance(points[i-1], points[i]) 31 | assert length <= config.MAX_LENGTH, ( 32 | f'Length of a scribble should be under {config.MAX_LENGTH} meters') 33 | return points 34 | 35 | 36 | class NewScribble(Identification): 37 | style: Annotated[str, Field(examples=['track'])] 38 | color: Annotated[Optional[str], Field( 39 | pattern='^[0-9a-fA-F]{6}$', description='Color in hex format RRGGBB')] = None 40 | dashed: bool = False 41 | thin: bool = True 42 | points: Annotated[list[tuple[float, float]], 43 | Len(min_length=2, max_length=config.MAX_POINTS), 44 | AfterValidator(validate_length), 45 | Field( 46 | description='Points as (lon, lat) for a line', 47 | examples=[[[10.1, 55.2], [10.2, 55.1]]]) 48 | ] 49 | 50 | 51 | class NewLabel(Identification): 52 | location: Annotated[tuple[float, float], Field(examples=[[10.1, 55.2]])] 53 | text: Annotated[str, Field( 54 | min_length=1, max_length=40, examples=['fence'])] 55 | color: Annotated[Optional[str], Field( 56 | pattern='^[0-9a-fA-F]{6}$', description='Color in hex format RRGGBB')] = None 57 | 58 | 59 | class Deletion(Identification): 60 | id: Annotated[int, Field( 61 | description='Unique id of the scribble', examples=[45, 91])] 62 | deleted: Annotated[bool, Field( 63 | description='Should be true for deleting an element')] 64 | 65 | 66 | class Scribble(NewScribble): 67 | id: Annotated[int, Field( 68 | description='Unique id of the scribble', examples=[45, 46])] 69 | created: PastDatetime 70 | 71 | 72 | class Label(NewLabel): 73 | id: Annotated[int, Field( 74 | description='Unique id of the scribble', examples=[46])] 75 | created: PastDatetime 76 | 77 | 78 | class Box(BaseModel): 79 | box: Annotated[list[float], Field(description='Bounding box')] 80 | minage: int 81 | 82 | 83 | class Task(BaseModel): 84 | id: Annotated[int, Field( 85 | description='Unique id of the task', examples=[21])] 86 | location: Annotated[tuple[float, float], Field(examples=[[10.1, 55.2]])] 87 | location_str: Annotated[Optional[str], Field( 88 | description='Country and city of the task')] 89 | scribbles: Annotated[int, Field( 90 | description='Count of scribbles in this task', examples=[1])] 91 | username: Annotated[str, Field(description='OSM user name')] 92 | user_id: Annotated[int, Field( 93 | gt=0, description='OSM user id', examples=[1234])] 94 | created: PastDatetime 95 | processed: Optional[PastDatetime] 96 | processed_by_id: Annotated[Optional[int], Field( 97 | gt=0, description='OSM user who closed the task', examples=[1234])] 98 | 99 | 100 | class Feature(BaseModel): 101 | f_type: Annotated[str, Field("Feature", serialization_alias='type')] 102 | geometry: dict 103 | properties: dict 104 | 105 | 106 | class FeatureCollection(BaseModel): 107 | features: list[Feature] 108 | -------------------------------------------------------------------------------- /web/wms.py: -------------------------------------------------------------------------------- 1 | from fastapi import HTTPException 2 | from io import BytesIO 3 | from . import config 4 | from .crs import CRS_LIST, BBox 5 | from .models import Scribble, Label, Box 6 | from .db import query 7 | from .dashed_draw import DashedImageDraw 8 | from PIL import Image, ImageDraw, ImageFont 9 | from typing import Union 10 | 11 | 12 | def get_capabilities(endpoint: str) -> str: 13 | srs = '\n'.join([f'{k}' for k in CRS_LIST.keys()]) 14 | xml = """ 15 | 16 | 17 | WMS 18 | GeoScribbles 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | application/vnd.ogc.wms_xml 27 | 28 | 29 | 31 | 32 | 33 | 34 | 35 | image/png 36 | 37 | 38 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | application/vnd.ogc.se_xml 47 | application/vnd.ogc.se_inimage 48 | application/vnd.ogc.se_blank 49 | 50 | 51 | 52 | 53 | 54 | GeoScribbles 55 | {srs} 56 | 57 | 59 | 60 | scribbles 61 | Scribbles and labels 62 | 63 | 64 | latest 65 | Scribbles less than a month old 66 | 67 | 68 | 69 | 70 | """.format(url=str(endpoint).rstrip('/'), srs=srs) 71 | return xml 72 | 73 | 74 | async def get_map(params: dict[str, str]) -> bytes: 75 | # Fist get CRS because everything depends on it. 76 | crs = CRS_LIST.get(params.get('crs', params.get('srs', '')).upper()) 77 | if not crs: 78 | raise HTTPException(422, 'Unsupported CRS') 79 | 80 | # Now parse width and height and check they're inside max values. 81 | try: 82 | width = int(params['width']) 83 | height = int(params['height']) 84 | except ValueError: 85 | raise HTTPException(422, 'Width and height should be integer numbers') 86 | if width < 100 or width > config.MAX_IMAGE_WIDTH: 87 | raise HTTPException(422, f'Max width is {config.MAX_IMAGE_WIDTH}') 88 | if height < 100 or height > config.MAX_IMAGE_HEIGHT: 89 | raise HTTPException(422, f'Max height is {config.MAX_IMAGE_HEIGHT}') 90 | 91 | # Bounding box - reproject to 4326. 92 | try: 93 | bbox = [float(p) for p in params['bbox'].split(',')] 94 | except ValueError: 95 | bbox = [] 96 | if len(bbox) != 4: 97 | raise HTTPException(422, 'Expecting 4 numbers for bbox') 98 | 99 | bbox_obj = BBox(crs, bbox) 100 | maxage = 21 if 'latest' in params.get('layers', '') else None 101 | 102 | try: 103 | user_id: int | None = int(params.get('user_id', '')) 104 | except ValueError: 105 | user_id = None 106 | 107 | scribbles = await query( 108 | bbox_obj.to_4326(), maxage=maxage, 109 | user_id=user_id, username=params.get('username'), 110 | ) 111 | out = Image.new('RGBA', (width, height)) 112 | render_image(out, bbox_obj, scribbles) 113 | content = BytesIO() 114 | out.save(content, 'PNG') 115 | return content.getvalue() 116 | 117 | 118 | def render_image(image: Image, bbox: BBox, scribbles: list[Union[Scribble, Label]]): 119 | age_colors = { 120 | 3: '#ffffb2', 121 | 7: '#fed976', 122 | 14: '#feb24c', 123 | 30: '#fd8d3c', 124 | 61: '#f03b20', 125 | 1000: '#bd0026', 126 | } 127 | draw = DashedImageDraw(image) 128 | for s in scribbles: 129 | if isinstance(s, Scribble): 130 | coords = [bbox.to_pixel(c) for c in s.points] 131 | coords = [(round(c[0] * image.width), round(c[1] * image.height)) for c in coords] 132 | width = 3 if s.thin else 5 133 | if s.dashed: 134 | draw.dashed_line(coords, (10, 10), fill=f'#{s.color}', width=width) 135 | else: 136 | draw.line(coords, fill=f'#{s.color}', width=width) 137 | elif isinstance(s, Box): 138 | x1, y1 = bbox.to_pixel((s.box[0], s.box[1])) 139 | x2, y2 = bbox.to_pixel((s.box[2], s.box[3])) 140 | xy = [min(x1, x2), min(y1, y2), max(x1, x2), max(y1, y2)] 141 | color = '#ffffff' # not used 142 | for k in sorted(age_colors.keys()): 143 | if s.minage <= k or k == 1000: 144 | color = age_colors[k] 145 | break 146 | draw.rectangle(xy, fill=color, width=0) 147 | for s in scribbles: 148 | # Drawing labels always after geometries 149 | if isinstance(s, Label): 150 | coord = bbox.to_pixel(s.location) 151 | coord = (round(coord[0] * image.width), round(coord[1] * image.height)) 152 | r = 3 153 | elcoord = [ 154 | (coord[0] - r, coord[1] - r), 155 | (coord[0] + r, coord[1] + r), 156 | ] 157 | draw.ellipse(elcoord, outline='#000000', fill='#e0ffe0', width=1) 158 | try: 159 | font = ImageFont.truetype(config.FONT, size=14) 160 | except OSError: 161 | font = ImageFont.load_default() 162 | # Draw semi-transparent background 163 | expand = 3 164 | torig = [coord[0] + expand, coord[1] - expand] 165 | tbox = font.getbbox(s.text) 166 | tbounds = [ 167 | tbox[0] + torig[0] - expand, -tbox[3] + torig[1] - expand, 168 | tbox[2] + torig[0] + expand, tbox[1] + torig[1] + expand, 169 | ] 170 | draw.rounded_rectangle(tbounds, 5, fill='#00000050') 171 | draw.text(torig, s.text, fill='#ffffff', font=font, anchor='lb') 172 | -------------------------------------------------------------------------------- /web/main.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | from typing import Annotated, Union, Optional 4 | from fastapi import FastAPI, Query, HTTPException, Response, Request 5 | from fastapi.middleware.cors import CORSMiddleware 6 | from fastapi.responses import HTMLResponse, RedirectResponse 7 | from fastapi.templating import Jinja2Templates 8 | from starlette.middleware.sessions import SessionMiddleware 9 | from authlib.integrations.starlette_client import OAuth 10 | from authlib.common.errors import AuthlibBaseError 11 | from xml.etree import ElementTree as etree 12 | from . import config 13 | from .models import ( 14 | NewScribble, NewLabel, Deletion, Scribble, Label, 15 | FeatureCollection, Feature, Box, Task, 16 | ) 17 | from .db import ( 18 | init_database, query, get_cursor, 19 | insert_scribble, insert_label, delete_scribble, 20 | list_tasks, mark_processed, 21 | ) 22 | from .wms import get_map, get_capabilities 23 | 24 | 25 | logging.basicConfig(level=logging.INFO, format='[%(levelname)s] %(message)s') 26 | app = FastAPI() 27 | app.add_middleware(CORSMiddleware, allow_origins=['*']) 28 | app.add_middleware(SessionMiddleware, secret_key=config.SECRET_KEY, max_age=3600*24*365) 29 | templates = Jinja2Templates(directory=os.path.join(os.path.dirname(__file__), 'templates')) 30 | 31 | oauth = OAuth() 32 | oauth.register( 33 | 'openstreetmap', 34 | api_base_url='https://api.openstreetmap.org/api/0.6/', 35 | access_token_url='https://www.openstreetmap.org/oauth2/token', 36 | authorize_url='https://www.openstreetmap.org/oauth2/authorize', 37 | client_id=config.OAUTH_ID, 38 | client_secret=config.OAUTH_SECRET, 39 | client_kwargs={'scope': 'read_prefs'}, 40 | ) 41 | 42 | 43 | @app.on_event('startup') 44 | async def startup(): 45 | await init_database() 46 | 47 | 48 | @app.get('/map', response_class=HTMLResponse, include_in_schema=False) 49 | async def root(request: Request): 50 | return templates.TemplateResponse(request=request, name='browse.html') 51 | 52 | 53 | def format_date(d) -> str: 54 | return d.strftime('%d %b %H:%M') 55 | 56 | 57 | templates.env.filters['format_date'] = format_date 58 | 59 | 60 | @app.get('/', response_class=HTMLResponse, include_in_schema=False) 61 | async def list_edits(request: Request): 62 | user_id = request.session.get('user_id') 63 | nofilter = bool(request.session.get('nofilter')) 64 | edits = await list_tasks(user_id=None if nofilter else user_id) 65 | 66 | context = { 67 | 'edits': edits, 68 | 'username': request.session.get('username'), 69 | 'nofilter': nofilter, 70 | } 71 | return templates.TemplateResponse( 72 | request=request, name='list.html', context=context) 73 | 74 | 75 | @app.get('/task_process', include_in_schema=False) 76 | async def process_task(task_id: int, request: Request): 77 | user_id = request.session.get('user_id') 78 | if user_id: 79 | await mark_processed(task_id, user_id) 80 | return RedirectResponse(request.url_for('list_edits')) 81 | 82 | 83 | @app.get('/task_unprocess', include_in_schema=False) 84 | async def unprocess_task(task_id: int, request: Request): 85 | user_id = request.session.get('user_id') 86 | if user_id: 87 | await mark_processed(task_id, None) 88 | return RedirectResponse(request.url_for('list_edits')) 89 | 90 | 91 | @app.get('/toggle_filter', include_in_schema=False) 92 | async def toggle_filter(request: Request): 93 | request.session['nofilter'] = not request.session.get('nofilter') 94 | return RedirectResponse(request.url_for('list_edits')) 95 | 96 | 97 | @app.get("/login", include_in_schema=False) 98 | async def login_via_osm(request: Request): 99 | redirect_uri = request.url_for('auth_via_osm') 100 | return await oauth.openstreetmap.authorize_redirect(request, redirect_uri) 101 | 102 | 103 | @app.get("/auth", include_in_schema=False) 104 | async def auth_via_osm(request: Request): 105 | try: 106 | token = await oauth.openstreetmap.authorize_access_token(request) 107 | except AuthlibBaseError: 108 | return HTMLResponse('Denied. Go back.') 109 | 110 | response = await oauth.openstreetmap.get('user/details', token=token) 111 | user_details = etree.fromstring(response.content) 112 | request.session['username'] = user_details[0].get('display_name') 113 | request.session['user_id'] = int(user_details[0].get('id') or 1) 114 | 115 | return RedirectResponse(request.url_for('list_edits')) 116 | 117 | 118 | @app.get('/logout', include_in_schema=False) 119 | async def logout(request: Request): 120 | request.session.pop('username') 121 | request.session.pop('user_id') 122 | return RedirectResponse(request.url_for('list_edits'), 302) 123 | 124 | 125 | @app.get('/tasks') 126 | async def tasks( 127 | bbox: Annotated[Optional[str], Query(pattern=r'^-?\d+(?:\.\d+)?(,-?\d+(?:\.\d+)?){3}$')], 128 | username: Optional[str] = None, user_id: Optional[int] = None, 129 | maxage: Optional[int] = None) -> list[Task]: 130 | """List tasks (grouped scribbles) by user and date.""" 131 | box = None if not bbox else [float(part.strip()) for part in bbox.split(',')] 132 | return await list_tasks(box, username, user_id, maxage) 133 | 134 | 135 | @app.get('/scribbles') 136 | async def scribbles( 137 | bbox: Annotated[str, Query(pattern=r'^-?\d+(?:\.\d+)?(,-?\d+(?:\.\d+)?){3}$')], 138 | username: Optional[str] = None, user_id: Optional[int] = None, 139 | maxage: Optional[int] = None) -> list[Union[Scribble, Label, Box]]: 140 | """Return scribbles for a given area, in a raw json format.""" 141 | box = [float(part.strip()) for part in bbox.split(',')] 142 | return await query(box, username, user_id, None, maxage) 143 | 144 | 145 | @app.get('/geojson') 146 | async def geojson( 147 | bbox: Annotated[str, Query(pattern=r'^-?\d+(?:\.\d+)?(,-?\d+(?:\.\d+)?){3}$')], 148 | username: Optional[str] = None, user_id: Optional[str] = None, 149 | maxage: Optional[int] = None) -> FeatureCollection: 150 | """Return scribbles for a given area as GeoJSON.""" 151 | scr = await scribbles(bbox, username, user_id, maxage) 152 | features = [] 153 | for s in scr: 154 | if isinstance(s, Scribble): 155 | features.append(Feature( 156 | f_type='Feature', 157 | geometry={'type': 'LineString', 'coordinates': s.points}, 158 | properties={ 159 | 'type': 'scribble', 160 | 'id': s.id, 161 | 'style': s.style, 162 | 'color': None if not s.color else f'#{s.color}', 163 | 'dashed': s.dashed, 164 | 'thin': s.thin, 165 | 'userName': s.username, 166 | 'userId': s.user_id, 167 | 'editor': s.editor, 168 | 'created': s.created, 169 | } 170 | )) 171 | elif isinstance(s, Label): 172 | features.append(Feature( 173 | f_type='Feature', 174 | geometry={'type': 'Point', 'coordinates': s.location}, 175 | properties={ 176 | 'type': 'label', 177 | 'id': s.id, 178 | 'color': None if not s.color else f'#{s.color}', 179 | 'text': s.text, 180 | 'username': s.username, 181 | 'userId': s.user_id, 182 | 'editor': s.editor, 183 | 'created': s.created, 184 | } 185 | )) 186 | elif isinstance(s, Box): 187 | features.append(Feature( 188 | f_type='Feature', 189 | geometry={ 190 | 'type': 'Polygon', 191 | 'coordinates': [[ 192 | [s.box[0], s.box[1]], 193 | [s.box[2], s.box[1]], 194 | [s.box[2], s.box[3]], 195 | [s.box[0], s.box[3]], 196 | [s.box[0], s.box[1]], 197 | ]], 198 | }, 199 | properties={ 200 | 'type': 'box', 201 | 'minAge': s.minage, 202 | }, 203 | )) 204 | return FeatureCollection(features=features) 205 | 206 | 207 | @app.get('/wms') 208 | async def wms(request: Request): 209 | """WMS endpoint for editors.""" 210 | params = {k.lower(): v for k, v in request.query_params.items()} 211 | if params.get('request') == 'GetCapabilities': 212 | if params.get('service', 'WMS').lower() != 'wms': 213 | raise HTTPException(422, "Please use WMS for service") 214 | base_url = config.BASE_URL or request.scope.get('root_path') or request.base_url 215 | xml = get_capabilities(base_url) # TODO: url behind proxy 216 | return Response(content=xml, media_type='application/xml') 217 | elif params.get('request') == 'GetMap': 218 | if any([k not in params for k in ('format', 'bbox', 'width', 'height', 'layers')]): 219 | raise HTTPException(422, "Missing parameter for GetMap") 220 | if params.get('format') != 'image/png': 221 | raise HTTPException(422, "GetMap supports only PNG images") 222 | data = await get_map(params) 223 | return Response(content=data, media_type='image/png') 224 | else: 225 | raise HTTPException(422, "This server supports only GetCapabilities and GetMap") 226 | 227 | 228 | @app.post('/upload') 229 | async def put_scribbles( 230 | scribbles: list[Union[NewScribble, NewLabel, Deletion]] 231 | ) -> list[Optional[int]]: 232 | """Batch upload scribbles, labels, and deletions.""" 233 | # Check that at least the user is the same 234 | for i in range(1, len(scribbles)): 235 | if (scribbles[i].user_id != scribbles[0].user_id or 236 | scribbles[i].username != scribbles[0].username or 237 | scribbles[i].editor != scribbles[0].editor): 238 | raise HTTPException(401, "User and editor should be the same for all elements") 239 | 240 | new_ids: list[Optional[int]] = [] 241 | async with get_cursor(True) as cur: 242 | for s in scribbles: 243 | if isinstance(s, NewScribble): 244 | new_ids.append(await insert_scribble(cur, s)) 245 | elif isinstance(s, NewLabel): 246 | new_ids.append(await insert_label(cur, s)) 247 | elif isinstance(s, Deletion): 248 | new_ids.append(await delete_scribble(cur, s)) 249 | return new_ids 250 | 251 | 252 | @app.put('/new') 253 | async def put_one_scribble(scribble: Union[NewScribble, NewLabel]) -> int: 254 | """Upload one scribble or label.""" 255 | async with get_cursor(True) as cur: 256 | if isinstance(scribble, NewScribble): 257 | return await insert_scribble(cur, scribble) 258 | elif isinstance(scribble, NewLabel): 259 | return await insert_label(cur, scribble) 260 | raise HTTPException(401) # TODO: proper error 261 | -------------------------------------------------------------------------------- /web/db.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import aiohttp 3 | import os 4 | import json 5 | import logging 6 | from datetime import datetime, timedelta 7 | from contextlib import asynccontextmanager 8 | from psycopg_pool import AsyncConnectionPool 9 | from psycopg.rows import dict_row 10 | from typing import Union, Optional 11 | from . import config 12 | from .models import Scribble, Label, NewLabel, NewScribble, Deletion, Box, Task 13 | 14 | 15 | pool = AsyncConnectionPool( 16 | kwargs={ 17 | 'host': config.PG_HOST, 18 | 'port': config.PG_PORT, 19 | 'dbname': config.PG_DATABASE, 20 | 'user': config.PG_USER, 21 | 'row_factory': dict_row, 22 | }, 23 | open=False, 24 | ) 25 | 26 | 27 | async def check_connections(): 28 | while True: 29 | await asyncio.sleep(600) 30 | await pool.check() 31 | 32 | 33 | @asynccontextmanager 34 | async def get_cursor(commit: bool = False): 35 | async with pool.connection() as conn: 36 | cursor = conn.cursor() 37 | try: 38 | yield cursor 39 | if commit: 40 | await conn.commit() 41 | finally: 42 | await cursor.close() 43 | 44 | 45 | async def create_table(): 46 | async with get_cursor(True) as cur: 47 | await cur.execute( 48 | "select 1 from pg_tables where schemaname='public' and tablename='tasks'") 49 | if not await cur.fetchone(): 50 | # Table is missing, run the script 51 | filename = os.path.join(os.path.dirname(__file__), 'v1_init_tables.sql') 52 | with open(filename, 'r') as f: 53 | sql = f.read() 54 | await cur.execute(sql) 55 | 56 | 57 | async def init_database(): 58 | await pool.open() 59 | asyncio.create_task(check_connections()) 60 | await create_table() 61 | 62 | 63 | def bbox_too_big(box: list[float]) -> bool: 64 | if (abs(box[2] - box[0]) > config.MAX_COORD_SPAN or 65 | abs(box[3] - box[1]) > config.MAX_COORD_SPAN): 66 | return True 67 | return False 68 | 69 | 70 | def geohash_digits(box: list[float]) -> int: 71 | # area in square degrees. 72 | area = abs(box[2] - box[0]) * abs(box[3] - box[1]) 73 | if area > 1000: 74 | return 3 75 | if area > 40: 76 | return 4 77 | return 5 78 | 79 | 80 | async def query(bbox: list[float], username: Optional[str] = None, 81 | user_id: Optional[int] = None, 82 | editor: Optional[str] = None, 83 | maxage: Optional[int] = None 84 | ) -> list[Union[Scribble, Label, Box]]: 85 | age = maxage or config.DEFAULT_AGE 86 | result: list[Union[Scribble, Label]] = [] 87 | async with get_cursor() as cur: 88 | params = [*bbox, timedelta(days=age)] 89 | add_queries = [] 90 | 91 | if username: 92 | add_queries.append('and username = %s') 93 | params.append(username) 94 | if user_id: 95 | add_queries.append('and user_id = %s') 96 | params.append(user_id) 97 | if editor: 98 | add_queries.append('and editor = %s') 99 | params.append(editor) 100 | 101 | overview = bbox_too_big(bbox) 102 | if not overview: 103 | sql = """select *, ST_AsGeoJSON(geom) json from scribbles 104 | where ST_Intersects(geom, ST_MakeEnvelope(%s, %s, %s, %s, 4326)) 105 | and created >= now() - %s and deleted is null 106 | {q}""".format(q=' '.join(add_queries)) 107 | else: 108 | params.insert(0, geohash_digits(bbox)) 109 | sql = """with t as ( 110 | select ST_GeoHash(geom, %s) hash, min(now()-created) age 111 | from scribbles 112 | where geom && ST_MakeEnvelope(%s, %s, %s, %s, 4326) 113 | and created >= now() - %s and deleted is null 114 | {q} group by 1) 115 | select ST_AsGeoJSON(ST_GeomFromGeoHash(hash)) json, 116 | extract(day from age) age from t 117 | """.format(q=' '.join(add_queries)) 118 | 119 | await cur.execute(sql, params) 120 | 121 | async for row in cur: 122 | geom = json.loads(row['json']) 123 | if overview: 124 | c = geom['coordinates'][0] 125 | result.append(Box( 126 | minage=row['age'], 127 | box=[c[0][0], c[0][1], c[2][0], c[2][1]], 128 | )) 129 | elif geom['type'] == 'Point': 130 | result.append(Label( 131 | id=row['scribble_id'], 132 | created=row['created'], 133 | username=row['username'], 134 | user_id=row['user_id'], 135 | editor=row['editor'] or '', 136 | location=(geom['coordinates'][0], 137 | geom['coordinates'][1]), 138 | color=row['color'], 139 | text=row['label'], 140 | )) 141 | else: 142 | result.append(Scribble( 143 | id=row['scribble_id'], 144 | created=row['created'], 145 | username=row['username'], 146 | user_id=row['user_id'], 147 | editor=row['editor'] or '', 148 | style=row['style'], 149 | color=row['color'], 150 | dashed=row['dashed'], 151 | thin=row['thin'], 152 | points=geom['coordinates'], 153 | )) 154 | return result 155 | 156 | 157 | async def insert_scribble(cur, s: NewScribble) -> int: 158 | await cur.execute( 159 | """insert into scribbles 160 | (user_id, username, editor, style, color, thin, dashed, geom) 161 | values (%s, %s, %s, %s, %s, %s, %s, ST_SetSRID(ST_GeomFromGeoJSON(%s), 4326)) 162 | returning scribble_id""", 163 | (s.user_id, s.username, s.editor, s.style, s.color, s.thin, s.dashed, 164 | json.dumps({'type': 'LineString', 'coordinates': s.points}))) 165 | return (await cur.fetchone())['scribble_id'] 166 | 167 | 168 | async def insert_label(cur, s: NewLabel) -> int: 169 | await cur.execute( 170 | """insert into scribbles 171 | (user_id, username, editor, color, label, geom) 172 | values (%s, %s, %s, %s, %s, ST_Point(%s, %s, 4326)) 173 | returning scribble_id""", 174 | (s.user_id, s.username, s.editor, s.color, s.text, *s.location)) 175 | return (await cur.fetchone())['scribble_id'] 176 | 177 | 178 | async def delete_scribble(cur, s: Deletion) -> int: 179 | await cur.execute( 180 | "update scribbles set deleted = now(), deleted_by_id = %s where scribble_id = %s", 181 | (s.user_id, s.id)) 182 | return s.id 183 | 184 | 185 | async def reverse_geocode(lon: float, lat: float) -> Optional[str]: 186 | async with aiohttp.ClientSession() as session: 187 | endpoint = 'https://nominatim.openstreetmap.org/reverse' 188 | params = { 189 | 'format': 'jsonv2', 190 | 'lat': lat, 191 | 'lon': lon, 192 | 'accept-language': 'en', 193 | 'zoom': '12', 194 | 'layer': 'address', 195 | 'email': config.EMAIL, 196 | } 197 | async with session.get(endpoint, params=params) as response: 198 | if response.status == 200: 199 | data = await response.json() 200 | return data.get('display_name') 201 | else: 202 | logging.warn('Could not geocode %s: %s', 203 | response.url, await response.text()) 204 | return None 205 | 206 | 207 | async def update_tasks() -> None: 208 | async with get_cursor(True) as cur: 209 | # Run the script from the file. 210 | filename = os.path.join(os.path.dirname(__file__), 'update_tasks.sql') 211 | with open(filename, 'r') as f: 212 | sql = f.read() 213 | await cur.execute(sql) 214 | 215 | # Get max task_id for reverse geocoding. 216 | await cur.execute("select max(task_id) t from tasks where location_str is not null") 217 | last_geocoded = (await cur.fetchone())['t'] or 0 218 | 219 | # Reverse geocode the new tasks. 220 | await cur.execute( 221 | "select task_id, ST_X(location) lon, ST_Y(location) lat " 222 | "from tasks where task_id > %s and task_id <= %s", 223 | (last_geocoded, last_geocoded + config.MAX_GEOCODE)) 224 | locs: list[tuple[str, int]] = [] 225 | async for row in cur: 226 | loc = await reverse_geocode(row['lon'], row['lat']) 227 | if loc: 228 | locs.append((loc, row['task_id'])) 229 | await asyncio.sleep(1.2) 230 | await cur.executemany("update tasks set location_str = %s where task_id = %s", locs) 231 | 232 | 233 | async def list_tasks(bbox: Optional[list[float]] = None, 234 | username: Optional[str] = None, user_id: Optional[int] = None, 235 | maxage: Optional[int] = None, 236 | since: Optional[datetime] = None, limit: int = 100) -> list[Task]: 237 | age = maxage or config.DEFAULT_AGE 238 | if not since: 239 | since = datetime.now() - timedelta(days=age) 240 | 241 | params: list = [since] 242 | add_queries: list[str] = [] 243 | 244 | if bbox: 245 | add_queries.append('and ST_Intersects(location, ST_MakeEnvelope(%s, %s, %s, %s, 4326))') 246 | params.extend(bbox) 247 | if username: 248 | add_queries.append('and username = %s') 249 | params.append(username) 250 | if user_id: 251 | add_queries.append('and user_id = %s') 252 | params.append(user_id) 253 | 254 | sql = """select *, ST_AsGeoJSON(location) json from tasks 255 | where created >= %s {q} order by created desc limit {limit}""".format( 256 | q=' '.join(add_queries), limit=limit) 257 | 258 | result: list[Task] = [] 259 | async with get_cursor() as cur: 260 | await cur.execute(sql, params) 261 | async for row in cur: 262 | geom = json.loads(row['json']) 263 | result.append(Task( 264 | id=row['task_id'], 265 | location=(geom['coordinates'][0], 266 | geom['coordinates'][1]), 267 | location_str=row['location_str'], 268 | scribbles=row['scribbles'], 269 | username=row['username'], 270 | user_id=row['user_id'], 271 | created=row['created'], 272 | processed=row['processed'], 273 | processed_by_id=row['processed_by_id'], 274 | )) 275 | 276 | return result 277 | 278 | 279 | async def mark_processed(task_id: int, user_id: Optional[int]) -> None: 280 | async with get_cursor(True) as cur: 281 | if user_id: 282 | sql = """update tasks set processed = now(), processed_by_id = %(u)s 283 | where task_id = %(t)s""" 284 | else: 285 | sql = """update tasks set processed = null, processed_by_id = null 286 | where task_id = %(t)s""" 287 | await cur.execute(sql, {'t': task_id, 'u': user_id}) 288 | --------------------------------------------------------------------------------