├── worker ├── dev_assets │ └── .gitkeep ├── local_assets │ └── .gitkeep ├── .prettierrc ├── .editorconfig ├── vitest.config.js ├── package.json ├── public │ └── index.html ├── README.md ├── test │ └── index.spec.js ├── .gitignore ├── wrangler.jsonc ├── scripts │ └── update-db.js └── src │ └── index.js ├── .gitignore ├── data_curation ├── utils │ ├── __init__.py │ └── extraction_utils.py ├── generated_data │ └── .gitkeep ├── wikipedia_data │ └── .gitkeep ├── requirements.txt ├── .gitignore └── README.md ├── docs ├── data_organization.md ├── assets │ ├── geoquash.jpg │ ├── geoquah_tree.png │ ├── completed_tree.jpg │ ├── dates_by_year.png │ ├── geohash_level0.jpg │ ├── geohash_level1.jpg │ ├── geoquash_tree.png │ ├── map_geoquashes.jpg │ ├── event_screenshot.jpeg │ ├── pages_until_year.png │ ├── geoquash_tree_zooms.jpg │ ├── pushing_down_the_tree.jpg │ └── map_geoquashes_completed.jpg ├── README.md ├── economics_of_landnotes.md ├── user_experience.md ├── future_directions.md └── displaying_on_the_map.md ├── website ├── public │ ├── icon.png │ ├── icon-192x192.png │ ├── icon-512x512.png │ ├── icons │ │ ├── README.md │ │ ├── search.svg │ │ ├── flag.svg │ │ ├── expand-vertical.svg │ │ ├── mountain-snow.svg │ │ ├── external-link.svg │ │ ├── expand.svg │ │ ├── book-marked.svg │ │ ├── person-standing.svg │ │ ├── shrink.svg │ │ ├── square-user-round.svg │ │ ├── text-search.svg │ │ ├── newspaper.svg │ │ ├── skull.svg │ │ ├── trees.svg │ │ ├── calendar-fold.svg │ │ ├── pin.svg │ │ ├── briefcase-business.svg │ │ ├── train-front.svg │ │ ├── luggage.svg │ │ ├── landmark.svg │ │ ├── baby.svg │ │ ├── city.svg │ │ ├── plane-takeoff.svg │ │ ├── waves.svg │ │ ├── map.svg │ │ ├── school.svg │ │ ├── church.svg │ │ ├── tree-palm.svg │ │ ├── trophy.svg │ │ ├── building.svg │ │ └── menu.svg │ └── manifest.webmanifest ├── README.md ├── src │ ├── main.js │ ├── vite-env.d.ts │ ├── app.css │ ├── lib │ │ ├── data │ │ │ ├── page_data.svelte.js │ │ │ ├── utils.js │ │ │ ├── events_data.js │ │ │ ├── date_utils.js │ │ │ ├── places_data.js │ │ │ ├── mapEntries.svelte.js │ │ │ └── geohash.js │ │ ├── sliding_pane │ │ │ ├── About.svelte │ │ │ ├── SlidingPaneHeader.svelte │ │ │ ├── SlidingPane.svelte │ │ │ ├── PageEvents.svelte │ │ │ └── SameLocationEvents.svelte │ │ ├── menu │ │ │ ├── DropdownMenu.svelte │ │ │ ├── DatePicker.svelte │ │ │ └── MenuDropdown.svelte │ │ ├── map │ │ │ ├── createMarker.js │ │ │ ├── MarkerIcon.svelte │ │ │ ├── MapPopup.svelte │ │ │ └── WikiPreview.svelte │ │ └── appState.svelte.js │ └── App.svelte ├── svelte.config.js ├── .gitignore ├── package.json ├── vite.config.js ├── jsconfig.json ├── index.html └── notes.md ├── LICENCE.txt └── README.md /worker/dev_assets/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /worker/local_assets/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode 2 | .DS_Store -------------------------------------------------------------------------------- /data_curation/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data_curation/generated_data/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data_curation/wikipedia_data/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/data_organization.md: -------------------------------------------------------------------------------- 1 | # Data Organization in Landnotes 2 | 3 | Section in progress. -------------------------------------------------------------------------------- /data_curation/requirements.txt: -------------------------------------------------------------------------------- 1 | cartopy 2 | wiki_dump_extractor[llm] 3 | sqlalchemy 4 | rapidfuzz -------------------------------------------------------------------------------- /website/public/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zulko/landnotes/HEAD/website/public/icon.png -------------------------------------------------------------------------------- /docs/assets/geoquash.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zulko/landnotes/HEAD/docs/assets/geoquash.jpg -------------------------------------------------------------------------------- /docs/assets/geoquah_tree.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zulko/landnotes/HEAD/docs/assets/geoquah_tree.png -------------------------------------------------------------------------------- /docs/assets/completed_tree.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zulko/landnotes/HEAD/docs/assets/completed_tree.jpg -------------------------------------------------------------------------------- /docs/assets/dates_by_year.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zulko/landnotes/HEAD/docs/assets/dates_by_year.png -------------------------------------------------------------------------------- /docs/assets/geohash_level0.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zulko/landnotes/HEAD/docs/assets/geohash_level0.jpg -------------------------------------------------------------------------------- /docs/assets/geohash_level1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zulko/landnotes/HEAD/docs/assets/geohash_level1.jpg -------------------------------------------------------------------------------- /docs/assets/geoquash_tree.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zulko/landnotes/HEAD/docs/assets/geoquash_tree.png -------------------------------------------------------------------------------- /docs/assets/map_geoquashes.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zulko/landnotes/HEAD/docs/assets/map_geoquashes.jpg -------------------------------------------------------------------------------- /website/public/icon-192x192.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zulko/landnotes/HEAD/website/public/icon-192x192.png -------------------------------------------------------------------------------- /website/public/icon-512x512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zulko/landnotes/HEAD/website/public/icon-512x512.png -------------------------------------------------------------------------------- /docs/assets/event_screenshot.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zulko/landnotes/HEAD/docs/assets/event_screenshot.jpeg -------------------------------------------------------------------------------- /docs/assets/pages_until_year.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zulko/landnotes/HEAD/docs/assets/pages_until_year.png -------------------------------------------------------------------------------- /docs/assets/geoquash_tree_zooms.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zulko/landnotes/HEAD/docs/assets/geoquash_tree_zooms.jpg -------------------------------------------------------------------------------- /worker/.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "printWidth": 140, 3 | "singleQuote": true, 4 | "semi": true, 5 | "useTabs": true 6 | } 7 | -------------------------------------------------------------------------------- /docs/assets/pushing_down_the_tree.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zulko/landnotes/HEAD/docs/assets/pushing_down_the_tree.jpg -------------------------------------------------------------------------------- /docs/assets/map_geoquashes_completed.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Zulko/landnotes/HEAD/docs/assets/map_geoquashes_completed.jpg -------------------------------------------------------------------------------- /website/README.md: -------------------------------------------------------------------------------- 1 | # The Landnotes app 2 | 3 | For local development use: 4 | 5 | ``` 6 | npm install 7 | npm run dev 8 | ``` 9 | 10 | Pushes to master trigger a redeployment of the website. 11 | -------------------------------------------------------------------------------- /website/public/icons/README.md: -------------------------------------------------------------------------------- 1 | # Icons from Lucide 2 | 3 | The icons in this folder come from the open source Lucide icons library and are under the ISC License (see https://lucide.dev/license for license details). -------------------------------------------------------------------------------- /website/src/main.js: -------------------------------------------------------------------------------- 1 | import { mount } from 'svelte' 2 | import './app.css' 3 | import App from './App.svelte' 4 | 5 | const app = mount(App, { 6 | target: document.getElementById('app'), 7 | }) 8 | 9 | export default app 10 | -------------------------------------------------------------------------------- /worker/.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | root = true 3 | 4 | [*] 5 | indent_style = tab 6 | end_of_line = lf 7 | charset = utf-8 8 | trim_trailing_whitespace = true 9 | insert_final_newline = true 10 | 11 | [*.yml] 12 | indent_style = space 13 | -------------------------------------------------------------------------------- /website/svelte.config.js: -------------------------------------------------------------------------------- 1 | import { vitePreprocess } from '@sveltejs/vite-plugin-svelte' 2 | 3 | export default { 4 | // Consult https://svelte.dev/docs#compile-time-svelte-preprocess 5 | // for more information about preprocessors 6 | preprocess: vitePreprocess(), 7 | } 8 | -------------------------------------------------------------------------------- /worker/vitest.config.js: -------------------------------------------------------------------------------- 1 | import { defineWorkersConfig } from '@cloudflare/vitest-pool-workers/config'; 2 | 3 | export default defineWorkersConfig({ 4 | test: { 5 | poolOptions: { 6 | workers: { 7 | wrangler: { configPath: './wrangler.jsonc' }, 8 | }, 9 | }, 10 | }, 11 | }); 12 | -------------------------------------------------------------------------------- /website/src/vite-env.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | /// 3 | 4 | interface ImportMeta { 5 | env: { 6 | BASE_URL: string; 7 | MODE: string; 8 | DEV: boolean; 9 | PROD: boolean; 10 | SSR: boolean; 11 | [key: string]: any; 12 | }; 13 | } -------------------------------------------------------------------------------- /website/public/icons/search.svg: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | -------------------------------------------------------------------------------- /website/public/icons/flag.svg: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | -------------------------------------------------------------------------------- /website/src/app.css: -------------------------------------------------------------------------------- 1 | :root { 2 | font-family: Inter, system-ui, Avenir, Helvetica, Arial, sans-serif; 3 | line-height: 1.5; 4 | font-weight: 400; 5 | } 6 | 7 | body { 8 | margin: 0; 9 | padding: 0; 10 | width: 100%; 11 | height: 100vh; 12 | overflow: hidden; 13 | } 14 | 15 | #app { 16 | width: 100%; 17 | height: 100%; 18 | margin: 0; 19 | padding: 0; 20 | } -------------------------------------------------------------------------------- /website/public/icons/expand-vertical.svg: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /website/.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | pnpm-debug.log* 8 | lerna-debug.log* 9 | 10 | node_modules 11 | dist 12 | dist-ssr 13 | *.local 14 | 15 | # Editor directories and files 16 | .vscode 17 | _geodata 18 | !.vscode/extensions.json 19 | .idea 20 | .DS_Store 21 | *.suo 22 | *.ntvs* 23 | *.njsproj 24 | *.sln 25 | *.sw? 26 | -------------------------------------------------------------------------------- /website/public/icons/mountain-snow.svg: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | -------------------------------------------------------------------------------- /worker/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "landnotes-worker", 3 | "version": "0.0.0", 4 | "private": true, 5 | "scripts": { 6 | "deploy": "wrangler deploy", 7 | "dev": "wrangler dev", 8 | "start": "wrangler dev", 9 | "test": "vitest" 10 | }, 11 | "devDependencies": { 12 | "@cloudflare/vitest-pool-workers": "^0.7.5", 13 | "vitest": "~3.0.7", 14 | "wrangler": "^4.7.2" 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /website/public/icons/external-link.svg: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /website/public/icons/expand.svg: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /website/public/icons/book-marked.svg: -------------------------------------------------------------------------------- 1 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /website/public/icons/person-standing.svg: -------------------------------------------------------------------------------- 1 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /website/public/icons/shrink.svg: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /worker/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Hello, World! 7 | 8 | 9 |

10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /website/public/icons/square-user-round.svg: -------------------------------------------------------------------------------- 1 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /website/public/icons/text-search.svg: -------------------------------------------------------------------------------- 1 | 4 | 5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | # Landnotes docs 2 | 3 | The following sections (still a work in progress) explain how Landnotes works: 4 | 5 | - [Event extraction with scripts and AI](./event_extraction.md) 6 | - [How markers are displayed on the map](./displaying_on_the_map.md) 7 | - [Economics of Landnotes](./economics_of_landnotes.md) 8 | - [Full documentation of the user experience (for developers and coding assistants)](./user_experience.md) 9 | - [Future directions](./future_directions.md) 10 | -------------------------------------------------------------------------------- /website/public/icons/newspaper.svg: -------------------------------------------------------------------------------- 1 | 3 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /website/public/icons/skull.svg: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /website/public/icons/trees.svg: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /website/public/icons/calendar-fold.svg: -------------------------------------------------------------------------------- 1 | 4 | 5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /website/public/icons/pin.svg: -------------------------------------------------------------------------------- 1 | 3 | 4 | 6 | -------------------------------------------------------------------------------- /website/public/icons/briefcase-business.svg: -------------------------------------------------------------------------------- 1 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /website/public/icons/train-front.svg: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /website/public/icons/luggage.svg: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /website/public/icons/landmark.svg: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /website/public/icons/baby.svg: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | 8 | -------------------------------------------------------------------------------- /website/public/icons/city.svg: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /website/public/icons/plane-takeoff.svg: -------------------------------------------------------------------------------- 1 | 3 | 4 | 6 | -------------------------------------------------------------------------------- /website/public/icons/waves.svg: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /website/public/icons/map.svg: -------------------------------------------------------------------------------- 1 | 3 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /website/public/icons/school.svg: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /website/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "landnotes", 3 | "private": true, 4 | "version": "0.1.0", 5 | "type": "module", 6 | "scripts": { 7 | "dev": "vite", 8 | "dev:remote-dbs": "REMOTE_DBS=true vite", 9 | "build": "vite build", 10 | "preview": "REMOTE_DBS=true vite preview" 11 | }, 12 | "devDependencies": { 13 | "@sveltejs/vite-plugin-svelte": "^5.0.3", 14 | "svelte": "^5.20.2", 15 | "vite": "^6.2.0" 16 | }, 17 | "dependencies": { 18 | "leaflet": "^1.9.4", 19 | "pako": "^2.1.0", 20 | "svelte-portal": "^2.2.1" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /website/public/icons/church.svg: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /website/public/manifest.webmanifest: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Landnotes", 3 | "short_name": "Landnotes", 4 | "start_url": "/", 5 | "display": "standalone", 6 | "background_color": "#ffffff", 7 | "theme_color": "#b2d2dd", 8 | "apple-mobile-web-app-capable": "yes", 9 | "apple-mobile-web-app-status-bar-style": "black", 10 | "icons": [ 11 | { 12 | "src": "icon-192x192.png", 13 | "sizes": "192x192", 14 | "type": "image/png" 15 | }, 16 | { 17 | "src": "icon-512x512.png", 18 | "sizes": "512x512", 19 | "type": "image/png" 20 | } 21 | ] 22 | } -------------------------------------------------------------------------------- /website/public/icons/tree-palm.svg: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /website/public/icons/trophy.svg: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /website/public/icons/building.svg: -------------------------------------------------------------------------------- 1 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /website/public/icons/menu.svg: -------------------------------------------------------------------------------- 1 | 3 | 5 | 6 | -------------------------------------------------------------------------------- /website/vite.config.js: -------------------------------------------------------------------------------- 1 | import { defineConfig } from "vite"; 2 | import { svelte } from "@sveltejs/vite-plugin-svelte"; 3 | 4 | console.log(process.env.NODE_ENV); 5 | // Detect custom flag 6 | const useRemoteDbs = process.env.REMOTE_DBS === "true"; 7 | export default defineConfig({ 8 | plugins: [svelte()], 9 | server: { 10 | proxy: 11 | useRemoteDbs 12 | ? { 13 | "/data": { 14 | target: "https://data.landnotes.org", 15 | changeOrigin: true, 16 | rewrite: (path) => path.replace(/^\/data/, ""), 17 | }, 18 | "/query": { 19 | target: "https://landnotes.org", 20 | changeOrigin: true, 21 | rewrite: (path) => path.replace(/^\/query/, "/query"), 22 | }, 23 | } 24 | : process.env.NODE_ENV === "development" ? { 25 | "/data": { 26 | target: "http://localhost:8787", 27 | rewrite: (path) => path.replace(/^\/data/, ""), 28 | }, 29 | "/query": "http://localhost:8787", 30 | } 31 | : {}, 32 | }, 33 | }); 34 | -------------------------------------------------------------------------------- /website/jsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "moduleResolution": "bundler", 4 | "target": "ESNext", 5 | "module": "ESNext", 6 | /** 7 | * svelte-preprocess cannot figure out whether you have 8 | * a value or a type, so tell TypeScript to enforce using 9 | * `import type` instead of `import` for Types. 10 | */ 11 | "verbatimModuleSyntax": true, 12 | "isolatedModules": true, 13 | "resolveJsonModule": true, 14 | /** 15 | * To have warnings / errors of the Svelte compiler at the 16 | * correct position, enable source maps by default. 17 | */ 18 | "sourceMap": true, 19 | "esModuleInterop": true, 20 | "skipLibCheck": true, 21 | /** 22 | * Typecheck JS in `.svelte` and `.js` files by default. 23 | * Disable this if you'd like to use dynamic types. 24 | */ 25 | "checkJs": true 26 | }, 27 | /** 28 | * Use global.d.ts instead of compilerOptions.types 29 | * to avoid limiting type declarations. 30 | */ 31 | "include": ["src/**/*.d.ts", "src/**/*.js", "src/**/*.svelte"] 32 | } 33 | -------------------------------------------------------------------------------- /LICENCE.txt: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2025 Zulko 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /data_curation/.gitignore: -------------------------------------------------------------------------------- 1 | # Keep only .gitkeep files in data directories 2 | wikipedia_data/* 3 | !wikipedia_data/.gitkeep 4 | 5 | generated_data/* 6 | !generated_data/.gitkeep 7 | 8 | # Python 9 | __pycache__/ 10 | *.py[cod] 11 | *$py.class 12 | *.so 13 | .Python 14 | build/ 15 | develop-eggs/ 16 | dist/ 17 | downloads/ 18 | eggs/ 19 | .eggs/ 20 | lib/ 21 | lib64/ 22 | parts/ 23 | sdist/ 24 | var/ 25 | wheels/ 26 | *.egg-info/ 27 | .installed.cfg 28 | *.egg 29 | 30 | # Jupyter Notebook 31 | .ipynb_checkpoints 32 | */.ipynb_checkpoints/* 33 | 34 | # IPython 35 | profile_default/ 36 | ipython_config.py 37 | 38 | # Environments 39 | .env 40 | .venv 41 | env/ 42 | venv/ 43 | ENV/ 44 | env.bak/ 45 | venv.bak/ 46 | 47 | # Distribution / packaging 48 | .Python 49 | build/ 50 | develop-eggs/ 51 | dist/ 52 | downloads/ 53 | eggs/ 54 | .eggs/ 55 | lib/ 56 | lib64/ 57 | parts/ 58 | sdist/ 59 | var/ 60 | wheels/ 61 | *.egg-info/ 62 | .installed.cfg 63 | *.egg 64 | 65 | # Unit test / coverage reports 66 | htmlcov/ 67 | .tox/ 68 | .nox/ 69 | .coverage 70 | .coverage.* 71 | .cache 72 | nosetests.xml 73 | coverage.xml 74 | *.cover 75 | .hypothesis/ 76 | .pytest_cache/ 77 | 78 | # mypy 79 | .mypy_cache/ 80 | .dmypy.json 81 | dmypy.json 82 | -------------------------------------------------------------------------------- /data_curation/README.md: -------------------------------------------------------------------------------- 1 | # Data curation for landnotes 2 | 3 | A lot is happening in this repo. Here are the notebooks, in the order in which they should be run: 4 | 5 | - `wikipedia_dump_extraction.ipynb`: downloads and pre-processes the wikipedia dump. 6 | - Downloads the wikipedia dump 7 | - Extracts "redirect" pages to a key-value store (Bombay -> Mumbai) 8 | - Converts the dump to avro format 9 | - Indexes the dump so it can be queried by page title 10 | - Extracts the interpage links (e.g. a page might display "Capitole" but the word has a link to "Toulouse Capitole" which is useful and page-specific context). 11 | - Lists pages which are "disambiguation" pages" (we don't want these pages to appear in Landnotes) 12 | - Extracts and parses the infoboxes from the pages. 13 | - `geodata_curation.ipynb`: Creates the `places` database. 14 | - Downloads the SQL wikipedia dump for "geotagged places" 15 | - Parses it to a CSV/dataframe 16 | - Downloads the corresponding SQL dump with page titles and reconstructs the full dataset. 17 | - Filter out places which might come from lists or non-earth location (Mars craters etc) 18 | - Attribute a geohash to every place. 19 | - Compute the score of every page (this is the length of the site's page, using the Wikipedia page dump, after some parsing) 20 | - Hierachize the places to decide which appear at low or high zoom levels. 21 | - Write the places to a sqlite file for local testing. 22 | - Write the corresponding SQL for Cloudflare upload. 23 | - Create a database of the pages with geolocation -------------------------------------------------------------------------------- /data_curation/utils/extraction_utils.py: -------------------------------------------------------------------------------- 1 | from wiki_dump_extractor import date_utils, page_utils 2 | import zlib 3 | import json 4 | 5 | 6 | def find_dates_in_pages(pages_and_index): 7 | def extract_and_compress_dates(text): 8 | text = page_utils.remove_appendix_sections(text) 9 | text = page_utils.remove_comments_and_citations(text) 10 | dates, _errors = date_utils.extract_dates(text) 11 | json_data = json.dumps([d.to_dict() for d in dates]) 12 | return zlib.compress(json_data.encode()) 13 | 14 | pages, _ = pages_and_index 15 | return {pg.title: extract_and_compress_dates(pg.text) for pg in pages} 16 | 17 | 18 | def find_links_in_pages(index_and_pages): 19 | pages, index = index_and_pages 20 | 21 | def extract_and_compress_links(text): 22 | links = page_utils.extract_links(text) 23 | json_data = json.dumps(links).encode() 24 | return zlib.compress(json_data) 25 | 26 | return [(pg.title.encode(), extract_and_compress_links(pg.text)) for pg in pages] 27 | 28 | 29 | def parse_infoboxes(batch_and_index): 30 | pages, index = batch_and_index 31 | 32 | def process_page(page): 33 | data, _ = page_utils.parse_infobox(page.text) 34 | if not data: 35 | return None 36 | json_data = json.dumps(data) 37 | return zlib.compress(json_data.encode()) 38 | 39 | records = [ 40 | (page.title.encode(), data) 41 | for page in pages 42 | if not page.redirect_title and page.text 43 | if (data := process_page(page)) is not None 44 | ] 45 | return records 46 | -------------------------------------------------------------------------------- /worker/README.md: -------------------------------------------------------------------------------- 1 | # Database of events for cloudflare 2 | 3 | ## Prerequisites 4 | 5 | - Your SQLite file ready (let’s say it's called events.sqlite) 6 | - A Cloudflare account 7 | - wrangler CLI installed (`npm install -g wrangler`) 8 | - Logged in via `wrangler login` 9 | 10 | ## 1 Create the database 11 | 12 | ``` 13 | wrangler d1 create events-db 14 | ``` 15 | 16 | This will: 17 | 18 | - Create a D1 database named events-db 19 | - Show the binding string you’ll use in Workers 20 | 21 | # 2 upload the SQLITE file 22 | 23 | **Warning:** this will overwrite the remote DB, so only do it once to initialize. 24 | 25 | wrangler d1 create landnotes-geo-db 26 | wrangler d1 execute landnotes-geo-db --file=geo_db.sqlite 27 | wrangler d1 execute landnotes-geo-db --file=geo_db.sqlite --remote 28 | 29 | ``` 30 | wrangler d1 execute events-db --file=./events.sqlite 31 | ``` 32 | 33 | Result: 34 | 35 | ``` 36 | { 37 | "d1_databases": [ 38 | { 39 | "binding": "DB", 40 | "database_name": "landnotes-geo-db", 41 | "database_id": "1aa44a6d-dd4f-4d51-8e9b-115a653538c0" 42 | } 43 | ] 44 | } 45 | { 46 | "d1_databases": [ 47 | { 48 | "binding": "DB", 49 | "database_name": "events-db", 50 | "database_id": "e9fa680b-3f78-48e8-abd6-4eb17c6d24c6" 51 | } 52 | ] 53 | } 54 | ``` 55 | 56 | ## 3 Create the Worker 57 | 58 | ``` 59 | wrangler init querier 60 | cd event-querier 61 | ``` 62 | 63 | # Deploy 64 | 65 | Upload the data files: 66 | 67 | ``` 68 | rclone copy dev_assets/landnotes-data-files r2:landnotes-data-files 69 | ``` 70 | 71 | And deploy: 72 | 73 | ``` 74 | wrangler deploy 75 | ``` 76 | -------------------------------------------------------------------------------- /docs/economics_of_landnotes.md: -------------------------------------------------------------------------------- 1 | # The economics of Landnotes 2 | 3 | ## AI costs 4 | 5 | - We process one page per request. The page is ~3000 tokens in average, the prompt adds ~500 tokens. The output size is very variable but seems to be around 1000 tokens in average. 6 | - We use Gemini Flash 2.0 (see [Extracting events with AI](./event_extraction.md)) in batch mode which costs 7.5c/million input tokens and 30c/million output tokens. This brings us to a cost of $70/100,000 pages. 7 | - Gemini offers a promotional $300 credit to new customers, and these were used to process the first ~400,000 pages. 8 | - Processing the ~4 million wikipedia pages which are estimated to contain dates (from a regular-expression search on the 12 million pages of English wikipedia) would therefore cost ~$3000. 9 | 10 | ## Server costs 11 | 12 | Landnotes uses cloudflare for hosting. The base cost is 5$/month. 13 | 14 | _Data storage:_ There is an allowance of 5Gb, then $0.75/GB/month. Right now the project uses less than 5Gb but if we were storing all events accross 4 million pages, that would be around 25-30Gb so in the $20/month. 15 | 16 | For the next server cost computations, let's say that in a typical session, a user makes 100 requests to the server, each request reading ~100 rows in average. 17 | 18 | _Rows read:_ 25 billion free every month + 0.001$/million rows read. so this is 2.5 million free sessions a month and then 0.01$ per 1000 sessions. Basically, it comes free. 19 | 20 | _Worker requests:_ 10 million requests/month + $0.3/million requests. So taking the numbers from above, that's 100k free sessions a month, then $0.03 per 1000 sessions. A steeper cost line but a good problem to have. 21 | -------------------------------------------------------------------------------- /website/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | Landnotes 21 | 22 | 23 |
24 | 25 | 26 | -------------------------------------------------------------------------------- /website/src/lib/data/page_data.svelte.js: -------------------------------------------------------------------------------- 1 | import { inflate } from "pako"; 2 | import { fetchFromBucket, queryWithCache } from "./utils"; 3 | 4 | const pageEventsCache = new Map(); 5 | 6 | async function queryPageEventsLists(pageTitles) { 7 | try { 8 | const response = await fetch("query/events-by-page", { 9 | method: "POST", 10 | headers: { 11 | "Content-Type": "application/json", 12 | }, 13 | body: JSON.stringify(pageTitles), 14 | }); 15 | 16 | if (!response.ok) { 17 | throw new Error(`Failed to fetch events: ${response.statusText}`); 18 | } 19 | 20 | const queryJSON = await response.json(); 21 | const promisedEventsByPage = queryJSON.results.map(async (result) => { 22 | const decodedData = atob(result.zlib_json_blob); 23 | let compressedData; 24 | if (decodedData.startsWith("file:")) { 25 | const path = decodedData.slice(5); 26 | compressedData = await fetchFromBucket(path); 27 | } else { 28 | compressedData = new Uint8Array( 29 | Array.from(decodedData, (c) => c.charCodeAt(0)) 30 | ); 31 | } 32 | const decompressed = inflate(compressedData, { to: "string" }); 33 | const events = JSON.parse(decompressed); 34 | const page_title = result.page_title; 35 | return { page_title, events }; 36 | }); 37 | const eventsByPage = await Promise.all(promisedEventsByPage); 38 | return eventsByPage; 39 | } catch (error) { 40 | console.error("Error fetching page events:", error); 41 | return []; 42 | } 43 | } 44 | 45 | export async function getPageEvents(pageTitle) { 46 | const eventsByPage = await queryWithCache({ 47 | queries: [pageTitle], 48 | queryFn: queryPageEventsLists, 49 | cachedQueries: pageEventsCache, 50 | resultId: "page_title", 51 | }); 52 | if (eventsByPage.length > 0) { 53 | return eventsByPage[0].events; 54 | } else { 55 | return []; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /worker/test/index.spec.js: -------------------------------------------------------------------------------- 1 | import { env, createExecutionContext, waitOnExecutionContext, SELF } from 'cloudflare:test'; 2 | import { describe, it, expect } from 'vitest'; 3 | import worker from '../src'; 4 | 5 | describe('Hello World user worker', () => { 6 | describe('request for /message', () => { 7 | it('/ responds with "Hello, World!" (unit style)', async () => { 8 | const request = new Request('http://example.com/message'); 9 | // Create an empty context to pass to `worker.fetch()`. 10 | const ctx = createExecutionContext(); 11 | const response = await worker.fetch(request, env, ctx); 12 | // Wait for all `Promise`s passed to `ctx.waitUntil()` to settle before running test assertions 13 | await waitOnExecutionContext(ctx); 14 | expect(await response.text()).toMatchInlineSnapshot(`"Hello, World!"`); 15 | }); 16 | 17 | it('responds with "Hello, World!" (integration style)', async () => { 18 | const request = new Request('http://example.com/message'); 19 | const response = await SELF.fetch(request); 20 | expect(await response.text()).toMatchInlineSnapshot(`"Hello, World!"`); 21 | }); 22 | }); 23 | 24 | describe('request for /random', () => { 25 | it('/ responds with a random UUID (unit style)', async () => { 26 | const request = new Request('http://example.com/random'); 27 | // Create an empty context to pass to `worker.fetch()`. 28 | const ctx = createExecutionContext(); 29 | const response = await worker.fetch(request, env, ctx); 30 | // Wait for all `Promise`s passed to `ctx.waitUntil()` to settle before running test assertions 31 | await waitOnExecutionContext(ctx); 32 | expect(await response.text()).toMatch(/[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}/); 33 | }); 34 | 35 | it('responds with a random UUID (integration style)', async () => { 36 | const request = new Request('http://example.com/random'); 37 | const response = await SELF.fetch(request); 38 | expect(await response.text()).toMatch(/[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}/); 39 | }); 40 | }); 41 | }); 42 | -------------------------------------------------------------------------------- /website/src/lib/sliding_pane/About.svelte: -------------------------------------------------------------------------------- 1 | 5 | 6 |
7 |

Welcome to Landnotes

8 | 9 |

10 | Landnotes shows Wikipedia places and events on a map (see the menu 11 | menu 12 | for options). 13 |

14 | 15 |

16 | There are currently 6.5 million events extracted from 400,000 articles using 17 | Google Gemini. If anything doesn't make sense, blame it on the AI! There 18 | might be 10 times more events to extract by scanning all of Wikipedia - 19 | learn more on the project's Github page. 23 |

24 | 25 | 26 |
27 | 28 | 90 | -------------------------------------------------------------------------------- /website/src/lib/data/utils.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Queries data with caching support to minimize redundant requests 3 | * 4 | * @param {Object} options - Query options 5 | * @param {Array} options.queries - List of query identifiers to fetch 6 | * @param {Function} options.queryFn - Function to execute for queries not in cache 7 | * @param {Map} options.cachedQueries - Cache storage (Map) of previous query results 8 | * @param {string} options.resultId - Property name to use as the unique key for caching results 9 | * @returns {Promise} Combined results from cache and new queries 10 | */ 11 | export async function queryWithCache({ 12 | queries, 13 | queryFn, 14 | cachedQueries, 15 | resultId, 16 | }) { 17 | // Separate queries into those already in cache and those that need fetching 18 | const queriesInCache = queries.filter((query) => cachedQueries.has(query)); 19 | const notInCache = queries.filter((query) => !cachedQueries.has(query)); 20 | 21 | // Get results for queries that are in cache 22 | const cachedResults = queriesInCache 23 | .map((query) => cachedQueries.get(query)) 24 | .filter((entry) => entry !== null); 25 | 26 | // If all queries were in cache, return only cached results 27 | if (notInCache.length === 0) { 28 | return cachedResults; 29 | } 30 | 31 | // Fetch new results for queries not in cache 32 | const newResults = await queryFn(notInCache); 33 | 34 | // Update cache with new results 35 | newResults.forEach((newResult) => { 36 | cachedQueries.set(newResult[resultId], newResult); 37 | }); 38 | 39 | // Mark queries with no results as null in cache to avoid repeated fetching 40 | notInCache.forEach((query) => { 41 | if (!cachedQueries.has(query)) { 42 | cachedQueries.set(query, null); 43 | } 44 | }); 45 | 46 | // Return combined results from cache and new queries 47 | return [...cachedResults, ...newResults]; 48 | } 49 | 50 | export async function fetchFromBucket(path) { 51 | const bucketName = "landnotes-data-files"; 52 | 53 | const endpoint = import.meta.env.DEV 54 | ? `/data/${path}` 55 | : `https://data.landnotes.org/${path}`; 56 | 57 | try { 58 | const response = await fetch(endpoint); 59 | if (!response.ok) { 60 | throw new Error( 61 | `Failed to fetch ${bucketName}/${path}: ${response.status}` 62 | ); 63 | } 64 | return response.arrayBuffer(); 65 | } catch (error) { 66 | console.error(`Error fetching data from bucket ${bucketName}:`, error); 67 | throw error; 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Landnotes 2 | 3 | Landnotes shows Wikipedia places and events on top of OpenStreetMap - try it out! 4 | 5 | There are currently 6.5 million events extracted from 400,000 articles using Google Gemini. If anything doesn't make sense, blame it on the AI! There might be 10 times more events to extract by scanning all of Wikipedia. Learn more about future directions in the [project's documentation](./docs/README.md). 6 | 7 |

8 |

9 | 10 | screenshot 11 | 12 |
13 |

14 | 15 |

16 |

17 | 18 | screenshot 19 | 20 |
21 |

22 | 23 | 24 | ## Stack 25 | 26 | Landnotes is built on top of the following technologies :pray: 27 | 28 | Web application: 29 | - [OpenStreetMap](https://www.openstreetmap.org/) for the maps, via [LeafletJS](https://leafletjs.com/) 30 | - [Svelte 5](https://svelte.dev/) for the components and interactions 31 | - [Cloudflare](https://www.cloudflare.com/) (D1, R2, Workers) for the infrastructure. 32 | - And of course [Wikipedia](https://www.wikipedia.org/) and its [API](https://en.wikipedia.org/api/) 33 | 34 | Data curation: 35 | - [Google Gemini](https://gemini.google.com/) for the event extraction 36 | - [wiki-dump-extractor](https://github.com/zulko/wiki_dump_extractor) (written specially for this project) for handling, indexing, parsing the wikipedia page dump. 37 | - [mwparserfromhell](https://mwparserfromhell.readthedocs.io/en/latest/) for parsing Wikipedia markup into plain text. 38 | 39 | 40 | ## Development 41 | 42 | Run the website locally with: 43 | 44 | ``` 45 | cd website 46 | npm run dev:remote-dbs 47 | ``` 48 | 49 | Or to run with local databases: 50 | 51 | ``` 52 | cd website 53 | npm run dev 54 | ``` 55 | 56 | However for the local-database mode to run properly you need a local database worker (from Cloudflare): 57 | 58 | - First build a database using the scripts in `data_curation` (that's the hard part! Working on providing a test database). 59 | - Then install `wrangler` on your machine. 60 | 61 | You can run the database worker with 62 | 63 | ``` 64 | cd worker 65 | wrangler dev 66 | ``` 67 | -------------------------------------------------------------------------------- /worker/.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | 3 | logs 4 | _.log 5 | npm-debug.log_ 6 | yarn-debug.log* 7 | yarn-error.log* 8 | lerna-debug.log* 9 | .pnpm-debug.log* 10 | 11 | # Diagnostic reports (https://nodejs.org/api/report.html) 12 | 13 | report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json 14 | 15 | # Runtime data 16 | 17 | pids 18 | _.pid 19 | _.seed 20 | \*.pid.lock 21 | 22 | # Directory for instrumented libs generated by jscoverage/JSCover 23 | 24 | lib-cov 25 | 26 | # Coverage directory used by tools like istanbul 27 | 28 | coverage 29 | \*.lcov 30 | 31 | # nyc test coverage 32 | 33 | .nyc_output 34 | 35 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 36 | 37 | .grunt 38 | 39 | # Bower dependency directory (https://bower.io/) 40 | 41 | bower_components 42 | 43 | # node-waf configuration 44 | 45 | .lock-wscript 46 | 47 | # Compiled binary addons (https://nodejs.org/api/addons.html) 48 | 49 | build/Release 50 | 51 | # Dependency directories 52 | 53 | node_modules/ 54 | jspm_packages/ 55 | 56 | # Snowpack dependency directory (https://snowpack.dev/) 57 | 58 | web_modules/ 59 | 60 | # TypeScript cache 61 | 62 | \*.tsbuildinfo 63 | 64 | # Optional npm cache directory 65 | 66 | .npm 67 | 68 | # Optional eslint cache 69 | 70 | .eslintcache 71 | 72 | # Optional stylelint cache 73 | 74 | .stylelintcache 75 | 76 | # Microbundle cache 77 | 78 | .rpt2_cache/ 79 | .rts2_cache_cjs/ 80 | .rts2_cache_es/ 81 | .rts2_cache_umd/ 82 | 83 | # Optional REPL history 84 | 85 | .node_repl_history 86 | 87 | # Output of 'npm pack' 88 | 89 | \*.tgz 90 | 91 | # Yarn Integrity file 92 | 93 | .yarn-integrity 94 | 95 | # dotenv environment variable files 96 | 97 | .env 98 | .env.development.local 99 | .env.test.local 100 | .env.production.local 101 | .env.local 102 | 103 | # parcel-bundler cache (https://parceljs.org/) 104 | 105 | .cache 106 | .parcel-cache 107 | 108 | # Next.js build output 109 | 110 | .next 111 | out 112 | 113 | # Nuxt.js build / generate output 114 | 115 | .nuxt 116 | dist 117 | 118 | # Gatsby files 119 | 120 | .cache/ 121 | 122 | # Comment in the public line in if your project uses Gatsby and not Next.js 123 | 124 | # https://nextjs.org/blog/next-9-1#public-directory-support 125 | 126 | # public 127 | 128 | # vuepress build output 129 | 130 | .vuepress/dist 131 | 132 | # vuepress v2.x temp and cache directory 133 | 134 | .temp 135 | .cache 136 | 137 | # Docusaurus cache and generated files 138 | 139 | .docusaurus 140 | 141 | # Serverless directories 142 | 143 | .serverless/ 144 | 145 | # FuseBox cache 146 | 147 | .fusebox/ 148 | 149 | # DynamoDB Local files 150 | 151 | .dynamodb/ 152 | 153 | # TernJS port file 154 | 155 | .tern-port 156 | 157 | # Stores VSCode versions used for testing VSCode extensions 158 | 159 | .vscode-test 160 | 161 | # yarn v2 162 | 163 | .yarn/cache 164 | .yarn/unplugged 165 | .yarn/build-state.yml 166 | .yarn/install-state.gz 167 | .pnp.\* 168 | 169 | # wrangler project 170 | 171 | .dev.vars 172 | .wrangler/ 173 | 174 | .sqlite 175 | .sql 176 | # Ignore contents of asset directories but keep the directories 177 | dev_assets/* 178 | !dev_assets/.gitkeep 179 | local_assets/* 180 | !local_assets/.gitkeep 181 | -------------------------------------------------------------------------------- /docs/user_experience.md: -------------------------------------------------------------------------------- 1 | This document attempts to capture all the specs of the user experience in the app. 2 | It aims to be a useful reference for human developers and AI coding assistants. 3 | 4 | # The App 5 | 6 | - Landnotes is mostly a map with "markers" indicating places or regions. 7 | - The user can zoom in and out of the map, revealing markers at different levels of detail. 8 | - Hovering or clicking on a marker reveals information from wikipedia (the exact behavior varies on desktop and mobile). 9 | - The app has two main modes: showing geographic places, or showing historical events. 10 | 11 | ## Shareable links 12 | 13 | - At any point, a user can share the link in the URL bar with anyone else who will be able to see the same map (location, date, selected marker, etc.) 14 | 15 | ## Browsing history 16 | 17 | - Every time a user moves the map or selects a marker, this adds to the browsing history (and updates the URL). 18 | - When the user uses their browser's back button, the map will move back to the previous recorded state. 19 | - Currently, the use of Wikipedia iframes makes the "back button" behavior a bit complicated and browser-dependent. 20 | 21 | ## Layout 22 | 23 | - The main component, taking almost the full screen, is the map with markers. 24 | - Overlaid at the top of the map, there is a "Menu" with a search bar, a date selector (in "events" mode), and drop-down options which mainly serves to switch between the two . 25 | - There is a sliding pane that will appear (on the left on mobile, on the bottom on desktop) to show wikipedia pages and lists of events. 26 | 27 | # Markers 28 | 29 | Marker hover/click behavior is different between mobile/desktop and between place/event modes. 30 | This explains the general complexity of the code around these. 31 | 32 | On mobile devices: 33 | 34 | - Tapping a marker centers the map on it and reveals its popup 35 | - Tapping the marker again reveals its wikipedia page. 36 | 37 | On desktop: 38 | 39 | - Hovering over a marker reveals its popup 40 | - Clicking on a marker centers the map on it and reveals its wikipedia page. 41 | 42 | For place markers: 43 | 44 | - The popup is a wikipedia summary with a photo and a short text cut at ~300 characters. 45 | - The popup is not enterable on desktop, leaving the marker closes the popup. 46 | 47 | For event markers: 48 | 49 | - The popup is a card who/where/what. 50 | - The popup has links to the people and places mentioned in the popup. 51 | - Clicking on the links in the popup opens their wikipedia pages in the side pane. 52 | - The popup can be entered on desktop, leaving the marker or the popup closes the popup. 53 | - On desktop, hovering the links in the popup reveals a tooltip with the wikipedia summary. 54 | 55 | ## Geographic places 56 | 57 | - Places with a larger wikipedia page are shown at higher levels of zoom. 58 | 59 | # Historical events 60 | 61 | ## Side pane 62 | 63 | The side pane is used to show the wikipedia page. 64 | 65 | On wide screens, the side pane appears from the left, on narrow screens it appears from the bottom. 66 | 67 | On narrow screens, the side pane takes the full screen first. 68 | 69 | The side pane has tabs, 70 | 71 | - "Wikipedia" (shows the wikipedia page) 72 | - "Events" (shows the events associated with the page, either because their were extracted from the page because they happen at the location that the page represents or involved the person that the page represents). In that tab, the app first fetches a list of event IDs by year. Then it displays the events in collapsible sections by year. If the total number of events is less than 500, it fetches all event infos at once. Else the event data is only fetched when the user clicks on a year section. 73 | 74 | The side pane is also used in for two other displays: 75 | 76 | - When a user clicks "see more events at this location" in the map, it shows the events that happened at the same place at the same date specified by the user. 77 | - The "About" tab also shows in the side pane. 78 | -------------------------------------------------------------------------------- /website/src/lib/data/events_data.js: -------------------------------------------------------------------------------- 1 | import { cachedDecodeHybridGeohash } from "./places_data"; 2 | import { queryWithCache } from "./utils"; 3 | let worker = null; 4 | // Store for pending request promises 5 | let workerPromises = {}; 6 | const cachedEventsById = new Map(); 7 | 8 | function initEventsWorker() { 9 | // Create a single worker instance that will be reused 10 | 11 | console.log("Initializing worker"); 12 | if (worker === null) { 13 | worker = new Worker(new URL("./events_worker.js", import.meta.url), { 14 | type: "module", 15 | }); 16 | 17 | // Set up message handler for worker responses 18 | worker.addEventListener("message", (event) => { 19 | const { type, requestId, events, dotEvents, error } = event.data; 20 | 21 | if (!requestId || !workerPromises[requestId]) { 22 | console.error( 23 | "Received worker message with no matching request", 24 | event.data 25 | ); 26 | return; 27 | } 28 | 29 | const { resolve, reject } = workerPromises[requestId]; 30 | 31 | if (type === "response") { 32 | resolve({ events, dotEvents }); 33 | } else if (type === "error") { 34 | reject(new Error(error)); 35 | } 36 | 37 | // Clean up the promise 38 | delete workerPromises[requestId]; 39 | }); 40 | 41 | // Handle worker errors 42 | worker.addEventListener("error", (error) => { 43 | console.error("Worker error:", error); 44 | }); 45 | } 46 | } 47 | 48 | initEventsWorker(); 49 | 50 | /** 51 | * Get events for the given bounds and date parameters 52 | * @param {Object} params - Parameters for fetching events 53 | * @param {Object} params.bounds - Map bounds (north, south, east, west) 54 | * @param {number} params.zoom - Current zoom level 55 | * @param {Object} params.date - Date object with year and month 56 | * @param {boolean} params.strictDate - Whether to strictly match the date 57 | * @returns {Promise<{events: Array, dotEvents: Array}>} Promise resolving to an array of events 58 | */ 59 | export async function getEventsForBoundsAndDate({ 60 | bounds, 61 | zoom, 62 | date, 63 | strictDate, 64 | }) { 65 | // Create a unique request ID 66 | const requestId = `query_${Date.now()}_${Math.random()}`; 67 | 68 | // Create a promise that will be resolved when the worker responds 69 | const requestPromise = new Promise((resolve, reject) => { 70 | workerPromises[requestId] = { resolve, reject }; 71 | }); 72 | 73 | // Send the request to the worker 74 | worker.postMessage({ 75 | type: "getEventsForBoundsAndDate", 76 | requestId, 77 | bounds, 78 | zoom, 79 | date, 80 | strictDate, 81 | }); 82 | 83 | // Set a timeout to reject the promise if the worker doesn't respond 84 | const timeoutId = setTimeout(() => { 85 | if (workerPromises[requestId]) { 86 | const { reject } = workerPromises[requestId]; 87 | reject(new Error("Worker request timed out")); 88 | delete workerPromises[requestId]; 89 | } 90 | }, 30000); // 30 second timeout 91 | 92 | try { 93 | // Wait for the worker to respond 94 | const result = await requestPromise; 95 | clearTimeout(timeoutId); 96 | return result; 97 | } catch (error) { 98 | clearTimeout(timeoutId); 99 | throw error; 100 | } 101 | } 102 | 103 | async function queryEventsById(eventIds) { 104 | eventIds.sort(); 105 | const response = await fetch("query/events-by-id", { 106 | method: "POST", 107 | headers: { 108 | "Content-Type": "application/json", 109 | }, 110 | body: JSON.stringify(eventIds), 111 | }); 112 | const queryJSON = await response.json(); 113 | const entries = queryJSON.results; 114 | entries.forEach((entry) => { 115 | entry.geohash4 = entry.geohash4.split("|"); 116 | entry.locations_latlon = entry.geohash4.map(cachedDecodeHybridGeohash); 117 | entry.subevents = []; 118 | }); 119 | return entries; 120 | } 121 | 122 | export async function getEventsById(eventIds) { 123 | return await queryWithCache({ 124 | queries: eventIds, 125 | queryFn: queryEventsById, 126 | cachedQueries: cachedEventsById, 127 | resultId: "event_id", 128 | }); 129 | } 130 | -------------------------------------------------------------------------------- /worker/wrangler.jsonc: -------------------------------------------------------------------------------- 1 | /** 2 | * For more details on how to configure Wrangler, refer to: 3 | * https://developers.cloudflare.com/workers/wrangler/configuration/ 4 | */ 5 | { 6 | "$schema": "node_modules/wrangler/config-schema.json", 7 | "name": "landnotes-worker", 8 | "main": "src/index.js", 9 | "compatibility_date": "2025-04-04", 10 | "compatibility_flags": [ 11 | "nodejs_compat" 12 | ], 13 | "assets": { 14 | "binding": "ASSETS", 15 | "directory": "./public" 16 | }, 17 | "observability": { 18 | "enabled": true, 19 | "head_sampling_rate": 0.1 20 | }, 21 | "env": { 22 | "dev": { 23 | "assets": { 24 | "directory": "dev_assets", 25 | "binding": "DEV_FILES", 26 | "not_found_handling": "404-page" 27 | }, 28 | "d1_databases": [ 29 | { 30 | "binding": "geoDB", 31 | "database_name": "landnotes-geo-db", 32 | "database_id": "77e642fc-e1d6-4996-bc1d-11bcb077a838", 33 | "preview_database_id": "local-landnotes-geo-db" 34 | }, 35 | { 36 | "binding": "eventsDB", 37 | "database_name": "landnotes-events-db", 38 | "database_id": "2a596312-20f2-4c93-8ffd-8fa6ef9c5c12", 39 | "preview_database_id": "local-landnotes-events-db" 40 | }, 41 | { 42 | "binding": "eventsByPageDB", 43 | "database_name": "landnotes-events-by-page-db", 44 | "database_id": "000860d8-e326-4784-96c0-452f0753c261", 45 | "preview_database_id": "local-landnotes-events-by-page-db" 46 | }, 47 | { 48 | "binding": "eventsByMonthDB", 49 | "database_name": "landnotes-events-by-month-db", 50 | "database_id": "06cf9626-8e7a-475a-86b5-ea3025824703", 51 | "preview_database_id": "local-landnotes-events-by-month-db" 52 | } 53 | ] 54 | } 55 | }, 56 | /** 57 | * D1 Database Binding 58 | * https://developers.cloudflare.com/d1/get-started/ 59 | */ 60 | "d1_databases": [ 61 | { 62 | "binding": "geoDB", 63 | "database_name": "landnotes-geo-db", 64 | "database_id": "77e642fc-e1d6-4996-bc1d-11bcb077a838", 65 | "preview_database_id": "local-landnotes-geo-db" 66 | }, 67 | { 68 | "binding": "eventsDB", 69 | "database_name": "landnotes-events-db", 70 | "database_id": "2a596312-20f2-4c93-8ffd-8fa6ef9c5c12", 71 | "preview_database_id": "local-landnotes-events-db" 72 | }, 73 | { 74 | "binding": "eventsByPageDB", 75 | "database_name": "landnotes-events-by-page-db", 76 | "database_id": "000860d8-e326-4784-96c0-452f0753c261", 77 | "preview_database_id": "local-landnotes-events-by-page-db" 78 | }, 79 | { 80 | "binding": "eventsByMonthDB", 81 | "database_name": "landnotes-events-by-month-db", 82 | "database_id": "06cf9626-8e7a-475a-86b5-ea3025824703", 83 | "preview_database_id": "local-landnotes-events-by-month-db" 84 | } 85 | ], 86 | "r2_buckets": [ 87 | { 88 | "bucket_name": "landnotes-data-files", 89 | "binding": "landnotesDataFiles" 90 | } 91 | ] 92 | /** 93 | * Smart Placement 94 | * Docs: https://developers.cloudflare.com/workers/configuration/smart-placement/#smart-placement 95 | */ 96 | // "placement": { "mode": "smart" }, 97 | 98 | /** 99 | * Bindings 100 | * Bindings allow your Worker to interact with resources on the Cloudflare Developer Platform, including 101 | * databases, object storage, AI inference, real-time communication and more. 102 | * https://developers.cloudflare.com/workers/runtime-apis/bindings/ 103 | */ 104 | 105 | /** 106 | * Environment Variables 107 | * https://developers.cloudflare.com/workers/wrangler/configuration/#environment-variables 108 | */ 109 | // "vars": { "MY_VARIABLE": "production_value" }, 110 | /** 111 | * Note: Use secrets to store sensitive data. 112 | * https://developers.cloudflare.com/workers/configuration/secrets/ 113 | */ 114 | 115 | /** 116 | * Static Assets 117 | * https://developers.cloudflare.com/workers/static-assets/binding/ 118 | */ 119 | // "assets": { "directory": "./public/", "binding": "ASSETS" }, 120 | 121 | /** 122 | * Service Bindings (communicate between multiple Workers) 123 | * https://developers.cloudflare.com/workers/wrangler/configuration/#service-bindings 124 | */ 125 | // "services": [{ "binding": "MY_SERVICE", "service": "my-service" }] 126 | } 127 | -------------------------------------------------------------------------------- /docs/future_directions.md: -------------------------------------------------------------------------------- 1 | # Ideas of future directions for Landnotes 2 | 3 | ## Scanning all of the English Wikipedia 4 | 5 | The most obvious future direction is to continue feeding the English Wikipedia to an AI for extraction. So far 400,000 pages have been processed, but based on a simple search using regular-expressions there are at least 2 million pages other pages just for events happening between 250 and 1950, and another 2 million pages for events happening after 1950. It would take a bit over $3000 to process all these pages, and then the database would cost around ~30$/month to host, see [the economics of Landnotes]. This is a bit steep for a project at this stage, so the progress will depend on user interest and whether there is interest in crowd-sourcing, credit sponsoring from, or interest from the wikimedia foundation etc. 6 | 7 | ## Non-English wikipedias 8 | 9 | In general, I think AI has a card to play in terms of homogenizing the quality of wikipedia articles accross languages, typically to to find facts reported in some languages and not others. For instance, you can expect the French article on the Eiffel Tower to have more facts than the English one, because there may be more french-speaking people available to edit the page than english-speaking people, and because most sources are in French. On the other hand, the different language versions reflect cultural interests and blindspots. The French wikipedia page for the Eiffel Tower makes no mention of Haiti, but the English one has a whole paragraph on how some of the financing came from predatory loans that Haiti was subjected to, citing the New York Times. 10 | 11 | In terms of data volume, and looking at the size of the backup files, non-English wikipedia is not much bigger than English wikipedia. French/Spanish/German/Chinese wikipedia combined are about the same size as the English wikipedia. 12 | 13 | But extending the project to non-English wikipedia won't be as simple as pushing a button: 14 | 15 | - When looking for events using scripts, or when classifying events and places into categories, we will probably need to adapt the scripts to each language so that the current script looking for "birth" in english will look for "naissance" in french for instance. 16 | - We'll need to evaluate which AI models can extract data from these languages, and whether they still can extract data in a JSON format ({who, when, where, what}) without being confused. 17 | - We'll need to make user-experience decisions on how to use the events extracted from non-English wikipedia. Do we translate them to english? Do we display them with the other events? Does the user need to select and unselect languages? 18 | 19 | ## Beyond wikipedia 20 | 21 | If the project goes this far, it would be interesting to extract events from sources other than wikipedia. For instance, history books. If we fed Mark Duncan's Lafayette biography to Gemini, it would no doubt extract hundreds of interesting events. Fiction books would work too, so you could follow your favorite historical novel displayed on the map along with the real events. The Gutenberg project has around 20Gb of public domain texts. By identifying the ones with dates and feeding them to Gemini, how many events would we get? 22 | 23 | ## Historical borders 24 | 25 | Landnote displays historical events on Open Street Map, so the old settlements, battles and travels are superimposed on today's streets, fields and highways. But this deprives us of important historical context. Some regions of the world were very different just one or two centuries ago, and borders have moved a lot. Famous German scientist Max Born and Alois Alzheimer were born in the German city of Breslau, which is now called Wroclaw in Poland. Verdi was born in a small Italian village that was technically in France at the time. 26 | 27 | Having a layer showing historical borders on the map depending on the year would go a long way to give a better understanding of the historical events. This data is mostly available. The [historical-basemaps](https://github.com/aourednik/historical-basemaps?tab=readme-ov-file) project on Github has a great data collection and a [demo for it](https://historicborders.app/?lng=6.4513466&lat=39.3328695&zoom=4.1297515&year=1492) but the viral GPL license makes me a bit worried. There are many other projects like [Open Historical Maps](https://openhistoricalmaps.org/) and [OldMapsOnline.org](https://oldmapsonline.org/) that make me think it is achievable. 28 | -------------------------------------------------------------------------------- /website/src/App.svelte: -------------------------------------------------------------------------------- 1 | 76 | 77 | 78 | 79 |
85 |
86 | {#if appState.wikiPage || appState.paneTab === "same-location-events" || appState.paneTab === "about"} 87 |
88 | 89 |
90 | {/if} 91 | 92 |
93 | 94 |
95 | 96 |
97 |
98 |
99 |
100 | 101 | 169 | -------------------------------------------------------------------------------- /website/notes.md: -------------------------------------------------------------------------------- 1 | every time the event parameters change, 2 | 3 | - get all the missing event data dumps for the months and regions at hand and depending on regions (boundaries) 4 | - store the month-region results in cachedEventsByMonthRegion 5 | - For each region, build a LOD geokey system that is specific to the date parameters by 6 | - Assigning a score to each event depending on how far they are from the date. 7 | - Building the hybrid geotree where the best events get to go first to get the best geokeys 8 | - Storing the geotrees under geotreesByRegion["a"] 9 | - Every time the date parameters change 10 | - download whichever month-region results are missing for the regions in current bounds. Cache these 11 | - compute again all the geotrees for all the regions at hand. 12 | - Every time the bounds change: 13 | - Download the missing month-region results for the new regions, cache them. 14 | - Compute the geotrees for these new regions only. 15 | 16 | These functions and their cache probably live in a webworker. 17 | 18 | page side (in App.js): 19 | 20 | ```python 21 | 22 | cachedEvents = Map() 23 | 24 | 25 | async def getEventsFromWorker(): 26 | requestId = `query_${Date.now()}_${Math.random()}` 27 | const queryPromise = new Promise((resolve, reject) => { 28 | window.eventsWorkerPromises[requestId] = { resolve, reject }; 29 | }); 30 | window.postMessage("getEventIdsForGeokeys", {geokeys, date, strictDate}); 31 | return await queryPromise 32 | 33 | 34 | async def getEvents(bounds, zoom, date, strictDate): 35 | events = await getEventsFromWorker(bounds, zoom, date, strictDate) 36 | eventsById = {event.id: event for event in events} 37 | eventsWithInfo = queryEventsById(eventsById.keys(), cachedEvents) 38 | for event in eventsWithInfo: 39 | eventsById[event.id] = {...eventsById[event.id], event} 40 | markers = eventsById.values() 41 | dotMarkers = events.map(event=> event.subevents).flat() 42 | ``` 43 | 44 | Worker pseudo code (should be javascript): 45 | 46 | ```python 47 | 48 | currentDate = None 49 | currentStrictDate = None 50 | processedRegions = Set() 51 | cachedEventsByMonthRegion = Map() 52 | eventsByGeoKeyForDate = Map() 53 | 54 | 55 | self.addEventListener('message', async (event) => { 56 | if (event.data.type === "getEventIdsForGeokeys"): 57 | const { geokeys, date, strictDate } = event.data; 58 | const events = await getEventsForGeokeys(geokeys, date, strictDate); 59 | }) 60 | 61 | async def getEventsForGeokeys(geokeys, date, strictDate): 62 | if currentDate != date or currentStrictDate != strictDate: 63 | # reset the computed events geokeys! 64 | currentDate = date 65 | currentStrictDate = strictDate 66 | eventsByGeoKeyForDate = Map() 67 | regions = set([g[0] for g in geokeys]) 68 | missing_regions = [r for r in regions if r not in processedRegions] 69 | # regions with empty geotrees simply receive {} 70 | await Promise.all([processRegion(region, date, strictDate) for region in missing_regions]) 71 | events = [eventsByGeoKeyForDate[g] for g in geokeys] 72 | return events 73 | 74 | async def processRegion(region, date, strictDate): 75 | months = getMonths(date, strictDate) 76 | missing_month_regions = [region + month in months] 77 | events = await getEvents({missing_month_regions, cachedQueries: cachedEventsByMonthRegion}) 78 | assignGeokeysToEvents(events, date, strictDate) 79 | 80 | def assignGeokeysToEvents(events, date, strictDate): 81 | 82 | # sort the events by order of proximity to the date 83 | for event in events: 84 | event.score = scoreEventByProximityToDate(event, date, strictDate) 85 | events.sort(key=lambda e: e.score, reverse=True) 86 | 87 | for event in events: 88 | prefix = "" 89 | for char in event.geohash: 90 | prefix += char 91 | 92 | if prefix not in eventsByGeoKeyForDate: 93 | eventsByGeoKeyForDate[prefix] = {event, subevents: [], same_location_events: []} 94 | else: 95 | event_for_geokey = eventsByGeoKeyForDate[prefix] 96 | if len(event_for_geokey["subevents"]) < 10: 97 | event_for_geokey["subevents"].append(event) 98 | if event.geohash == event_for_geokey["event"].geohash: 99 | event_for_geokey["same_location_events"].append(event) 100 | 101 | 102 | def scoreEventByProximityToDate(event, date, strictDate): 103 | # sum the difference beteen the event start date and the date and also end date 104 | score = (event.start_date - date).days + (event.end_date - date).days 105 | return score 106 | ``` 107 | -------------------------------------------------------------------------------- /website/src/lib/sliding_pane/SlidingPaneHeader.svelte: -------------------------------------------------------------------------------- 1 | 11 | 12 |
13 | {#if appState.paneTab === "wikipedia" || appState.paneTab === "events"} 14 |
15 | 23 | 31 |
32 | {/if} 33 | 34 |
35 | 47 | 48 | 63 | 64 | 79 | 82 |
83 |
84 | 85 | 185 | -------------------------------------------------------------------------------- /website/src/lib/data/date_utils.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Configuration for date validation constraints 3 | * Defines thresholds for when month and day values become required 4 | */ 5 | export const dateConstraints = { 6 | firstYearRequiringMonth: 3000, 7 | firstYearRequiringDay: 3000, 8 | }; 9 | 10 | /** 11 | * Calculates the number of days in a specific month and year 12 | * 13 | * @param {number} year - The year (handles both positive and negative years) 14 | * @param {number} month - The month (1-12) 15 | * @returns {number} The number of days in the specified month 16 | */ 17 | function getDaysInMonth(year, month) { 18 | return new Date(year < 0 ? year + 1 : year, month, 0).getDate(); 19 | } 20 | 21 | /** 22 | * Ensures a date object conforms to defined constraints. 23 | * - Enforces month requirements for years beyond threshold 24 | * - Enforces day requirements for years beyond threshold 25 | * - Ensures day values are within valid range for the month/year 26 | * 27 | * @param {Object} date - Date object with year, month, and day properties 28 | * @param {number} date.year - Year value 29 | * @param {number|string} date.month - Month value (1-12 or "all") 30 | * @param {number|string} date.day - Day value (1-31 or "all") 31 | * @returns {Object} The constrained date object 32 | */ 33 | export function constrainedDate(date) { 34 | // Enforce month requirement for years beyond threshold 35 | const newDate = { ...date }; 36 | if ( 37 | date.year > dateConstraints.firstYearRequiringMonth && 38 | date.month === "all" 39 | ) { 40 | newDate.month = 1; 41 | } 42 | 43 | // Enforce day requirement for years beyond threshold 44 | if (date.year > dateConstraints.firstYearRequiringDay && date.day === "all") { 45 | newDate.day = 1; 46 | } 47 | 48 | // Ensure day is within valid range for the month 49 | if (date.day !== "all" && date.month !== "all") { 50 | const daysInMonth = getDaysInMonth(date.year, date.month); 51 | newDate.day = Math.min(Math.max(date.day, 1), daysInMonth); 52 | } 53 | 54 | return newDate; 55 | } 56 | 57 | export function parseEventDate(date) { 58 | const isApproximate = date.includes("(~)"); 59 | const [yearStr, monthStr, dayStr] = date.replace(" (~)", "").split("/"); 60 | let year; 61 | if (yearStr.endsWith(" BC")) { 62 | year = -parseInt(yearStr.slice(0, -3)); 63 | } else { 64 | year = parseInt(yearStr); 65 | } 66 | const month = parseInt(monthStr); 67 | const day = parseInt(dayStr); 68 | return { year, month, day, isApproximate }; 69 | } 70 | 71 | export function daysBetweenTwoDates(date1, date2) { 72 | const yearGap = 365 * (date1.year - date2.year); 73 | const monthGap = (date1.month - date2.month) * 30; 74 | const dayGap = date1.day - date2.day; 75 | return yearGap + monthGap + dayGap; 76 | } 77 | 78 | export function isAfter(date1, date2) { 79 | return ( 80 | date1.year > date2.year || 81 | (date1.year === date2.year && 82 | (date1.month > date2.month || 83 | (date1.month === date2.month && 84 | (date1.day > date2.day || date1.day === date2.day)))) 85 | ); 86 | } 87 | 88 | export function dateToUrlString(date) { 89 | if (!date) { 90 | return null; 91 | } 92 | if (date.month === "all") { 93 | return `${date.year}`; 94 | } else if (date.day === "all") { 95 | return `${date.year}--${date.month}`; 96 | } else { 97 | return `${date.year}--${date.month}--${date.day}`; 98 | } 99 | } 100 | 101 | export function parseUrlDate(dateString) { 102 | if (!dateString) { 103 | return null; 104 | } 105 | let date = null; 106 | const components = dateString.split("--").map((e) => parseInt(e)); 107 | if (components.length === 1) { 108 | let [year] = components; 109 | date = { year, month: "all", day: "all" }; 110 | } else if (components.length === 2) { 111 | let [year, month] = components; 112 | date = { year, month, day: "all" }; 113 | } else if (components.length === 3) { 114 | let [year, month, day] = components; 115 | date = { year, month, day }; 116 | } 117 | return constrainedDate(date); 118 | } 119 | 120 | export function startAndEndDateToDateSetting(startDate, endDate) { 121 | if ( 122 | startDate.month === 1 && 123 | startDate.day === 1 && 124 | (startDate.year < endDate.year || 125 | (startDate.year === endDate.year && 126 | endDate.month === 12 && 127 | endDate.day === 31)) 128 | ) { 129 | return { 130 | year: startDate.year, 131 | month: "all", 132 | day: "all", 133 | }; 134 | } else if ( 135 | startDate.year === endDate.year && 136 | startDate.month === endDate.month && 137 | startDate.day === 1 && 138 | (endDate.month > startDate.month || 139 | (endDate.month === startDate.month && 140 | endDate.day === getDaysInMonth(endDate.year, endDate.month))) 141 | ) { 142 | return { 143 | year: startDate.year, 144 | month: startDate.month, 145 | day: "all", 146 | }; 147 | } else { 148 | return { 149 | year: startDate.year, 150 | month: startDate.month, 151 | day: startDate.day, 152 | }; 153 | } 154 | } 155 | -------------------------------------------------------------------------------- /worker/scripts/update-db.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Update Geo Database Script 3 | * ========================== 4 | * 5 | * This script updates the geographic database by executing SQL files. 6 | * It can be run in either local or remote mode and supports various parameters. 7 | * 8 | * Usage: 9 | * node update-db.js [options] 10 | * 11 | * Options: 12 | * --remote Execute against the remote database (default: local) 13 | * --delete Drop existing tables before creating new ones 14 | * --start-index=N Skip the first N SQL files (useful for resuming after errors) 15 | * --db=NAME Specify the database name (default: landnotes-geo-db) 16 | * --sql-dir=PATH Specify the directory containing SQL files (required) 17 | * 18 | * Examples: 19 | * node scripts/update-db.js --remote --delete --db=landnotes-events-by-month-db --sql-dir=local_assets/sql/events_by_month_region/ 20 | * node scripts/update-db.js --remote --delete --db=landnotes-events-by-page-db --sql-dir=local_assets/sql/events_by_page_and_year/ 21 | * node scripts/update-db.js --remote --delete --db=landnotes-events-db --sql-dir=local_assets/sql/events/ 22 | * node scripts/update-db.js --remote --delete --db=landnotes-geo-db --sql-dir=local_assets/sql/places_db/ 23 | * 24 | * Notes: 25 | * - SQL files are read from the directory specified by --sql-dir 26 | * - The script processes all .sql files in alphabetical order 27 | * - If an error occurs, the script will retry after a 5-second delay 28 | */ 29 | 30 | const { execSync } = require('child_process'); 31 | const fs = require('fs'); 32 | const path = require('path'); 33 | 34 | // Check if --remote flag is provided 35 | const isRemote = process.argv.includes('--remote'); 36 | const executionMode = isRemote ? '--remote' : ''; 37 | console.log(`Running in ${isRemote ? 'remote' : 'local'} mode`); 38 | 39 | // Check if --delete flag is provided 40 | const deleteTables = process.argv.includes('--delete'); 41 | console.log(`Table deletion is ${deleteTables ? 'enabled' : 'disabled'}`); 42 | 43 | // Parse the start-index parameter 44 | const startIndexArg = process.argv.find((arg) => arg.startsWith('--start-index=')); 45 | const startIndex = startIndexArg ? parseInt(startIndexArg.split('=')[1], 10) || 0 : 0; 46 | console.log(`Skipping first ${startIndex} files`); 47 | 48 | // Parse the db parameter 49 | const dbArg = process.argv.find((arg) => arg.startsWith('--db=')); 50 | if (!dbArg) { 51 | console.error('Error: Database name must be specified with --db=NAME'); 52 | process.exit(1); 53 | } 54 | const dbName = dbArg.split('=')[1]; 55 | console.log(`Using database: ${dbName}`); 56 | 57 | // Parse the sql-dir parameter 58 | const sqlDirArg = process.argv.find((arg) => arg.startsWith('--sql-dir=')); 59 | if (!sqlDirArg) { 60 | console.error('Error: SQL directory must be specified with --sql-dir=PATH'); 61 | process.exit(1); 62 | } 63 | const sqlDir = sqlDirArg.split('=')[1]; 64 | console.log(`Using SQL directory: ${sqlDir}`); 65 | 66 | const files = fs 67 | .readdirSync(sqlDir) 68 | .filter((file) => file.endsWith('.sql')) 69 | .sort(); 70 | console.log(files); 71 | 72 | // Only execute deletion commands if --delete flag is provided 73 | if (deleteTables) { 74 | const tablesToDrop = ['geodata', 'places', 'events', 'pages', 'events_by_month_region', 'text_search']; 75 | 76 | console.log(`Dropping ${tablesToDrop.length} tables...`); 77 | 78 | for (let i = 0; i < tablesToDrop.length; i++) { 79 | const tableName = tablesToDrop[i]; 80 | console.log(`Dropping table ${i + 1}/${tablesToDrop.length}: ${tableName}`); 81 | execSync(`npx wrangler d1 execute ${executionMode} ${dbName} --command "DROP TABLE IF EXISTS ${tableName};"`); 82 | } 83 | 84 | console.log('All tables dropped successfully.'); 85 | } 86 | 87 | // Skip the first N files by using slice 88 | const filesToProcess = files.slice(startIndex); 89 | console.log(`Processing ${filesToProcess.length} out of ${files.length} files`); 90 | 91 | // Convert to an async function to handle the retry delay 92 | const processFiles = async () => { 93 | for (const file of filesToProcess) { 94 | const filePath = path.join(sqlDir, file); 95 | const command = `npx wrangler d1 execute ${executionMode} -y ${dbName} --file="${filePath}"`; 96 | console.log(`Executing ${filePath}...`); 97 | console.log(command); 98 | 99 | let success = false; 100 | try { 101 | execSync(command, { stdio: 'inherit' }); 102 | success = true; 103 | } catch (error) { 104 | console.error(`Error executing ${filePath}:`, error.message); 105 | console.log('Waiting 5 seconds before retry...'); 106 | 107 | // Wait 5 seconds before retry 108 | await new Promise((resolve) => setTimeout(resolve, 5000)); 109 | 110 | console.log(`Retrying ${filePath}...`); 111 | try { 112 | execSync(command, { stdio: 'inherit' }); 113 | success = true; 114 | } catch (retryError) { 115 | console.error(`Retry failed for ${filePath}:`, retryError.message); 116 | } 117 | } 118 | 119 | if (success) { 120 | console.log(`Successfully processed ${filePath}`); 121 | } else { 122 | console.error(`Failed to process ${filePath} after retry`); 123 | } 124 | } 125 | }; 126 | 127 | // Execute the async function 128 | processFiles().catch((error) => { 129 | console.error('Processing failed:', error); 130 | process.exit(1); 131 | }); 132 | -------------------------------------------------------------------------------- /website/src/lib/sliding_pane/SlidingPane.svelte: -------------------------------------------------------------------------------- 1 | 109 | 110 | e.key === "Escape" && closePane()} 112 | onresize={handleResize} 113 | /> 114 | 115 |
116 |
117 | 118 | 119 |
120 | {#if appState.paneTab === "wikipedia" && appState.wikiPage} 121 | 131 | {:else if appState.paneTab === "events" && appState.wikiPage} 132 | 133 | {:else if appState.paneTab === "same-location-events"} 134 | 135 | {:else if appState.paneTab === "about"} 136 | 137 | {:else} 138 |

No page specified

139 | {/if} 140 |
141 |
142 |
143 | 144 | 185 | -------------------------------------------------------------------------------- /website/src/lib/data/places_data.js: -------------------------------------------------------------------------------- 1 | import { inflate } from "pako"; 2 | 3 | import { decodeHybridGeohash, getOverlappingGeoEncodings } from "./geohash"; 4 | import { queryWithCache } from "./utils"; 5 | const decodeHybridGeohashCache = new Map(); 6 | 7 | export function cachedDecodeHybridGeohash(geohash) { 8 | if (decodeHybridGeohashCache.has(geohash)) { 9 | return decodeHybridGeohashCache.get(geohash); 10 | } 11 | const result = decodeHybridGeohash(geohash); 12 | decodeHybridGeohashCache.set(geohash, result); 13 | return result; 14 | } 15 | 16 | export function addLatLonToEntry(entry) { 17 | // Todo: get rid of full_hybrid_geohash 18 | const full_geohash = 19 | entry.geohash4 || 20 | entry.full_hybrid_geohash || 21 | `${entry.geokey}${entry.geokey_complement}`; 22 | const coords = cachedDecodeHybridGeohash(full_geohash); 23 | entry.lat = coords.lat; 24 | entry.lon = coords.lon; 25 | } 26 | 27 | function processEntriesUnderGeokey(entry) { 28 | if (entry.dots) { 29 | // Decompress dots if it's compressed with zlib 30 | const decodedData = atob(entry.dots); 31 | const compressedData = new Uint8Array( 32 | Array.from(decodedData, (c) => c.charCodeAt(0)) 33 | ); 34 | const decompressed = inflate(compressedData, { to: "string" }); 35 | const entriesUnderGeokey = JSON.parse(decompressed); 36 | entry.dots = entriesUnderGeokey; 37 | 38 | // Convert geohashes to lat/lon coordinates for each key in dots 39 | for (const key in entry.dots) { 40 | if (entry.dots.hasOwnProperty(key)) { 41 | const geohashes = entry.dots[key]; 42 | entry.dots[key] = geohashes.map((geohash) => { 43 | const coords = decodeHybridGeohash(geohash); 44 | return { geokey: `dot-${geohash}`, ...coords }; 45 | }); 46 | } 47 | } 48 | } 49 | } 50 | 51 | async function queryPlacesByGeokey(geokeys) { 52 | geokeys.sort(); 53 | const response = await fetch("query/places-by-geokey", { 54 | method: "POST", 55 | headers: { 56 | "Content-Type": "application/json", 57 | }, 58 | body: JSON.stringify(geokeys), 59 | }); 60 | const queryJSON = await response.json(); 61 | const entries = queryJSON.results; 62 | entries.forEach(addLatLonToEntry); 63 | entries.forEach(processEntriesUnderGeokey); 64 | return entries; 65 | } 66 | /** 67 | * Fetches geodata for a list of geokeys, using cached entries when available 68 | * 69 | * @param {Object} params - Object containing geokeys and cachedQueries 70 | * @param {Array} params.geokeys - Array of geokeys to fetch 71 | * @param {Map} params.cachedQueries - Map of already cached entries 72 | * @returns {Promise} - Combined array of cached and newly fetched entries 73 | */ 74 | export async function getPlaceDataFromGeokeys({ geokeys, cachedQueries }) { 75 | return await queryWithCache({ 76 | queries: geokeys, 77 | queryFn: queryPlacesByGeokey, 78 | cachedQueries, 79 | resultId: "geokey", 80 | }); 81 | } 82 | 83 | /** 84 | * Fetch geodata for the specified map bounds 85 | */ 86 | export async function getGeodataFromBounds({ 87 | bounds, 88 | maxZoomLevel, 89 | cachedQueries, 90 | }) { 91 | // Collect geokeys for all zoom levels up to maxZoomLevel 92 | const geokeys = Array.from({ length: maxZoomLevel }, (_, i) => i + 1).flatMap( 93 | (zoomLevel) => getOverlappingGeoEncodings(bounds, zoomLevel) 94 | ); 95 | if (geokeys.length > 1000) { 96 | throw new Error("geokeys issue"); 97 | } 98 | const geokeyResults = await getPlaceDataFromGeokeys({ 99 | geokeys, 100 | cachedQueries, 101 | }); 102 | 103 | // Create a list of dot markers from dots 104 | const dots = []; 105 | const seenCoordinates = new Set(); // Track coordinates we've already processed 106 | let totalEntries = 0; 107 | for (const result of geokeyResults) { 108 | if (!result.dots) continue; 109 | if (!result.dots[maxZoomLevel]) continue; 110 | // Process each entry at this zoom level 111 | for (const entry of result.dots[maxZoomLevel]) { 112 | totalEntries++; 113 | if (seenCoordinates.has(entry.geokey)) continue; 114 | seenCoordinates.add(entry.geokey); 115 | 116 | // Check if the coordinates are within bounds 117 | if ( 118 | entry.lat >= bounds.minLat && 119 | entry.lat <= bounds.maxLat && 120 | entry.lon >= bounds.minLon && 121 | entry.lon <= bounds.maxLon 122 | ) { 123 | dots.push(entry); 124 | } 125 | } 126 | } 127 | 128 | const entryInfos = geokeyResults.filter((entry) => { 129 | return ( 130 | entry.lat >= bounds.minLat && 131 | entry.lat <= bounds.maxLat && 132 | entry.lon >= bounds.minLon && 133 | entry.lon <= bounds.maxLon 134 | ); 135 | }); 136 | return { entryInfos, dots }; 137 | } 138 | 139 | export async function getEntriesfromText(searchText, mode = "places") { 140 | try { 141 | const response = await fetch(`/query/${mode}-textsearch`, { 142 | method: "POST", 143 | headers: { 144 | "Content-Type": "application/json", 145 | }, 146 | body: JSON.stringify({ searchText }), 147 | }); 148 | 149 | if (!response.ok) { 150 | throw new Error(`Search request failed with status ${response.status}`); 151 | } 152 | 153 | const data = await response.json(); 154 | const entries = data["results"]; 155 | if (mode === "places") { 156 | entries.forEach(addLatLonToEntry); 157 | } 158 | return entries; 159 | } catch (error) { 160 | console.error("Error searching for locations:", error); 161 | return []; 162 | } 163 | } 164 | -------------------------------------------------------------------------------- /website/src/lib/sliding_pane/PageEvents.svelte: -------------------------------------------------------------------------------- 1 | 69 | 70 |
71 |

Events in {wikiPage}

72 | 73 | {#if loadingEvents} 74 |
Loading events...
75 | {:else if Object.keys(eventIdsByYear).length === 0} 76 |
No events found for this page.
77 | {:else} 78 | {#each Object.entries(eventIdsByYear).sort(([yearA], [yearB]) => Number(yearA) - Number(yearB)) as [year, yearEventIds]} 79 |
80 |
toggleYear(year)} 83 | onkeydown={(e) => e.key === "Enter" && toggleYear(year)} 84 | role="button" 85 | tabindex="0" 86 | > 87 |

{year}

88 | {yearEventIds.length} event{yearEventIds.length !== 1 90 | ? "s" 91 | : ""} 93 | {expandedYears[year] ? "▼" : "►"} 94 |
95 | 96 | {#if expandedYears[year]} 97 |
98 | {#each dataLoadedByYear[year] as event} 99 |
100 | 105 |
106 | {/each} 107 |
108 | {/if} 109 |
110 | {/each} 111 | {/if} 112 |
113 | 114 | 196 | -------------------------------------------------------------------------------- /website/src/lib/menu/DropdownMenu.svelte: -------------------------------------------------------------------------------- 1 | 40 | 41 | 42 | 43 |
48 | 72 | 73 | {#if isOpen} 74 |
75 | {#each options as option} 76 | 85 | {/each} 86 |
87 | {/if} 88 |
89 | 90 | 245 | -------------------------------------------------------------------------------- /website/src/lib/map/createMarker.js: -------------------------------------------------------------------------------- 1 | import L from "leaflet"; 2 | // @ts-ignore 3 | import MarkerIcon from "./MarkerIcon.svelte"; 4 | import { uiGlobals } from "../appState.svelte"; 5 | import { mount, unmount } from "svelte"; 6 | import { appState } from "../appState.svelte"; 7 | 8 | const iconSizesByDisplayClass = { 9 | dot: [18, 18], 10 | reduced: [28, 28], 11 | full: [128, 32], 12 | selected: [128, 32], 13 | }; 14 | 15 | // Store component references to properly unmount them 16 | const markerComponents = new WeakMap(); 17 | 18 | /** 19 | * Creates a marker with appropriate behavior based on type 20 | * @param {Object} options - Options object 21 | * @param {Object} options.entry - Normalized marker data 22 | * @returns {L.Marker} - Leaflet marker 23 | */ 24 | export function createMarker({ entry }) { 25 | // No longer need to normalize here as the entry should already be normalized 26 | const { divIcon, markerComponent } = createDivIcon({ 27 | entry, 28 | displayClass: entry.displayClass, 29 | }); 30 | const marker = L.marker([entry.lat, entry.lon], { 31 | icon: divIcon, 32 | pane: entry.displayClass + "MarkersPane", 33 | }); 34 | 35 | // Store the current display class on the marker 36 | // @ts-ignore - adding custom property 37 | marker._currentDisplayClass = entry.displayClass; 38 | 39 | // Store the component reference 40 | markerComponents.set(marker, markerComponent); 41 | 42 | bindHoverEvents({ marker, entry }); 43 | 44 | return marker; 45 | } 46 | 47 | export function updateMarkerIcon({ marker, entry }) { 48 | // Unmount the old component before creating a new one 49 | const oldComponent = markerComponents.get(marker); 50 | if (oldComponent) { 51 | unmount(oldComponent); 52 | } 53 | 54 | const { divIcon, markerComponent } = createDivIcon({ 55 | entry, 56 | displayClass: entry.displayClass, 57 | }); 58 | marker.setIcon(divIcon); 59 | 60 | // Update the stored display class 61 | // @ts-ignore - adding custom property 62 | marker._currentDisplayClass = entry.displayClass; 63 | 64 | // Store the new component reference 65 | markerComponents.set(marker, markerComponent); 66 | } 67 | 68 | export function updateMarkerPane(marker, pane) { 69 | uiGlobals.leafletMap.removeLayer(marker); 70 | marker.options.pane = pane; 71 | marker.addTo(uiGlobals.leafletMap); 72 | } 73 | 74 | export function cleanupMarker(marker) { 75 | // Unmount the Svelte component when the marker is removed 76 | const component = markerComponents.get(marker); 77 | if (component) { 78 | unmount(component); 79 | markerComponents.delete(marker); 80 | } 81 | } 82 | 83 | function createDivIcon({ entry, displayClass }) { 84 | const markerDiv = document.createElement("div"); 85 | const markerComponent = mount(MarkerIcon, { 86 | target: markerDiv, 87 | props: { entry, onClick: () => onClick(entry) }, 88 | }); 89 | 90 | return { 91 | divIcon: L.divIcon({ 92 | className: "custom-div-icon", 93 | html: markerDiv, 94 | iconSize: iconSizesByDisplayClass[displayClass], 95 | }), 96 | markerComponent, 97 | }; 98 | } 99 | 100 | function onClick(entry) { 101 | if (uiGlobals.isTouchDevice) { 102 | selectMarkerAndCenterOnIt({ entry, selectDelay: 350 }); 103 | if (entry.displayClass == "selected") { 104 | appState.wikiSection = entry.page_section; 105 | appState.wikiPage = entry.pageTitle; 106 | appState.paneTab = "wikipedia"; 107 | } 108 | } else { 109 | selectMarkerAndCenterOnIt({ entry, selectDelay: 0 }); 110 | appState.wikiSection = entry.page_section; 111 | appState.wikiPage = entry.pageTitle; 112 | appState.paneTab = "wikipedia"; 113 | } 114 | } 115 | 116 | /** 117 | * Create appropriate popup based on marker type and bind click events 118 | * @param {Object} options - Configuration options 119 | * @param {L.Marker} options.marker - Leaflet marker object 120 | * @param {Object} options.entry - Normalized marker data 121 | */ 122 | function bindHoverEvents({ marker, entry }) { 123 | let isHovered = false; 124 | let unhoverTimeout = null; 125 | 126 | if (!uiGlobals.isTouchDevice) { 127 | marker.on("mouseover", () => { 128 | clearTimeout(unhoverTimeout); 129 | if (isHovered) return; 130 | 131 | // Update icon size directly without creating a new icon 132 | const [width, height] = iconSizesByDisplayClass["full"]; 133 | marker.options.icon.options.iconSize = [width, height]; 134 | 135 | // Force Leaflet to update the icon element 136 | // @ts-ignore - accessing internal Leaflet property 137 | const iconElement = marker._icon; 138 | if (iconElement) { 139 | iconElement.style.width = width + 'px'; 140 | iconElement.style.height = height + 'px'; 141 | iconElement.style.marginLeft = -(width / 2) + 'px'; 142 | iconElement.style.marginTop = -(height / 2) + 'px'; 143 | } 144 | 145 | // Move to top pane 146 | updateMarkerPane(marker, "topPane"); 147 | isHovered = true; 148 | }); 149 | 150 | marker.on("mouseout", () => { 151 | unhoverTimeout = setTimeout(() => { 152 | // Use the current display class stored on the marker 153 | // @ts-ignore - accessing custom property 154 | const currentDisplayClass = marker._currentDisplayClass || 'reduced'; 155 | 156 | // Use the icon size for the current display class 157 | const [width, height] = iconSizesByDisplayClass[currentDisplayClass]; 158 | marker.options.icon.options.iconSize = [width, height]; 159 | 160 | // Force Leaflet to update the icon element 161 | // @ts-ignore - accessing internal Leaflet property 162 | const iconElement = marker._icon; 163 | if (iconElement) { 164 | iconElement.style.width = width + 'px'; 165 | iconElement.style.height = height + 'px'; 166 | iconElement.style.marginLeft = -(width / 2) + 'px'; 167 | iconElement.style.marginTop = -(height / 2) + 'px'; 168 | } 169 | 170 | // Restore original pane 171 | const targetPane = currentDisplayClass + "MarkersPane"; 172 | updateMarkerPane(marker, targetPane); 173 | 174 | isHovered = false; 175 | }, 250); // Increase timeout to be longer than MapPopup's 200ms grace period 176 | }); 177 | } 178 | } 179 | 180 | function selectMarkerAndCenterOnIt({ entry, selectDelay = 0 }) { 181 | setTimeout(() => { 182 | appState.selectedMarkerId = entry.id; 183 | }, selectDelay); 184 | uiGlobals.mapTravel({ 185 | location: { lat: entry.lat, lon: entry.lon }, 186 | flyDuration: 0.3, 187 | }); 188 | } 189 | -------------------------------------------------------------------------------- /website/src/lib/sliding_pane/SameLocationEvents.svelte: -------------------------------------------------------------------------------- 1 | 83 | 84 |
85 |

Events at this location

86 |
87 | Showing events for 88 | 89 | {appState.date.year} 90 | {#if typeof appState.date.month === "number"} 91 | / {appState.date.month} 92 | {#if typeof appState.date.day === "number"} 93 | / {appState.date.day} 94 | {/if} 95 | {/if} 96 | 97 | {#if !appState.strictDate} 98 | and any time range containing this date 101 | {/if} 102 |
103 | 104 | {#if loadingEvents} 105 |
Loading events...
106 | {:else if Object.keys(eventIdsByMonth).length === 0} 107 |
No events found at this location.
108 | {:else} 109 | {#each Object.entries(eventIdsByMonth).sort( ([monthA], [monthB]) => monthA.localeCompare(monthB) ) as [month, monthEventIds]} 110 |
111 |
toggleMonth(month)} 114 | onkeydown={(e) => e.key === "Enter" && toggleMonth(month)} 115 | role="button" 116 | tabindex="0" 117 | > 118 |

{formatMonth(month)}

119 | {monthEventIds.length} event{monthEventIds.length !== 1 121 | ? "s" 122 | : ""} 124 | {expandedMonths[month] ? "▼" : "►"} 125 |
126 | 127 | {#if expandedMonths[month]} 128 |
129 | {#each dataLoadedByMonth[month] as event} 130 |
131 | 136 |
137 | {/each} 138 |
139 | {/if} 140 |
141 | {/each} 142 | {/if} 143 |
144 | 145 | 223 | -------------------------------------------------------------------------------- /worker/src/index.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Welcome to Cloudflare Workers! This is your first worker. 3 | * 4 | * - Run `npm run dev` in your terminal to start a development server 5 | * - Open a browser tab at http://localhost:8787/ to see your worker in action 6 | * - Run `npm run deploy` to publish your worker 7 | * 8 | * Learn more at https://developers.cloudflare.com/workers/ 9 | */ 10 | 11 | export default { 12 | async fetch(request, env, ctx) { 13 | const url = new URL(request.url); 14 | let result, searchText; 15 | switch (url.pathname) { 16 | case '/query/places-by-geokey': 17 | const geokeys = await request.json(); 18 | result = await queryPlacesFromGeokeysByBatch(geokeys, env.geoDB); 19 | return resultsToResponse(result); 20 | case '/query/places-textsearch': 21 | ({ searchText } = await request.json()); 22 | result = await queryPlacesFromText(searchText, env.geoDB); 23 | return resultsToResponse(result); 24 | case '/query/pages-textsearch': 25 | ({ searchText } = await request.json()); 26 | result = await queryPagesFromText(searchText, env.eventsByPageDB); 27 | return resultsToResponse(result); 28 | case '/query/events-by-month-region': 29 | const monthRegions = await request.json(); 30 | result = await queryEventsByMonthRegionByBatch(monthRegions, env.eventsByMonthDB); 31 | return resultsToResponse(result); 32 | case '/query/events-by-id': 33 | const eventIds = await request.json(); 34 | result = await queryEventsByIdByBatch( 35 | eventIds.map((id) => id.replace(' ', '_')), 36 | env.eventsDB 37 | ); 38 | return resultsToResponse(result); 39 | case '/query/events-by-page': 40 | const pageTitles = await request.json(); 41 | result = await queryEventsByPage(pageTitles, env.eventsByPageDB); 42 | return resultsToResponse(result); 43 | default: 44 | return new Response(`Landnotes endpoint ${url.pathname} not found`, { status: 404 }); 45 | } 46 | }, 47 | }; 48 | 49 | function resultsToResponse(results) { 50 | return new Response(JSON.stringify(results), { headers: { 'Content-Type': 'application/json' } }); 51 | } 52 | 53 | async function queryByBatch({ queryFn, params, db, batchSize = 80 }) { 54 | let allResults = { rowsRead: 0, results: [] }; 55 | 56 | if (params.length <= batchSize) { 57 | return await queryFn(params, db); 58 | } else { 59 | const batchPromises = []; 60 | for (let i = 0; i < params.length; i += batchSize) { 61 | const batch = params.slice(i, i + batchSize); 62 | batchPromises.push(queryFn(batch, db)); 63 | } 64 | const batchResults = await Promise.all(batchPromises); 65 | for (const batchResult of batchResults) { 66 | allResults.rowsRead += batchResult.rowsRead; 67 | allResults.results = allResults.results.concat(batchResult.results); 68 | } 69 | return allResults; 70 | } 71 | } 72 | 73 | async function queryPlacesFromGeokeys(geokeys, db) { 74 | const placeholders = geokeys.map(() => '?').join(','); 75 | const stmt = db.prepare(`SELECT * from places WHERE geokey IN (${placeholders})`); 76 | const result = await stmt.bind(...geokeys).all(); 77 | // Cloudflare returns zipped blobs as int arrays which isn't great forJSON, so let's 78 | // convert them to base64 strings 79 | result.results.forEach((entry) => { 80 | const dots = entry.dots; 81 | if (dots) { 82 | entry.dots = btoa(String.fromCharCode(...dots)); 83 | } 84 | }); 85 | return { results: result.results, rowsRead: result.meta.rows_read }; 86 | } 87 | 88 | async function queryPlacesFromGeokeysByBatch(geokeys, geoDB) { 89 | return await queryByBatch({ queryFn: queryPlacesFromGeokeys, params: geokeys, db: geoDB }); 90 | } 91 | 92 | async function queryPlacesFromText(searchText, geoDB) { 93 | const escapedSearchText = searchText.replace(/[-"]/g, (match) => `"${match}"`); 94 | const stmt = geoDB.prepare( 95 | 'SELECT places.* ' + 96 | 'FROM places ' + 97 | 'JOIN (SELECT rowid ' + 98 | ' FROM text_search ' + 99 | ' WHERE text_search MATCH ? ' + 100 | ' LIMIT 10) AS top_matches ' + 101 | 'ON places.rowid = top_matches.rowid ' 102 | ); 103 | return await stmt.bind(escapedSearchText + (escapedSearchText.length > 2 ? '*' : '')).all(); 104 | } 105 | 106 | async function queryPagesFromText(searchText, pagesDB) { 107 | const escapedSearchText = searchText.replace(/[-"]/g, (match) => `"${match}"`); 108 | const stmt = pagesDB.prepare( 109 | 'SELECT pages.page_title, pages.n_events ' + 110 | 'FROM pages ' + 111 | 'JOIN (SELECT rowid ' + 112 | ' FROM text_search ' + 113 | ' WHERE text_search MATCH ? ' + 114 | ' LIMIT 10) AS top_matches ' + 115 | 'ON pages.rowid = top_matches.rowid ' 116 | ); 117 | return await stmt.bind(escapedSearchText + (escapedSearchText.length > 2 ? '*' : '')).all(); 118 | } 119 | 120 | async function queryEventsByMonthRegion(monthRegions, eventsByMonthDB) { 121 | console.log('queryEventsByMonthRegion', monthRegions, eventsByMonthDB); 122 | const placeholders = monthRegions.map(() => '?').join(','); 123 | const stmt = eventsByMonthDB.prepare(`SELECT * from events_by_month_region WHERE month_region IN (${placeholders})`); 124 | const result = await stmt.bind(...monthRegions).all(); 125 | result.results.forEach((entry) => { 126 | entry.zlib_json_blob = arrayBufferToBase64(entry.zlib_json_blob); 127 | }); 128 | return { results: result.results, rowsRead: result.meta.rows_read }; 129 | } 130 | 131 | async function queryEventsByMonthRegionByBatch(monthRegions, eventsByMonthDB) { 132 | return await queryByBatch({ queryFn: queryEventsByMonthRegion, params: monthRegions, db: eventsByMonthDB }); 133 | } 134 | 135 | async function queryEventsById(eventIds, eventsDB) { 136 | const placeholders = eventIds.map(() => '?').join(','); 137 | const stmt = eventsDB.prepare(`SELECT * from events WHERE event_id IN (${placeholders})`); 138 | const result = await stmt.bind(...eventIds).all(); 139 | return { results: result.results, rowsRead: result.meta.rows_read }; 140 | } 141 | 142 | async function queryEventsByIdByBatch(eventIds, eventsDB) { 143 | return await queryByBatch({ queryFn: queryEventsById, params: eventIds, db: eventsDB }); 144 | } 145 | 146 | async function queryEventsByPage(pageTitles, eventsByPageDB) { 147 | const placeholders = pageTitles.map(() => '?').join(','); 148 | const stmt = eventsByPageDB.prepare(`SELECT * from pages WHERE page_title IN (${placeholders})`); 149 | const result = await stmt.bind(...pageTitles).all(); 150 | result.results.forEach((entry) => { 151 | entry.zlib_json_blob = arrayBufferToBase64(entry.zlib_json_blob); 152 | }); 153 | return { results: result.results, rowsRead: result.meta.rows_read }; 154 | } 155 | 156 | function arrayBufferToBase64(buffer) { 157 | const chunkSize = 8192; // Process in manageable chunks 158 | let binary = ''; 159 | for (let i = 0; i < buffer.length; i += chunkSize) { 160 | const chunk = buffer.slice(i, i + chunkSize); 161 | binary += String.fromCharCode.apply(null, chunk); 162 | } 163 | return btoa(binary); 164 | } 165 | -------------------------------------------------------------------------------- /website/src/lib/appState.svelte.js: -------------------------------------------------------------------------------- 1 | /** 2 | * State Management Module 3 | * 4 | * This module manages the central state for the application and handles the 5 | * bidirectional synchronization between the application state and URL parameters. 6 | * 7 | * Key responsibilities: 8 | * - Maintains the global application state using Svelte's reactive state system 9 | * - Provides functions to update the URL based on state changes 10 | * - Parses URL parameters to restore application state 11 | * - Handles browser history integration for navigation 12 | * 13 | * The state includes information about: 14 | * - Current view mode (places/events) 15 | * - Map position and zoom level 16 | * - Selected markers and content 17 | * - Date filters for events mode 18 | * - Other application-wide settings 19 | */ 20 | 21 | import { geohashToLatLon, latLonToGeohash } from "./data/geohash"; 22 | import { 23 | constrainedDate, 24 | dateToUrlString, 25 | parseUrlDate, 26 | } from "./data/date_utils"; 27 | const stateDefaults = { 28 | mode: "places", 29 | strictDate: true, 30 | date: { year: 1949, month: 3, day: "all" }, 31 | zoom: 1, 32 | location: null, 33 | selectedMarkerId: null, 34 | wikiPage: "", 35 | wikiSection: "", 36 | paneTab: "wikipedia", 37 | }; 38 | 39 | export const appState = $state(stateDefaults); 40 | export const uiGlobals = { 41 | isNarrowScreen: false, 42 | leafletMap: null, 43 | mapTravel: null, 44 | isTouchDevice: 45 | typeof window !== "undefined" && 46 | ("ontouchstart" in window || navigator.maxTouchPoints > 0), 47 | isSafariOrFirefox: 48 | typeof window !== "undefined" && 49 | (/^((?!chrome|android).)*safari/i.test(navigator.userAgent) || 50 | /Firefox/i.test(navigator.userAgent)), 51 | }; 52 | export const uiState = $state({ 53 | sameLocationEvents: null, 54 | dataIsLoading: false, 55 | }); 56 | let dontPushToHistory = $state(false); 57 | let currentMode = $state("places"); 58 | 59 | $effect.root(() => { 60 | // When the app state changes in any way, update the URL params 61 | $effect(() => { 62 | const state = $state.snapshot(appState); 63 | debouncedUpdateURLParams(state); 64 | }); 65 | $effect(() => { 66 | if (appState.mode !== currentMode) { 67 | appState.selectedMarkerId = null; 68 | currentMode = appState.mode; 69 | } 70 | }); 71 | }); 72 | 73 | /** 74 | * Updates the URL parameters based on the current application state 75 | * 76 | * This function converts the application state into URL parameters and updates 77 | * the browser's URL without reloading the page. It handles map position, selected 78 | * markers, and event-specific parameters like date and strictDate. 79 | * 80 | * @param {Object} state - The current application state 81 | * @param {Object} [state.location] - The current map location (lat/lon) 82 | * @param {number} [state.zoom] - The current map zoom level 83 | * @param {string} [state.selectedMarkerId] - ID of the currently selected marker 84 | * @param {string} [state.mode] - Current application mode (e.g., "places" or "events") 85 | * @param {Object} [state.date] - Current date selection for events mode 86 | * @param {string} [state.paneTab] - The current pane tab 87 | * @param {boolean} [state.strictDate] - Whether to use strict date matching for events 88 | * @param {string} [state.wikiPage] - The current wiki page 89 | * @param {string} [state.wikiSection] - The current wiki section 90 | * @param {boolean} [addToHistory=true] - Whether to add the state to browser history 91 | * (true = pushState, false = replaceState) 92 | */ 93 | export function updateURLParams(state, addToHistory = true) { 94 | const { 95 | location, 96 | zoom, 97 | selectedMarkerId, 98 | mode, 99 | date, 100 | paneTab, 101 | strictDate, 102 | wikiPage, 103 | wikiSection, 104 | } = state; 105 | const params = new URLSearchParams(); 106 | 107 | // Add map position parameters if they exist 108 | if (location) { 109 | const geohash = latLonToGeohash(location.lat, location.lon, 8); 110 | params.set("location", `${geohash}-${zoom}`); 111 | } 112 | if (selectedMarkerId) { 113 | params.set("selected", selectedMarkerId); 114 | } 115 | if (mode === "events") { 116 | if (date) { 117 | params.set("date", dateToUrlString(date)); 118 | } 119 | if (strictDate) { 120 | params.set("strictDate", strictDate ? "true" : "false"); 121 | } 122 | } 123 | if (wikiPage) { 124 | params.set("wikiPage", wikiPage); 125 | } 126 | if (paneTab && paneTab !== "wikipedia") { 127 | params.set("paneTab", paneTab); 128 | } 129 | if (wikiSection) { 130 | params.set("wikiSection", wikiSection); 131 | } 132 | 133 | // Update URL without reloading the page 134 | const newUrl = `${window.location.pathname}?${params.toString()}`; 135 | 136 | if (params.toString() === window.location.search.slice(1)) { 137 | console.log("Skipping pushstate because the url is the same"); 138 | return; 139 | } 140 | 141 | if (addToHistory) { 142 | window.history.pushState(state, "", newUrl); 143 | } else { 144 | window.history.replaceState(state, "", newUrl); 145 | } 146 | } 147 | 148 | function debounce(func, wait) { 149 | let timeout; 150 | return function (...args) { 151 | clearTimeout(timeout); 152 | timeout = setTimeout(() => func.apply(this, args), wait); 153 | }; 154 | } 155 | function updateURLParamsOnStateChange(appState) { 156 | if (dontPushToHistory) return; 157 | updateURLParams(appState, true); 158 | } 159 | 160 | const debouncedUpdateURLParams = debounce(updateURLParamsOnStateChange, 500); 161 | 162 | // ------------------------------------------ 163 | // Reading the URL params into the app state 164 | // ------------------------------------------ 165 | 166 | export function setStateFromURLParams() { 167 | dontPushToHistory = true; 168 | const urlState = readURLParams(); 169 | if (urlState.date) { 170 | urlState.date = constrainedDate(urlState.date); 171 | } 172 | Object.assign(appState, { ...stateDefaults, ...urlState }); 173 | setTimeout(() => { 174 | dontPushToHistory = false; 175 | }, 3000); 176 | return urlState; 177 | } 178 | 179 | /** 180 | * Reads application state from URL parameters 181 | * 182 | * @returns {Object} - Object containing parsed parameters and selected marker 183 | */ 184 | export function readURLParams() { 185 | const params = new URLSearchParams(window.location.search); 186 | 187 | // Get map position parameters 188 | const result = {}; 189 | const location = params.get("location"); 190 | if (location) { 191 | const [geohash, zoom] = location.split("-"); 192 | result.location = geohashToLatLon(geohash); 193 | result.zoom = parseInt(zoom); 194 | } 195 | result.selectedMarkerId = params.get("selected"); 196 | const date = params.get("date"); 197 | if (date) { 198 | result.mode = "events"; 199 | result.date = parseUrlDate(date); 200 | result.strictDate = params.get("strictDate") === "true"; 201 | } else { 202 | result.mode = "places"; 203 | result; 204 | } 205 | result.wikiPage = params.get("wikiPage"); 206 | result.paneTab = params.get("paneTab") || "wikipedia"; 207 | return result; 208 | } 209 | -------------------------------------------------------------------------------- /docs/displaying_on_the_map.md: -------------------------------------------------------------------------------- 1 | # How places and events are displayed on the map in Landnotes 2 | 3 | Just like in any map website, as a user zooms in on smaller regions, new markers forplaces and events start appearing. The implementation of this mechanism (often referred to as "Levels of Detail") impacts has a big impact on the user experience: 4 | 5 | - It is opinionated: markers that appear at the world level will be shown to more users than the ones that only appear at the level of a city district. We need to ensure that the pages at the top levels are the most interesting ones. 6 | - It is database-intensive: as the users zoom and pan all over the map, we want to make sure the queries made in the background are as efficient as possible, in part to keep the website fluid and reactive, and in greater part because the database provider, CloudFlare, charges per database row scanned. 7 | 8 | This page describes how Levels of Detail works in Landnotes. 9 | 10 | ## Quadrant-based geohashing 11 | 12 | To explain how we make new markers progressively appear as a user zooms in, we have to first talk about geohashing, whereby we attribute a short string to different regions of the world. 13 | 14 | In the most common geocoding method, we first split the world in a grid of 32 regions and each region receives a character (note that some characters like A, I, O are not used for legibility reasons): 15 | 16 |
Geohashing grid
17 | 18 | This division of the world is actually great from a Wikipedia data perspective, where the regions with the most pages in the English wikipedia are the UK, the US East Coast, the USE West Coast, India, Japan... These fall in different squares, which will make it easy to split some of our datasets along these regions without having one dataset being much larger than the others. 19 | 20 | In the standard geohash, the next step is to split each of the 32 regions in 32 smaller regions, for a total of 1024 regions, for instance here is region D, and it's subdivisions D0, D1, ... DZ: 21 | 22 |
Geohashing grid
23 | 24 | By iterating the division process just 8 times, we can get a geohash like `u09tvmqr` which points to a site on the map with a resolution of just a couple meters, a degree of precision largely sufficient for the purposes of Landnotes. 25 | 26 | While the division of each region in 32 smaller regions is neat and efficient, it is not the best to go with map zooming. In the map framework used by Landnotes, LeafletJS, increasing the zoom level by one means halving the latitude and longitude resolution. It is as if we were going one level deeper, not by diving into a 32 times small subregion, but a 4 times smaller subregion. 27 | 28 | Therefore, after the initial 32-region partition, we divide each of the 32 regions using a nesting of quadrants: the region is divided in 4 quadrants, then each quadrant is divided in 4 subquadrants, and so on: 29 | 30 |
Geohashing grid
31 | 32 | There might not be a name for this type of geohashing, so let's call it "geoquashing". It are less concise than its 32-based cousin: achieving a resolution of a few meters now requires to have 17 characters which will look like `D2012302320010232`. On the other hand, they are practical because they follow the zoom level resolution. For instance we could decide that when the zoom level is 4, we will subdivide the screen using 4-length geoquashes (`G1201`, `G1202`, ...) and when the zoom level is 7 we'll look at the content of regions defined by 7-length geoquashes (`G1201322`, `G1201323`...) 33 | 34 | Now the question becomes: what do we show when we show the content of `G1201322`? 35 | 36 | ## Hierarchizing locations 37 | 38 | The different levels of geohash regions can be nicely represented as a tree: 39 | 40 |
Geohashing grid
41 | 42 | Note that in practice, we never really need to encode a tree because geohashes are predictible: it is always obvious what the children of `G12312` are (just add 0, 1, 2 or 3) and who its parent is (remove the last character). But trees are a nice way to represent what's going on. 43 | 44 | Now say that we have ranked the places and events in the world from the most interesting to the least interesting (more on this in the next section), and we want the most interesting ones at the highest zoom level. The way we do this is by pushing the events down the geohash tree, by order of importance, each event claiming the regions at various levels that have not been claimed yet: 45 | 46 |
Geohashing grid
47 | 48 | Once we are done, all subregions at all levels of the tree should have their "best entry": 49 | 50 |
Geohashing grid
51 | 52 | And now we know exactly what to show at each zoom level: 53 | 54 |
Geohashing grid
55 | 56 | Each site (Cap Canaveral, Statue of Liberty, etc) can now be stored in a database with an index corresponding to the highest geohash for which it is the best entry. Cap Canaveral will be indexed as "D", the Statue of liberty as "D2", The white house as "D23", and so on. 57 | 58 | In addition but not covered in details here, through the same tree-pushing algorithm, we also log the "next 10" entries for each node, which we will use to display "dot markers" as small wisual cues of what regions have more sites at lower zoom levels. 59 | 60 | ## Retrieving the entries to display in a given region 61 | 62 | Say a user is currently viewing the region corresponding to latitude [45-55], longitude [-5 - 10], with a zoom level of 5. 63 | It is relatively easy, with an algorithm, to determine all the geoquashes of size 5 or less that overlap with this region. 64 | 65 |
Geohashing grid
66 | 67 | Now all we have to do is to request these ~30 entries to the database, which will be very efficient since the database is already indexed by "highest geoquash" and will typically take less than 50 row scans. 68 | 69 | We can now display the markers for the sites that have been retrieved, along with their corresponding "dot markers" for other sites in the top 10 in each region. 70 | 71 |
Geohashing grid
72 | 73 | ## Ranking places and events 74 | 75 | Which places should be first, and have a marker at the world level, while other places only appear at street level? 76 | 77 | The answer is subjective and depends on goals. Landote tries to foster discovery of interesting topics and well-documented wikipedia pages, and so the criteria is simply "the size of the wikipedia page". 78 | 79 | Initially, Landnotes used the total number of characters in a page as a the score (the wikipedia dumps provide that number for each wikipedia page). But the longest pages often owe their length to a high number of journalistic references, because they cover current events and controversial subjects (conflicts, assassinations, politicians). Having these pages appear first made for a grim world, so the page length is now measured after removing all references (using a regex), which favors pages with actual wikipedia content. 80 | 81 | There could be alternative criteria for page ranking, based for instance on the number of wikipedia pages that link to a given page, similar to how Google used to rank the internet back in the 2000s. 82 | -------------------------------------------------------------------------------- /website/src/lib/map/MarkerIcon.svelte: -------------------------------------------------------------------------------- 1 | 69 | 70 | {#snippet popupContent(isOpen)} 71 | {#if entry.isEvent} 72 | 73 | {:else} 74 | 75 | {/if} 76 | {/snippet} 77 | 78 | 85 |
92 |
93 | icon 94 | {#if entry.isEvent && entry.same_location_events && entry.same_location_events.length > 0} 95 |
96 | +{entry.same_location_events.length} 97 |
98 | {/if} 99 |
100 | 101 |
102 |
{label()}
103 |
{label()}
104 |
105 |
106 | 107 | 108 | 261 | -------------------------------------------------------------------------------- /website/src/lib/menu/DatePicker.svelte: -------------------------------------------------------------------------------- 1 | 97 | 98 |
99 | {#if date.month !== "all"} 100 | 107 | {/if} 108 | 109 | 116 | 117 |
118 | updateDate("year", parseInt(e.currentTarget.value))} 122 | min="-10000" 123 | max="2000" 124 | placeholder="Year" 125 | aria-label="Year" 126 | class="date-input year-input" 127 | /> 128 |
129 | 150 | 171 |
172 |
173 |
174 | 175 | 285 | -------------------------------------------------------------------------------- /website/src/lib/data/mapEntries.svelte.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Map Entries Management Module 3 | * 4 | * This module is responsible for updating the map entries (markers and dots) in response to 5 | * changes in the application state and map bounds. It handles: 6 | * 7 | * - Fetching and displaying place data when in "places" mode 8 | * - Fetching and displaying event data when in "events" mode 9 | * - Maintaining cached data to improve performance 10 | * - Updating marker display classes based on selection state 11 | * - Synchronizing map entries with the current view parameters (zoom, bounds, date filters) 12 | * 13 | * The module uses Svelte's reactive system to automatically update the map when relevant 14 | * state changes occur, such as mode switches, date changes, or map movement. 15 | */ 16 | 17 | import { appState, uiState } from "../appState.svelte"; 18 | import { getPlaceDataFromGeokeys, getGeodataFromBounds } from "./places_data"; 19 | import { getEventsById, getEventsForBoundsAndDate } from "./events_data"; 20 | 21 | export const mapEntries = $state({ markerInfos: [], dots: [] }); 22 | export const mapBounds = $state({}); 23 | 24 | let cachedPlaceData = new Map(); 25 | 26 | $effect.root(() => { 27 | $effect(() => { 28 | handleNewSelectedMarker(appState.selectedMarkerId); 29 | }); 30 | $effect(() => { 31 | const _mapBounds = $state.snapshot(mapBounds); 32 | if (Object.keys(_mapBounds).length === 0) return; 33 | if (appState.mode === "events") { 34 | const { zoom, date, strictDate } = appState; 35 | const _date = $state.snapshot(date); 36 | updateMapEventEntries({ 37 | zoom, 38 | date: _date, 39 | mapBounds: _mapBounds, 40 | strictDate, 41 | }); 42 | } 43 | }); 44 | $effect(() => { 45 | const _mapBounds = $state.snapshot(mapBounds); 46 | if (Object.keys(_mapBounds).length === 0) return; 47 | if (appState.mode === "places") { 48 | const { zoom } = appState; // variables observed + mapBounds 49 | updateMapPlaceEntries({ zoom, mapBounds: _mapBounds }); 50 | } 51 | }); 52 | }); 53 | 54 | /** 55 | * Fetches and updates map markers with geographical data based on the current map bounds and zoom level. 56 | * @param {Object} options - The options object 57 | * @param {Object} options.mapBounds - The current bounds of the map view 58 | * @param {number} options.zoom - The current zoom level of the map 59 | */ 60 | async function updateMapPlaceEntries({ mapBounds, zoom }) { 61 | const willBeLoadingTimeOut = setTimeout(() => { 62 | uiState.dataIsLoading = true; 63 | }, 500); 64 | const { entryInfos, dots } = await getGeodataFromBounds({ 65 | bounds: mapBounds, 66 | maxZoomLevel: zoom - 1, 67 | cachedQueries: cachedPlaceData, 68 | }); 69 | 70 | // Make sure the selected marker is included 71 | if ( 72 | appState.selectedMarkerId && 73 | !entryInfos.some((entry) => entry.geokey === appState.selectedMarkerId) 74 | ) { 75 | const selectedMarker = await getPlaceDataFromGeokeys({ 76 | geokeys: [appState.selectedMarkerId], 77 | cachedQueries: cachedPlaceData, 78 | }); 79 | entryInfos.push(selectedMarker[0]); 80 | } 81 | updateMapEntriesFromQueryResults({ entryInfos, dots }); 82 | clearTimeout(willBeLoadingTimeOut); 83 | uiState.dataIsLoading = false; 84 | } 85 | 86 | /** 87 | * Fetches and updates map markers with event data based on the current map bounds, zoom level, date, and strict date. 88 | * @param {Object} options - The options object 89 | * @param {Object} options.mapBounds - The current bounds of the map view 90 | * @param {number} options.zoom - The current zoom level of the map 91 | * @param {Object} options.date - The current date of the map 92 | * @param {boolean} options.strictDate - Whether to use strict date filtering 93 | * @returns {Promise} - Object containing event information and dots for the map 94 | */ 95 | async function updateMapEventEntries({ mapBounds, zoom, date, strictDate }) { 96 | const willBeLoadingTimeOut = setTimeout(() => { 97 | uiState.dataIsLoading = true; 98 | }, 500); 99 | const { events, dotEvents } = await getEventsForBoundsAndDate({ 100 | date, 101 | bounds: mapBounds, 102 | zoom: zoom - 1, 103 | strictDate, 104 | }); 105 | const eventIds = events.map((event) => event.event_id); 106 | const eventInfos = await getEventsById(eventIds); 107 | // Add type annotation to make it clear this is a Map of objects 108 | const eventInfosById = new Map( 109 | eventInfos.map((eventInfo) => [eventInfo.event_id, eventInfo]) 110 | ); 111 | 112 | const eventsWithInfos = events.map((event) => { 113 | // Cast to object type or use type assertion 114 | const eventInfo = eventInfosById.get(event.event_id) || {}; 115 | return { ...eventInfo, ...event }; 116 | }); 117 | 118 | updateMapEntriesFromQueryResults({ 119 | entryInfos: eventsWithInfos, 120 | dots: dotEvents, 121 | }); 122 | clearTimeout(willBeLoadingTimeOut); 123 | uiState.dataIsLoading = false; 124 | } 125 | 126 | function updateMapEntriesFromQueryResults({ entryInfos, dots }) { 127 | const normalizedInfos = entryInfos.map(normalizeMapEntryInfo); 128 | const preExistingEntries = mapEntries.markerInfos.filter((entry) => 129 | normalizedInfos.some((e) => e.id === entry.id) 130 | ); 131 | const newEntries = normalizedInfos.filter( 132 | (entry) => !mapEntries.markerInfos.some((e) => e.id === entry.id) 133 | ); 134 | const allMarkerInfos = [...preExistingEntries, ...newEntries]; 135 | updateDisplayClasses(allMarkerInfos); 136 | mapEntries.markerInfos = allMarkerInfos; 137 | mapEntries.dots = dots; 138 | } 139 | 140 | function updateDisplayClasses(entries) { 141 | const selectedId = appState.selectedMarkerId; 142 | for (const entry of entries) { 143 | if (selectedId && entry.id == selectedId) { 144 | entry.displayClass = "selected"; 145 | } else if (appState.zoom > 17 || entry.geokey.length <= appState.zoom - 2) { 146 | entry.displayClass = "full"; 147 | } else if (entry.geokey.length <= appState.zoom - 1) { 148 | entry.displayClass = "reduced"; 149 | } else { 150 | entry.displayClass = "dot"; 151 | } 152 | } 153 | } 154 | 155 | async function handleNewSelectedMarker(selectedMarkerId) { 156 | if (selectedMarkerId === appState.selectedMarkerId) return; 157 | if (!selectedMarkerId) { 158 | // a marker got deselected. Let's just update the display classes 159 | updateDisplayClasses(mapEntries.markerInfos); 160 | return; 161 | } 162 | 163 | let query; 164 | if (appState.mode === "places") { 165 | query = await getPlaceDataFromGeokeys({ 166 | geokeys: [selectedMarkerId], 167 | cachedQueries: cachedPlaceData, 168 | }); 169 | } else { 170 | query = await getEventsById([selectedMarkerId]); 171 | } 172 | 173 | const selectedMarker = normalizeMapEntryInfo(query[0]); 174 | appState.wikiSection = selectedMarker.page_section; 175 | appState.wikiPage = selectedMarker.pageTitle; 176 | appState.paneTab = "wikipedia"; 177 | const newMarkers = [...mapEntries.markerInfos]; 178 | if ( 179 | !mapEntries.markerInfos.some((marker) => marker.id === selectedMarkerId) 180 | ) { 181 | newMarkers.push(selectedMarker); 182 | } 183 | updateDisplayClasses(newMarkers); 184 | mapEntries.markerInfos = newMarkers; 185 | } 186 | 187 | /** 188 | * Adapts marker data to a consistent format regardless of type (place or event) 189 | * @param {Object} entry - Original data entry 190 | * @returns {Object} - Normalized marker data 191 | */ 192 | export function normalizeMapEntryInfo(entry) { 193 | // Determine marker type 194 | const isEvent = Boolean(entry.when); 195 | 196 | // Return a normalized object with consistent property names 197 | const pageTitle = entry.page_title 198 | ? entry.page_title.replaceAll("_", " ") 199 | : ""; 200 | return { 201 | ...entry, 202 | id: isEvent ? entry.event_id : entry.geokey, 203 | name: entry.name || pageTitle, 204 | pageTitle, 205 | displayClass: entry.displayClass || "dot", 206 | category: entry.category || "other", 207 | isEvent, 208 | }; 209 | } 210 | -------------------------------------------------------------------------------- /website/src/lib/menu/MenuDropdown.svelte: -------------------------------------------------------------------------------- 1 | 97 | 98 |
99 |
100 | 101 |
102 | Show 103 |
104 | 110 | 116 |
117 |
118 | 119 | 120 | {#if mode === "events"} 121 |
122 | Date filter 123 | 130 |
131 | {/if} 132 | 133 | 134 |
135 | { 138 | if (e.key === "Enter" || e.key === " ") { 139 | handleShareLink(); 140 | } 141 | }} 142 | class="menu-item" 143 | role="button" 144 | tabindex="0" 145 | > 146 | Share the link for the current view 147 | 148 | { 150 | appState.paneTab = "about"; 151 | onCloseMenu(); 152 | }} 153 | onkeydown={(e) => { 154 | if (e.key === "Enter" || e.key === " ") { 155 | appState.paneTab = "about"; 156 | onCloseMenu(); 157 | } 158 | }} 159 | class="menu-item" 160 | role="button" 161 | tabindex="0" 162 | > 163 | About Landnotes 164 | 165 |
166 |
167 |
168 | 169 | 301 | -------------------------------------------------------------------------------- /website/src/lib/data/geohash.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Encodes latitude and longitude coordinates into a standard base-32 geohash 3 | * 4 | * @param {number} lat - Latitude coordinate (-90 to 90) 5 | * @param {number} lon - Longitude coordinate (-180 to 180) 6 | * @param {number} precision - Length of the geohash to generate (default: 9) 7 | * @returns {string} - The geohash string 8 | */ 9 | export function latLonToGeohash(lat, lon, precision = 9) { 10 | if (lat < -90 || lat > 90) { 11 | throw new Error("Latitude must be between -90 and 90"); 12 | } 13 | if (lon < -180 || lon > 180) { 14 | throw new Error("Longitude must be between -180 and 180"); 15 | } 16 | 17 | const base32 = "0123456789bcdefghjkmnpqrstuvwxyz"; 18 | let geohash = ""; 19 | 20 | // Start with full ranges 21 | let latRange = [-90, 90]; 22 | let lonRange = [-180, 180]; 23 | 24 | // Each character encodes 5 bits (2.5 iterations of lat/lon) 25 | let isEven = true; 26 | let bit = 0; 27 | let charIndex = 0; 28 | 29 | while (geohash.length < precision) { 30 | if (isEven) { 31 | // Longitude 32 | const mid = (lonRange[0] + lonRange[1]) / 2; 33 | if (lon >= mid) { 34 | charIndex = (charIndex << 1) + 1; 35 | lonRange[0] = mid; 36 | } else { 37 | charIndex = (charIndex << 1) + 0; 38 | lonRange[1] = mid; 39 | } 40 | } else { 41 | // Latitude 42 | const mid = (latRange[0] + latRange[1]) / 2; 43 | if (lat >= mid) { 44 | charIndex = (charIndex << 1) + 1; 45 | latRange[0] = mid; 46 | } else { 47 | charIndex = (charIndex << 1) + 0; 48 | latRange[1] = mid; 49 | } 50 | } 51 | 52 | isEven = !isEven; 53 | bit++; 54 | 55 | // Every 5 bits, append a character 56 | if (bit === 5) { 57 | geohash += base32.charAt(charIndex); 58 | bit = 0; 59 | charIndex = 0; 60 | } 61 | } 62 | 63 | return geohash; 64 | } 65 | 66 | /** 67 | * Converts a geohash string to latitude and longitude coordinates 68 | * 69 | * @param {string} geohash - The geohash string to decode 70 | * @returns {Object} - Object containing {lat, lon} coordinates of the geohash center 71 | */ 72 | export function geohashToLatLon(geohash) { 73 | if (!geohash || geohash.length === 0) { 74 | throw new Error("Invalid geohash: empty string"); 75 | } 76 | 77 | const base32 = "0123456789bcdefghjkmnpqrstuvwxyz"; 78 | 79 | // Start with full ranges 80 | let latRange = [-90, 90]; 81 | let lonRange = [-180, 180]; 82 | 83 | let isEven = true; // longitude first 84 | 85 | for (let i = 0; i < geohash.length; i++) { 86 | const char = geohash.charAt(i); 87 | const charIndex = base32.indexOf(char); 88 | 89 | if (charIndex === -1) { 90 | throw new Error(`Invalid geohash character: ${char}`); 91 | } 92 | 93 | // Each character encodes 5 bits 94 | for (let bit = 0; bit < 5; bit++) { 95 | const bitValue = (charIndex >> (4 - bit)) & 1; 96 | 97 | if (isEven) { 98 | // Longitude 99 | const mid = (lonRange[0] + lonRange[1]) / 2; 100 | if (bitValue === 1) { 101 | lonRange[0] = mid; 102 | } else { 103 | lonRange[1] = mid; 104 | } 105 | } else { 106 | // Latitude 107 | const mid = (latRange[0] + latRange[1]) / 2; 108 | if (bitValue === 1) { 109 | latRange[0] = mid; 110 | } else { 111 | latRange[1] = mid; 112 | } 113 | } 114 | 115 | isEven = !isEven; 116 | } 117 | } 118 | 119 | // Return the center of the bounding box 120 | const lat = (latRange[0] + latRange[1]) / 2; 121 | const lon = (lonRange[0] + lonRange[1]) / 2; 122 | 123 | return { lat, lon }; 124 | } 125 | 126 | /** 127 | * Decodes a hybrid geohash to its center latitude and longitude coordinates 128 | * 129 | * @param {string} geohash - The hybrid geohash (first character base32, remaining characters base4) 130 | * @returns {Object} - Object containing {lat, lon} center coordinates 131 | */ 132 | export function decodeHybridGeohash(geohash) { 133 | if (!geohash || geohash.length === 0) { 134 | throw new Error("Invalid geohash: empty string"); 135 | } 136 | 137 | const base32 = "0123456789bcdefghjkmnpqrstuvwxyz"; 138 | 139 | // Start with full ranges 140 | let latRange = [-90, 90]; 141 | let lonRange = [-180, 180]; 142 | 143 | // Process the first character (base32) 144 | const firstChar = geohash.charAt(0); 145 | const firstCharIndex = base32.indexOf(firstChar); 146 | 147 | if (firstCharIndex === -1) { 148 | throw new Error(`Invalid base32 character: ${firstChar}`); 149 | } 150 | 151 | // Decode the 5 bits of the base32 character 152 | for (let bit = 0; bit < 5; bit++) { 153 | const bitValue = (firstCharIndex >> (4 - bit)) & 1; 154 | 155 | if (bit % 2 === 0) { 156 | // Even bits encode longitude 157 | const mid = (lonRange[0] + lonRange[1]) / 2; 158 | if (bitValue === 1) { 159 | lonRange[0] = mid; 160 | } else { 161 | lonRange[1] = mid; 162 | } 163 | } else { 164 | // Odd bits encode latitude 165 | const mid = (latRange[0] + latRange[1]) / 2; 166 | if (bitValue === 1) { 167 | latRange[0] = mid; 168 | } else { 169 | latRange[1] = mid; 170 | } 171 | } 172 | } 173 | 174 | // Process remaining characters (base4 quadtree) 175 | for (let i = 1; i < geohash.length; i++) { 176 | const quadrant = parseInt(geohash.charAt(i)); 177 | 178 | if (isNaN(quadrant) || quadrant < 0 || quadrant > 3) { 179 | throw new Error( 180 | `Invalid quadrant character at position ${i}: ${geohash.charAt(i)}` 181 | ); 182 | } 183 | 184 | const midLat = (latRange[0] + latRange[1]) / 2; 185 | const midLon = (lonRange[0] + lonRange[1]) / 2; 186 | 187 | // Adjust ranges based on quadrant 188 | // 0=SW, 1=SE, 2=NW, 3=NE 189 | if (quadrant & 2) { 190 | // North (2 or 3) 191 | latRange[0] = midLat; 192 | } else { 193 | // South (0 or 1) 194 | latRange[1] = midLat; 195 | } 196 | 197 | if (quadrant & 1) { 198 | // East (1 or 3) 199 | lonRange[0] = midLon; 200 | } else { 201 | // West (0 or 2) 202 | lonRange[1] = midLon; 203 | } 204 | } 205 | 206 | // Return the center point 207 | return { 208 | lat: (latRange[0] + latRange[1]) / 2, 209 | lon: (lonRange[0] + lonRange[1]) / 2, 210 | }; 211 | } 212 | 213 | /** 214 | * Returns all hybrid geoencodings of size 5 that overlap with the given bounds 215 | * @param {Object} bounds - The geographic bounds to search within {min_lat, min_lon, max_lat, max_lon} 216 | * @param {number} zoomLevel - The zoom level to search for 217 | * @returns {Array} - Array of hybrid geoencodings (strings) that overlap with the bounds 218 | */ 219 | export function getOverlappingGeoEncodings(bounds, zoomLevel) { 220 | const { minLat, minLon, maxLat, maxLon } = bounds; 221 | const base32 = "0123456789bcdefghjkmnpqrstuvwxyz"; 222 | const results = []; 223 | 224 | // Search through all possible first characters (base32) 225 | for (let i = 0; i < base32.length; i++) { 226 | const char = base32[i]; 227 | const charValue = i; 228 | 229 | // Calculate bounds for the first character (geohash-style) 230 | let latRange = [-90, 90]; 231 | let lonRange = [-180, 180]; 232 | 233 | // Decode the 5 bits of the base32 character 234 | for (let bit = 0; bit < 5; bit++) { 235 | const bitValue = (charValue >> (4 - bit)) & 1; 236 | 237 | if (bit % 2 === 0) { 238 | // Even bits encode longitude 239 | const mid = (lonRange[0] + lonRange[1]) / 2; 240 | if (bitValue === 1) { 241 | lonRange[0] = mid; 242 | } else { 243 | lonRange[1] = mid; 244 | } 245 | } else { 246 | // Odd bits encode latitude 247 | const mid = (latRange[0] + latRange[1]) / 2; 248 | if (bitValue === 1) { 249 | latRange[0] = mid; 250 | } else { 251 | latRange[1] = mid; 252 | } 253 | } 254 | } 255 | 256 | // Check if this first-level cell overlaps with our target bounds 257 | if ( 258 | !( 259 | latRange[1] < minLat || 260 | latRange[0] > maxLat || 261 | lonRange[1] < minLon || 262 | lonRange[0] > maxLon 263 | ) 264 | ) { 265 | // Explore the quadtree (base4) characters 266 | exploreQuadtree(char, latRange, lonRange, 1); 267 | } 268 | } 269 | 270 | // Recursive function to explore quadtree encodings 271 | function exploreQuadtree(prefix, latRange, lonRange, depth) { 272 | // If we've reached depth 5, we have a complete encoding 273 | if (depth === zoomLevel) { 274 | results.push(prefix); 275 | return; 276 | } 277 | 278 | // Try all four quadrants 279 | for (let quadrant = 0; quadrant < 4; quadrant++) { 280 | // Create new ranges for this quadrant 281 | const newLatRange = [...latRange]; 282 | const newLonRange = [...lonRange]; 283 | 284 | const midLat = (newLatRange[0] + newLatRange[1]) / 2; 285 | const midLon = (newLonRange[0] + newLonRange[1]) / 2; 286 | 287 | // Adjust ranges based on quadrant 288 | // 0=SW, 1=SE, 2=NW, 3=NE 289 | if (quadrant & 2) { 290 | // North (2 or 3) 291 | newLatRange[0] = midLat; 292 | } else { 293 | // South (0 or 1) 294 | newLatRange[1] = midLat; 295 | } 296 | 297 | if (quadrant & 1) { 298 | // East (1 or 3) 299 | newLonRange[0] = midLon; 300 | } else { 301 | // West (0 or 2) 302 | newLonRange[1] = midLon; 303 | } 304 | 305 | // Check if this quadrant overlaps with our target bounds 306 | if ( 307 | !( 308 | newLatRange[1] < minLat || 309 | newLatRange[0] > maxLat || 310 | newLonRange[1] < minLon || 311 | newLonRange[0] > maxLon 312 | ) 313 | ) { 314 | // Continue exploring this branch 315 | exploreQuadtree(prefix + quadrant, newLatRange, newLonRange, depth + 1); 316 | } 317 | } 318 | } 319 | 320 | return results; 321 | } 322 | -------------------------------------------------------------------------------- /website/src/lib/map/MapPopup.svelte: -------------------------------------------------------------------------------- 1 | 245 | 246 | {#if portalTarget} 247 |
{ 253 | if (enterable) { 254 | clearTimeout(closeTimeout); 255 | } 256 | }} 257 | onmouseleave={onMouseLeave} 258 | tabindex="-1" 259 | role="tooltip" 260 | > 261 | {@render popupContent(isOpen)} 262 |
263 | {:else} 264 |
{ 269 | if (enterable) { 270 | clearTimeout(closeTimeout); 271 | } 272 | }} 273 | onmouseleave={onMouseLeave} 274 | tabindex="-1" 275 | role="tooltip" 276 | > 277 | {@render popupContent(isOpen)} 278 |
279 | {/if} 280 | 281 | 290 | {@render children()} 291 | 292 | 293 | 309 | -------------------------------------------------------------------------------- /website/src/lib/map/WikiPreview.svelte: -------------------------------------------------------------------------------- 1 | 182 | 183 |
189 | {#if infosFetched} 190 | {#if thumbnail} 191 | {pageTitle} thumbnail 198 | {/if} 199 |
200 |

From Wikipedia

201 |
202 |
203 | {@html summary} 204 |
205 | {#if uiGlobals.isTouchDevice} 206 |
207 | 223 | {/if} 224 | {:else} 225 |
226 |
227 | 228 |
229 |
230 | {/if} 231 |
232 | 233 | 327 | --------------------------------------------------------------------------------