├── .nvmrc ├── .python-version ├── tilecloud_chain ├── py.typed ├── static │ └── admin.css ├── views │ └── __init__.py ├── filter │ ├── __init__.py │ └── error.py ├── tests │ ├── tilegeneration │ │ ├── hooks │ │ │ ├── post-restore-database │ │ │ └── post-create-database │ │ ├── wrong_sequence.yaml │ │ ├── wrong_map.yaml │ │ ├── test-redis-main.yaml │ │ ├── wrong_srs.yaml │ │ ├── wrong_srs_auth.yaml │ │ ├── wrong_srs_id.yaml │ │ ├── deploy.cfg │ │ ├── test-authorised.yaml │ │ ├── wrong_resolutions.yaml │ │ ├── wrong_mapnik_grid_meta.yaml │ │ ├── wrong_type.yaml │ │ ├── test-apache-s3-tilesurl.yaml │ │ ├── test-nodim.yaml │ │ ├── test-copy.yaml │ │ ├── test-capabilities.yaml │ │ ├── test-multi-grid.yaml │ │ ├── test-multidim.yaml │ │ ├── test-redis-project.yaml │ │ ├── test-int-grid.yaml │ │ ├── test-serve-wmtscapabilities.yaml │ │ ├── test-bsddb.yaml │ │ ├── test-serve.yaml │ │ ├── test-multigeom.yaml │ │ ├── test-redis.yaml │ │ ├── test-internal-mapcache.yaml │ │ ├── test-legends.yaml │ │ ├── test-nosns.yaml │ │ ├── test.yaml │ │ └── test-fix.yaml │ ├── index.expected.png │ ├── test.expected.png │ ├── not-login.expected.png │ ├── create_test_data.sh │ ├── apache.conf │ ├── mapfile │ │ ├── circle.svg │ │ ├── mapfile.map │ │ └── test.mapnik │ ├── test_config.py │ ├── test_ui.py │ ├── test_copy.py │ ├── test_postgresql.py │ └── test_expiretiles.py ├── HOST_LIMIT.md ├── format.py ├── host_limit.py ├── host-limit-schema.json ├── security.py ├── timedtilestore.py ├── templates │ └── openlayers.html ├── store │ ├── __init__.py │ ├── azure_storage_blob.py │ ├── url.py │ └── mapnik_.py ├── multitilestore.py ├── copy_.py ├── expiretiles.py └── database_logger.py ├── .secretsignore ├── docker ├── mapfile-docker │ ├── mapserver.conf │ ├── circle.svg │ ├── mapserver.map │ └── test.mapnik ├── test-db │ └── 10_init.sql └── run ├── .prettierignore ├── .sonarcloud.properties ├── .coveragerc ├── Chaines.dia ├── admin-screenshot.png ├── requirements.txt ├── package.json ├── setup.cfg ├── ci ├── requirements.txt └── config.yaml ├── .github ├── spell-ignore-words.txt ├── publish.yaml ├── workflows │ ├── rebuild.yaml │ ├── pull-request-automation.yaml │ └── main.yaml └── renovate.json5 ├── .dockerignore ├── .hadolint.yaml ├── .gitattributes ├── .gitignore ├── .editorconfig ├── SECURITY.md ├── .prospector.yaml ├── jsonschema-gentypes.yaml ├── logging.yaml ├── screenshot.js ├── docker-compose.override.sample.yaml ├── LICENSE ├── Makefile ├── README.md ├── CHANGES.md ├── docker-compose.yaml ├── example └── tilegeneration │ ├── config.yaml │ └── config-postgresql.yaml ├── .pre-commit-config.yaml ├── pyproject.toml └── Dockerfile /.nvmrc: -------------------------------------------------------------------------------- 1 | 18 2 | -------------------------------------------------------------------------------- /.python-version: -------------------------------------------------------------------------------- 1 | 3.13 2 | -------------------------------------------------------------------------------- /tilecloud_chain/py.typed: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tilecloud_chain/static/admin.css: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tilecloud_chain/views/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tilecloud_chain/filter/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.secretsignore: -------------------------------------------------------------------------------- 1 | [secrets] 2 | 1234567890123456789 3 | -------------------------------------------------------------------------------- /docker/mapfile-docker/mapserver.conf: -------------------------------------------------------------------------------- 1 | CONFIG 2 | END 3 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | poetry.lock 2 | tilecloud_chain/CONFIG.md 3 | -------------------------------------------------------------------------------- /.sonarcloud.properties: -------------------------------------------------------------------------------- 1 | sonar.exclusions=tilecloud_chain/tests 2 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | source = tilecloud_chain 3 | omit = */tests/* 4 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/hooks/post-restore-database: -------------------------------------------------------------------------------- 1 | echo SUCCESS 2 | -------------------------------------------------------------------------------- /Chaines.dia: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/camptocamp/tilecloud-chain/HEAD/Chaines.dia -------------------------------------------------------------------------------- /admin-screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/camptocamp/tilecloud-chain/HEAD/admin-screenshot.png -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/hooks/post-create-database: -------------------------------------------------------------------------------- 1 | sudo -u postgresql dropdb tests-deploy 2 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/wrong_sequence.yaml: -------------------------------------------------------------------------------- 1 | grids: 2 | test: 3 | resolutions: test 4 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/wrong_map.yaml: -------------------------------------------------------------------------------- 1 | layers: 2 | test: 3 | empty_tile_detection: test 4 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | poetry==2.2.1 2 | poetry-plugin-export==1.9.0 3 | poetry-dynamic-versioning==1.9.1 4 | pip==25.3 5 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/index.expected.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/camptocamp/tilecloud-chain/HEAD/tilecloud_chain/tests/index.expected.png -------------------------------------------------------------------------------- /tilecloud_chain/tests/test.expected.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/camptocamp/tilecloud-chain/HEAD/tilecloud_chain/tests/test.expected.png -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "devDependencies": { 3 | "commander": "14.0.2", 4 | "puppeteer": "24.27.0" 5 | }, 6 | "type": "module" 7 | } 8 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/not-login.expected.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/camptocamp/tilecloud-chain/HEAD/tilecloud_chain/tests/not-login.expected.png -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/test-redis-main.yaml: -------------------------------------------------------------------------------- 1 | redis: 2 | sentinels: 3 | - - redis_sentinel 4 | - 26379 5 | queue: tilecloud 6 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [nosetests] 2 | match = ^test 3 | where = tilecloud_chain/tests 4 | cover-package = tilecloud_chain 5 | with-coverage = 1 6 | cover-erase = 1 7 | -------------------------------------------------------------------------------- /ci/requirements.txt: -------------------------------------------------------------------------------- 1 | c2cciutils==1.7.5 2 | poetry-dynamic-versioning==1.9.1 3 | poetry-plugin-export==1.9.0 4 | pre-commit==4.3.0 5 | importlib-metadata<8.7.1 6 | tag-publish==1.1.1 7 | -------------------------------------------------------------------------------- /.github/spell-ignore-words.txt: -------------------------------------------------------------------------------- 1 | ro 2 | backend 3 | center 4 | Gio 5 | SQLAlchemy 6 | FreeTileGrid 7 | Proj4 8 | PostgreSQL 9 | Mapnik 10 | Redis 11 | CloudFront 12 | OpenLayers 13 | pyproj 14 | geoms 15 | bbox 16 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/wrong_srs.yaml: -------------------------------------------------------------------------------- 1 | grids: 2 | swissgrid_01: 3 | resolutions: [1] 4 | bbox: [420000, 30000, 900000, 350000] 5 | srs: EPSG21781 6 | 7 | caches: {} 8 | 9 | layers: {} 10 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/wrong_srs_auth.yaml: -------------------------------------------------------------------------------- 1 | grids: 2 | swissgrid_01: 3 | resolutions: [1] 4 | bbox: [420000, 30000, 900000, 350000] 5 | srs: toto:21781 6 | 7 | caches: {} 8 | 9 | layers: {} 10 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/wrong_srs_id.yaml: -------------------------------------------------------------------------------- 1 | grids: 2 | swissgrid_01: 3 | resolutions: [1] 4 | bbox: [420000, 30000, 900000, 350000] 5 | srs: EPSG:21781a 6 | 7 | caches: {} 8 | 9 | layers: {} 10 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | * 2 | !requirements.txt 3 | !poetry.lock 4 | !pyproject.toml 5 | !README.md 6 | !CHANGES.md 7 | !MANIFEST.in 8 | !tilecloud_chain/* 9 | !docker/run 10 | !logging.yaml 11 | !package*.json 12 | !.nvmrc 13 | !screenshot.js 14 | -------------------------------------------------------------------------------- /.hadolint.yaml: -------------------------------------------------------------------------------- 1 | ignored: 2 | - DL3008 # Pin versions in apt get install. Instead of `apt-get install ` use `apt-get install =` 3 | - DL3042 # Avoid use of cache directory with pip. Use `pip install --no-cache-dir ` 4 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/create_test_data.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | createdb -E UTF8 -T template0 tests-deploy 4 | psql -q -d tests-deploy -c "CREATE TABLE test (name varchar(10));" 5 | psql -q -d tests-deploy -c "INSERT INTO test VALUES ('referance');" 6 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | * text whitespace=trailing-space,tab-in-indent,cr-at-eol,tabwidth=4 eol=lf 2 | .gitmodules text whitespace=indent-with-non-tab,tabwidth=2 3 | Makefile text whitespace=indent-with-non-tab,tabwidth=2 4 | *.rst tex conflict-marker-size=100 5 | *.png binary 6 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/deploy.cfg: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | project = test 3 | 4 | [main] 5 | hookdir = %(here)s/hooks 6 | 7 | [files] 8 | active = false 9 | 10 | [databases] 11 | active = true 12 | names = tests-deploy 13 | 14 | [code] 15 | active = false 16 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/test-authorised.yaml: -------------------------------------------------------------------------------- 1 | grids: {} 2 | 3 | caches: {} 4 | 5 | layers: {} 6 | 7 | generation: 8 | authorised_user: www-data 9 | 10 | sns: 11 | topic: arn:aws:sns:eu-west-1:your-account-id:tilecloud 12 | region: eu-west-1 13 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/wrong_resolutions.yaml: -------------------------------------------------------------------------------- 1 | grids: 2 | swissgrid_01: 3 | resolutions: [1, 0.2, 0.1] 4 | resolution_scale: 5 5 | bbox: [420000, 30000, 900000, 350000] 6 | srs: EPSG:21781 7 | 8 | caches: {} 9 | 10 | layers: {} 11 | -------------------------------------------------------------------------------- /.github/publish.yaml: -------------------------------------------------------------------------------- 1 | # yaml-language-server: $schema=https://raw.githubusercontent.com/camptocamp/tag-publish/1.1.1/tag_publish/schema.json 2 | 3 | docker: 4 | images: 5 | - name: camptocamp/tilecloud-chain 6 | pypi: 7 | packages: 8 | - {} 9 | dispatch: 10 | - {} 11 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/apache.conf: -------------------------------------------------------------------------------- 1 | ScriptAlias /mapserv /usr/lib/cgi-bin/mapserv 2 | 3 | SetHandler fcgid-script 4 | SetEnv MS_MAPFILE /home/travis/build/camptocamp/tilecloud-chain/tilecloud_chain/tests/mapfile/mapfile.map 5 | Require all granted 6 | 7 | -------------------------------------------------------------------------------- /docker/mapfile-docker/circle.svg: -------------------------------------------------------------------------------- 1 | 2 | 6 | 11 | 12 | -------------------------------------------------------------------------------- /ci/config.yaml: -------------------------------------------------------------------------------- 1 | # yaml-language-server: $schema=https://raw.githubusercontent.com/camptocamp/c2cciutils/1.7.5/c2cciutils/schema.json 2 | 3 | publish: 4 | docker: 5 | dispatch: {} 6 | images: 7 | - name: camptocamp/tilecloud-chain 8 | pypi: 9 | versions: 10 | - version_tag 11 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/mapfile/circle.svg: -------------------------------------------------------------------------------- 1 | 2 | 6 | 11 | 12 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[co] 2 | /dist 3 | /.build 4 | /.mypy_cache/ 5 | /.github/changelog-generator-cache 6 | /PYPI.md 7 | /tilecloud_chain.egg-info/ 8 | /tilecloud_chain/tests/error.list 9 | /tilecloud_chain/tests/mapcache.xml 10 | /tilecloud_chain/tests/tiles.conf 11 | /node_modules/ 12 | /results/ 13 | /docker-compose.override.yaml 14 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | end_of_line = lf 5 | insert_final_newline = true 6 | charset = utf-8 7 | indent_style = space 8 | indent_size = 4 9 | trim_trailing_whitespace = true 10 | max_line_length = 110 11 | quote_type = single 12 | 13 | [{Makefile,*.mk}] 14 | indent_style = tab 15 | 16 | [*.{json,json5,js,yaml,md,html,whitesource}] 17 | indent_size = 2 18 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Supported Versions 4 | 5 | | Version | Supported Until | 6 | | ------- | --------------- | 7 | | <=1.16 | Unsupported | 8 | | 1.17 | 23/06/2025 | 9 | | 1.18 | Unsupported | 10 | | 1.19 | Unsupported | 11 | | 1.20 | Best effort | 12 | | 1.21 | Best effort | 13 | | 1.22 | Best effort | 14 | | 1.23 | Unsupported | 15 | -------------------------------------------------------------------------------- /.prospector.yaml: -------------------------------------------------------------------------------- 1 | inherits: 2 | - utils:base 3 | - utils:no-design-checks 4 | - utils:fix 5 | - utils:c2cwsgiutils 6 | - duplicated 7 | 8 | ignore-paths: 9 | - tilecloud_chain/configuration.py 10 | 11 | pylint: 12 | disable: 13 | - cyclic-import 14 | 15 | mypy: 16 | options: 17 | python-version: '3.10' 18 | 19 | ruff: 20 | options: 21 | target-version: py310 22 | disable: 23 | - PLC0415 # `import` should be at the top-level of a file (present in pre-commit but not in checks) 24 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/wrong_mapnik_grid_meta.yaml: -------------------------------------------------------------------------------- 1 | grids: 2 | swissgrid_1: 3 | resolutions: [1] 4 | bbox: [420000, 30000, 900000, 350000] 5 | srs: EPSG:21781 6 | 7 | caches: {} 8 | 9 | defaults: 10 | layer: &layer 11 | grid: swissgrid_1 12 | type: mapnik 13 | meta: true 14 | mapfile: test.mapnik 15 | wmts_style: default 16 | 17 | layers: 18 | a: 19 | <<: *layer 20 | output_format: png 21 | extension: png 22 | mime_type: image/png 23 | b: 24 | <<: *layer 25 | output_format: grid 26 | extension: json 27 | mime_type: application/json 28 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/wrong_type.yaml: -------------------------------------------------------------------------------- 1 | grids: 2 | swissgrid_1: 3 | resolutions: [2] 4 | resolution_scale: 5.5 5 | bbox: [a, b, c] 6 | srs: ['EPSG:21781'] 7 | swissgrid_2: 8 | resolutions: [1] 9 | bbox: [123] 10 | srs: {} 11 | swissgrid_3: 12 | srs: epsg:21781 13 | swissgrid_4: 14 | srs: epsg21781 15 | swissgrid_5: {} 16 | swissgrid_6: 17 | swissgrid!: {} 18 | 19 | caches: {} 20 | 21 | layers: 22 | hi!: 23 | wmts_style: yo! 24 | dimensions: 25 | - name: DATE! 26 | default: '2010!' 27 | generate: ['2012!'] 28 | values: ['2005!', '2010!', '2012!'] 29 | - name: time 30 | default: 1 31 | generate: [1] 32 | values: [1] 33 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/test-apache-s3-tilesurl.yaml: -------------------------------------------------------------------------------- 1 | grids: 2 | swissgrid_5: 3 | resolutions: [100, 50, 20, 10, 5] 4 | bbox: [420000, 30000, 900000, 350000] 5 | tile_size: 256 6 | srs: EPSG:21781 7 | 8 | caches: 9 | s3-tilesurl: 10 | type: s3 11 | http_url: https://%(host)s/%(bucket)s/%(folder)s/ 12 | tiles_url: http://tiles.example.com/ 13 | host: s3-eu-west-1.amazonaws.com 14 | bucket: tiles 15 | folder: tiles 16 | 17 | layers: 18 | point: 19 | url: http://mapserver:8080/ 20 | layers: point 21 | min_resolution_seed: 10 22 | grids: [swissgrid_5] 23 | type: wms 24 | wmts_style: default 25 | mime_type: image/png 26 | extension: png 27 | 28 | generation: 29 | default_cache: s3-tilesurl 30 | default_layers: [point] 31 | -------------------------------------------------------------------------------- /tilecloud_chain/HOST_LIMIT.md: -------------------------------------------------------------------------------- 1 | # TileCloud-chain host limit configuration 2 | 3 | _The configuration of the concurrent request limit on a host_ 4 | 5 | ## Properties 6 | 7 | - **`default`** _(object)_ 8 | - **`concurrent`** _(integer)_: Default limit of concurrent request on the same host (can be set with the `TILECLOUD_CHAIN_HOST_CONCURRENT` environment variable. Default: `1`. 9 | - **`hosts`** _(object)_: Can contain additional properties. 10 | - **Additional properties** _(object)_ 11 | - **`concurrent`** _(integer)_: Limit of concurrent request on the host. 12 | 13 | ## Definitions 14 | -------------------------------------------------------------------------------- /jsonschema-gentypes.yaml: -------------------------------------------------------------------------------- 1 | # yaml-language-server: $schema=https://raw.githubusercontent.com/sbrunner/jsonschema-gentypes/2.12.0/jsonschema_gentypes/schema.json 2 | 3 | headers: | 4 | """ 5 | Automatically generated file from a JSON schema. 6 | """ 7 | 8 | pre_commit: 9 | enabled: true 10 | hooks_skip: 11 | - jsonschema-gentypes 12 | - shellcheck 13 | arguments: 14 | - --color=never 15 | 16 | python_version: '3.10' 17 | 18 | generate: 19 | - source: tilecloud_chain/schema.json 20 | destination: tilecloud_chain/configuration.py 21 | root_name: Configuration 22 | api_arguments: 23 | additional_properties: Only explicit 24 | - source: tilecloud_chain/host-limit-schema.json 25 | destination: tilecloud_chain/host_limit.py 26 | root_name: HostLimit 27 | api_arguments: 28 | additional_properties: Only explicit 29 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/test-nodim.yaml: -------------------------------------------------------------------------------- 1 | grids: 2 | swissgrid_5: 3 | resolutions: [100, 50, 20, 10, 5] 4 | bbox: [420000, 30000, 900000, 350000] 5 | tile_size: 256 6 | srs: EPSG:21781 7 | 8 | caches: 9 | s3: 10 | type: s3 11 | http_url: https://%(host)s/%(bucket)s/%(folder)s/ 12 | host: s3-eu-west-1.amazonaws.com 13 | bucket: tiles 14 | folder: tiles 15 | 16 | defaults: 17 | layer: &layer 18 | grids: [swissgrid_5] 19 | type: wms 20 | url: http://mapserver:8080/ 21 | headers: 22 | Host: example.com 23 | Cache-Control: no-cache, no-store 24 | Pragma: no-cache 25 | wmts_style: default 26 | mime_type: image/png 27 | extension: png 28 | meta: true 29 | meta_size: 8 30 | meta_buffer: 128 31 | 32 | layers: 33 | nodim: 34 | <<: *layer 35 | layers: default 36 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/test_config.py: -------------------------------------------------------------------------------- 1 | import os 2 | from pathlib import Path 3 | 4 | from testfixtures import LogCapture 5 | 6 | from tilecloud_chain import controller 7 | from tilecloud_chain.tests import CompareCase 8 | 9 | 10 | class TestConfig(CompareCase): 11 | def setUp(self) -> None: # noqa 12 | self.maxDiff = None 13 | 14 | @classmethod 15 | def setUpClass(cls): # noqa 16 | os.chdir(Path(__file__).parent) 17 | 18 | @classmethod 19 | def tearDownClass(cls): # noqa 20 | os.chdir(Path(__file__).parent.parent.parent) 21 | 22 | def test_int_grid(self) -> None: 23 | with LogCapture("tilecloud_chain") as log_capture: 24 | self.run_cmd( 25 | cmd=".build/venv/bin/generate-controller -c tilegeneration/test-int-grid.yaml --dump-config", 26 | main_func=controller.main, 27 | ) 28 | log_capture.check() 29 | -------------------------------------------------------------------------------- /logging.yaml: -------------------------------------------------------------------------------- 1 | ### 2 | # Logging configuration 3 | # http://docs.pylonsproject.org/projects/pyramid/en/latest/narr/logging.html 4 | ### 5 | 6 | version: 1 7 | disable_existing_loggers: false 8 | 9 | formatters: 10 | generic: 11 | class: logging.Formatter 12 | format: '%(levelname)-5.5s %(name)s %(message)s' 13 | datefmt: '[%Y-%m-%d %H:%M:%S %z]' 14 | 15 | handlers: 16 | console: 17 | class: logging.StreamHandler 18 | formatter: generic 19 | 20 | loggers: 21 | uvicorn.error: 22 | level: INFO 23 | 24 | uvicorn.access: 25 | level: INFO 26 | 27 | sqlalchemy.engine: 28 | # "level = INFO" logs SQL queries. 29 | # "level = DEBUG" logs SQL queries and results. 30 | # "level = WARNING" logs neither. (Recommended for production systems.) 31 | level: INFO 32 | 33 | c2casgiutils: 34 | level: INFO 35 | 36 | tilecloud: 37 | level: INFO 38 | 39 | tilecloud_chain: 40 | level: INFO 41 | 42 | root: 43 | level: WARNING 44 | handlers: 45 | - console 46 | -------------------------------------------------------------------------------- /docker/test-db/10_init.sql: -------------------------------------------------------------------------------- 1 | CREATE EXTENSION postgis; 2 | 3 | CREATE SCHEMA tests; 4 | 5 | CREATE TABLE tests.point (gid serial Primary KEY, name varchar(10)); 6 | SELECT AddGeometryColumn('tests', 'point','the_geom',21781,'POINT',2); 7 | 8 | CREATE TABLE tests.line (gid serial Primary KEY, name varchar(10)); 9 | SELECT AddGeometryColumn('tests', 'line','the_geom',21781,'LINESTRING',2); 10 | 11 | CREATE TABLE tests.polygon (gid serial Primary KEY, name varchar(10)); 12 | SELECT AddGeometryColumn('tests', 'polygon','the_geom',21781,'POLYGON',2); 13 | 14 | 15 | INSERT INTO tests.point VALUES (0, 'point1', ST_GeomFromText('POINT (600000 200000)', 21781)); 16 | INSERT INTO tests.point VALUES (1, 'point2', ST_GeomFromText('POINT (530000 150000)', 21781)); 17 | 18 | INSERT INTO tests.line VALUES (0, 'line1', ST_GeomFromText('LINESTRING (600000 200000,530000 150000)', 21781)); 19 | 20 | INSERT INTO tests.polygon VALUES (0, 'polygon1', ST_GeomFromText('POLYGON ((600000 200000,600000 150000,530000 150000, 530000 200000, 600000 200000))', 21781)); 21 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/test-copy.yaml: -------------------------------------------------------------------------------- 1 | grids: 2 | '21781': 3 | resolutions: [1000] 4 | bbox: [420000, 30000, 900000, 350000] 5 | tile_size: 256 6 | srs: EPSG:21781 7 | 8 | caches: 9 | src: 10 | type: filesystem 11 | folder: /tmp/tiles/src 12 | dst: 13 | type: filesystem 14 | folder: /tmp/tiles/dst 15 | 16 | defaults: 17 | layer: &layer 18 | grids: ['21781'] 19 | type: wms 20 | url: http://mapserver:8080/ 21 | wmts_style: default 22 | mime_type: image/png 23 | extension: png 24 | meta: true 25 | meta_size: 8 26 | meta_buffer: 128 27 | empty_tile_detection: 28 | size: 334 29 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 30 | 31 | layers: 32 | point_hash: 33 | <<: *layer 34 | layers: point 35 | 36 | process: 37 | optipng: 38 | - cmd: optipng %(args)s -zc9 -zm8 -zs3 -f5 -out %(out)s %(in)s 39 | need_out: true 40 | arg: 41 | default: '-q' 42 | quiet: '-q' 43 | debug: '' 44 | verbose: '' 45 | -------------------------------------------------------------------------------- /tilecloud_chain/format.py: -------------------------------------------------------------------------------- 1 | """Format functions.""" 2 | 3 | from datetime import timedelta 4 | 5 | 6 | def default_int(number_array: tuple[float, float, float, float]) -> tuple[int, int, int, int]: 7 | """Convert an array of float in an array of int.""" 8 | return (int(number_array[0]), int(number_array[1]), int(number_array[2]), int(number_array[3])) 9 | 10 | 11 | def size_format(number: float) -> str: 12 | """Get human readable size.""" 13 | for unit in ["o", "Kio", "Mio", "Gio", "Tio"]: 14 | if number < 1024.0: 15 | if number < 10: 16 | return f"{number:.1f} {unit}" 17 | return f"{number:.0f} {unit}" 18 | number /= 1024.0 19 | return f"{number:.0f} Tio" 20 | 21 | 22 | def duration_format(duration: timedelta) -> str: 23 | """Get human readable duration.""" 24 | hours, remainder = divmod(duration.seconds, 3600) 25 | minutes, seconds = divmod(remainder, 60) 26 | if duration.days > 0: 27 | return f"{duration.days} {hours}:{minutes:02d}:{seconds:02d}" 28 | return f"{hours}:{minutes:02d}:{seconds:02d}" 29 | -------------------------------------------------------------------------------- /screenshot.js: -------------------------------------------------------------------------------- 1 | import puppeteer from 'puppeteer'; 2 | import { program } from 'commander'; 3 | 4 | program 5 | .option('--url ', 'The URL') 6 | .option('--output ', 'The output filename') 7 | .option('--width ', 'The page width', 800) 8 | .option('--height ', 'The page height', 600); 9 | 10 | program.parse(); 11 | 12 | const options = program.opts(); 13 | 14 | function delay(ms) { 15 | return new Promise((resolve) => setTimeout(resolve, ms)); 16 | } 17 | 18 | (async () => { 19 | const browser = await puppeteer.launch({ 20 | headless: true, 21 | args: ['--no-sandbox'], 22 | }); 23 | const page = await browser.newPage(); 24 | page.setDefaultNavigationTimeout(10000); 25 | await page.goto(options.url, { timeout: 5000 }); 26 | await delay(1000); 27 | await page.setViewport({ 28 | width: parseInt(options.width), 29 | height: parseInt(options.height), 30 | }); 31 | await page.screenshot({ 32 | path: options.output, 33 | clip: { x: 0, y: 0, width: parseInt(options.width), height: parseInt(options.height) }, 34 | }); 35 | await browser.close(); 36 | })(); 37 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/test-capabilities.yaml: -------------------------------------------------------------------------------- 1 | grids: 2 | swissgrid: 3 | resolutions: [100, 10] 4 | bbox: [420000, 30000, 900000, 350000] 5 | tile_size: 256 6 | srs: EPSG:21781 7 | 8 | caches: 9 | local: 10 | type: filesystem 11 | http_url: http://wmts1/tiles/ 12 | folder: /tmp/tiles 13 | wmtscapabilities_file: 1.0.0/WMTSCapabilities.xml 14 | 15 | defaults: 16 | layer: &layer 17 | url: http://mapserver:8080/ 18 | layers: point 19 | grids: [swissgrid] 20 | type: wms 21 | wmts_style: default 22 | mime_type: image/png 23 | extension: png 24 | 25 | layers: 26 | no_dim: *layer 27 | one: 28 | <<: *layer 29 | dimensions: 30 | - name: DATE 31 | default: '2012' 32 | generate: ['2012'] 33 | values: ['2012'] 34 | two: 35 | <<: *layer 36 | dimensions: 37 | - name: DATE 38 | default: '2012' 39 | generate: ['2012'] 40 | values: ['2012'] 41 | - name: LEVEL 42 | default: '1' 43 | generate: ['1'] 44 | values: ['1', '2'] 45 | 46 | generation: 47 | default_cache: local 48 | -------------------------------------------------------------------------------- /tilecloud_chain/host_limit.py: -------------------------------------------------------------------------------- 1 | """ 2 | Automatically generated file from a JSON schema. 3 | """ 4 | 5 | from typing import TypedDict 6 | 7 | DEFAULT_CONCURRENT_LIMIT_DEFAULT = 1 8 | """ Default value of the field path 'Default values concurrent' """ 9 | 10 | 11 | class DefaultValues(TypedDict, total=False): 12 | """Default values.""" 13 | 14 | concurrent: int 15 | """ 16 | Default concurrent limit. 17 | 18 | Default limit of concurrent request on the same host (can be set with the `TILECLOUD_CHAIN_HOST_CONCURRENT` environment variable. 19 | 20 | default: 1 21 | """ 22 | 23 | 24 | class Host(TypedDict, total=False): 25 | """Host.""" 26 | 27 | concurrent: int 28 | """ 29 | Concurrent limit. 30 | 31 | Limit of concurrent request on the host 32 | """ 33 | 34 | 35 | class HostLimit(TypedDict, total=False): 36 | """ 37 | TileCloud-chain host limit configuration. 38 | 39 | The configuration of the concurrent request limit on a host 40 | """ 41 | 42 | default: "DefaultValues" 43 | """ Default values. """ 44 | 45 | hosts: dict[str, "Host"] 46 | """ Hosts. """ 47 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/test-multi-grid.yaml: -------------------------------------------------------------------------------- 1 | grids: 2 | swissgrid_21781: 3 | resolutions: [1000, 500, 200, 100, 50, 20, 10, 5] 4 | bbox: [420000, 30000, 900000, 350000] 5 | tile_size: 256 6 | srs: EPSG:21781 7 | swissgrid_2056: 8 | resolutions: [1000, 500, 200, 100, 50, 20, 10, 5] 9 | bbox: [2420000, 1030000, 2900000, 1350000] 10 | tile_size: 256 11 | srs: EPSG:2056 12 | 13 | caches: 14 | local: 15 | type: filesystem 16 | http_url: http://wmts1/tiles/ 17 | folder: /tmp/tiles 18 | 19 | defaults: 20 | layer: &layer 21 | type: wms 22 | url: http://mapserver:8080/ 23 | wmts_style: default 24 | mime_type: image/png 25 | extension: png 26 | dimensions: 27 | - name: DATE 28 | default: '2012' 29 | generate: ['2012'] 30 | values: ['2005', '2010', '2012'] 31 | meta: true 32 | meta_size: 2 33 | meta_buffer: 128 34 | 35 | layers: 36 | all: 37 | <<: *layer 38 | layers: point 39 | one: 40 | <<: *layer 41 | layers: point 42 | grids: 43 | - swissgrid_2056 44 | 45 | generation: 46 | default_cache: local 47 | maxconsecutive_errors: 2 48 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/test-multidim.yaml: -------------------------------------------------------------------------------- 1 | grids: 2 | swissgrid: 3 | resolutions: [100, 50, 20] 4 | bbox: [420000, 30000, 900000, 350000] 5 | tile_size: 256 6 | srs: EPSG:21781 7 | 8 | caches: 9 | local: 10 | type: filesystem 11 | http_url: http://wmts1/tiles/ 12 | folder: /tmp/tiles 13 | wmtscapabilities_file: 1.0.0/WMTSCapabilities.xml 14 | 15 | layers: 16 | multi: 17 | url: http://mapserver:8080/ 18 | layers: point_multi 19 | grids: [swissgrid] 20 | type: wms 21 | wmts_style: default 22 | mime_type: image/png 23 | extension: png 24 | meta: true 25 | meta_size: 8 26 | meta_buffer: 128 27 | dimensions: 28 | - name: POINT_NAME 29 | default: point1 30 | generate: [point1, point2] 31 | values: [point1, point2] 32 | geoms: 33 | - sql: the_geom AS geom FROM tests.point 34 | connection: user=postgresql password=postgresql dbname=tests host=db 35 | empty_metatile_detection: 36 | size: 20743 37 | hash: 01062bb3b25dcead792d7824f9a7045f0dd92992 38 | empty_tile_detection: 39 | size: 334 40 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 41 | 42 | generation: 43 | default_cache: local 44 | -------------------------------------------------------------------------------- /docker-compose.override.sample.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | application: &app 3 | command: 4 | - uvicorn 5 | - tilecloud_chain.main:app 6 | - --host=0.0.0.0 7 | - --port=8080 8 | - --log-config=logging.yaml 9 | - --reload 10 | - --reload-dir=/app/ 11 | - --reload-include=tilecloud_chain/**/*.py 12 | - --reload-dir=/venv/lib/python3.12/site-packages/ 13 | - --reload-include=tilecloud/**/*.py 14 | - --reload-include=c2casgiutils/**/*.py 15 | environment: 16 | - DEVELOPMENT=TRUE 17 | volumes: 18 | - ./tilecloud_chain:/app/tilecloud_chain:ro 19 | # - ../tilecloud/tilecloud:/usr/local/lib/python3.10/dist-packages/tilecloud:ro 20 | - ../c2casgiutils/c2casgiutils:/usr/local/lib/python3.10/dist-packages/c2casgiutils:ro 21 | 22 | app_test_user: 23 | <<: *app 24 | 25 | app_postgresql: 26 | volumes: 27 | - ./tilecloud_chain:/app/tilecloud_chain:ro 28 | - ../c2casgiutils/c2casgiutils:/usr/local/lib/python3.10/dist-packages/c2casgiutils:ro 29 | 30 | slave: 31 | volumes: 32 | - ./tilecloud_chain:/app/tilecloud_chain:ro 33 | 34 | # test: 35 | # volumes: 36 | # - ./tilecloud_chain:/app/tilecloud_chain:rw 37 | # - ../tilecloud/tilecloud:/usr/local/lib/python3.10/dist-packages/tilecloud:ro 38 | -------------------------------------------------------------------------------- /tilecloud_chain/host-limit-schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-07/schema", 3 | "$id": "https://raw.githubusercontent.com/camptocamp/tilecloud-chain/master/tilecloud_chain/schema.json", 4 | "type": "object", 5 | "title": "TileCloud-chain host limit configuration", 6 | "description": "The configuration of the concurrent request limit on a host", 7 | "additionalProperties": false, 8 | "definitions": {}, 9 | "properties": { 10 | "default": { 11 | "type": "object", 12 | "title": "Default values", 13 | "properties": { 14 | "concurrent": { 15 | "type": "integer", 16 | "title": "Default concurrent limit", 17 | "description": "Default limit of concurrent request on the same host (can be set with the `TILECLOUD_CHAIN_HOST_CONCURRENT` environment variable.", 18 | "default": 1 19 | } 20 | } 21 | }, 22 | "hosts": { 23 | "type": "object", 24 | "title": "Hosts", 25 | "additionalProperties": { 26 | "type": "object", 27 | "title": "Host", 28 | "properties": { 29 | "concurrent": { 30 | "type": "integer", 31 | "title": "Concurrent limit", 32 | "description": "Limit of concurrent request on the host" 33 | } 34 | } 35 | } 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/test-redis-project.yaml: -------------------------------------------------------------------------------- 1 | grids: 2 | swissgrid_5: 3 | resolutions: [100, 50, 20, 10, 5] 4 | bbox: [420000, 30000, 900000, 350000] 5 | tile_size: 256 6 | srs: EPSG:21781 7 | 8 | caches: 9 | local: 10 | type: filesystem 11 | http_url: http://wmts1/tiles/ 12 | folder: /tmp/tiles 13 | wmtscapabilities_file: 1.0.0/WMTSCapabilities.xml 14 | 15 | defaults: 16 | layer: &layer 17 | grids: [swissgrid_5] 18 | type: wms 19 | url: http://mapserver:8080/ 20 | wmts_style: default 21 | mime_type: image/png 22 | extension: png 23 | dimensions: 24 | - name: DATE 25 | default: '2012' 26 | generate: ['2012'] 27 | values: ['2012'] 28 | meta: true 29 | meta_size: 8 30 | meta_buffer: 128 31 | cost: 32 | # [ms] 33 | tileonly_generation_time: 60 34 | # [ms] 35 | tile_generation_time: 30 36 | # [ms] 37 | metatile_generation_time: 30 38 | # [ko] 39 | tile_size: 20 40 | 41 | layers: 42 | point: 43 | <<: *layer 44 | layers: point 45 | geoms: 46 | - sql: the_geom AS geom FROM tests.point 47 | connection: user=postgresql password=postgresql dbname=tests host=db 48 | min_resolution_seed: 10 49 | 50 | generation: 51 | default_cache: local 52 | default_layers: [point] 53 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012-2025, Camptocamp SA 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | 1. Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 2. Redistributions in binary form must reproduce the above copyright notice, 10 | this list of conditions and the following disclaimer in the documentation 11 | and/or other materials provided with the distribution. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 14 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 15 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 16 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 17 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 18 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 19 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 20 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 21 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 22 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/test-int-grid.yaml: -------------------------------------------------------------------------------- 1 | grids: 2 | '21781': 3 | resolutions: [100, 50, 20, 10, 5] 4 | bbox: [420000, 30000, 900000, 350000] 5 | tile_size: 256 6 | srs: EPSG:21781 7 | 8 | caches: 9 | bsddb: 10 | type: bsddb 11 | http_url: http://wmts1/tiles/ 12 | folder: /tmp/tiles/bsddb 13 | 14 | defaults: 15 | layer: &layer_defaults 16 | grids: ['21781'] 17 | type: wms 18 | url: http://mapserver:8080/ 19 | wmts_style: default 20 | mime_type: image/png 21 | extension: png 22 | dimensions: 23 | - name: DATE 24 | default: '2012' 25 | generate: ['2012'] 26 | values: ['2005', '2010', '2012'] 27 | meta: true 28 | meta_size: 8 29 | meta_buffer: 128 30 | 31 | layers: 32 | point_hash: 33 | <<: *layer_defaults 34 | layers: point 35 | query_layers: point 36 | geoms: 37 | - sql: the_geom AS geom FROM tests.point 38 | connection: user=postgresql password=postgresql dbname=tests host=db 39 | min_resolution_seed: 10 40 | empty_metatile_detection: 41 | size: 20743 42 | hash: 01062bb3b25dcead792d7824f9a7045f0dd92992 43 | empty_tile_detection: 44 | size: 334 45 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 46 | 47 | generation: 48 | default_cache: bsddb 49 | maxconsecutive_errors: 2 50 | 51 | server: {} 52 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/test-serve-wmtscapabilities.yaml: -------------------------------------------------------------------------------- 1 | grids: 2 | swissgrid_5: 3 | resolutions: [100, 50, 20, 10, 5] 4 | bbox: [420000, 30000, 900000, 350000] 5 | tile_size: 256 6 | srs: EPSG:21781 7 | 8 | caches: 9 | local: 10 | type: filesystem 11 | http_url: http://wmts1/tiles/ 12 | folder: /tmp/tiles-ondemend 13 | 14 | defaults: 15 | layer: &layer 16 | type: wms 17 | url: http://mapserver:8080/ 18 | wmts_style: default 19 | mime_type: image/png 20 | extension: png 21 | dimensions: 22 | - name: DATE 23 | default: '2012' 24 | generate: ['2012'] 25 | values: ['2005', '2010', '2012'] 26 | meta: true 27 | meta_size: 8 28 | meta_buffer: 128 29 | 30 | layers: 31 | point_hash: 32 | <<: *layer 33 | layers: point 34 | query_layers: point 35 | geoms: 36 | - sql: the_geom AS geom FROM tests.point 37 | connection: user=postgresql password=postgresql dbname=tests host=db 38 | min_resolution_seed: 10 39 | empty_metatile_detection: 40 | size: 20743 41 | hash: 01062bb3b25dcead792d7824f9a7045f0dd92992 42 | empty_tile_detection: 43 | size: 334 44 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 45 | generation: 46 | default_cache: local 47 | maxconsecutive_errors: 2 48 | 49 | server: 50 | geoms_redirect: true 51 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/test-bsddb.yaml: -------------------------------------------------------------------------------- 1 | grids: 2 | swissgrid_5: 3 | resolutions: [100, 50, 20, 10, 5] 4 | bbox: [420000, 30000, 900000, 350000] 5 | tile_size: 256 6 | srs: EPSG:21781 7 | 8 | caches: 9 | bsddb: 10 | type: bsddb 11 | http_url: http://wmts1/tiles/ 12 | folder: /tmp/tiles/bsddb 13 | wmtscapabilities_file: 1.0.0/WMTSCapabilities.xml 14 | 15 | defaults: 16 | layer: &layer 17 | grids: [swissgrid_5] 18 | type: wms 19 | url: http://mapserver:8080/ 20 | wmts_style: default 21 | mime_type: image/png 22 | extension: png 23 | dimensions: 24 | - name: DATE 25 | default: '2012' 26 | generate: ['2012'] 27 | values: ['2005', '2010', '2012'] 28 | meta: true 29 | meta_size: 8 30 | meta_buffer: 128 31 | 32 | layers: 33 | point_hash: 34 | <<: *layer 35 | layers: point 36 | query_layers: point 37 | geoms: 38 | - sql: the_geom AS geom FROM tests.point 39 | connection: user=postgresql password=postgresql dbname=tests host=db 40 | min_resolution_seed: 10 41 | empty_metatile_detection: 42 | size: 20743 43 | hash: 01062bb3b25dcead792d7824f9a7045f0dd92992 44 | empty_tile_detection: 45 | size: 334 46 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 47 | 48 | generation: 49 | default_cache: bsddb 50 | maxconsecutive_errors: 2 51 | 52 | server: {} 53 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/test_ui.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | from pathlib import Path 3 | 4 | import pytest 5 | import skimage.io 6 | from c2cwsgiutils.acceptance.image import check_image 7 | 8 | REGENERATE = False 9 | 10 | 11 | def test_should_not_commit(): 12 | assert REGENERATE is False 13 | 14 | 15 | @pytest.mark.parametrize( 16 | ("url", "expected_file_name", "height", "width"), 17 | [ 18 | pytest.param("http://application:8080/admin/", "not-login", 250, 800, id="not-login"), 19 | pytest.param("http://application:8080/admin/test", "test", 800, 800, id="test-not-login"), 20 | pytest.param("http://app_test_user:8080/admin", "index", 1250, 1000, id="index"), 21 | pytest.param("http://app_test_user:8080/admin/test", "test", 800, 800, id="test"), 22 | ], 23 | ) 24 | def test_ui(url, expected_file_name, height, width): 25 | subprocess.run( 26 | [ 27 | "node", 28 | "screenshot.js", 29 | f"--url={url}", 30 | f"--width={width}", 31 | f"--height={height}", 32 | f"--output=/tmp/{expected_file_name}.png", 33 | ], 34 | check=True, 35 | ) 36 | check_image( 37 | "/results", 38 | skimage.io.imread(f"/tmp/{expected_file_name}.png")[:, :, :3], 39 | Path(__file__).parent / f"{expected_file_name}.expected.png", 40 | generate_expected_image=REGENERATE, 41 | ) 42 | -------------------------------------------------------------------------------- /.github/workflows/rebuild.yaml: -------------------------------------------------------------------------------- 1 | name: Rebuild 2 | 3 | on: 4 | schedule: 5 | - cron: 30 2 * * * 6 | 7 | jobs: 8 | rebuild: 9 | name: Rebuild 10 | runs-on: ubuntu-24.04 11 | timeout-minutes: 20 12 | 13 | strategy: 14 | fail-fast: false 15 | matrix: 16 | include: 17 | - branch: '1.17' 18 | 19 | env: 20 | REDIS_URL: redis://localhost:6379 21 | 22 | steps: 23 | - run: docker system prune --all --force 24 | - uses: actions/checkout@v5 25 | with: 26 | ref: ${{ matrix.branch }} 27 | 28 | - uses: camptocamp/initialise-gopass-summon-action@v2 29 | with: 30 | ci-gpg-private-key: ${{secrets.CI_GPG_PRIVATE_KEY}} 31 | github-gopass-ci-token: ${{secrets.GOPASS_CI_GITHUB_TOKEN}} 32 | patterns: pypi docker 33 | 34 | - uses: actions/setup-python@v6 35 | with: 36 | python-version: '3.13' 37 | - run: python3 -m pip install setuptools==76.1.0 wheel==0.45.1 38 | - run: python3 -m pip install --user --requirement=ci/requirements.txt 39 | 40 | - name: Checks 41 | run: c2cciutils-checks 42 | 43 | - name: Build 44 | run: make build 45 | 46 | - name: Tests 47 | run: make tests 48 | 49 | - run: c2cciutils-docker-logs 50 | if: always() 51 | 52 | - name: Publish 53 | run: c2cciutils-publish --type=rebuild --branch=${{ matrix.branch }} 54 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/test-serve.yaml: -------------------------------------------------------------------------------- 1 | grids: 2 | swissgrid_5: 3 | resolutions: [100, 50, 20, 10, 5] 4 | bbox: [420000, 30000, 900000, 350000] 5 | tile_size: 256 6 | srs: EPSG:21781 7 | 8 | caches: 9 | local: 10 | type: filesystem 11 | http_url: http://wmts1/tiles/ 12 | folder: /tmp/tiles 13 | mbtiles: 14 | type: mbtiles 15 | http_url: http://wmts1/tiles/ 16 | folder: /tmp/tiles/mbtiles 17 | 18 | defaults: 19 | layer: &layer 20 | grids: [swissgrid_5] 21 | type: wms 22 | url: http://mapserver:8080/ 23 | wmts_style: default 24 | mime_type: image/png 25 | extension: png 26 | dimensions: 27 | - name: DATE 28 | default: '2012' 29 | generate: ['2012'] 30 | values: ['2005', '2010', '2012'] 31 | meta: true 32 | meta_size: 8 33 | meta_buffer: 128 34 | 35 | layers: 36 | point_hash: 37 | <<: *layer 38 | layers: point 39 | query_layers: point 40 | geoms: 41 | - sql: the_geom AS geom FROM tests.point 42 | connection: user=postgresql password=postgresql dbname=tests host=db 43 | min_resolution_seed: 10 44 | empty_metatile_detection: 45 | size: 20743 46 | hash: 01062bb3b25dcead792d7824f9a7045f0dd92992 47 | empty_tile_detection: 48 | size: 334 49 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 50 | generation: 51 | default_cache: mbtiles 52 | maxconsecutive_errors: 2 53 | 54 | server: 55 | geoms_redirect: true 56 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/test-multigeom.yaml: -------------------------------------------------------------------------------- 1 | grids: 2 | swissgrid_5: 3 | resolutions: [100, 50, 20, 10] 4 | bbox: [420000, 30000, 900000, 350000] 5 | tile_size: 256 6 | srs: EPSG:21781 7 | 8 | caches: 9 | local: 10 | type: filesystem 11 | http_url: http://wmts1/tiles/ 12 | folder: /tmp/tiles 13 | wmtscapabilities_file: 1.0.0/WMTSCapabilities.xml 14 | 15 | defaults: 16 | layer: &layer 17 | grids: [swissgrid_5] 18 | type: wms 19 | url: http://mapserver:8080/ 20 | wmts_style: default 21 | mime_type: image/png 22 | extension: png 23 | dimensions: 24 | - name: DATE 25 | default: '2012' 26 | generate: ['2012'] 27 | values: ['2005', '2010', '2012'] 28 | meta: false 29 | meta_size: 8 30 | meta_buffer: 128 31 | empty_metatile_detection: 32 | size: 20743 33 | hash: 01062bb3b25dcead792d7824f9a7045f0dd92992 34 | empty_tile_detection: 35 | size: 334 36 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 37 | 38 | layers: 39 | pp: 40 | <<: *layer 41 | layers: point,polygon 42 | geoms: 43 | - sql: the_geom AS geom FROM tests.polygon 44 | connection: user=postgresql password=postgresql dbname=tests host=db 45 | - sql: the_geom AS geom FROM tests.point 46 | connection: user=postgresql password=postgresql dbname=tests host=db 47 | min_resolution: 10 48 | max_resolution: 20 49 | 50 | generation: 51 | default_cache: local 52 | maxconsecutive_errors: 2 53 | -------------------------------------------------------------------------------- /docker/run: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import os 4 | import stat 5 | import subprocess # nosec 6 | import sys 7 | from pathlib import Path 8 | 9 | if "GROUP_ID" not in os.environ: 10 | sys.exit("The GROUP_ID environment variable is required") 11 | 12 | if "USER_ID" not in os.environ: 13 | sys.exit("The USER_ID environment variable is required") 14 | 15 | if "USER_NAME" not in os.environ: 16 | sys.exit("The USER_NAME environment variable is required") 17 | 18 | if "UMASK" not in os.environ: 19 | sys.exit("The UMASK environment variable is required") 20 | 21 | subprocess.check_call(["groupadd", "-g", os.environ["GROUP_ID"], "geomapfish"]) # noqa: S603, S607 22 | subprocess.check_call( # noqa: S603 # nosec 23 | [ # noqa: S607 24 | "useradd", 25 | "--shell", 26 | "/bin/bash", 27 | "--uid", 28 | os.environ["USER_ID"], 29 | "--gid", 30 | os.environ["GROUP_ID"], 31 | os.environ["USER_NAME"], 32 | ], 33 | ) 34 | 35 | run_file_name = Path("/tmp/run") # noqa: S108 36 | with run_file_name.open("w", encoding="utf-8") as run_file: 37 | run_file.write("#!/usr/bin/python\n") 38 | run_file.write("import subprocess, os\n") 39 | run_file.write(f"os.umask(0o{os.environ['UMASK']})\n") 40 | run_file.write(f"subprocess.check_call({sys.argv[1:]!r})\n") 41 | 42 | run_file_name.chmod( 43 | stat.S_IRUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH, 44 | ) 45 | subprocess.check_call(["su", os.environ["USER_NAME"], "-c", run_file_name]) # noqa: S603, S607 46 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/test-redis.yaml: -------------------------------------------------------------------------------- 1 | grids: 2 | swissgrid_5: 3 | resolutions: [100, 50, 20, 10, 5] 4 | bbox: [420000, 30000, 900000, 350000] 5 | tile_size: 256 6 | srs: EPSG:21781 7 | 8 | caches: 9 | local: 10 | type: filesystem 11 | http_url: http://wmts1/tiles/ 12 | folder: /tmp/tiles 13 | wmtscapabilities_file: 1.0.0/WMTSCapabilities.xml 14 | 15 | defaults: 16 | layer: &layer 17 | grids: [swissgrid_5] 18 | type: wms 19 | url: http://mapserver:8080/ 20 | wmts_style: default 21 | mime_type: image/png 22 | extension: png 23 | dimensions: 24 | - name: DATE 25 | default: '2012' 26 | generate: ['2012'] 27 | values: ['2012'] 28 | meta: true 29 | meta_size: 8 30 | meta_buffer: 128 31 | cost: 32 | # [ms] 33 | tileonly_generation_time: 60 34 | # [ms] 35 | tile_generation_time: 30 36 | # [ms] 37 | metatile_generation_time: 30 38 | # [ko] 39 | tile_size: 20 40 | 41 | layers: 42 | point: 43 | <<: *layer 44 | layers: point 45 | geoms: 46 | - sql: the_geom AS geom FROM tests.point 47 | connection: user=postgresql password=postgresql dbname=tests host=db 48 | min_resolution_seed: 10 49 | generation: 50 | default_cache: local 51 | default_layers: [point] 52 | maxconsecutive_errors: 2 53 | error_file: error.list 54 | 55 | cost: 56 | # [nb/month] 57 | request_per_layers: 10000000 58 | 59 | redis: 60 | sentinels: 61 | - - redis_sentinel 62 | - 26379 63 | queue: tilecloud 64 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/test-internal-mapcache.yaml: -------------------------------------------------------------------------------- 1 | grids: 2 | swissgrid_5: 3 | resolutions: [100, 50, 20, 10, 5] 4 | bbox: [420000, 30000, 900000, 350000] 5 | tile_size: 256 6 | srs: EPSG:21781 7 | 8 | caches: 9 | local: 10 | type: filesystem 11 | http_url: http://wmts1/tiles/ 12 | folder: /tmp/tiles 13 | wmtscapabilities_file: 1.0.0/WMTSCapabilities.xml 14 | 15 | defaults: 16 | layer: &layer 17 | grids: [swissgrid_5] 18 | type: wms 19 | url: http://mapserver:8080/ 20 | wmts_style: default 21 | mime_type: image/png 22 | extension: png 23 | dimensions: 24 | - name: DATE 25 | default: '2012' 26 | generate: ['2012'] 27 | values: ['2005', '2010', '2012'] 28 | meta: true 29 | meta_size: 8 30 | meta_buffer: 128 31 | cost: 32 | # [ms] 33 | tileonly_generation_time: 60 34 | # [ms] 35 | tile_generation_time: 30 36 | # [ms] 37 | metatile_generation_time: 30 38 | # [ko] 39 | tile_size: 20 40 | 41 | layers: 42 | point: 43 | <<: *layer 44 | layers: point 45 | geoms: 46 | - sql: the_geom AS geom FROM tests.point 47 | connection: user=postgresql password=postgresql dbname=tests host=db 48 | min_resolution_seed: 100 49 | 50 | generation: 51 | default_cache: local 52 | default_layers: [point] 53 | maxconsecutive_errors: 2 54 | 55 | openlayers: 56 | srs: EPSG:21781 57 | center_x: 600000 58 | center_y: 200000 59 | 60 | cost: 61 | # [nb/month] 62 | request_per_layers: 10000000 63 | 64 | mapcache: 65 | config_file: mapcache.xml 66 | memcache_host: memcached 67 | 68 | server: 69 | geoms_redirect: true 70 | mapcache_base: http://mapcache:8080/ 71 | 72 | redis: 73 | sentinels: 74 | - - redis_sentinel 75 | - 26379 76 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | export DOCKER_BUILDKIT=1 2 | VERSION = $(strip $(shell poetry version --short)) 3 | 4 | .PHONY: help 5 | help: ## Display this help message 6 | @echo "Usage: make " 7 | @echo 8 | @echo "Available targets:" 9 | @grep --extended-regexp --no-filename '^[a-zA-Z_-]+:.*## ' $(MAKEFILE_LIST) | sort | \ 10 | awk 'BEGIN {FS = ":.*?## "}; {printf " %-20s%s\n", $$1, $$2}' 11 | 12 | .PHONY: build 13 | build: ## Build all Docker images 14 | docker build --tag=camptocamp/tilecloud-chain-tests --target=tests . 15 | docker build --tag=camptocamp/tilecloud-chain --build-arg=VERSION=$(VERSION) . 16 | 17 | .PHONY: checks 18 | checks: prospector ## Run the checks 19 | 20 | .PHONY: prospector 21 | prospector: ## Run Prospector 22 | docker run --rm --volume=${PWD}:/app camptocamp/tilecloud-chain-tests prospector --output-format=pylint --die-on-tool-error 23 | 24 | .PHONY: tests 25 | tests: build ## Run the unit tests 26 | docker compose stop --timeout=0 27 | docker compose down || true 28 | docker compose up -d 29 | 30 | # Wait for DB to be up 31 | while ! docker compose exec -T test psql -h db -p 5432 -U postgresql -v ON_ERROR_STOP=1 -c "SELECT 1" -d tests; \ 32 | do \ 33 | echo "Waiting for DB to be UP"; \ 34 | sleep 1; \ 35 | done 36 | 37 | c2cciutils-docker-logs 38 | 39 | docker compose exec -T test pytest -vvv --color=yes ${PYTEST_ARGS} 40 | 41 | c2cciutils-docker-logs 42 | docker compose down 43 | 44 | PYTEST_ARGS ?= --last-failed --full-trace 45 | 46 | .PHONY: tests-fast 47 | tests-fast: 48 | docker compose up -d 49 | 50 | # Wait for DB to be up 51 | while ! docker compose exec -T test psql -h db -p 5432 -U postgresql -v ON_ERROR_STOP=1 -c "SELECT 1" -d tests; \ 52 | do \ 53 | echo "Waiting for DB to be UP"; \ 54 | sleep 1; \ 55 | done 56 | 57 | docker compose exec -T test pytest -vvv --color=yes $(PYTEST_ARGS) 58 | -------------------------------------------------------------------------------- /.github/workflows/pull-request-automation.yaml: -------------------------------------------------------------------------------- 1 | name: Auto reviews, merge and close pull requests 2 | 3 | on: 4 | pull_request: 5 | types: 6 | - opened 7 | - reopened 8 | jobs: 9 | auto-merge: 10 | name: Auto reviews pull requests from bots 11 | runs-on: ubuntu-22.04 12 | timeout-minutes: 5 13 | 14 | permissions: 15 | pull-requests: write 16 | steps: 17 | - name: Print event 18 | run: echo "${GITHUB}" | jq 19 | env: 20 | GITHUB: ${{ toJson(github) }} 21 | - name: Print context 22 | uses: actions/github-script@v8 23 | with: 24 | script: |- 25 | console.log(context); 26 | - name: Auto reviews GHCI updates 27 | uses: actions/github-script@v8 28 | if: |- 29 | startsWith(github.head_ref, 'ghci/audit/') 30 | && (github.event.pull_request.user.login == 'geo-ghci-test[bot]' 31 | || github.event.pull_request.user.login == 'geo-ghci-int[bot]' 32 | || github.event.pull_request.user.login == 'geo-ghci[bot]') 33 | with: 34 | script: |- 35 | github.rest.pulls.createReview({ 36 | owner: context.repo.owner, 37 | repo: context.repo.repo, 38 | pull_number: context.payload.pull_request.number, 39 | event: 'APPROVE', 40 | }) 41 | - name: Auto reviews Renovate updates 42 | uses: actions/github-script@v8 43 | if: |- 44 | github.event.pull_request.user.login == 'renovate[bot]' 45 | with: 46 | script: |- 47 | github.rest.pulls.createReview({ 48 | owner: context.repo.owner, 49 | repo: context.repo.repo, 50 | pull_number: context.payload.pull_request.number, 51 | event: 'APPROVE', 52 | }) 53 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # TileCloud-chain 2 | 3 | TileCloud Chain is a comprehensive toolset for managing tile generation workflows. It supports various source and destination formats, making it a versatile solution for map tile management. 4 | 5 | ## Sources 6 | 7 | - Web Map Service (WMS) 8 | - Mapnik rendering engine 9 | 10 | ## Destination Formats and Storage 11 | 12 | - Web Map Tile Service (WMTS) layout 13 | - Amazon S3 storage 14 | - Azure Blob storage 15 | - Local filesystem 16 | 17 | ## Key Features 18 | 19 | - Tile generation with configurable parameters 20 | - Automatic removal of empty tiles 21 | - Geographic filtering (bbox and geometry-based) 22 | - MetaTile support for efficient generation 23 | - Legend image generation 24 | - GetCapabilities document 25 | - OpenLayers demo page 26 | - Empty tile detection via hashing 27 | - Cache synchronization 28 | - Post-processing capabilities 29 | 30 | ## Legacy Support 31 | 32 | Note: The following features are maintained for backward compatibility: 33 | 34 | - Berkeley DB integration 35 | - SQLite (MBTiles) support 36 | - Mapnik rendering (Python 3 update pending) 37 | 38 | ## Visual Preview 39 | 40 | The admin interface with PostgreSQL queue integration: 41 | 42 | ![TileCloud Chain Admin Interface](./admin-screenshot.png) 43 | 44 | ## Getting Started 45 | 46 | Create a configuration file at `tilegeneration/config.yaml`. 47 | 48 | Reference the [example configuration](https://github.com/camptocamp/tilecloud-chain/blob/master/example/tilegeneration/config.yaml). 49 | 50 | ## Support Policy 51 | 52 | Only the latest release receives active support. Versions prior to 1.11 contain security vulnerabilities and should not be used. 53 | 54 | ## Development 55 | 56 | ### Building 57 | 58 | ```bash 59 | make build 60 | ``` 61 | 62 | ### Quality Assurance 63 | 64 | ```bash 65 | make prospector 66 | ``` 67 | 68 | ### Testing 69 | 70 | ```bash 71 | make tests 72 | ``` 73 | 74 | ## Documentation 75 | 76 | - [Usage Guide](https://github.com/camptocamp/tilecloud-chain/blob/master/tilecloud_chain/USAGE.rst) 77 | - [Configuration Reference](https://github.com/camptocamp/tilecloud-chain/blob/master/tilecloud_chain/CONFIG.md) 78 | 79 | ## Contributing 80 | 81 | Set up pre-commit hooks: 82 | 83 | ```bash 84 | pip install pre-commit 85 | pre-commit install --allow-missing-config 86 | ``` 87 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/test-legends.yaml: -------------------------------------------------------------------------------- 1 | grids: 2 | swissgrid: 3 | resolutions: [100, 50, 20, 10, 5] 4 | bbox: [420000, 30000, 900000, 350000] 5 | tile_size: 256 6 | srs: EPSG:21781 7 | 8 | caches: 9 | local: 10 | type: filesystem 11 | http_url: http://wmts1/tiles/ 12 | folder: /tmp/tiles 13 | wmtscapabilities_file: 1.0.0/WMTSCapabilities.xml 14 | 15 | defaults: 16 | layer: &layer 17 | grids: [swissgrid] 18 | type: wms 19 | legend_mime: image/png 20 | legend_extension: png 21 | wmts_style: default 22 | mime_type: image/png 23 | extension: png 24 | dimensions: 25 | - name: DATE 26 | default: '2012' 27 | generate: ['2012'] 28 | values: ['2005', '2010', '2012'] 29 | meta: true 30 | meta_size: 8 31 | meta_buffer: 128 32 | cost: 33 | # [ms] 34 | tileonly_generation_time: 60 35 | # [ms] 36 | tile_generation_time: 30 37 | # [ms] 38 | metatile_generation_time: 30 39 | # [ko] 40 | tile_size: 20 41 | 42 | layers: 43 | point: 44 | <<: *layer 45 | url: http://mapserver:8080/ 46 | layers: point 47 | geoms: 48 | - sql: the_geom AS geom FROM tests.point 49 | connection: user=postgresql password=postgresql dbname=tests host=db 50 | min_resolution_seed: 10 51 | line: 52 | <<: *layer 53 | url: http://mapserver:8080/ 54 | layers: line 55 | headers: 56 | Cache-Control: no-cache 57 | params: 58 | PARAM: value 59 | geoms: 60 | - sql: the_geom AS geom FROM tests.line 61 | connection: user=postgresql password=postgresql dbname=tests host=db 62 | empty_metatile_detection: 63 | size: 20743 64 | hash: 01062bb3b25dcead792d7824f9a7045f0dd92992 65 | empty_tile_detection: 66 | size: 334 67 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 68 | polygon: 69 | <<: *layer 70 | url: http://mapserver:8080/ 71 | layers: polygon 72 | meta: false 73 | geoms: 74 | - sql: the_geom AS geom FROM tests.polygon 75 | connection: user=postgresql password=postgresql dbname=tests host=db 76 | empty_metatile_detection: 77 | size: 20743 78 | hash: 01062bb3b25dcead792d7824f9a7045f0dd92992 79 | empty_tile_detection: 80 | size: 334 81 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 82 | all: 83 | <<: *layer 84 | url: http://mapserver:8080/ 85 | layers: point,line,polygon 86 | meta: false 87 | bbox: [550000.0, 170000.0, 560000.0, 180000.0] 88 | 89 | generation: 90 | default_cache: local 91 | 92 | sqs: 93 | queue: sqs_point 94 | -------------------------------------------------------------------------------- /.github/renovate.json5: -------------------------------------------------------------------------------- 1 | { 2 | $schema: 'https://docs.renovatebot.com/renovate-schema.json', 3 | extends: [ 4 | 'github>camptocamp/gs-renovate-config-preset:base.json5#1.5.0', 5 | 'github>camptocamp/gs-renovate-config-preset:group.json5#1.5.0', 6 | 'github>camptocamp/gs-renovate-config-preset:ci.json5#1.5.0', 7 | 'github>camptocamp/gs-renovate-config-preset:preset.json5#1.5.0', 8 | 'github>camptocamp/gs-renovate-config-preset:pre-commit.json5#1.5.0', 9 | 'github>camptocamp/gs-renovate-config-preset:python.json5#1.5.0', 10 | 'github>camptocamp/gs-renovate-config-preset:docker.json5#1.5.0', 11 | 'github>camptocamp/gs-renovate-config-preset:json-schema.json5#1.5.0', 12 | 'github>camptocamp/gs-renovate-config-preset:shellcheck.json5#1.5.0', 13 | 'github>camptocamp/gs-renovate-config-preset:stabilization-branches.json5#1.5.0', 14 | 'github>camptocamp/gs-renovate-config-preset:own.json5#1.5.0', 15 | 'github>camptocamp/gs-renovate-config-preset:security.json5#1.5.0', 16 | ], 17 | baseBranchPatterns: ['1.17', '1.20', '1.21', '1.22', '1.23', 'master'], 18 | customManagers: [ 19 | /** Manage unpkg */ 20 | { 21 | matchStrings: ['unpkg\\.com/(?[^@]+)@(?[^/]+)'], 22 | datasourceTemplate: 'npm', 23 | customType: 'regex', 24 | managerFilePatterns: ['/.*\\.html$/', '/.*\\.py$/'], 25 | }, 26 | /** Manage jsdelivr */ 27 | { 28 | matchStrings: ['cdn\\.jsdelivr\\.net/npm/(?[^@]+)@(?[^/]+)'], 29 | datasourceTemplate: 'npm', 30 | customType: 'regex', 31 | managerFilePatterns: ['/.*\\.html$/', '/.*\\.py$/'], 32 | }, 33 | ], 34 | packageRules: [ 35 | /** Docker images versioning */ 36 | { 37 | matchDatasources: ['docker'], 38 | matchDepNames: ['camptocamp/mapserver'], 39 | versioning: 'regex:^(?\\d+)\\.(?\\d+)$', 40 | }, 41 | { 42 | matchDatasources: ['docker'], 43 | matchDepNames: ['ghcr.io/osgeo/gdal'], 44 | versioning: 'regex:^(?.*)-(?\\d+)\\.(?\\d+)\\.(?\\d+)?$', 45 | }, 46 | /** Disable types-request update on version <= 1.21 */ 47 | { 48 | matchDepNames: ['types-requests'], 49 | matchBaseBranches: ['1.17', '1.18', '1.19', '1.20', '1.21'], 50 | enabled: false, 51 | }, 52 | /** Ungroup Gdal */ 53 | { 54 | matchDepNames: ['ghcr.io/osgeo/gdal'], 55 | groupName: 'Gdal', 56 | }, 57 | /** Python should be x.y */ 58 | { 59 | matchDepNames: ['python'], 60 | matchFileNames: ['.python-version$'], 61 | versioning: 'regex:^(?\\d+)\\.(?\\d+)$', 62 | }, 63 | ], 64 | } 65 | -------------------------------------------------------------------------------- /tilecloud_chain/filter/error.py: -------------------------------------------------------------------------------- 1 | """Module includes filters for dealing with errors in tiles.""" 2 | 3 | from tilecloud import Tile 4 | 5 | 6 | class MaximumConsecutiveErrors: 7 | """ 8 | Create a filter that limit the consecutive errors. 9 | 10 | Raises a :class:`TooManyErrors` exception when there are ``max_consecutive_errors`` 11 | consecutive errors. 12 | 13 | max_consecutive_errors: 14 | The max number of permitted consecutive errors. Once 15 | exceeded a :class:`TooManyErrors` exception is raised. 16 | """ 17 | 18 | def __init__(self, max_consecutive_errors: int) -> None: 19 | self.max_consecutive_errors = max_consecutive_errors 20 | self.consecutive_errors = 0 21 | 22 | def __call__(self, tile: Tile) -> Tile: 23 | """Call the filter.""" 24 | if tile and tile.error: 25 | self.consecutive_errors += 1 26 | if self.consecutive_errors > self.max_consecutive_errors: 27 | if isinstance(tile.error, Exception): 28 | raise TooManyError(tile) from tile.error 29 | raise TooManyError(tile) 30 | else: 31 | self.consecutive_errors = 0 32 | return tile 33 | 34 | 35 | class MaximumErrorRate: 36 | """ 37 | Create a filter that limit the error rate. 38 | 39 | Raises a :class:`TooManyErrors` exception when the total error rate exceeds ``max_error_rate``. 40 | 41 | max_error_rate: 42 | The maximum error rate. Once exceeded a :class:`TooManyErrors` 43 | exception is raised. 44 | min_tiles: 45 | The minimum number of received tiles before a :class:`TooManyErrors` 46 | exception can be raised. Defaults to 8. 47 | """ 48 | 49 | def __init__(self, max_error_rate: float, min_tiles: int = 8) -> None: 50 | self.max_error_rate = max_error_rate 51 | self.min_tiles = min_tiles 52 | self.tile_count = 0 53 | self.error_count = 0 54 | 55 | def __call__(self, tile: Tile) -> Tile: 56 | """Call the filter.""" 57 | self.tile_count += 1 58 | if tile and tile.error: 59 | self.error_count += 1 60 | if ( 61 | self.tile_count >= self.min_tiles 62 | and self.error_count >= self.max_error_rate * self.tile_count 63 | ): 64 | if isinstance(tile.error, Exception): 65 | raise TooManyError(tile) from tile.error 66 | raise TooManyError(tile) 67 | return tile 68 | 69 | 70 | class TooManyError(RuntimeError): 71 | """TooManyErrors exception class.""" 72 | 73 | def __init__(self, tile: Tile) -> None: 74 | self.last_tile = tile 75 | super().__init__( 76 | f"Too many errors, last tile in error {tile.tilecoord} {tile.formated_metadata}\n{tile.error}", 77 | ) 78 | -------------------------------------------------------------------------------- /.github/workflows/main.yaml: -------------------------------------------------------------------------------- 1 | name: Continuous integration 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | - '[0-9]+.[0-9]+' 8 | tags: 9 | - '*' 10 | pull_request: 11 | 12 | env: 13 | HAS_SECRETS: ${{ secrets.HAS_SECRETS }} 14 | 15 | jobs: 16 | main: 17 | name: Continuous integration 18 | runs-on: ubuntu-24.04 19 | timeout-minutes: 30 20 | if: "!startsWith(github.event.head_commit.message, '[skip ci] ')" 21 | 22 | env: 23 | REDIS_URL: redis://localhost:6379 24 | 25 | steps: 26 | - run: docker system prune --all --force 27 | - uses: actions/checkout@v5 28 | with: 29 | fetch-depth: 0 30 | 31 | - uses: camptocamp/initialise-gopass-summon-action@v2 32 | with: 33 | ci-gpg-private-key: ${{secrets.CI_GPG_PRIVATE_KEY}} 34 | github-gopass-ci-token: ${{secrets.GOPASS_CI_GITHUB_TOKEN}} 35 | patterns: pypi docker 36 | if: env.HAS_SECRETS == 'HAS_SECRETS' 37 | 38 | - uses: actions/setup-python@v6 39 | - run: python3 -m pip install --requirement=ci/requirements.txt 40 | 41 | - uses: actions/cache@v4 42 | with: 43 | path: ~/.cache/pre-commit 44 | key: pre-commit-${{ hashFiles('.pre-commit-config.yaml') }} 45 | restore-keys: "pre-commit-${{ hashFiles('.pre-commit-config.yaml') }}\npre-commit-" 46 | - run: pre-commit run --all-files --color=always 47 | - run: git diff --exit-code --patch > /tmp/pre-commit.patch; git diff --color; git reset --hard || true 48 | if: failure() 49 | - uses: actions/upload-artifact@v4 50 | with: 51 | name: Apply pre-commit fix.patch 52 | path: /tmp/pre-commit.patch 53 | retention-days: 1 54 | if: failure() 55 | 56 | - name: Print environment information 57 | run: c2cciutils-env 58 | env: 59 | GITHUB_EVENT: ${{ toJson(github) }} 60 | 61 | - name: Build 62 | run: make build 63 | 64 | - name: Checks 65 | run: make checks 66 | 67 | - name: Tests 68 | run: make tests 69 | 70 | - run: c2cciutils-docker-logs 71 | if: always() 72 | 73 | - uses: actions/upload-artifact@v4 74 | with: 75 | name: results 76 | path: results 77 | if-no-files-found: ignore 78 | retention-days: 5 79 | if: failure() 80 | 81 | - run: git reset --hard 82 | - name: Publish 83 | run: tag-publish 84 | if: env.HAS_SECRETS == 'HAS_SECRETS' 85 | env: 86 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 87 | - run: git diff --exit-code --patch > /tmp/dpkg-versions.patch; git diff --color; git reset --hard || true 88 | if: failure() 89 | - uses: actions/upload-artifact@v4 90 | with: 91 | name: Update dpkg versions list.patch 92 | path: /tmp/dpkg-versions.patch 93 | retention-days: 1 94 | if: failure() 95 | permissions: 96 | contents: write 97 | packages: write 98 | id-token: write 99 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/mapfile/mapfile.map: -------------------------------------------------------------------------------- 1 | MAP 2 | NAME "Tests" 3 | STATUS ON 4 | EXTENT 420000 30000 900000 350000 5 | MAXSIZE 2500 6 | 7 | WEB 8 | TEMPLATE dummyTemplateForWmsGetFeatureInfo 9 | METADATA 10 | "ows_title" "Tests" 11 | "ows_encoding" "UTF-8" 12 | "wms_enable_request" "*" 13 | END 14 | END 15 | 16 | PROJECTION 17 | "init=epsg:21781" 18 | END 19 | 20 | SYMBOL 21 | NAME "circle" 22 | TYPE ellipse 23 | POINTS 24 | 1 1 25 | END 26 | FILLED true 27 | END 28 | 29 | LAYER 30 | NAME "point" 31 | TYPE POINT 32 | CONNECTIONTYPE postgis 33 | CONNECTION "user=postgresql password=postgresql dbname=tests host=db" 34 | DATA "the_geom FROM tests.point" 35 | TEMPLATE fooOnlyForWMSGetFeatureInfo # For GetFeatureInfo 36 | STATUS ON 37 | METADATA 38 | "gml_include_items" "all" # For GetFeatureInfo 39 | "ows_geom_type" "point" # For returning geometries in GetFeatureInfo 40 | "ows_geometries" "the_geom" # For returning geometries in GetFeatureInfo 41 | END 42 | CLASS 43 | NAME "Point" 44 | COLOR 255 0 0 45 | SIZE 10 46 | SYMBOL "circle" 47 | END 48 | END 49 | 50 | LAYER 51 | NAME "point_multi" 52 | TYPE POINT 53 | CONNECTIONTYPE postgis 54 | CONNECTION "user=postgresql password=postgresql dbname=tests host=db" 55 | DATA "the_geom FROM (SELECT * FROM tests.point WHERE name='%POINT_NAME%') AS foo USING unique gid" 56 | TEMPLATE fooOnlyForWMSGetFeatureInfo # For GetFeatureInfo 57 | STATUS ON 58 | METADATA 59 | "gml_include_items" "all" # For GetFeatureInfo 60 | "ows_geom_type" "point" # For returning geometries in GetFeatureInfo 61 | "ows_geometries" "the_geom" # For returning geometries in GetFeatureInfo 62 | END 63 | VALIDATION 64 | "POINT_NAME" "[a-z0-9]+" 65 | "default_POINT_NAME" "point1" 66 | END 67 | CLASS 68 | NAME "Point" 69 | COLOR 255 0 0 70 | SIZE 10 71 | SYMBOL "circle" 72 | END 73 | END 74 | 75 | LAYER 76 | NAME "line" 77 | TYPE LINE 78 | CONNECTIONTYPE postgis 79 | CONNECTION "user=postgresql password=postgresql dbname=tests host=db" 80 | DATA "the_geom FROM tests.line" 81 | STATUS ON 82 | CLASS 83 | NAME "Line 1" 84 | STYLE 85 | COLOR 0 0 255 86 | WIDTH 5 87 | MINSCALEDENOM 100000 88 | END 89 | END 90 | CLASS 91 | NAME "Line 2" 92 | STYLE 93 | COLOR 0 0 255 94 | WIDTH 5 95 | MAXSCALEDENOM 100000 96 | END 97 | END 98 | END 99 | 100 | LAYER 101 | NAME "polygon" 102 | TYPE POLYGON 103 | CONNECTIONTYPE postgis 104 | CONNECTION "user=postgresql password=postgresql dbname=tests host=db" 105 | DATA "the_geom FROM tests.polygon" 106 | STATUS ON 107 | CLASS 108 | NAME "Polygon" 109 | STYLE 110 | OUTLINECOLOR 0 255 0 111 | COLOR 255 255 0 112 | END 113 | END 114 | END 115 | END 116 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/test_copy.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | from pathlib import Path 4 | 5 | import requests 6 | 7 | from tilecloud_chain import copy_ 8 | from tilecloud_chain.tests import CompareCase 9 | 10 | 11 | class TestGenerate(CompareCase): 12 | def setUp(self) -> None: 13 | self.maxDiff = None 14 | 15 | @classmethod 16 | def setUpClass(cls): 17 | os.chdir(Path(__file__).parent) 18 | if Path("/tmp/tiles").exists(): 19 | shutil.rmtree("/tmp/tiles") 20 | Path("/tmp/tiles/src/1.0.0/point_hash/default/21781/0/0/").mkdir(parents=True) 21 | 22 | @classmethod 23 | def tearDownClass(cls): 24 | os.chdir(Path(__file__).parent.parent.parent) 25 | if Path("/tmp/tiles").exists(): 26 | shutil.rmtree("/tmp/tiles") 27 | 28 | def test_copy(self) -> None: 29 | with Path("/tmp/tiles/src/1.0.0/point_hash/default/21781/0/0/0.png").open("w") as f: 30 | f.write("test image") 31 | 32 | for d in ("-d", "-q", "-v"): 33 | self.assert_cmd_equals( 34 | cmd=f".build/venv/bin/generate-copy {d} -c tilegeneration/test-copy.yaml src dst", 35 | main_func=copy_.main, 36 | regex=True, 37 | expected=( 38 | """The tile copy of layer 'point_hash' is finish 39 | Nb copy tiles: 1 40 | Nb errored tiles: 0 41 | Nb dropped tiles: 0 42 | Total time: 0:00:[0-9][0-9] 43 | Total size: 10 o 44 | Time per tile: [0-9]+ ms 45 | Size per tile: 10(.0)? o 46 | 47 | """ 48 | if d != "-q" 49 | else "" 50 | ), 51 | empty_err=True, 52 | ) 53 | with Path("/tmp/tiles/dst/1.0.0/point_hash/default/21781/0/0/0.png").open() as f: 54 | assert f.read() == "test image" 55 | 56 | def test_process(self) -> None: 57 | for d in ("-vd", "-q", "-v", ""): 58 | response = requests.get( 59 | "http://mapserver:8080/?STYLES=default&SERVICE=WMS&FORMAT=\ 60 | image%2Fpng&REQUEST=GetMap&HEIGHT=256&WIDTH=256&VERSION=1.1.1&BBOX=\ 61 | %28560800.0%2C+158000.0%2C+573600.0%2C+170800.0%29&LAYERS=point&SRS=EPSG%3A21781", 62 | ) 63 | response.raise_for_status() 64 | with Path("/tmp/tiles/src/1.0.0/point_hash/default/21781/0/0/0.png").open("wb") as out: 65 | out.write(response.content) 66 | statinfo = Path( 67 | "/tmp/tiles/src/1.0.0/point_hash/default/21781/0/0/0.png", 68 | ).stat() 69 | assert statinfo.st_size == 755 70 | 71 | self.assert_cmd_equals( 72 | cmd=f".build/venv/bin/generate-process {d} -c " 73 | "tilegeneration/test-copy.yaml --cache src optipng", 74 | main_func=copy_.process, 75 | regex=True, 76 | expected=( 77 | """The tile process of layer 'point_hash' is finish 78 | Nb process tiles: 1 79 | Nb errored tiles: 0 80 | Nb dropped tiles: 0 81 | Total time: 0:00:[0-9][0-9] 82 | Total size: 103 o 83 | Time per tile: [0-9]+ ms 84 | Size per tile: 103(.0)? o 85 | 86 | """ 87 | if d != "-q" 88 | else "" 89 | ), 90 | empty_err=True, 91 | ) 92 | statinfo = Path( 93 | "/tmp/tiles/src/1.0.0/point_hash/default/21781/0/0/0.png", 94 | ).stat() 95 | assert statinfo.st_size == 103 96 | -------------------------------------------------------------------------------- /docker/mapfile-docker/mapserver.map: -------------------------------------------------------------------------------- 1 | MAP 2 | NAME "Tests" 3 | STATUS ON 4 | EXTENT 420000 30000 900000 350000 5 | MAXSIZE 2500 6 | 7 | WEB 8 | TEMPLATE dummyTemplateForWmsGetFeatureInfo 9 | METADATA 10 | "ows_title" "Tests" 11 | "ows_encoding" "UTF-8" 12 | "wms_enable_request" "*" 13 | "wms_srs" "EPSG:21781 EPSG:2056 EPSG:3857" 14 | END 15 | END 16 | 17 | PROJECTION 18 | "init=epsg:21781" 19 | END 20 | 21 | SYMBOL 22 | NAME "circle" 23 | TYPE ellipse 24 | POINTS 25 | 1 1 26 | END 27 | FILLED true 28 | END 29 | 30 | OUTPUTFORMAT 31 | NAME "webp" 32 | DRIVER "GDAL/WEBP" 33 | MIMETYPE "image/webp" 34 | IMAGEMODE RGBA 35 | EXTENSION "webp" 36 | END 37 | 38 | LAYER 39 | NAME "point" 40 | TYPE POINT 41 | CONNECTIONTYPE postgis 42 | CONNECTION "user=postgresql password=postgresql dbname=tests host=db" 43 | DATA "the_geom FROM tests.point" 44 | TEMPLATE fooOnlyForWMSGetFeatureInfo # For GetFeatureInfo 45 | STATUS ON 46 | METADATA 47 | "gml_include_items" "all" # For GetFeatureInfo 48 | "ows_geom_type" "point" # For returning geometries in GetFeatureInfo 49 | "ows_geometries" "the_geom" # For returning geometries in GetFeatureInfo 50 | END 51 | CLASS 52 | NAME "Point" 53 | STYLE 54 | COLOR 255 0 0 55 | SIZE 10 56 | SYMBOL "circle" 57 | END 58 | END 59 | END 60 | 61 | LAYER 62 | NAME "point_multi" 63 | TYPE POINT 64 | CONNECTIONTYPE postgis 65 | CONNECTION "user=postgresql password=postgresql dbname=tests host=db" 66 | DATA "the_geom FROM (SELECT * FROM tests.point WHERE name='%POINT_NAME%') AS foo USING unique gid" 67 | TEMPLATE fooOnlyForWMSGetFeatureInfo # For GetFeatureInfo 68 | STATUS ON 69 | METADATA 70 | "gml_include_items" "all" # For GetFeatureInfo 71 | "ows_geom_type" "point" # For returning geometries in GetFeatureInfo 72 | "ows_geometries" "the_geom" # For returning geometries in GetFeatureInfo 73 | END 74 | VALIDATION 75 | "POINT_NAME" "[a-z0-9]+" 76 | "default_POINT_NAME" "point1" 77 | END 78 | CLASS 79 | NAME "Point" 80 | STYLE 81 | COLOR 255 0 0 82 | SIZE 10 83 | SYMBOL "circle" 84 | END 85 | END 86 | END 87 | 88 | LAYER 89 | NAME "line" 90 | TYPE LINE 91 | CONNECTIONTYPE postgis 92 | CONNECTION "user=postgresql password=postgresql dbname=tests host=db" 93 | DATA "the_geom FROM tests.line" 94 | STATUS ON 95 | CLASS 96 | NAME "Line 1" 97 | STYLE 98 | COLOR 0 255 0 99 | WIDTH 5 100 | MINSCALEDENOM 100000 101 | END 102 | END 103 | CLASS 104 | NAME "Line 2" 105 | STYLE 106 | COLOR 0 0 255 107 | WIDTH 5 108 | MAXSCALEDENOM 100000 109 | END 110 | END 111 | END 112 | 113 | LAYER 114 | NAME "polygon" 115 | TYPE POLYGON 116 | CONNECTIONTYPE postgis 117 | CONNECTION "user=postgresql password=postgresql dbname=tests host=db" 118 | DATA "the_geom FROM tests.polygon" 119 | STATUS ON 120 | CLASS 121 | NAME "Polygon" 122 | STYLE 123 | OUTLINECOLOR 0 255 0 124 | COLOR 255 255 0 125 | END 126 | END 127 | END 128 | END 129 | -------------------------------------------------------------------------------- /docker/mapfile-docker/test.mapnik: -------------------------------------------------------------------------------- 1 | 2 | 3 | 8 | 13 | 18 | 19 | polygon 20 | 21 | tests.polygon 22 | the_geom 23 | 21781 24 | postgis 25 | tests 26 | db 27 | 5432 28 | postgresql 29 | postgresql 30 | false 31 | 420000, 30000, 900000, 350000 32 | 33 | 34 | 35 | line 36 | 37 | tests.line 38 | the_geom 39 | 21781 40 | postgis 41 | tests 42 | db 43 | 5432 44 | postgresql 45 | postgresql 46 | false 47 | 420000, 30000, 900000, 350000 48 | 49 | 50 | 51 | point 52 | 53 | tests.point 54 | the_geom 55 | 21781 56 | postgis 57 | tests 58 | db 59 | 5432 60 | postgresql 61 | postgresql 62 | false 63 | 420000, 30000, 900000, 350000 64 | 65 | 66 | 67 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/mapfile/test.mapnik: -------------------------------------------------------------------------------- 1 | 2 | 3 | 8 | 13 | 18 | 19 | polygon 20 | 21 | tests.polygon 22 | the_geom 23 | 21781 24 | postgis 25 | tests 26 | db 27 | 5432 28 | postgresql 29 | postgresql 30 | false 31 | 420000, 30000, 900000, 350000 32 | 33 | 34 | 35 | line 36 | 37 | tests.line 38 | the_geom 39 | 21781 40 | postgis 41 | tests 42 | db 43 | 5432 44 | postgresql 45 | postgresql 46 | false 47 | 420000, 30000, 900000, 350000 48 | 49 | 50 | 51 | point 52 | 53 | tests.point 54 | the_geom 55 | 21781 56 | postgis 57 | tests 58 | db 59 | 5432 60 | postgresql 61 | postgresql 62 | false 63 | 420000, 30000, 900000, 350000 64 | 65 | 66 | 67 | -------------------------------------------------------------------------------- /tilecloud_chain/security.py: -------------------------------------------------------------------------------- 1 | """Security policy for the pyramid application.""" 2 | 3 | import os 4 | 5 | import c2cwsgiutils.auth 6 | import pyramid.request 7 | from c2cwsgiutils.auth import AuthConfig 8 | from pyramid.security import Allowed, Denied 9 | 10 | 11 | class User: 12 | """The user definition.""" 13 | 14 | login: str | None 15 | name: str | None 16 | url: str | None 17 | is_auth: bool 18 | token: str | None 19 | is_admin: bool 20 | request: pyramid.request.Request 21 | 22 | def __init__( 23 | self, 24 | auth_type: str, 25 | login: str | None, 26 | name: str | None, 27 | url: str | None, 28 | is_auth: bool, 29 | token: str | None, 30 | request: pyramid.request.Request, 31 | ) -> None: 32 | self.auth_type = auth_type 33 | self.login = login 34 | self.name = name 35 | self.url = url 36 | self.is_auth = is_auth 37 | self.token = token 38 | self.request = request 39 | self.is_admin = c2cwsgiutils.auth.check_access(self.request) 40 | 41 | def has_access(self, auth_config: AuthConfig) -> bool: 42 | """Check if the user has access to the tenant.""" 43 | if self.is_admin: 44 | return True 45 | if "github_repository" in auth_config: 46 | return c2cwsgiutils.auth.check_access_config(self.request, auth_config) 47 | 48 | return False 49 | 50 | 51 | class SecurityPolicy: 52 | """The pyramid security policy.""" 53 | 54 | def identity(self, request: pyramid.request.Request) -> User: 55 | """Return app-specific user object.""" 56 | if not hasattr(request, "user"): 57 | if "TEST_USER" in os.environ: 58 | user = User( 59 | auth_type="test_user", 60 | login=os.environ["TEST_USER"], 61 | name=os.environ["TEST_USER"], 62 | url="https://example.com/user", 63 | is_auth=True, 64 | token=None, 65 | request=request, 66 | ) 67 | else: 68 | is_auth, c2cuser = c2cwsgiutils.auth.is_auth_user(request) 69 | user = User( 70 | "github_oauth", 71 | c2cuser.get("login"), 72 | c2cuser.get("name"), 73 | c2cuser.get("url"), 74 | is_auth, 75 | c2cuser.get("token"), 76 | request, 77 | ) 78 | request.user = user 79 | return request.user # type: ignore[no-any-return] 80 | 81 | def authenticated_userid(self, request: pyramid.request.Request) -> str | None: 82 | """Return a string ID for the user.""" 83 | identity = self.identity(request) 84 | 85 | if identity is None: 86 | return None 87 | 88 | return identity.login 89 | 90 | def permits( 91 | self, 92 | request: pyramid.request.Request, 93 | context: AuthConfig, 94 | permission: str, 95 | ) -> Allowed | Denied: 96 | """Allow access to everything if signed in.""" 97 | identity = self.identity(request) 98 | 99 | if identity is None: 100 | return Denied("User is not signed in.") 101 | if identity.auth_type in ("test_user",): 102 | return Allowed(f"All access auth type: {identity.auth_type}") 103 | if identity.is_admin: 104 | return Allowed("The User is admin.") 105 | if permission == "all": 106 | return Denied("Root access is required.") 107 | if identity.has_access(context): 108 | return Allowed("The User has access.") 109 | return Denied(f"The User has no access to source {permission}.") 110 | -------------------------------------------------------------------------------- /CHANGES.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## Release 1.17 4 | 5 | 1. Change the validator and parser => duplicate key generate an error: on/off are no more considered as boolean. 6 | 2. The argument --layer is no more used when we use the parameter --tiles, we get the information from the 7 | tiles file. 8 | 3. Be able to mutualise the service. 9 | 4. Add Azure blob storage 10 | 5. Remove Apache and MapCache 11 | 6. Remove the `log_format` in the `generation` configuration, nor we use the logging configuration from the 12 | `development.ini` file. 13 | 14 | ## Release 1.16 15 | 16 | 1. Change the config validator who is a little bit more strict. 17 | 18 | ## Release 1.4 19 | 20 | 1. Add optional `metadata` section to the config file. See the scaffolds for example. 21 | 22 | ## Release 0.9 23 | 24 | 1. Correct some error with slash. 25 | 2. Better error handling. 26 | 3. Be able to have one error file per layer. 27 | 28 | ## Release 0.8 29 | 30 | 1. Correct some error with slash. 31 | 2. Add `pre_hash_post_process` and `post_process`. 32 | 3. Add copy command. 33 | 34 | ## Release 0.7 35 | 36 | 1. Support of deferent geoms per layers, requires configuration changes, old version: 37 | 38 | > ```yaml 39 | > connection: user=www-data password=www-data dbname= host=localhost 40 | > sql: AS geom FROM 41 | > ``` 42 | > 43 | > to new version: 44 | > 45 | > ```yaml 46 | > connection: user=www-data password=www-data dbname= host=localhost 47 | > geoms: 48 | > - sql: AS geom FROM
49 | > ``` 50 | > 51 | > More information in the **Configure geom/sql** chapter. 52 | 53 | 2. Update from `optparse` to `argparse`, and some argument refactoring, use `--help` to see the new version. 54 | 3. Add support of Blackbery DB (`bsddb`). 55 | 4. The tile `server` is completely rewrite, now it support all cache, `REST` and `KVP` interface, 56 | `GetFeatureInfo` request, and it can be used as a pyramid view or as a `WSGI` server. More information in 57 | the **istribute the tiles** chapter. 58 | 5. Add three strategy to bypass the proxy/cache: Use the headers `Cache-Control: no-cache, no-store`, 59 | `Pragma: no-cache` (default). Use localhost in the URL and the header `Host: ` (recommended). 60 | Add a `SALT` random argument (if the above don't work). More information in the **Proxy/cache issue** 61 | chapter. 62 | 6. Improve the dimensions usage by adding it ti the WMS requests, And add a `--dimensions` argument of 63 | `generate-tiles` to change the dimensions values. 64 | 7. Extract `generate_cost` and `generate_amazon` from `generate_controler`. 65 | 8. Now we can creates legends, see the **Legends** chapter. 66 | 9. Now the tiles generation display generation statistics at the ends. 67 | 10. The EC2 configuration is moved in a separate structure, see README for more information. 68 | 69 | ## Release 0.6 70 | 71 | 1. Now the apache configuration can be generated with 72 | `.build/venv/bin/generate-controller --generate-apache-config`, it support `filesystem` `cache` and 73 | `MapCache`. 74 | 2. Windows fixes. 75 | 3. Use console rewrite (r) to log generated tiles coordinates. 76 | 4. Now if no layers is specified in `generation:default_layers` we generate all layers by default. 77 | 5. Now bbox to be floats. 78 | 6. New `--get-bbox` option to get the bbox of a tile. 79 | 7. Add coveralls support (). 80 | 8. Add an config option `generation:error_file` and a command option `--tiles` to store and regenerate 81 | errored tiles. 82 | 83 | ## Release 0.5 84 | 85 | 1. SQS config change: 86 | 87 | ```yaml 88 | layers: 89 | layer_name: 90 | sqs: 91 | # The region where the SQS queue is 92 | region: eu-west-1 93 | # The SQS queue name, it should already exists 94 | queue: the_name 95 | ``` 96 | 97 | 2. Add debug option (`--debug`), please use it to report issue. 98 | 3. Now the `sql` request can return a set of geometries in a column names geom but the syntax change a little 99 | bit => ` AS geom FROM
` 100 | -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | db: 3 | image: camptocamp/postgres:17-postgis-3 4 | environment: 5 | POSTGRES_USER: postgresql 6 | POSTGRES_PASSWORD: postgresql 7 | POSTGRES_DB: tests 8 | volumes: 9 | - ./docker/test-db:/docker-entrypoint-initdb.d:ro 10 | 11 | mapserver: 12 | image: camptocamp/mapserver:8.4 13 | environment: 14 | MS_DEBUGLEVEL: '5' 15 | MAPSERVER_CATCH_SEGV: '1' 16 | MS_MAPFILE: /etc/mapserver/mapserver.map 17 | MS_MAP_PATTERN: ^/etc/mapserver/ 18 | volumes: 19 | - ./docker/mapfile-docker:/etc/mapserver:ro 20 | links: 21 | - db:db 22 | user: www-data 23 | ports: 24 | - 8080:8080 25 | 26 | redis_master: 27 | image: bitnamilegacy/redis:8.2.1 28 | environment: 29 | - REDIS_REPLICATION_MODE=master 30 | - ALLOW_EMPTY_PASSWORD=yes 31 | 32 | redis_slave: 33 | image: bitnamilegacy/redis:8.2.1 34 | environment: 35 | - REDIS_REPLICATION_MODE=slave 36 | - REDIS_MASTER_HOST=redis_master 37 | - ALLOW_EMPTY_PASSWORD=yes 38 | depends_on: 39 | - redis_master 40 | 41 | redis_sentinel: 42 | image: bitnamilegacy/redis-sentinel:8.2.1 43 | environment: 44 | - REDIS_MASTER_HOST=redis_master 45 | - REDIS_MASTER_SET=mymaster 46 | - ALLOW_EMPTY_PASSWORD=yes 47 | depends_on: 48 | - redis_master 49 | - redis_slave 50 | 51 | application: &app 52 | image: camptocamp/tilecloud-chain 53 | environment: &app-env 54 | TILECLOUD_LOG_LEVEL: INFO 55 | TILECLOUD_CHAIN_LOG_LEVEL: INFO 56 | TILECLOUD_CHAIN_SESSION_SECRET: '1234' 57 | TILECLOUD_CHAIN_SESSION_SALT: '1234' 58 | C2C__AUTH__GITHUB__REPOSITORY: camptocamp/tilecloud-chain 59 | C2C__AUTH__GITHUB__CLIENT_ID: '1234' 60 | C2C__AUTH__GITHUB__CLIENT_SECRET: '1234' 61 | C2C__PROMETHEUS__PORT: '9110' 62 | HTTP: TRUE 63 | ports: 64 | - '9050:8080' 65 | links: 66 | - db 67 | - redis_sentinel 68 | volumes: 69 | - ./example/tilegeneration/config.yaml:/etc/tilegeneration/config.yaml:ro 70 | 71 | app_test_user: 72 | <<: *app 73 | environment: 74 | <<: *app-env 75 | C2C__AUTH__TEST__USERNAME: Test 76 | ports: 77 | - '9051:8080' 78 | 79 | app_postgresql: 80 | <<: *app 81 | environment: &app-pg-env 82 | <<: *app-env 83 | C2C__AUTH__TEST__USERNAME: Test 84 | SQL_LOG_LEVEL: DEBUG 85 | TILECLOUD_CHAIN_SQLALCHEMY_URL: postgresql+asyncpg://postgresql:postgresql@db:5432/tests 86 | PGHOST: db 87 | PGUSER: postgresql 88 | PGPASSWORD: postgresql 89 | PGDATABASE: tests 90 | volumes: 91 | - ./example/tilegeneration/config-postgresql.yaml:/etc/tilegeneration/config.yaml:ro 92 | ports: 93 | - '9052:8080' 94 | 95 | slave: 96 | <<: *app 97 | command: 98 | - /venv/bin/generate-tiles 99 | - '--role=slave' 100 | - '--daemon' 101 | environment: 102 | <<: *app-pg-env 103 | TILECLOUD_CHAIN_SLAVE: 'TRUE' 104 | volumes: 105 | - ./example/tilegeneration/config-postgresql.yaml:/etc/tilegeneration/config.yaml:ro 106 | ports: [] 107 | 108 | test: 109 | image: camptocamp/tilecloud-chain-tests 110 | working_dir: /app 111 | environment: 112 | CI: 'true' 113 | TESTS: 'true' 114 | PGPASSWORD: postgresql 115 | TILECLOUD_LOG_LEVEL: DEBUG 116 | TILECLOUD_CHAIN_LOG_LEVEL: DEBUG 117 | TILECLOUD_CHAIN_SESSION_SALT: a-long-secret-a-long-secret 118 | TILECLOUD_CHAIN_SQLALCHEMY_URL: postgresql+psycopg://postgresql:postgresql@db:5432/tests 119 | command: 120 | - sleep 121 | - infinity 122 | links: 123 | - db 124 | - redis_sentinel 125 | volumes: 126 | - ./results:/results 127 | - ./tilecloud_chain:/app/tilecloud_chain 128 | # - ../tilecloud/tilecloud:/usr/local/lib/python3.8/dist-packages/tilecloud 129 | 130 | shell: 131 | image: camptocamp/postgres:17-postgis-3 132 | command: 133 | - tail 134 | - -f 135 | - /dev/null 136 | environment: 137 | - PGHOST=db 138 | - PGUSER=postgresql 139 | - PGPASSWORD=postgresql 140 | - PGDATABASE=tests 141 | - PGPORT=5432 142 | -------------------------------------------------------------------------------- /tilecloud_chain/timedtilestore.py: -------------------------------------------------------------------------------- 1 | """A wrapper around a TileStore that adds timer metrics.""" 2 | 3 | import time 4 | from collections.abc import AsyncGenerator, AsyncIterator 5 | from typing import Any, TypeVar 6 | 7 | from prometheus_client import Summary 8 | from tilecloud import Tile 9 | 10 | from tilecloud_chain.store import AsyncTileStore 11 | 12 | _OptionalTileOrNot = TypeVar("_OptionalTileOrNot", Tile | None, Tile) 13 | 14 | _TILESTORE_OPERATION_SUMMARY = Summary( 15 | "tilecloud_chain_tilestore", 16 | "Number of tilestore contains", 17 | ["layer", "host", "store", "operation"], 18 | ) 19 | 20 | 21 | class TimedTileStoreWrapper(AsyncTileStore): 22 | """A wrapper around a TileStore that adds timer metrics.""" 23 | 24 | def __init__(self, tile_store: AsyncTileStore, store_name: str) -> None: 25 | """Initialize.""" 26 | super().__init__() 27 | self._tile_store = tile_store 28 | self._store_name = store_name 29 | 30 | async def _time_iteration(self) -> AsyncGenerator[_OptionalTileOrNot]: 31 | while True: 32 | start = time.perf_counter() 33 | try: 34 | tile = await anext(self._tile_store.list()) 35 | except StopAsyncIteration: 36 | break 37 | except RuntimeError as exception: 38 | if isinstance(exception.__cause__, StopAsyncIteration): 39 | # since python 3.7, a StopIteration is wrapped in a RuntimeError (PEP 479) 40 | break 41 | raise 42 | _TILESTORE_OPERATION_SUMMARY.labels( 43 | tile.metadata.get("layer", "none"), 44 | tile.metadata.get("host", "none"), 45 | self._store_name, 46 | "list", 47 | ).observe(time.perf_counter() - start) 48 | yield tile 49 | 50 | async def __contains__(self, tile: Tile) -> bool: 51 | """See in superclass.""" 52 | with _TILESTORE_OPERATION_SUMMARY.labels( 53 | tile.metadata.get("layer", "none"), 54 | tile.metadata.get("host", "none"), 55 | self._store_name, 56 | "contains", 57 | ).time(): 58 | return await self._tile_store.__contains__(tile) 59 | 60 | async def delete_one(self, tile: Tile) -> Tile: 61 | """See in superclass.""" 62 | with _TILESTORE_OPERATION_SUMMARY.labels( 63 | tile.metadata.get("layer", "none"), 64 | tile.metadata.get("host", "none"), 65 | self._store_name, 66 | "delete_one", 67 | ).time(): 68 | return await self._tile_store.delete_one(tile) 69 | 70 | async def list(self) -> AsyncIterator[Tile]: 71 | """See in superclass.""" 72 | async for tile in self._time_iteration(): 73 | yield tile 74 | 75 | async def get_one(self, tile: Tile) -> Tile | None: 76 | """See in superclass.""" 77 | with _TILESTORE_OPERATION_SUMMARY.labels( 78 | tile.metadata.get("layer", "none"), 79 | tile.metadata.get("host", "none"), 80 | self._store_name, 81 | "get_one", 82 | ).time(): 83 | return await self._tile_store.get_one(tile) 84 | 85 | async def put_one(self, tile: Tile) -> Tile: 86 | """See in superclass.""" 87 | with _TILESTORE_OPERATION_SUMMARY.labels( 88 | tile.metadata.get("layer", "none"), 89 | tile.metadata.get("host", "none"), 90 | self._store_name, 91 | "put_one", 92 | ).time(): 93 | return await self._tile_store.put_one(tile) 94 | 95 | async def get(self, tiles: AsyncIterator[Tile]) -> AsyncIterator[Tile | None]: 96 | """See in superclass.""" 97 | with _TILESTORE_OPERATION_SUMMARY.labels("none", "none", self._store_name, "get").time(): 98 | async for tile in self._tile_store.get(tiles): 99 | yield tile 100 | 101 | def __getattr__(self, item: str) -> Any: 102 | """See in superclass.""" 103 | return getattr(self._tile_store, item) 104 | 105 | def __str__(self) -> str: 106 | """Get string representation.""" 107 | return f"{self.__class__.__name__}({self._store_name}: {self._tile_store}" 108 | 109 | def __repr__(self) -> str: 110 | """Get string representation.""" 111 | return f"{self.__class__.__name__}({self._store_name}: {self._tile_store!r})" 112 | -------------------------------------------------------------------------------- /tilecloud_chain/templates/openlayers.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 9 | 10 | TileCloud-chain test page 11 | 18 | 25 | 26 | 33 | 40 | 58 | 59 | 60 |
61 | 67 | 73 | 74 | 80 | 81 | 144 | 145 | 146 | -------------------------------------------------------------------------------- /tilecloud_chain/store/__init__.py: -------------------------------------------------------------------------------- 1 | from collections.abc import AsyncIterator, Callable 2 | from typing import Any 3 | 4 | from tilecloud import Tile, TileCoord, TileStore 5 | 6 | 7 | class AsyncTileStore: 8 | """A tile store.""" 9 | 10 | async def __contains__(self, tile: Tile) -> bool: 11 | """ 12 | Return true if this store contains ``tile``. 13 | 14 | Attributes 15 | ---------- 16 | tile: Tile 17 | 18 | """ 19 | raise NotImplementedError 20 | 21 | async def delete_one(self, tile: Tile) -> Tile: 22 | """ 23 | Delete ``tile`` and return ``tile``. 24 | 25 | Attributes 26 | ---------- 27 | tile: Tile 28 | 29 | """ 30 | raise NotImplementedError 31 | 32 | async def get_one(self, tile: Tile) -> Tile | None: 33 | """ 34 | Add data to ``tile``, or return ``None`` if ``tile`` is not in the store. 35 | 36 | Attributes 37 | ---------- 38 | tile: Tile 39 | 40 | """ 41 | raise NotImplementedError 42 | 43 | async def get(self, tiles: AsyncIterator[Tile]) -> AsyncIterator[Tile | None]: 44 | """ 45 | Add data to the tiles, or return ``None`` if the tile is not in the store. 46 | 47 | Attributes 48 | ---------- 49 | tiles: AsyncIterator[Tile] 50 | 51 | """ 52 | del tiles 53 | raise NotImplementedError 54 | yield Tile(TileCoord(0, 0, 0)) # pylint: disable=unreachable 55 | # async for tile in tiles: 56 | # yield await self.get_one(tile) 57 | 58 | async def list(self) -> AsyncIterator[Tile]: 59 | """Generate all the tiles in the store, but without their data.""" 60 | raise NotImplementedError 61 | yield Tile(TileCoord(0, 0, 0)) # pylint: disable=unreachable 62 | 63 | async def put_one(self, tile: Tile) -> Tile: 64 | """ 65 | Store ``tile`` in the store. 66 | 67 | Attributes 68 | ---------- 69 | tile: Tile 70 | 71 | """ 72 | raise NotImplementedError 73 | 74 | 75 | class TileStoreWrapper(AsyncTileStore): 76 | """Wrap a TileStore.""" 77 | 78 | def __init__(self, tile_store: TileStore) -> None: 79 | """Initialize.""" 80 | self.tile_store = tile_store 81 | 82 | async def __contains__(self, tile: Tile) -> bool: 83 | """See in superclass.""" 84 | return self.tile_store.__contains__(tile) 85 | 86 | async def delete_one(self, tile: Tile) -> Tile: 87 | """See in superclass.""" 88 | return self.tile_store.delete_one(tile) 89 | 90 | async def get_one(self, tile: Tile) -> Tile | None: 91 | """See in superclass.""" 92 | return self.tile_store.get_one(tile) 93 | 94 | async def get(self, tiles: AsyncIterator[Tile]) -> AsyncIterator[Tile | None]: 95 | """See in superclass.""" 96 | all_tiles = [] 97 | all_tiles = [tile async for tile in tiles] 98 | for new_tile in self.tile_store.get(all_tiles): 99 | yield new_tile 100 | 101 | async def list(self) -> AsyncIterator[Tile]: 102 | """See in superclass.""" 103 | for tile in self.tile_store.list(): 104 | yield tile 105 | 106 | async def put_one(self, tile: Tile) -> Tile: 107 | """See in superclass.""" 108 | return self.tile_store.put_one(tile) 109 | 110 | def __getattr__(self, item: str) -> Any: 111 | """See in superclass.""" 112 | return getattr(self.tile_store, item) 113 | 114 | 115 | class NoneTileStore(AsyncTileStore): 116 | """A tile store that does nothing.""" 117 | 118 | async def __contains__(self, tile: Tile) -> bool: 119 | """See in superclass.""" 120 | raise NotImplementedError 121 | 122 | async def delete_one(self, tile: Tile) -> Tile: 123 | """See in superclass.""" 124 | raise NotImplementedError 125 | 126 | async def get_one(self, tile: Tile) -> Tile | None: 127 | """See in superclass.""" 128 | return tile 129 | 130 | async def list(self) -> AsyncIterator[Tile]: 131 | """See in superclass.""" 132 | raise NotImplementedError 133 | yield Tile(TileCoord(0, 0, 0)) # pylint: disable=unreachable 134 | 135 | async def get(self, tiles: AsyncIterator[Tile]) -> AsyncIterator[Tile | None]: 136 | """See in superclass.""" 137 | async for tile in tiles: 138 | yield tile 139 | 140 | 141 | class CallWrapper: 142 | """Wrap a function call.""" 143 | 144 | def __init__(self, function: Callable[[Tile], Tile | None]) -> None: 145 | """Initialize.""" 146 | self.function = function 147 | 148 | async def __call__(self, tile: Tile) -> Tile | None: 149 | """See in superclass.""" 150 | return self.function(tile) 151 | 152 | 153 | class AsyncTilesIterator: 154 | """An async iterator.""" 155 | 156 | def __init__(self, tiles: list[Tile]) -> None: 157 | """Initialize.""" 158 | self._tiles = tiles 159 | 160 | async def __call__(self) -> AsyncIterator[Tile]: 161 | """Async iterator of the tiles.""" 162 | for tile in self._tiles: 163 | yield tile 164 | -------------------------------------------------------------------------------- /tilecloud_chain/multitilestore.py: -------------------------------------------------------------------------------- 1 | """Redirect to the corresponding Tilestore for the layer and config file.""" 2 | 3 | import logging 4 | from collections.abc import AsyncIterator, Awaitable, Callable 5 | from dataclasses import dataclass 6 | from pathlib import Path 7 | 8 | from tilecloud import Tile 9 | 10 | from tilecloud_chain.store import AsyncTilesIterator, AsyncTileStore 11 | 12 | logger = logging.getLogger(__name__) 13 | 14 | 15 | @dataclass 16 | class _DatedStore: 17 | """Store the date and the store.""" 18 | 19 | mtime: float 20 | store: AsyncTileStore 21 | 22 | 23 | class MultiTileStore(AsyncTileStore): 24 | """Redirect to the corresponding Tilestore for the layer and config file.""" 25 | 26 | def __init__( 27 | self, 28 | get_store: Callable[[Path, str, str | None], Awaitable[AsyncTileStore | None]], 29 | ) -> None: 30 | """Initialize.""" 31 | self.get_store = get_store 32 | self.stores: dict[tuple[Path, str, str], _DatedStore | None] = {} 33 | 34 | async def _get_store(self, config_file: Path, layer: str, grid_name: str) -> AsyncTileStore | None: 35 | config_path = Path(config_file) 36 | mtime = config_path.stat().st_mtime 37 | store = self.stores.get((config_file, layer, grid_name)) 38 | if store is not None and store.mtime != mtime: 39 | store = None 40 | if store is None: 41 | tile_store = await self.get_store(config_file, layer, grid_name) 42 | if tile_store is not None: 43 | store = _DatedStore(mtime, tile_store) 44 | self.stores[(config_file, layer, grid_name)] = store 45 | return store.store if store is not None else None 46 | 47 | async def _get_store_tile(self, tile: Tile) -> AsyncTileStore | None: 48 | """Return the store corresponding to the tile.""" 49 | layer = tile.metadata["layer"] 50 | grid = tile.metadata["grid"] 51 | config_file = Path(tile.metadata["config_file"]) 52 | return await self._get_store(config_file, layer, grid) 53 | 54 | async def __contains__(self, tile: Tile) -> bool: 55 | """ 56 | Return true if this store contains ``tile``. 57 | 58 | Arguments: 59 | tile: Tile 60 | """ 61 | store = await self._get_store_tile(tile) 62 | assert store is not None 63 | return tile in store 64 | 65 | async def delete_one(self, tile: Tile) -> Tile: 66 | """ 67 | Delete ``tile`` and return ``tile``. 68 | 69 | Arguments: 70 | tile: Tile 71 | """ 72 | store = await self._get_store_tile(tile) 73 | assert store is not None 74 | return await store.delete_one(tile) 75 | 76 | async def list(self) -> AsyncIterator[Tile]: 77 | """Generate all the tiles in the store, but without their data.""" 78 | # Too dangerous to list all tiles in all stores. Return an empty iterator instead 79 | while False: 80 | yield 81 | 82 | async def put_one(self, tile: Tile) -> Tile: 83 | """ 84 | Store ``tile`` in the store. 85 | 86 | Arguments: 87 | tile: Tile 88 | """ 89 | store = await self._get_store_tile(tile) 90 | assert store is not None 91 | return await store.put_one(tile) 92 | 93 | async def get_one(self, tile: Tile) -> Tile | None: 94 | """ 95 | Add data to ``tile``, or return ``None`` if ``tile`` is not in the store. 96 | 97 | Arguments: 98 | tile: Tile 99 | """ 100 | store = await self._get_store_tile(tile) 101 | assert store is not None 102 | return await store.get_one(tile) 103 | 104 | async def get(self, tiles: AsyncIterator[Tile]) -> AsyncIterator[Tile | None]: 105 | """ 106 | Add data to the tiles, or return ``None`` if the tile is not in the store. 107 | 108 | Arguments: 109 | tiles: AsyncIterator[Tile] 110 | """ 111 | async for tile in tiles: 112 | store = await self._get_store_tile(tile) 113 | assert store is not None, f"No store found for tile {tile.tilecoord} {tile.formated_metadata}" 114 | 115 | async for new_tile in store.get(AsyncTilesIterator([tile])()): 116 | yield new_tile 117 | 118 | def __str__(self) -> str: 119 | """Return a string representation of the object.""" 120 | stores = {str(store) for store in self.stores.values()} 121 | keys = {f"{config_file}:{layer}:{grid}" for config_file, layer, grid in self.stores} 122 | return f"{self.__class__.__name__}({', '.join(stores)} - {', '.join(keys)})" 123 | 124 | def __repr__(self) -> str: 125 | """Return a string representation of the object.""" 126 | stores = {repr(store) for store in self.stores.values()} 127 | keys = {f"{config_file}:{layer}:{grid}" for config_file, layer, grid in self.stores} 128 | return f"{self.__class__.__name__}({', '.join(stores)} - {', '.join(keys)})" 129 | 130 | @staticmethod 131 | def _get_layer(tile: Tile | None) -> tuple[str, str]: 132 | assert tile is not None 133 | return (tile.metadata["config_file"], tile.metadata["layer"]) 134 | -------------------------------------------------------------------------------- /example/tilegeneration/config.yaml: -------------------------------------------------------------------------------- 1 | grids: 2 | # grid name, I just recommends to add the min resolution because it's common to not generate all the layers at the same resolution. 3 | swissgrid_05: 4 | # resolutions [required] 5 | resolutions: [1000, 500, 250, 100, 50, 20, 10, 5, 2, 1, 0.5] 6 | # bbox [required] 7 | bbox: [420000, 30000, 900000, 350000] 8 | # srs [required] 9 | srs: EPSG:21781 10 | 11 | caches: 12 | local: 13 | type: filesystem 14 | folder: /var/sig/tiles 15 | # for GetCapabilities 16 | http_url: https://%(host)s/tiles/ 17 | hosts: 18 | - wmts0.example.com 19 | - wmts1.example.com 20 | - wmts2.example.com 21 | - wmts3.example.com 22 | - wmts4.example.com 23 | s3: 24 | type: s3 25 | bucket: tiles 26 | folder: '' 27 | # for GetCapabilities 28 | http_url: https://%(host)s/%(bucket)s/%(folder)s/ 29 | cache_control: 'public, max-age=14400' 30 | hosts: 31 | - wmts0. 32 | 33 | # this defines some defaults values for all the layers 34 | defaults: 35 | layer: &layer 36 | type: wms 37 | grid: swissgrid_05 38 | # The minimum resolution to seed, useful to use with mapcache, optional. 39 | # min_resolution_seed: 1 40 | # the URL of the WMS server to used 41 | url: http://mapserver/ 42 | # Set the headers to get the right virtual host, and don't get any cached result 43 | headers: 44 | Cache-Control: no-cache, no-store 45 | Pragma: no-cache 46 | # file name extension 47 | extension: png 48 | # the bbox there we want to generate tiles 49 | #bbox: [493000, 114000, 586000, 204000] 50 | 51 | # mime type used for the WMS request and the WMTS capabilities generation 52 | mime_type: image/png 53 | wmts_style: default 54 | # the WMTS dimensions definition [default is []] 55 | #dimensions: 56 | # - name: DATE 57 | # # the default value for the WMTS capabilities 58 | # default: '2012' 59 | # # the generated values 60 | # generate: ['2012'] 61 | # # all the available values in the WMTS capabilities 62 | # values: ['2012'] 63 | # the meta tiles definition [default is off] 64 | meta: true 65 | # the meta tiles size [default is 8] 66 | meta_size: 8 67 | # the meta tiles buffer [default is 128] 68 | meta_buffer: 128 69 | # connection an sql to get geometries (in column named geom) where we want to generate tiles 70 | # Warn: too complex result can slow down the application 71 | # connection: user=www-data password=www-data dbname= host=localhost 72 | # geoms: 73 | # - sql: AS geom FROM
74 | # size and hash used to detect empty tiles and metatiles [optional, default is None] 75 | empty_metatile_detection: 76 | size: 740 77 | hash: 3237839c217b51b8a9644d596982f342f8041546 78 | empty_tile_detection: 79 | size: 921 80 | hash: 1e3da153be87a493c4c71198366485f290cad43c 81 | 82 | layers: 83 | plan: 84 | <<: *layer 85 | layers: plan 86 | ortho: 87 | <<: *layer 88 | layers: ortho 89 | extension: jpeg 90 | mime_type: image/jpeg 91 | # no buffer needed on rater sources 92 | meta_buffer: 0 93 | empty_metatile_detection: 94 | size: 66163 95 | hash: a9d16a1794586ef92129a2fb41a739451ed09914 96 | empty_tile_detection: 97 | size: 1651 98 | hash: 2892fea0a474228f5d66a534b0b5231d923696da 99 | 100 | generation: 101 | default_cache: local 102 | # used to allowed only a specific user to generate tiles (for rights issue) 103 | authorised_user: www-data 104 | 105 | # maximum allowed consecutive errors, after it exit [default is 10] 106 | maxconsecutive_errors: 10 107 | 108 | process: 109 | optipng_test: 110 | - cmd: optipng -o7 -simulate %(in)s 111 | optipng: 112 | - cmd: optipng %(args)s -zc9 -zm8 -zs3 -f5 %(in)s 113 | arg: 114 | default: '-q' 115 | quiet: '-q' 116 | jpegoptim: 117 | - cmd: jpegoptim %(args)s --strip-all --all-normal -m 90 %(in)s 118 | arg: 119 | default: '-q' 120 | quiet: '-q' 121 | 122 | openlayers: 123 | # srs, center_x, center_y [required] 124 | srs: EPSG:21781 125 | center_x: 600000 126 | center_y: 200000 127 | 128 | metadata: 129 | title: Some title 130 | abstract: Some abstract 131 | servicetype: OGC WMTS 132 | keywords: 133 | - some 134 | - keywords 135 | fees: None 136 | access_constraints: None 137 | 138 | provider: 139 | name: The provider name 140 | url: The provider URL 141 | contact: 142 | name: The contact name 143 | position: The position name 144 | info: 145 | phone: 146 | voice: +41 11 222 33 44 147 | fax: +41 11 222 33 44 148 | address: 149 | delivery: Address delivery 150 | city: Berne 151 | area: BE 152 | postal_code: 3000 153 | country: Switzerland 154 | email: info@example.com 155 | 156 | redis: 157 | socket_timeout: 30 158 | sentinels: 159 | - - redis_sentinel 160 | - 26379 161 | service_name: mymaster 162 | db: 1 163 | 164 | server: 165 | predefined_commands: 166 | - name: Generation all layers 167 | command: generate-tiles 168 | - name: Generation layer plan 169 | command: generate-tiles --layer=plan 170 | - name: Generation layer ortho 171 | command: generate-tiles --layer=ortho 172 | - name: Generate the legend images 173 | command: generate-controller --generate-legend-images 174 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | exclude: |- 2 | (?x)^( 3 | tilecloud_chain/configuration\.py 4 | )$ 5 | 6 | default_language_version: 7 | python: '3.13' 8 | 9 | repos: 10 | - repo: https://github.com/pre-commit/pre-commit-hooks 11 | rev: v6.0.0 12 | hooks: 13 | - id: detect-private-key 14 | - id: check-merge-conflict 15 | - id: check-ast 16 | - id: debug-statements 17 | - id: check-toml 18 | - id: check-yaml 19 | - id: check-json 20 | - id: end-of-file-fixer 21 | - id: trailing-whitespace 22 | - id: mixed-line-ending 23 | - repo: https://github.com/sbrunner/integrity-updater 24 | rev: 1.0.2 25 | hooks: 26 | - id: integrity-updater 27 | - repo: https://github.com/mheap/json-schema-spell-checker 28 | rev: main 29 | hooks: 30 | - id: json-schema-spell-checker 31 | files: tilecloud_chain/schema.json 32 | args: 33 | - --fields=description 34 | - --ignore-numbers 35 | - --ignore-acronyms 36 | - --en-us 37 | - --spelling=.github/spell-ignore-words.txt 38 | - repo: https://github.com/mheap/json-schema-spell-checker 39 | rev: main 40 | hooks: 41 | - id: json-schema-spell-checker 42 | files: tilecloud_chain/host-limit-schema.json 43 | args: 44 | - --fields=description 45 | - --ignore-numbers 46 | - --ignore-acronyms 47 | - --en-us 48 | - --spelling=.github/spell-ignore-words.txt 49 | - repo: https://github.com/pre-commit/mirrors-prettier 50 | rev: v3.1.0 51 | hooks: 52 | - id: prettier 53 | additional_dependencies: 54 | - prettier@2.8.4 55 | - repo: https://github.com/camptocamp/jsonschema-gentypes 56 | rev: 2.12.0 57 | hooks: 58 | - id: jsonschema-gentypes 59 | files: |- 60 | (?x)^( 61 | jsonschema-gentypes\.yaml 62 | |^tilecloud_chain/schema\.json 63 | |^tilecloud_chain/.*-schema\.json 64 | )$ 65 | - repo: https://github.com/sbrunner/jsonschema2md2 66 | rev: 1.7.0 67 | hooks: 68 | - id: jsonschema2md 69 | files: tilecloud_chain/schema\.json 70 | args: 71 | - --pre-commit 72 | - tilecloud_chain/schema.json 73 | - tilecloud_chain/CONFIG.md 74 | - id: jsonschema2md 75 | files: tilecloud_chain/host-limit-schema\.json 76 | args: 77 | - --pre-commit 78 | - tilecloud_chain/host-limit-schema.json 79 | - tilecloud_chain/HOST_LIMIT.md 80 | - repo: https://github.com/sbrunner/hooks 81 | rev: 1.6.1 82 | hooks: 83 | - id: copyright 84 | - id: poetry2-lock 85 | additional_dependencies: 86 | - poetry==2.2.1 # pypi 87 | - id: canonicalize 88 | - id: prospector-to-ruff 89 | additional_dependencies: 90 | - prospector-profile-duplicated==1.10.5 # pypi 91 | - prospector-profile-utils==1.26.5 # pypi 92 | args: 93 | - --test=utils:tests 94 | - repo: https://github.com/codespell-project/codespell 95 | rev: v2.4.1 96 | hooks: 97 | - id: codespell 98 | exclude: ^(.*/)?poetry\.lock$ 99 | args: 100 | - --ignore-words=.github/spell-ignore-words.txt 101 | - repo: https://github.com/shellcheck-py/shellcheck-py 102 | rev: v0.11.0.1 103 | hooks: 104 | - id: shellcheck 105 | - repo: https://github.com/python-jsonschema/check-jsonschema 106 | rev: 0.34.1 107 | hooks: 108 | - id: check-github-actions 109 | - id: check-github-workflows 110 | - id: check-jsonschema 111 | name: Check GitHub Workflows set timeout-minutes 112 | files: ^\.github/workflows/[^/]+$ 113 | types: 114 | - yaml 115 | args: 116 | - --builtin-schema 117 | - github-workflows-require-timeout 118 | - repo: https://github.com/sirwart/ripsecrets 119 | rev: v0.1.9 120 | hooks: 121 | - id: ripsecrets 122 | - repo: https://github.com/astral-sh/ruff-pre-commit 123 | rev: v0.14.3 124 | hooks: 125 | - id: ruff-format 126 | - repo: https://github.com/PyCQA/prospector 127 | rev: v1.17.3 128 | hooks: 129 | - id: prospector 130 | args: 131 | - --profile=utils:pre-commit 132 | - --profile=.prospector.yaml 133 | - --die-on-tool-error 134 | - --output-format=pylint 135 | exclude: |- 136 | (?x)^( 137 | tilecloud_chain/tests/.* 138 | )$ 139 | additional_dependencies: 140 | - prospector-profile-duplicated==1.10.5 # pypi 141 | - prospector-profile-utils==1.26.5 # pypi 142 | - pylint[spelling]==3.3.9 # pypi 143 | - ruff==0.14.3 # pypi 144 | - id: prospector 145 | args: 146 | - --die-on-tool-error 147 | - --output-format=pylint 148 | - --profile=utils:tests 149 | - --profile=utils:pre-commit 150 | additional_dependencies: 151 | - prospector-profile-utils==1.26.5 # pypi 152 | - repo: https://github.com/sbrunner/jsonschema-validator 153 | rev: 1.0.0 154 | hooks: 155 | - id: jsonschema-validator 156 | files: |- 157 | (?x)^( 158 | ci/config\.yaml 159 | |\.github/publish\.yaml 160 | |jsonschema\-gentypes\.yaml 161 | )$ 162 | - repo: https://github.com/renovatebot/pre-commit-hooks 163 | rev: 41.168.5 164 | hooks: 165 | - id: renovate-config-validator 166 | - repo: https://github.com/sbrunner/python-versions-hook 167 | rev: 1.1.2 168 | hooks: 169 | - id: python-versions 170 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.ruff] 2 | line-length = 110 3 | target-version = "py310" 4 | 5 | [tool.ruff.lint] 6 | select = [] 7 | ignore = ["ANN401", "C90", "D100", "D104", "D105", "D107", "D200", "D202", "D207", "D208", "D212", "E501", "EM101", "EM102", "ERA001", "FA", "FBT001", "FBT002", "FIX002", "I001", "PERF203", "PLC0415", "PLR09", "PLR2004", "Q000", "S101", "S113", "SIM105", "T201", "TD002", "W293"] 8 | extend-select = ["UP", "I", "S", "B", "ALL"] 9 | 10 | [tool.ruff.lint.pydocstyle] 11 | convention = "numpy" 12 | 13 | [tool.ruff.lint.extend-per-file-ignores] 14 | "tests/**" = ["ANN", "ARG", "ASYNC", "BLE001", "D", "DTZ", "E722", "FBT003", "INP001", "N", "PGH003", "PLC", "PLR2004", "PLW0603", "PLW1641", "RET", "RUF012", "RUF100", "S104", "S105", "S106", "S108", "S113", "S324", "S603", "S607", "S608", "SLF", "TRY003", "TRY301"] 15 | "**/tests/**" = ["ANN", "ARG", "ASYNC", "BLE001", "D", "DTZ", "E722", "FBT003", "INP001", "N", "PGH003", "PLC", "PLR2004", "PLW0603", "PLW1641", "RET", "RUF012", "RUF100", "S104", "S105", "S106", "S108", "S113", "S324", "S603", "S607", "S608", "SLF", "TRY003", "TRY301"] 16 | "**/test_*.py" = ["ANN", "ARG", "ASYNC", "BLE001", "D", "DTZ", "E722", "FBT003", "INP001", "N", "PGH003", "PLC", "PLR2004", "PLW0603", "PLW1641", "RET", "RUF012", "RUF100", "S104", "S105", "S106", "S108", "S113", "S324", "S603", "S607", "S608", "SLF", "TRY003", "TRY301"] 17 | "**/*_test.py" = ["ANN", "ARG", "ASYNC", "BLE001", "D", "DTZ", "E722", "FBT003", "INP001", "N", "PGH003", "PLC", "PLR2004", "PLW0603", "PLW1641", "RET", "RUF012", "RUF100", "S104", "S105", "S106", "S108", "S113", "S324", "S603", "S607", "S608", "SLF", "TRY003", "TRY301"] 18 | 19 | [tool.pytest.ini_options] 20 | asyncio_mode = "auto" 21 | 22 | [tool.poetry] 23 | version = "0.0.0" 24 | 25 | [tool.poetry.dependencies] 26 | # Minimal version should also be set in the jsonschema-gentypes.yaml file 27 | python = ">=3.11,<3.14" 28 | c2casgiutils = { version = "0.5.0", extras = ["all"] } 29 | fastapi = {extras = ["standard"], version = "0.120.4"} 30 | python-dateutil = "2.9.0.post0" 31 | tilecloud = { version = "1.13.5", extras = ["azure", "aws", "redis", "wsgi"] } 32 | Jinja2 = "3.1.6" 33 | PyYAML = "6.0.3" 34 | Shapely = "2.1.2" 35 | jsonschema = "4.25.1" 36 | jsonschema-validator-new = "0.3.2" 37 | azure-storage-blob = "12.27.1" 38 | certifi = "2025.10.5" 39 | Paste = "3.10.1" 40 | psutil = "7.1.2" 41 | pyproj = "3.7.2" 42 | aiohttp = "3.13.2" 43 | sqlalchemy = { version = "2.0.44", extras = ["asyncio"] } 44 | aiofiles = "24.1.0" 45 | asyncpg = "0.30.0" 46 | html-sanitizer = "2.6.0" 47 | 48 | [tool.poetry.group.dev.dependencies] 49 | prospector = { extras = ["with_mypy", "with_bandit", "with_pyroma", "with_ruff"], version = "1.17.3" } 50 | prospector-profile-duplicated = "1.10.5" 51 | prospector-profile-utils = "1.26.5" 52 | c2cwsgiutils = { version = "6.1.8", extras = ["test_images"] } 53 | scikit-image = { version = "0.25.2" } 54 | pytest = "8.4.2" 55 | testfixtures = "10.0.0" 56 | coverage = "7.11.0" 57 | types-redis = "4.6.0.20241004" 58 | types-requests = "2.32.4.20250913" 59 | types-aiofiles = "25.1.0.20251011" 60 | pytest-asyncio = "1.2.0" 61 | pytest-check = "2.6.0" 62 | types-pyyaml = "6.0.12.20250915" 63 | 64 | [tool.poetry-dynamic-versioning] 65 | enable = true 66 | vcs = "git" 67 | pattern = "^(?P\\d+(\\.\\d+)*)" 68 | format-jinja = """ 69 | {%- if env.get("VERSION_TYPE") == "default_branch" -%} 70 | {{serialize_pep440(bump_version(base, 1), dev=distance)}} 71 | {%- elif env.get("VERSION_TYPE") == "stabilization_branch" -%} 72 | {{serialize_pep440(bump_version(base, 2), dev=distance)}} 73 | {%- elif distance == 0 -%} 74 | {{serialize_pep440(base)}} 75 | {%- else -%} 76 | {{serialize_pep440(bump_version(base), dev=distance)}} 77 | {%- endif -%} 78 | """ 79 | 80 | [tool.poetry-plugin-tweak-dependencies-version] 81 | default = "present" 82 | 83 | [project] 84 | classifiers = [ 85 | 'Development Status :: 5 - Production/Stable', 86 | 'Environment :: Web Environment', 87 | 'Framework :: Pyramid', 88 | 'Intended Audience :: Other Audience', 89 | 'Operating System :: OS Independent', 90 | 'Programming Language :: Python', 91 | 'Programming Language :: Python :: 3', 92 | 'Programming Language :: Python :: 3.10', 93 | 'Programming Language :: Python :: 3.11', 94 | 'Programming Language :: Python :: 3.12', 95 | 'Programming Language :: Python :: 3.13', 96 | 'Topic :: Scientific/Engineering :: GIS', 97 | 'Typing :: Typed', 98 | ] 99 | dynamic = ["dependencies", "version"] 100 | name = "tilecloud-chain" 101 | description = "Tools to generate tiles from WMS or Mapnik, to S3, Berkeley DB, MBTiles, or local filesystem in WMTS layout using Amazon cloud services." 102 | readme = "README.md" 103 | keywords = ["gis", "tilecloud", "chain"] 104 | license = "BSD-2-Clause" 105 | authors = [{name = "Camptocamp",email = "info@camptocamp.com"}] 106 | packages = [{ include = "tilecloud_chain" }] 107 | include = ["tilecloud_chain/py.typed", "tilecloud_chain/*.rst", "tilecloud_chain/*.md"] 108 | requires-python = ">=3.10" 109 | dependencies = ["c2cwsgiutils[broadcast,debug,oauth2,standard]", "python-dateutil", "tilecloud[aws,azure,redis,wsgi]", "Jinja2", "PyYAML", "Shapely", "jsonschema", "jsonschema-validator-new", "azure-storage-blob", "certifi", "Paste", "psutil", "pyproj", "psycopg[binary]", "aiohttp", "sqlalchemy[asyncio]", "pytest-asyncio", "aiofiles", "asyncpg", "nest-asyncio", "c2casgiutils[all]", "fastapi[standard]", "html-sanitizer"] 110 | 111 | [project.urls] 112 | repository = "https://github.com/camptocamp/tilecloud-chain" 113 | "Bug Tracker" = "https://github.com/camptocamp/tilecloud-chain/issues" 114 | 115 | [project.scripts] 116 | generate-tiles = "tilecloud_chain.generate:main" 117 | generate-controller = "tilecloud_chain.controller:main" 118 | generate-cost = "tilecloud_chain.cost:main" 119 | generate-copy = "tilecloud_chain.copy_:main" 120 | generate-process = "tilecloud_chain.copy_:process" 121 | import-expiretiles = "tilecloud_chain.expiretiles:main" 122 | 123 | [build-system] 124 | requires = ["poetry-core>=1.0.0", "poetry-dynamic-versioning"] 125 | build-backend = "poetry.core.masonry.api" 126 | -------------------------------------------------------------------------------- /tilecloud_chain/store/azure_storage_blob.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | from collections.abc import AsyncIterator 4 | 5 | from azure.identity import DefaultAzureCredential 6 | from azure.storage.blob import ContentSettings 7 | from azure.storage.blob.aio import BlobServiceClient, ContainerClient 8 | from tilecloud import Tile, TileLayout 9 | 10 | from tilecloud_chain.store import AsyncTileStore 11 | 12 | _LOGGER = logging.getLogger(__name__) 13 | 14 | 15 | class AzureStorageBlobTileStore(AsyncTileStore): 16 | """Tiles stored in Azure storage blob.""" 17 | 18 | def __init__( 19 | self, 20 | tilelayout: TileLayout, 21 | container: str | None = None, 22 | dry_run: bool = False, 23 | cache_control: str | None = None, 24 | container_client: ContainerClient | None = None, 25 | ) -> None: 26 | """Initialize.""" 27 | if container_client is None: 28 | if "AZURE_STORAGE_CONNECTION_STRING" in os.environ: 29 | assert container is not None 30 | self.container_client = BlobServiceClient.from_connection_string( 31 | os.environ["AZURE_STORAGE_CONNECTION_STRING"], 32 | ).get_container_client(container=container) 33 | elif "AZURE_STORAGE_BLOB_CONTAINER_URL" in os.environ: 34 | self.container_client = ContainerClient.from_container_url( 35 | os.environ["AZURE_STORAGE_BLOB_CONTAINER_URL"], 36 | ) 37 | if os.environ.get("AZURE_STORAGE_BLOB_VALIDATE_CONTAINER_NAME", "false").lower() == "true": 38 | assert container == self.container_client.container_name 39 | else: 40 | assert container is not None 41 | self.container_client = BlobServiceClient( 42 | account_url=os.environ["AZURE_STORAGE_ACCOUNT_URL"], 43 | credential=DefaultAzureCredential(), # type: ignore[arg-type] 44 | ).get_container_client(container=container) 45 | else: 46 | self.container_client = container_client 47 | 48 | self.tilelayout = tilelayout 49 | self.dry_run = dry_run 50 | self.cache_control = cache_control 51 | 52 | async def __contains__(self, tile: Tile) -> bool: 53 | """Return true if this store contains ``tile``.""" 54 | if not tile: 55 | return False 56 | key_name = self.tilelayout.filename(tile.tilecoord, tile.metadata) 57 | blob = self.container_client.get_blob_client(blob=key_name) 58 | return await blob.exists() 59 | 60 | async def delete_one(self, tile: Tile) -> Tile: 61 | """Delete a tile from the store.""" 62 | try: 63 | key_name = self.tilelayout.filename(tile.tilecoord, tile.metadata) 64 | if not self.dry_run: 65 | blob = self.container_client.get_blob_client(blob=key_name) 66 | if blob.exists(): 67 | blob.delete_blob() 68 | except Exception as exc: # pylint: disable=broad-except # noqa: BLE001 69 | _LOGGER.warning("Failed to delete tile %s", tile.tilecoord, exc_info=exc) 70 | tile.error = exc 71 | return tile 72 | 73 | async def get_one(self, tile: Tile) -> Tile | None: 74 | """Get a tile from the store.""" 75 | key_name = self.tilelayout.filename(tile.tilecoord, tile.metadata) 76 | try: 77 | blob = self.container_client.get_blob_client(blob=key_name) 78 | if not await blob.exists(): 79 | return None 80 | download_result = await blob.download_blob() 81 | data = await download_result.readall() 82 | assert isinstance(data, bytes) or data is None, type(data) 83 | tile.data = data 84 | properties = await blob.get_blob_properties() 85 | tile.content_encoding = properties.content_settings.content_encoding 86 | tile.content_type = properties.content_settings.content_type 87 | except Exception as exc: # pylint: disable=broad-except # noqa: BLE001 88 | _LOGGER.warning("Failed to get tile %s", tile.tilecoord, exc_info=exc) 89 | tile.error = exc 90 | return tile 91 | 92 | async def get(self, tiles: AsyncIterator[Tile]) -> AsyncIterator[Tile | None]: 93 | """Get tiles from the store.""" 94 | async for tile in tiles: 95 | yield await self.get_one(tile) 96 | 97 | async def list(self) -> AsyncIterator[Tile]: 98 | """List all the tiles in the store.""" 99 | prefix = getattr(self.tilelayout, "prefix", "") 100 | 101 | async for blob in self.container_client.list_blobs(name_starts_with=prefix): 102 | try: 103 | assert isinstance(blob.name, str) 104 | tilecoord = self.tilelayout.tilecoord(blob.name) 105 | except ValueError: 106 | continue 107 | blob_data = self.container_client.get_blob_client(blob=blob.name) 108 | yield Tile(tilecoord, data=await (await blob_data.download_blob()).readall()) 109 | 110 | async def put_one(self, tile: Tile) -> Tile: 111 | """Store ``tile`` in the store.""" 112 | assert tile.data is not None 113 | key_name = self.tilelayout.filename(tile.tilecoord, tile.metadata) 114 | if not self.dry_run: 115 | try: 116 | blob = self.container_client.get_blob_client(blob=key_name) 117 | await blob.upload_blob( 118 | tile.data, 119 | overwrite=True, 120 | content_settings=ContentSettings( 121 | content_type=tile.content_type, 122 | content_encoding=tile.content_encoding, 123 | cache_control=self.cache_control, 124 | ), 125 | ) 126 | except Exception as exc: # pylint: disable=broad-except # noqa: BLE001 127 | _LOGGER.warning("Failed to put tile %s", tile.tilecoord, exc_info=exc) 128 | tile.error = exc 129 | 130 | return tile 131 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Base of all section, install the apt packages 2 | FROM ghcr.io/osgeo/gdal:ubuntu-small-3.11.4 AS base-all 3 | LABEL org.opencontainers.image.authors="Camptocamp " 4 | 5 | # Fail on error on pipe, see: https://github.com/hadolint/hadolint/wiki/DL4006. 6 | # Treat unset variables as an error when substituting. 7 | # Print commands and their arguments as they are executed. 8 | SHELL ["/bin/bash", "-o", "pipefail", "-cux"] 9 | 10 | RUN --mount=type=cache,target=/var/lib/apt/lists \ 11 | --mount=type=cache,target=/var/cache,sharing=locked \ 12 | apt-get update \ 13 | && apt-get upgrade --assume-yes \ 14 | && apt-get install --assume-yes --no-install-recommends \ 15 | python3-mapnik \ 16 | libdb5.3 \ 17 | fonts-dejavu \ 18 | optipng jpegoptim pngquant \ 19 | postgresql-client net-tools iputils-ping \ 20 | python3-pip python3-venv \ 21 | && python3 -m venv --system-site-packages /venv 22 | 23 | ENV PATH=/venv/bin:$PATH 24 | 25 | # Used to convert the locked packages by poetry to pip requirements format 26 | # We don't directly use `poetry install` because it force to use a virtual environment. 27 | FROM base-all AS poetry 28 | 29 | # Install Poetry 30 | WORKDIR /tmp 31 | COPY requirements.txt ./ 32 | RUN --mount=type=cache,target=/root/.cache \ 33 | python3 -m pip install --disable-pip-version-check --requirement=requirements.txt 34 | 35 | # Do the conversion 36 | COPY poetry.lock pyproject.toml ./ 37 | ENV POETRY_DYNAMIC_VERSIONING_BYPASS=0.0.0 38 | RUN poetry export --output=requirements.txt \ 39 | && poetry export --with=dev --output=requirements-dev.txt 40 | 41 | # Base, the biggest thing is to install the Python packages 42 | FROM base-all AS base 43 | 44 | # hadolint ignore=SC2086 45 | RUN --mount=type=cache,target=/var/lib/apt/lists \ 46 | --mount=type=cache,target=/var/cache,sharing=locked \ 47 | --mount=type=cache,target=/root/.cache \ 48 | --mount=type=bind,from=poetry,source=/tmp,target=/poetry \ 49 | DEV_PACKAGES="python3-dev build-essential libgeos-dev libpq-dev" \ 50 | && apt-get update \ 51 | && apt-get install --assume-yes --no-install-recommends ${DEV_PACKAGES} \ 52 | && python3 -m pip install --disable-pip-version-check --no-deps --requirement=/poetry/requirements.txt \ 53 | && python3 -m compileall /venv/lib/python* /usr/lib/python* \ 54 | && strip /venv/lib/python*/site-packages/shapely/*.so \ 55 | && apt-get remove --purge --autoremove --yes ${DEV_PACKAGES} binutils 56 | 57 | # From c2cwsgiutils 58 | 59 | CMD ["/venv/bin/pserve", "c2c:///app/application.ini"] 60 | 61 | ENV LOG_TYPE=console \ 62 | DEVELOPMENT=0 \ 63 | PKG_CONFIG_ALLOW_SYSTEM_LIBS=OHYESPLEASE 64 | 65 | # End from c2cwsgiutils 66 | 67 | ENV TILEGENERATION_CONFIGFILE=/etc/tilegeneration/config.yaml \ 68 | TILEGENERATION_MAIN_CONFIGFILE=/etc/tilegeneration/config.yaml \ 69 | TILEGENERATION_HOSTSFILE=/etc/tilegeneration/hosts.yaml \ 70 | TILECLOUD_CHAIN_LOG_LEVEL=INFO \ 71 | TILECLOUD_LOG_LEVEL=INFO \ 72 | C2CWSGIUTILS_LOG_LEVEL=WARN \ 73 | WAITRESS_LOG_LEVEL=INFO \ 74 | WSGI_LOG_LEVEL=INFO \ 75 | SQL_LOG_LEVEL=WARN \ 76 | OTHER_LOG_LEVEL=WARN \ 77 | VISIBLE_ENTRY_POINT=/ \ 78 | TILE_SERVER_LOGLEVEL=quiet \ 79 | TILE_MAPCACHE_LOGLEVEL=verbose \ 80 | WAITRESS_THREADS=10 \ 81 | PYRAMID_INCLUDES= \ 82 | DEBUGTOOLBAR_HOSTS= 83 | 84 | EXPOSE 8080 85 | 86 | WORKDIR /app/ 87 | 88 | # The final part 89 | FROM base AS runner 90 | 91 | COPY . /app/ 92 | ARG VERSION=dev 93 | ENV POETRY_DYNAMIC_VERSIONING_BYPASS=dev 94 | RUN --mount=type=cache,target=/root/.cache \ 95 | POETRY_DYNAMIC_VERSIONING_BYPASS=${VERSION} python3 -m pip install --disable-pip-version-check --no-deps --editable=. \ 96 | && mv docker/run /usr/bin/ \ 97 | && python3 -m compileall -q /app/tilecloud_chain 98 | 99 | CMD ["uvicorn", "tilecloud_chain.main:app", "--host=0.0.0.0", "--port=8080", "--log-config=/app/logging.yaml"] 100 | 101 | EXPOSE 8080 102 | 103 | # Do the lint, used by the tests 104 | FROM base AS tests 105 | 106 | # Fail on error on pipe, see: https://github.com/hadolint/hadolint/wiki/DL4006. 107 | # Treat unset variables as an error when substituting. 108 | # Print commands and their arguments as they are executed. 109 | SHELL ["/bin/bash", "-o", "pipefail", "-cux"] 110 | 111 | RUN --mount=type=cache,target=/var/lib/apt/lists \ 112 | --mount=type=cache,target=/var/cache,sharing=locked \ 113 | apt-get update \ 114 | && apt-get install --assume-yes --no-install-recommends software-properties-common gpg-agent \ 115 | && add-apt-repository ppa:savoury1/pipewire \ 116 | && add-apt-repository ppa:savoury1/chromium \ 117 | && apt-get install --assume-yes --no-install-recommends chromium-browser git curl gnupg 118 | COPY .nvmrc /tmp 119 | RUN --mount=type=cache,target=/var/lib/apt/lists \ 120 | --mount=type=cache,target=/var/cache,sharing=locked \ 121 | NODE_MAJOR="$(cat /tmp/.nvmrc)" \ 122 | && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_MAJOR}.x nodistro main" > /etc/apt/sources.list.d/nodesource.list \ 123 | && curl --silent https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor --output=/etc/apt/keyrings/nodesource.gpg \ 124 | && apt-get update \ 125 | && apt-get install --assume-yes --no-install-recommends "nodejs=${NODE_MAJOR}.*" 126 | COPY package.json package-lock.json ./ 127 | RUN --mount=type=cache,target=/root/.npm \ 128 | npm install --include=dev --ignore-scripts 129 | ENV PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser 130 | 131 | RUN --mount=type=cache,target=/root/.cache \ 132 | --mount=type=bind,from=poetry,source=/tmp,target=/poetry \ 133 | python3 -m pip install --disable-pip-version-check --no-deps --requirement=/poetry/requirements-dev.txt 134 | 135 | COPY . ./ 136 | RUN --mount=type=cache,target=/root/.cache \ 137 | POETRY_DYNAMIC_VERSIONING_BYPASS=0.0.0 python3 -m pip install --disable-pip-version-check --no-deps --editable=. \ 138 | && python3 -m pip freeze > /requirements.txt 139 | 140 | ENV TILEGENERATION_MAIN_CONFIGFILE= 141 | 142 | # Set runner as final 143 | FROM runner 144 | -------------------------------------------------------------------------------- /example/tilegeneration/config-postgresql.yaml: -------------------------------------------------------------------------------- 1 | queue_store: postgresql 2 | 3 | grids: 4 | # grid name, I just recommends to add the min resolution because it's common to not generate all the layers at the same resolution. 5 | swissgrid_05: 6 | # resolutions [required] 7 | resolutions: [1000, 500, 250, 100, 50, 20, 10, 5, 2, 1, 0.5] 8 | # bbox [required] 9 | bbox: [420000, 30000, 900000, 350000] 10 | # srs [required] 11 | srs: EPSG:21781 12 | # grid name, I just recommends to add the min resolution because it's common to not generate all the layers at the same resolution. 13 | swissgrid_2056_05: 14 | # resolutions [required] 15 | resolutions: [1000, 500, 250, 100, 50, 20, 10, 5, 2, 1, 0.5] 16 | # bbox [required] 17 | bbox: [2420000, 1030000, 2900000, 1350000] 18 | # srs [required] 19 | srs: EPSG:2056 20 | 21 | caches: 22 | local: 23 | type: filesystem 24 | folder: /var/sig/tiles 25 | # for GetCapabilities 26 | http_url: http://%(host)s/tiles/ 27 | hosts: 28 | - localhost:9052 29 | s3: 30 | type: s3 31 | bucket: tiles 32 | folder: '' 33 | # for GetCapabilities 34 | http_url: https://%(host)s/%(bucket)s/%(folder)s/ 35 | cache_control: 'public, max-age=14400' 36 | hosts: 37 | - wmts0. 38 | 39 | # this defines some defaults values for all the layers 40 | defaults: 41 | layer: &layer 42 | type: wms 43 | # The minimum resolution to seed, useful to use with mapcache, optional. 44 | # min_resolution_seed: 1 45 | # the URL of the WMS server to used 46 | url: http://mapserver:8080/ 47 | # Set the headers to get the right virtual host, and don't get any cached result 48 | headers: 49 | Cache-Control: no-cache, no-store 50 | Pragma: no-cache 51 | # file name extension 52 | extension: png 53 | # the bbox there we want to generate tiles 54 | #bbox: [493000, 114000, 586000, 204000] 55 | 56 | # mime type used for the WMS request and the WMTS capabilities generation 57 | mime_type: image/png 58 | wmts_style: default 59 | # the WMTS dimensions definition [default is []] 60 | #dimensions: 61 | # - name: DATE 62 | # # the default value for the WMTS capabilities 63 | # default: '2012' 64 | # # the generated values 65 | # generate: ['2012'] 66 | # # all the available values in the WMTS capabilities 67 | # values: ['2012'] 68 | # the meta tiles definition [default is off] 69 | meta: true 70 | # the meta tiles size [default is 8] 71 | meta_size: 8 72 | # the meta tiles buffer [default is 128] 73 | meta_buffer: 128 74 | # connection an sql to get geometries (in column named geom) where we want to generate tiles 75 | # Warn: too complex result can slow down the application 76 | # connection: user=www-data password=www-data dbname= host=localhost 77 | # geoms: 78 | # - sql: AS geom FROM
79 | # size and hash used to detect empty tiles and metatiles [optional, default is None] 80 | empty_metatile_detection: 81 | size: 740 82 | hash: 3237839c217b51b8a9644d596982f342f8041546 83 | empty_tile_detection: 84 | size: 921 85 | hash: 1e3da153be87a493c4c71198366485f290cad43c 86 | 87 | layers: 88 | plan: 89 | <<: *layer 90 | layers: line 91 | ortho: 92 | <<: *layer 93 | layers: point 94 | extension: jpeg 95 | mime_type: image/jpeg 96 | # no buffer needed on rater sources 97 | meta_buffer: 0 98 | empty_metatile_detection: 99 | size: 66163 100 | hash: a9d16a1794586ef92129a2fb41a739451ed09914 101 | empty_tile_detection: 102 | size: 1651 103 | hash: 2892fea0a474228f5d66a534b0b5231d923696da 104 | 105 | generation: 106 | default_cache: local 107 | # used to allowed only a specific user to generate tiles (for rights issue) 108 | authorised_user: www-data 109 | 110 | # maximum allowed consecutive errors, after it exit [default is 10] 111 | maxconsecutive_errors: 10 112 | 113 | process: 114 | optipng_test: 115 | - cmd: optipng -o7 -simulate %(in)s 116 | optipng: 117 | - cmd: optipng %(args)s -zc9 -zm8 -zs3 -f5 %(in)s 118 | arg: 119 | default: '-q' 120 | quiet: '-q' 121 | jpegoptim: 122 | - cmd: jpegoptim %(args)s --strip-all --all-normal -m 90 %(in)s 123 | arg: 124 | default: '-q' 125 | quiet: '-q' 126 | 127 | openlayers: 128 | # srs, center_x, center_y [required] 129 | srs: EPSG:21781 130 | center_x: 600000 131 | center_y: 200000 132 | 133 | metadata: 134 | title: Some title 135 | abstract: Some abstract 136 | servicetype: OGC WMTS 137 | keywords: 138 | - some 139 | - keywords 140 | fees: None 141 | access_constraints: None 142 | 143 | provider: 144 | name: The provider name 145 | url: The provider URL 146 | contact: 147 | name: The contact name 148 | position: The position name 149 | info: 150 | phone: 151 | voice: +41 11 222 33 44 152 | fax: +41 11 222 33 44 153 | address: 154 | delivery: Address delivery 155 | city: Berne 156 | area: BE 157 | postal_code: 3000 158 | country: Switzerland 159 | email: info@example.com 160 | 161 | postgresql: {} 162 | 163 | redis: 164 | socket_timeout: 30 165 | sentinels: 166 | - - redis_sentinel 167 | - 26379 168 | service_name: mymaster 169 | db: 1 170 | 171 | server: 172 | predefined_commands: 173 | - name: Generation all layers 174 | command: generate-tiles 175 | - name: Generation layer plan 176 | command: generate-tiles --layer=plan 177 | - name: Generation layer ortho 178 | command: generate-tiles --layer=ortho 179 | - name: Generate the legend images 180 | command: generate-controller --generate-legend-images 181 | - name: Get the hash of plan 182 | command: generate-tiles --layer=plan --get-hash=10/0/0 183 | - name: Get bbox 184 | command: generate-tiles --layer=plan --get-bbox=10/0/0 185 | 186 | admin_footer: The old jobs will be automatically removed 187 | admin_footer_classes: alert alert-dark 188 | -------------------------------------------------------------------------------- /tilecloud_chain/copy_.py: -------------------------------------------------------------------------------- 1 | """Copy the tiles from a cache to an other.""" 2 | 3 | import asyncio 4 | import logging 5 | import os 6 | import sys 7 | from argparse import ArgumentParser, Namespace 8 | from typing import TYPE_CHECKING, cast 9 | 10 | from tilecloud_chain import Count, DropEmpty, HashDropper, TileGeneration, add_common_options 11 | from tilecloud_chain.format import duration_format, size_format 12 | 13 | if TYPE_CHECKING: 14 | import tilecloud_chain.configuration 15 | 16 | 17 | _logger = logging.getLogger(__name__) 18 | 19 | 20 | class Copy: 21 | """Copy the tiles from a cache to an other.""" 22 | 23 | count = None 24 | 25 | async def copy( 26 | self, 27 | options: Namespace, 28 | gene: TileGeneration, 29 | layer_name: str, 30 | source: str, 31 | destination: str, 32 | task_name: str, 33 | ) -> None: 34 | """Copy the tiles from a cache to an other.""" 35 | assert gene.config_file 36 | config = await gene.get_config(gene.config_file) 37 | layer = config.config["layers"][layer_name] 38 | # disable metatiles 39 | cast("tilecloud_chain.configuration.LayerWms", layer)["meta"] = False 40 | count_tiles_dropped = Count() 41 | 42 | await gene.create_log_tiles_error(layer_name) 43 | source_tilestore = gene.get_tilesstore(source) 44 | dest_tilestore = gene.get_tilesstore(destination) 45 | gene.init_tilecoords(config, layer_name, options.grid) 46 | gene.add_geom_filter() 47 | gene.add_logger() 48 | gene.get(source_tilestore, "Get the tiles") 49 | gene.imap(DropEmpty(gene)) 50 | # Discard tiles with certain content 51 | if "empty_tile_detection" in layer: 52 | empty_tile = layer["empty_tile_detection"] 53 | 54 | gene.imap( 55 | HashDropper( 56 | empty_tile["size"], 57 | empty_tile["hash"], 58 | store=dest_tilestore, 59 | count=count_tiles_dropped, 60 | ), 61 | ) 62 | 63 | if options.process: 64 | await gene.process(options.process) 65 | 66 | gene.imap(DropEmpty(gene)) 67 | self.count = gene.counter_size() 68 | gene.put(dest_tilestore, "Store the tiles") 69 | await gene.consume() 70 | if not options.quiet: 71 | print( 72 | f"""The tile {task_name} of layer '{layer_name}' is finish 73 | Nb {task_name} tiles: {self.count.nb} 74 | Nb errored tiles: {gene.error} 75 | Nb dropped tiles: {count_tiles_dropped.nb} 76 | Total time: {duration_format(gene.duration)} 77 | Total size: {size_format(self.count.size)} 78 | Time per tile: {(gene.duration / self.count.nb * 1000).seconds if self.count.nb != 0 else 0} ms 79 | Size per tile: {self.count.size / self.count.nb if self.count.nb != 0 else -1} o 80 | """, 81 | ) 82 | 83 | 84 | def main() -> None: 85 | """Copy the tiles from a cache to an other.""" 86 | asyncio.run(_async_main()) 87 | 88 | 89 | async def _async_main() -> None: 90 | """Copy the tiles from a cache to an other.""" 91 | try: 92 | parser = ArgumentParser( 93 | description="Used to copy the tiles from a cache to an other", 94 | prog=sys.argv[0], 95 | ) 96 | add_common_options(parser, near=False, time=False, dimensions=True, cache=False, grid=True) 97 | parser.add_argument("--process", dest="process", metavar="NAME", help="The process name to do") 98 | parser.add_argument("source", metavar="SOURCE", help="The source cache") 99 | parser.add_argument("dest", metavar="DEST", help="The destination cache") 100 | 101 | options = parser.parse_args() 102 | 103 | gene = TileGeneration(options.config, options, multi_task=False) 104 | await gene.ainit() 105 | assert gene.config_file 106 | config = await gene.get_config(gene.config_file) 107 | 108 | if options.layer: 109 | copy = Copy() 110 | await copy.copy(options, gene, options.layer, options.source, options.dest, "copy") 111 | else: 112 | config = await gene.get_config(gene.config_file) 113 | layers = ( 114 | config.config["generation"]["default_layers"] 115 | if "default_layers" in config.config["generation"] 116 | else config.config["layers"].keys() 117 | ) 118 | for layer in layers: 119 | copy = Copy() 120 | await copy.copy(options, gene, layer, options.source, options.dest, "copy") 121 | except SystemExit: 122 | raise 123 | except: # pylint: disable=bare-except 124 | _logger.exception("Exit with exception") 125 | if os.environ.get("TESTS", "false").lower() == "true": 126 | raise 127 | sys.exit(1) 128 | 129 | 130 | def process() -> None: 131 | """Copy the tiles from a cache to an other.""" 132 | asyncio.run(_async_process()) 133 | 134 | 135 | async def _async_process() -> None: 136 | """Copy the tiles from a cache to an other.""" 137 | try: 138 | parser = ArgumentParser( 139 | description="Used to copy the tiles from a cache to an other", 140 | prog=sys.argv[0], 141 | ) 142 | add_common_options(parser, near=False, time=False, dimensions=True) 143 | parser.add_argument("process", metavar="PROCESS", help="The process name to do") 144 | 145 | options = parser.parse_args() 146 | 147 | gene = TileGeneration(options.config, options, multi_task=False) 148 | await gene.ainit() 149 | 150 | copy = Copy() 151 | if options.layer: 152 | await copy.copy(options, gene, options.layer, options.cache, options.cache, "process") 153 | else: 154 | assert gene.config_file 155 | config = await gene.get_config(gene.config_file) 156 | layers_name = ( 157 | config.config["generation"]["default_layers"] 158 | if "default_layers" in config.config.get("generation", {}) 159 | else config.config["layers"].keys() 160 | ) 161 | for layer in layers_name: 162 | await copy.copy(options, gene, layer, options.cache, options.cache, "process") 163 | except SystemExit: 164 | raise 165 | except: # pylint: disable=bare-except 166 | _logger.exception("Exit with exception") 167 | sys.exit(1) 168 | -------------------------------------------------------------------------------- /tilecloud_chain/store/url.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import json 3 | import logging 4 | import os 5 | import pkgutil 6 | import urllib.parse 7 | from collections.abc import AsyncGenerator, Iterable 8 | from pathlib import Path 9 | from typing import Any, cast 10 | 11 | import aiofiles 12 | import aiohttp 13 | import jsonschema_validator 14 | from ruamel.yaml import YAML 15 | from tilecloud import BoundingPyramid, Tile, TileCoord, TileLayout 16 | 17 | from tilecloud_chain import host_limit 18 | from tilecloud_chain.store import AsyncTileStore 19 | 20 | _LOGGER = logging.getLogger(__name__) 21 | 22 | 23 | class _DatedConfig: 24 | """Loaded config with timestamps to be able to invalidate it on configuration file change.""" 25 | 26 | def __init__(self) -> None: 27 | self.config: host_limit.HostLimit = {} 28 | self.mtime = 0.0 29 | 30 | 31 | class URLTileStore(AsyncTileStore): 32 | """A tile store that reads and writes tiles from a formatted URL.""" 33 | 34 | def __init__( 35 | self, 36 | tile_layouts: Iterable[TileLayout], 37 | headers: Any | None = None, 38 | allows_no_contenttype: bool = False, 39 | bounding_pyramid: BoundingPyramid | None = None, 40 | ) -> None: 41 | self._allows_no_contenttype = allows_no_contenttype 42 | self._tile_layouts = tuple(tile_layouts) 43 | self._bounding_pyramid = bounding_pyramid 44 | self._session = aiohttp.ClientSession() 45 | self._hosts_semaphore: dict[str, asyncio.Semaphore] = {} 46 | self._hosts_limit = _DatedConfig() 47 | if headers is not None: 48 | self._session.headers.update(headers) 49 | 50 | async def _get_hosts_limit(self) -> host_limit.HostLimit: 51 | """Initialize the store.""" 52 | host_limit_path = Path( 53 | os.environ.get( 54 | "TILEGENERATION_HOSTS_LIMIT", 55 | "/etc/tilegeneration/hosts_limit.yaml", 56 | ), 57 | ) 58 | if host_limit_path.exists() and self._hosts_limit.mtime != host_limit_path.stat().st_mtime: 59 | yaml = YAML(typ="safe") 60 | async with aiofiles.open(host_limit_path, encoding="utf-8") as f: 61 | content = await f.read() 62 | self._hosts_limit.config = yaml.load(content) 63 | self._hosts_limit.mtime = host_limit_path.stat().st_mtime 64 | 65 | schema_data = pkgutil.get_data("tilecloud_chain", "host-limit-schema.json") 66 | assert schema_data 67 | errors, _ = jsonschema_validator.validate( 68 | str(host_limit_path), 69 | cast("dict[str, Any]", self._hosts_limit), 70 | json.loads(schema_data), 71 | ) 72 | 73 | if errors: 74 | _LOGGER.error("The host limit file is invalid, ignoring:\n%s", "\n".join(errors)) 75 | self._hosts_limit.config = {} 76 | return self._hosts_limit.config 77 | 78 | async def get_one(self, tile: Tile) -> Tile | None: 79 | """See in superclass.""" 80 | if tile is None: 81 | return None 82 | if self._bounding_pyramid is not None and tile.tilecoord not in self._bounding_pyramid: 83 | return None 84 | tilelayout = self._tile_layouts[hash(tile.tilecoord) % len(self._tile_layouts)] 85 | try: 86 | url = tilelayout.filename(tile.tilecoord, tile.metadata) 87 | except Exception as exception: # pylint: disable=broad-except # noqa: BLE001 88 | _LOGGER.warning("Error while getting tile %s", tile, exc_info=True) 89 | tile.error = exception 90 | return tile 91 | 92 | url_split = urllib.parse.urlparse(url) 93 | assert url_split.hostname is not None 94 | if url_split.hostname in self._hosts_semaphore: 95 | semaphore = self._hosts_semaphore[url_split.hostname] 96 | else: 97 | limit = ( 98 | (await self._get_hosts_limit()) 99 | .get("hosts", {}) 100 | .get(url_split.hostname, {}) 101 | .get( 102 | "concurrent", 103 | (await self._get_hosts_limit()) 104 | .get("default", {}) 105 | .get( 106 | "concurrent", 107 | host_limit.DEFAULT_CONCURRENT_LIMIT_DEFAULT, 108 | ), 109 | ) 110 | ) 111 | semaphore = asyncio.Semaphore(limit) 112 | self._hosts_semaphore[url_split.hostname] = semaphore 113 | 114 | async with semaphore: 115 | _LOGGER.info("GET %s", url) 116 | try: 117 | async with self._session.get(url) as response: 118 | if response.status in (404, 204): 119 | _LOGGER.debug("Got empty tile from %s: %s", url, response.status) 120 | return None 121 | tile.content_encoding = response.headers.get("Content-Encoding") 122 | tile.content_type = response.headers.get("Content-Type") 123 | if response.status < 300: 124 | if response.status != 200: 125 | tile.error = ( 126 | f"URL: {url}\nUnsupported status code {response.status}: {response.reason}" 127 | ) 128 | if tile.content_type: 129 | if tile.content_type.startswith("image/"): 130 | tile.data = await response.read() 131 | else: 132 | tile.error = f"URL: {url}\n{await response.text()}" 133 | elif self._allows_no_contenttype: 134 | tile.data = await response.read() 135 | else: 136 | tile.error = f"URL: {url}\nThe Content-Type header is missing" 137 | 138 | else: 139 | tile.error = f"URL: {url}\n{response.status}: {response.reason}\n{response.text}" 140 | except aiohttp.ClientError as exception: 141 | _LOGGER.warning("Error while getting tile %s", tile, exc_info=True) 142 | tile.error = exception 143 | return tile 144 | 145 | async def __contains__(self, tile: Tile) -> bool: 146 | """See in superclass.""" 147 | raise NotImplementedError 148 | 149 | async def list(self) -> AsyncGenerator[Tile]: 150 | """See in superclass.""" 151 | raise NotImplementedError 152 | yield Tile(TileCoord(0, 0, 0)) # pylint: disable=unreachable 153 | 154 | async def put_one(self, tile: Tile) -> Tile: 155 | """See in superclass.""" 156 | raise NotImplementedError 157 | 158 | async def delete_one(self, tile: Tile) -> Tile: 159 | """See in superclass.""" 160 | raise NotImplementedError 161 | -------------------------------------------------------------------------------- /tilecloud_chain/store/mapnik_.py: -------------------------------------------------------------------------------- 1 | """MapnikTileStore with drop action if the generated tile is empty.""" 2 | 3 | import logging 4 | from collections.abc import AsyncGenerator, Callable 5 | from json import dumps 6 | from typing import Any 7 | 8 | import mapnik # pylint: disable=import-error 9 | from tilecloud import Tile, TileCoord, TileGrid 10 | 11 | from tilecloud_chain.store import AsyncTileStore 12 | 13 | _LOGGER = logging.getLogger(__name__) 14 | 15 | 16 | class MapnikTileStore(AsyncTileStore): 17 | """ 18 | Tile store that renders tiles with Mapnik. 19 | 20 | requires mapnik: https://python-mapnik.readthedocs.io/ 21 | """ 22 | 23 | def __init__( 24 | self, 25 | tilegrid: TileGrid, 26 | mapfile: str, 27 | data_buffer: int = 128, 28 | image_buffer: int = 0, 29 | output_format: str = "png256", 30 | resolution: int = 2, 31 | layers_fields: dict[str, list[str]] | None = None, 32 | drop_empty_utfgrid: bool = False, 33 | proj4_literal: str | None = None, 34 | **kwargs: Any, 35 | ) -> None: 36 | """ 37 | Construct a MapnikTileStore. 38 | 39 | tilegrid: the tilegrid. 40 | mapfile: the file used to render the tiles. 41 | buffer_size: the image buffer size default is 128. 42 | output_format: the output format, 43 | possible values 'jpeg', 'png', 'png256', 'grid', 44 | default is 'png256' 45 | layers_fields: the layers and fields used in the grid generation, 46 | example: { 'my_layer': ['my_first_field', 'my_segonf_field']}, 47 | default is {}. 48 | **kwargs: for extended class. 49 | """ 50 | if layers_fields is None: 51 | layers_fields = {} 52 | 53 | AsyncTileStore.__init__(self, **kwargs) 54 | self.tilegrid = tilegrid 55 | self.buffer = image_buffer 56 | self.output_format = output_format 57 | self.resolution = resolution 58 | self.layers_fields = layers_fields 59 | self.drop_empty_utfgrid = drop_empty_utfgrid 60 | 61 | self.mapnik = mapnik.Map(tilegrid.tile_size, tilegrid.tile_size) # pylint: disable=no-member 62 | mapnik.load_map(self.mapnik, mapfile, True) # noqa: FBT003 # pylint: disable=no-member 63 | self.mapnik.buffer_size = data_buffer 64 | if proj4_literal is not None: 65 | self.mapnik.srs = proj4_literal 66 | 67 | async def get_one(self, tile: Tile) -> Tile | None: 68 | """See in superclass.""" 69 | bbox = self.tilegrid.extent(tile.tilecoord, self.buffer) 70 | bbox2d = mapnik.Box2d(bbox[0], bbox[1], bbox[2], bbox[3]) # pylint: disable=no-member 71 | 72 | size = tile.tilecoord.n * self.tilegrid.tile_size + 2 * self.buffer 73 | self.mapnik.resize(size, size) 74 | self.mapnik.zoom_to_box(bbox2d) 75 | 76 | if self.output_format == "grid": 77 | grid = mapnik.Grid(self.tilegrid.tile_size, self.tilegrid.tile_size) # pylint: disable=no-member 78 | for number, layer in enumerate(self.mapnik.layers): 79 | if layer.name in self.layers_fields: 80 | mapnik.render_layer( # pylint: disable=no-member 81 | self.mapnik, 82 | grid, 83 | layer=number, 84 | fields=self.layers_fields[layer.name], 85 | ) 86 | 87 | encode = grid.encode("utf", resolution=self.resolution) 88 | if self.drop_empty_utfgrid and len(encode["data"].keys()) == 0: 89 | return None 90 | tile.data = dumps(encode).encode() 91 | else: 92 | # Render image with default Agg renderer 93 | image = mapnik.Image(size, size) # pylint: disable=no-member 94 | mapnik.render(self.mapnik, image) # pylint: disable=no-member 95 | tile.data = image.tostring(self.output_format) 96 | 97 | return tile 98 | 99 | async def put_one(self, tile: Tile) -> Tile: 100 | """See in superclass.""" 101 | raise NotImplementedError 102 | 103 | async def delete_one(self, tile: Tile) -> Tile: 104 | """See in superclass.""" 105 | raise NotImplementedError 106 | 107 | async def __contains__(self, tile: Tile) -> bool: 108 | """See in superclass.""" 109 | raise NotImplementedError 110 | 111 | async def list(self) -> AsyncGenerator[Tile]: 112 | """See in superclass.""" 113 | raise NotImplementedError 114 | yield Tile(TileCoord(0, 0, 0)) # pylint: disable=unreachable 115 | 116 | 117 | class MapnikDropActionTileStore(MapnikTileStore): 118 | """MapnikTileStore with drop action if the generated tile is empty.""" 119 | 120 | def __init__( 121 | self, 122 | store: AsyncTileStore | None = None, 123 | queue_store: AsyncTileStore | None = None, 124 | count: list[Callable[[Tile | None], Any]] | None = None, 125 | **kwargs: Any, 126 | ) -> None: 127 | """Initialize.""" 128 | self.store = store 129 | self.queue_store = queue_store 130 | self.count = count or [] 131 | MapnikTileStore.__init__(self, **kwargs) 132 | 133 | async def get_one(self, tile: Tile) -> Tile | None: 134 | """See in superclass.""" 135 | result = await MapnikTileStore.get_one(self, tile) 136 | if result is None: 137 | if self.store is not None: 138 | if tile.tilecoord.n != 1: 139 | for tilecoord in tile.tilecoord: 140 | await self.store.delete_one(Tile(tilecoord)) 141 | else: 142 | await self.store.delete_one(tile) 143 | _LOGGER.info("The tile %s %s is dropped", tile.tilecoord, tile.formated_metadata) 144 | if hasattr(tile, "metatile"): 145 | metatile: Tile = tile.metatile 146 | metatile.elapsed_togenerate -= 1 # type: ignore[attr-defined] 147 | if metatile.elapsed_togenerate == 0 and self.queue_store is not None: # type: ignore[attr-defined] 148 | await self.queue_store.delete_one(metatile) 149 | elif self.queue_store is not None: 150 | await self.queue_store.delete_one(tile) 151 | 152 | for count in self.count: 153 | count(None) 154 | return result 155 | 156 | async def __contains__(self, tile: Tile) -> bool: 157 | """See in superclass.""" 158 | raise NotImplementedError 159 | 160 | async def put_one(self, tile: Tile) -> Tile: 161 | """See in superclass.""" 162 | raise NotImplementedError 163 | 164 | async def delete_one(self, tile: Tile) -> Tile: 165 | """See in superclass.""" 166 | raise NotImplementedError 167 | 168 | async def list(self) -> AsyncGenerator[Tile]: 169 | """See in superclass.""" 170 | raise NotImplementedError 171 | yield Tile(TileCoord(0, 0, 0)) # pylint: disable=unreachable 172 | -------------------------------------------------------------------------------- /tilecloud_chain/expiretiles.py: -------------------------------------------------------------------------------- 1 | """Import the osm2pgsql expire-tiles file to Postgres.""" 2 | 3 | import logging 4 | import sys 5 | from argparse import ArgumentParser 6 | from pathlib import Path 7 | 8 | import psycopg2.sql 9 | from shapely.geometry import MultiPolygon, Polygon 10 | from shapely.ops import unary_union 11 | from tilecloud.grid.quad import QuadTileGrid 12 | 13 | from tilecloud_chain import parse_tilecoord 14 | 15 | logger = logging.getLogger(__name__) 16 | 17 | 18 | def main() -> None: 19 | """Import the osm2pgsql expire-tiles file to Postgres.""" 20 | try: 21 | parser = ArgumentParser( 22 | description="Used to import the osm2pgsql expire-tiles file to Postgres", 23 | prog=sys.argv[0], 24 | ) 25 | parser.add_argument( 26 | "--buffer", 27 | type=float, 28 | default=0.0, 29 | help="Extent buffer to the tiles [m], default is 0", 30 | ) 31 | parser.add_argument( 32 | "--simplify", 33 | type=float, 34 | default=0.0, 35 | help="Simplify the result geometry [m], default is 0", 36 | ) 37 | parser.add_argument( 38 | "--create", 39 | default=False, 40 | action="store_true", 41 | help="create the table if not exists", 42 | ) 43 | parser.add_argument( 44 | "--delete", 45 | default=False, 46 | action="store_true", 47 | help="empty the table", 48 | ) 49 | parser.add_argument( 50 | "file", 51 | type=Path, 52 | metavar="FILE", 53 | help="The osm2pgsql expire-tiles file", 54 | ) 55 | parser.add_argument( 56 | "connection", 57 | metavar="CONNECTION", 58 | help=( 59 | "The PostgreSQL connection string e.g. " 60 | '"user=www-data password=www-data dbname=sig host=localhost"' 61 | ), 62 | ) 63 | parser.add_argument( 64 | "table", 65 | metavar="TABLE", 66 | help="The PostgreSQL table to fill", 67 | ) 68 | parser.add_argument( 69 | "--schema", 70 | default="public", 71 | help="The PostgreSQL schema to use (should already exists), default is public", 72 | ) 73 | parser.add_argument( 74 | "column", 75 | metavar="COLUMN", 76 | default="geom", 77 | nargs="?", 78 | help='The PostgreSQL column, default is "geom"', 79 | ) 80 | parser.add_argument( 81 | "--srid", 82 | type=int, 83 | default=3857, 84 | nargs="?", 85 | help="The stored geometry SRID, no conversion by default (3857)", 86 | ) 87 | options = parser.parse_args() 88 | 89 | connection = psycopg2.connect(options.connection) 90 | cursor = connection.cursor() 91 | 92 | if options.create: 93 | cursor.execute( 94 | "SELECT count(*) FROM pg_tables WHERE schemaname=%(schema)s AND tablename=%(table)s", 95 | {"schema": options.schema, "table": options.table}, 96 | ) 97 | if cursor.fetchone()[0] == 0: 98 | cursor.execute( 99 | psycopg2.sql.SQL("CREATE TABLE IF NOT EXISTS {}.{} (id serial)").format( 100 | psycopg2.sql.Identifier(options.schema), 101 | psycopg2.sql.Identifier(options.table), 102 | ), 103 | ) 104 | cursor.execute( 105 | "SELECT AddGeometryColumn(%(schema)s, %(table)s, %(column)s, %(srid)s, 'MULTIPOLYGON', 2)", 106 | { 107 | "schema": options.schema, 108 | "table": options.table, 109 | "column": options.column, 110 | "srid": options.srid, 111 | }, 112 | ) 113 | 114 | if options.delete: 115 | cursor.execute(psycopg2.sql.SQL("DELETE FROM {}").format(psycopg2.sql.Identifier(options.table))) 116 | 117 | geoms = [] 118 | grid = QuadTileGrid( 119 | max_extent=(-20037508.34, -20037508.34, 20037508.34, 20037508.34), 120 | ) 121 | with options.file.open(encoding="utf-8") as f: 122 | for coord in f: 123 | extent = grid.extent(parse_tilecoord(coord), options.buffer) 124 | geoms.append( 125 | Polygon( 126 | ( 127 | (extent[0], extent[1]), 128 | (extent[0], extent[3]), 129 | (extent[2], extent[3]), 130 | (extent[2], extent[1]), 131 | ), 132 | ), 133 | ) 134 | if len(geoms) == 0: 135 | print("No coords found") 136 | connection.commit() 137 | cursor.close() 138 | connection.close() 139 | sys.exit(0) 140 | geom = unary_union(geoms) 141 | if geom.geom_type == "Polygon": 142 | geom = MultiPolygon((geom,)) 143 | 144 | if options.simplify > 0: 145 | geom.simplify(options.simplify) 146 | 147 | if options.srid <= 0: 148 | cursor.execute( 149 | psycopg2.sql.SQL("INSERT INTO {} ({}) VALUES (ST_GeomFromText(%(geom)s))").format( 150 | psycopg2.sql.Identifier(options.table), 151 | psycopg2.sql.Identifier(options.column), 152 | ), 153 | { 154 | "geom": geom.wkt, 155 | }, 156 | ) 157 | 158 | elif options.srid != 3857: 159 | cursor.execute( 160 | psycopg2.sql.SQL( 161 | "INSERT INTO {} ({}) VALUES (ST_Transform(ST_GeomFromText(%(geom)s, 3857), %(srid)s))", 162 | ).format( 163 | psycopg2.sql.Identifier(options.table), 164 | psycopg2.sql.Identifier(options.column), 165 | ), 166 | { 167 | "geom": geom.wkt, 168 | "srid": options.srid, 169 | }, 170 | ) 171 | else: 172 | cursor.execute( 173 | psycopg2.sql.SQL("INSERT INTO {} ({}) VALUES (ST_GeomFromText(%(geom)s, 3857))").format( 174 | psycopg2.sql.Identifier(options.table), 175 | psycopg2.sql.Identifier(options.column), 176 | ), 177 | { 178 | "geom": geom.wkt, 179 | "srid": options.srid, 180 | }, 181 | ) 182 | 183 | connection.commit() 184 | cursor.close() 185 | connection.close() 186 | print("Import successful") 187 | except SystemExit: 188 | raise 189 | except: # pylint: disable=bare-except 190 | logger.exception("Exit with exception") 191 | sys.exit(1) 192 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/test-nosns.yaml: -------------------------------------------------------------------------------- 1 | grids: 2 | swissgrid_5: 3 | resolutions: [100, 50, 20, 10, 5] 4 | bbox: [420000, 30000, 900000, 350000] 5 | tile_size: 256 6 | srs: EPSG:21781 7 | 8 | swissgrid_01: 9 | resolutions: [1, 0.2, 0.1] 10 | bbox: [420000, 30000, 900000, 350000] 11 | tile_size: 256 12 | srs: EPSG:21781 13 | matrix_identifier: resolution 14 | 15 | swissgrid_2_5: 16 | resolutions: [2.5] 17 | bbox: [420000, 30000, 900000, 350000] 18 | tile_size: 256 19 | srs: EPSG:21781 20 | matrix_identifier: resolution 21 | 22 | swissgrid_025: 23 | resolutions: [0.25] 24 | bbox: [420000, 30000, 900000, 350000] 25 | tile_size: 256 26 | srs: EPSG:21781 27 | matrix_identifier: resolution 28 | 29 | caches: 30 | local: 31 | type: filesystem 32 | http_url: http://wmts1/tiles/ 33 | folder: /tmp/tiles 34 | multi_host: 35 | type: filesystem 36 | http_url: http://%(host)s/tiles/ 37 | folder: /tmp/tiles 38 | hosts: 39 | - wmts1 40 | - wmts2 41 | - wmts3 42 | multi_url: 43 | type: filesystem 44 | http_urls: 45 | - http://wmts1/tiles/ 46 | - http://wmts2/tiles/ 47 | - http://wmts3/tiles/ 48 | folder: /tmp/tiles 49 | mbtiles: 50 | type: mbtiles 51 | http_url: http://wmts1/tiles/ 52 | folder: /tmp/tiles/mbtiles 53 | bsddb: 54 | type: bsddb 55 | http_url: http://wmts1/tiles/ 56 | folder: /tmp/tiles/bsddb 57 | s3: 58 | type: s3 59 | http_url: https://%(host)s/%(bucket)s/%(folder)s/ 60 | host: s3-eu-west-1.amazonaws.com 61 | bucket: tiles 62 | folder: tiles 63 | 64 | defaults: 65 | all_layer: &all_layer 66 | grids: [swissgrid_5] 67 | wmts_style: default 68 | mime_type: image/png 69 | extension: png 70 | dimensions: 71 | - name: DATE 72 | default: '2012' 73 | generate: ['2012'] 74 | values: ['2005', '2010', '2012'] 75 | meta: true 76 | meta_size: 8 77 | meta_buffer: 128 78 | cost: 79 | # [ms] 80 | tileonly_generation_time: 60 81 | # [ms] 82 | tile_generation_time: 30 83 | # [ms] 84 | metatile_generation_time: 30 85 | # [ko] 86 | tile_size: 20 87 | layer: &layer 88 | <<: *all_layer 89 | type: wms 90 | url: http://mapserver:8080/ 91 | 92 | layers: 93 | point: 94 | <<: *layer 95 | layers: point 96 | geoms: 97 | - sql: the_geom AS geom FROM tests.point 98 | connection: user=postgresql password=postgresql dbname=tests host=db 99 | min_resolution_seed: 10 100 | point_error: 101 | <<: *layer 102 | layers: point_error 103 | point_px_buffer: 104 | <<: *layer 105 | layers: point 106 | px_buffer: 100 107 | geoms: 108 | - sql: the_geom AS geom FROM tests.point 109 | connection: user=postgresql password=postgresql dbname=tests host=db 110 | empty_metatile_detection: 111 | size: 20743 112 | hash: 01062bb3b25dcead792d7824f9a7045f0dd92992 113 | empty_tile_detection: 114 | size: 334 115 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 116 | point_hash: &point_hash 117 | <<: *layer 118 | layers: point 119 | geoms: 120 | - sql: the_geom AS geom FROM tests.point 121 | connection: user=postgresql password=postgresql dbname=tests host=db 122 | min_resolution_seed: 10 123 | empty_metatile_detection: 124 | size: 20743 125 | hash: 01062bb3b25dcead792d7824f9a7045f0dd92992 126 | empty_tile_detection: 127 | size: 334 128 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 129 | point_hash_no_meta: 130 | <<: *layer 131 | layers: point 132 | meta: false 133 | empty_tile_detection: 134 | size: 334 135 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 136 | line: 137 | <<: *layer 138 | layers: line 139 | generate_salt: true 140 | geoms: 141 | - sql: the_geom AS geom FROM tests.line 142 | connection: user=postgresql password=postgresql dbname=tests host=db 143 | empty_metatile_detection: 144 | size: 20743 145 | hash: 01062bb3b25dcead792d7824f9a7045f0dd92992 146 | empty_tile_detection: 147 | size: 334 148 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 149 | polygon: 150 | <<: *layer 151 | layers: polygon 152 | meta: false 153 | geoms: 154 | - sql: the_geom AS geom FROM tests.polygon 155 | connection: user=postgresql password=postgresql dbname=tests host=db 156 | empty_metatile_detection: 157 | size: 20743 158 | hash: 01062bb3b25dcead792d7824f9a7045f0dd92992 159 | empty_tile_detection: 160 | size: 334 161 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 162 | polygon2: 163 | <<: *layer 164 | layers: polygon 165 | grids: [swissgrid_01] 166 | geoms: 167 | - sql: the_geom AS geom FROM tests.polygon 168 | connection: user=postgresql password=postgresql dbname=tests host=db 169 | empty_metatile_detection: 170 | size: 20743 171 | hash: 01062bb3b25dcead792d7824f9a7045f0dd92992 172 | empty_tile_detection: 173 | size: 334 174 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 175 | point_webp: 176 | <<: *point_hash 177 | mime_type: image/webp 178 | extension: webp 179 | meta_size: 1 180 | bbox: [420000, 30000, 600000, 150000] 181 | all: 182 | <<: *layer 183 | layers: point,line,polygon 184 | meta: false 185 | bbox: [550000.0, 170000.0, 560000.0, 180000.0] 186 | mapnik: 187 | <<: *all_layer 188 | type: mapnik 189 | mapfile: mapfile/test.mapnik 190 | meta: false 191 | data_buffer: 128 192 | output_format: png 193 | geoms: 194 | - sql: the_geom AS geom FROM tests.polygon 195 | connection: user=postgresql password=postgresql dbname=tests host=db 196 | mapnik_grid: 197 | <<: *all_layer 198 | type: mapnik 199 | mapfile: mapfile/test.mapnik 200 | meta: false 201 | data_buffer: 128 202 | output_format: grid 203 | mime_type: application/utfgrid 204 | extension: json 205 | resolution: 16 206 | geoms: 207 | - sql: the_geom AS geom FROM tests.polygon 208 | connection: user=postgresql password=postgresql dbname=tests host=db 209 | layers_fields: 210 | point: 211 | - name 212 | line: 213 | - name 214 | polygon: 215 | - name 216 | mapnik_grid_drop: 217 | <<: *all_layer 218 | type: mapnik 219 | mapfile: mapfile/test.mapnik 220 | meta: false 221 | meta_buffer: 0 222 | data_buffer: 128 223 | output_format: grid 224 | mime_type: application/utfgrid 225 | extension: json 226 | drop_empty_utfgrid: true 227 | resolution: 16 228 | geoms: 229 | - sql: the_geom AS geom FROM tests.polygon 230 | connection: user=postgresql password=postgresql dbname=tests host=db 231 | layers_fields: 232 | point: 233 | - name 234 | generation: 235 | default_cache: local 236 | default_layers: [line, polygon] 237 | maxconsecutive_errors: 2 238 | error_file: error.list 239 | number_process: 2 240 | 241 | openlayers: 242 | srs: EPSG:21781 243 | center_x: 600000 244 | center_y: 200000 245 | 246 | cost: 247 | # [nb/month] 248 | request_per_layers: 10000000 249 | 250 | server: {} 251 | 252 | sqs: 253 | queue: sqs_point 254 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/test_postgresql.py: -------------------------------------------------------------------------------- 1 | import os 2 | from datetime import datetime, timedelta, timezone 3 | from pathlib import Path 4 | 5 | import pytest 6 | import pytest_asyncio 7 | from sqlalchemy import and_ 8 | from sqlalchemy.engine import create_engine 9 | from sqlalchemy.orm import sessionmaker 10 | from tilecloud import Tile, TileCoord 11 | 12 | from tilecloud_chain import DatedConfig 13 | from tilecloud_chain.store.postgresql import ( 14 | _STATUS_CANCELLED, 15 | _STATUS_CREATED, 16 | _STATUS_DONE, 17 | _STATUS_ERROR, 18 | _STATUS_PENDING, 19 | _STATUS_STARTED, 20 | Job, 21 | PostgresqlTileStore, 22 | Queue, 23 | get_postgresql_queue_store, 24 | ) 25 | 26 | 27 | @pytest_asyncio.fixture 28 | async def tilestore() -> PostgresqlTileStore: 29 | return await get_postgresql_queue_store(DatedConfig({}, 0, "config.yaml")) 30 | 31 | 32 | @pytest.fixture 33 | def SessionMaker() -> sessionmaker: 34 | engine = create_engine(os.environ["TILECLOUD_CHAIN_SQLALCHEMY_URL"]) 35 | return sessionmaker(engine) # noqa 36 | 37 | 38 | @pytest_asyncio.fixture 39 | async def queue(SessionMaker: sessionmaker, tilestore: PostgresqlTileStore) -> tuple[int, int, int]: 40 | with SessionMaker() as session: 41 | for job in session.query(Job).filter(Job.name == "test").all(): 42 | session.delete(job) 43 | session.commit() 44 | await tilestore.create_job("test", "generate-tiles", Path("config.yaml")) 45 | with SessionMaker() as session: 46 | job = session.query(Job).filter(Job.name == "test").one() 47 | job.status = _STATUS_STARTED 48 | job_id = job.id 49 | session.commit() 50 | 51 | await tilestore.put_one( 52 | Tile( 53 | TileCoord(0, 0, 0), 54 | metadata={ 55 | "job_id": job_id, 56 | }, 57 | ), 58 | ) 59 | await tilestore.put_one( 60 | Tile( 61 | TileCoord(1, 0, 0), 62 | metadata={ 63 | "job_id": job_id, 64 | }, 65 | ), 66 | ) 67 | 68 | with SessionMaker() as session: 69 | metatile_0_id = session.query(Queue.id).filter(and_(Queue.job_id == job_id, Queue.zoom == 0)).one()[0] 70 | metatile_1_id = session.query(Queue.id).filter(and_(Queue.job_id == job_id, Queue.zoom == 1)).one()[0] 71 | 72 | yield job_id, metatile_0_id, metatile_1_id 73 | 74 | with SessionMaker() as session: 75 | session.query(Queue).filter(Queue.job_id == job_id).delete() 76 | session.query(Job).filter(Job.id == job_id).delete() 77 | session.commit() 78 | 79 | 80 | @pytest.mark.asyncio 81 | async def test_retry(queue: tuple[int, int, int], SessionMaker: sessionmaker, tilestore: PostgresqlTileStore): 82 | job_id, _, _ = queue 83 | 84 | tile_1 = await anext(tilestore.list()) 85 | tile_2 = await anext(tilestore.list()) 86 | 87 | tile_1.error = "test error" 88 | 89 | await tilestore.delete_one(tile_1) 90 | await tilestore.delete_one(tile_2) 91 | 92 | with SessionMaker() as session: 93 | metatiles = session.query(Queue).filter(Queue.job_id == job_id).all() 94 | assert len(metatiles) == 1 95 | assert metatiles[0].error == "test error" 96 | 97 | await tilestore._maintenance() 98 | 99 | with SessionMaker() as session: 100 | job = session.query(Job).filter(Job.id == job_id).one() 101 | assert job.status == _STATUS_ERROR 102 | 103 | await tilestore.retry(job_id, Path("config.yaml")) 104 | 105 | with SessionMaker() as session: 106 | job = session.query(Job).filter(Job.id == job_id).one() 107 | assert job.status == _STATUS_CREATED 108 | metatiles = session.query(Queue).filter(Queue.job_id == job_id).all() 109 | assert len(metatiles) == 1 110 | assert metatiles[0].status == _STATUS_CREATED 111 | 112 | 113 | @pytest.mark.asyncio 114 | async def test_cancel( 115 | queue: tuple[int, int, int], 116 | SessionMaker: sessionmaker, 117 | tilestore: PostgresqlTileStore, 118 | ): 119 | job_id, _, _ = queue 120 | 121 | tile_1 = await anext(tilestore.list()) 122 | 123 | await tilestore.delete_one(tile_1) 124 | 125 | with SessionMaker() as session: 126 | metatiles = session.query(Queue).filter(Queue.job_id == job_id).all() 127 | assert len(metatiles) == 1 128 | 129 | await tilestore.cancel(job_id, Path("config.yaml")) 130 | 131 | with SessionMaker() as session: 132 | metatiles = session.query(Queue).filter(Queue.job_id == job_id).all() 133 | assert len(metatiles) == 0 134 | job = session.query(Job).filter(Job.id == job_id).one() 135 | assert job.status == _STATUS_CANCELLED 136 | 137 | 138 | @pytest.mark.asyncio 139 | async def test_maintenance_status_done( 140 | queue: tuple[int, int, int], 141 | SessionMaker: sessionmaker, 142 | tilestore: PostgresqlTileStore, 143 | ): 144 | job_id, _, _ = queue 145 | 146 | tile_1 = await anext(tilestore.list()) 147 | tile_2 = await anext(tilestore.list()) 148 | 149 | await tilestore.delete_one(tile_1) 150 | await tilestore.delete_one(tile_2) 151 | 152 | with SessionMaker() as session: 153 | metatiles = session.query(Queue).filter(Queue.job_id == job_id).all() 154 | assert len(metatiles) == 0 155 | 156 | await tilestore._maintenance() 157 | 158 | with SessionMaker() as session: 159 | job = session.query(Job).filter(Job.id == job_id).one() 160 | assert job.status == _STATUS_DONE 161 | 162 | 163 | @pytest.mark.asyncio 164 | async def test_maintenance_pending_tile( 165 | queue: tuple[int, int, int], 166 | SessionMaker: sessionmaker, 167 | tilestore: PostgresqlTileStore, 168 | ): 169 | _job_id, metatile_0_id, metatile_1_id = queue 170 | 171 | with SessionMaker() as session: 172 | metatile_0 = session.query(Queue).filter(Queue.id == metatile_0_id).one() 173 | metatile_0.status = _STATUS_PENDING 174 | metatile_0.started_at = datetime.now(tz=timezone.utc) - timedelta(hours=1) 175 | metatile_1 = session.query(Queue).filter(Queue.id == metatile_1_id).one() 176 | metatile_1.status = _STATUS_PENDING 177 | metatile_1.started_at = datetime.now(tz=timezone.utc) - timedelta(hours=1) 178 | session.commit() 179 | 180 | await tilestore._maintenance() 181 | with SessionMaker() as session: 182 | metatile_0 = session.query(Queue).filter(Queue.id == metatile_0_id).one() 183 | assert metatile_0.status == _STATUS_CREATED 184 | metatile_1 = session.query(Queue).filter(Queue.id == metatile_1_id).one() 185 | assert metatile_1.status == _STATUS_CREATED 186 | 187 | 188 | @pytest.mark.asyncio 189 | async def test_maintenance_pending_job( 190 | queue: tuple[int, int, int], 191 | SessionMaker: sessionmaker, 192 | tilestore: PostgresqlTileStore, 193 | ): 194 | job_id, _metatile_0_id, _metatile_1_id = queue 195 | with SessionMaker() as session: 196 | job = session.query(Job).filter(Job.id == job_id).one() 197 | job.status = _STATUS_PENDING 198 | job.started_at = datetime.now(tz=timezone.utc) - timedelta(hours=1) 199 | 200 | await tilestore._maintenance() 201 | 202 | with SessionMaker() as session: 203 | job = session.query(Job).filter(Job.id == job_id).one() 204 | assert job.status == _STATUS_STARTED 205 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/test.yaml: -------------------------------------------------------------------------------- 1 | grids: 2 | swissgrid_5: 3 | resolutions: [100, 50, 20, 10, 5] 4 | bbox: [420000, 30000, 900000, 350000] 5 | tile_size: 256 6 | srs: EPSG:21781 7 | 8 | swissgrid_01: 9 | resolutions: [1, 0.2, 0.1] 10 | bbox: [420000, 30000, 900000, 350000] 11 | tile_size: 256 12 | srs: EPSG:21781 13 | matrix_identifier: resolution 14 | 15 | swissgrid_2_5: 16 | resolutions: [2.5] 17 | bbox: [420000, 30000, 900000, 350000] 18 | tile_size: 256 19 | srs: EPSG:21781 20 | matrix_identifier: resolution 21 | 22 | swissgrid_025: 23 | resolutions: [0.25] 24 | bbox: [420000, 30000, 900000, 350000] 25 | tile_size: 256 26 | srs: EPSG:21781 27 | matrix_identifier: resolution 28 | 29 | caches: 30 | local: 31 | type: filesystem 32 | http_url: http://wmts1/tiles/ 33 | folder: /tmp/tiles 34 | wmtscapabilities_file: 1.0.0/WMTSCapabilities.xml 35 | multi_host: 36 | type: filesystem 37 | http_url: http://%(host)s/tiles/ 38 | folder: /tmp/tiles 39 | hosts: 40 | - wmts1 41 | - wmts2 42 | - wmts3 43 | multi_url: 44 | type: filesystem 45 | http_urls: 46 | - http://wmts1/tiles/ 47 | - http://wmts2/tiles/ 48 | - http://wmts3/tiles/ 49 | folder: /tmp/tiles 50 | mbtiles: 51 | type: mbtiles 52 | http_url: http://wmts1/tiles/ 53 | folder: /tmp/tiles/mbtiles 54 | bsddb: 55 | type: bsddb 56 | http_url: http://wmts1/tiles/ 57 | folder: /tmp/tiles/bsddb 58 | s3: 59 | type: s3 60 | http_url: https://%(host)s/%(bucket)s/%(folder)s/ 61 | host: s3-eu-west-1.amazonaws.com 62 | bucket: tiles 63 | folder: tiles 64 | 65 | defaults: 66 | base_layer: &base_layer 67 | grids: [swissgrid_5] 68 | wmts_style: default 69 | mime_type: image/png 70 | extension: png 71 | dimensions: 72 | - name: DATE 73 | default: '2012' 74 | generate: ['2012'] 75 | values: ['2005', '2010', '2012'] 76 | meta: true 77 | meta_size: 8 78 | meta_buffer: 128 79 | cost: 80 | # [ms] 81 | tileonly_generation_time: 60 82 | # [ms] 83 | tile_generation_time: 30 84 | # [ms] 85 | metatile_generation_time: 30 86 | # [ko] 87 | tile_size: 20 88 | mapserver_layer: &mapserver_layer 89 | <<: *base_layer 90 | type: wms 91 | url: http://mapserver:8080/ 92 | 93 | layers: 94 | point: 95 | <<: *mapserver_layer 96 | layers: point 97 | geoms: 98 | - sql: the_geom AS geom FROM tests.point 99 | connection: user=postgresql password=postgresql dbname=tests host=db 100 | min_resolution_seed: 10 101 | point_error: 102 | <<: *mapserver_layer 103 | layers: point_error 104 | point_px_buffer: 105 | <<: *mapserver_layer 106 | layers: point 107 | px_buffer: 100 108 | geoms: 109 | - sql: the_geom AS geom FROM tests.point 110 | connection: user=postgresql password=postgresql dbname=tests host=db 111 | empty_metatile_detection: 112 | size: 20743 113 | hash: 01062bb3b25dcead792d7824f9a7045f0dd92992 114 | empty_tile_detection: 115 | size: 334 116 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 117 | point_hash: 118 | <<: *mapserver_layer 119 | layers: point 120 | geoms: 121 | - sql: the_geom AS geom FROM tests.point 122 | connection: user=postgresql password=postgresql dbname=tests host=db 123 | min_resolution_seed: 10 124 | empty_metatile_detection: 125 | size: 20743 126 | hash: 01062bb3b25dcead792d7824f9a7045f0dd92992 127 | empty_tile_detection: 128 | size: 334 129 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 130 | point_hash_no_meta: 131 | <<: *mapserver_layer 132 | layers: point 133 | meta: false 134 | empty_tile_detection: 135 | size: 334 136 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 137 | line: 138 | <<: *mapserver_layer 139 | layers: line 140 | geoms: 141 | - sql: the_geom AS geom FROM tests.line 142 | connection: user=postgresql password=postgresql dbname=tests host=db 143 | empty_metatile_detection: 144 | size: 20743 145 | hash: 01062bb3b25dcead792d7824f9a7045f0dd92992 146 | empty_tile_detection: 147 | size: 334 148 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 149 | polygon: 150 | <<: *mapserver_layer 151 | layers: polygon 152 | meta: false 153 | geoms: 154 | - sql: the_geom AS geom FROM tests.polygon 155 | connection: user=postgresql password=postgresql dbname=tests host=db 156 | empty_metatile_detection: 157 | size: 20743 158 | hash: 01062bb3b25dcead792d7824f9a7045f0dd92992 159 | empty_tile_detection: 160 | size: 334 161 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 162 | polygon2: 163 | <<: *mapserver_layer 164 | layers: polygon 165 | grids: [swissgrid_01] 166 | geoms: 167 | - sql: the_geom AS geom FROM tests.polygon 168 | connection: user=postgresql password=postgresql dbname=tests host=db 169 | empty_metatile_detection: 170 | size: 20743 171 | hash: 01062bb3b25dcead792d7824f9a7045f0dd92992 172 | empty_tile_detection: 173 | size: 334 174 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 175 | all: 176 | <<: *mapserver_layer 177 | layers: point,line,polygon 178 | meta: false 179 | bbox: [550000.0, 170000.0, 560000.0, 180000.0] 180 | mapnik: 181 | <<: *base_layer 182 | type: mapnik 183 | mapfile: mapfile/test.mapnik 184 | meta: false 185 | data_buffer: 128 186 | output_format: png 187 | geoms: 188 | - sql: the_geom AS geom FROM tests.polygon 189 | connection: user=postgresql password=postgresql dbname=tests host=db 190 | wms_url: http://mapserver:8080/ 191 | mapnik_grid: 192 | <<: *base_layer 193 | type: mapnik 194 | mapfile: mapfile/test.mapnik 195 | meta: false 196 | data_buffer: 128 197 | output_format: grid 198 | mime_type: application/utfgrid 199 | extension: json 200 | resolution: 16 201 | geoms: 202 | - sql: the_geom AS geom FROM tests.polygon 203 | connection: user=postgresql password=postgresql dbname=tests host=db 204 | layers_fields: 205 | point: 206 | - name 207 | line: 208 | - name 209 | polygon: 210 | - name 211 | mapnik_grid_drop: 212 | <<: *base_layer 213 | type: mapnik 214 | mapfile: mapfile/test.mapnik 215 | meta: false 216 | meta_buffer: 0 217 | data_buffer: 128 218 | output_format: grid 219 | mime_type: application/utfgrid 220 | extension: json 221 | drop_empty_utfgrid: true 222 | resolution: 16 223 | geoms: 224 | - sql: the_geom AS geom FROM tests.polygon 225 | connection: user=postgresql password=postgresql dbname=tests host=db 226 | layers_fields: 227 | point: 228 | - name 229 | generation: 230 | default_cache: local 231 | default_layers: [line, polygon] 232 | maxconsecutive_errors: 2 233 | error_file: error.list 234 | 235 | openlayers: 236 | srs: EPSG:21781 237 | center_x: 600000 238 | center_y: 200000 239 | 240 | cost: 241 | # [nb/month] 242 | request_per_layers: 10000000 243 | 244 | sns: 245 | topic: arn:aws:sns:eu-west-1:your-account-id:tilecloud 246 | region: eu-west-1 247 | 248 | sqs: 249 | queue: sqs_point 250 | -------------------------------------------------------------------------------- /tilecloud_chain/database_logger.py: -------------------------------------------------------------------------------- 1 | """Log the generated tiles in a database.""" 2 | 3 | import asyncio 4 | import logging 5 | import sys 6 | 7 | import psycopg.sql 8 | from prometheus_client import Summary 9 | from tilecloud import Tile 10 | 11 | import tilecloud_chain.configuration 12 | 13 | _LOGGER = logging.getLogger(__name__) 14 | 15 | _INSERT_SUMMARY = Summary("tilecloud_chain_database_logger", "Number of database inserts", ["layer"]) 16 | 17 | 18 | class DatabaseLoggerCommon: 19 | """Log the generated tiles in a database.""" 20 | 21 | def __init__(self, config: tilecloud_chain.configuration.Logging, daemon: bool) -> None: 22 | self._db_params = config["database"] 23 | self._daemon = daemon 24 | self.connection: psycopg.AsyncConnection | None = None 25 | self.schema: str | None = None 26 | self.table: str | None = None 27 | 28 | async def _init(self) -> None: 29 | while True: 30 | try: 31 | self.connection = await psycopg.AsyncConnection.connect( 32 | dbname=self._db_params["dbname"], 33 | host=self._db_params.get("host"), 34 | port=self._db_params.get("port"), 35 | user=self._db_params.get("user"), 36 | password=self._db_params.get("password"), 37 | ) 38 | break 39 | except psycopg.OperationalError: 40 | _LOGGER.warning("Failed connecting to the database. Will try again in 1s", exc_info=True) 41 | if self._daemon: 42 | await asyncio.sleep(1) 43 | else: 44 | sys.exit(2) 45 | assert self.connection is not None 46 | 47 | if "." in self._db_params["table"]: 48 | schema, table = self._db_params["table"].split(".") 49 | else: 50 | schema = "public" 51 | table = self._db_params["table"] 52 | 53 | async with self.connection.cursor() as cursor: 54 | await cursor.execute( 55 | "SELECT EXISTS(SELECT 1 FROM pg_tables WHERE schemaname=%s AND tablename=%s)", 56 | (schema, table), 57 | ) 58 | schema = psycopg.sql.quote(schema, self.connection) 59 | table = psycopg.sql.quote(table, self.connection) 60 | 61 | elem = await cursor.fetchone() 62 | assert elem is not None 63 | if not elem[0]: 64 | try: 65 | await cursor.execute( 66 | psycopg.sql.SQL( 67 | "CREATE TABLE {}.{} (" 68 | " id BIGSERIAL PRIMARY KEY," 69 | " layer CHARACTER VARYING(80) NOT NULL," 70 | " run INTEGER NOT NULL," 71 | " action CHARACTER VARYING(7) NOT NULL," 72 | " tile TEXT NOT NULL," 73 | " UNIQUE (layer, run, tile))", 74 | ).format(psycopg.sql.Identifier(schema), psycopg.sql.Identifier(table)), 75 | ) 76 | await self.connection.commit() 77 | except psycopg.DatabaseError: 78 | _LOGGER.exception("Unable to create table %s.%s", schema, table) 79 | sys.exit(1) 80 | else: 81 | try: 82 | await cursor.execute( 83 | psycopg.sql.SQL( 84 | "INSERT INTO {}.{}(layer, run, action, tile) VALUES (%s, %s, %s, %s)", 85 | ).format(psycopg.sql.Identifier(schema), psycopg.sql.Identifier(table)), 86 | ("test_layer", -1, "test", "-1x-1"), 87 | ) 88 | except psycopg.DatabaseError: 89 | _LOGGER.exception("Unable to insert logging data into %s.%s", schema, table) 90 | sys.exit(1) 91 | finally: 92 | await self.connection.rollback() 93 | 94 | self.schema = schema 95 | self.table = table 96 | 97 | 98 | class DatabaseLoggerInit(DatabaseLoggerCommon): 99 | """Log the generated tiles in a database.""" 100 | 101 | def __init__(self, config: tilecloud_chain.configuration.Logging, daemon: bool) -> None: 102 | super().__init__(config, daemon) 103 | 104 | self.init = False 105 | self.run = -1 106 | 107 | async def _init(self) -> None: 108 | assert self.connection is not None 109 | assert self.schema is not None 110 | assert self.table is not None 111 | 112 | async with self.connection.cursor() as cursor: 113 | await cursor.execute( 114 | psycopg.sql.SQL("SELECT COALESCE(MAX(run), 0) + 1 FROM {}.{}").format( 115 | psycopg.sql.Identifier(self.schema), 116 | psycopg.sql.Identifier(self.table), 117 | ), 118 | ) 119 | elem = await cursor.fetchone() 120 | assert elem is not None 121 | (self.run,) = elem 122 | self.init = True 123 | 124 | async def __call__(self, tile: Tile) -> Tile: 125 | """Log the generated tiles in a database.""" 126 | if not self.init: 127 | await self._init() 128 | tile.metadata["run"] = self.run # type: ignore[assignment] 129 | return tile 130 | 131 | 132 | class DatabaseLogger(DatabaseLoggerCommon): 133 | """Log the generated tiles in a database.""" 134 | 135 | async def __call__(self, tile: Tile) -> Tile: 136 | """Log the generated tiles in a database.""" 137 | if self.connection is None: 138 | await self._init() 139 | assert self.connection is not None 140 | assert self.schema is not None 141 | assert self.table is not None 142 | 143 | if tile is None: 144 | _LOGGER.warning("The tile is None") 145 | return None 146 | 147 | if tile.error: 148 | action = "error" 149 | elif tile.data: 150 | action = "create" 151 | else: 152 | action = "delete" 153 | 154 | layer = tile.metadata.get("layer", "- No layer -") 155 | run = tile.metadata.get("run", -1) 156 | 157 | with _INSERT_SUMMARY.labels(layer).time(): 158 | async with self.connection.cursor() as cursor: 159 | try: 160 | await cursor.execute( 161 | psycopg.sql.SQL( 162 | "INSERT INTO {} (layer, run, action, tile) " 163 | "VALUES (%(layer)s, %(run)s, %(action)s::varchar(7), %(tile)s)", 164 | ).format(psycopg.sql.Identifier(self.schema), psycopg.sql.Identifier(self.table)), 165 | {"layer": layer, "action": action, "tile": str(tile.tilecoord), "run": run}, 166 | ) 167 | except psycopg.IntegrityError: 168 | await self.connection.rollback() 169 | await cursor.execute( 170 | psycopg.sql.SQL( 171 | "UPDATE {} SET action = %(action)s " 172 | "WHERE layer = %(layer)s AND run = %(run)s AND tile = %(tile)s", 173 | ).format(psycopg.sql.Identifier(self.schema), psycopg.sql.Identifier(self.table)), 174 | {"layer": layer, "action": action, "tile": str(tile.tilecoord), "run": run}, 175 | ) 176 | 177 | await self.connection.commit() 178 | 179 | return tile 180 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/test_expiretiles.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import psycopg2 4 | import pytest 5 | from testfixtures import LogCapture 6 | 7 | from tilecloud_chain import expiretiles 8 | from tilecloud_chain.tests import CompareCase, MatchRegex 9 | 10 | 11 | class TestExpireTiles(CompareCase): 12 | def setUp(self) -> None: 13 | self.maxDiff = None 14 | 15 | @classmethod 16 | def setUpClass(cls): 17 | with Path("/tmp/expired").open("w") as f: 18 | f.write("18/135900/92720\n") 19 | f.write("18/135900/92721\n") 20 | f.write("18/135900/92722\n") 21 | f.write("18/135901/92721\n") 22 | f.write("18/135901/92722\n") 23 | f.write("18/135902/92722\n") 24 | 25 | with Path("/tmp/expired-empty").open("w"): 26 | pass 27 | 28 | @classmethod 29 | def tearDownClass(cls): 30 | Path("/tmp/expired").unlink() 31 | Path("/tmp/expired-empty").unlink() 32 | 33 | def test_expire_tiles( 34 | self, 35 | ) -> None: 36 | with LogCapture("tilecloud_chain", level=30) as log_capture: 37 | geom_re = MatchRegex(r"MULTIPOLYGON\(\(\(([0-9\. ,]+)\)\)\)") 38 | geom_coords = [ 39 | pytest.approx([538274.006497397, 151463.940954133], abs=1e-6), 40 | pytest.approx([538272.927475664, 151358.882137848], abs=1e-6), 41 | pytest.approx([538167.532395446, 151359.965536437], abs=1e-6), 42 | pytest.approx([538062.137334338, 151361.050781072], abs=1e-6), 43 | pytest.approx([537956.742292377, 151362.137871759], abs=1e-6), 44 | pytest.approx([537957.826834589, 151467.19663084], abs=1e-6), 45 | pytest.approx([537958.911357866, 151572.253567259], abs=1e-6), 46 | pytest.approx([537959.995862209, 151677.308681051], abs=1e-6), 47 | pytest.approx([538065.385383791, 151676.221647663], abs=1e-6), 48 | pytest.approx([538064.302719542, 151571.166514773], abs=1e-6), 49 | pytest.approx([538169.694100363, 151570.08130827], abs=1e-6), 50 | pytest.approx([538168.61325734, 151465.024333685], abs=1e-6), 51 | pytest.approx([538274.006497397, 151463.940954133], abs=1e-6), 52 | ] 53 | 54 | self.assert_cmd_equals( 55 | cmd=[ 56 | ".build/venv/bin/import_expiretiles", 57 | "--create", 58 | "--delete", 59 | "--srid", 60 | "21781", 61 | "/tmp/expired", 62 | "user=postgresql password=postgresql dbname=tests host=db", 63 | "expired", 64 | "the_geom", 65 | ], 66 | main_func=expiretiles.main, 67 | expected="""Import successful 68 | """, 69 | ) 70 | connection = psycopg2.connect("user=postgresql password=postgresql dbname=tests host=db") 71 | cursor = connection.cursor() 72 | cursor.execute("SELECT ST_AsText(the_geom) FROM expired") 73 | geoms = [str(r[0]) for r in cursor.fetchall()] 74 | assert [geom_re] == geoms 75 | 76 | def parse_coord(coord: str) -> tuple[float, float]: 77 | coord_split = coord.split(" ") 78 | return [float(c) for c in coord_split] 79 | 80 | assert [parse_coord(e) for e in geom_re.match(geoms[0]).group(1).split(",")] == geom_coords 81 | 82 | self.assert_cmd_equals( 83 | cmd=[ 84 | ".build/venv/bin/import_expiretiles", 85 | "--create", 86 | "--delete", 87 | "--srid", 88 | "21781", 89 | "/tmp/expired", 90 | "user=postgresql password=postgresql dbname=tests host=db", 91 | "expired", 92 | "the_geom", 93 | ], 94 | main_func=expiretiles.main, 95 | expected="""Import successful 96 | """, 97 | ) 98 | connection = psycopg2.connect("user=postgresql password=postgresql dbname=tests host=db") 99 | cursor = connection.cursor() 100 | cursor.execute("SELECT ST_AsText(the_geom) FROM expired") 101 | geoms = [str(r[0]) for r in cursor.fetchall()] 102 | assert [geom_re] == geoms 103 | assert [parse_coord(e) for e in geom_re.match(geoms[0]).group(1).split(",")] == geom_coords 104 | 105 | self.assert_cmd_equals( 106 | cmd=[ 107 | ".build/venv/bin/import_expiretiles", 108 | "--simplify", 109 | "1000", 110 | "--create", 111 | "--delete", 112 | "/tmp/expired", 113 | "user=postgresql password=postgresql dbname=tests host=db", 114 | "expired2", 115 | ], 116 | main_func=expiretiles.main, 117 | expected="""Import successful 118 | """, 119 | ) 120 | connection = psycopg2.connect("user=postgresql password=postgresql dbname=tests host=db") 121 | cursor = connection.cursor() 122 | cursor.execute("SELECT ST_AsText(geom) FROM expired2") 123 | geoms = [str(r[0]) for r in cursor.fetchall()] 124 | geom_coords = [ 125 | pytest.approx([738534.567188568, 5862720.06865692], abs=1e-6), 126 | pytest.approx([738534.567188568, 5862567.19460037], abs=1e-6), 127 | pytest.approx([738381.693132021, 5862567.19460037], abs=1e-6), 128 | pytest.approx([738228.819075469, 5862567.19460037], abs=1e-6), 129 | pytest.approx([738075.945018921, 5862567.19460037], abs=1e-6), 130 | pytest.approx([738075.945018921, 5862720.06865692], abs=1e-6), 131 | pytest.approx([738075.945018921, 5862872.94271347], abs=1e-6), 132 | pytest.approx([738075.945018921, 5863025.81677002], abs=1e-6), 133 | pytest.approx([738228.819075469, 5863025.81677002], abs=1e-6), 134 | pytest.approx([738228.819075469, 5862872.94271347], abs=1e-6), 135 | pytest.approx([738381.693132021, 5862872.94271347], abs=1e-6), 136 | pytest.approx([738381.693132021, 5862720.06865692], abs=1e-6), 137 | pytest.approx([738534.567188568, 5862720.06865692], abs=1e-6), 138 | ] 139 | assert [geom_re] == geoms 140 | assert [parse_coord(e) for e in geom_re.match(geoms[0]).group(1).split(",")] == geom_coords 141 | 142 | log_capture.check() 143 | 144 | def test_expire_tiles_empty(self) -> None: 145 | with LogCapture("tilecloud_chain", level=30): 146 | self.assert_cmd_equals( 147 | cmd=[ 148 | ".build/venv/bin/import_expiretiles", 149 | "--create", 150 | "--delete", 151 | "--srid", 152 | "21781", 153 | "/tmp/expired-empty", 154 | "user=postgresql password=postgresql dbname=tests host=db", 155 | "expired", 156 | "the_geom", 157 | ], 158 | main_func=expiretiles.main, 159 | expected="""No coords found 160 | """, 161 | ) 162 | connection = psycopg2.connect("user=postgresql password=postgresql dbname=tests host=db") 163 | cursor = connection.cursor() 164 | cursor.execute("SELECT the_geom FROM expired") 165 | geoms = cursor.fetchall() 166 | assert len(geoms) == 0 167 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/test-fix.yaml: -------------------------------------------------------------------------------- 1 | grids: 2 | swissgrid_5: 3 | resolutions: [100, 50, 20, 10, 5] 4 | bbox: [420000, 30000, 900000, 350000] 5 | tile_size: 256 6 | srs: EPSG:21781 7 | 8 | swissgrid_01: 9 | resolutions: [1, 0.2, 0.1] 10 | bbox: [420000, 30000, 900000, 350000] 11 | tile_size: 256 12 | srs: EPSG:21781 13 | matrix_identifier: resolution 14 | 15 | swissgrid_2_5: 16 | resolutions: [2.5] 17 | bbox: [420000, 30000, 900000, 350000] 18 | tile_size: 256 19 | srs: EPSG:21781 20 | matrix_identifier: resolution 21 | 22 | swissgrid_025: 23 | resolutions: [0.25] 24 | bbox: [420000, 30000, 900000, 350000] 25 | tile_size: 256 26 | srs: EPSG:21781 27 | matrix_identifier: resolution 28 | 29 | caches: 30 | local: 31 | type: filesystem 32 | http_url: http://wmts1/tiles/ 33 | folder: /tmp/tiles 34 | wmtscapabilities_file: 1.0.0/WMTSCapabilities.xml 35 | multi_host: 36 | type: filesystem 37 | http_url: http://%(host)s/tiles/ 38 | folder: /tmp/tiles 39 | hosts: 40 | - wmts1 41 | - wmts2 42 | - wmts3 43 | multi_url: 44 | type: filesystem 45 | http_urls: 46 | - http://wmts1/tiles/ 47 | - http://wmts2/tiles/ 48 | - http://wmts3/tiles/ 49 | folder: /tmp/tiles 50 | mbtiles: 51 | type: mbtiles 52 | http_url: http://wmts1/tiles/ 53 | folder: /tmp/tiles/mbtiles 54 | s3: 55 | type: s3 56 | http_url: https://%(host)s/%(bucket)s/%(folder)s/ 57 | host: s3-eu-west-1.amazonaws.com 58 | bucket: tiles 59 | folder: tiles 60 | cache_control: 'public, max-age=14400' 61 | 62 | defaults: 63 | layer: &layer 64 | grids: [swissgrid_5] 65 | type: wms 66 | wmts_style: default 67 | mime_type: image/png 68 | extension: png 69 | dimensions: 70 | - name: DATE 71 | default: '2012' 72 | generate: ['2012'] 73 | values: ['2005', '2010', '2012'] 74 | meta: true 75 | meta_size: 8 76 | meta_buffer: 128 77 | cost: 78 | # [ms] 79 | tileonly_generation_time: 60 80 | # [ms] 81 | tile_generation_time: 30 82 | # [ms] 83 | metatile_generation_time: 30 84 | # [ko] 85 | tile_size: 20 86 | 87 | layers: 88 | point: 89 | <<: *layer 90 | url: http://mapserver:8080/ 91 | layers: point 92 | geoms: 93 | - sql: the_geom AS geom FROM tests.point 94 | connection: user=postgresql password=postgresql dbname=tests host=db 95 | min_resolution_seed: 10 96 | point_px_buffer: 97 | <<: *layer 98 | url: http://mapserver:8080/ 99 | layers: point 100 | px_buffer: 100 101 | geoms: 102 | - sql: the_geom AS geom FROM tests.point 103 | connection: user=postgresql password=postgresql dbname=tests host=db 104 | empty_metatile_detection: 105 | size: 20743 106 | hash: 01062bb3b25dcead792d7824f9a7045f0dd92992 107 | empty_tile_detection: 108 | size: 334 109 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 110 | point_hash: 111 | <<: *layer 112 | url: http://mapserver:8080/ 113 | layers: point 114 | geoms: 115 | - sql: the_geom AS geom FROM tests.point 116 | connection: user=postgresql password=postgresql dbname=tests host=db 117 | min_resolution_seed: 10 118 | empty_metatile_detection: 119 | size: 20743 120 | hash: 01062bb3b25dcead792d7824f9a7045f0dd92992 121 | empty_tile_detection: 122 | size: 334 123 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 124 | point_hash_no_meta: 125 | <<: *layer 126 | url: http://mapserver:8080/ 127 | layers: point 128 | meta: false 129 | empty_tile_detection: 130 | size: 334 131 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 132 | line: 133 | <<: *layer 134 | url: http://mapserver:8080/ 135 | layers: line 136 | headers: 137 | Cache-Control: no-cache 138 | params: 139 | PARAM: value 140 | geoms: 141 | - sql: the_geom AS geom FROM tests.line 142 | connection: user=postgresql password=postgresql dbname=tests host=db 143 | empty_metatile_detection: 144 | size: 20743 145 | hash: 01062bb3b25dcead792d7824f9a7045f0dd92992 146 | empty_tile_detection: 147 | size: 334 148 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 149 | polygon: 150 | <<: *layer 151 | url: http://mapserver:8080/ 152 | layers: polygon 153 | meta: false 154 | geoms: 155 | - sql: the_geom AS geom FROM tests.polygon 156 | connection: user=postgresql password=postgresql dbname=tests host=db 157 | empty_metatile_detection: 158 | size: 20743 159 | hash: 01062bb3b25dcead792d7824f9a7045f0dd92992 160 | empty_tile_detection: 161 | size: 334 162 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 163 | polygon2: 164 | <<: *layer 165 | url: http://mapserver:8080/ 166 | layers: polygon 167 | grids: [swissgrid_01] 168 | geoms: 169 | - sql: the_geom AS geom FROM tests.polygon 170 | connection: user=postgresql password=postgresql dbname=tests host=db 171 | empty_metatile_detection: 172 | size: 20743 173 | hash: 01062bb3b25dcead792d7824f9a7045f0dd92992 174 | empty_tile_detection: 175 | size: 334 176 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 177 | all: 178 | <<: *layer 179 | url: http://mapserver:8080/ 180 | layers: point,line,polygon 181 | meta: false 182 | bbox: [550000.0, 170000.0, 560000.0, 180000.0] 183 | mapnik: 184 | <<: *layer 185 | type: mapnik 186 | mapfile: mapfile/test.mapnik 187 | meta: false 188 | data_buffer: 128 189 | output_format: png 190 | geoms: 191 | - sql: the_geom AS geom FROM tests.polygon 192 | connection: user=postgresql password=postgresql dbname=tests host=db 193 | mapnik_grid: 194 | <<: *layer 195 | type: mapnik 196 | mapfile: mapfile/test.mapnik 197 | meta: false 198 | data_buffer: 128 199 | output_format: grid 200 | mime_type: application/utfgrid 201 | extension: json 202 | resolution: 16 203 | geoms: 204 | - sql: the_geom AS geom FROM tests.polygon 205 | connection: user=postgresql password=postgresql dbname=tests host=db 206 | layers_fields: 207 | point: 208 | - name 209 | line: 210 | - name 211 | polygon: 212 | - name 213 | mapnik_grid_drop: 214 | <<: *layer 215 | type: mapnik 216 | mapfile: mapfile/test.mapnik 217 | meta: false 218 | meta_buffer: 0 219 | data_buffer: 128 220 | output_format: grid 221 | mime_type: application/utfgrid 222 | extension: json 223 | drop_empty_utfgrid: true 224 | resolution: 16 225 | geoms: 226 | - sql: the_geom AS geom FROM tests.polygon 227 | connection: user=postgresql password=postgresql dbname=tests host=db 228 | layers_fields: 229 | point: 230 | - name 231 | generation: 232 | default_cache: local 233 | default_layers: [line, polygon] 234 | maxconsecutive_errors: 2 235 | error_file: error.list 236 | 237 | openlayers: 238 | srs: EPSG:21781 239 | center_x: 600000 240 | center_y: 200000 241 | 242 | cost: 243 | # [nb/month] 244 | request_per_layers: 10000000 245 | s3: {} 246 | cloudfront: {} 247 | sqs: {} 248 | 249 | sqs: 250 | queue: sqs_point 251 | 252 | sns: 253 | topic: arn:aws:sns:eu-west-1:your-account-id:tilecloud 254 | region: eu-west-1 255 | 256 | metadata: 257 | title: Some title 258 | abstract: Some abstract 259 | keywords: 260 | - some 261 | - keywords 262 | fees: None 263 | access_constraints: None 264 | 265 | provider: 266 | name: The provider name 267 | url: The provider URL 268 | contact: 269 | name: The contact name 270 | position: The position name 271 | info: 272 | phone: 273 | voice: +41 11 222 33 44 274 | fax: +41 11 222 33 44 275 | address: 276 | delivery: Address delivery 277 | city: Berne 278 | area: BE 279 | postal_code: 3000 280 | country: Switzerland 281 | email: info@example.com 282 | --------------------------------------------------------------------------------