├── .coveragerc ├── .dockerignore ├── .editorconfig ├── .gitattributes ├── .github ├── dpkg-versions.yaml ├── publish.yaml ├── renovate.json5 ├── spell-ignore-words.txt └── workflows │ ├── main.yaml │ ├── pull-request-automation.yaml │ └── rebuild.yaml ├── .gitignore ├── .hadolint.yaml ├── .nvmrc ├── .pre-commit-config.yaml ├── .prettierignore ├── .prospector.yaml ├── .python-version ├── .secretsignore ├── .sonarcloud.properties ├── CHANGES.md ├── Chaines.dia ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── SECURITY.md ├── admin-screenshot.png ├── application.ini ├── ci ├── config.yaml └── requirements.txt ├── docker-compose.override.sample.yaml ├── docker-compose.yaml ├── docker ├── mapfile-docker │ ├── circle.svg │ ├── mapserver.conf │ ├── mapserver.map │ └── test.mapnik ├── run └── test-db │ └── 10_init.sql ├── example └── tilegeneration │ ├── config-postgresql.yaml │ └── config.yaml ├── jsonschema-gentypes.yaml ├── package-lock.json ├── package.json ├── poetry.lock ├── pyproject.toml ├── requirements.txt ├── screenshot.js ├── setup.cfg └── tilecloud_chain ├── CONFIG.md ├── HOST_LIMIT.md ├── USAGE.rst ├── __init__.py ├── configuration.py ├── controller.py ├── copy_.py ├── cost.py ├── database_logger.py ├── expiretiles.py ├── filter ├── __init__.py └── error.py ├── format.py ├── generate.py ├── host-limit-schema.json ├── host_limit.py ├── internal_mapcache.py ├── multitilestore.py ├── py.typed ├── schema.json ├── security.py ├── server.py ├── static ├── favicon-16x16.png └── favicon-32x32.png ├── store ├── __init__.py ├── azure_storage_blob.py ├── mapnik_.py ├── postgresql.py └── url.py ├── templates ├── admin_index.html └── openlayers.html ├── tests ├── __init__.py ├── apache.conf ├── create_test_data.sh ├── index.expected.png ├── mapfile │ ├── circle.svg │ ├── mapfile.map │ └── test.mapnik ├── not-login.expected.png ├── test.expected.png ├── test_config.py ├── test_controller.py ├── test_copy.py ├── test_cost.py ├── test_error.py ├── test_expiretiles.py ├── test_generate.py ├── test_postgresql.py ├── test_serve.py ├── test_ui.py └── tilegeneration │ ├── deploy.cfg │ ├── hooks │ ├── post-create-database │ └── post-restore-database │ ├── test-apache-s3-tilesurl.yaml │ ├── test-authorised.yaml │ ├── test-bsddb.yaml │ ├── test-capabilities.yaml │ ├── test-copy.yaml │ ├── test-fix.yaml │ ├── test-int-grid.yaml │ ├── test-internal-mapcache.yaml │ ├── test-legends.yaml │ ├── test-multidim.yaml │ ├── test-multigeom.yaml │ ├── test-nodim.yaml │ ├── test-nosns.yaml │ ├── test-redis-main.yaml │ ├── test-redis-project.yaml │ ├── test-redis.yaml │ ├── test-serve-wmtscapabilities.yaml │ ├── test-serve.yaml │ ├── test.yaml │ ├── wrong_map.yaml │ ├── wrong_mapnik_grid_meta.yaml │ ├── wrong_resolutions.yaml │ ├── wrong_sequence.yaml │ ├── wrong_srs.yaml │ ├── wrong_srs_auth.yaml │ ├── wrong_srs_id.yaml │ └── wrong_type.yaml ├── timedtilestore.py ├── views ├── __init__.py └── admin.py └── wmts_get_capabilities.jinja /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | source = tilecloud_chain 3 | omit = */tests/* 4 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | * 2 | !requirements.txt 3 | !poetry.lock 4 | !pyproject.toml 5 | !README.md 6 | !CHANGES.md 7 | !MANIFEST.in 8 | !tilecloud_chain/* 9 | !docker/run 10 | !application.ini 11 | !package*.json 12 | !.nvmrc 13 | !screenshot.js 14 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | end_of_line = lf 5 | insert_final_newline = true 6 | charset = utf-8 7 | indent_style = space 8 | indent_size = 4 9 | trim_trailing_whitespace = true 10 | max_line_length = 110 11 | quote_type = single 12 | 13 | [{Makefile,*.mk}] 14 | indent_style = tab 15 | 16 | [*.{json,json5,js,yaml,md,html,whitesource}] 17 | indent_size = 2 18 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | * text whitespace=trailing-space,tab-in-indent,cr-at-eol,tabwidth=4 eol=lf 2 | .gitmodules text whitespace=indent-with-non-tab,tabwidth=2 3 | Makefile text whitespace=indent-with-non-tab,tabwidth=2 4 | *.rst tex conflict-marker-size=100 5 | *.png binary 6 | -------------------------------------------------------------------------------- /.github/publish.yaml: -------------------------------------------------------------------------------- 1 | # yaml-language-server: $schema=https://raw.githubusercontent.com/camptocamp/tag-publish/1.0.1/tag_publish/schema.json 2 | 3 | docker: 4 | images: 5 | - name: camptocamp/tilecloud-chain 6 | pypi: 7 | packages: 8 | - {} 9 | dispatch: 10 | - {} 11 | -------------------------------------------------------------------------------- /.github/renovate.json5: -------------------------------------------------------------------------------- 1 | { 2 | extends: [ 3 | 'github>camptocamp/gs-renovate-config-preset:base.json5#1.1.2', 4 | 'github>camptocamp/gs-renovate-config-preset:group.json5#1.1.2', 5 | 'github>camptocamp/gs-renovate-config-preset:stabilization-branches.json5#1.1.2', 6 | 'github>camptocamp/gs-renovate-config-preset:ci.json5#1.1.2', 7 | 'github>camptocamp/gs-renovate-config-preset:preset.json5#1.1.2', 8 | 'github>camptocamp/gs-renovate-config-preset:pre-commit.json5#1.1.2', 9 | 'github>camptocamp/gs-renovate-config-preset:python.json5#1.1.2', 10 | 'github>camptocamp/gs-renovate-config-preset:security.json5#1.1.2', 11 | 'github>camptocamp/gs-renovate-config-preset:docker.json5#1.1.2', 12 | 'github>camptocamp/gs-renovate-config-preset:own.json5#1.1.2', 13 | 'github>camptocamp/gs-renovate-config-preset:json-schema.json5#1.1.2', 14 | 'github>camptocamp/gs-renovate-config-preset:shellcheck.json5#1.1.2', 15 | ], 16 | baseBranches: ['1.17', '1.19', '1.20', '1.21', '1.22', 'master'], 17 | customManagers: [ 18 | /** Manage unpkg */ 19 | { 20 | fileMatch: ['.*\\.html$'], 21 | matchStrings: ['unpkg\\.com/(?[^@]+)@(?[^/]+)'], 22 | datasourceTemplate: 'npm', 23 | customType: 'regex', 24 | }, 25 | /** Manage jsdelivr */ 26 | { 27 | fileMatch: ['.*\\.html$'], 28 | matchStrings: ['cdn\\.jsdelivr\\.net/npm/(?[^@]+)@(?[^/]+)'], 29 | datasourceTemplate: 'npm', 30 | customType: 'regex', 31 | }, 32 | ], 33 | packageRules: [ 34 | /** Docker images versioning */ 35 | { 36 | matchDatasources: ['docker'], 37 | versioning: 'loose', 38 | }, 39 | { 40 | matchDatasources: ['docker'], 41 | matchDepNames: ['camptocamp/mapserver'], 42 | versioning: 'regex:^(?\\d+)\\.(?\\d+)$', 43 | }, 44 | { 45 | matchDatasources: ['docker'], 46 | matchDepNames: ['redis'], 47 | versioning: 'regex:^(?\\d+)\\.(?\\d+)\\.(?\\d+)$', 48 | }, 49 | { 50 | matchDatasources: ['docker'], 51 | matchDepNames: ['ghcr.io/osgeo/gdal'], 52 | versioning: 'regex:^(?.*)-(?\\d+)\\.(?\\d+)\\.(?\\d+)?$', 53 | }, 54 | /** Automerge the patch, the minor and the dev dependency */ 55 | { 56 | matchUpdateTypes: ['minor', 'patch'], 57 | automerge: true, 58 | }, 59 | /** Accept only the patch on stabilization branches */ 60 | { 61 | matchBaseBranches: ['/^[0-9]+\\.[0-9]+$/'], 62 | matchUpdateTypes: ['major', 'minor', 'pin', 'digest', 'lockFileMaintenance', 'rollback', 'bump'], 63 | enabled: false, 64 | }, 65 | /** Disable upgrading the supported Python version */ 66 | { 67 | matchFileNames: ['pyproject.toml'], 68 | matchDepNames: ['python'], 69 | enabled: false, 70 | }, 71 | /** Only LTS version of Node */ 72 | { 73 | allowedVersions: '/(0|2|4|6|8)$/', 74 | matchDepNames: ['node'], 75 | enabled: false, 76 | }, 77 | /** Disable types-request update on version <= 1.21 */ 78 | { 79 | matchDepNames: ['types-requests'], 80 | matchBaseBranches: ['1.17', '1.18', '1.19', '1.20', '1.21'], 81 | enabled: false, 82 | }, 83 | /** Automatically update all versions of cryptography to fic CVE */ 84 | { 85 | matchDepNames: ['cryptography'], 86 | enabled: true, 87 | automerge: true, 88 | schedule: 'at any time', 89 | }, 90 | /** Packages published very recently are not pushed to stabilization branches for security reasons */ 91 | { 92 | matchBaseBranches: ['/^[0-9]+\\.[0-9]+$/'], 93 | minimumReleaseAge: '7 days', 94 | }, 95 | /** Ungroup Gdal */ 96 | { 97 | matchDepNames: ['ghcr.io/osgeo/gdal'], 98 | groupName: 'Gdal', 99 | }, 100 | /** In file `.python-version`, use the `.` version */ 101 | { 102 | matchFileNames: ['.python-version'], 103 | versioning: 'regex:^(?\\d+)\\.(?\\d+)$', 104 | }, 105 | ], 106 | $schema: 'https://docs.renovatebot.com/renovate-schema.json', 107 | } 108 | -------------------------------------------------------------------------------- /.github/spell-ignore-words.txt: -------------------------------------------------------------------------------- 1 | ro 2 | backend 3 | center 4 | Gio 5 | SQLAlchemy 6 | FreeTileGrid 7 | Proj4 8 | PostgreSQL 9 | Mapnik 10 | Redis 11 | CloudFront 12 | OpenLayers 13 | pyproj 14 | -------------------------------------------------------------------------------- /.github/workflows/main.yaml: -------------------------------------------------------------------------------- 1 | name: Continuous integration 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | - '[0-9]+.[0-9]+' 8 | tags: 9 | - '*' 10 | pull_request: 11 | 12 | env: 13 | HAS_SECRETS: ${{ secrets.HAS_SECRETS }} 14 | 15 | jobs: 16 | main: 17 | name: Continuous integration 18 | runs-on: ubuntu-24.04 19 | timeout-minutes: 30 20 | if: "!startsWith(github.event.head_commit.message, '[skip ci] ')" 21 | 22 | env: 23 | REDIS_URL: redis://localhost:6379 24 | 25 | steps: 26 | - run: docker system prune --all --force 27 | - uses: actions/checkout@v4 28 | with: 29 | fetch-depth: 0 30 | 31 | - uses: camptocamp/initialise-gopass-summon-action@v2 32 | with: 33 | ci-gpg-private-key: ${{secrets.CI_GPG_PRIVATE_KEY}} 34 | github-gopass-ci-token: ${{secrets.GOPASS_CI_GITHUB_TOKEN}} 35 | patterns: pypi docker 36 | if: env.HAS_SECRETS == 'HAS_SECRETS' 37 | 38 | - uses: actions/setup-python@v5 39 | with: 40 | python-version: '3.12' 41 | - run: python3 -m pip install --requirement=ci/requirements.txt 42 | 43 | - uses: actions/cache@v4 44 | with: 45 | path: ~/.cache/pre-commit 46 | key: pre-commit-${{ hashFiles('.pre-commit-config.yaml') }} 47 | restore-keys: "pre-commit-${{ hashFiles('.pre-commit-config.yaml') }}\npre-commit-" 48 | - run: pre-commit run --all-files --color=always 49 | - run: git diff --exit-code --patch > /tmp/pre-commit.patch; git diff --color; git reset --hard || true 50 | if: failure() 51 | - uses: actions/upload-artifact@v4 52 | with: 53 | name: Apply pre-commit fix.patch 54 | path: /tmp/pre-commit.patch 55 | retention-days: 1 56 | if: failure() 57 | 58 | - name: Print environment information 59 | run: c2cciutils-env 60 | env: 61 | GITHUB_EVENT: ${{ toJson(github) }} 62 | 63 | - name: Build 64 | run: make build 65 | 66 | - name: Checks 67 | run: make checks 68 | 69 | - name: Tests 70 | run: make tests 71 | 72 | - run: c2cciutils-docker-logs 73 | if: always() 74 | 75 | - uses: actions/upload-artifact@v4 76 | with: 77 | name: results 78 | path: results 79 | if-no-files-found: ignore 80 | retention-days: 5 81 | if: failure() 82 | 83 | - run: git reset --hard 84 | - name: Publish 85 | run: tag-publish 86 | if: env.HAS_SECRETS == 'HAS_SECRETS' 87 | env: 88 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 89 | - run: git diff --exit-code --patch > /tmp/dpkg-versions.patch; git diff --color; git reset --hard || true 90 | if: failure() 91 | - uses: actions/upload-artifact@v4 92 | with: 93 | name: Update dpkg versions list.patch 94 | path: /tmp/dpkg-versions.patch 95 | retention-days: 1 96 | if: failure() 97 | permissions: 98 | contents: write 99 | packages: write 100 | id-token: write 101 | -------------------------------------------------------------------------------- /.github/workflows/pull-request-automation.yaml: -------------------------------------------------------------------------------- 1 | name: Auto reviews, merge and close pull requests 2 | 3 | on: 4 | pull_request: 5 | types: 6 | - opened 7 | - reopened 8 | jobs: 9 | auto-merge: 10 | name: Auto reviews pull requests from bots 11 | runs-on: ubuntu-24.04 12 | timeout-minutes: 5 13 | 14 | steps: 15 | - name: Print event 16 | run: echo "${GITHUB}" | jq 17 | env: 18 | GITHUB: ${{ toJson(github) }} 19 | - name: Print context 20 | uses: actions/github-script@v7 21 | with: 22 | script: |- 23 | console.log(context); 24 | - name: Auto reviews GHCI updates 25 | uses: actions/github-script@v7 26 | with: 27 | script: |- 28 | github.rest.pulls.createReview({ 29 | owner: context.repo.owner, 30 | repo: context.repo.repo, 31 | pull_number: context.payload.pull_request.number, 32 | event: 'APPROVE', 33 | }) 34 | if: |- 35 | startsWith(github.head_ref, 'ghci/audit/') 36 | && (github.event.pull_request.user.login == 'geo-ghci-test[bot]' 37 | || github.event.pull_request.user.login == 'geo-ghci-int[bot]' 38 | || github.event.pull_request.user.login == 'geo-ghci[bot]') 39 | - name: Auto reviews Renovate updates 40 | uses: actions/github-script@v7 41 | with: 42 | script: |- 43 | github.rest.pulls.createReview({ 44 | owner: context.repo.owner, 45 | repo: context.repo.repo, 46 | pull_number: context.payload.pull_request.number, 47 | event: 'APPROVE', 48 | }) 49 | if: |- 50 | github.event.pull_request.user.login == 'renovate[bot]' 51 | -------------------------------------------------------------------------------- /.github/workflows/rebuild.yaml: -------------------------------------------------------------------------------- 1 | name: Rebuild 2 | 3 | on: 4 | schedule: 5 | - cron: 30 2 * * * 6 | 7 | jobs: 8 | rebuild: 9 | name: Rebuild 10 | runs-on: ubuntu-24.04 11 | timeout-minutes: 20 12 | 13 | strategy: 14 | fail-fast: false 15 | matrix: 16 | include: 17 | - branch: '1.17' 18 | 19 | env: 20 | REDIS_URL: redis://localhost:6379 21 | 22 | steps: 23 | - run: docker system prune --all --force 24 | - uses: actions/checkout@v4 25 | with: 26 | ref: ${{ matrix.branch }} 27 | 28 | - uses: camptocamp/initialise-gopass-summon-action@v2 29 | with: 30 | ci-gpg-private-key: ${{secrets.CI_GPG_PRIVATE_KEY}} 31 | github-gopass-ci-token: ${{secrets.GOPASS_CI_GITHUB_TOKEN}} 32 | patterns: pypi docker 33 | 34 | - uses: actions/setup-python@v5 35 | with: 36 | python-version: '3.10' 37 | - run: python3 -m pip install setuptools==76.1.0 wheel==0.45.1 38 | - run: python3 -m pip install --user --requirement=ci/requirements.txt 39 | 40 | - name: Checks 41 | run: c2cciutils-checks 42 | 43 | - name: Build 44 | run: make build 45 | 46 | - name: Tests 47 | run: make tests 48 | 49 | - run: c2cciutils-docker-logs 50 | if: always() 51 | 52 | - name: Publish 53 | run: c2cciutils-publish --type=rebuild --branch=${{ matrix.branch }} 54 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[co] 2 | /dist 3 | /.build 4 | /.mypy_cache/ 5 | /.github/changelog-generator-cache 6 | /PYPI.md 7 | /tilecloud_chain.egg-info/ 8 | /tilecloud_chain/tests/error.list 9 | /tilecloud_chain/tests/mapcache.xml 10 | /tilecloud_chain/tests/tiles.conf 11 | /node_modules/ 12 | /results/ 13 | /docker-compose.override.yaml 14 | -------------------------------------------------------------------------------- /.hadolint.yaml: -------------------------------------------------------------------------------- 1 | ignored: 2 | - DL3008 # Pin versions in apt get install. Instead of `apt-get install ` use `apt-get install =` 3 | - DL3042 # Avoid use of cache directory with pip. Use `pip install --no-cache-dir ` 4 | -------------------------------------------------------------------------------- /.nvmrc: -------------------------------------------------------------------------------- 1 | 18 2 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | exclude: |- 2 | (?x)^( 3 | tilecloud_chain/configuration\.py 4 | )$ 5 | 6 | repos: 7 | - repo: https://github.com/pre-commit/pre-commit-hooks 8 | rev: v5.0.0 9 | hooks: 10 | - id: detect-private-key 11 | - id: check-merge-conflict 12 | - id: check-ast 13 | - id: debug-statements 14 | - id: check-toml 15 | - id: check-yaml 16 | - id: check-json 17 | - id: end-of-file-fixer 18 | - id: trailing-whitespace 19 | - id: mixed-line-ending 20 | - repo: https://github.com/sbrunner/integrity-updater 21 | rev: 1.0.2 22 | hooks: 23 | - id: integrity-updater 24 | - repo: https://github.com/mheap/json-schema-spell-checker 25 | rev: main 26 | hooks: 27 | - id: json-schema-spell-checker 28 | files: tilecloud_chain/schema.json 29 | args: 30 | - --fields=description 31 | - --ignore-numbers 32 | - --ignore-acronyms 33 | - --en-us 34 | - --spelling=.github/spell-ignore-words.txt 35 | - repo: https://github.com/mheap/json-schema-spell-checker 36 | rev: main 37 | hooks: 38 | - id: json-schema-spell-checker 39 | files: tilecloud_chain/host-limit-schema.json 40 | args: 41 | - --fields=description 42 | - --ignore-numbers 43 | - --ignore-acronyms 44 | - --en-us 45 | - --spelling=.github/spell-ignore-words.txt 46 | - repo: https://github.com/pre-commit/mirrors-prettier 47 | rev: v3.1.0 48 | hooks: 49 | - id: prettier 50 | additional_dependencies: 51 | - prettier@2.8.4 52 | - repo: https://github.com/camptocamp/jsonschema-gentypes 53 | rev: 2.11.0 54 | hooks: 55 | - id: jsonschema-gentypes 56 | files: |- 57 | (?x)^( 58 | jsonschema-gentypes\.yaml 59 | |^tilecloud_chain/schema\.json 60 | |^tilecloud_chain/.*-schema\.json 61 | )$ 62 | - repo: https://github.com/sbrunner/jsonschema2md2 63 | rev: 1.5.2 64 | hooks: 65 | - id: jsonschema2md 66 | files: tilecloud_chain/schema\.json 67 | args: 68 | - --pre-commit 69 | - tilecloud_chain/schema.json 70 | - tilecloud_chain/CONFIG.md 71 | - id: jsonschema2md 72 | files: tilecloud_chain/host-limit-schema\.json 73 | args: 74 | - --pre-commit 75 | - tilecloud_chain/host-limit-schema.json 76 | - tilecloud_chain/HOST_LIMIT.md 77 | - repo: https://github.com/sbrunner/hooks 78 | rev: 1.4.1 79 | hooks: 80 | - id: copyright 81 | - id: poetry2-lock 82 | additional_dependencies: 83 | - poetry==2.1.3 # pypi 84 | - id: canonicalize 85 | - repo: https://github.com/codespell-project/codespell 86 | rev: v2.4.1 87 | hooks: 88 | - id: codespell 89 | exclude: ^(.*/)?poetry\.lock$ 90 | args: 91 | - --ignore-words=.github/spell-ignore-words.txt 92 | - repo: https://github.com/shellcheck-py/shellcheck-py 93 | rev: v0.10.0.1 94 | hooks: 95 | - id: shellcheck 96 | - repo: https://github.com/python-jsonschema/check-jsonschema 97 | rev: 0.33.0 98 | hooks: 99 | - id: check-github-actions 100 | - id: check-github-workflows 101 | - id: check-jsonschema 102 | name: Check GitHub Workflows set timeout-minutes 103 | files: ^\.github/workflows/[^/]+$ 104 | types: 105 | - yaml 106 | args: 107 | - --builtin-schema 108 | - github-workflows-require-timeout 109 | - repo: https://github.com/sirwart/ripsecrets 110 | rev: v0.1.9 111 | hooks: 112 | - id: ripsecrets 113 | - repo: https://github.com/astral-sh/ruff-pre-commit 114 | rev: v0.11.12 115 | hooks: 116 | - id: ruff-format 117 | - repo: https://github.com/PyCQA/prospector 118 | rev: v1.17.1 119 | hooks: 120 | - id: prospector 121 | args: 122 | - --profile=utils:pre-commit 123 | - --profile=.prospector.yaml 124 | - --die-on-tool-error 125 | - --output-format=pylint 126 | exclude: |- 127 | (?x)^( 128 | tilecloud_chain/tests/.* 129 | )$ 130 | additional_dependencies: 131 | - prospector-profile-duplicated==1.10.5 # pypi 132 | - prospector-profile-utils==1.22.3 # pypi 133 | - pylint[spelling]==3.3.7 # pypi 134 | - ruff==0.11.12 # pypi 135 | - id: prospector 136 | args: 137 | - --die-on-tool-error 138 | - --output-format=pylint 139 | - --profile=utils:tests 140 | - --profile=utils:pre-commit 141 | additional_dependencies: 142 | - prospector-profile-utils==1.22.3 # pypi 143 | - repo: https://github.com/sbrunner/jsonschema-validator 144 | rev: 1.0.0 145 | hooks: 146 | - id: jsonschema-validator 147 | files: |- 148 | (?x)^( 149 | ci/config\.yaml 150 | |\.github/publish\.yaml 151 | |jsonschema\-gentypes\.yaml 152 | )$ 153 | - repo: https://github.com/renovatebot/pre-commit-hooks 154 | rev: 40.36.8 155 | hooks: 156 | - id: renovate-config-validator 157 | - repo: https://github.com/sbrunner/python-versions-hook 158 | rev: 1.1.2 159 | hooks: 160 | - id: python-versions 161 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | poetry.lock 2 | -------------------------------------------------------------------------------- /.prospector.yaml: -------------------------------------------------------------------------------- 1 | inherits: 2 | - utils:base 3 | - utils:no-design-checks 4 | - utils:fix 5 | - utils:c2cwsgiutils 6 | - duplicated 7 | 8 | ignore-paths: 9 | - tilecloud_chain/configuration.py 10 | 11 | pylint: 12 | disable: 13 | - cyclic-import 14 | 15 | mypy: 16 | options: 17 | python-version: '3.10' 18 | 19 | ruff: 20 | options: 21 | target-version: py310 22 | -------------------------------------------------------------------------------- /.python-version: -------------------------------------------------------------------------------- 1 | 3.13 2 | -------------------------------------------------------------------------------- /.secretsignore: -------------------------------------------------------------------------------- 1 | [secrets] 2 | 1234567890123456789 3 | -------------------------------------------------------------------------------- /.sonarcloud.properties: -------------------------------------------------------------------------------- 1 | sonar.exclusions=tilecloud_chain/tests 2 | -------------------------------------------------------------------------------- /CHANGES.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## Release 1.17 4 | 5 | 1. Change the validator and parser => duplicate key generate an error: on/off are no more considered as boolean. 6 | 2. The argument --layer is no more used when we use the parameter --tiles, we get the information from the 7 | tiles file. 8 | 3. Be able to mutualise the service. 9 | 4. Add Azure blob storage 10 | 5. Remove Apache and MapCache 11 | 6. Remove the `log_format` in the `generation` configuration, nor we use the logging configuration from the 12 | `development.ini` file. 13 | 14 | ## Release 1.16 15 | 16 | 1. Change the config validator who is a little bit more strict. 17 | 18 | ## Release 1.4 19 | 20 | 1. Add optional `metadata` section to the config file. See the scaffolds for example. 21 | 22 | ## Release 0.9 23 | 24 | 1. Correct some error with slash. 25 | 2. Better error handling. 26 | 3. Be able to have one error file per layer. 27 | 28 | ## Release 0.8 29 | 30 | 1. Correct some error with slash. 31 | 2. Add `pre_hash_post_process` and `post_process`. 32 | 3. Add copy command. 33 | 34 | ## Release 0.7 35 | 36 | 1. Support of deferent geoms per layers, requires configuration changes, old version: 37 | 38 | > ```yaml 39 | > connection: user=www-data password=www-data dbname= host=localhost 40 | > sql: AS geom FROM 41 | > ``` 42 | > 43 | > to new version: 44 | > 45 | > ```yaml 46 | > connection: user=www-data password=www-data dbname= host=localhost 47 | > geoms: 48 | > - sql: AS geom FROM
49 | > ``` 50 | > 51 | > More information in the **Configure geom/sql** chapter. 52 | 53 | 2. Update from `optparse` to `argparse`, and some argument refactoring, use `--help` to see the new version. 54 | 3. Add support of Blackbery DB (`bsddb`). 55 | 4. The tile `server` is completely rewrite, now it support all cache, `REST` and `KVP` interface, 56 | `GetFeatureInfo` request, and it can be used as a pyramid view or as a `WSGI` server. More information in 57 | the **istribute the tiles** chapter. 58 | 5. Add three strategy to bypass the proxy/cache: Use the headers `Cache-Control: no-cache, no-store`, 59 | `Pragma: no-cache` (default). Use localhost in the URL and the header `Host: ` (recommended). 60 | Add a `SALT` random argument (if the above don't work). More information in the **Proxy/cache issue** 61 | chapter. 62 | 6. Improve the dimensions usage by adding it ti the WMS requests, And add a `--dimensions` argument of 63 | `generate_tiles` to change the dimensions values. 64 | 7. Extract generate_cost and generate_amazon from generate_controler. 65 | 8. Now we can creates legends, see the **Legends** chapter. 66 | 9. Now the tiles generation display generation statistics at the ends. 67 | 10. The EC2 configuration is moved in a separate structure, see README for more information. 68 | 69 | ## Release 0.6 70 | 71 | 1. Now the apache configuration can be generated with 72 | `.build/venv/bin/generate_controller --generate-apache-config`, it support `filesystem` `cache` and 73 | `MapCache`. 74 | 2. Windows fixes. 75 | 3. Use console rewrite (r) to log generated tiles coordinates. 76 | 4. Now if no layers is specified in `generation:default_layers` we generate all layers by default. 77 | 5. Now bbox to be floats. 78 | 6. New `--get-bbox` option to get the bbox of a tile. 79 | 7. Add coveralls support (). 80 | 8. Add an config option `generation:error_file` and a command option `--tiles` to store and regenerate 81 | errored tiles. 82 | 83 | ## Release 0.5 84 | 85 | 1. SQS config change: 86 | 87 | ```yaml 88 | layers: 89 | layer_name: 90 | sqs: 91 | # The region where the SQS queue is 92 | region: eu-west-1 93 | # The SQS queue name, it should already exists 94 | queue: the_name 95 | ``` 96 | 97 | 2. Add debug option (`--debug`), please use it to report issue. 98 | 3. Now the `sql` request can return a set of geometries in a column names geom but the syntax change a little 99 | bit => ` AS geom FROM
` 100 | -------------------------------------------------------------------------------- /Chaines.dia: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/camptocamp/tilecloud-chain/2f84239147eb4025af917a862d8b3cb38f4a88e1/Chaines.dia -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Base of all section, install the apt packages 2 | FROM ghcr.io/osgeo/gdal:ubuntu-small-3.11.0 AS base-all 3 | LABEL org.opencontainers.image.authors="Camptocamp " 4 | 5 | # Fail on error on pipe, see: https://github.com/hadolint/hadolint/wiki/DL4006. 6 | # Treat unset variables as an error when substituting. 7 | # Print commands and their arguments as they are executed. 8 | SHELL ["/bin/bash", "-o", "pipefail", "-cux"] 9 | 10 | RUN --mount=type=cache,target=/var/lib/apt/lists \ 11 | --mount=type=cache,target=/var/cache,sharing=locked \ 12 | apt-get update \ 13 | && apt-get upgrade --assume-yes \ 14 | && apt-get install --assume-yes --no-install-recommends \ 15 | libmapnik3.1 mapnik-utils \ 16 | libdb5.3 \ 17 | fonts-dejavu \ 18 | optipng jpegoptim pngquant \ 19 | postgresql-client net-tools iputils-ping \ 20 | python3-pip python3-venv \ 21 | && python3 -m venv /venv 22 | 23 | ENV PATH=/venv/bin:$PATH 24 | 25 | # Used to convert the locked packages by poetry to pip requirements format 26 | # We don't directly use `poetry install` because it force to use a virtual environment. 27 | FROM base-all AS poetry 28 | 29 | # Install Poetry 30 | WORKDIR /tmp 31 | COPY requirements.txt ./ 32 | RUN --mount=type=cache,target=/root/.cache \ 33 | python3 -m pip install --disable-pip-version-check --requirement=requirements.txt 34 | 35 | # Do the conversion 36 | COPY poetry.lock pyproject.toml ./ 37 | ENV POETRY_DYNAMIC_VERSIONING_BYPASS=0.0.0 38 | RUN poetry export --output=requirements.txt \ 39 | && poetry export --with=dev --output=requirements-dev.txt 40 | 41 | # Base, the biggest thing is to install the Python packages 42 | FROM base-all AS base 43 | 44 | # hadolint ignore=SC2086 45 | RUN --mount=type=cache,target=/var/lib/apt/lists \ 46 | --mount=type=cache,target=/var/cache,sharing=locked \ 47 | --mount=type=cache,target=/root/.cache \ 48 | --mount=type=bind,from=poetry,source=/tmp,target=/poetry \ 49 | DEV_PACKAGES="python3-dev build-essential libgeos-dev libmapnik-dev libpq-dev build-essential" \ 50 | && apt-get update \ 51 | && apt-get install --assume-yes --no-install-recommends ${DEV_PACKAGES} \ 52 | && python3 -m pip install --disable-pip-version-check --no-deps --requirement=/poetry/requirements.txt \ 53 | && python3 -m compileall /venv/lib/python* /usr/lib/python* \ 54 | && strip /venv/lib/python*/site-packages/shapely/*.so \ 55 | && apt-get remove --purge --autoremove --yes ${DEV_PACKAGES} binutils 56 | 57 | # From c2cwsgiutils 58 | 59 | CMD ["/venv/bin/pserve", "c2c:///app/application.ini"] 60 | 61 | ENV LOG_TYPE=console \ 62 | DEVELOPMENT=0 \ 63 | PKG_CONFIG_ALLOW_SYSTEM_LIBS=OHYESPLEASE 64 | 65 | ENV C2C_SECRET= \ 66 | C2C_BASE_PATH=/c2c \ 67 | C2C_REDIS_URL= \ 68 | C2C_REDIS_SENTINELS= \ 69 | C2C_REDIS_TIMEOUT=3 \ 70 | C2C_REDIS_SERVICENAME=mymaster \ 71 | C2C_REDIS_DB=0 \ 72 | C2C_BROADCAST_PREFIX=broadcast_api_ \ 73 | C2C_REQUEST_ID_HEADER= \ 74 | C2C_REQUESTS_DEFAULT_TIMEOUT= \ 75 | C2C_SQL_PROFILER_ENABLED=0 \ 76 | C2C_PROFILER_PATH= \ 77 | C2C_PROFILER_MODULES= \ 78 | C2C_DEBUG_VIEW_ENABLED=0 \ 79 | C2C_ENABLE_EXCEPTION_HANDLING=0 80 | 81 | # End from c2cwsgiutils 82 | 83 | ENV TILEGENERATION_CONFIGFILE=/etc/tilegeneration/config.yaml \ 84 | TILEGENERATION_MAIN_CONFIGFILE=/etc/tilegeneration/config.yaml \ 85 | TILEGENERATION_HOSTSFILE=/etc/tilegeneration/hosts.yaml \ 86 | TILECLOUD_CHAIN_LOG_LEVEL=INFO \ 87 | TILECLOUD_LOG_LEVEL=INFO \ 88 | C2CWSGIUTILS_LOG_LEVEL=WARN \ 89 | WAITRESS_LOG_LEVEL=INFO \ 90 | WSGI_LOG_LEVEL=INFO \ 91 | SQL_LOG_LEVEL=WARN \ 92 | OTHER_LOG_LEVEL=WARN \ 93 | VISIBLE_ENTRY_POINT=/ \ 94 | TILE_NB_THREAD=2 \ 95 | METATILE_NB_THREAD=25 \ 96 | SERVER_NB_THREAD=10 \ 97 | TILE_QUEUE_SIZE=2 \ 98 | TILE_CHUNK_SIZE=1 \ 99 | TILE_SERVER_LOGLEVEL=quiet \ 100 | TILE_MAPCACHE_LOGLEVEL=verbose \ 101 | WAITRESS_THREADS=10 \ 102 | PYRAMID_INCLUDES= \ 103 | DEBUGTOOLBAR_HOSTS= 104 | 105 | EXPOSE 8080 106 | 107 | WORKDIR /app/ 108 | 109 | # The final part 110 | FROM base AS runner 111 | 112 | COPY . /app/ 113 | ARG VERSION=dev 114 | ENV POETRY_DYNAMIC_VERSIONING_BYPASS=dev 115 | RUN --mount=type=cache,target=/root/.cache \ 116 | POETRY_DYNAMIC_VERSIONING_BYPASS=${VERSION} python3 -m pip install --disable-pip-version-check --no-deps --editable=. \ 117 | && mv docker/run /usr/bin/ \ 118 | && python3 -m compileall -q /app/tilecloud_chain 119 | 120 | # Do the lint, used by the tests 121 | FROM base AS tests 122 | 123 | # Fail on error on pipe, see: https://github.com/hadolint/hadolint/wiki/DL4006. 124 | # Treat unset variables as an error when substituting. 125 | # Print commands and their arguments as they are executed. 126 | SHELL ["/bin/bash", "-o", "pipefail", "-cux"] 127 | 128 | RUN --mount=type=cache,target=/var/lib/apt/lists \ 129 | --mount=type=cache,target=/var/cache,sharing=locked \ 130 | apt-get update \ 131 | && apt-get install --assume-yes --no-install-recommends software-properties-common gpg-agent \ 132 | && add-apt-repository ppa:savoury1/pipewire \ 133 | && add-apt-repository ppa:savoury1/chromium \ 134 | && apt-get install --assume-yes --no-install-recommends chromium-browser git curl gnupg 135 | COPY .nvmrc /tmp 136 | RUN --mount=type=cache,target=/var/lib/apt/lists \ 137 | --mount=type=cache,target=/var/cache,sharing=locked \ 138 | NODE_MAJOR="$(cat /tmp/.nvmrc)" \ 139 | && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_MAJOR}.x nodistro main" > /etc/apt/sources.list.d/nodesource.list \ 140 | && curl --silent https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor --output=/etc/apt/keyrings/nodesource.gpg \ 141 | && apt-get update \ 142 | && apt-get install --assume-yes --no-install-recommends "nodejs=${NODE_MAJOR}.*" 143 | COPY package.json package-lock.json ./ 144 | RUN npm install --dev --ignore-scripts 145 | ENV PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser 146 | 147 | RUN --mount=type=cache,target=/root/.cache \ 148 | --mount=type=bind,from=poetry,source=/tmp,target=/poetry \ 149 | python3 -m pip install --disable-pip-version-check --no-deps --requirement=/poetry/requirements-dev.txt 150 | 151 | COPY . ./ 152 | RUN --mount=type=cache,target=/root/.cache \ 153 | POETRY_DYNAMIC_VERSIONING_BYPASS=0.0.0 python3 -m pip install --disable-pip-version-check --no-deps --editable=. \ 154 | && python3 -m pip freeze > /requirements.txt 155 | 156 | ENV TILEGENERATION_MAIN_CONFIGFILE= 157 | 158 | # Set runner as final 159 | FROM runner 160 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012-2025, Camptocamp SA 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | 1. Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 2. Redistributions in binary form must reproduce the above copyright notice, 10 | this list of conditions and the following disclaimer in the documentation 11 | and/or other materials provided with the distribution. 12 | 13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 14 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 15 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 16 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 17 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 18 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 19 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 20 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 21 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 22 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 23 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | export DOCKER_BUILDKIT=1 2 | VERSION = $(strip $(shell poetry version --short)) 3 | 4 | .PHONY: help 5 | help: ## Display this help message 6 | @echo "Usage: make " 7 | @echo 8 | @echo "Available targets:" 9 | @grep --extended-regexp --no-filename '^[a-zA-Z_-]+:.*## ' $(MAKEFILE_LIST) | sort | \ 10 | awk 'BEGIN {FS = ":.*?## "}; {printf " %-20s%s\n", $$1, $$2}' 11 | 12 | .PHONY: build 13 | build: ## Build all Docker images 14 | docker build --tag=camptocamp/tilecloud-chain-tests --target=tests . 15 | docker build --tag=camptocamp/tilecloud-chain --build-arg=VERSION=$(VERSION) . 16 | 17 | .PHONY: checks 18 | checks: prospector ## Run the checks 19 | 20 | .PHONY: prospector 21 | prospector: ## Run Prospector 22 | docker run --rm --volume=${PWD}:/app camptocamp/tilecloud-chain-tests prospector --output-format=pylint --die-on-tool-error 23 | 24 | .PHONY: tests 25 | tests: build ## Run the unit tests 26 | docker compose stop --timeout=0 27 | docker compose down || true 28 | docker compose up -d 29 | 30 | # Wait for DB to be up 31 | while ! docker compose exec -T test psql -h db -p 5432 -U postgresql -v ON_ERROR_STOP=1 -c "SELECT 1" -d tests; \ 32 | do \ 33 | echo "Waiting for DB to be UP"; \ 34 | sleep 1; \ 35 | done 36 | 37 | c2cciutils-docker-logs 38 | 39 | docker compose exec -T test pytest -vvv --color=yes 40 | 41 | c2cciutils-docker-logs 42 | docker compose down 43 | 44 | PYTEST_ARGS ?= --last-failed --full-trace 45 | 46 | .PHONY: tests-fast 47 | tests-fast: 48 | docker compose up -d 49 | 50 | # Wait for DB to be up 51 | while ! docker compose exec -T test psql -h db -p 5432 -U postgresql -v ON_ERROR_STOP=1 -c "SELECT 1" -d tests; \ 52 | do \ 53 | echo "Waiting for DB to be UP"; \ 54 | sleep 1; \ 55 | done 56 | 57 | docker compose exec -T test pytest -vvv --color=yes $(PYTEST_ARGS) 58 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # TileCloud-chain 2 | 3 | TileCloud Chain is a comprehensive toolset for managing tile generation workflows. It supports various source and destination formats, making it a versatile solution for map tile management. 4 | 5 | ## Sources 6 | 7 | - Web Map Service (WMS) 8 | - Mapnik rendering engine 9 | 10 | ## Destination Formats and Storage 11 | 12 | - Web Map Tile Service (WMTS) layout 13 | - Amazon S3 storage 14 | - Azure Blob storage 15 | - Local filesystem 16 | 17 | ## Key Features 18 | 19 | - Tile generation with configurable parameters 20 | - Automatic removal of empty tiles 21 | - Geographic filtering (bbox and geometry-based) 22 | - MetaTile support for efficient generation 23 | - Legend image generation 24 | - GetCapabilities document 25 | - OpenLayers demo page 26 | - Empty tile detection via hashing 27 | - Cache synchronization 28 | - Post-processing capabilities 29 | 30 | ## Legacy Support 31 | 32 | Note: The following features are maintained for backward compatibility: 33 | 34 | - Berkeley DB integration 35 | - SQLite (MBTiles) support 36 | - Mapnik rendering (Python 3 update pending) 37 | 38 | ## Visual Preview 39 | 40 | The admin interface with PostgreSQL queue integration: 41 | 42 | ![TileCloud Chain Admin Interface](./admin-screenshot.png) 43 | 44 | ## Getting Started 45 | 46 | Create a configuration file at `tilegeneration/config.yaml`. 47 | 48 | Reference the [example configuration](https://github.com/camptocamp/tilecloud-chain/blob/master/example/tilegeneration/config.yaml). 49 | 50 | ## Support Policy 51 | 52 | Only the latest release receives active support. Versions prior to 1.11 contain security vulnerabilities and should not be used. 53 | 54 | ## Development 55 | 56 | ### Building 57 | 58 | ```bash 59 | make build 60 | ``` 61 | 62 | ### Quality Assurance 63 | 64 | ```bash 65 | make prospector 66 | ``` 67 | 68 | ### Testing 69 | 70 | ```bash 71 | make tests 72 | ``` 73 | 74 | ## Documentation 75 | 76 | - [Usage Guide](https://github.com/camptocamp/tilecloud-chain/blob/master/tilecloud_chain/USAGE.rst) 77 | - [Configuration Reference](https://github.com/camptocamp/tilecloud-chain/blob/master/tilecloud_chain/CONFIG.md) 78 | 79 | ## Contributing 80 | 81 | Set up pre-commit hooks: 82 | 83 | ```bash 84 | pip install pre-commit 85 | pre-commit install --allow-missing-config 86 | ``` 87 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Supported Versions 4 | 5 | | Version | Supported Until | 6 | | ------- | --------------- | 7 | | <=1.16 | Unsupported | 8 | | 1.17 | 23/06/2025 | 9 | | 1.18 | Unsupported | 10 | | 1.19 | Best effort | 11 | | 1.20 | Best effort | 12 | | 1.21 | Best effort | 13 | | 1.22 | Best effort | 14 | -------------------------------------------------------------------------------- /admin-screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/camptocamp/tilecloud-chain/2f84239147eb4025af917a862d8b3cb38f4a88e1/admin-screenshot.png -------------------------------------------------------------------------------- /application.ini: -------------------------------------------------------------------------------- 1 | ### 2 | # app configuration 3 | # http://docs.pylonsproject.org/projects/pyramid/en/1.6-branch/narr/environment.html 4 | ### 5 | 6 | [app:app] 7 | use = egg:tilecloud-chain 8 | filter-with = proxy-prefix 9 | 10 | pyramid.reload_templates = %(DEVELOPMENT)s 11 | pyramid.debug_authorization = %(DEVELOPMENT)s 12 | pyramid.debug_notfound = %(DEVELOPMENT)s 13 | pyramid.debug_routematch = %(DEVELOPMENT)s 14 | pyramid.debug_templates = %(DEVELOPMENT)s 15 | pyramid.default_locale_name = en 16 | 17 | c2c.base_path = /c2c 18 | 19 | tilegeneration_configfile = %(TILEGENERATION_CONFIGFILE)s 20 | 21 | pyramid.includes = %(PYRAMID_INCLUDES)s 22 | # Be careful when manipulate this, 23 | # if a hacker has access to the debug toolbar, 24 | # it's a severe security issue 25 | # With the provided values, the debug toolbar is only available from the Docker network 26 | debugtoolbar.hosts = %(DEBUGTOOLBAR_HOSTS)s 27 | 28 | [filter:translogger] 29 | use = egg:Paste#translogger 30 | setup_console_handler = False 31 | 32 | [filter:proxy-prefix] 33 | use = egg:PasteDeploy#prefix 34 | prefix = %(VISIBLE_ENTRY_POINT)s 35 | 36 | [pipeline:main] 37 | pipeline = translogger egg:c2cwsgiutils#client_info egg:c2cwsgiutils#sentry app 38 | 39 | [server:main] 40 | use = egg:waitress#main 41 | listen = *:8080 42 | threads = %(WAITRESS_THREADS)s 43 | trusted_proxy = True 44 | clear_untrusted_proxy_headers = False 45 | 46 | ### 47 | # Logging configuration 48 | # http://docs.pylonsproject.org/projects/pyramid/en/1.6-branch/narr/logging.html 49 | ### 50 | 51 | [loggers] 52 | keys = root, waitress, wsgi, c2cwsgiutils, tilecloud, tilecloud_chain, sqlalchemy 53 | 54 | [handlers] 55 | keys = console, json 56 | 57 | [formatters] 58 | keys = generic 59 | format = %(levelname)-5.5s [%(name)s] %(message)s 60 | 61 | [logger_root] 62 | level = %(OTHER_LOG_LEVEL)s 63 | handlers = %(LOG_TYPE)s 64 | 65 | [logger_tilecloud] 66 | level = %(TILECLOUD_LOG_LEVEL)s 67 | handlers = 68 | qualname = tilecloud 69 | 70 | [logger_tilecloud_chain] 71 | level = %(TILECLOUD_CHAIN_LOG_LEVEL)s 72 | handlers = 73 | qualname = tilecloud_chain 74 | 75 | [logger_c2cwsgiutils] 76 | level = %(C2CWSGIUTILS_LOG_LEVEL)s 77 | handlers = 78 | qualname = c2cwsgiutils 79 | 80 | [logger_sqlalchemy] 81 | level = %(SQL_LOG_LEVEL)s 82 | handlers = 83 | qualname = sqlalchemy.engine 84 | # "level = INFO" logs SQL queries. 85 | # "level = DEBUG" logs SQL queries and results. 86 | # "level = WARN" logs neither. (Recommended for production systems.) 87 | 88 | [logger_wsgi] 89 | level = %(WSGI_LOG_LEVEL)s 90 | handlers = 91 | qualname = wsgi 92 | 93 | [logger_waitress] 94 | level = %(WAITRESS_LOG_LEVEL)s 95 | handlers = 96 | qualname = waitress 97 | 98 | [handler_console] 99 | class = StreamHandler 100 | kwargs = {'stream': 'ext://sys.stdout'} 101 | level = NOTSET 102 | formatter = generic 103 | 104 | [formatter_generic] 105 | format = %(levelname)-5.5s %(name)s %(message)s 106 | 107 | [handler_json] 108 | class = tilecloud_chain.JsonLogHandler 109 | kwargs = {'stream': 'ext://sys.stdout'} 110 | level = NOTSET 111 | -------------------------------------------------------------------------------- /ci/config.yaml: -------------------------------------------------------------------------------- 1 | # yaml-language-server: $schema=https://raw.githubusercontent.com/camptocamp/c2cciutils/1.7.3/c2cciutils/schema.json 2 | 3 | publish: 4 | docker: 5 | dispatch: {} 6 | images: 7 | - name: camptocamp/tilecloud-chain 8 | pypi: 9 | versions: 10 | - version_tag 11 | -------------------------------------------------------------------------------- /ci/requirements.txt: -------------------------------------------------------------------------------- 1 | c2cciutils==1.7.3 2 | poetry-dynamic-versioning==1.8.2 3 | poetry-plugin-export==1.9.0 4 | pre-commit==4.2.0 5 | importlib-metadata<8.7.1 6 | tag-publish==1.0.1 7 | -------------------------------------------------------------------------------- /docker-compose.override.sample.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | application: &app 3 | ports: 4 | - '9050:8080' 5 | command: 6 | - /venv/bin/pserve 7 | - --reload 8 | - c2c:///app/application.ini 9 | environment: 10 | - DEVELOPMENT=TRUE 11 | volumes: 12 | - ./tilecloud_chain:/app/tilecloud_chain:ro 13 | # - ../tilecloud/tilecloud:/usr/local/lib/python3.10/dist-packages/tilecloud:ro 14 | 15 | app_test_user: 16 | <<: *app 17 | ports: 18 | - '9051:8080' 19 | 20 | slave: 21 | volumes: 22 | - ./tilecloud_chain:/app/tilecloud_chain:ro 23 | 24 | # test: 25 | # volumes: 26 | # - ./tilecloud_chain:/app/tilecloud_chain:ro 27 | # - ../tilecloud/tilecloud:/usr/local/lib/python3.10/dist-packages/tilecloud:ro 28 | -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | services: 2 | db: 3 | image: camptocamp/postgres:17-postgis-3 4 | environment: 5 | POSTGRES_USER: postgresql 6 | POSTGRES_PASSWORD: postgresql 7 | POSTGRES_DB: tests 8 | volumes: 9 | - ./docker/test-db:/docker-entrypoint-initdb.d:ro 10 | 11 | mapserver: 12 | image: camptocamp/mapserver:8.4 13 | environment: 14 | MS_DEBUGLEVEL: '5' 15 | MAPSERVER_CATCH_SEGV: '1' 16 | MS_MAPFILE: /etc/mapserver/mapserver.map 17 | MS_MAP_PATTERN: ^/etc/mapserver/ 18 | volumes: 19 | - ./docker/mapfile-docker:/etc/mapserver:ro 20 | links: 21 | - db:db 22 | user: www-data 23 | 24 | redis_master: 25 | image: bitnami/redis:8.0.2 26 | environment: 27 | - REDIS_REPLICATION_MODE=master 28 | - ALLOW_EMPTY_PASSWORD=yes 29 | 30 | redis_slave: 31 | image: bitnami/redis:8.0.2 32 | environment: 33 | - REDIS_REPLICATION_MODE=slave 34 | - REDIS_MASTER_HOST=redis_master 35 | - ALLOW_EMPTY_PASSWORD=yes 36 | depends_on: 37 | - redis_master 38 | 39 | redis_sentinel: 40 | image: bitnami/redis-sentinel:8.0.2 41 | environment: 42 | - REDIS_MASTER_HOST=redis_master 43 | - REDIS_MASTER_SET=mymaster 44 | - ALLOW_EMPTY_PASSWORD=yes 45 | depends_on: 46 | - redis_master 47 | - redis_slave 48 | 49 | application: &app 50 | image: camptocamp/tilecloud-chain 51 | environment: &app-env 52 | TILECLOUD_LOG_LEVEL: INFO 53 | TILECLOUD_CHAIN_LOG_LEVEL: INFO 54 | TILECLOUD_CHAIN_SESSION_SECRET: '1234' 55 | TILECLOUD_CHAIN_SESSION_SALT: '1234' 56 | C2C_AUTH_GITHUB_REPOSITORY: camptocamp/tilecloud-chain 57 | C2C_AUTH_GITHUB_SECRET: '1234567890123456789' 58 | C2C_AUTH_GITHUB_CLIENT_ID: '1234' 59 | C2C_AUTH_GITHUB_CLIENT_SECRET: '1234' 60 | C2C_PROMETHEUS_PORT: '9110' 61 | links: 62 | - db 63 | - redis_sentinel 64 | volumes: 65 | - ./example/tilegeneration/config.yaml:/etc/tilegeneration/config.yaml:ro 66 | 67 | app_test_user: 68 | <<: *app 69 | environment: 70 | <<: *app-env 71 | TEST_USER: Test 72 | 73 | app_postgresql: 74 | <<: *app 75 | environment: 76 | <<: *app-env 77 | TEST_USER: Test 78 | SQL_LOG_LEVEL: DEBUG 79 | TILECLOUD_CHAIN_SQLALCHEMY_URL: postgresql+pyscopg://postgresql:postgresql@db:5432/tests 80 | volumes: 81 | - ./example/tilegeneration/config-postgresql.yaml:/etc/tilegeneration/config.yaml:ro 82 | ports: 83 | - '9052:8080' 84 | 85 | slave: 86 | <<: *app 87 | command: 88 | - /venv/bin/generate-tiles 89 | - '--role=slave' 90 | - '--daemon' 91 | environment: 92 | <<: *app-env 93 | TILECLOUD_CHAIN_SLAVE: 'TRUE' 94 | 95 | test: 96 | image: camptocamp/tilecloud-chain-tests 97 | working_dir: /app 98 | environment: 99 | CI: 'true' 100 | TESTS: 'true' 101 | PGPASSWORD: postgresql 102 | TILE_NB_THREAD: 2 103 | METATILE_NB_THREAD: 2 104 | SERVER_NB_THREAD: 2 105 | TILECLOUD_LOG_LEVEL: DEBUG 106 | TILECLOUD_CHAIN_LOG_LEVEL: DEBUG 107 | TILECLOUD_CHAIN_SESSION_SALT: a-long-secret-a-long-secret 108 | TILECLOUD_CHAIN_SQLALCHEMY_URL: postgresql+psycopg://postgresql:postgresql@db:5432/tests 109 | command: 110 | - sleep 111 | - infinity 112 | links: 113 | - db 114 | - redis_sentinel 115 | volumes: 116 | - ./results:/results 117 | - ./tilecloud_chain:/app/tilecloud_chain 118 | # - ../tilecloud/tilecloud:/usr/local/lib/python3.8/dist-packages/tilecloud 119 | 120 | shell: 121 | image: camptocamp/postgres:17-postgis-3 122 | command: 123 | - tail 124 | - -f 125 | - /dev/null 126 | environment: 127 | - PGHOST=db 128 | - PGUSER=postgresql 129 | - PGPASSWORD=postgresql 130 | - PGDATABASE=tests 131 | - PGPORT=5432 132 | -------------------------------------------------------------------------------- /docker/mapfile-docker/circle.svg: -------------------------------------------------------------------------------- 1 | 2 | 6 | 11 | 12 | -------------------------------------------------------------------------------- /docker/mapfile-docker/mapserver.conf: -------------------------------------------------------------------------------- 1 | CONFIG 2 | END 3 | -------------------------------------------------------------------------------- /docker/mapfile-docker/mapserver.map: -------------------------------------------------------------------------------- 1 | MAP 2 | NAME "Tests" 3 | STATUS ON 4 | EXTENT 420000 30000 900000 350000 5 | MAXSIZE 2500 6 | 7 | WEB 8 | TEMPLATE dummyTemplateForWmsGetFeatureInfo 9 | METADATA 10 | "ows_title" "Tests" 11 | "ows_encoding" "UTF-8" 12 | "wms_enable_request" "*" 13 | END 14 | END 15 | 16 | PROJECTION 17 | "init=epsg:21781" 18 | END 19 | 20 | SYMBOL 21 | NAME "circle" 22 | TYPE ellipse 23 | POINTS 24 | 1 1 25 | END 26 | FILLED true 27 | END 28 | 29 | LAYER 30 | NAME "point" 31 | TYPE POINT 32 | CONNECTIONTYPE postgis 33 | CONNECTION "user=postgresql password=postgresql dbname=tests host=db" 34 | DATA "the_geom FROM tests.point" 35 | TEMPLATE fooOnlyForWMSGetFeatureInfo # For GetFeatureInfo 36 | STATUS ON 37 | METADATA 38 | "gml_include_items" "all" # For GetFeatureInfo 39 | "ows_geom_type" "point" # For returning geometries in GetFeatureInfo 40 | "ows_geometries" "the_geom" # For returning geometries in GetFeatureInfo 41 | END 42 | CLASS 43 | NAME "Point" 44 | STYLE 45 | COLOR 255 0 0 46 | SIZE 10 47 | SYMBOL "circle" 48 | END 49 | END 50 | END 51 | 52 | LAYER 53 | NAME "point_multi" 54 | TYPE POINT 55 | CONNECTIONTYPE postgis 56 | CONNECTION "user=postgresql password=postgresql dbname=tests host=db" 57 | DATA "the_geom FROM (SELECT * FROM tests.point WHERE name='%POINT_NAME%') AS foo USING unique gid" 58 | TEMPLATE fooOnlyForWMSGetFeatureInfo # For GetFeatureInfo 59 | STATUS ON 60 | METADATA 61 | "gml_include_items" "all" # For GetFeatureInfo 62 | "ows_geom_type" "point" # For returning geometries in GetFeatureInfo 63 | "ows_geometries" "the_geom" # For returning geometries in GetFeatureInfo 64 | END 65 | VALIDATION 66 | "POINT_NAME" "[a-z0-9]+" 67 | "default_POINT_NAME" "point1" 68 | END 69 | CLASS 70 | NAME "Point" 71 | STYLE 72 | COLOR 255 0 0 73 | SIZE 10 74 | SYMBOL "circle" 75 | END 76 | END 77 | END 78 | 79 | LAYER 80 | NAME "line" 81 | TYPE LINE 82 | CONNECTIONTYPE postgis 83 | CONNECTION "user=postgresql password=postgresql dbname=tests host=db" 84 | DATA "the_geom FROM tests.line" 85 | STATUS ON 86 | CLASS 87 | NAME "Line 1" 88 | STYLE 89 | COLOR 0 255 0 90 | WIDTH 5 91 | MINSCALEDENOM 100000 92 | END 93 | END 94 | CLASS 95 | NAME "Line 2" 96 | STYLE 97 | COLOR 0 0 255 98 | WIDTH 5 99 | MAXSCALEDENOM 100000 100 | END 101 | END 102 | END 103 | 104 | LAYER 105 | NAME "polygon" 106 | TYPE POLYGON 107 | CONNECTIONTYPE postgis 108 | CONNECTION "user=postgresql password=postgresql dbname=tests host=db" 109 | DATA "the_geom FROM tests.polygon" 110 | STATUS ON 111 | CLASS 112 | NAME "Polygon" 113 | STYLE 114 | OUTLINECOLOR 0 255 0 115 | COLOR 255 255 0 116 | END 117 | END 118 | END 119 | END 120 | -------------------------------------------------------------------------------- /docker/mapfile-docker/test.mapnik: -------------------------------------------------------------------------------- 1 | 2 | 3 | 8 | 13 | 18 | 19 | polygon 20 | 21 | tests.polygon 22 | the_geom 23 | 21781 24 | postgis 25 | tests 26 | db 27 | 5432 28 | postgresql 29 | postgresql 30 | false 31 | 420000, 30000, 900000, 350000 32 | 33 | 34 | 35 | line 36 | 37 | tests.line 38 | the_geom 39 | 21781 40 | postgis 41 | tests 42 | db 43 | 5432 44 | postgresql 45 | postgresql 46 | false 47 | 420000, 30000, 900000, 350000 48 | 49 | 50 | 51 | point 52 | 53 | tests.point 54 | the_geom 55 | 21781 56 | postgis 57 | tests 58 | db 59 | 5432 60 | postgresql 61 | postgresql 62 | false 63 | 420000, 30000, 900000, 350000 64 | 65 | 66 | 67 | -------------------------------------------------------------------------------- /docker/run: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import os 4 | import stat 5 | import subprocess # nosec 6 | import sys 7 | from pathlib import Path 8 | 9 | if "GROUP_ID" not in os.environ: 10 | sys.exit("The GROUP_ID environment variable is required") 11 | 12 | if "USER_ID" not in os.environ: 13 | sys.exit("The USER_ID environment variable is required") 14 | 15 | if "USER_NAME" not in os.environ: 16 | sys.exit("The USER_NAME environment variable is required") 17 | 18 | if "UMASK" not in os.environ: 19 | sys.exit("The UMASK environment variable is required") 20 | 21 | subprocess.check_call(["groupadd", "-g", os.environ["GROUP_ID"], "geomapfish"]) # noqa: S603, S607 22 | subprocess.check_call( # noqa: S603 # nosec 23 | [ # noqa: S607 24 | "useradd", 25 | "--shell", 26 | "/bin/bash", 27 | "--uid", 28 | os.environ["USER_ID"], 29 | "--gid", 30 | os.environ["GROUP_ID"], 31 | os.environ["USER_NAME"], 32 | ], 33 | ) 34 | 35 | run_file_name = Path("/tmp/run") # noqa: S108 36 | with run_file_name.open("w", encoding="utf-8") as run_file: 37 | run_file.write("#!/usr/bin/python\n") 38 | run_file.write("import subprocess, os\n") 39 | run_file.write(f"os.umask(0o{os.environ['UMASK']})\n") 40 | run_file.write(f"subprocess.check_call({sys.argv[1:]!r})\n") 41 | 42 | run_file_name.chmod( 43 | stat.S_IRUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH, 44 | ) 45 | subprocess.check_call(["su", os.environ["USER_NAME"], "-c", run_file_name]) # noqa: S603, S607 46 | -------------------------------------------------------------------------------- /docker/test-db/10_init.sql: -------------------------------------------------------------------------------- 1 | CREATE EXTENSION postgis; 2 | 3 | CREATE SCHEMA tests; 4 | 5 | CREATE TABLE tests.point (gid serial Primary KEY, name varchar(10)); 6 | SELECT AddGeometryColumn('tests', 'point','the_geom',21781,'POINT',2); 7 | 8 | CREATE TABLE tests.line (gid serial Primary KEY, name varchar(10)); 9 | SELECT AddGeometryColumn('tests', 'line','the_geom',21781,'LINESTRING',2); 10 | 11 | CREATE TABLE tests.polygon (gid serial Primary KEY, name varchar(10)); 12 | SELECT AddGeometryColumn('tests', 'polygon','the_geom',21781,'POLYGON',2); 13 | 14 | 15 | INSERT INTO tests.point VALUES (0, 'point1', ST_GeomFromText('POINT (600000 200000)', 21781)); 16 | INSERT INTO tests.point VALUES (1, 'point2', ST_GeomFromText('POINT (530000 150000)', 21781)); 17 | 18 | INSERT INTO tests.line VALUES (0, 'line1', ST_GeomFromText('LINESTRING (600000 200000,530000 150000)', 21781)); 19 | 20 | INSERT INTO tests.polygon VALUES (0, 'polygon1', ST_GeomFromText('POLYGON ((600000 200000,600000 150000,530000 150000, 530000 200000, 600000 200000))', 21781)); 21 | -------------------------------------------------------------------------------- /example/tilegeneration/config-postgresql.yaml: -------------------------------------------------------------------------------- 1 | grids: 2 | # grid name, I just recommends to add the min resolution because it's common to not generate all the layers at the same resolution. 3 | swissgrid_05: 4 | # resolutions [required] 5 | resolutions: [1000, 500, 250, 100, 50, 20, 10, 5, 2, 1, 0.5] 6 | # bbox [required] 7 | bbox: [420000, 30000, 900000, 350000] 8 | # srs [required] 9 | srs: EPSG:21781 10 | 11 | caches: 12 | local: 13 | type: filesystem 14 | folder: /var/sig/tiles 15 | # for GetCapabilities 16 | http_url: https://%(host)s/tiles/ 17 | hosts: 18 | - wmts0.example.com 19 | - wmts1.example.com 20 | - wmts2.example.com 21 | - wmts3.example.com 22 | - wmts4.example.com 23 | s3: 24 | type: s3 25 | bucket: tiles 26 | folder: '' 27 | # for GetCapabilities 28 | http_url: https://%(host)s/%(bucket)s/%(folder)s/ 29 | cache_control: 'public, max-age=14400' 30 | hosts: 31 | - wmts0. 32 | 33 | # this defines some defaults values for all the layers 34 | defaults: 35 | layer: &layer 36 | type: wms 37 | grid: swissgrid_05 38 | # The minimum resolution to seed, useful to use with mapcache, optional. 39 | # min_resolution_seed: 1 40 | # the URL of the WMS server to used 41 | url: http://mapserver/ 42 | # Set the headers to get the right virtual host, and don't get any cached result 43 | headers: 44 | Cache-Control: no-cache, no-store 45 | Pragma: no-cache 46 | # file name extension 47 | extension: png 48 | # the bbox there we want to generate tiles 49 | #bbox: [493000, 114000, 586000, 204000] 50 | 51 | # mime type used for the WMS request and the WMTS capabilities generation 52 | mime_type: image/png 53 | wmts_style: default 54 | # the WMTS dimensions definition [default is []] 55 | #dimensions: 56 | # - name: DATE 57 | # # the default value for the WMTS capabilities 58 | # default: '2012' 59 | # # the generated values 60 | # generate: ['2012'] 61 | # # all the available values in the WMTS capabilities 62 | # values: ['2012'] 63 | # the meta tiles definition [default is off] 64 | meta: true 65 | # the meta tiles size [default is 8] 66 | meta_size: 8 67 | # the meta tiles buffer [default is 128] 68 | meta_buffer: 128 69 | # connection an sql to get geometries (in column named geom) where we want to generate tiles 70 | # Warn: too complex result can slow down the application 71 | # connection: user=www-data password=www-data dbname= host=localhost 72 | # geoms: 73 | # - sql: AS geom FROM
74 | # size and hash used to detect empty tiles and metatiles [optional, default is None] 75 | empty_metatile_detection: 76 | size: 740 77 | hash: 3237839c217b51b8a9644d596982f342f8041546 78 | empty_tile_detection: 79 | size: 921 80 | hash: 1e3da153be87a493c4c71198366485f290cad43c 81 | 82 | layers: 83 | plan: 84 | <<: *layer 85 | layers: plan 86 | ortho: 87 | <<: *layer 88 | layers: ortho 89 | extension: jpeg 90 | mime_type: image/jpeg 91 | # no buffer needed on rater sources 92 | meta_buffer: 0 93 | empty_metatile_detection: 94 | size: 66163 95 | hash: a9d16a1794586ef92129a2fb41a739451ed09914 96 | empty_tile_detection: 97 | size: 1651 98 | hash: 2892fea0a474228f5d66a534b0b5231d923696da 99 | 100 | generation: 101 | default_cache: local 102 | # used to allowed only a specific user to generate tiles (for rights issue) 103 | authorised_user: www-data 104 | 105 | # maximum allowed consecutive errors, after it exit [default is 10] 106 | maxconsecutive_errors: 10 107 | 108 | process: 109 | optipng_test: 110 | - cmd: optipng -o7 -simulate %(in)s 111 | optipng: 112 | - cmd: optipng %(args)s -zc9 -zm8 -zs3 -f5 %(in)s 113 | arg: 114 | default: '-q' 115 | quiet: '-q' 116 | jpegoptim: 117 | - cmd: jpegoptim %(args)s --strip-all --all-normal -m 90 %(in)s 118 | arg: 119 | default: '-q' 120 | quiet: '-q' 121 | 122 | openlayers: 123 | # srs, center_x, center_y [required] 124 | srs: EPSG:21781 125 | center_x: 600000 126 | center_y: 200000 127 | 128 | metadata: 129 | title: Some title 130 | abstract: Some abstract 131 | servicetype: OGC WMTS 132 | keywords: 133 | - some 134 | - keywords 135 | fees: None 136 | access_constraints: None 137 | 138 | provider: 139 | name: The provider name 140 | url: The provider URL 141 | contact: 142 | name: The contact name 143 | position: The position name 144 | info: 145 | phone: 146 | voice: +41 11 222 33 44 147 | fax: +41 11 222 33 44 148 | address: 149 | delivery: Address delivery 150 | city: Berne 151 | area: BE 152 | postal_code: 3000 153 | country: Switzerland 154 | email: info@example.com 155 | 156 | postgresql: {} 157 | 158 | redis: 159 | socket_timeout: 30 160 | sentinels: 161 | - - redis_sentinel 162 | - 26379 163 | service_name: mymaster 164 | db: 1 165 | 166 | server: 167 | predefined_commands: 168 | - name: Generation all layers 169 | command: generate-tiles 170 | - name: Generation layer plan 171 | command: generate-tiles --layer=plan 172 | - name: Generation layer ortho 173 | command: generate-tiles --layer=ortho 174 | - name: Generate the legend images 175 | command: generate-controller --generate-legend-images 176 | - name: Get the hash of plan 177 | command: generate-tiles --layer plan --get-hash 10/0/0 178 | 179 | admin_footer: The old jobs will be automatically removed 180 | admin_footer_classes: alert alert-dark 181 | -------------------------------------------------------------------------------- /example/tilegeneration/config.yaml: -------------------------------------------------------------------------------- 1 | grids: 2 | # grid name, I just recommends to add the min resolution because it's common to not generate all the layers at the same resolution. 3 | swissgrid_05: 4 | # resolutions [required] 5 | resolutions: [1000, 500, 250, 100, 50, 20, 10, 5, 2, 1, 0.5] 6 | # bbox [required] 7 | bbox: [420000, 30000, 900000, 350000] 8 | # srs [required] 9 | srs: EPSG:21781 10 | 11 | caches: 12 | local: 13 | type: filesystem 14 | folder: /var/sig/tiles 15 | # for GetCapabilities 16 | http_url: https://%(host)s/tiles/ 17 | hosts: 18 | - wmts0.example.com 19 | - wmts1.example.com 20 | - wmts2.example.com 21 | - wmts3.example.com 22 | - wmts4.example.com 23 | s3: 24 | type: s3 25 | bucket: tiles 26 | folder: '' 27 | # for GetCapabilities 28 | http_url: https://%(host)s/%(bucket)s/%(folder)s/ 29 | cache_control: 'public, max-age=14400' 30 | hosts: 31 | - wmts0. 32 | 33 | # this defines some defaults values for all the layers 34 | defaults: 35 | layer: &layer 36 | type: wms 37 | grid: swissgrid_05 38 | # The minimum resolution to seed, useful to use with mapcache, optional. 39 | # min_resolution_seed: 1 40 | # the URL of the WMS server to used 41 | url: http://mapserver/ 42 | # Set the headers to get the right virtual host, and don't get any cached result 43 | headers: 44 | Cache-Control: no-cache, no-store 45 | Pragma: no-cache 46 | # file name extension 47 | extension: png 48 | # the bbox there we want to generate tiles 49 | #bbox: [493000, 114000, 586000, 204000] 50 | 51 | # mime type used for the WMS request and the WMTS capabilities generation 52 | mime_type: image/png 53 | wmts_style: default 54 | # the WMTS dimensions definition [default is []] 55 | #dimensions: 56 | # - name: DATE 57 | # # the default value for the WMTS capabilities 58 | # default: '2012' 59 | # # the generated values 60 | # generate: ['2012'] 61 | # # all the available values in the WMTS capabilities 62 | # values: ['2012'] 63 | # the meta tiles definition [default is off] 64 | meta: true 65 | # the meta tiles size [default is 8] 66 | meta_size: 8 67 | # the meta tiles buffer [default is 128] 68 | meta_buffer: 128 69 | # connection an sql to get geometries (in column named geom) where we want to generate tiles 70 | # Warn: too complex result can slow down the application 71 | # connection: user=www-data password=www-data dbname= host=localhost 72 | # geoms: 73 | # - sql: AS geom FROM
74 | # size and hash used to detect empty tiles and metatiles [optional, default is None] 75 | empty_metatile_detection: 76 | size: 740 77 | hash: 3237839c217b51b8a9644d596982f342f8041546 78 | empty_tile_detection: 79 | size: 921 80 | hash: 1e3da153be87a493c4c71198366485f290cad43c 81 | 82 | layers: 83 | plan: 84 | <<: *layer 85 | layers: plan 86 | ortho: 87 | <<: *layer 88 | layers: ortho 89 | extension: jpeg 90 | mime_type: image/jpeg 91 | # no buffer needed on rater sources 92 | meta_buffer: 0 93 | empty_metatile_detection: 94 | size: 66163 95 | hash: a9d16a1794586ef92129a2fb41a739451ed09914 96 | empty_tile_detection: 97 | size: 1651 98 | hash: 2892fea0a474228f5d66a534b0b5231d923696da 99 | 100 | generation: 101 | default_cache: local 102 | # used to allowed only a specific user to generate tiles (for rights issue) 103 | authorised_user: www-data 104 | 105 | # maximum allowed consecutive errors, after it exit [default is 10] 106 | maxconsecutive_errors: 10 107 | 108 | process: 109 | optipng_test: 110 | - cmd: optipng -o7 -simulate %(in)s 111 | optipng: 112 | - cmd: optipng %(args)s -zc9 -zm8 -zs3 -f5 %(in)s 113 | arg: 114 | default: '-q' 115 | quiet: '-q' 116 | jpegoptim: 117 | - cmd: jpegoptim %(args)s --strip-all --all-normal -m 90 %(in)s 118 | arg: 119 | default: '-q' 120 | quiet: '-q' 121 | 122 | openlayers: 123 | # srs, center_x, center_y [required] 124 | srs: EPSG:21781 125 | center_x: 600000 126 | center_y: 200000 127 | 128 | metadata: 129 | title: Some title 130 | abstract: Some abstract 131 | servicetype: OGC WMTS 132 | keywords: 133 | - some 134 | - keywords 135 | fees: None 136 | access_constraints: None 137 | 138 | provider: 139 | name: The provider name 140 | url: The provider URL 141 | contact: 142 | name: The contact name 143 | position: The position name 144 | info: 145 | phone: 146 | voice: +41 11 222 33 44 147 | fax: +41 11 222 33 44 148 | address: 149 | delivery: Address delivery 150 | city: Berne 151 | area: BE 152 | postal_code: 3000 153 | country: Switzerland 154 | email: info@example.com 155 | 156 | redis: 157 | socket_timeout: 30 158 | sentinels: 159 | - - redis_sentinel 160 | - 26379 161 | service_name: mymaster 162 | db: 1 163 | 164 | server: 165 | predefined_commands: 166 | - name: Generation all layers 167 | command: generate-tiles 168 | - name: Generation layer plan 169 | command: generate-tiles --layer=plan 170 | - name: Generation layer ortho 171 | command: generate-tiles --layer=ortho 172 | - name: Generate the legend images 173 | command: generate-controller --generate-legend-images 174 | -------------------------------------------------------------------------------- /jsonschema-gentypes.yaml: -------------------------------------------------------------------------------- 1 | # yaml-language-server: $schema=https://raw.githubusercontent.com/sbrunner/jsonschema-gentypes/2.11.0/jsonschema_gentypes/schema.json 2 | 3 | headers: | 4 | """ 5 | Automatically generated file from a JSON schema. 6 | """ 7 | 8 | pre_commit: 9 | enabled: true 10 | hooks_skip: 11 | - jsonschema-gentypes 12 | - shellcheck 13 | arguments: 14 | - --color=never 15 | 16 | python_version: '3.10' 17 | 18 | generate: 19 | - source: tilecloud_chain/schema.json 20 | destination: tilecloud_chain/configuration.py 21 | root_name: Configuration 22 | api_arguments: 23 | additional_properties: Only explicit 24 | - source: tilecloud_chain/host-limit-schema.json 25 | destination: tilecloud_chain/host_limit.py 26 | root_name: HostLimit 27 | api_arguments: 28 | additional_properties: Only explicit 29 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "devDependencies": { 3 | "commander": "14.0.0", 4 | "puppeteer": "24.10.0" 5 | }, 6 | "type": "module" 7 | } 8 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.ruff] 2 | line-length = 110 3 | target-version = "py310" 4 | 5 | [tool.ruff.lint.pydocstyle] 6 | convention = "numpy" 7 | 8 | [tool.pytest.ini_options] 9 | asyncio_mode = "auto" 10 | 11 | [tool.poetry] 12 | version = "0.0.0" 13 | 14 | [tool.poetry.plugins."pyramid.scaffold"] 15 | tilecloud_chain = "tilecloud_chain.scaffolds:Create" 16 | 17 | [tool.poetry.plugins."paste.app_factory"] 18 | main = "tilecloud_chain.server:main" 19 | 20 | [tool.poetry.dependencies] 21 | # Minimal version should also be set in the jsonschema-gentypes.yaml file 22 | python = ">=3.10,<3.13" 23 | c2cwsgiutils = { version = "6.1.7", extras = ["standard", "broadcast", "oauth2", "debug"] } 24 | pyramid-mako = "1.1.0" 25 | python-dateutil = "2.9.0.post0" 26 | tilecloud = { version = "1.13.1", extras = ["azure", "aws", "redis", "wsgi"] } 27 | Jinja2 = "3.1.6" 28 | PyYAML = "6.0.2" 29 | Shapely = "2.1.1" 30 | jsonschema = "4.24.0" 31 | pyramid = "2.0.2" 32 | jsonschema-validator-new = "0.3.2" 33 | azure-storage-blob = "12.25.1" 34 | waitress = "3.0.2" 35 | certifi = "2025.4.26" 36 | Paste = "3.10.1" 37 | psutil = "7.0.0" 38 | pyproj = "3.7.1" 39 | psycopg = { version = "3.2.9", extras = ["binary"] } 40 | aiohttp = "3.12.6" 41 | sqlalchemy = { version = "2.0.41", extras = ["asyncio"] } 42 | pytest-asyncio = "1.0.0" 43 | aiofiles = "24.1.0" 44 | 45 | [tool.poetry.group.dev.dependencies] 46 | prospector = { extras = ["with_mypy", "with_bandit", "with_pyroma", "with_ruff"], version = "1.17.1" } 47 | prospector-profile-duplicated = "1.10.5" 48 | prospector-profile-utils = "1.22.3" 49 | c2cwsgiutils = { version = "6.1.7", extras = ["test_images"] } 50 | scikit-image = { version = "0.25.2" } 51 | pytest = "8.3.5" 52 | testfixtures = "8.3.0" 53 | coverage = "7.8.2" 54 | types-redis = "4.6.0.20241004" 55 | types-requests = "2.32.0.20250515" 56 | types-aiofiles = "24.1.0.20250516" 57 | pytest-asyncio = "1.0.0" 58 | 59 | [tool.poetry-dynamic-versioning] 60 | enable = true 61 | vcs = "git" 62 | pattern = "^(?P\\d+(\\.\\d+)*)" 63 | format-jinja = """ 64 | {%- if env.get("VERSION_TYPE") == "default_branch" -%} 65 | {{serialize_pep440(bump_version(base, 1), dev=distance)}} 66 | {%- elif env.get("VERSION_TYPE") == "stabilization_branch" -%} 67 | {{serialize_pep440(bump_version(base, 2), dev=distance)}} 68 | {%- elif distance == 0 -%} 69 | {{serialize_pep440(base)}} 70 | {%- else -%} 71 | {{serialize_pep440(bump_version(base), dev=distance)}} 72 | {%- endif -%} 73 | """ 74 | 75 | [tool.poetry-plugin-tweak-dependencies-version] 76 | default = "present" 77 | 78 | [project] 79 | classifiers = [ 80 | 'Development Status :: 5 - Production/Stable', 81 | 'Environment :: Web Environment', 82 | 'Framework :: Pyramid', 83 | 'Intended Audience :: Other Audience', 84 | 'License :: OSI Approved :: BSD License', 85 | 'Operating System :: OS Independent', 86 | 'Programming Language :: Python', 87 | 'Programming Language :: Python :: 3', 88 | 'Programming Language :: Python :: 3.10', 89 | 'Programming Language :: Python :: 3.11', 90 | 'Programming Language :: Python :: 3.12', 91 | 'Programming Language :: Python :: 3.13', 92 | 'Topic :: Scientific/Engineering :: GIS', 93 | 'Typing :: Typed', 94 | ] 95 | dynamic = ["dependencies", "version"] 96 | name = "tilecloud-chain" 97 | description = "Tools to generate tiles from WMS or Mapnik, to S3, Berkeley DB, MBTiles, or local filesystem in WMTS layout using Amazon cloud services." 98 | readme = "README.md" 99 | keywords = ["gis", "tilecloud", "chain"] 100 | license = "BSD-2-Clause" 101 | authors = [{name = "Camptocamp",email = "info@camptocamp.com"}] 102 | packages = [{ include = "tilecloud_chain" }] 103 | include = ["tilecloud_chain/py.typed", "tilecloud_chain/*.rst", "tilecloud_chain/*.md"] 104 | requires-python = ">=3.10" 105 | dependencies = ["c2cwsgiutils[broadcast,debug,oauth2,standard]", "pyramid-mako", "python-dateutil", "tilecloud[aws,azure,redis,wsgi]", "Jinja2", "PyYAML", "Shapely", "jsonschema", "pyramid", "jsonschema-validator-new", "azure-storage-blob", "waitress", "certifi", "Paste", "psutil", "pyproj", "psycopg[binary]", "aiohttp", "sqlalchemy[asyncio]", "pytest-asyncio", "aiofiles"] 106 | 107 | [project.urls] 108 | repository = "https://github.com/camptocamp/tilecloud-chain" 109 | "Bug Tracker" = "https://github.com/camptocamp/tilecloud-chain/issues" 110 | 111 | [project.scripts] 112 | generate-tiles = "tilecloud_chain.generate:main" 113 | generate-controller = "tilecloud_chain.controller:main" 114 | generate-cost = "tilecloud_chain.cost:main" 115 | generate-copy = "tilecloud_chain.copy_:main" 116 | generate-process = "tilecloud_chain.copy_:process" 117 | import-expiretiles = "tilecloud_chain.expiretiles:main" 118 | 119 | [build-system] 120 | requires = ["poetry-core>=1.0.0", "poetry-dynamic-versioning"] 121 | build-backend = "poetry.core.masonry.api" 122 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | poetry==2.1.3 2 | poetry-plugin-export==1.9.0 3 | poetry-dynamic-versioning==1.8.2 4 | pip==25.1.1 5 | -------------------------------------------------------------------------------- /screenshot.js: -------------------------------------------------------------------------------- 1 | import puppeteer from 'puppeteer'; 2 | import { program } from 'commander'; 3 | 4 | program 5 | .option('--url ', 'The URL') 6 | .option('--output ', 'The output filename') 7 | .option('--width ', 'The page width', 800) 8 | .option('--height ', 'The page height', 600); 9 | 10 | program.parse(); 11 | 12 | const options = program.opts(); 13 | 14 | function delay(ms) { 15 | return new Promise((resolve) => setTimeout(resolve, ms)); 16 | } 17 | 18 | (async () => { 19 | const browser = await puppeteer.launch({ 20 | headless: true, 21 | args: ['--no-sandbox'], 22 | }); 23 | const page = await browser.newPage(); 24 | page.setDefaultNavigationTimeout(10000); 25 | await page.goto(options.url, { timeout: 5000 }); 26 | await delay(1000); 27 | await page.setViewport({ 28 | width: parseInt(options.width), 29 | height: parseInt(options.height), 30 | }); 31 | await page.screenshot({ 32 | path: options.output, 33 | clip: { x: 0, y: 0, width: parseInt(options.width), height: parseInt(options.height) }, 34 | }); 35 | await browser.close(); 36 | })(); 37 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [nosetests] 2 | match = ^test 3 | where = tilecloud_chain/tests 4 | cover-package = tilecloud_chain 5 | with-coverage = 1 6 | cover-erase = 1 7 | -------------------------------------------------------------------------------- /tilecloud_chain/HOST_LIMIT.md: -------------------------------------------------------------------------------- 1 | # TileCloud-chain host limit configuration 2 | 3 | _The configuration of the concurrent request limit on a host_ 4 | 5 | ## Properties 6 | 7 | - **`default`** _(object)_ 8 | - **`concurrent`** _(integer)_: Default limit of concurrent request on the same host (can be set with the `TILECLOUD_CHAIN_HOST_CONCURRENT` environment variable. Default: `1`. 9 | - **`hosts`** _(object)_: Can contain additional properties. 10 | - **Additional properties** _(object)_ 11 | - **`concurrent`** _(integer)_: Limit of concurrent request on the host. 12 | 13 | ## Definitions 14 | -------------------------------------------------------------------------------- /tilecloud_chain/copy_.py: -------------------------------------------------------------------------------- 1 | """Copy the tiles from a cache to an other.""" 2 | 3 | import asyncio 4 | import logging 5 | import os 6 | import sys 7 | from argparse import ArgumentParser, Namespace 8 | from typing import TYPE_CHECKING, cast 9 | 10 | from tilecloud_chain import Count, DropEmpty, HashDropper, TileGeneration, add_common_options 11 | from tilecloud_chain.format import duration_format, size_format 12 | 13 | if TYPE_CHECKING: 14 | import tilecloud_chain.configuration 15 | 16 | 17 | _logger = logging.getLogger(__name__) 18 | 19 | 20 | class Copy: 21 | """Copy the tiles from a cache to an other.""" 22 | 23 | count = None 24 | 25 | async def copy( 26 | self, 27 | options: Namespace, 28 | gene: TileGeneration, 29 | layer_name: str, 30 | source: str, 31 | destination: str, 32 | task_name: str, 33 | ) -> None: 34 | """Copy the tiles from a cache to an other.""" 35 | assert gene.config_file 36 | config = gene.get_config(gene.config_file) 37 | layer = config.config["layers"][layer_name] 38 | # disable metatiles 39 | cast("tilecloud_chain.configuration.LayerWms", layer)["meta"] = False 40 | count_tiles_dropped = Count() 41 | 42 | gene.create_log_tiles_error(layer_name) 43 | source_tilestore = gene.get_tilesstore(source) 44 | dest_tilestore = gene.get_tilesstore(destination) 45 | gene.init_tilecoords(config, layer_name) 46 | gene.add_geom_filter() 47 | gene.add_logger() 48 | gene.get(source_tilestore, "Get the tiles") 49 | gene.imap(DropEmpty(gene)) 50 | # Discard tiles with certain content 51 | if "empty_tile_detection" in layer: 52 | empty_tile = layer["empty_tile_detection"] 53 | 54 | gene.imap( 55 | HashDropper( 56 | empty_tile["size"], 57 | empty_tile["hash"], 58 | store=dest_tilestore, 59 | count=count_tiles_dropped, 60 | ), 61 | ) 62 | 63 | if options.process: 64 | gene.process(options.process) 65 | 66 | gene.imap(DropEmpty(gene)) 67 | self.count = gene.counter_size() 68 | gene.put(dest_tilestore, "Store the tiles") 69 | await gene.consume() 70 | if not options.quiet: 71 | print( 72 | f"""The tile {task_name} of layer '{layer_name}' is finish 73 | Nb {task_name} tiles: {self.count.nb} 74 | Nb errored tiles: {gene.error} 75 | Nb dropped tiles: {count_tiles_dropped.nb} 76 | Total time: {duration_format(gene.duration)} 77 | Total size: {size_format(self.count.size)} 78 | Time per tile: {(gene.duration / self.count.nb * 1000).seconds if self.count.nb != 0 else 0} ms 79 | Size per tile: {self.count.size / self.count.nb if self.count.nb != 0 else -1} o 80 | """, 81 | ) 82 | 83 | 84 | def main() -> None: 85 | """Copy the tiles from a cache to an other.""" 86 | asyncio.run(_async_main()) 87 | 88 | 89 | async def _async_main() -> None: 90 | """Copy the tiles from a cache to an other.""" 91 | try: 92 | parser = ArgumentParser( 93 | description="Used to copy the tiles from a cache to an other", 94 | prog=sys.argv[0], 95 | ) 96 | add_common_options(parser, near=False, time=False, dimensions=True, cache=False) 97 | parser.add_argument("--process", dest="process", metavar="NAME", help="The process name to do") 98 | parser.add_argument("source", metavar="SOURCE", help="The source cache") 99 | parser.add_argument("dest", metavar="DEST", help="The destination cache") 100 | 101 | options = parser.parse_args() 102 | 103 | gene = TileGeneration(options.config, options) 104 | assert gene.config_file 105 | config = gene.get_config(gene.config_file) 106 | 107 | if options.layer: 108 | copy = Copy() 109 | await copy.copy(options, gene, options.layer, options.source, options.dest, "copy") 110 | else: 111 | layers = ( 112 | config.config["generation"]["default_layers"] 113 | if "default_layers" in config.config["generation"] 114 | else config.config["layers"].keys() 115 | ) 116 | for layer in layers: 117 | copy = Copy() 118 | await copy.copy(options, gene, layer, options.source, options.dest, "copy") 119 | except SystemExit: 120 | raise 121 | except: # pylint: disable=bare-except 122 | _logger.exception("Exit with exception") 123 | if os.environ.get("TESTS", "false").lower() == "true": 124 | raise 125 | sys.exit(1) 126 | 127 | 128 | def process() -> None: 129 | """Copy the tiles from a cache to an other.""" 130 | asyncio.run(_async_process()) 131 | 132 | 133 | async def _async_process() -> None: 134 | """Copy the tiles from a cache to an other.""" 135 | try: 136 | parser = ArgumentParser( 137 | description="Used to copy the tiles from a cache to an other", 138 | prog=sys.argv[0], 139 | ) 140 | add_common_options(parser, near=False, time=False, dimensions=True) 141 | parser.add_argument("process", metavar="PROCESS", help="The process name to do") 142 | 143 | options = parser.parse_args() 144 | 145 | gene = TileGeneration(options.config, options, multi_task=False) 146 | 147 | copy = Copy() 148 | if options.layer: 149 | await copy.copy(options, gene, options.layer, options.cache, options.cache, "process") 150 | else: 151 | assert gene.config_file 152 | config = gene.get_config(gene.config_file) 153 | layers_name = ( 154 | config.config["generation"]["default_layers"] 155 | if "default_layers" in config.config.get("generation", {}) 156 | else config.config["layers"].keys() 157 | ) 158 | for layer in layers_name: 159 | await copy.copy(options, gene, layer, options.cache, options.cache, "process") 160 | except SystemExit: 161 | raise 162 | except: # pylint: disable=bare-except 163 | _logger.exception("Exit with exception") 164 | sys.exit(1) 165 | -------------------------------------------------------------------------------- /tilecloud_chain/database_logger.py: -------------------------------------------------------------------------------- 1 | """Log the generated tiles in a database.""" 2 | 3 | import asyncio 4 | import logging 5 | import sys 6 | 7 | import psycopg.sql 8 | from prometheus_client import Summary 9 | from tilecloud import Tile 10 | 11 | import tilecloud_chain.configuration 12 | 13 | _LOGGER = logging.getLogger(__name__) 14 | 15 | _INSERT_SUMMARY = Summary("tilecloud_chain_database_logger", "Number of database inserts", ["layer"]) 16 | 17 | 18 | class DatabaseLoggerCommon: 19 | """Log the generated tiles in a database.""" 20 | 21 | def __init__(self, config: tilecloud_chain.configuration.Logging, daemon: bool) -> None: 22 | self._db_params = config["database"] 23 | self._daemon = daemon 24 | self.connection: psycopg.AsyncConnection | None = None 25 | self.schema: str | None = None 26 | self.table: str | None = None 27 | 28 | async def _init(self) -> None: 29 | while True: 30 | try: 31 | self.connection = await psycopg.AsyncConnection.connect( 32 | dbname=self._db_params["dbname"], 33 | host=self._db_params.get("host"), 34 | port=self._db_params.get("port"), 35 | user=self._db_params.get("user"), 36 | password=self._db_params.get("password"), 37 | ) 38 | break 39 | except psycopg.OperationalError: 40 | _LOGGER.warning("Failed connecting to the database. Will try again in 1s", exc_info=True) 41 | if self._daemon: 42 | await asyncio.sleep(1) 43 | else: 44 | sys.exit(2) 45 | assert self.connection is not None 46 | 47 | if "." in self._db_params["table"]: 48 | schema, table = self._db_params["table"].split(".") 49 | else: 50 | schema = "public" 51 | table = self._db_params["table"] 52 | 53 | async with self.connection.cursor() as cursor: 54 | await cursor.execute( 55 | "SELECT EXISTS(SELECT 1 FROM pg_tables WHERE schemaname=%s AND tablename=%s)", 56 | (schema, table), 57 | ) 58 | schema = psycopg.sql.quote(schema, self.connection) 59 | table = psycopg.sql.quote(table, self.connection) 60 | 61 | elem = await cursor.fetchone() 62 | assert elem is not None 63 | if not elem[0]: 64 | try: 65 | await cursor.execute( 66 | psycopg.sql.SQL( 67 | "CREATE TABLE {}.{} (" 68 | " id BIGSERIAL PRIMARY KEY," 69 | " layer CHARACTER VARYING(80) NOT NULL," 70 | " run INTEGER NOT NULL," 71 | " action CHARACTER VARYING(7) NOT NULL," 72 | " tile TEXT NOT NULL," 73 | " UNIQUE (layer, run, tile))", 74 | ).format(psycopg.sql.Identifier(schema), psycopg.sql.Identifier(table)), 75 | ) 76 | await self.connection.commit() 77 | except psycopg.DatabaseError: 78 | _LOGGER.exception("Unable to create table %s.%s", schema, table) 79 | sys.exit(1) 80 | else: 81 | try: 82 | await cursor.execute( 83 | psycopg.sql.SQL( 84 | "INSERT INTO {}.{}(layer, run, action, tile) VALUES (%s, %s, %s, %s)", 85 | ).format(psycopg.sql.Identifier(schema), psycopg.sql.Identifier(table)), 86 | ("test_layer", -1, "test", "-1x-1"), 87 | ) 88 | except psycopg.DatabaseError: 89 | _LOGGER.exception("Unable to insert logging data into %s.%s", schema, table) 90 | sys.exit(1) 91 | finally: 92 | await self.connection.rollback() 93 | 94 | self.schema = schema 95 | self.table = table 96 | 97 | 98 | class DatabaseLoggerInit(DatabaseLoggerCommon): 99 | """Log the generated tiles in a database.""" 100 | 101 | def __init__(self, config: tilecloud_chain.configuration.Logging, daemon: bool) -> None: 102 | super().__init__(config, daemon) 103 | 104 | self.init = False 105 | self.run = -1 106 | 107 | async def _init(self) -> None: 108 | assert self.connection is not None 109 | assert self.schema is not None 110 | assert self.table is not None 111 | 112 | async with self.connection.cursor() as cursor: 113 | await cursor.execute( 114 | psycopg.sql.SQL("SELECT COALESCE(MAX(run), 0) + 1 FROM {}.{}").format( 115 | psycopg.sql.Identifier(self.schema), 116 | psycopg.sql.Identifier(self.table), 117 | ), 118 | ) 119 | elem = await cursor.fetchone() 120 | assert elem is not None 121 | (self.run,) = elem 122 | self.init = True 123 | 124 | async def __call__(self, tile: Tile) -> Tile: 125 | """Log the generated tiles in a database.""" 126 | if not self.init: 127 | await self._init() 128 | tile.metadata["run"] = self.run # type: ignore[assignment] 129 | return tile 130 | 131 | 132 | class DatabaseLogger(DatabaseLoggerCommon): 133 | """Log the generated tiles in a database.""" 134 | 135 | async def __call__(self, tile: Tile) -> Tile: 136 | """Log the generated tiles in a database.""" 137 | if self.connection is None: 138 | await self._init() 139 | assert self.connection is not None 140 | assert self.schema is not None 141 | assert self.table is not None 142 | 143 | if tile is None: 144 | _LOGGER.warning("The tile is None") 145 | return None 146 | 147 | if tile.error: 148 | action = "error" 149 | elif tile.data: 150 | action = "create" 151 | else: 152 | action = "delete" 153 | 154 | layer = tile.metadata.get("layer", "- No layer -") 155 | run = tile.metadata.get("run", -1) 156 | 157 | with _INSERT_SUMMARY.labels(layer).time(): 158 | async with self.connection.cursor() as cursor: 159 | try: 160 | await cursor.execute( 161 | psycopg.sql.SQL( 162 | "INSERT INTO {} (layer, run, action, tile) " 163 | "VALUES (%(layer)s, %(run)s, %(action)s::varchar(7), %(tile)s)", 164 | ).format(psycopg.sql.Identifier(self.schema), psycopg.sql.Identifier(self.table)), 165 | {"layer": layer, "action": action, "tile": str(tile.tilecoord), "run": run}, 166 | ) 167 | except psycopg.IntegrityError: 168 | await self.connection.rollback() 169 | await cursor.execute( 170 | psycopg.sql.SQL( 171 | "UPDATE {} SET action = %(action)s " 172 | "WHERE layer = %(layer)s AND run = %(run)s AND tile = %(tile)s", 173 | ).format(psycopg.sql.Identifier(self.schema), psycopg.sql.Identifier(self.table)), 174 | {"layer": layer, "action": action, "tile": str(tile.tilecoord), "run": run}, 175 | ) 176 | 177 | await self.connection.commit() 178 | 179 | return tile 180 | -------------------------------------------------------------------------------- /tilecloud_chain/expiretiles.py: -------------------------------------------------------------------------------- 1 | """Import the osm2pgsql expire-tiles file to Postgres.""" 2 | 3 | import logging 4 | import sys 5 | from argparse import ArgumentParser 6 | from pathlib import Path 7 | 8 | import psycopg2.sql 9 | from shapely.geometry import MultiPolygon, Polygon 10 | from shapely.ops import unary_union 11 | from tilecloud.grid.quad import QuadTileGrid 12 | 13 | from tilecloud_chain import parse_tilecoord 14 | 15 | logger = logging.getLogger(__name__) 16 | 17 | 18 | def main() -> None: 19 | """Import the osm2pgsql expire-tiles file to Postgres.""" 20 | try: 21 | parser = ArgumentParser( 22 | description="Used to import the osm2pgsql expire-tiles file to Postgres", 23 | prog=sys.argv[0], 24 | ) 25 | parser.add_argument( 26 | "--buffer", 27 | type=float, 28 | default=0.0, 29 | help="Extent buffer to the tiles [m], default is 0", 30 | ) 31 | parser.add_argument( 32 | "--simplify", 33 | type=float, 34 | default=0.0, 35 | help="Simplify the result geometry [m], default is 0", 36 | ) 37 | parser.add_argument( 38 | "--create", 39 | default=False, 40 | action="store_true", 41 | help="create the table if not exists", 42 | ) 43 | parser.add_argument( 44 | "--delete", 45 | default=False, 46 | action="store_true", 47 | help="empty the table", 48 | ) 49 | parser.add_argument( 50 | "file", 51 | type=Path, 52 | metavar="FILE", 53 | help="The osm2pgsql expire-tiles file", 54 | ) 55 | parser.add_argument( 56 | "connection", 57 | metavar="CONNECTION", 58 | help=( 59 | "The PostgreSQL connection string e.g. " 60 | '"user=www-data password=www-data dbname=sig host=localhost"' 61 | ), 62 | ) 63 | parser.add_argument( 64 | "table", 65 | metavar="TABLE", 66 | help="The PostgreSQL table to fill", 67 | ) 68 | parser.add_argument( 69 | "--schema", 70 | default="public", 71 | help="The PostgreSQL schema to use (should already exists), default is public", 72 | ) 73 | parser.add_argument( 74 | "column", 75 | metavar="COLUMN", 76 | default="geom", 77 | nargs="?", 78 | help='The PostgreSQL column, default is "geom"', 79 | ) 80 | parser.add_argument( 81 | "--srid", 82 | type=int, 83 | default=3857, 84 | nargs="?", 85 | help="The stored geometry SRID, no conversion by default (3857)", 86 | ) 87 | options = parser.parse_args() 88 | 89 | connection = psycopg2.connect(options.connection) 90 | cursor = connection.cursor() 91 | 92 | if options.create: 93 | cursor.execute( 94 | "SELECT count(*) FROM pg_tables WHERE schemaname=%(schema)s AND tablename=%(table)s", 95 | {"schema": options.schema, "table": options.table}, 96 | ) 97 | if cursor.fetchone()[0] == 0: 98 | cursor.execute( 99 | psycopg2.sql.SQL("CREATE TABLE IF NOT EXISTS {}.{} (id serial)").format( 100 | psycopg2.sql.Identifier(options.schema), 101 | psycopg2.sql.Identifier(options.table), 102 | ), 103 | ) 104 | cursor.execute( 105 | "SELECT AddGeometryColumn(%(schema)s, %(table)s, %(column)s, %(srid)s, 'MULTIPOLYGON', 2)", 106 | { 107 | "schema": options.schema, 108 | "table": options.table, 109 | "column": options.column, 110 | "srid": options.srid, 111 | }, 112 | ) 113 | 114 | if options.delete: 115 | cursor.execute(psycopg2.sql.SQL("DELETE FROM {}").format(psycopg2.sql.Identifier(options.table))) 116 | 117 | geoms = [] 118 | grid = QuadTileGrid( 119 | max_extent=(-20037508.34, -20037508.34, 20037508.34, 20037508.34), 120 | ) 121 | with options.file.open(encoding="utf-8") as f: 122 | for coord in f: 123 | extent = grid.extent(parse_tilecoord(coord), options.buffer) 124 | geoms.append( 125 | Polygon( 126 | ( 127 | (extent[0], extent[1]), 128 | (extent[0], extent[3]), 129 | (extent[2], extent[3]), 130 | (extent[2], extent[1]), 131 | ), 132 | ), 133 | ) 134 | if len(geoms) == 0: 135 | print("No coords found") 136 | connection.commit() 137 | cursor.close() 138 | connection.close() 139 | sys.exit(0) 140 | geom = unary_union(geoms) 141 | if geom.geom_type == "Polygon": 142 | geom = MultiPolygon((geom,)) 143 | 144 | if options.simplify > 0: 145 | geom.simplify(options.simplify) 146 | 147 | if options.srid <= 0: 148 | cursor.execute( 149 | psycopg2.sql.SQL("INSERT INTO {} ({}) VALUES (ST_GeomFromText(%(geom)s))").format( 150 | psycopg2.sql.Identifier(options.table), 151 | psycopg2.sql.Identifier(options.column), 152 | ), 153 | { 154 | "geom": geom.wkt, 155 | }, 156 | ) 157 | 158 | elif options.srid != 3857: 159 | cursor.execute( 160 | psycopg2.sql.SQL( 161 | "INSERT INTO {} ({}) VALUES (ST_Transform(ST_GeomFromText(%(geom)s, 3857), %(srid)s))", 162 | ).format( 163 | psycopg2.sql.Identifier(options.table), 164 | psycopg2.sql.Identifier(options.column), 165 | ), 166 | { 167 | "geom": geom.wkt, 168 | "srid": options.srid, 169 | }, 170 | ) 171 | else: 172 | cursor.execute( 173 | psycopg2.sql.SQL("INSERT INTO {} ({}) VALUES (ST_GeomFromText(%(geom)s, 3857))").format( 174 | psycopg2.sql.Identifier(options.table), 175 | psycopg2.sql.Identifier(options.column), 176 | ), 177 | { 178 | "geom": geom.wkt, 179 | "srid": options.srid, 180 | }, 181 | ) 182 | 183 | connection.commit() 184 | cursor.close() 185 | connection.close() 186 | print("Import successful") 187 | except SystemExit: 188 | raise 189 | except: # pylint: disable=bare-except 190 | logger.exception("Exit with exception") 191 | sys.exit(1) 192 | -------------------------------------------------------------------------------- /tilecloud_chain/filter/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/camptocamp/tilecloud-chain/2f84239147eb4025af917a862d8b3cb38f4a88e1/tilecloud_chain/filter/__init__.py -------------------------------------------------------------------------------- /tilecloud_chain/filter/error.py: -------------------------------------------------------------------------------- 1 | """Module includes filters for dealing with errors in tiles.""" 2 | 3 | from tilecloud import Tile 4 | 5 | 6 | class MaximumConsecutiveErrors: 7 | """ 8 | Create a filter that limit the consecutive errors. 9 | 10 | Raises a :class:`TooManyErrors` exception when there are ``max_consecutive_errors`` 11 | consecutive errors. 12 | 13 | max_consecutive_errors: 14 | The max number of permitted consecutive errors. Once 15 | exceeded a :class:`TooManyErrors` exception is raised. 16 | """ 17 | 18 | def __init__(self, max_consecutive_errors: int) -> None: 19 | self.max_consecutive_errors = max_consecutive_errors 20 | self.consecutive_errors = 0 21 | 22 | def __call__(self, tile: Tile) -> Tile: 23 | """Call the filter.""" 24 | if tile and tile.error: 25 | self.consecutive_errors += 1 26 | if self.consecutive_errors > self.max_consecutive_errors: 27 | if isinstance(tile.error, Exception): 28 | raise TooManyError(tile) from tile.error 29 | raise TooManyError(tile) 30 | else: 31 | self.consecutive_errors = 0 32 | return tile 33 | 34 | 35 | class MaximumErrorRate: 36 | """ 37 | Create a filter that limit the error rate. 38 | 39 | Raises a :class:`TooManyErrors` exception when the total error rate exceeds ``max_error_rate``. 40 | 41 | max_error_rate: 42 | The maximum error rate. Once exceeded a :class:`TooManyErrors` 43 | exception is raised. 44 | min_tiles: 45 | The minimum number of received tiles before a :class:`TooManyErrors` 46 | exception can be raised. Defaults to 8. 47 | """ 48 | 49 | def __init__(self, max_error_rate: float, min_tiles: int = 8) -> None: 50 | self.max_error_rate = max_error_rate 51 | self.min_tiles = min_tiles 52 | self.tile_count = 0 53 | self.error_count = 0 54 | 55 | def __call__(self, tile: Tile) -> Tile: 56 | """Call the filter.""" 57 | self.tile_count += 1 58 | if tile and tile.error: 59 | self.error_count += 1 60 | if ( 61 | self.tile_count >= self.min_tiles 62 | and self.error_count >= self.max_error_rate * self.tile_count 63 | ): 64 | if isinstance(tile.error, Exception): 65 | raise TooManyError(tile) from tile.error 66 | raise TooManyError(tile) 67 | return tile 68 | 69 | 70 | class TooManyError(RuntimeError): 71 | """TooManyErrors exception class.""" 72 | 73 | def __init__(self, tile: Tile) -> None: 74 | self.last_tile = tile 75 | super().__init__( 76 | f"Too many errors, last tile in error {tile.tilecoord} {tile.formated_metadata}\n{tile.error}", 77 | ) 78 | -------------------------------------------------------------------------------- /tilecloud_chain/format.py: -------------------------------------------------------------------------------- 1 | """Format functions.""" 2 | 3 | from datetime import timedelta 4 | 5 | 6 | def default_int(number_array: tuple[float, float, float, float]) -> tuple[int, int, int, int]: 7 | """Convert an array of float in an array of int.""" 8 | return (int(number_array[0]), int(number_array[1]), int(number_array[2]), int(number_array[3])) 9 | 10 | 11 | def size_format(number: float) -> str: 12 | """Get human readable size.""" 13 | for unit in ["o", "Kio", "Mio", "Gio", "Tio"]: 14 | if number < 1024.0: 15 | if number < 10: 16 | return f"{number:.1f} {unit}" 17 | return f"{number:.0f} {unit}" 18 | number /= 1024.0 19 | return f"{number:.0f} Tio" 20 | 21 | 22 | def duration_format(duration: timedelta) -> str: 23 | """Get human readable duration.""" 24 | hours, remainder = divmod(duration.seconds, 3600) 25 | minutes, seconds = divmod(remainder, 60) 26 | if duration.days > 0: 27 | return f"{duration.days} {hours}:{minutes:02d}:{seconds:02d}" 28 | return f"{hours}:{minutes:02d}:{seconds:02d}" 29 | -------------------------------------------------------------------------------- /tilecloud_chain/host-limit-schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-07/schema", 3 | "$id": "https://raw.githubusercontent.com/camptocamp/tilecloud-chain/master/tilecloud_chain/schema.json", 4 | "type": "object", 5 | "title": "TileCloud-chain host limit configuration", 6 | "description": "The configuration of the concurrent request limit on a host", 7 | "additionalProperties": false, 8 | "definitions": {}, 9 | "properties": { 10 | "default": { 11 | "type": "object", 12 | "title": "Default values", 13 | "properties": { 14 | "concurrent": { 15 | "type": "integer", 16 | "title": "Default concurrent limit", 17 | "description": "Default limit of concurrent request on the same host (can be set with the `TILECLOUD_CHAIN_HOST_CONCURRENT` environment variable.", 18 | "default": 1 19 | } 20 | } 21 | }, 22 | "hosts": { 23 | "type": "object", 24 | "title": "Hosts", 25 | "additionalProperties": { 26 | "type": "object", 27 | "title": "Host", 28 | "properties": { 29 | "concurrent": { 30 | "type": "integer", 31 | "title": "Concurrent limit", 32 | "description": "Limit of concurrent request on the host" 33 | } 34 | } 35 | } 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /tilecloud_chain/host_limit.py: -------------------------------------------------------------------------------- 1 | """ 2 | Automatically generated file from a JSON schema. 3 | """ 4 | 5 | from typing import TypedDict 6 | 7 | DEFAULT_CONCURRENT_LIMIT_DEFAULT = 1 8 | """ Default value of the field path 'Default values concurrent' """ 9 | 10 | 11 | class DefaultValues(TypedDict, total=False): 12 | """Default values.""" 13 | 14 | concurrent: int 15 | """ 16 | Default concurrent limit. 17 | 18 | Default limit of concurrent request on the same host (can be set with the `TILECLOUD_CHAIN_HOST_CONCURRENT` environment variable. 19 | 20 | default: 1 21 | """ 22 | 23 | 24 | class Host(TypedDict, total=False): 25 | """Host.""" 26 | 27 | concurrent: int 28 | """ 29 | Concurrent limit. 30 | 31 | Limit of concurrent request on the host 32 | """ 33 | 34 | 35 | class HostLimit(TypedDict, total=False): 36 | """ 37 | TileCloud-chain host limit configuration. 38 | 39 | The configuration of the concurrent request limit on a host 40 | """ 41 | 42 | default: "DefaultValues" 43 | """ Default values. """ 44 | 45 | hosts: dict[str, "Host"] 46 | """ Hosts. """ 47 | -------------------------------------------------------------------------------- /tilecloud_chain/multitilestore.py: -------------------------------------------------------------------------------- 1 | """Redirect to the corresponding Tilestore for the layer and config file.""" 2 | 3 | import logging 4 | from collections.abc import AsyncIterator, Callable 5 | from dataclasses import dataclass 6 | from pathlib import Path 7 | 8 | from tilecloud import Tile 9 | 10 | from tilecloud_chain.store import AsyncTilesIterator, AsyncTileStore 11 | 12 | logger = logging.getLogger(__name__) 13 | 14 | 15 | @dataclass 16 | class _DatedStore: 17 | """Store the date and the store.""" 18 | 19 | mtime: float 20 | store: AsyncTileStore 21 | 22 | 23 | class MultiTileStore(AsyncTileStore): 24 | """Redirect to the corresponding Tilestore for the layer and config file.""" 25 | 26 | def __init__(self, get_store: Callable[[Path, str], AsyncTileStore | None]) -> None: 27 | """Initialize.""" 28 | self.get_store = get_store 29 | self.stores: dict[tuple[Path, str], _DatedStore | None] = {} 30 | 31 | def _get_store(self, config_file: Path, layer: str) -> AsyncTileStore | None: 32 | config_path = Path(config_file) 33 | mtime = config_path.stat().st_mtime 34 | store = self.stores.get((config_file, layer)) 35 | if store is not None and store.mtime != mtime: 36 | store = None 37 | if store is None: 38 | tile_store = self.get_store(config_file, layer) 39 | if tile_store is not None: 40 | store = _DatedStore(mtime, tile_store) 41 | self.stores[(config_file, layer)] = store 42 | return store.store if store is not None else None 43 | 44 | def _get_store_tile(self, tile: Tile) -> AsyncTileStore | None: 45 | """Return the store corresponding to the tile.""" 46 | layer = tile.metadata["layer"] 47 | config_file = Path(tile.metadata["config_file"]) 48 | return self._get_store(config_file, layer) 49 | 50 | async def __contains__(self, tile: Tile) -> bool: 51 | """ 52 | Return true if this store contains ``tile``. 53 | 54 | Arguments: 55 | tile: Tile 56 | """ 57 | store = self._get_store_tile(tile) 58 | assert store is not None 59 | return tile in store 60 | 61 | async def delete_one(self, tile: Tile) -> Tile: 62 | """ 63 | Delete ``tile`` and return ``tile``. 64 | 65 | Arguments: 66 | tile: Tile 67 | """ 68 | store = self._get_store_tile(tile) 69 | assert store is not None 70 | return await store.delete_one(tile) 71 | 72 | async def list(self) -> AsyncIterator[Tile]: 73 | """Generate all the tiles in the store, but without their data.""" 74 | # Too dangerous to list all tiles in all stores. Return an empty iterator instead 75 | while False: 76 | yield 77 | 78 | async def put_one(self, tile: Tile) -> Tile: 79 | """ 80 | Store ``tile`` in the store. 81 | 82 | Arguments: 83 | tile: Tile 84 | """ 85 | store = self._get_store_tile(tile) 86 | assert store is not None 87 | return await store.put_one(tile) 88 | 89 | async def get_one(self, tile: Tile) -> Tile | None: 90 | """ 91 | Add data to ``tile``, or return ``None`` if ``tile`` is not in the store. 92 | 93 | Arguments: 94 | tile: Tile 95 | """ 96 | store = self._get_store_tile(tile) 97 | assert store is not None 98 | return await store.get_one(tile) 99 | 100 | async def get(self, tiles: AsyncIterator[Tile]) -> AsyncIterator[Tile | None]: 101 | """ 102 | Add data to the tiles, or return ``None`` if the tile is not in the store. 103 | 104 | Arguments: 105 | tiles: AsyncIterator[Tile] 106 | """ 107 | async for tile in tiles: 108 | store = self._get_store_tile(tile) 109 | assert store is not None, f"No store found for tile {tile.tilecoord} {tile.formated_metadata}" 110 | 111 | async for new_tile in store.get(AsyncTilesIterator([tile])()): 112 | yield new_tile 113 | 114 | def __str__(self) -> str: 115 | """Return a string representation of the object.""" 116 | stores = {str(store) for store in self.stores.values()} 117 | keys = {f"{config_file}:{layer}" for config_file, layer in self.stores} 118 | return f"{self.__class__.__name__}({', '.join(stores)} - {', '.join(keys)})" 119 | 120 | def __repr__(self) -> str: 121 | """Return a string representation of the object.""" 122 | stores = {repr(store) for store in self.stores.values()} 123 | keys = {f"{config_file}:{layer}" for config_file, layer in self.stores} 124 | return f"{self.__class__.__name__}({', '.join(stores)} - {', '.join(keys)})" 125 | 126 | @staticmethod 127 | def _get_layer(tile: Tile | None) -> tuple[str, str]: 128 | assert tile is not None 129 | return (tile.metadata["config_file"], tile.metadata["layer"]) 130 | -------------------------------------------------------------------------------- /tilecloud_chain/py.typed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/camptocamp/tilecloud-chain/2f84239147eb4025af917a862d8b3cb38f4a88e1/tilecloud_chain/py.typed -------------------------------------------------------------------------------- /tilecloud_chain/security.py: -------------------------------------------------------------------------------- 1 | """Security policy for the pyramid application.""" 2 | 3 | import os 4 | 5 | import c2cwsgiutils.auth 6 | import pyramid.request 7 | from c2cwsgiutils.auth import AuthConfig 8 | from pyramid.security import Allowed, Denied 9 | 10 | 11 | class User: 12 | """The user definition.""" 13 | 14 | login: str | None 15 | name: str | None 16 | url: str | None 17 | is_auth: bool 18 | token: str | None 19 | is_admin: bool 20 | request: pyramid.request.Request 21 | 22 | def __init__( 23 | self, 24 | auth_type: str, 25 | login: str | None, 26 | name: str | None, 27 | url: str | None, 28 | is_auth: bool, 29 | token: str | None, 30 | request: pyramid.request.Request, 31 | ) -> None: 32 | self.auth_type = auth_type 33 | self.login = login 34 | self.name = name 35 | self.url = url 36 | self.is_auth = is_auth 37 | self.token = token 38 | self.request = request 39 | self.is_admin = c2cwsgiutils.auth.check_access(self.request) 40 | 41 | def has_access(self, auth_config: AuthConfig) -> bool: 42 | """Check if the user has access to the tenant.""" 43 | if self.is_admin: 44 | return True 45 | if "github_repository" in auth_config: 46 | return c2cwsgiutils.auth.check_access_config(self.request, auth_config) 47 | 48 | return False 49 | 50 | 51 | class SecurityPolicy: 52 | """The pyramid security policy.""" 53 | 54 | def identity(self, request: pyramid.request.Request) -> User: 55 | """Return app-specific user object.""" 56 | if not hasattr(request, "user"): 57 | if "TEST_USER" in os.environ: 58 | user = User( 59 | auth_type="test_user", 60 | login=os.environ["TEST_USER"], 61 | name=os.environ["TEST_USER"], 62 | url="https://example.com/user", 63 | is_auth=True, 64 | token=None, 65 | request=request, 66 | ) 67 | else: 68 | is_auth, c2cuser = c2cwsgiutils.auth.is_auth_user(request) 69 | user = User( 70 | "github_oauth", 71 | c2cuser.get("login"), 72 | c2cuser.get("name"), 73 | c2cuser.get("url"), 74 | is_auth, 75 | c2cuser.get("token"), 76 | request, 77 | ) 78 | request.user = user 79 | return request.user # type: ignore[no-any-return] 80 | 81 | def authenticated_userid(self, request: pyramid.request.Request) -> str | None: 82 | """Return a string ID for the user.""" 83 | identity = self.identity(request) 84 | 85 | if identity is None: 86 | return None 87 | 88 | return identity.login 89 | 90 | def permits( 91 | self, 92 | request: pyramid.request.Request, 93 | context: AuthConfig, 94 | permission: str, 95 | ) -> Allowed | Denied: 96 | """Allow access to everything if signed in.""" 97 | identity = self.identity(request) 98 | 99 | if identity is None: 100 | return Denied("User is not signed in.") 101 | if identity.auth_type in ("test_user",): 102 | return Allowed(f"All access auth type: {identity.auth_type}") 103 | if identity.is_admin: 104 | return Allowed("The User is admin.") 105 | if permission == "all": 106 | return Denied("Root access is required.") 107 | if identity.has_access(context): 108 | return Allowed("The User has access.") 109 | return Denied(f"The User has no access to source {permission}.") 110 | -------------------------------------------------------------------------------- /tilecloud_chain/static/favicon-16x16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/camptocamp/tilecloud-chain/2f84239147eb4025af917a862d8b3cb38f4a88e1/tilecloud_chain/static/favicon-16x16.png -------------------------------------------------------------------------------- /tilecloud_chain/static/favicon-32x32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/camptocamp/tilecloud-chain/2f84239147eb4025af917a862d8b3cb38f4a88e1/tilecloud_chain/static/favicon-32x32.png -------------------------------------------------------------------------------- /tilecloud_chain/store/__init__.py: -------------------------------------------------------------------------------- 1 | from collections.abc import AsyncIterator, Callable 2 | from typing import Any 3 | 4 | from tilecloud import Tile, TileCoord, TileStore 5 | 6 | 7 | class AsyncTileStore: 8 | """A tile store.""" 9 | 10 | async def __contains__(self, tile: Tile) -> bool: 11 | """ 12 | Return true if this store contains ``tile``. 13 | 14 | Attributes 15 | ---------- 16 | tile: Tile 17 | 18 | """ 19 | raise NotImplementedError 20 | 21 | async def delete_one(self, tile: Tile) -> Tile: 22 | """ 23 | Delete ``tile`` and return ``tile``. 24 | 25 | Attributes 26 | ---------- 27 | tile: Tile 28 | 29 | """ 30 | raise NotImplementedError 31 | 32 | async def get_one(self, tile: Tile) -> Tile | None: 33 | """ 34 | Add data to ``tile``, or return ``None`` if ``tile`` is not in the store. 35 | 36 | Attributes 37 | ---------- 38 | tile: Tile 39 | 40 | """ 41 | raise NotImplementedError 42 | 43 | async def get(self, tiles: AsyncIterator[Tile]) -> AsyncIterator[Tile | None]: 44 | """ 45 | Add data to the tiles, or return ``None`` if the tile is not in the store. 46 | 47 | Attributes 48 | ---------- 49 | tiles: AsyncIterator[Tile] 50 | 51 | """ 52 | del tiles 53 | raise NotImplementedError 54 | yield Tile(TileCoord(0, 0, 0)) # pylint: disable=unreachable 55 | # async for tile in tiles: 56 | # yield await self.get_one(tile) 57 | 58 | async def list(self) -> AsyncIterator[Tile]: 59 | """Generate all the tiles in the store, but without their data.""" 60 | raise NotImplementedError 61 | yield Tile(TileCoord(0, 0, 0)) # pylint: disable=unreachable 62 | 63 | async def put_one(self, tile: Tile) -> Tile: 64 | """ 65 | Store ``tile`` in the store. 66 | 67 | Attributes 68 | ---------- 69 | tile: Tile 70 | 71 | """ 72 | raise NotImplementedError 73 | 74 | 75 | class TileStoreWrapper(AsyncTileStore): 76 | """Wrap a TileStore.""" 77 | 78 | def __init__(self, tile_store: TileStore) -> None: 79 | """Initialize.""" 80 | self.tile_store = tile_store 81 | 82 | async def __contains__(self, tile: Tile) -> bool: 83 | """See in superclass.""" 84 | return self.tile_store.__contains__(tile) 85 | 86 | async def delete_one(self, tile: Tile) -> Tile: 87 | """See in superclass.""" 88 | return self.tile_store.delete_one(tile) 89 | 90 | async def get_one(self, tile: Tile) -> Tile | None: 91 | """See in superclass.""" 92 | return self.tile_store.get_one(tile) 93 | 94 | async def get(self, tiles: AsyncIterator[Tile]) -> AsyncIterator[Tile | None]: 95 | """See in superclass.""" 96 | all_tiles = [] 97 | all_tiles = [tile async for tile in tiles] 98 | for new_tile in self.tile_store.get(all_tiles): 99 | yield new_tile 100 | 101 | async def list(self) -> AsyncIterator[Tile]: 102 | """See in superclass.""" 103 | for tile in self.tile_store.list(): 104 | yield tile 105 | 106 | async def put_one(self, tile: Tile) -> Tile: 107 | """See in superclass.""" 108 | return self.tile_store.put_one(tile) 109 | 110 | def __getattr__(self, item: str) -> Any: 111 | """See in superclass.""" 112 | return getattr(self.tile_store, item) 113 | 114 | 115 | class NoneTileStore(AsyncTileStore): 116 | """A tile store that does nothing.""" 117 | 118 | async def __contains__(self, tile: Tile) -> bool: 119 | """See in superclass.""" 120 | raise NotImplementedError 121 | 122 | async def delete_one(self, tile: Tile) -> Tile: 123 | """See in superclass.""" 124 | raise NotImplementedError 125 | 126 | async def get_one(self, tile: Tile) -> Tile | None: 127 | """See in superclass.""" 128 | return tile 129 | 130 | async def list(self) -> AsyncIterator[Tile]: 131 | """See in superclass.""" 132 | raise NotImplementedError 133 | yield Tile(TileCoord(0, 0, 0)) # pylint: disable=unreachable 134 | 135 | async def get(self, tiles: AsyncIterator[Tile]) -> AsyncIterator[Tile | None]: 136 | """See in superclass.""" 137 | async for tile in tiles: 138 | yield tile 139 | 140 | 141 | class CallWrapper: 142 | """Wrap a function call.""" 143 | 144 | def __init__(self, function: Callable[[Tile], Tile | None]) -> None: 145 | """Initialize.""" 146 | self.function = function 147 | 148 | async def __call__(self, tile: Tile) -> Tile | None: 149 | """See in superclass.""" 150 | return self.function(tile) 151 | 152 | 153 | class AsyncTilesIterator: 154 | """An async iterator.""" 155 | 156 | def __init__(self, tiles: list[Tile]) -> None: 157 | """Initialize.""" 158 | self._tiles = tiles 159 | 160 | async def __call__(self) -> AsyncIterator[Tile]: 161 | """Async iterator of the tiles.""" 162 | for tile in self._tiles: 163 | yield tile 164 | -------------------------------------------------------------------------------- /tilecloud_chain/store/azure_storage_blob.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | from collections.abc import AsyncIterator 4 | 5 | from azure.identity import DefaultAzureCredential 6 | from azure.storage.blob import ContentSettings 7 | from azure.storage.blob.aio import BlobServiceClient, ContainerClient 8 | from tilecloud import Tile, TileLayout 9 | 10 | from tilecloud_chain.store import AsyncTileStore 11 | 12 | _LOGGER = logging.getLogger(__name__) 13 | 14 | 15 | class AzureStorageBlobTileStore(AsyncTileStore): 16 | """Tiles stored in Azure storage blob.""" 17 | 18 | def __init__( 19 | self, 20 | tilelayout: TileLayout, 21 | container: str | None = None, 22 | dry_run: bool = False, 23 | cache_control: str | None = None, 24 | container_client: ContainerClient | None = None, 25 | ) -> None: 26 | """Initialize.""" 27 | if container_client is None: 28 | if "AZURE_STORAGE_CONNECTION_STRING" in os.environ: 29 | assert container is not None 30 | self.container_client = BlobServiceClient.from_connection_string( 31 | os.environ["AZURE_STORAGE_CONNECTION_STRING"], 32 | ).get_container_client(container=container) 33 | elif "AZURE_STORAGE_BLOB_CONTAINER_URL" in os.environ: 34 | self.container_client = ContainerClient.from_container_url( 35 | os.environ["AZURE_STORAGE_BLOB_CONTAINER_URL"], 36 | ) 37 | if os.environ.get("AZURE_STORAGE_BLOB_VALIDATE_CONTAINER_NAME", "false").lower() == "true": 38 | assert container == self.container_client.container_name 39 | else: 40 | assert container is not None 41 | self.container_client = BlobServiceClient( 42 | account_url=os.environ["AZURE_STORAGE_ACCOUNT_URL"], 43 | credential=DefaultAzureCredential(), # type: ignore[arg-type] 44 | ).get_container_client(container=container) 45 | else: 46 | self.container_client = container_client 47 | 48 | self.tilelayout = tilelayout 49 | self.dry_run = dry_run 50 | self.cache_control = cache_control 51 | 52 | async def __contains__(self, tile: Tile) -> bool: 53 | """Return true if this store contains ``tile``.""" 54 | if not tile: 55 | return False 56 | key_name = self.tilelayout.filename(tile.tilecoord, tile.metadata) 57 | blob = self.container_client.get_blob_client(blob=key_name) 58 | return await blob.exists() 59 | 60 | async def delete_one(self, tile: Tile) -> Tile: 61 | """Delete a tile from the store.""" 62 | try: 63 | key_name = self.tilelayout.filename(tile.tilecoord, tile.metadata) 64 | if not self.dry_run: 65 | blob = self.container_client.get_blob_client(blob=key_name) 66 | if blob.exists(): 67 | blob.delete_blob() 68 | except Exception as exc: # pylint: disable=broad-except # noqa: BLE001 69 | _LOGGER.warning("Failed to delete tile %s", tile.tilecoord, exc_info=exc) 70 | tile.error = exc 71 | return tile 72 | 73 | async def get_one(self, tile: Tile) -> Tile | None: 74 | """Get a tile from the store.""" 75 | key_name = self.tilelayout.filename(tile.tilecoord, tile.metadata) 76 | try: 77 | blob = self.container_client.get_blob_client(blob=key_name) 78 | if not blob.exists(): 79 | return None 80 | data = (await blob.download_blob()).readall() 81 | assert isinstance(data, bytes) or data is None 82 | tile.data = data 83 | properties = await blob.get_blob_properties() 84 | tile.content_encoding = properties.content_settings.content_encoding 85 | tile.content_type = properties.content_settings.content_type 86 | except Exception as exc: # pylint: disable=broad-except # noqa: BLE001 87 | _LOGGER.warning("Failed to get tile %s", tile.tilecoord, exc_info=exc) 88 | tile.error = exc 89 | return tile 90 | 91 | async def get(self, tiles: AsyncIterator[Tile]) -> AsyncIterator[Tile | None]: 92 | """Get tiles from the store.""" 93 | async for tile in tiles: 94 | yield await self.get_one(tile) 95 | 96 | async def list(self) -> AsyncIterator[Tile]: 97 | """List all the tiles in the store.""" 98 | prefix = getattr(self.tilelayout, "prefix", "") 99 | 100 | async for blob in self.container_client.list_blobs(name_starts_with=prefix): 101 | try: 102 | assert isinstance(blob.name, str) 103 | tilecoord = self.tilelayout.tilecoord(blob.name) 104 | except ValueError: 105 | continue 106 | blob_data = self.container_client.get_blob_client(blob=blob.name) 107 | yield Tile(tilecoord, data=await (await blob_data.download_blob()).readall()) 108 | 109 | async def put_one(self, tile: Tile) -> Tile: 110 | """Store ``tile`` in the store.""" 111 | assert tile.data is not None 112 | key_name = self.tilelayout.filename(tile.tilecoord, tile.metadata) 113 | if not self.dry_run: 114 | try: 115 | blob = self.container_client.get_blob_client(blob=key_name) 116 | await blob.upload_blob( 117 | tile.data, 118 | overwrite=True, 119 | content_settings=ContentSettings( 120 | content_type=tile.content_type, 121 | content_encoding=tile.content_encoding, 122 | cache_control=self.cache_control, 123 | ), 124 | ) 125 | except Exception as exc: # pylint: disable=broad-except # noqa: BLE001 126 | _LOGGER.warning("Failed to put tile %s", tile.tilecoord, exc_info=exc) 127 | tile.error = exc 128 | 129 | return tile 130 | -------------------------------------------------------------------------------- /tilecloud_chain/store/mapnik_.py: -------------------------------------------------------------------------------- 1 | """MapnikTileStore with drop action if the generated tile is empty.""" 2 | 3 | import logging 4 | from collections.abc import AsyncGenerator, Callable 5 | from json import dumps 6 | from typing import Any 7 | 8 | from tilecloud import Tile, TileCoord, TileGrid 9 | 10 | from tilecloud_chain.store import AsyncTileStore 11 | 12 | try: 13 | import mapnik2 as mapnik 14 | except ImportError: 15 | import mapnik 16 | 17 | 18 | _LOGGER = logging.getLogger(__name__) 19 | 20 | 21 | class MapnikTileStore(AsyncTileStore): 22 | """ 23 | Tile store that renders tiles with Mapnik. 24 | 25 | requires mapnik2: http://pypi.python.org/pypi/mapnik2 26 | """ 27 | 28 | def __init__( 29 | self, 30 | tilegrid: TileGrid, 31 | mapfile: str, 32 | data_buffer: int = 128, 33 | image_buffer: int = 0, 34 | output_format: str = "png256", 35 | resolution: int = 2, 36 | layers_fields: dict[str, list[str]] | None = None, 37 | drop_empty_utfgrid: bool = False, 38 | proj4_literal: str | None = None, 39 | **kwargs: Any, 40 | ) -> None: 41 | """ 42 | Construct a MapnikTileStore. 43 | 44 | tilegrid: the tilegrid. 45 | mapfile: the file used to render the tiles. 46 | buffer_size: the image buffer size default is 128. 47 | output_format: the output format, 48 | possible values 'jpeg', 'png', 'png256', 'grid', 49 | default is 'png256' 50 | layers_fields: the layers and fields used in the grid generation, 51 | example: { 'my_layer': ['my_first_field', 'my_segonf_field']}, 52 | default is {}. 53 | **kwargs: for extended class. 54 | """ 55 | if layers_fields is None: 56 | layers_fields = {} 57 | 58 | AsyncTileStore.__init__(self, **kwargs) 59 | self.tilegrid = tilegrid 60 | self.buffer = image_buffer 61 | self.output_format = output_format 62 | self.resolution = resolution 63 | self.layers_fields = layers_fields 64 | self.drop_empty_utfgrid = drop_empty_utfgrid 65 | 66 | self.mapnik = mapnik.Map(tilegrid.tile_size, tilegrid.tile_size) 67 | mapnik.load_map(self.mapnik, mapfile, True) # noqa: FBT003 68 | self.mapnik.buffer_size = data_buffer 69 | if proj4_literal is not None: 70 | self.mapnik.srs = proj4_literal 71 | 72 | async def get_one(self, tile: Tile) -> Tile | None: 73 | """See in superclass.""" 74 | bbox = self.tilegrid.extent(tile.tilecoord, self.buffer) 75 | bbox2d = mapnik.Box2d(bbox[0], bbox[1], bbox[2], bbox[3]) 76 | 77 | size = tile.tilecoord.n * self.tilegrid.tile_size + 2 * self.buffer 78 | self.mapnik.resize(size, size) 79 | self.mapnik.zoom_to_box(bbox2d) 80 | 81 | if self.output_format == "grid": 82 | grid = mapnik.Grid(self.tilegrid.tile_size, self.tilegrid.tile_size) 83 | for number, layer in enumerate(self.mapnik.layers): 84 | if layer.name in self.layers_fields: 85 | mapnik.render_layer( 86 | self.mapnik, 87 | grid, 88 | layer=number, 89 | fields=self.layers_fields[layer.name], 90 | ) 91 | 92 | encode = grid.encode("utf", resolution=self.resolution) 93 | if self.drop_empty_utfgrid and len(encode["data"].keys()) == 0: 94 | return None 95 | tile.data = dumps(encode).encode() 96 | else: 97 | # Render image with default Agg renderer 98 | image = mapnik.Image(size, size) 99 | mapnik.render(self.mapnik, image) 100 | tile.data = image.tostring(self.output_format) 101 | 102 | return tile 103 | 104 | async def put_one(self, tile: Tile) -> Tile: 105 | """See in superclass.""" 106 | raise NotImplementedError 107 | 108 | async def delete_one(self, tile: Tile) -> Tile: 109 | """See in superclass.""" 110 | raise NotImplementedError 111 | 112 | async def __contains__(self, tile: Tile) -> bool: 113 | """See in superclass.""" 114 | raise NotImplementedError 115 | 116 | async def list(self) -> AsyncGenerator[Tile]: 117 | """See in superclass.""" 118 | raise NotImplementedError 119 | yield Tile(TileCoord(0, 0, 0)) # pylint: disable=unreachable 120 | 121 | 122 | class MapnikDropActionTileStore(MapnikTileStore): 123 | """MapnikTileStore with drop action if the generated tile is empty.""" 124 | 125 | def __init__( 126 | self, 127 | store: AsyncTileStore | None = None, 128 | queue_store: AsyncTileStore | None = None, 129 | count: list[Callable[[Tile | None], Any]] | None = None, 130 | **kwargs: Any, 131 | ) -> None: 132 | """Initialize.""" 133 | self.store = store 134 | self.queue_store = queue_store 135 | self.count = count or [] 136 | MapnikTileStore.__init__(self, **kwargs) 137 | 138 | async def get_one(self, tile: Tile) -> Tile | None: 139 | """See in superclass.""" 140 | result = await MapnikTileStore.get_one(self, tile) 141 | if result is None: 142 | if self.store is not None: 143 | if tile.tilecoord.n != 1: 144 | for tilecoord in tile.tilecoord: 145 | await self.store.delete_one(Tile(tilecoord)) 146 | else: 147 | await self.store.delete_one(tile) 148 | _LOGGER.info("The tile %s %s is dropped", tile.tilecoord, tile.formated_metadata) 149 | if hasattr(tile, "metatile"): 150 | metatile: Tile = tile.metatile 151 | metatile.elapsed_togenerate -= 1 # type: ignore[attr-defined] 152 | if metatile.elapsed_togenerate == 0 and self.queue_store is not None: # type: ignore[attr-defined] 153 | await self.queue_store.delete_one(metatile) 154 | elif self.queue_store is not None: 155 | await self.queue_store.delete_one(tile) 156 | 157 | for count in self.count: 158 | count(None) 159 | return result 160 | 161 | async def __contains__(self, tile: Tile) -> bool: 162 | """See in superclass.""" 163 | raise NotImplementedError 164 | 165 | async def put_one(self, tile: Tile) -> Tile: 166 | """See in superclass.""" 167 | raise NotImplementedError 168 | 169 | async def delete_one(self, tile: Tile) -> Tile: 170 | """See in superclass.""" 171 | raise NotImplementedError 172 | 173 | async def list(self) -> AsyncGenerator[Tile]: 174 | """See in superclass.""" 175 | raise NotImplementedError 176 | yield Tile(TileCoord(0, 0, 0)) # pylint: disable=unreachable 177 | -------------------------------------------------------------------------------- /tilecloud_chain/store/url.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import json 3 | import logging 4 | import os 5 | import pkgutil 6 | import urllib.parse 7 | from collections.abc import AsyncGenerator, Iterable 8 | from pathlib import Path 9 | from typing import Any, cast 10 | 11 | import aiohttp 12 | import jsonschema_validator 13 | from ruamel.yaml import YAML 14 | from tilecloud import BoundingPyramid, Tile, TileCoord, TileLayout 15 | 16 | from tilecloud_chain import host_limit 17 | from tilecloud_chain.store import AsyncTileStore 18 | 19 | _LOGGER = logging.getLogger(__name__) 20 | 21 | 22 | class URLTileStore(AsyncTileStore): 23 | """A tile store that reads and writes tiles from a formatted URL.""" 24 | 25 | def __init__( 26 | self, 27 | tile_layouts: Iterable[TileLayout], 28 | headers: Any | None = None, 29 | allows_no_contenttype: bool = False, 30 | bounding_pyramid: BoundingPyramid | None = None, 31 | ) -> None: 32 | self._allows_no_contenttype = allows_no_contenttype 33 | self._tile_layouts = tuple(tile_layouts) 34 | self._bounding_pyramid = bounding_pyramid 35 | self._session = aiohttp.ClientSession() 36 | self._hosts_semaphore: dict[str, asyncio.Semaphore] = {} 37 | self._hosts_limit: host_limit.HostLimit = {} 38 | if headers is not None: 39 | self._session.headers.update(headers) 40 | host_limit_path = Path( 41 | os.environ.get( 42 | "TILEGENERATION_HOSTS_LIMIT", 43 | "/etc/tilegeneration/hosts_limit.yaml", 44 | ), 45 | ) 46 | if host_limit_path.exists(): 47 | yaml = YAML(typ="safe") 48 | with host_limit_path.open(encoding="utf-8") as f: 49 | self._hosts_limit = yaml.load(f) 50 | 51 | schema_data = pkgutil.get_data("tilecloud_chain", "host-limit-schema.json") 52 | assert schema_data 53 | errors, _ = jsonschema_validator.validate( 54 | str(host_limit_path), 55 | cast("dict[str, Any]", self._hosts_limit), 56 | json.loads(schema_data), 57 | ) 58 | 59 | if errors: 60 | _LOGGER.error("The host limit file is invalid, ignoring:\n%s", "\n".join(errors)) 61 | self._hosts_limit = {} 62 | 63 | async def get_one(self, tile: Tile) -> Tile | None: 64 | """See in superclass.""" 65 | if tile is None: 66 | return None 67 | if self._bounding_pyramid is not None and tile.tilecoord not in self._bounding_pyramid: 68 | return None 69 | tilelayout = self._tile_layouts[hash(tile.tilecoord) % len(self._tile_layouts)] 70 | try: 71 | url = tilelayout.filename(tile.tilecoord, tile.metadata) 72 | except Exception as exception: # pylint: disable=broad-except # noqa: BLE001 73 | _LOGGER.warning("Error while getting tile %s", tile, exc_info=True) 74 | tile.error = exception 75 | return tile 76 | 77 | url_split = urllib.parse.urlparse(url) 78 | assert url_split.hostname is not None 79 | if url_split.hostname in self._hosts_semaphore: 80 | semaphore = self._hosts_semaphore[url_split.hostname] 81 | else: 82 | limit = ( 83 | self._hosts_limit.get("hosts", {}) 84 | .get(url_split.hostname, {}) 85 | .get( 86 | "concurrent", 87 | self._hosts_limit.get("default", {}).get( 88 | "concurrent", 89 | host_limit.DEFAULT_CONCURRENT_LIMIT_DEFAULT, 90 | ), 91 | ) 92 | ) 93 | semaphore = asyncio.Semaphore(limit) 94 | self._hosts_semaphore[url_split.hostname] = semaphore 95 | 96 | async with semaphore: 97 | _LOGGER.info("GET %s", url) 98 | try: 99 | async with self._session.get(url) as response: 100 | if response.status in (404, 204): 101 | _LOGGER.debug("Got empty tile from %s: %s", url, response.status) 102 | return None 103 | tile.content_encoding = response.headers.get("Content-Encoding") 104 | tile.content_type = response.headers.get("Content-Type") 105 | if response.status < 300: 106 | if response.status != 200: 107 | tile.error = ( 108 | f"URL: {url}\nUnsupported status code {response.status}: {response.reason}" 109 | ) 110 | if tile.content_type: 111 | if tile.content_type.startswith("image/"): 112 | tile.data = await response.read() 113 | else: 114 | tile.error = f"URL: {url}\n{await response.text()}" 115 | elif self._allows_no_contenttype: 116 | tile.data = await response.read() 117 | else: 118 | tile.error = f"URL: {url}\nThe Content-Type header is missing" 119 | 120 | else: 121 | tile.error = f"URL: {url}\n{response.status}: {response.reason}\n{response.text}" 122 | except aiohttp.ClientError as exception: 123 | _LOGGER.warning("Error while getting tile %s", tile, exc_info=True) 124 | tile.error = exception 125 | return tile 126 | 127 | async def __contains__(self, tile: Tile) -> bool: 128 | """See in superclass.""" 129 | raise NotImplementedError 130 | 131 | async def list(self) -> AsyncGenerator[Tile]: 132 | """See in superclass.""" 133 | raise NotImplementedError 134 | yield Tile(TileCoord(0, 0, 0)) # pylint: disable=unreachable 135 | 136 | async def put_one(self, tile: Tile) -> Tile: 137 | """See in superclass.""" 138 | raise NotImplementedError 139 | 140 | async def delete_one(self, tile: Tile) -> Tile: 141 | """See in superclass.""" 142 | raise NotImplementedError 143 | -------------------------------------------------------------------------------- /tilecloud_chain/templates/openlayers.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 9 | 10 | TileCloud-chain test page 11 | 18 | 25 | 26 | 33 | 40 | 58 | 59 | 60 |
61 | 67 | 73 | 74 | 80 | 81 | 144 | 145 | 146 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/apache.conf: -------------------------------------------------------------------------------- 1 | ScriptAlias /mapserv /usr/lib/cgi-bin/mapserv 2 | 3 | SetHandler fcgid-script 4 | SetEnv MS_MAPFILE /home/travis/build/camptocamp/tilecloud-chain/tilecloud_chain/tests/mapfile/mapfile.map 5 | Require all granted 6 | 7 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/create_test_data.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | createdb -E UTF8 -T template0 tests-deploy 4 | psql -q -d tests-deploy -c "CREATE TABLE test (name varchar(10));" 5 | psql -q -d tests-deploy -c "INSERT INTO test VALUES ('referance');" 6 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/index.expected.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/camptocamp/tilecloud-chain/2f84239147eb4025af917a862d8b3cb38f4a88e1/tilecloud_chain/tests/index.expected.png -------------------------------------------------------------------------------- /tilecloud_chain/tests/mapfile/circle.svg: -------------------------------------------------------------------------------- 1 | 2 | 6 | 11 | 12 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/mapfile/mapfile.map: -------------------------------------------------------------------------------- 1 | MAP 2 | NAME "Tests" 3 | STATUS ON 4 | EXTENT 420000 30000 900000 350000 5 | MAXSIZE 2500 6 | 7 | WEB 8 | TEMPLATE dummyTemplateForWmsGetFeatureInfo 9 | METADATA 10 | "ows_title" "Tests" 11 | "ows_encoding" "UTF-8" 12 | "wms_enable_request" "*" 13 | END 14 | END 15 | 16 | PROJECTION 17 | "init=epsg:21781" 18 | END 19 | 20 | SYMBOL 21 | NAME "circle" 22 | TYPE ellipse 23 | POINTS 24 | 1 1 25 | END 26 | FILLED true 27 | END 28 | 29 | LAYER 30 | NAME "point" 31 | TYPE POINT 32 | CONNECTIONTYPE postgis 33 | CONNECTION "user=postgresql password=postgresql dbname=tests host=db" 34 | DATA "the_geom FROM tests.point" 35 | TEMPLATE fooOnlyForWMSGetFeatureInfo # For GetFeatureInfo 36 | STATUS ON 37 | METADATA 38 | "gml_include_items" "all" # For GetFeatureInfo 39 | "ows_geom_type" "point" # For returning geometries in GetFeatureInfo 40 | "ows_geometries" "the_geom" # For returning geometries in GetFeatureInfo 41 | END 42 | CLASS 43 | NAME "Point" 44 | COLOR 255 0 0 45 | SIZE 10 46 | SYMBOL "circle" 47 | END 48 | END 49 | 50 | LAYER 51 | NAME "point_multi" 52 | TYPE POINT 53 | CONNECTIONTYPE postgis 54 | CONNECTION "user=postgresql password=postgresql dbname=tests host=db" 55 | DATA "the_geom FROM (SELECT * FROM tests.point WHERE name='%POINT_NAME%') AS foo USING unique gid" 56 | TEMPLATE fooOnlyForWMSGetFeatureInfo # For GetFeatureInfo 57 | STATUS ON 58 | METADATA 59 | "gml_include_items" "all" # For GetFeatureInfo 60 | "ows_geom_type" "point" # For returning geometries in GetFeatureInfo 61 | "ows_geometries" "the_geom" # For returning geometries in GetFeatureInfo 62 | END 63 | VALIDATION 64 | "POINT_NAME" "[a-z0-9]+" 65 | "default_POINT_NAME" "point1" 66 | END 67 | CLASS 68 | NAME "Point" 69 | COLOR 255 0 0 70 | SIZE 10 71 | SYMBOL "circle" 72 | END 73 | END 74 | 75 | LAYER 76 | NAME "line" 77 | TYPE LINE 78 | CONNECTIONTYPE postgis 79 | CONNECTION "user=postgresql password=postgresql dbname=tests host=db" 80 | DATA "the_geom FROM tests.line" 81 | STATUS ON 82 | CLASS 83 | NAME "Line 1" 84 | STYLE 85 | COLOR 0 0 255 86 | WIDTH 5 87 | MINSCALEDENOM 100000 88 | END 89 | END 90 | CLASS 91 | NAME "Line 2" 92 | STYLE 93 | COLOR 0 0 255 94 | WIDTH 5 95 | MAXSCALEDENOM 100000 96 | END 97 | END 98 | END 99 | 100 | LAYER 101 | NAME "polygon" 102 | TYPE POLYGON 103 | CONNECTIONTYPE postgis 104 | CONNECTION "user=postgresql password=postgresql dbname=tests host=db" 105 | DATA "the_geom FROM tests.polygon" 106 | STATUS ON 107 | CLASS 108 | NAME "Polygon" 109 | STYLE 110 | OUTLINECOLOR 0 255 0 111 | COLOR 255 255 0 112 | END 113 | END 114 | END 115 | END 116 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/mapfile/test.mapnik: -------------------------------------------------------------------------------- 1 | 2 | 3 | 8 | 13 | 18 | 19 | polygon 20 | 21 | tests.polygon 22 | the_geom 23 | 21781 24 | postgis 25 | tests 26 | db 27 | 5432 28 | postgresql 29 | postgresql 30 | false 31 | 420000, 30000, 900000, 350000 32 | 33 | 34 | 35 | line 36 | 37 | tests.line 38 | the_geom 39 | 21781 40 | postgis 41 | tests 42 | db 43 | 5432 44 | postgresql 45 | postgresql 46 | false 47 | 420000, 30000, 900000, 350000 48 | 49 | 50 | 51 | point 52 | 53 | tests.point 54 | the_geom 55 | 21781 56 | postgis 57 | tests 58 | db 59 | 5432 60 | postgresql 61 | postgresql 62 | false 63 | 420000, 30000, 900000, 350000 64 | 65 | 66 | 67 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/not-login.expected.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/camptocamp/tilecloud-chain/2f84239147eb4025af917a862d8b3cb38f4a88e1/tilecloud_chain/tests/not-login.expected.png -------------------------------------------------------------------------------- /tilecloud_chain/tests/test.expected.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/camptocamp/tilecloud-chain/2f84239147eb4025af917a862d8b3cb38f4a88e1/tilecloud_chain/tests/test.expected.png -------------------------------------------------------------------------------- /tilecloud_chain/tests/test_config.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from testfixtures import LogCapture 4 | 5 | from tilecloud_chain import controller 6 | from tilecloud_chain.tests import CompareCase 7 | 8 | 9 | class TestConfig(CompareCase): 10 | def setUp(self) -> None: # noqa 11 | self.maxDiff = None 12 | 13 | @classmethod 14 | def setUpClass(cls): # noqa 15 | os.chdir(os.path.dirname(__file__)) 16 | 17 | @classmethod 18 | def tearDownClass(cls): # noqa 19 | os.chdir(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) 20 | 21 | def test_int_grid(self) -> None: 22 | with LogCapture("tilecloud_chain") as log_capture: 23 | self.run_cmd( 24 | cmd=".build/venv/bin/generate_controller -c tilegeneration/test-int-grid.yaml --dump-config", 25 | main_func=controller.main, 26 | ) 27 | log_capture.check() 28 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/test_copy.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | 4 | import requests 5 | 6 | from tilecloud_chain import copy_ 7 | from tilecloud_chain.tests import CompareCase 8 | 9 | 10 | class TestGenerate(CompareCase): 11 | def setUp(self) -> None: # noqa 12 | self.maxDiff = None 13 | 14 | @classmethod 15 | def setUpClass(cls): # noqa 16 | os.chdir(os.path.dirname(__file__)) 17 | if os.path.exists("/tmp/tiles"): 18 | shutil.rmtree("/tmp/tiles") 19 | os.makedirs("/tmp/tiles/src/1.0.0/point_hash/default/21781/0/0/") 20 | 21 | @classmethod 22 | def tearDownClass(cls): # noqa 23 | os.chdir(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) 24 | if os.path.exists("/tmp/tiles"): 25 | shutil.rmtree("/tmp/tiles") 26 | 27 | def test_copy(self) -> None: 28 | with open("/tmp/tiles/src/1.0.0/point_hash/default/21781/0/0/0.png", "w") as f: 29 | f.write("test image") 30 | 31 | for d in ("-d", "-q", "-v"): 32 | self.assert_cmd_equals( 33 | cmd=f".build/venv/bin/generate_copy {d} -c tilegeneration/test-copy.yaml src dst", 34 | main_func=copy_.main, 35 | regex=True, 36 | expected=( 37 | """The tile copy of layer 'point_hash' is finish 38 | Nb copy tiles: 1 39 | Nb errored tiles: 0 40 | Nb dropped tiles: 0 41 | Total time: 0:00:[0-9][0-9] 42 | Total size: 10 o 43 | Time per tile: [0-9]+ ms 44 | Size per tile: 10(.0)? o 45 | 46 | """ 47 | if d != "-q" 48 | else "" 49 | ), 50 | empty_err=True, 51 | ) 52 | with open("/tmp/tiles/dst/1.0.0/point_hash/default/21781/0/0/0.png") as f: 53 | assert f.read() == "test image" 54 | 55 | def test_process(self) -> None: 56 | for d in ("-vd", "-q", "-v", ""): 57 | response = requests.get( 58 | "http://mapserver:8080/?STYLES=default&SERVICE=WMS&FORMAT=\ 59 | image%2Fpng&REQUEST=GetMap&HEIGHT=256&WIDTH=256&VERSION=1.1.1&BBOX=\ 60 | %28560800.0%2C+158000.0%2C+573600.0%2C+170800.0%29&LAYERS=point&SRS=EPSG%3A21781" 61 | ) 62 | response.raise_for_status() 63 | with open("/tmp/tiles/src/1.0.0/point_hash/default/21781/0/0/0.png", "wb") as out: 64 | out.write(response.content) 65 | statinfo = os.stat( 66 | "/tmp/tiles/src/1.0.0/point_hash/default/21781/0/0/0.png", 67 | ) 68 | assert statinfo.st_size == 755 69 | 70 | self.assert_cmd_equals( 71 | cmd=f".build/venv/bin/generate_process {d} -c " 72 | "tilegeneration/test-copy.yaml --cache src optipng", 73 | main_func=copy_.process, 74 | regex=True, 75 | expected=( 76 | """The tile process of layer 'point_hash' is finish 77 | Nb process tiles: 1 78 | Nb errored tiles: 0 79 | Nb dropped tiles: 0 80 | Total time: 0:00:[0-9][0-9] 81 | Total size: 103 o 82 | Time per tile: [0-9]+ ms 83 | Size per tile: 103(.0)? o 84 | 85 | """ 86 | if d != "-q" 87 | else "" 88 | ), 89 | empty_err=True, 90 | ) 91 | statinfo = os.stat( 92 | "/tmp/tiles/src/1.0.0/point_hash/default/21781/0/0/0.png", 93 | ) 94 | assert statinfo.st_size == 103 95 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/test_expiretiles.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import psycopg2 4 | import pytest 5 | from testfixtures import LogCapture 6 | 7 | from tilecloud_chain import expiretiles 8 | from tilecloud_chain.tests import CompareCase, MatchRegex 9 | 10 | 11 | class TestExpireTiles(CompareCase): 12 | def setUp(self) -> None: # noqa 13 | self.maxDiff = None 14 | 15 | @classmethod 16 | def setUpClass(cls): # noqa 17 | with open("/tmp/expired", "w") as f: 18 | f.write("18/135900/92720\n") 19 | f.write("18/135900/92721\n") 20 | f.write("18/135900/92722\n") 21 | f.write("18/135901/92721\n") 22 | f.write("18/135901/92722\n") 23 | f.write("18/135902/92722\n") 24 | 25 | with open("/tmp/expired-empty", "w"): 26 | pass 27 | 28 | @classmethod 29 | def tearDownClass(cls): # noqa 30 | os.remove("/tmp/expired") 31 | os.remove("/tmp/expired-empty") 32 | 33 | def test_expire_tiles( 34 | self, 35 | ) -> None: 36 | with LogCapture("tilecloud_chain", level=30) as log_capture: 37 | geom_re = MatchRegex(r"MULTIPOLYGON\(\(\(([0-9\. ,]+)\)\)\)") 38 | geom_coords = [ 39 | pytest.approx([538274.006497397, 151463.940954133], abs=1e-6), 40 | pytest.approx([538272.927475664, 151358.882137848], abs=1e-6), 41 | pytest.approx([538167.532395446, 151359.965536437], abs=1e-6), 42 | pytest.approx([538062.137334338, 151361.050781072], abs=1e-6), 43 | pytest.approx([537956.742292377, 151362.137871759], abs=1e-6), 44 | pytest.approx([537957.826834589, 151467.19663084], abs=1e-6), 45 | pytest.approx([537958.911357866, 151572.253567259], abs=1e-6), 46 | pytest.approx([537959.995862209, 151677.308681051], abs=1e-6), 47 | pytest.approx([538065.385383791, 151676.221647663], abs=1e-6), 48 | pytest.approx([538064.302719542, 151571.166514773], abs=1e-6), 49 | pytest.approx([538169.694100363, 151570.08130827], abs=1e-6), 50 | pytest.approx([538168.61325734, 151465.024333685], abs=1e-6), 51 | pytest.approx([538274.006497397, 151463.940954133], abs=1e-6), 52 | ] 53 | 54 | self.assert_cmd_equals( 55 | cmd=[ 56 | ".build/venv/bin/import_expiretiles", 57 | "--create", 58 | "--delete", 59 | "--srid", 60 | "21781", 61 | "/tmp/expired", 62 | "user=postgresql password=postgresql dbname=tests host=db", 63 | "expired", 64 | "the_geom", 65 | ], 66 | main_func=expiretiles.main, 67 | expected="""Import successful 68 | """, 69 | ) 70 | connection = psycopg2.connect("user=postgresql password=postgresql dbname=tests host=db") 71 | cursor = connection.cursor() 72 | cursor.execute("SELECT ST_AsText(the_geom) FROM expired") 73 | geoms = [str(r[0]) for r in cursor.fetchall()] 74 | assert [geom_re] == geoms 75 | 76 | def parse_coord(coord: str) -> tuple[float, float]: 77 | coord_split = coord.split(" ") 78 | return [float(c) for c in coord_split] 79 | 80 | assert [parse_coord(e) for e in geom_re.match(geoms[0]).group(1).split(",")] == geom_coords 81 | 82 | self.assert_cmd_equals( 83 | cmd=[ 84 | ".build/venv/bin/import_expiretiles", 85 | "--create", 86 | "--delete", 87 | "--srid", 88 | "21781", 89 | "/tmp/expired", 90 | "user=postgresql password=postgresql dbname=tests host=db", 91 | "expired", 92 | "the_geom", 93 | ], 94 | main_func=expiretiles.main, 95 | expected="""Import successful 96 | """, 97 | ) 98 | connection = psycopg2.connect("user=postgresql password=postgresql dbname=tests host=db") 99 | cursor = connection.cursor() 100 | cursor.execute("SELECT ST_AsText(the_geom) FROM expired") 101 | geoms = [str(r[0]) for r in cursor.fetchall()] 102 | assert [geom_re] == geoms 103 | assert [parse_coord(e) for e in geom_re.match(geoms[0]).group(1).split(",")] == geom_coords 104 | 105 | self.assert_cmd_equals( 106 | cmd=[ 107 | ".build/venv/bin/import_expiretiles", 108 | "--simplify", 109 | "1000", 110 | "--create", 111 | "--delete", 112 | "/tmp/expired", 113 | "user=postgresql password=postgresql dbname=tests host=db", 114 | "expired2", 115 | ], 116 | main_func=expiretiles.main, 117 | expected="""Import successful 118 | """, 119 | ) 120 | connection = psycopg2.connect("user=postgresql password=postgresql dbname=tests host=db") 121 | cursor = connection.cursor() 122 | cursor.execute("SELECT ST_AsText(geom) FROM expired2") 123 | geoms = [str(r[0]) for r in cursor.fetchall()] 124 | geom_coords = [ 125 | pytest.approx([738534.567188568, 5862720.06865692], abs=1e-6), 126 | pytest.approx([738534.567188568, 5862567.19460037], abs=1e-6), 127 | pytest.approx([738381.693132021, 5862567.19460037], abs=1e-6), 128 | pytest.approx([738228.819075469, 5862567.19460037], abs=1e-6), 129 | pytest.approx([738075.945018921, 5862567.19460037], abs=1e-6), 130 | pytest.approx([738075.945018921, 5862720.06865692], abs=1e-6), 131 | pytest.approx([738075.945018921, 5862872.94271347], abs=1e-6), 132 | pytest.approx([738075.945018921, 5863025.81677002], abs=1e-6), 133 | pytest.approx([738228.819075469, 5863025.81677002], abs=1e-6), 134 | pytest.approx([738228.819075469, 5862872.94271347], abs=1e-6), 135 | pytest.approx([738381.693132021, 5862872.94271347], abs=1e-6), 136 | pytest.approx([738381.693132021, 5862720.06865692], abs=1e-6), 137 | pytest.approx([738534.567188568, 5862720.06865692], abs=1e-6), 138 | ] 139 | assert [geom_re] == geoms 140 | assert [parse_coord(e) for e in geom_re.match(geoms[0]).group(1).split(",")] == geom_coords 141 | 142 | log_capture.check() 143 | 144 | def test_expire_tiles_empty(self) -> None: 145 | with LogCapture("tilecloud_chain", level=30): 146 | self.assert_cmd_equals( 147 | cmd=[ 148 | ".build/venv/bin/import_expiretiles", 149 | "--create", 150 | "--delete", 151 | "--srid", 152 | "21781", 153 | "/tmp/expired-empty", 154 | "user=postgresql password=postgresql dbname=tests host=db", 155 | "expired", 156 | "the_geom", 157 | ], 158 | main_func=expiretiles.main, 159 | expected="""No coords found 160 | """, 161 | ) 162 | connection = psycopg2.connect("user=postgresql password=postgresql dbname=tests host=db") 163 | cursor = connection.cursor() 164 | cursor.execute("SELECT the_geom FROM expired") 165 | geoms = cursor.fetchall() 166 | assert len(geoms) == 0 167 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/test_postgresql.py: -------------------------------------------------------------------------------- 1 | import os 2 | from datetime import datetime, timedelta 3 | from pathlib import Path 4 | 5 | import pytest 6 | import pytest_asyncio 7 | from sqlalchemy import and_ 8 | from sqlalchemy.engine import create_engine 9 | from sqlalchemy.orm import sessionmaker 10 | from tilecloud import Tile, TileCoord 11 | 12 | from tilecloud_chain import DatedConfig 13 | from tilecloud_chain.store.postgresql import ( 14 | _STATUS_CANCELLED, 15 | _STATUS_CREATED, 16 | _STATUS_DONE, 17 | _STATUS_ERROR, 18 | _STATUS_PENDING, 19 | _STATUS_STARTED, 20 | Job, 21 | PostgresqlTileStore, 22 | Queue, 23 | get_postgresql_queue_store, 24 | ) 25 | 26 | 27 | @pytest_asyncio.fixture 28 | async def tilestore() -> PostgresqlTileStore: 29 | return await get_postgresql_queue_store(DatedConfig({}, 0, "config.yaml")) 30 | 31 | 32 | @pytest.fixture 33 | def SessionMaker() -> sessionmaker: 34 | engine = create_engine(os.environ["TILECLOUD_CHAIN_SQLALCHEMY_URL"]) 35 | return sessionmaker(engine) # noqa 36 | 37 | 38 | @pytest_asyncio.fixture 39 | async def queue(SessionMaker: sessionmaker, tilestore: PostgresqlTileStore) -> tuple[int, int, int]: 40 | with SessionMaker() as session: 41 | for job in session.query(Job).filter(Job.name == "test").all(): 42 | session.delete(job) 43 | session.commit() 44 | await tilestore.create_job("test", "generate-tiles", "config.yaml") 45 | with SessionMaker() as session: 46 | job = session.query(Job).filter(Job.name == "test").one() 47 | job.status = _STATUS_STARTED 48 | job_id = job.id 49 | session.commit() 50 | 51 | await tilestore.put_one( 52 | Tile( 53 | TileCoord(0, 0, 0), 54 | metadata={ 55 | "job_id": job_id, 56 | }, 57 | ) 58 | ) 59 | await tilestore.put_one( 60 | Tile( 61 | TileCoord(1, 0, 0), 62 | metadata={ 63 | "job_id": job_id, 64 | }, 65 | ) 66 | ) 67 | 68 | with SessionMaker() as session: 69 | metatile_0_id = session.query(Queue.id).filter(and_(Queue.job_id == job_id, Queue.zoom == 0)).one()[0] 70 | metatile_1_id = session.query(Queue.id).filter(and_(Queue.job_id == job_id, Queue.zoom == 1)).one()[0] 71 | 72 | yield job_id, metatile_0_id, metatile_1_id 73 | 74 | with SessionMaker() as session: 75 | session.query(Queue).filter(Queue.job_id == job_id).delete() 76 | session.query(Job).filter(Job.id == job_id).delete() 77 | session.commit() 78 | 79 | 80 | @pytest.mark.asyncio 81 | async def test_retry(queue: tuple[int, int, int], SessionMaker: sessionmaker, tilestore: PostgresqlTileStore): 82 | job_id, _, _ = queue 83 | 84 | tile_1 = await anext(tilestore.list()) 85 | tile_2 = await anext(tilestore.list()) 86 | 87 | tile_1.error = "test error" 88 | 89 | await tilestore.delete_one(tile_1) 90 | await tilestore.delete_one(tile_2) 91 | 92 | with SessionMaker() as session: 93 | metatiles = session.query(Queue).filter(Queue.job_id == job_id).all() 94 | assert len(metatiles) == 1 95 | assert metatiles[0].error == "test error" 96 | 97 | await tilestore._maintenance() 98 | 99 | with SessionMaker() as session: 100 | job = session.query(Job).filter(Job.id == job_id).one() 101 | assert job.status == _STATUS_ERROR 102 | 103 | await tilestore.retry(job_id, Path("config.yaml")) 104 | 105 | with SessionMaker() as session: 106 | job = session.query(Job).filter(Job.id == job_id).one() 107 | assert job.status == _STATUS_CREATED 108 | metatiles = session.query(Queue).filter(Queue.job_id == job_id).all() 109 | assert len(metatiles) == 1 110 | assert metatiles[0].status == _STATUS_CREATED 111 | 112 | 113 | @pytest.mark.asyncio 114 | async def test_cancel( 115 | queue: tuple[int, int, int], SessionMaker: sessionmaker, tilestore: PostgresqlTileStore 116 | ): 117 | job_id, _, _ = queue 118 | 119 | tile_1 = await anext(tilestore.list()) 120 | 121 | await tilestore.delete_one(tile_1) 122 | 123 | with SessionMaker() as session: 124 | metatiles = session.query(Queue).filter(Queue.job_id == job_id).all() 125 | assert len(metatiles) == 1 126 | 127 | await tilestore.cancel(job_id, Path("config.yaml")) 128 | 129 | with SessionMaker() as session: 130 | metatiles = session.query(Queue).filter(Queue.job_id == job_id).all() 131 | assert len(metatiles) == 0 132 | job = session.query(Job).filter(Job.id == job_id).one() 133 | assert job.status == _STATUS_CANCELLED 134 | 135 | 136 | @pytest.mark.asyncio 137 | async def test_maintenance_status_done( 138 | queue: tuple[int, int, int], SessionMaker: sessionmaker, tilestore: PostgresqlTileStore 139 | ): 140 | job_id, _, _ = queue 141 | 142 | tile_1 = await anext(tilestore.list()) 143 | tile_2 = await anext(tilestore.list()) 144 | 145 | await tilestore.delete_one(tile_1) 146 | await tilestore.delete_one(tile_2) 147 | 148 | with SessionMaker() as session: 149 | metatiles = session.query(Queue).filter(Queue.job_id == job_id).all() 150 | assert len(metatiles) == 0 151 | 152 | await tilestore._maintenance() 153 | 154 | with SessionMaker() as session: 155 | job = session.query(Job).filter(Job.id == job_id).one() 156 | assert job.status == _STATUS_DONE 157 | 158 | 159 | @pytest.mark.asyncio 160 | async def test_maintenance_pending_tile( 161 | queue: tuple[int, int, int], SessionMaker: sessionmaker, tilestore: PostgresqlTileStore 162 | ): 163 | job_id, metatile_0_id, metatile_1_id = queue 164 | 165 | with SessionMaker() as session: 166 | metatile_0 = session.query(Queue).filter(Queue.id == metatile_0_id).one() 167 | metatile_0.status = _STATUS_PENDING 168 | metatile_0.started_at = datetime.now() - timedelta(hours=1) 169 | metatile_1 = session.query(Queue).filter(Queue.id == metatile_1_id).one() 170 | metatile_1.status = _STATUS_PENDING 171 | metatile_1.started_at = datetime.now() - timedelta(hours=1) 172 | session.commit() 173 | 174 | await tilestore._maintenance() 175 | with SessionMaker() as session: 176 | metatile_0 = session.query(Queue).filter(Queue.id == metatile_0_id).one() 177 | assert metatile_0.status == _STATUS_CREATED 178 | metatile_1 = session.query(Queue).filter(Queue.id == metatile_1_id).one() 179 | assert metatile_1.status == _STATUS_CREATED 180 | 181 | 182 | @pytest.mark.asyncio 183 | async def test_maintenance_pending_job( 184 | queue: tuple[int, int, int], SessionMaker: sessionmaker, tilestore: PostgresqlTileStore 185 | ): 186 | job_id, metatile_0_id, metatile_1_id = queue 187 | with SessionMaker() as session: 188 | job = session.query(Job).filter(Job.id == job_id).one() 189 | job.status = _STATUS_PENDING 190 | job.started_at = datetime.now() - timedelta(hours=1) 191 | 192 | await tilestore._maintenance() 193 | 194 | with SessionMaker() as session: 195 | job = session.query(Job).filter(Job.id == job_id).one() 196 | assert job.status == _STATUS_STARTED 197 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/test_ui.py: -------------------------------------------------------------------------------- 1 | import os 2 | import subprocess 3 | 4 | import pytest 5 | import skimage.io 6 | from c2cwsgiutils.acceptance.image import check_image 7 | 8 | REGENERATE = False 9 | 10 | 11 | def test_should_not_commit(): 12 | assert REGENERATE is False 13 | 14 | 15 | @pytest.mark.parametrize( 16 | ("url", "expected_file_name", "height", "width"), 17 | [ 18 | pytest.param("http://application:8080/admin/", "not-login", 250, 800, id="not-login"), 19 | pytest.param("http://application:8080/admin/test", "test", 800, 800, id="test-not-login"), 20 | pytest.param("http://app_test_user:8080/admin", "index", 1250, 1000, id="index"), 21 | pytest.param("http://app_test_user:8080/admin/test", "test", 800, 800, id="test"), 22 | ], 23 | ) 24 | def test_ui(url, expected_file_name, height, width): 25 | subprocess.run( 26 | [ 27 | "node", 28 | "screenshot.js", 29 | f"--url={url}", 30 | f"--width={width}", 31 | f"--height={height}", 32 | f"--output=/tmp/{expected_file_name}.png", 33 | ], 34 | check=True, 35 | ) 36 | check_image( 37 | "/results", 38 | skimage.io.imread(f"/tmp/{expected_file_name}.png")[:, :, :3], 39 | os.path.join(os.path.dirname(__file__), f"{expected_file_name}.expected.png"), 40 | generate_expected_image=REGENERATE, 41 | ) 42 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/deploy.cfg: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | project = test 3 | 4 | [main] 5 | hookdir = %(here)s/hooks 6 | 7 | [files] 8 | active = false 9 | 10 | [databases] 11 | active = true 12 | names = tests-deploy 13 | 14 | [code] 15 | active = false 16 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/hooks/post-create-database: -------------------------------------------------------------------------------- 1 | sudo -u postgresql dropdb tests-deploy 2 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/hooks/post-restore-database: -------------------------------------------------------------------------------- 1 | echo SUCCESS 2 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/test-apache-s3-tilesurl.yaml: -------------------------------------------------------------------------------- 1 | grids: 2 | swissgrid_5: 3 | resolutions: [100, 50, 20, 10, 5] 4 | bbox: [420000, 30000, 900000, 350000] 5 | tile_size: 256 6 | srs: EPSG:21781 7 | 8 | caches: 9 | s3-tilesurl: 10 | type: s3 11 | http_url: https://%(host)s/%(bucket)s/%(folder)s/ 12 | tiles_url: http://tiles.example.com/ 13 | host: s3-eu-west-1.amazonaws.com 14 | bucket: tiles 15 | folder: tiles 16 | 17 | layers: 18 | point: 19 | url: http://mapserver:8080/ 20 | layers: point 21 | min_resolution_seed: 10 22 | grid: swissgrid_5 23 | type: wms 24 | wmts_style: default 25 | mime_type: image/png 26 | extension: png 27 | 28 | generation: 29 | default_cache: s3-tilesurl 30 | default_layers: [point] 31 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/test-authorised.yaml: -------------------------------------------------------------------------------- 1 | grids: {} 2 | 3 | caches: {} 4 | 5 | layers: {} 6 | 7 | generation: 8 | authorised_user: www-data 9 | 10 | sns: 11 | topic: arn:aws:sns:eu-west-1:your-account-id:tilecloud 12 | region: eu-west-1 13 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/test-bsddb.yaml: -------------------------------------------------------------------------------- 1 | grids: 2 | swissgrid_5: 3 | resolutions: [100, 50, 20, 10, 5] 4 | bbox: [420000, 30000, 900000, 350000] 5 | tile_size: 256 6 | srs: EPSG:21781 7 | 8 | caches: 9 | bsddb: 10 | type: bsddb 11 | http_url: http://wmts1/tiles/ 12 | folder: /tmp/tiles/bsddb 13 | wmtscapabilities_file: 1.0.0/WMTSCapabilities.xml 14 | 15 | defaults: 16 | layer: &layer 17 | grid: swissgrid_5 18 | type: wms 19 | url: http://mapserver:8080/ 20 | wmts_style: default 21 | mime_type: image/png 22 | extension: png 23 | dimensions: 24 | - name: DATE 25 | default: '2012' 26 | generate: ['2012'] 27 | values: ['2005', '2010', '2012'] 28 | meta: true 29 | meta_size: 8 30 | meta_buffer: 128 31 | 32 | layers: 33 | point_hash: 34 | <<: *layer 35 | layers: point 36 | query_layers: point 37 | geoms: 38 | - sql: the_geom AS geom FROM tests.point 39 | connection: user=postgresql password=postgresql dbname=tests host=db 40 | min_resolution_seed: 10 41 | empty_metatile_detection: 42 | size: 20743 43 | hash: 01062bb3b25dcead792d7824f9a7045f0dd92992 44 | empty_tile_detection: 45 | size: 334 46 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 47 | 48 | generation: 49 | default_cache: bsddb 50 | maxconsecutive_errors: 2 51 | 52 | server: {} 53 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/test-capabilities.yaml: -------------------------------------------------------------------------------- 1 | grids: 2 | swissgrid: 3 | resolutions: [100, 10] 4 | bbox: [420000, 30000, 900000, 350000] 5 | tile_size: 256 6 | srs: EPSG:21781 7 | 8 | caches: 9 | local: 10 | type: filesystem 11 | http_url: http://wmts1/tiles/ 12 | folder: /tmp/tiles 13 | wmtscapabilities_file: 1.0.0/WMTSCapabilities.xml 14 | 15 | defaults: 16 | layer: &layer 17 | url: http://mapserver:8080/ 18 | layers: point 19 | grid: swissgrid 20 | type: wms 21 | wmts_style: default 22 | mime_type: image/png 23 | extension: png 24 | 25 | layers: 26 | no_dim: *layer 27 | one: 28 | <<: *layer 29 | dimensions: 30 | - name: DATE 31 | default: '2012' 32 | generate: ['2012'] 33 | values: ['2012'] 34 | two: 35 | <<: *layer 36 | dimensions: 37 | - name: DATE 38 | default: '2012' 39 | generate: ['2012'] 40 | values: ['2012'] 41 | - name: LEVEL 42 | default: '1' 43 | generate: ['1'] 44 | values: ['1', '2'] 45 | 46 | generation: 47 | default_cache: local 48 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/test-copy.yaml: -------------------------------------------------------------------------------- 1 | grids: 2 | '21781': 3 | resolutions: [1000] 4 | bbox: [420000, 30000, 900000, 350000] 5 | tile_size: 256 6 | srs: EPSG:21781 7 | 8 | caches: 9 | src: 10 | type: filesystem 11 | folder: /tmp/tiles/src 12 | dst: 13 | type: filesystem 14 | folder: /tmp/tiles/dst 15 | 16 | defaults: 17 | layer: &layer 18 | grid: '21781' 19 | type: wms 20 | url: http://mapserver:8080/ 21 | wmts_style: default 22 | mime_type: image/png 23 | extension: png 24 | meta: true 25 | meta_size: 8 26 | meta_buffer: 128 27 | empty_tile_detection: 28 | size: 334 29 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 30 | 31 | layers: 32 | point_hash: 33 | <<: *layer 34 | layers: point 35 | 36 | process: 37 | optipng: 38 | - cmd: optipng %(args)s -zc9 -zm8 -zs3 -f5 -out %(out)s %(in)s 39 | need_out: true 40 | arg: 41 | default: '-q' 42 | quiet: '-q' 43 | debug: '' 44 | verbose: '' 45 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/test-fix.yaml: -------------------------------------------------------------------------------- 1 | grids: 2 | swissgrid_5: 3 | resolutions: [100, 50, 20, 10, 5] 4 | bbox: [420000, 30000, 900000, 350000] 5 | tile_size: 256 6 | srs: EPSG:21781 7 | 8 | swissgrid_01: 9 | resolutions: [1, 0.2, 0.1] 10 | bbox: [420000, 30000, 900000, 350000] 11 | tile_size: 256 12 | srs: EPSG:21781 13 | matrix_identifier: resolution 14 | 15 | swissgrid_2_5: 16 | resolutions: [2.5] 17 | bbox: [420000, 30000, 900000, 350000] 18 | tile_size: 256 19 | srs: EPSG:21781 20 | matrix_identifier: resolution 21 | 22 | swissgrid_025: 23 | resolutions: [0.25] 24 | bbox: [420000, 30000, 900000, 350000] 25 | tile_size: 256 26 | srs: EPSG:21781 27 | matrix_identifier: resolution 28 | 29 | caches: 30 | local: 31 | type: filesystem 32 | http_url: http://wmts1/tiles/ 33 | folder: /tmp/tiles 34 | wmtscapabilities_file: 1.0.0/WMTSCapabilities.xml 35 | multi_host: 36 | type: filesystem 37 | http_url: http://%(host)s/tiles/ 38 | folder: /tmp/tiles 39 | hosts: 40 | - wmts1 41 | - wmts2 42 | - wmts3 43 | multi_url: 44 | type: filesystem 45 | http_urls: 46 | - http://wmts1/tiles/ 47 | - http://wmts2/tiles/ 48 | - http://wmts3/tiles/ 49 | folder: /tmp/tiles 50 | mbtiles: 51 | type: mbtiles 52 | http_url: http://wmts1/tiles/ 53 | folder: /tmp/tiles/mbtiles 54 | s3: 55 | type: s3 56 | http_url: https://%(host)s/%(bucket)s/%(folder)s/ 57 | host: s3-eu-west-1.amazonaws.com 58 | bucket: tiles 59 | folder: tiles 60 | cache_control: 'public, max-age=14400' 61 | 62 | defaults: 63 | layer: &layer 64 | grid: swissgrid_5 65 | type: wms 66 | wmts_style: default 67 | mime_type: image/png 68 | extension: png 69 | dimensions: 70 | - name: DATE 71 | default: '2012' 72 | generate: ['2012'] 73 | values: ['2005', '2010', '2012'] 74 | meta: true 75 | meta_size: 8 76 | meta_buffer: 128 77 | cost: 78 | # [ms] 79 | tileonly_generation_time: 60 80 | # [ms] 81 | tile_generation_time: 30 82 | # [ms] 83 | metatile_generation_time: 30 84 | # [ko] 85 | tile_size: 20 86 | 87 | layers: 88 | point: 89 | <<: *layer 90 | url: http://mapserver:8080/ 91 | layers: point 92 | geoms: 93 | - sql: the_geom AS geom FROM tests.point 94 | connection: user=postgresql password=postgresql dbname=tests host=db 95 | min_resolution_seed: 10 96 | point_px_buffer: 97 | <<: *layer 98 | url: http://mapserver:8080/ 99 | layers: point 100 | px_buffer: 100 101 | geoms: 102 | - sql: the_geom AS geom FROM tests.point 103 | connection: user=postgresql password=postgresql dbname=tests host=db 104 | empty_metatile_detection: 105 | size: 20743 106 | hash: 01062bb3b25dcead792d7824f9a7045f0dd92992 107 | empty_tile_detection: 108 | size: 334 109 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 110 | point_hash: 111 | <<: *layer 112 | url: http://mapserver:8080/ 113 | layers: point 114 | geoms: 115 | - sql: the_geom AS geom FROM tests.point 116 | connection: user=postgresql password=postgresql dbname=tests host=db 117 | min_resolution_seed: 10 118 | empty_metatile_detection: 119 | size: 20743 120 | hash: 01062bb3b25dcead792d7824f9a7045f0dd92992 121 | empty_tile_detection: 122 | size: 334 123 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 124 | point_hash_no_meta: 125 | <<: *layer 126 | url: http://mapserver:8080/ 127 | layers: point 128 | meta: false 129 | empty_tile_detection: 130 | size: 334 131 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 132 | line: 133 | <<: *layer 134 | url: http://mapserver:8080/ 135 | layers: line 136 | headers: 137 | Cache-Control: no-cache 138 | params: 139 | PARAM: value 140 | geoms: 141 | - sql: the_geom AS geom FROM tests.line 142 | connection: user=postgresql password=postgresql dbname=tests host=db 143 | empty_metatile_detection: 144 | size: 20743 145 | hash: 01062bb3b25dcead792d7824f9a7045f0dd92992 146 | empty_tile_detection: 147 | size: 334 148 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 149 | polygon: 150 | <<: *layer 151 | url: http://mapserver:8080/ 152 | layers: polygon 153 | meta: false 154 | geoms: 155 | - sql: the_geom AS geom FROM tests.polygon 156 | connection: user=postgresql password=postgresql dbname=tests host=db 157 | empty_metatile_detection: 158 | size: 20743 159 | hash: 01062bb3b25dcead792d7824f9a7045f0dd92992 160 | empty_tile_detection: 161 | size: 334 162 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 163 | polygon2: 164 | <<: *layer 165 | url: http://mapserver:8080/ 166 | layers: polygon 167 | grid: swissgrid_01 168 | geoms: 169 | - sql: the_geom AS geom FROM tests.polygon 170 | connection: user=postgresql password=postgresql dbname=tests host=db 171 | empty_metatile_detection: 172 | size: 20743 173 | hash: 01062bb3b25dcead792d7824f9a7045f0dd92992 174 | empty_tile_detection: 175 | size: 334 176 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 177 | all: 178 | <<: *layer 179 | url: http://mapserver:8080/ 180 | layers: point,line,polygon 181 | meta: false 182 | bbox: [550000.0, 170000.0, 560000.0, 180000.0] 183 | mapnik: 184 | <<: *layer 185 | type: mapnik 186 | mapfile: mapfile/test.mapnik 187 | meta: false 188 | data_buffer: 128 189 | output_format: png 190 | geoms: 191 | - sql: the_geom AS geom FROM tests.polygon 192 | connection: user=postgresql password=postgresql dbname=tests host=db 193 | mapnik_grid: 194 | <<: *layer 195 | type: mapnik 196 | mapfile: mapfile/test.mapnik 197 | meta: false 198 | data_buffer: 128 199 | output_format: grid 200 | mime_type: application/utfgrid 201 | extension: json 202 | resolution: 16 203 | geoms: 204 | - sql: the_geom AS geom FROM tests.polygon 205 | connection: user=postgresql password=postgresql dbname=tests host=db 206 | layers_fields: 207 | point: 208 | - name 209 | line: 210 | - name 211 | polygon: 212 | - name 213 | mapnik_grid_drop: 214 | <<: *layer 215 | type: mapnik 216 | mapfile: mapfile/test.mapnik 217 | meta: false 218 | meta_buffer: 0 219 | data_buffer: 128 220 | output_format: grid 221 | mime_type: application/utfgrid 222 | extension: json 223 | drop_empty_utfgrid: true 224 | resolution: 16 225 | geoms: 226 | - sql: the_geom AS geom FROM tests.polygon 227 | connection: user=postgresql password=postgresql dbname=tests host=db 228 | layers_fields: 229 | point: 230 | - name 231 | generation: 232 | default_cache: local 233 | default_layers: [line, polygon] 234 | maxconsecutive_errors: 2 235 | error_file: error.list 236 | 237 | openlayers: 238 | srs: EPSG:21781 239 | center_x: 600000 240 | center_y: 200000 241 | 242 | cost: 243 | # [nb/month] 244 | request_per_layers: 10000000 245 | s3: {} 246 | cloudfront: {} 247 | sqs: {} 248 | 249 | sqs: 250 | queue: sqs_point 251 | 252 | sns: 253 | topic: arn:aws:sns:eu-west-1:your-account-id:tilecloud 254 | region: eu-west-1 255 | 256 | metadata: 257 | title: Some title 258 | abstract: Some abstract 259 | keywords: 260 | - some 261 | - keywords 262 | fees: None 263 | access_constraints: None 264 | 265 | provider: 266 | name: The provider name 267 | url: The provider URL 268 | contact: 269 | name: The contact name 270 | position: The position name 271 | info: 272 | phone: 273 | voice: +41 11 222 33 44 274 | fax: +41 11 222 33 44 275 | address: 276 | delivery: Address delivery 277 | city: Berne 278 | area: BE 279 | postal_code: 3000 280 | country: Switzerland 281 | email: info@example.com 282 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/test-int-grid.yaml: -------------------------------------------------------------------------------- 1 | grids: 2 | '21781': 3 | resolutions: [100, 50, 20, 10, 5] 4 | bbox: [420000, 30000, 900000, 350000] 5 | tile_size: 256 6 | srs: EPSG:21781 7 | 8 | caches: 9 | bsddb: 10 | type: bsddb 11 | http_url: http://wmts1/tiles/ 12 | folder: /tmp/tiles/bsddb 13 | 14 | defaults: 15 | layer: &layer_defaults 16 | grid: '21781' 17 | type: wms 18 | url: http://mapserver:8080/ 19 | wmts_style: default 20 | mime_type: image/png 21 | extension: png 22 | dimensions: 23 | - name: DATE 24 | default: '2012' 25 | generate: ['2012'] 26 | values: ['2005', '2010', '2012'] 27 | meta: true 28 | meta_size: 8 29 | meta_buffer: 128 30 | 31 | layers: 32 | point_hash: 33 | <<: *layer_defaults 34 | layers: point 35 | query_layers: point 36 | geoms: 37 | - sql: the_geom AS geom FROM tests.point 38 | connection: user=postgresql password=postgresql dbname=tests host=db 39 | min_resolution_seed: 10 40 | empty_metatile_detection: 41 | size: 20743 42 | hash: 01062bb3b25dcead792d7824f9a7045f0dd92992 43 | empty_tile_detection: 44 | size: 334 45 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 46 | 47 | generation: 48 | default_cache: bsddb 49 | maxconsecutive_errors: 2 50 | 51 | server: {} 52 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/test-internal-mapcache.yaml: -------------------------------------------------------------------------------- 1 | grids: 2 | swissgrid_5: 3 | resolutions: [100, 50, 20, 10, 5] 4 | bbox: [420000, 30000, 900000, 350000] 5 | tile_size: 256 6 | srs: EPSG:21781 7 | 8 | caches: 9 | local: 10 | type: filesystem 11 | http_url: http://wmts1/tiles/ 12 | folder: /tmp/tiles 13 | wmtscapabilities_file: 1.0.0/WMTSCapabilities.xml 14 | 15 | defaults: 16 | layer: &layer 17 | grid: swissgrid_5 18 | type: wms 19 | url: http://mapserver:8080/ 20 | wmts_style: default 21 | mime_type: image/png 22 | extension: png 23 | dimensions: 24 | - name: DATE 25 | default: '2012' 26 | generate: ['2012'] 27 | values: ['2005', '2010', '2012'] 28 | meta: true 29 | meta_size: 8 30 | meta_buffer: 128 31 | cost: 32 | # [ms] 33 | tileonly_generation_time: 60 34 | # [ms] 35 | tile_generation_time: 30 36 | # [ms] 37 | metatile_generation_time: 30 38 | # [ko] 39 | tile_size: 20 40 | 41 | layers: 42 | point: 43 | <<: *layer 44 | layers: point 45 | geoms: 46 | - sql: the_geom AS geom FROM tests.point 47 | connection: user=postgresql password=postgresql dbname=tests host=db 48 | min_resolution_seed: 100 49 | 50 | generation: 51 | default_cache: local 52 | default_layers: [point] 53 | maxconsecutive_errors: 2 54 | 55 | openlayers: 56 | srs: EPSG:21781 57 | center_x: 600000 58 | center_y: 200000 59 | 60 | cost: 61 | # [nb/month] 62 | request_per_layers: 10000000 63 | 64 | mapcache: 65 | config_file: mapcache.xml 66 | memcache_host: memcached 67 | 68 | server: 69 | geoms_redirect: true 70 | mapcache_base: http://mapcache:8080/ 71 | 72 | redis: 73 | sentinels: 74 | - - redis_sentinel 75 | - 26379 76 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/test-legends.yaml: -------------------------------------------------------------------------------- 1 | grids: 2 | swissgrid: 3 | resolutions: [100, 50, 20, 10, 5] 4 | bbox: [420000, 30000, 900000, 350000] 5 | tile_size: 256 6 | srs: EPSG:21781 7 | 8 | caches: 9 | local: 10 | type: filesystem 11 | http_url: http://wmts1/tiles/ 12 | folder: /tmp/tiles 13 | wmtscapabilities_file: 1.0.0/WMTSCapabilities.xml 14 | 15 | defaults: 16 | layer: &layer 17 | grid: swissgrid 18 | type: wms 19 | legend_mime: image/png 20 | legend_extension: png 21 | wmts_style: default 22 | mime_type: image/png 23 | extension: png 24 | dimensions: 25 | - name: DATE 26 | default: '2012' 27 | generate: ['2012'] 28 | values: ['2005', '2010', '2012'] 29 | meta: true 30 | meta_size: 8 31 | meta_buffer: 128 32 | cost: 33 | # [ms] 34 | tileonly_generation_time: 60 35 | # [ms] 36 | tile_generation_time: 30 37 | # [ms] 38 | metatile_generation_time: 30 39 | # [ko] 40 | tile_size: 20 41 | 42 | layers: 43 | point: 44 | <<: *layer 45 | url: http://mapserver:8080/ 46 | layers: point 47 | geoms: 48 | - sql: the_geom AS geom FROM tests.point 49 | connection: user=postgresql password=postgresql dbname=tests host=db 50 | min_resolution_seed: 10 51 | line: 52 | <<: *layer 53 | url: http://mapserver:8080/ 54 | layers: line 55 | headers: 56 | Cache-Control: no-cache 57 | params: 58 | PARAM: value 59 | geoms: 60 | - sql: the_geom AS geom FROM tests.line 61 | connection: user=postgresql password=postgresql dbname=tests host=db 62 | empty_metatile_detection: 63 | size: 20743 64 | hash: 01062bb3b25dcead792d7824f9a7045f0dd92992 65 | empty_tile_detection: 66 | size: 334 67 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 68 | polygon: 69 | <<: *layer 70 | url: http://mapserver:8080/ 71 | layers: polygon 72 | meta: false 73 | geoms: 74 | - sql: the_geom AS geom FROM tests.polygon 75 | connection: user=postgresql password=postgresql dbname=tests host=db 76 | empty_metatile_detection: 77 | size: 20743 78 | hash: 01062bb3b25dcead792d7824f9a7045f0dd92992 79 | empty_tile_detection: 80 | size: 334 81 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 82 | all: 83 | <<: *layer 84 | url: http://mapserver:8080/ 85 | layers: point,line,polygon 86 | meta: false 87 | bbox: [550000.0, 170000.0, 560000.0, 180000.0] 88 | 89 | generation: 90 | default_cache: local 91 | 92 | sqs: 93 | queue: sqs_point 94 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/test-multidim.yaml: -------------------------------------------------------------------------------- 1 | grids: 2 | swissgrid: 3 | resolutions: [100, 50, 20] 4 | bbox: [420000, 30000, 900000, 350000] 5 | tile_size: 256 6 | srs: EPSG:21781 7 | 8 | caches: 9 | local: 10 | type: filesystem 11 | http_url: http://wmts1/tiles/ 12 | folder: /tmp/tiles 13 | wmtscapabilities_file: 1.0.0/WMTSCapabilities.xml 14 | 15 | layers: 16 | multi: 17 | url: http://mapserver:8080/ 18 | layers: point_multi 19 | grid: swissgrid 20 | type: wms 21 | wmts_style: default 22 | mime_type: image/png 23 | extension: png 24 | meta: true 25 | meta_size: 8 26 | meta_buffer: 128 27 | dimensions: 28 | - name: POINT_NAME 29 | default: point1 30 | generate: [point1, point2] 31 | values: [point1, point2] 32 | geoms: 33 | - sql: the_geom AS geom FROM tests.point 34 | connection: user=postgresql password=postgresql dbname=tests host=db 35 | empty_metatile_detection: 36 | size: 20743 37 | hash: 01062bb3b25dcead792d7824f9a7045f0dd92992 38 | empty_tile_detection: 39 | size: 334 40 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 41 | 42 | generation: 43 | default_cache: local 44 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/test-multigeom.yaml: -------------------------------------------------------------------------------- 1 | grids: 2 | swissgrid_5: 3 | resolutions: [100, 50, 20, 10] 4 | bbox: [420000, 30000, 900000, 350000] 5 | tile_size: 256 6 | srs: EPSG:21781 7 | 8 | caches: 9 | local: 10 | type: filesystem 11 | http_url: http://wmts1/tiles/ 12 | folder: /tmp/tiles 13 | wmtscapabilities_file: 1.0.0/WMTSCapabilities.xml 14 | 15 | defaults: 16 | layer: &layer 17 | grid: swissgrid_5 18 | type: wms 19 | url: http://mapserver:8080/ 20 | wmts_style: default 21 | mime_type: image/png 22 | extension: png 23 | dimensions: 24 | - name: DATE 25 | default: '2012' 26 | generate: ['2012'] 27 | values: ['2005', '2010', '2012'] 28 | meta: false 29 | meta_size: 8 30 | meta_buffer: 128 31 | empty_metatile_detection: 32 | size: 20743 33 | hash: 01062bb3b25dcead792d7824f9a7045f0dd92992 34 | empty_tile_detection: 35 | size: 334 36 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 37 | 38 | layers: 39 | pp: 40 | <<: *layer 41 | layers: point,polygon 42 | geoms: 43 | - sql: the_geom AS geom FROM tests.polygon 44 | connection: user=postgresql password=postgresql dbname=tests host=db 45 | - sql: the_geom AS geom FROM tests.point 46 | connection: user=postgresql password=postgresql dbname=tests host=db 47 | min_resolution: 10 48 | max_resolution: 20 49 | 50 | generation: 51 | default_cache: local 52 | maxconsecutive_errors: 2 53 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/test-nodim.yaml: -------------------------------------------------------------------------------- 1 | grids: 2 | swissgrid_5: 3 | resolutions: [100, 50, 20, 10, 5] 4 | bbox: [420000, 30000, 900000, 350000] 5 | tile_size: 256 6 | srs: EPSG:21781 7 | 8 | caches: 9 | s3: 10 | type: s3 11 | http_url: https://%(host)s/%(bucket)s/%(folder)s/ 12 | host: s3-eu-west-1.amazonaws.com 13 | bucket: tiles 14 | folder: tiles 15 | 16 | defaults: 17 | layer: &layer 18 | grid: swissgrid_5 19 | type: wms 20 | url: http://mapserver:8080/ 21 | headers: 22 | Host: example.com 23 | Cache-Control: no-cache, no-store 24 | Pragma: no-cache 25 | wmts_style: default 26 | mime_type: image/png 27 | extension: png 28 | meta: true 29 | meta_size: 8 30 | meta_buffer: 128 31 | 32 | layers: 33 | nodim: 34 | <<: *layer 35 | layers: default 36 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/test-nosns.yaml: -------------------------------------------------------------------------------- 1 | grids: 2 | swissgrid_5: 3 | resolutions: [100, 50, 20, 10, 5] 4 | bbox: [420000, 30000, 900000, 350000] 5 | tile_size: 256 6 | srs: EPSG:21781 7 | 8 | swissgrid_01: 9 | resolutions: [1, 0.2, 0.1] 10 | bbox: [420000, 30000, 900000, 350000] 11 | tile_size: 256 12 | srs: EPSG:21781 13 | matrix_identifier: resolution 14 | 15 | swissgrid_2_5: 16 | resolutions: [2.5] 17 | bbox: [420000, 30000, 900000, 350000] 18 | tile_size: 256 19 | srs: EPSG:21781 20 | matrix_identifier: resolution 21 | 22 | swissgrid_025: 23 | resolutions: [0.25] 24 | bbox: [420000, 30000, 900000, 350000] 25 | tile_size: 256 26 | srs: EPSG:21781 27 | matrix_identifier: resolution 28 | 29 | caches: 30 | local: 31 | type: filesystem 32 | http_url: http://wmts1/tiles/ 33 | folder: /tmp/tiles 34 | multi_host: 35 | type: filesystem 36 | http_url: http://%(host)s/tiles/ 37 | folder: /tmp/tiles 38 | hosts: 39 | - wmts1 40 | - wmts2 41 | - wmts3 42 | multi_url: 43 | type: filesystem 44 | http_urls: 45 | - http://wmts1/tiles/ 46 | - http://wmts2/tiles/ 47 | - http://wmts3/tiles/ 48 | folder: /tmp/tiles 49 | mbtiles: 50 | type: mbtiles 51 | http_url: http://wmts1/tiles/ 52 | folder: /tmp/tiles/mbtiles 53 | bsddb: 54 | type: bsddb 55 | http_url: http://wmts1/tiles/ 56 | folder: /tmp/tiles/bsddb 57 | s3: 58 | type: s3 59 | http_url: https://%(host)s/%(bucket)s/%(folder)s/ 60 | host: s3-eu-west-1.amazonaws.com 61 | bucket: tiles 62 | folder: tiles 63 | 64 | defaults: 65 | all_layer: &all_layer 66 | grid: swissgrid_5 67 | wmts_style: default 68 | mime_type: image/png 69 | extension: png 70 | dimensions: 71 | - name: DATE 72 | default: '2012' 73 | generate: ['2012'] 74 | values: ['2005', '2010', '2012'] 75 | meta: true 76 | meta_size: 8 77 | meta_buffer: 128 78 | cost: 79 | # [ms] 80 | tileonly_generation_time: 60 81 | # [ms] 82 | tile_generation_time: 30 83 | # [ms] 84 | metatile_generation_time: 30 85 | # [ko] 86 | tile_size: 20 87 | layer: &layer 88 | <<: *all_layer 89 | type: wms 90 | url: http://mapserver:8080/ 91 | 92 | layers: 93 | point: 94 | <<: *layer 95 | layers: point 96 | geoms: 97 | - sql: the_geom AS geom FROM tests.point 98 | connection: user=postgresql password=postgresql dbname=tests host=db 99 | min_resolution_seed: 10 100 | point_error: 101 | <<: *layer 102 | layers: point_error 103 | point_px_buffer: 104 | <<: *layer 105 | layers: point 106 | px_buffer: 100 107 | geoms: 108 | - sql: the_geom AS geom FROM tests.point 109 | connection: user=postgresql password=postgresql dbname=tests host=db 110 | empty_metatile_detection: 111 | size: 20743 112 | hash: 01062bb3b25dcead792d7824f9a7045f0dd92992 113 | empty_tile_detection: 114 | size: 334 115 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 116 | point_hash: 117 | <<: *layer 118 | layers: point 119 | geoms: 120 | - sql: the_geom AS geom FROM tests.point 121 | connection: user=postgresql password=postgresql dbname=tests host=db 122 | min_resolution_seed: 10 123 | empty_metatile_detection: 124 | size: 20743 125 | hash: 01062bb3b25dcead792d7824f9a7045f0dd92992 126 | empty_tile_detection: 127 | size: 334 128 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 129 | point_hash_no_meta: 130 | <<: *layer 131 | layers: point 132 | meta: false 133 | empty_tile_detection: 134 | size: 334 135 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 136 | line: 137 | <<: *layer 138 | layers: line 139 | generate_salt: true 140 | geoms: 141 | - sql: the_geom AS geom FROM tests.line 142 | connection: user=postgresql password=postgresql dbname=tests host=db 143 | empty_metatile_detection: 144 | size: 20743 145 | hash: 01062bb3b25dcead792d7824f9a7045f0dd92992 146 | empty_tile_detection: 147 | size: 334 148 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 149 | polygon: 150 | <<: *layer 151 | layers: polygon 152 | meta: false 153 | geoms: 154 | - sql: the_geom AS geom FROM tests.polygon 155 | connection: user=postgresql password=postgresql dbname=tests host=db 156 | empty_metatile_detection: 157 | size: 20743 158 | hash: 01062bb3b25dcead792d7824f9a7045f0dd92992 159 | empty_tile_detection: 160 | size: 334 161 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 162 | polygon2: 163 | <<: *layer 164 | layers: polygon 165 | grid: swissgrid_01 166 | geoms: 167 | - sql: the_geom AS geom FROM tests.polygon 168 | connection: user=postgresql password=postgresql dbname=tests host=db 169 | empty_metatile_detection: 170 | size: 20743 171 | hash: 01062bb3b25dcead792d7824f9a7045f0dd92992 172 | empty_tile_detection: 173 | size: 334 174 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 175 | all: 176 | <<: *layer 177 | layers: point,line,polygon 178 | meta: false 179 | bbox: [550000.0, 170000.0, 560000.0, 180000.0] 180 | mapnik: 181 | <<: *all_layer 182 | type: mapnik 183 | mapfile: mapfile/test.mapnik 184 | meta: false 185 | data_buffer: 128 186 | output_format: png 187 | geoms: 188 | - sql: the_geom AS geom FROM tests.polygon 189 | connection: user=postgresql password=postgresql dbname=tests host=db 190 | mapnik_grid: 191 | <<: *all_layer 192 | type: mapnik 193 | mapfile: mapfile/test.mapnik 194 | meta: false 195 | data_buffer: 128 196 | output_format: grid 197 | mime_type: application/utfgrid 198 | extension: json 199 | resolution: 16 200 | geoms: 201 | - sql: the_geom AS geom FROM tests.polygon 202 | connection: user=postgresql password=postgresql dbname=tests host=db 203 | layers_fields: 204 | point: 205 | - name 206 | line: 207 | - name 208 | polygon: 209 | - name 210 | mapnik_grid_drop: 211 | <<: *all_layer 212 | type: mapnik 213 | mapfile: mapfile/test.mapnik 214 | meta: false 215 | meta_buffer: 0 216 | data_buffer: 128 217 | output_format: grid 218 | mime_type: application/utfgrid 219 | extension: json 220 | drop_empty_utfgrid: true 221 | resolution: 16 222 | geoms: 223 | - sql: the_geom AS geom FROM tests.polygon 224 | connection: user=postgresql password=postgresql dbname=tests host=db 225 | layers_fields: 226 | point: 227 | - name 228 | generation: 229 | default_cache: local 230 | default_layers: [line, polygon] 231 | maxconsecutive_errors: 2 232 | error_file: error.list 233 | number_process: 2 234 | 235 | openlayers: 236 | srs: EPSG:21781 237 | center_x: 600000 238 | center_y: 200000 239 | 240 | cost: 241 | # [nb/month] 242 | request_per_layers: 10000000 243 | 244 | server: {} 245 | 246 | sqs: 247 | queue: sqs_point 248 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/test-redis-main.yaml: -------------------------------------------------------------------------------- 1 | redis: 2 | sentinels: 3 | - - redis_sentinel 4 | - 26379 5 | queue: tilecloud 6 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/test-redis-project.yaml: -------------------------------------------------------------------------------- 1 | grids: 2 | swissgrid_5: 3 | resolutions: [100, 50, 20, 10, 5] 4 | bbox: [420000, 30000, 900000, 350000] 5 | tile_size: 256 6 | srs: EPSG:21781 7 | 8 | caches: 9 | local: 10 | type: filesystem 11 | http_url: http://wmts1/tiles/ 12 | folder: /tmp/tiles 13 | wmtscapabilities_file: 1.0.0/WMTSCapabilities.xml 14 | 15 | defaults: 16 | layer: &layer 17 | grid: swissgrid_5 18 | type: wms 19 | url: http://mapserver:8080/ 20 | wmts_style: default 21 | mime_type: image/png 22 | extension: png 23 | dimensions: 24 | - name: DATE 25 | default: '2012' 26 | generate: ['2012'] 27 | values: ['2012'] 28 | meta: true 29 | meta_size: 8 30 | meta_buffer: 128 31 | cost: 32 | # [ms] 33 | tileonly_generation_time: 60 34 | # [ms] 35 | tile_generation_time: 30 36 | # [ms] 37 | metatile_generation_time: 30 38 | # [ko] 39 | tile_size: 20 40 | 41 | layers: 42 | point: 43 | <<: *layer 44 | layers: point 45 | geoms: 46 | - sql: the_geom AS geom FROM tests.point 47 | connection: user=postgresql password=postgresql dbname=tests host=db 48 | min_resolution_seed: 10 49 | 50 | generation: 51 | default_cache: local 52 | default_layers: [point] 53 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/test-redis.yaml: -------------------------------------------------------------------------------- 1 | grids: 2 | swissgrid_5: 3 | resolutions: [100, 50, 20, 10, 5] 4 | bbox: [420000, 30000, 900000, 350000] 5 | tile_size: 256 6 | srs: EPSG:21781 7 | 8 | caches: 9 | local: 10 | type: filesystem 11 | http_url: http://wmts1/tiles/ 12 | folder: /tmp/tiles 13 | wmtscapabilities_file: 1.0.0/WMTSCapabilities.xml 14 | 15 | defaults: 16 | layer: &layer 17 | grid: swissgrid_5 18 | type: wms 19 | url: http://mapserver:8080/ 20 | wmts_style: default 21 | mime_type: image/png 22 | extension: png 23 | dimensions: 24 | - name: DATE 25 | default: '2012' 26 | generate: ['2012'] 27 | values: ['2012'] 28 | meta: true 29 | meta_size: 8 30 | meta_buffer: 128 31 | cost: 32 | # [ms] 33 | tileonly_generation_time: 60 34 | # [ms] 35 | tile_generation_time: 30 36 | # [ms] 37 | metatile_generation_time: 30 38 | # [ko] 39 | tile_size: 20 40 | 41 | layers: 42 | point: 43 | <<: *layer 44 | layers: point 45 | geoms: 46 | - sql: the_geom AS geom FROM tests.point 47 | connection: user=postgresql password=postgresql dbname=tests host=db 48 | min_resolution_seed: 10 49 | generation: 50 | default_cache: local 51 | default_layers: [point] 52 | maxconsecutive_errors: 2 53 | error_file: error.list 54 | 55 | cost: 56 | # [nb/month] 57 | request_per_layers: 10000000 58 | 59 | redis: 60 | sentinels: 61 | - - redis_sentinel 62 | - 26379 63 | queue: tilecloud 64 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/test-serve-wmtscapabilities.yaml: -------------------------------------------------------------------------------- 1 | grids: 2 | swissgrid_5: 3 | resolutions: [100, 50, 20, 10, 5] 4 | bbox: [420000, 30000, 900000, 350000] 5 | tile_size: 256 6 | srs: EPSG:21781 7 | 8 | caches: 9 | local: 10 | type: filesystem 11 | http_url: http://wmts1/tiles/ 12 | folder: /tmp/tiles-ondemend 13 | 14 | defaults: 15 | layer: &layer 16 | grid: swissgrid_5 17 | type: wms 18 | url: http://mapserver:8080/ 19 | wmts_style: default 20 | mime_type: image/png 21 | extension: png 22 | dimensions: 23 | - name: DATE 24 | default: '2012' 25 | generate: ['2012'] 26 | values: ['2005', '2010', '2012'] 27 | meta: true 28 | meta_size: 8 29 | meta_buffer: 128 30 | 31 | layers: 32 | point_hash: 33 | <<: *layer 34 | layers: point 35 | query_layers: point 36 | geoms: 37 | - sql: the_geom AS geom FROM tests.point 38 | connection: user=postgresql password=postgresql dbname=tests host=db 39 | min_resolution_seed: 10 40 | empty_metatile_detection: 41 | size: 20743 42 | hash: 01062bb3b25dcead792d7824f9a7045f0dd92992 43 | empty_tile_detection: 44 | size: 334 45 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 46 | generation: 47 | default_cache: local 48 | maxconsecutive_errors: 2 49 | 50 | server: 51 | geoms_redirect: true 52 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/test-serve.yaml: -------------------------------------------------------------------------------- 1 | grids: 2 | swissgrid_5: 3 | resolutions: [100, 50, 20, 10, 5] 4 | bbox: [420000, 30000, 900000, 350000] 5 | tile_size: 256 6 | srs: EPSG:21781 7 | 8 | caches: 9 | local: 10 | type: filesystem 11 | http_url: http://wmts1/tiles/ 12 | folder: /tmp/tiles 13 | mbtiles: 14 | type: mbtiles 15 | http_url: http://wmts1/tiles/ 16 | folder: /tmp/tiles/mbtiles 17 | 18 | defaults: 19 | layer: &layer 20 | grid: swissgrid_5 21 | type: wms 22 | url: http://mapserver:8080/ 23 | wmts_style: default 24 | mime_type: image/png 25 | extension: png 26 | dimensions: 27 | - name: DATE 28 | default: '2012' 29 | generate: ['2012'] 30 | values: ['2005', '2010', '2012'] 31 | meta: true 32 | meta_size: 8 33 | meta_buffer: 128 34 | 35 | layers: 36 | point_hash: 37 | <<: *layer 38 | layers: point 39 | query_layers: point 40 | geoms: 41 | - sql: the_geom AS geom FROM tests.point 42 | connection: user=postgresql password=postgresql dbname=tests host=db 43 | min_resolution_seed: 10 44 | empty_metatile_detection: 45 | size: 20743 46 | hash: 01062bb3b25dcead792d7824f9a7045f0dd92992 47 | empty_tile_detection: 48 | size: 334 49 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 50 | generation: 51 | default_cache: mbtiles 52 | maxconsecutive_errors: 2 53 | 54 | server: 55 | geoms_redirect: true 56 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/test.yaml: -------------------------------------------------------------------------------- 1 | grids: 2 | swissgrid_5: 3 | resolutions: [100, 50, 20, 10, 5] 4 | bbox: [420000, 30000, 900000, 350000] 5 | tile_size: 256 6 | srs: EPSG:21781 7 | 8 | swissgrid_01: 9 | resolutions: [1, 0.2, 0.1] 10 | bbox: [420000, 30000, 900000, 350000] 11 | tile_size: 256 12 | srs: EPSG:21781 13 | matrix_identifier: resolution 14 | 15 | swissgrid_2_5: 16 | resolutions: [2.5] 17 | bbox: [420000, 30000, 900000, 350000] 18 | tile_size: 256 19 | srs: EPSG:21781 20 | matrix_identifier: resolution 21 | 22 | swissgrid_025: 23 | resolutions: [0.25] 24 | bbox: [420000, 30000, 900000, 350000] 25 | tile_size: 256 26 | srs: EPSG:21781 27 | matrix_identifier: resolution 28 | 29 | caches: 30 | local: 31 | type: filesystem 32 | http_url: http://wmts1/tiles/ 33 | folder: /tmp/tiles 34 | wmtscapabilities_file: 1.0.0/WMTSCapabilities.xml 35 | multi_host: 36 | type: filesystem 37 | http_url: http://%(host)s/tiles/ 38 | folder: /tmp/tiles 39 | hosts: 40 | - wmts1 41 | - wmts2 42 | - wmts3 43 | multi_url: 44 | type: filesystem 45 | http_urls: 46 | - http://wmts1/tiles/ 47 | - http://wmts2/tiles/ 48 | - http://wmts3/tiles/ 49 | folder: /tmp/tiles 50 | mbtiles: 51 | type: mbtiles 52 | http_url: http://wmts1/tiles/ 53 | folder: /tmp/tiles/mbtiles 54 | bsddb: 55 | type: bsddb 56 | http_url: http://wmts1/tiles/ 57 | folder: /tmp/tiles/bsddb 58 | s3: 59 | type: s3 60 | http_url: https://%(host)s/%(bucket)s/%(folder)s/ 61 | host: s3-eu-west-1.amazonaws.com 62 | bucket: tiles 63 | folder: tiles 64 | 65 | defaults: 66 | base_layer: &base_layer 67 | grid: swissgrid_5 68 | wmts_style: default 69 | mime_type: image/png 70 | extension: png 71 | dimensions: 72 | - name: DATE 73 | default: '2012' 74 | generate: ['2012'] 75 | values: ['2005', '2010', '2012'] 76 | meta: true 77 | meta_size: 8 78 | meta_buffer: 128 79 | cost: 80 | # [ms] 81 | tileonly_generation_time: 60 82 | # [ms] 83 | tile_generation_time: 30 84 | # [ms] 85 | metatile_generation_time: 30 86 | # [ko] 87 | tile_size: 20 88 | mapserver_layer: &mapserver_layer 89 | <<: *base_layer 90 | type: wms 91 | url: http://mapserver:8080/ 92 | 93 | layers: 94 | point: 95 | <<: *mapserver_layer 96 | layers: point 97 | geoms: 98 | - sql: the_geom AS geom FROM tests.point 99 | connection: user=postgresql password=postgresql dbname=tests host=db 100 | min_resolution_seed: 10 101 | point_error: 102 | <<: *mapserver_layer 103 | layers: point_error 104 | point_px_buffer: 105 | <<: *mapserver_layer 106 | layers: point 107 | px_buffer: 100 108 | geoms: 109 | - sql: the_geom AS geom FROM tests.point 110 | connection: user=postgresql password=postgresql dbname=tests host=db 111 | empty_metatile_detection: 112 | size: 20743 113 | hash: 01062bb3b25dcead792d7824f9a7045f0dd92992 114 | empty_tile_detection: 115 | size: 334 116 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 117 | point_hash: 118 | <<: *mapserver_layer 119 | layers: point 120 | geoms: 121 | - sql: the_geom AS geom FROM tests.point 122 | connection: user=postgresql password=postgresql dbname=tests host=db 123 | min_resolution_seed: 10 124 | empty_metatile_detection: 125 | size: 20743 126 | hash: 01062bb3b25dcead792d7824f9a7045f0dd92992 127 | empty_tile_detection: 128 | size: 334 129 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 130 | point_hash_no_meta: 131 | <<: *mapserver_layer 132 | layers: point 133 | meta: false 134 | empty_tile_detection: 135 | size: 334 136 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 137 | line: 138 | <<: *mapserver_layer 139 | layers: line 140 | geoms: 141 | - sql: the_geom AS geom FROM tests.line 142 | connection: user=postgresql password=postgresql dbname=tests host=db 143 | empty_metatile_detection: 144 | size: 20743 145 | hash: 01062bb3b25dcead792d7824f9a7045f0dd92992 146 | empty_tile_detection: 147 | size: 334 148 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 149 | polygon: 150 | <<: *mapserver_layer 151 | layers: polygon 152 | meta: false 153 | geoms: 154 | - sql: the_geom AS geom FROM tests.polygon 155 | connection: user=postgresql password=postgresql dbname=tests host=db 156 | empty_metatile_detection: 157 | size: 20743 158 | hash: 01062bb3b25dcead792d7824f9a7045f0dd92992 159 | empty_tile_detection: 160 | size: 334 161 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 162 | polygon2: 163 | <<: *mapserver_layer 164 | layers: polygon 165 | grid: swissgrid_01 166 | geoms: 167 | - sql: the_geom AS geom FROM tests.polygon 168 | connection: user=postgresql password=postgresql dbname=tests host=db 169 | empty_metatile_detection: 170 | size: 20743 171 | hash: 01062bb3b25dcead792d7824f9a7045f0dd92992 172 | empty_tile_detection: 173 | size: 334 174 | hash: dd6cb45962bccb3ad2450ab07011ef88f766eda8 175 | all: 176 | <<: *mapserver_layer 177 | layers: point,line,polygon 178 | meta: false 179 | bbox: [550000.0, 170000.0, 560000.0, 180000.0] 180 | mapnik: 181 | <<: *base_layer 182 | type: mapnik 183 | mapfile: mapfile/test.mapnik 184 | meta: false 185 | data_buffer: 128 186 | output_format: png 187 | geoms: 188 | - sql: the_geom AS geom FROM tests.polygon 189 | connection: user=postgresql password=postgresql dbname=tests host=db 190 | wms_url: http://mapserver:8080/ 191 | mapnik_grid: 192 | <<: *base_layer 193 | type: mapnik 194 | mapfile: mapfile/test.mapnik 195 | meta: false 196 | data_buffer: 128 197 | output_format: grid 198 | mime_type: application/utfgrid 199 | extension: json 200 | resolution: 16 201 | geoms: 202 | - sql: the_geom AS geom FROM tests.polygon 203 | connection: user=postgresql password=postgresql dbname=tests host=db 204 | layers_fields: 205 | point: 206 | - name 207 | line: 208 | - name 209 | polygon: 210 | - name 211 | mapnik_grid_drop: 212 | <<: *base_layer 213 | type: mapnik 214 | mapfile: mapfile/test.mapnik 215 | meta: false 216 | meta_buffer: 0 217 | data_buffer: 128 218 | output_format: grid 219 | mime_type: application/utfgrid 220 | extension: json 221 | drop_empty_utfgrid: true 222 | resolution: 16 223 | geoms: 224 | - sql: the_geom AS geom FROM tests.polygon 225 | connection: user=postgresql password=postgresql dbname=tests host=db 226 | layers_fields: 227 | point: 228 | - name 229 | generation: 230 | default_cache: local 231 | default_layers: [line, polygon] 232 | maxconsecutive_errors: 2 233 | error_file: error.list 234 | 235 | openlayers: 236 | srs: EPSG:21781 237 | center_x: 600000 238 | center_y: 200000 239 | 240 | cost: 241 | # [nb/month] 242 | request_per_layers: 10000000 243 | 244 | sns: 245 | topic: arn:aws:sns:eu-west-1:your-account-id:tilecloud 246 | region: eu-west-1 247 | 248 | sqs: 249 | queue: sqs_point 250 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/wrong_map.yaml: -------------------------------------------------------------------------------- 1 | layers: 2 | test: 3 | empty_tile_detection: test 4 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/wrong_mapnik_grid_meta.yaml: -------------------------------------------------------------------------------- 1 | grids: 2 | swissgrid_1: 3 | resolutions: [1] 4 | bbox: [420000, 30000, 900000, 350000] 5 | srs: EPSG:21781 6 | 7 | caches: {} 8 | 9 | defaults: 10 | layer: &layer 11 | grid: swissgrid_1 12 | type: mapnik 13 | meta: true 14 | mapfile: test.mapnik 15 | wmts_style: default 16 | 17 | layers: 18 | a: 19 | <<: *layer 20 | output_format: png 21 | extension: png 22 | mime_type: image/png 23 | b: 24 | <<: *layer 25 | output_format: grid 26 | extension: json 27 | mime_type: application/json 28 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/wrong_resolutions.yaml: -------------------------------------------------------------------------------- 1 | grids: 2 | swissgrid_01: 3 | resolutions: [1, 0.2, 0.1] 4 | resolution_scale: 5 5 | bbox: [420000, 30000, 900000, 350000] 6 | srs: EPSG:21781 7 | 8 | caches: {} 9 | 10 | layers: {} 11 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/wrong_sequence.yaml: -------------------------------------------------------------------------------- 1 | grids: 2 | test: 3 | resolutions: test 4 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/wrong_srs.yaml: -------------------------------------------------------------------------------- 1 | grids: 2 | swissgrid_01: 3 | resolutions: [1] 4 | bbox: [420000, 30000, 900000, 350000] 5 | srs: EPSG21781 6 | 7 | caches: {} 8 | 9 | layers: {} 10 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/wrong_srs_auth.yaml: -------------------------------------------------------------------------------- 1 | grids: 2 | swissgrid_01: 3 | resolutions: [1] 4 | bbox: [420000, 30000, 900000, 350000] 5 | srs: toto:21781 6 | 7 | caches: {} 8 | 9 | layers: {} 10 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/wrong_srs_id.yaml: -------------------------------------------------------------------------------- 1 | grids: 2 | swissgrid_01: 3 | resolutions: [1] 4 | bbox: [420000, 30000, 900000, 350000] 5 | srs: EPSG:21781a 6 | 7 | caches: {} 8 | 9 | layers: {} 10 | -------------------------------------------------------------------------------- /tilecloud_chain/tests/tilegeneration/wrong_type.yaml: -------------------------------------------------------------------------------- 1 | grids: 2 | swissgrid_1: 3 | resolutions: [2] 4 | resolution_scale: 5.5 5 | bbox: [a, b, c] 6 | srs: ['EPSG:21781'] 7 | swissgrid_2: 8 | resolutions: [1] 9 | bbox: [123] 10 | srs: {} 11 | swissgrid_3: 12 | srs: epsg:21781 13 | swissgrid_4: 14 | srs: epsg21781 15 | swissgrid_5: {} 16 | swissgrid_6: 17 | swissgrid!: {} 18 | 19 | caches: {} 20 | 21 | layers: 22 | hi!: 23 | wmts_style: yo! 24 | dimensions: 25 | - name: DATE! 26 | default: '2010!' 27 | generate: ['2012!'] 28 | values: ['2005!', '2010!', '2012!'] 29 | - name: time 30 | default: 1 31 | generate: [1] 32 | values: [1] 33 | -------------------------------------------------------------------------------- /tilecloud_chain/timedtilestore.py: -------------------------------------------------------------------------------- 1 | """A wrapper around a TileStore that adds timer metrics.""" 2 | 3 | import time 4 | from collections.abc import AsyncGenerator, AsyncIterator 5 | from typing import Any, TypeVar 6 | 7 | from prometheus_client import Summary 8 | from tilecloud import Tile 9 | 10 | from tilecloud_chain.store import AsyncTileStore 11 | 12 | _OptionalTileOrNot = TypeVar("_OptionalTileOrNot", Tile | None, Tile) 13 | 14 | _TILESTORE_OPERATION_SUMMARY = Summary( 15 | "tilecloud_chain_tilestore", 16 | "Number of tilestore contains", 17 | ["layer", "host", "store", "operation"], 18 | ) 19 | 20 | 21 | class TimedTileStoreWrapper(AsyncTileStore): 22 | """A wrapper around a TileStore that adds timer metrics.""" 23 | 24 | def __init__(self, tile_store: AsyncTileStore, store_name: str) -> None: 25 | """Initialize.""" 26 | super().__init__() 27 | self._tile_store = tile_store 28 | self._store_name = store_name 29 | 30 | async def _time_iteration(self) -> AsyncGenerator[_OptionalTileOrNot]: 31 | while True: 32 | start = time.perf_counter() 33 | try: 34 | tile = await anext(self._tile_store.list()) 35 | except StopAsyncIteration: 36 | break 37 | except RuntimeError as exception: 38 | if isinstance(exception.__cause__, StopAsyncIteration): 39 | # since python 3.7, a StopIteration is wrapped in a RuntimeError (PEP 479) 40 | break 41 | raise 42 | _TILESTORE_OPERATION_SUMMARY.labels( 43 | tile.metadata.get("layer", "none"), 44 | tile.metadata.get("host", "none"), 45 | self._store_name, 46 | "list", 47 | ).observe(time.perf_counter() - start) 48 | yield tile 49 | 50 | async def __contains__(self, tile: Tile) -> bool: 51 | """See in superclass.""" 52 | with _TILESTORE_OPERATION_SUMMARY.labels( 53 | tile.metadata.get("layer", "none"), 54 | tile.metadata.get("host", "none"), 55 | self._store_name, 56 | "contains", 57 | ).time(): 58 | return await self._tile_store.__contains__(tile) 59 | 60 | async def delete_one(self, tile: Tile) -> Tile: 61 | """See in superclass.""" 62 | with _TILESTORE_OPERATION_SUMMARY.labels( 63 | tile.metadata.get("layer", "none"), 64 | tile.metadata.get("host", "none"), 65 | self._store_name, 66 | "delete_one", 67 | ).time(): 68 | return await self._tile_store.delete_one(tile) 69 | 70 | async def list(self) -> AsyncIterator[Tile]: 71 | """See in superclass.""" 72 | async for tile in self._time_iteration(): 73 | yield tile 74 | 75 | async def get_one(self, tile: Tile) -> Tile | None: 76 | """See in superclass.""" 77 | with _TILESTORE_OPERATION_SUMMARY.labels( 78 | tile.metadata.get("layer", "none"), 79 | tile.metadata.get("host", "none"), 80 | self._store_name, 81 | "get_one", 82 | ).time(): 83 | return await self._tile_store.get_one(tile) 84 | 85 | async def put_one(self, tile: Tile) -> Tile: 86 | """See in superclass.""" 87 | with _TILESTORE_OPERATION_SUMMARY.labels( 88 | tile.metadata.get("layer", "none"), 89 | tile.metadata.get("host", "none"), 90 | self._store_name, 91 | "put_one", 92 | ).time(): 93 | return await self._tile_store.put_one(tile) 94 | 95 | async def get(self, tiles: AsyncIterator[Tile]) -> AsyncIterator[Tile | None]: 96 | """See in superclass.""" 97 | with _TILESTORE_OPERATION_SUMMARY.labels("none", "none", self._store_name, "get").time(): 98 | async for tile in self._tile_store.get(tiles): 99 | yield tile 100 | 101 | def __getattr__(self, item: str) -> Any: 102 | """See in superclass.""" 103 | return getattr(self._tile_store, item) 104 | 105 | def __str__(self) -> str: 106 | """Get string representation.""" 107 | return f"{self.__class__.__name__}({self._store_name}: {self._tile_store}" 108 | 109 | def __repr__(self) -> str: 110 | """Get string representation.""" 111 | return f"{self.__class__.__name__}({self._store_name}: {self._tile_store!r})" 112 | -------------------------------------------------------------------------------- /tilecloud_chain/views/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/camptocamp/tilecloud-chain/2f84239147eb4025af917a862d8b3cb38f4a88e1/tilecloud_chain/views/__init__.py --------------------------------------------------------------------------------