├── .circleci └── config.yml ├── .dockerignore ├── .env.sample ├── .flake8 ├── .gitignore ├── .pre-commit-config.yaml ├── CHANGELOG.md ├── CONTRIBUTING.md ├── COPYRIGHT.txt ├── Dockerfile ├── LICENSE.txt ├── MANIFEST.in ├── README.md ├── VERSION ├── config.yaml.sample ├── logging.conf.sample ├── requirements.txt ├── setup.py ├── tests ├── __init__.py ├── fixtures │ └── sample_expire_list ├── queue │ ├── __init__.py │ ├── test_file.py │ ├── test_mapper.py │ ├── test_msg.py │ ├── test_sqs.py │ └── test_writer.py ├── test_buffer.py ├── test_command.py ├── test_config.py ├── test_metatile.py ├── test_metro_extract.py ├── test_mvt.py ├── test_process.py ├── test_query_common.py ├── test_query_fixture.py ├── test_query_rawr.py ├── test_query_split.py ├── test_rawr.py ├── test_store.py ├── test_tile.py ├── test_toi.py ├── test_utils.py ├── test_wof.py └── test_wof_http.py └── tilequeue ├── __init__.py ├── command.py ├── config.py ├── constants.py ├── format ├── OSciMap4 │ ├── GeomEncoder.py │ ├── StaticKeys │ │ └── __init__.py │ ├── StaticVals │ │ └── __init__.py │ ├── TagRewrite │ │ └── __init__.py │ ├── TileData_v4.proto │ ├── TileData_v4_pb2.py │ ├── __init__.py │ └── pbf_test.py ├── __init__.py ├── geojson.py ├── mvt.py ├── topojson.py └── vtm.py ├── log.py ├── metatile.py ├── metro_extract.py ├── process.py ├── query ├── __init__.py ├── common.py ├── fixture.py ├── pool.py ├── postgres.py ├── rawr.py └── split.py ├── queue ├── __init__.py ├── file.py ├── inflight.py ├── mapper.py ├── memory.py ├── message.py ├── redis_queue.py ├── sqs.py └── writer.py ├── rawr.py ├── stats.py ├── store.py ├── tile.py ├── toi ├── __init__.py ├── file.py └── s3.py ├── top_tiles.py ├── transform.py ├── utils.py ├── wof.py └── worker.py /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | jobs: 3 | build: 4 | docker: 5 | - image: circleci/python:2.7.15-stretch 6 | steps: 7 | - checkout 8 | - run: 9 | command: | 10 | cp .pre-commit-config.yaml pre-commit-cache-key.txt 11 | python --version --version >> pre-commit-cache-key.txt 12 | - run: 13 | name: Update packages 14 | command: sudo apt-get update 15 | - run: 16 | name: Install PROJ data 17 | command: sudo apt-get install proj-data 18 | - restore_cache: 19 | key: python-requirements-{{ .Branch }}-{{ checksum "requirements.txt" }} 20 | - run: 21 | name: Install Pip dependencies and pre-commit 22 | command: | 23 | virtualenv ~/env 24 | . ~/env/bin/activate 25 | pip install -Ur requirements.txt 26 | pip install pre-commit 27 | - save_cache: 28 | key: python-requirements-{{ .Branch }}-{{ checksum "requirements.txt" }} 29 | paths: 30 | - ~/.cache/pre-commit 31 | - "~/env" 32 | - run: 33 | name: Check Code Style using pre-commit 34 | command: | 35 | . ~/env/bin/activate 36 | pre-commit run --show-diff-on-failure --all-files 37 | - run: 38 | name: Setup.py develop 39 | command: | 40 | . ~/env/bin/activate 41 | python setup.py develop 42 | - run: 43 | name: Install Python packages for testing 44 | command: | 45 | . ~/env/bin/activate 46 | pip install -U 'mock==1.2.0' httptestserver 47 | # This section is commented out because now flake8 is part of an earlier step - `Check Code Style using pre-commit`. See https://github.com/tilezen/tilequeue/pull/403 48 | # - run: 49 | # name: Check PEP8 compliance 50 | # command: | 51 | # . ~/env/bin/activate 52 | # find . -not -path '*/.eggs/*' -not -path '*OSciMap4*' -name '*.py' | xargs flake8 53 | - run: 54 | name: Unit tests 55 | command: | 56 | . ~/env/bin/activate 57 | python setup.py test 58 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | .git 2 | -------------------------------------------------------------------------------- /.env.sample: -------------------------------------------------------------------------------- 1 | AWS_ACCESS_KEY_ID= 2 | AWS_SECRET_ACCESS_KEY= 3 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 120 3 | exclude = tests/* 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | tilequeue.egg-info 2 | *.pyc 3 | config.yaml 4 | logging.conf 5 | *.sw* 6 | build 7 | dist 8 | .env 9 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | default_language_version: 2 | python: python2.7 3 | repos: 4 | - repo: https://github.com/pre-commit/pre-commit-hooks 5 | rev: v2.5.0 6 | hooks: 7 | - id: check-docstring-first 8 | - id: check-yaml 9 | - id: debug-statements 10 | - id: check-case-conflict 11 | - id: check-executables-have-shebangs 12 | - id: check-json 13 | - id: end-of-file-fixer 14 | - id: trailing-whitespace 15 | - id: requirements-txt-fixer 16 | - id: double-quote-string-fixer 17 | - repo: https://github.com/asottile/reorder_python_imports 18 | rev: v1.9.0 19 | hooks: 20 | - id: reorder-python-imports 21 | - repo: https://github.com/pre-commit/mirrors-autopep8 22 | rev: v1.5.7 23 | hooks: 24 | - id: autopep8 25 | - repo: https://github.com/asottile/yesqa 26 | rev: v0.0.11 27 | hooks: 28 | - id: yesqa 29 | - repo: https://github.com/PyCQA/flake8 30 | rev: 3.9.2 31 | hooks: 32 | - id: flake8 33 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | ## Issues 4 | 5 | If you notice a problem, or would like to request a feature, create an 6 | issue within the [github issue 7 | tracker](https://github.com/mapzen/tilequeue/issues). 8 | 9 | ## Pull Requests 10 | 11 | Creating pull requests through github is easiest. 12 | 13 | * Fork the repository 14 | * Work off a branch 15 | * Add a test if possible 16 | * All tests should pass via `python setup.py test` 17 | * Ensure that the code is 18 | [pep8](http://legacy.python.org/dev/peps/pep-0008/) compliant (eg 19 | use [flake8](https://pypi.python.org/pypi/flake8)) 20 | - `find . -name '*.py' | xargs flake8` 21 | * Use atomic commits, and avoid adding multiple features or fixes in a 22 | single pull request. Open multiple pull requests, one for each 23 | semantic change. 24 | * Use good [commit messages](http://git-scm.com/book/ch5-2.html) 25 | * Create a pull request with a descriptive name. 26 | -------------------------------------------------------------------------------- /COPYRIGHT.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014 Mapzen 2 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:2 2 | 3 | RUN apt-get -y update \ 4 | && apt-get -y install \ 5 | libgeos-dev \ 6 | && rm -rf /var/lib/apt/lists/* 7 | 8 | RUN mkdir -p /usr/src/app 9 | WORKDIR /usr/src/app 10 | RUN mkdir -p /usr/src/vector-datasource \ 11 | && git clone --depth 1 https://github.com/mapzen/vector-datasource.git /usr/src/vector-datasource 12 | 13 | COPY . /usr/src/app 14 | RUN pip install --no-cache-dir -r requirements.txt \ 15 | && pip install --no-cache-dir -e . \ 16 | && pip install --no-cache-dir -e ../vector-datasource 17 | 18 | CMD [ "tilequeue", "process", "--config", "./config.yaml"] 19 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Mapzen 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include COPYRIGHT.txt LICENSE.txt README.md VERSION 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white)](https://github.com/pre-commit/pre-commit) 2 | 3 | # tilequeue 4 | 5 | A tile generator, used by itself for asyncronous tile generation or with [tileserver](https://github.com/tilezen/tileserver/) for serving tiles. 6 | 7 | ## Installation 8 | 9 | We recommend following the vector-datasource [installation instructions](https://github.com/tilezen/vector-datasource/wiki/Mapzen-Vector-Tile-Service). 10 | 11 | _Note: Installation has been tested using Python 2.7 and results with other versions may vary._ 12 | 13 | In addition to the dependencies in [requirements.txt](requirements.txt), tileserver requires 14 | 15 | * PostgreSQL client-side development library and headers (for psycopg) 16 | * GEOS library 17 | 18 | These can be installed on Debian-based systems with 19 | ``` 20 | sudo apt-get install libpq-dev libgeos-c1v5 21 | ``` 22 | 23 | Then install the python requirements with 24 | 25 | pip install -Ur requirements.txt 26 | 27 | Then: 28 | 29 | python setup.py develop 30 | 31 | ### Coanacatl 32 | 33 | Note that if you want to configure the `coanacatl` format (really an alternative MVT driver), you will need to install the [coanacatl](https://github.com/tilezen/coanacatl) library. This is totally optional and tilequeue will work fine with the regular `mvt` format, but can provide some robustness and speed improvements. 34 | 35 | ## Configuration 36 | 37 | See [`config.yaml.sample`](https://github.com/tilezen/tilequeue/blob/master/config.yaml.sample) 38 | 39 | ## Layer definitions 40 | 41 | To understand the language tilequeue layer definitions, it's best to look at the [Tilezen vector-datasource](https://github.com/tilezen/vector-datasource) 42 | 43 | ## Running 44 | 45 | A list of commands is available by running `tilequeue --help`. Each command also supports usage information by running `tilequeue --help`. All commands require a configuration file to be passed through the `--config` argument. A brief summary of commands: 46 | 47 | * `process`: Start the tilequeue worker process, reading jobs from the queue and processing them. 48 | * `seed`: Enqueue the tiles defined in the `tiles/seed` section of the config, and (if configured) add them to the TOI. 49 | * `dump-tiles-of-interest`: Write out the TOI to `toi.txt`. 50 | * `load-tiles-of-interest`: Replace the TOI with the contents of `toi.txt`. 51 | * `enqueue-tiles-of-interest`: Enqueue the TOI as a set of jobs. 52 | * `prune-tiles-of-interest`: Prune the TOI according to the rules in the `toi-prune` section of the config. 53 | * `wof-process-neighbourhoods`: Fetch the latest WOF neighbourhood data and update the database, enqueueing jobs for any changes. 54 | * `wof-load-initial-neighbourhoods`: Load WOF neighbourhood data into the database. 55 | * `consume-tile-traffic`: Read tile access log files and insert corresponding records into a PostgreSQL compatible database (we use AWS Redshift). 56 | * `stuck-tiles`: Find tiles which exist in the store, but are not in the TOI. These won't be updated when the data changes, so should be deleted. Tiles can become stuck due to race conditions between various components, e.g: a tile being dropped from the TOI while its job is still in the queue. Outputs a list of tiles to `stdout`. 57 | * `delete-stuck-tiles`: Read a list of tiles from `stdin` and delete them. Designed to be used in conjunction with `stuck-tiles`. 58 | * `rawr-process`: Read from RAWR tile queue and generate RAWR tiles. 59 | * `rawr-seed-toi`: Read the TOI and enqueue the corresponding RAWR tile jobs. 60 | * `tile-status`: Report the status of the given tiles in the store, queue and TOI. 61 | * `tile`: Render a single tile. 62 | * `rawr-enqueue`: Enqueue RAWR tiles corresponding to expired tiles. 63 | 64 | ### Testing 65 | 66 | You can run the tests with the command `python setup.py test` in the top level source directory. 67 | 68 | ### Code style 69 | 70 | We use `flake8` to check our source code is PEP8 compatible. You can run this using the command: 71 | 72 | ``` 73 | find . -not -path '*/.eggs/*' -not -path '*OSciMap4*' -not -path '*/venv/*' -name '*.py' | xargs flake8 74 | ``` 75 | 76 | You might find it useful to add that as a git pre-commit hook, or to run a PEP8 checker in your editor. 77 | 78 | ### Profiling 79 | 80 | A great way to get a high level view of the time consumed by the code is to run it via [`python-flamegraph`](https://github.com/evanhempel/python-flamegraph), which produces a profile suitable for processing into an SVG using another tool called [FlameGraph](http://www.brendangregg.com/flamegraphs.html). For example, to run a graph for a single tile: 81 | 82 | ``` 83 | python -m flamegraph -o perf.log `which tilequeue` tile --config config.yaml 10/163/395 84 | flamegraph.pl --title "Tilequeue 10/163/395" perf.log > perf.svg 85 | ``` 86 | 87 | Note that you may need to add the path to `flamegraph.pl` from Brendan Gregg's repository if you haven't installed it in your `$PATH`. 88 | 89 | ## License 90 | 91 | Tilequeue is available under [the MIT license](https://github.com/tilezen/tilequeue/blob/master/LICENSE.txt). 92 | -------------------------------------------------------------------------------- /VERSION: -------------------------------------------------------------------------------- 1 | 2.5.0 2 | -------------------------------------------------------------------------------- /logging.conf.sample: -------------------------------------------------------------------------------- 1 | [loggers] 2 | keys=root,process,seed,prune_tiles_of_interest,enqueue_tiles_of_interest,dump_tiles_of_interest,load_tiles_of_interest,wof_process_neighbourhoods,query,consume_tile_traffic,tile_status,stuck_tiles,delete_stuck_tiles,rawr_enqueue,rawr_process,rawr_seed 3 | 4 | [handlers] 5 | keys=consoleHandler,jsonConsoleHandler 6 | 7 | [formatters] 8 | keys=simpleFormatter,jsonFormatter 9 | 10 | [logger_root] 11 | level=WARNING 12 | handlers=consoleHandler 13 | 14 | [logger_prune_tiles_of_interest] 15 | level=INFO 16 | handlers=consoleHandler 17 | qualName=prune_tiles_of_interest 18 | propagate=0 19 | 20 | [logger_enqueue_tiles_of_interest] 21 | level=INFO 22 | handlers=consoleHandler 23 | qualName=enqueue_tiles_of_interest 24 | propagate=0 25 | 26 | [logger_dump_tiles_of_interest] 27 | level=INFO 28 | handlers=consoleHandler 29 | qualName=dump_tiles_of_interest 30 | propagate=0 31 | 32 | [logger_load_tiles_of_interest] 33 | level=INFO 34 | handlers=consoleHandler 35 | qualName=load_tiles_of_interest 36 | propagate=0 37 | 38 | [logger_process] 39 | level=INFO 40 | handlers=jsonConsoleHandler 41 | qualName=process 42 | propagate=0 43 | 44 | [logger_seed] 45 | level=INFO 46 | handlers=consoleHandler 47 | qualName=seed 48 | propagate=0 49 | 50 | [logger_wof_process_neighbourhoods] 51 | level=INFO 52 | handlers=consoleHandler 53 | qualName=wof_process_neighbourhoods 54 | propagate=0 55 | 56 | [logger_query] 57 | level=DEBUG 58 | handlers=consoleHandler 59 | qualName=query 60 | propagate=0 61 | 62 | [logger_consume_tile_traffic] 63 | level=INFO 64 | handlers=consoleHandler 65 | qualName=consume_tile_traffic 66 | propagate=0 67 | 68 | [logger_tile_status] 69 | level=INFO 70 | handlers=consoleHandler 71 | qualName=tile_status 72 | propagate=0 73 | 74 | [logger_stuck_tiles] 75 | level=INFO 76 | handlers=consoleHandler 77 | qualName=stuck_tiles 78 | propagate=0 79 | 80 | [logger_delete_stuck_tiles] 81 | level=INFO 82 | handlers=consoleHandler 83 | qualName=delete_stuck_tiles 84 | propagate=0 85 | 86 | [logger_rawr_enqueue] 87 | level=INFO 88 | handlers=consoleHandler 89 | qualName=rawr_enqueue 90 | propagate=0 91 | 92 | [logger_rawr_process] 93 | level=INFO 94 | handlers=jsonConsoleHandler 95 | qualName=rawr_process 96 | propagate=0 97 | 98 | [logger_rawr_seed] 99 | level=INFO 100 | handlers=consoleHandler 101 | qualName=rawr_seed 102 | propagate=0 103 | 104 | [handler_consoleHandler] 105 | class=StreamHandler 106 | formatter=simpleFormatter 107 | args=(sys.stdout,) 108 | 109 | [handler_jsonConsoleHandler] 110 | class=StreamHandler 111 | formatter=jsonFormatter 112 | args=(sys.stdout,) 113 | 114 | [formatter_jsonFormatter] 115 | format=%(asctime)s %(message)s 116 | datefmt=%Y-%m-%dT%H:%M:%S%z 117 | 118 | [formatter_simpleFormatter] 119 | format=%(asctime)s - %(name)s - %(levelname)s - %(message)s 120 | datefmt= 121 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | appdirs==1.4.3 2 | argparse==1.4.0 3 | boto==2.48.0 4 | boto3==1.15.18 5 | botocore==1.18.18 6 | edtf==2.6.0 7 | enum34==1.1.6 8 | future==0.16.0 9 | hiredis==0.2.0 10 | Jinja2==2.10.1 11 | mapbox-vector-tile==1.2.0 12 | MarkupSafe==1.0 13 | ModestMaps==1.4.7 14 | protobuf==3.4.0 15 | psycopg2==2.7.3.2 16 | pyclipper==1.0.6 17 | pycountry==17.9.23 18 | pyproj==2.1.0 19 | python-dateutil==2.6.1 20 | PyYAML==4.2b4 21 | git+https://github.com/tilezen/raw_tiles@master#egg=raw_tiles 22 | redis==2.10.6 23 | requests==2.25.1 24 | Shapely==1.6.2.post1 25 | six==1.11.0 26 | statsd==3.2.1 27 | StreetNames==0.1.5 28 | ujson==1.35 29 | Werkzeug==0.12.2 30 | wsgiref==0.1.2 31 | zope.dottedname==4.2 32 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | 3 | from setuptools import find_packages 4 | from setuptools import setup 5 | 6 | version_path = os.path.join(os.path.dirname(__file__), 'VERSION') 7 | with open(version_path) as fh: 8 | version = fh.read().strip() 9 | 10 | setup(name='tilequeue', 11 | version=version, 12 | description='Queue operations to manage the processes surrounding tile ' 13 | 'rendering.', 14 | long_description=open('README.md').read(), 15 | classifiers=[ 16 | # strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers 17 | 'Development Status :: 4 - Beta', 18 | 'Environment :: Console', 19 | 'Intended Audience :: Developers', 20 | 'Intended Audience :: System Administrators', 21 | 'License :: OSI Approved :: MIT License', 22 | 'Natural Language :: English', 23 | 'Operating System :: POSIX :: Linux', 24 | 'Programming Language :: Python :: 2.7', 25 | 'Programming Language :: Python :: Implementation :: CPython', 26 | 'Topic :: Internet :: WWW/HTTP :: Site Management', 27 | 'Topic :: Utilities', 28 | ], 29 | keywords='aws queue s3 sqs tile map', 30 | author='Robert Marianski, Mapzen', 31 | author_email='rob@mapzen.com', 32 | url='https://github.com/mapzen/tilequeue', 33 | license='MIT', 34 | packages=find_packages(exclude=['ez_setup', 'examples', 'tests']), 35 | include_package_data=True, 36 | zip_safe=False, 37 | install_requires=[ 38 | 'boto', 39 | 'boto3', 40 | 'edtf', 41 | 'enum34', 42 | 'hiredis', 43 | 'Jinja2>=2.10.1', 44 | 'mapbox-vector-tile', 45 | 'ModestMaps', 46 | 'protobuf', 47 | 'psycopg2', 48 | 'pyproj>=2.1.0', 49 | 'python-dateutil', 50 | 'PyYAML', 51 | 'raw_tiles', 52 | 'redis', 53 | 'requests', 54 | 'Shapely', 55 | 'statsd', 56 | 'ujson', 57 | 'zope.dottedname', 58 | ], 59 | test_suite='tests', 60 | tests_require=[ 61 | 'mock', 62 | 'httptestserver' 63 | ], 64 | entry_points=dict( 65 | console_scripts=[ 66 | 'tilequeue = tilequeue.command:tilequeue_main', 67 | ] 68 | ) 69 | ) 70 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | -------------------------------------------------------------------------------- /tests/fixtures/sample_expire_list: -------------------------------------------------------------------------------- 1 | 20/18/219103 2 | 20/19/219103 3 | 20/20/219102 4 | 20/20/219103 5 | 20/21/219102 6 | 20/21/219103 7 | 20/23/219101 8 | 20/22/219102 9 | 20/22/219103 10 | 20/23/219102 11 | 20/24/219101 12 | 20/25/219100 13 | 20/25/219101 14 | 20/24/219102 15 | 20/26/219100 16 | 20/26/219101 17 | 20/27/219100 18 | 20/28/219099 19 | 20/29/219099 20 | 20/30/219098 21 | 20/30/219099 22 | 20/31/219098 23 | 20/31/219099 24 | 20/28/219100 25 | 20/29/219100 26 | -------------------------------------------------------------------------------- /tests/queue/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tilezen/tilequeue/7dd1998e2c5c58182c9db49271b70cac4eab5c9d/tests/queue/__init__.py -------------------------------------------------------------------------------- /tests/queue/test_file.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Unit tests for `tilequeue.queue.file`. 3 | ''' 4 | import StringIO 5 | import unittest 6 | 7 | from ModestMaps.Core import Coordinate 8 | 9 | 10 | class TestQueue(unittest.TestCase): 11 | 12 | def setUp(self): 13 | from tilequeue import tile 14 | from tilequeue.queue import OutputFileQueue 15 | 16 | self.test_tile_coords = [ 17 | (0, 0, 0), 18 | (1, 2, 3), 19 | (4, 5, 6), 20 | (9, 3, 1), 21 | (4, 7, 1) 22 | ] 23 | self.test_tile_objs = [ 24 | Coordinate(row=r, column=c, zoom=z) 25 | for (z, c, r) in self.test_tile_coords] 26 | self.tile_coords_str = '\n'.join( 27 | map(tile.serialize_coord, self.test_tile_objs)) + '\n' 28 | self.tiles_fp = StringIO.StringIO() 29 | self.queue = OutputFileQueue(self.tiles_fp) 30 | 31 | def test_read(self): 32 | from tilequeue.tile import serialize_coord 33 | self._write_str_to_file(self.tile_coords_str) 34 | 35 | # Test `.read() for multiple records.` 36 | actual_coord_strs = [ 37 | msg.payload for msg in self.queue.read()] 38 | expected = map(serialize_coord, self.test_tile_objs) 39 | self.assertEqual( 40 | actual_coord_strs, expected, 'Reading multiple records failed') 41 | 42 | def test_enqueue_and_enqueue_batch(self): 43 | from tilequeue.tile import serialize_coord 44 | # Test `.enqueue_batch()`. 45 | num_to_enqueue = 3 46 | self.assertEqual( 47 | self.queue.enqueue_batch( 48 | map(serialize_coord, self.test_tile_objs[:num_to_enqueue])), 49 | (num_to_enqueue, 0), 50 | 'Return value of `enqueue_batch()` does not match expected' 51 | ) 52 | 53 | # Test `.enqueue()`. 54 | for coord in self.test_tile_objs[num_to_enqueue:]: 55 | self.queue.enqueue(serialize_coord(coord)) 56 | 57 | self.assertEqual( 58 | self.tiles_fp.getvalue(), 59 | self.tile_coords_str, 60 | 'Contents of file do not match expected') 61 | 62 | def test_clear(self): 63 | self._write_str_to_file(self.tile_coords_str) 64 | self.assertEqual( 65 | self.queue.clear(), -1, 66 | 'Return value of `clear()` does not match expected.') 67 | self.assertEqual( 68 | self.tiles_fp.getvalue(), '', '`clear()` did not clear the file!') 69 | 70 | def test_close(self): 71 | self.assertFalse( 72 | self.tiles_fp.closed, 73 | 'Sanity check failed: the test runner\'s file pointer appears to ' 74 | 'be closed. This shouldn\'t ever happen.') 75 | self.queue.close() 76 | self.assertTrue(self.tiles_fp.closed, 'File pointer was not closed!') 77 | 78 | def _write_str_to_file(self, string): 79 | self.tiles_fp.write(string) 80 | self.tiles_fp.seek(0) 81 | -------------------------------------------------------------------------------- /tests/queue/test_mapper.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | 4 | class SingleQueueMapperTest(unittest.TestCase): 5 | 6 | def make_queue_mapper(self, queue_name, tile_queue): 7 | from tilequeue.queue.mapper import SingleQueueMapper 8 | return SingleQueueMapper(queue_name, tile_queue) 9 | 10 | def test_single_queue_mapper(self): 11 | from mock import MagicMock 12 | from tilequeue.tile import deserialize_coord 13 | tile_queue_mock = MagicMock() 14 | queue_mapper = self.make_queue_mapper('queue_name', tile_queue_mock) 15 | coords = [deserialize_coord('1/1/1'), deserialize_coord('15/1/1')] 16 | coord_groups = list(queue_mapper.group(coords)) 17 | self.assertEquals(2, len(coord_groups)) 18 | cg1, cg2 = coord_groups 19 | self.assertEquals('queue_name', cg1.queue_id) 20 | self.assertEquals('queue_name', cg2.queue_id) 21 | self.assertEquals(1, len(cg1.coords)) 22 | self.assertEquals(coords[0], cg1.coords[0]) 23 | self.assertEquals(1, len(cg2.coords)) 24 | self.assertEquals(coords[1], cg2.coords[0]) 25 | 26 | self.assertIs(tile_queue_mock, queue_mapper.get_queue('queue_name')) 27 | 28 | qs = queue_mapper.queues_in_priority_order() 29 | self.assertEquals(1, len(qs)) 30 | qn, q = qs[0] 31 | self.assertEquals('queue_name', qn) 32 | self.assertIs(tile_queue_mock, q) 33 | 34 | 35 | class MultipleQueueMapperTest(unittest.TestCase): 36 | 37 | def make_queue_mapper(self, specs): 38 | from tilequeue.queue.mapper import ZoomRangeAndZoomGroupQueueMapper 39 | from tilequeue.queue.mapper import ZoomRangeQueueSpec 40 | zoom_range_specs = [] 41 | for zs, ze, qn, tq, gbz in specs: 42 | zrs = ZoomRangeQueueSpec(zs, ze, qn, tq, gbz) 43 | zoom_range_specs.append(zrs) 44 | qm = ZoomRangeAndZoomGroupQueueMapper(zoom_range_specs) 45 | return qm 46 | 47 | def test_group_coords(self): 48 | from tilequeue.tile import deserialize_coord 49 | specs = ( 50 | (0, 10, 'tile_queue_1', object(), None), 51 | (10, 16, 'tile_queue_2', object(), 10), 52 | ) 53 | qm = self.make_queue_mapper(specs) 54 | coord_strs = ( 55 | '1/1/1', 56 | '9/0/0', 57 | '10/0/0', 58 | '14/65536/65536', 59 | '15/0/0', 60 | ) 61 | coords = map(deserialize_coord, coord_strs) 62 | coord_groups = list(qm.group(coords)) 63 | assert len(coord_groups) == 4 64 | 65 | cg1, cg2, cg3, cg4 = coord_groups 66 | 67 | lo_zoom_queue_id = 0 68 | hi_zoom_queue_id = 1 69 | 70 | # low zooms are grouped separately 71 | self.assertEquals(1, len(cg1.coords)) 72 | self.assertEquals([deserialize_coord('1/1/1')], cg1.coords) 73 | self.assertEquals(lo_zoom_queue_id, cg1.queue_id) 74 | 75 | self.assertEquals(1, len(cg2.coords)) 76 | self.assertEquals([deserialize_coord('9/0/0')], cg2.coords) 77 | self.assertEquals(lo_zoom_queue_id, cg2.queue_id) 78 | 79 | # common z10 parents are grouped together 80 | self.assertEquals(2, len(cg3.coords)) 81 | self.assertEquals(map(deserialize_coord, ['10/0/0', '15/0/0']), 82 | cg3.coords) 83 | self.assertEquals(hi_zoom_queue_id, cg3.queue_id) 84 | 85 | # different z10 parent grouped separately, even though it 86 | # should be sent to the same queue 87 | self.assertEquals(1, len(cg4.coords)) 88 | self.assertEquals([deserialize_coord('14/65536/65536')], cg4.coords) 89 | self.assertEquals(hi_zoom_queue_id, cg4.queue_id) 90 | 91 | def test_group_coord_out_of_range(self): 92 | from tilequeue.tile import deserialize_coord 93 | specs = ( 94 | (0, 10, 'tile_queue_1', object(), None), 95 | (10, 16, 'tile_queue_2', object(), 10), 96 | ) 97 | qm = self.make_queue_mapper(specs) 98 | 99 | coords = [deserialize_coord('20/0/0')] 100 | coord_groups = list(qm.group(coords)) 101 | self.assertEquals(0, len(coord_groups)) 102 | 103 | coords = map(deserialize_coord, ['20/0/0', '1/1/1', '16/0/0']) 104 | coord_groups = list(qm.group(coords)) 105 | self.assertEquals(1, len(coord_groups)) 106 | self.assertEquals([deserialize_coord('1/1/1')], coord_groups[0].coords) 107 | self.assertEquals(0, coord_groups[0].queue_id) 108 | 109 | def test_queue_mappings(self): 110 | q1 = object() 111 | q2 = object() 112 | q3 = object() 113 | specs = ( 114 | (None, None, 'tile_queue_1', q1, None), 115 | (0, 10, 'tile_queue_2', q2, None), 116 | (10, 16, 'tile_queue_3', q3, 10), 117 | ) 118 | qm = self.make_queue_mapper(specs) 119 | 120 | q1_id, q2_id, q3_id = range(3) 121 | self.assertIs(q1, qm.get_queue(q1_id)) 122 | self.assertIs(q2, qm.get_queue(q2_id)) 123 | self.assertIs(q3, qm.get_queue(q3_id)) 124 | 125 | ordered_queue_result = list(qm.queues_in_priority_order()) 126 | self.assertEquals(3, len(ordered_queue_result)) 127 | r1_id, r1_q = ordered_queue_result[0] 128 | r2_id, r2_q = ordered_queue_result[1] 129 | r3_id, r3_q = ordered_queue_result[2] 130 | self.assertIs(q1, r1_q) 131 | self.assertEquals(q1_id, r1_id) 132 | self.assertIs(q2, r2_q) 133 | self.assertEquals(q2_id, r2_id) 134 | self.assertIs(q3, r3_q) 135 | self.assertEquals(q3_id, r3_id) 136 | 137 | from tilequeue.tile import deserialize_coord 138 | 139 | # verify that the queue ids line up with those that have zooms 140 | # specified 141 | coord_groups = list(qm.group([deserialize_coord('5/0/0')])) 142 | self.assertEquals(1, len(coord_groups)) 143 | self.assertEquals(1, coord_groups[0].queue_id) 144 | 145 | coord_groups = list(qm.group([deserialize_coord('15/0/0')])) 146 | self.assertEquals(1, len(coord_groups)) 147 | self.assertEquals(2, coord_groups[0].queue_id) 148 | 149 | def test_toi_priority(self): 150 | from tilequeue.queue.mapper import ZoomRangeAndZoomGroupQueueMapper 151 | from tilequeue.queue.mapper import ZoomRangeQueueSpec 152 | from tilequeue.tile import create_coord 153 | 154 | specs = [ 155 | ZoomRangeQueueSpec(0, 10, 'q1', object(), None, in_toi=True), 156 | ZoomRangeQueueSpec(0, 10, 'q2', object(), None, in_toi=False), 157 | ] 158 | 159 | coord_in_toi = create_coord(1, 1, 1) 160 | coord_not_in_toi = create_coord(2, 2, 2) 161 | 162 | class FakeToi(object): 163 | def __init__(self, toi): 164 | self.toi = toi 165 | 166 | def fetch_tiles_of_interest(self): 167 | return self.toi 168 | 169 | toi = FakeToi(set([coord_in_toi])) 170 | mapper = ZoomRangeAndZoomGroupQueueMapper(specs, toi=toi) 171 | 172 | for coord in (coord_in_toi, coord_not_in_toi): 173 | group = list(mapper.group([coord])) 174 | self.assertEquals(1, len(group)) 175 | self.assertEquals(coord == coord_in_toi, group[0].queue_id == 0) 176 | -------------------------------------------------------------------------------- /tests/queue/test_msg.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | 4 | class SingleMessageMarshallerTest(unittest.TestCase): 5 | 6 | def setUp(self): 7 | from tilequeue.queue.message import SingleMessageMarshaller 8 | self.msg_marshaller = SingleMessageMarshaller() 9 | 10 | def test_marshall_empty_list(self): 11 | with self.assertRaises(AssertionError): 12 | self.msg_marshaller.marshall([]) 13 | 14 | def test_marshall_multiple_coords(self): 15 | from tilequeue.tile import deserialize_coord 16 | coords = map(deserialize_coord, ('1/1/1', '2/2/2')) 17 | with self.assertRaises(AssertionError): 18 | self.msg_marshaller.marshall(coords) 19 | 20 | def test_marshall_single_coord(self): 21 | from tilequeue.tile import deserialize_coord 22 | result = self.msg_marshaller.marshall([deserialize_coord('1/1/1')]) 23 | self.assertEqual('1/1/1', result) 24 | 25 | def test_unmarshall_invalid(self): 26 | with self.assertRaises(AssertionError): 27 | self.msg_marshaller.unmarshall('invalid') 28 | 29 | def test_unmarshall_single(self): 30 | from tilequeue.tile import serialize_coord 31 | coords = self.msg_marshaller.unmarshall('1/1/1') 32 | self.assertEqual(1, len(coords)) 33 | self.assertEqual('1/1/1', serialize_coord(coords[0])) 34 | 35 | def test_unmarshall_multiple(self): 36 | with self.assertRaises(AssertionError): 37 | self.msg_marshaller.unmarshall('1/1/1,2/2/2') 38 | 39 | 40 | class MultipleMessageMarshallerTest(unittest.TestCase): 41 | 42 | def setUp(self): 43 | from tilequeue.queue.message import CommaSeparatedMarshaller 44 | self.msg_marshaller = CommaSeparatedMarshaller() 45 | 46 | def test_marshall_empty_list(self): 47 | actual = self.msg_marshaller.marshall([]) 48 | self.assertEqual('', actual) 49 | 50 | def test_marshall_multiple_coords(self): 51 | from tilequeue.tile import deserialize_coord 52 | coords = map(deserialize_coord, ('1/1/1', '2/2/2')) 53 | actual = self.msg_marshaller.marshall(coords) 54 | self.assertEqual('1/1/1,2/2/2', actual) 55 | 56 | def test_marshall_single_coord(self): 57 | from tilequeue.tile import deserialize_coord 58 | result = self.msg_marshaller.marshall([deserialize_coord('1/1/1')]) 59 | self.assertEqual('1/1/1', result) 60 | 61 | def test_unmarshall_invalid(self): 62 | with self.assertRaises(AssertionError): 63 | self.msg_marshaller.unmarshall('invalid') 64 | 65 | def test_unmarshall_empty(self): 66 | actual = self.msg_marshaller.unmarshall('') 67 | self.assertEqual([], actual) 68 | 69 | def test_unmarshall_single(self): 70 | from tilequeue.tile import serialize_coord 71 | coords = self.msg_marshaller.unmarshall('1/1/1') 72 | self.assertEqual(1, len(coords)) 73 | self.assertEqual('1/1/1', serialize_coord(coords[0])) 74 | 75 | def test_unmarshall_multiple(self): 76 | from tilequeue.tile import deserialize_coord 77 | actual = self.msg_marshaller.unmarshall('1/1/1,2/2/2') 78 | self.assertEqual(2, len(actual)) 79 | self.assertEqual(actual[0], deserialize_coord('1/1/1')) 80 | self.assertEqual(actual[1], deserialize_coord('2/2/2')) 81 | 82 | 83 | class SingleMessageTrackerTest(unittest.TestCase): 84 | 85 | def setUp(self): 86 | from tilequeue.queue.message import SingleMessagePerCoordTracker 87 | self.tracker = SingleMessagePerCoordTracker() 88 | 89 | def test_track_and_done(self): 90 | from tilequeue.tile import deserialize_coord 91 | from tilequeue.queue.message import QueueHandle 92 | queue_id = 1 93 | queue_handle = QueueHandle(queue_id, 'handle') 94 | coords = [deserialize_coord('1/1/1')] 95 | coord_handles = self.tracker.track(queue_handle, coords) 96 | self.assertEqual(1, len(coord_handles)) 97 | coord_handle = coord_handles[0] 98 | self.assertIs(queue_handle, coord_handle) 99 | 100 | track_result = self.tracker.done(coord_handle) 101 | self.assertIs(queue_handle, track_result.queue_handle) 102 | self.assertTrue(track_result.all_done) 103 | self.assertFalse(track_result.parent_tile) 104 | 105 | 106 | class MultipleMessageTrackerTest(unittest.TestCase): 107 | 108 | def setUp(self): 109 | from mock import MagicMock 110 | from tilequeue.queue.message import MultipleMessagesPerCoordTracker 111 | msg_tracker_logger = MagicMock() 112 | self.tracker = MultipleMessagesPerCoordTracker(msg_tracker_logger) 113 | 114 | def test_track_and_done_invalid_coord_handle(self): 115 | with self.assertRaises(ValueError): 116 | self.tracker.done('bogus-coord-handle') 117 | 118 | def test_track_and_done(self): 119 | from tilequeue.tile import deserialize_coord 120 | from tilequeue.queue.message import QueueHandle 121 | queue_id = 1 122 | queue_handle = QueueHandle(queue_id, 'handle') 123 | coords = map(deserialize_coord, ('1/1/1', '2/2/2')) 124 | parent_tile = deserialize_coord('1/1/1') 125 | self._assert_track_done(coords, queue_handle, parent_tile) 126 | 127 | def test_track_and_done_not_including_parent(self): 128 | from tilequeue.tile import deserialize_coord 129 | from tilequeue.queue.message import QueueHandle 130 | queue_id = 1 131 | queue_handle = QueueHandle(queue_id, 'handle') 132 | coords = map(deserialize_coord, ('2/2/3', '2/2/2')) 133 | parent_tile = deserialize_coord('1/1/1') 134 | self._assert_track_done(coords, queue_handle, parent_tile) 135 | 136 | def _assert_track_done(self, coords, queue_handle, parent_tile): 137 | coord_handles = self.tracker.track(queue_handle, coords, parent_tile) 138 | self.assertEqual(len(coords), len(coord_handles)) 139 | 140 | # all intermediate coords should not result in a done message. 141 | for coord in coord_handles[:-1]: 142 | track_result = self.tracker.done(coord) 143 | self.assertFalse(track_result.all_done) 144 | 145 | # final coord should complete the tracking 146 | track_result = self.tracker.done(coord_handles[-1]) 147 | self.assertIs(queue_handle, track_result.queue_handle) 148 | self.assertTrue(track_result.all_done) 149 | self.assertEqual(parent_tile, track_result.parent_tile) 150 | 151 | def test_track_and_done_asserts_on_duplicates(self): 152 | from tilequeue.tile import deserialize_coord 153 | from tilequeue.queue.message import QueueHandle 154 | queue_id = 1 155 | queue_handle = QueueHandle(queue_id, 'handle') 156 | coords = map(deserialize_coord, ('2/2/2', '2/2/2')) 157 | parent_tile = deserialize_coord('1/1/1') 158 | 159 | with self.assertRaises(AssertionError): 160 | self.tracker.track(queue_handle, coords, parent_tile) 161 | -------------------------------------------------------------------------------- /tests/queue/test_sqs.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | 4 | class TestQueue(unittest.TestCase): 5 | def setUp(self): 6 | from mock import MagicMock 7 | from tilequeue.queue import SqsQueue 8 | 9 | self.mockClient = MagicMock() 10 | visibility_mgr = None 11 | self.sqs = SqsQueue(self.mockClient, 'queue-url', 10, 20, 12 | visibility_mgr) 13 | 14 | def test_enqueue_batch_adds_tiles(self): 15 | from mock import MagicMock 16 | coords = ['1/1/1', '2/2/2'] 17 | self.mockClient.send_message_batch = MagicMock( 18 | return_value=dict(ResponseMetadata=dict(HTTPStatusCode=200)), 19 | ) 20 | self.sqs.enqueue_batch(coords) 21 | self.mockClient.send_message_batch.assert_called_with( 22 | QueueUrl='queue-url', 23 | Entries=[ 24 | {'Id': '0', 'MessageBody': '1/1/1'}, 25 | {'Id': '1', 'MessageBody': '2/2/2'}]) 26 | 27 | def test_enqueue_should_write_message_to_queue(self): 28 | from mock import MagicMock 29 | self.mockClient.send = MagicMock( 30 | return_value=dict(ResponseMetadata=dict(HTTPStatusCode=200)), 31 | ) 32 | self.sqs.enqueue('1/1/1') 33 | self.mockClient.send.assert_called_with( 34 | MessageBody='1/1/1', QueueUrl='queue-url') 35 | -------------------------------------------------------------------------------- /tests/queue/test_writer.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | 4 | class QueueWriterTest(unittest.TestCase): 5 | 6 | def make_queue_writer(self): 7 | from mock import MagicMock 8 | from tilequeue.queue.inflight import NoopInFlightManager 9 | from tilequeue.queue.mapper import SingleQueueMapper 10 | from tilequeue.queue.message import SingleMessageMarshaller 11 | from tilequeue.queue.writer import QueueWriter 12 | 13 | queue = MagicMock() 14 | queue_mapper = SingleQueueMapper('queue_name', queue) 15 | msg_marshaller = SingleMessageMarshaller() 16 | inflight_mgr = NoopInFlightManager() 17 | enqueue_batch_size = 10 18 | queue_writer = QueueWriter( 19 | queue_mapper, msg_marshaller, inflight_mgr, enqueue_batch_size) 20 | return queue_writer 21 | 22 | def test_write_coords(self): 23 | from tilequeue.tile import deserialize_coord 24 | coords = [deserialize_coord('1/1/1'), deserialize_coord('15/1/1')] 25 | queue_writer = self.make_queue_writer() 26 | n_enqueued, n_inflight = queue_writer.enqueue_batch(coords) 27 | self.assertEquals(2, n_enqueued) 28 | self.assertEquals(0, n_inflight) 29 | -------------------------------------------------------------------------------- /tests/test_buffer.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | 4 | class QueryBoundsTest(unittest.TestCase): 5 | 6 | def _call_fut(self, bounds, layer_name, buffer_cfg): 7 | from tilequeue.config import create_query_bounds_pad_fn 8 | fn = create_query_bounds_pad_fn(buffer_cfg, layer_name) 9 | result = fn(bounds, 1) 10 | return result 11 | 12 | def test_no_bounds(self): 13 | bounds = (0, 0, 1, 1) 14 | result = self._call_fut(bounds, 'foo', {}) 15 | self.assertEquals(bounds, result['point']) 16 | 17 | def test_layer_not_configured(self): 18 | bounds = (0, 0, 1, 1) 19 | buffer_cfg = dict(foo={}) 20 | result = self._call_fut(bounds, 'baz', buffer_cfg) 21 | self.assertEquals(bounds, result['point']) 22 | 23 | def test_layer_configured(self): 24 | bounds = (1, 1, 2, 2) 25 | buffer_cfg = { 26 | 'fmt': { 27 | 'layer': { 28 | 'foo': { 29 | 'point': 1 30 | } 31 | } 32 | } 33 | } 34 | result = self._call_fut(bounds, 'foo', buffer_cfg) 35 | exp_bounds = (0, 0, 3, 3) 36 | self.assertEquals(result['point'], exp_bounds) 37 | 38 | def test_geometry_configured(self): 39 | bounds = (1, 1, 2, 2) 40 | buffer_cfg = { 41 | 'fmt': { 42 | 'geometry': { 43 | 'line': 1 44 | } 45 | } 46 | } 47 | result = self._call_fut(bounds, 'foo', buffer_cfg) 48 | exp_bounds = (0, 0, 3, 3) 49 | self.assertEquals(result['line'], exp_bounds) 50 | 51 | def test_layer_trumps_geometry(self): 52 | bounds = (2, 2, 3, 3) 53 | buffer_cfg = { 54 | 'fmt': { 55 | 'layer': { 56 | 'foo': { 57 | 'polygon': 2 58 | } 59 | }, 60 | 'geometry': { 61 | 'polygon': 1 62 | } 63 | } 64 | } 65 | result = self._call_fut(bounds, 'foo', buffer_cfg) 66 | exp_bounds = (0, 0, 5, 5) 67 | self.assertEquals(result['polygon'], exp_bounds) 68 | 69 | def test_multiple_bounds(self): 70 | bounds = (2, 2, 3, 3) 71 | buffer_cfg = { 72 | 'fmt': { 73 | 'layer': { 74 | 'foo': { 75 | 'point': 1, 76 | 'polygon': 2, 77 | } 78 | } 79 | } 80 | } 81 | result = self._call_fut(bounds, 'foo', buffer_cfg) 82 | 83 | exp_bounds_poly = (0, 0, 5, 5) 84 | self.assertEquals(result['polygon'], exp_bounds_poly) 85 | 86 | exp_bounds_point = (1, 1, 4, 4) 87 | self.assertEquals(result['point'], exp_bounds_point) 88 | 89 | 90 | class ClipBoundsTest(unittest.TestCase): 91 | 92 | def _call_fut( 93 | self, bounds, ext, layer_name, geometry_type, meters_per_pixel, 94 | buffer_cfg): 95 | from tilequeue.transform import calc_buffered_bounds 96 | format = type(ext, (), dict(extension=ext)) 97 | result = calc_buffered_bounds( 98 | format, bounds, meters_per_pixel, layer_name, geometry_type, 99 | buffer_cfg) 100 | return result 101 | 102 | def test_empty(self): 103 | bounds = (1, 1, 2, 2) 104 | result = self._call_fut(bounds, 'foo', 'bar', 'Point', 1, {}) 105 | self.assertEquals(result, bounds) 106 | 107 | def test_diff_format(self): 108 | bounds = (1, 1, 2, 2) 109 | ext = 'quux' 110 | result = self._call_fut(bounds, ext, 'bar', 'Point', 1, dict(foo=42)) 111 | self.assertEquals(result, bounds) 112 | 113 | def test_layer_match(self): 114 | bounds = (1, 1, 2, 2) 115 | ext = 'fmt' 116 | layer_name = 'foo' 117 | buffer_cfg = { 118 | ext: { 119 | 'layer': { 120 | layer_name: { 121 | 'line': 1 122 | } 123 | } 124 | } 125 | } 126 | result = self._call_fut( 127 | bounds, ext, layer_name, 'LineString', 1, buffer_cfg) 128 | exp_bounds = (0, 0, 3, 3) 129 | self.assertEquals(result, exp_bounds) 130 | 131 | def test_geometry_match(self): 132 | bounds = (1, 1, 2, 2) 133 | ext = 'fmt' 134 | layer_name = 'foo' 135 | buffer_cfg = { 136 | ext: { 137 | 'geometry': { 138 | 'line': 1 139 | } 140 | } 141 | } 142 | result = self._call_fut( 143 | bounds, ext, layer_name, 'LineString', 1, buffer_cfg) 144 | exp_bounds = (0, 0, 3, 3) 145 | self.assertEquals(result, exp_bounds) 146 | 147 | def test_layer_trumps_geometry(self): 148 | bounds = (2, 2, 3, 3) 149 | ext = 'fmt' 150 | layer_name = 'foo' 151 | buffer_cfg = { 152 | ext: { 153 | 'layer': { 154 | layer_name: { 155 | 'line': 2 156 | } 157 | }, 158 | 'geometry': { 159 | 'line': 1 160 | } 161 | } 162 | } 163 | result = self._call_fut( 164 | bounds, ext, layer_name, 'LineString', 1, buffer_cfg) 165 | exp_bounds = (0, 0, 5, 5) 166 | self.assertEquals(result, exp_bounds) 167 | 168 | def test_multiple_layer_geometry_types(self): 169 | bounds = (2, 2, 3, 3) 170 | ext = 'fmt' 171 | layer_name = 'foo' 172 | buffer_cfg = { 173 | ext: { 174 | 'layer': { 175 | layer_name: { 176 | 'point': 1, 177 | 'line': 2, 178 | 'polygon': 3, 179 | } 180 | } 181 | } 182 | } 183 | result = self._call_fut( 184 | bounds, ext, layer_name, 'LineString', 1, buffer_cfg) 185 | exp_bounds = (0, 0, 5, 5) 186 | self.assertEquals(result, exp_bounds) 187 | 188 | def test_multi_geometry(self): 189 | bounds = (1, 1, 2, 2) 190 | ext = 'fmt' 191 | layer_name = 'foo' 192 | buffer_cfg = { 193 | ext: { 194 | 'geometry': { 195 | 'polygon': 1 196 | } 197 | } 198 | } 199 | result = self._call_fut(bounds, ext, layer_name, 'MultiPolygon', 1, 200 | buffer_cfg) 201 | exp_bounds = (0, 0, 3, 3) 202 | self.assertEquals(result, exp_bounds) 203 | 204 | def test_geometry_no_match(self): 205 | bounds = (1, 1, 2, 2) 206 | ext = 'fmt' 207 | layer_name = 'foo' 208 | buffer_cfg = { 209 | ext: { 210 | 'geometry': { 211 | 'polygon': 1 212 | } 213 | } 214 | } 215 | result = self._call_fut( 216 | bounds, ext, layer_name, 'LineString', 1, buffer_cfg) 217 | self.assertEquals(result, bounds) 218 | 219 | def test_meters_per_pixel(self): 220 | bounds = (2, 2, 3, 3) 221 | ext = 'fmt' 222 | layer_name = 'foo' 223 | meters_per_pixel = 2 224 | buffer_cfg = { 225 | ext: { 226 | 'geometry': { 227 | 'line': 1 228 | } 229 | } 230 | } 231 | result = self._call_fut( 232 | bounds, ext, layer_name, 'LineString', meters_per_pixel, 233 | buffer_cfg) 234 | exp_bounds = (0, 0, 5, 5) 235 | self.assertEquals(result, exp_bounds) 236 | 237 | 238 | class MetersPerPixelDimTest(unittest.TestCase): 239 | 240 | def _call_fut(self, zoom): 241 | from tilequeue.tile import calc_meters_per_pixel_dim 242 | result = calc_meters_per_pixel_dim(zoom) 243 | return result 244 | 245 | def test_z10(self): 246 | meters_per_pixel_dim = self._call_fut(10) 247 | self.assertAlmostEquals(152.746093811, meters_per_pixel_dim) 248 | 249 | def test_compare_z10_bounds(self): 250 | from tilequeue.tile import coord_to_mercator_bounds 251 | from tilequeue.tile import deserialize_coord 252 | 253 | coord = deserialize_coord('10/100/100') 254 | merc_bounds = coord_to_mercator_bounds(coord) 255 | tile_meters_wide = merc_bounds[2] - merc_bounds[0] 256 | exp_meters_per_pixel_dim = tile_meters_wide / 256 257 | 258 | act_meters_per_pixel_dim = self._call_fut(10) 259 | self.assertAlmostEquals( 260 | exp_meters_per_pixel_dim, act_meters_per_pixel_dim, places=0) 261 | 262 | 263 | class MetersPerPixelAreaTest(unittest.TestCase): 264 | 265 | def _call_fut(self, zoom): 266 | from tilequeue.tile import calc_meters_per_pixel_area 267 | result = calc_meters_per_pixel_area(zoom) 268 | return result 269 | 270 | def test_z10(self): 271 | meters_per_pixel_area = self._call_fut(10) 272 | self.assertAlmostEquals(23331.3691744, meters_per_pixel_area) 273 | 274 | def test_z10_compare_with_dim(self): 275 | from tilequeue.tile import calc_meters_per_pixel_dim 276 | 277 | meters_per_pixel_dim = calc_meters_per_pixel_dim(10) 278 | meters_per_pixel_area = self._call_fut(10) 279 | exp_meters_per_pixel_area = ( 280 | meters_per_pixel_dim * meters_per_pixel_dim) 281 | self.assertAlmostEquals( 282 | exp_meters_per_pixel_area, meters_per_pixel_area) 283 | -------------------------------------------------------------------------------- /tests/test_command.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | 4 | class tempdir(object): 5 | 6 | def __enter__(self): 7 | import tempfile 8 | self.tempdir = tempfile.mkdtemp() 9 | return self.tempdir 10 | 11 | def __exit__(self, type, value, traceback): 12 | import shutil 13 | shutil.rmtree(self.tempdir) 14 | 15 | 16 | class TestUniquifyGenerator(unittest.TestCase): 17 | 18 | enqueued_list = set() 19 | 20 | def fake_enqueue(self, value): 21 | self.enqueued_list.add(value) 22 | 23 | def fake_enqueue_batch(self, values): 24 | n = 0 25 | for value in values: 26 | n += 1 27 | self.fake_enqueue(value) 28 | return n, 0 29 | 30 | def test_uniquify_generator(self): 31 | from tilequeue.command import uniquify_generator 32 | from itertools import cycle, islice, tee 33 | gen = islice(cycle(xrange(5)), 10) 34 | gen, gencopy = tee(gen) 35 | uniqued_gen = uniquify_generator(gencopy) 36 | self.assertEqual(range(5) + range(5), list(gen)) 37 | self.assertEqual(range(5), list(uniqued_gen)) 38 | 39 | def test_tilequeue_explode_and_intersect(self): 40 | from tilequeue.command import explode_and_intersect 41 | from tilequeue.tile import coord_marshall_int 42 | from tilequeue.tile import coord_unmarshall_int 43 | from ModestMaps.Core import Coordinate 44 | sample_coord = Coordinate(zoom=14, column=250, row=250) 45 | sample_coord_int = coord_marshall_int(sample_coord) 46 | tiles_of_interest = [sample_coord_int] 47 | for i in (10, 11, 12, 13): 48 | coord = sample_coord.zoomTo(i) 49 | coord_int = coord_marshall_int(coord) 50 | tiles_of_interest.append(coord_int) 51 | exploded, metrics = explode_and_intersect( 52 | [sample_coord_int], tiles_of_interest, until=11) 53 | coord_ints = list(exploded) 54 | for coord_int in coord_ints: 55 | coord = coord_unmarshall_int(coord_int) 56 | self.failUnless(coord.zoom > 10) 57 | 58 | self.assertEqual(4, len(coord_ints)) 59 | 60 | self.assertEqual(4, metrics['hits']) 61 | self.assertEqual(0, metrics['misses']) 62 | self.assertEqual(4, metrics['total']) 63 | 64 | 65 | class ZoomToQueueNameMapTest(unittest.TestCase): 66 | 67 | def test_bad_map(self): 68 | from tilequeue.command import make_get_queue_name_for_zoom 69 | zoom_queue_map_cfg = {'badzoom-20': 'q1'} 70 | queue_names = ['q1'] 71 | with self.assertRaises(AssertionError): 72 | make_get_queue_name_for_zoom(zoom_queue_map_cfg, queue_names) 73 | 74 | def test_single_queue_name_for_zoom(self): 75 | from tilequeue.command import make_get_queue_name_for_zoom 76 | zoom_queue_map_cfg = {'0-20': 'q1'} 77 | queue_names = ['q1'] 78 | get_queue = make_get_queue_name_for_zoom( 79 | zoom_queue_map_cfg, queue_names) 80 | zoom = 7 81 | queue_name = get_queue(zoom) 82 | self.assertEqual(queue_name, 'q1') 83 | 84 | def test_multiple_queues(self): 85 | from tilequeue.command import make_get_queue_name_for_zoom 86 | zoom_queue_map_cfg = {'0-5': 'q1', '6-20': 'q2'} 87 | queue_names = ['q1', 'q2'] 88 | get_queue = make_get_queue_name_for_zoom( 89 | zoom_queue_map_cfg, queue_names) 90 | 91 | zoom = 5 92 | queue_name = get_queue(zoom) 93 | self.assertEqual(queue_name, 'q1') 94 | 95 | zoom = 15 96 | queue_name = get_queue(zoom) 97 | self.assertEqual(queue_name, 'q2') 98 | 99 | def test_missing_queue_name(self): 100 | from tilequeue.command import make_get_queue_name_for_zoom 101 | zoom_queue_map_cfg = {'0-5': 'q1', '6-20': 'q3'} 102 | queue_names = ['q1', 'q2'] 103 | with self.assertRaises(AssertionError): 104 | make_get_queue_name_for_zoom(zoom_queue_map_cfg, queue_names) 105 | 106 | def test_overlapping_queue_names(self): 107 | from tilequeue.command import make_get_queue_name_for_zoom 108 | zoom_queue_map_cfg = {'0-5': 'q1', '4-20': 'q2'} 109 | queue_names = ['q1', 'q2'] 110 | with self.assertRaises(AssertionError): 111 | make_get_queue_name_for_zoom(zoom_queue_map_cfg, queue_names) 112 | 113 | def test_zoom_invalid_lookup(self): 114 | from tilequeue.command import make_get_queue_name_for_zoom 115 | zoom_queue_map_cfg = {'0-20': 'q1'} 116 | queue_names = ['q1'] 117 | get_queue = make_get_queue_name_for_zoom( 118 | zoom_queue_map_cfg, queue_names) 119 | zoom = 21 120 | with self.assertRaises(AssertionError): 121 | get_queue(zoom) 122 | 123 | def test_zoom_out_of_range(self): 124 | from tilequeue.command import make_get_queue_name_for_zoom 125 | zoom_queue_map_cfg = {'0-5': 'q1'} 126 | queue_names = ['q1'] 127 | get_queue = make_get_queue_name_for_zoom( 128 | zoom_queue_map_cfg, queue_names) 129 | zoom = 7 130 | with self.assertRaises(AssertionError): 131 | get_queue(zoom) 132 | 133 | def test_zoom_is_long(self): 134 | # the zoom (or row/col) in a Coordinate can be a long simply because 135 | # the coordinate it was derived from in unmarshall_coord_int was a 136 | # long. 137 | from tilequeue.command import make_get_queue_name_for_zoom 138 | zoom_queue_map_cfg = {'0-20': 'q1'} 139 | queue_names = ['q1'] 140 | get_queue = make_get_queue_name_for_zoom( 141 | zoom_queue_map_cfg, queue_names) 142 | zoom = long(7) 143 | queue_name = get_queue(zoom) 144 | self.assertEqual(queue_name, 'q1') 145 | -------------------------------------------------------------------------------- /tests/test_config.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | 4 | class TestConfigMerge(unittest.TestCase): 5 | 6 | def _call_fut(self, destcfg, srccfg): 7 | from tilequeue.config import merge_cfg 8 | return merge_cfg(destcfg, srccfg) 9 | 10 | def test_both_empty(self): 11 | self.assertEqual({}, self._call_fut({}, {})) 12 | 13 | def test_complementary_scalar(self): 14 | src = dict(foo='bar') 15 | dest = dict(quux='morx') 16 | self.assertEqual(dict(foo='bar', quux='morx'), 17 | self._call_fut(dest, src)) 18 | 19 | def test_nested_complementary(self): 20 | src = dict(foo=dict(bar='baz')) 21 | dest = dict(quux=dict(morx='fleem')) 22 | self.assertEqual( 23 | dict(foo=dict(bar='baz'), 24 | quux=dict(morx='fleem')), 25 | self._call_fut(dest, src)) 26 | 27 | def test_merge_complementary(self): 28 | src = dict(foo=dict(bar='baz')) 29 | dest = dict(foo=dict(morx='fleem')) 30 | self.assertEqual( 31 | dict(foo=dict(bar='baz', morx='fleem')), 32 | self._call_fut(dest, src)) 33 | 34 | def test_merge_override(self): 35 | src = dict(foo=dict(bar='baz')) 36 | dest = dict(foo=dict(bar='fleem')) 37 | self.assertEqual( 38 | dict(foo=dict(bar='baz')), 39 | self._call_fut(dest, src)) 40 | 41 | 42 | class TestCliConfiguration(unittest.TestCase): 43 | 44 | def _call_fut(self, config_dict): 45 | from tilequeue.config import make_config_from_argparse 46 | from yaml import dump 47 | from cStringIO import StringIO 48 | raw_yaml = dump(config_dict) 49 | raw_yaml_file_obj = StringIO(raw_yaml) 50 | return make_config_from_argparse(raw_yaml_file_obj) 51 | 52 | def _assert_cfg(self, cfg, to_check): 53 | # cfg is the config object to validate 54 | # to_check is a dict of key, values to check in cfg 55 | for k, v in to_check.items(): 56 | cfg_val = getattr(cfg, k) 57 | self.assertEqual(v, cfg_val) 58 | 59 | def test_no_config(self): 60 | cfg = self._call_fut(dict(config=None)) 61 | # just assert some of the defaults are set 62 | self._assert_cfg(cfg, 63 | dict(s3_path='osm', 64 | output_formats=['json'], 65 | seed_all_zoom_start=None, 66 | seed_all_zoom_until=None)) 67 | 68 | def test_config_osm_path_modified(self): 69 | cfg = self._call_fut( 70 | dict(store=dict(path='custompath'))) 71 | self._assert_cfg(cfg, 72 | dict(s3_path='custompath', 73 | output_formats=['json'], 74 | seed_all_zoom_start=None, 75 | seed_all_zoom_until=None)) 76 | 77 | 78 | class TestMetatileConfiguration(unittest.TestCase): 79 | 80 | def _call_fut(self, config_dict): 81 | from tilequeue.config import make_config_from_argparse 82 | from yaml import dump 83 | from cStringIO import StringIO 84 | raw_yaml = dump(config_dict) 85 | raw_yaml_file_obj = StringIO(raw_yaml) 86 | return make_config_from_argparse(raw_yaml_file_obj) 87 | 88 | def test_metatile_size_default(self): 89 | config_dict = {} 90 | cfg = self._call_fut(config_dict) 91 | self.assertIsNone(cfg.metatile_size) 92 | self.assertEquals(cfg.metatile_zoom, 0) 93 | self.assertEquals(cfg.tile_sizes, [256]) 94 | 95 | def test_metatile_size_1(self): 96 | config_dict = dict(metatile=dict(size=1)) 97 | cfg = self._call_fut(config_dict) 98 | self.assertEquals(cfg.metatile_size, 1) 99 | self.assertEquals(cfg.metatile_zoom, 0) 100 | self.assertEquals(cfg.tile_sizes, [256]) 101 | 102 | def test_metatile_size_2(self): 103 | config_dict = dict(metatile=dict(size=2)) 104 | cfg = self._call_fut(config_dict) 105 | self.assertEquals(cfg.metatile_size, 2) 106 | self.assertEquals(cfg.metatile_zoom, 1) 107 | self.assertEquals(cfg.tile_sizes, [512, 256]) 108 | 109 | def test_metatile_size_4(self): 110 | config_dict = dict(metatile=dict(size=4)) 111 | cfg = self._call_fut(config_dict) 112 | self.assertEquals(cfg.metatile_size, 4) 113 | self.assertEquals(cfg.metatile_zoom, 2) 114 | self.assertEquals(cfg.tile_sizes, [1024, 512, 256]) 115 | 116 | def test_max_zoom(self): 117 | config_dict = dict(metatile=dict(size=2)) 118 | cfg = self._call_fut(config_dict) 119 | self.assertEquals(cfg.max_zoom, 15) 120 | -------------------------------------------------------------------------------- /tests/test_metatile.py: -------------------------------------------------------------------------------- 1 | import cStringIO as StringIO 2 | import unittest 3 | import zipfile 4 | 5 | from ModestMaps.Core import Coordinate 6 | 7 | from tilequeue.format import json_format 8 | from tilequeue.format import topojson_format 9 | from tilequeue.format import zip_format 10 | from tilequeue.metatile import extract_metatile 11 | from tilequeue.metatile import make_metatiles 12 | 13 | 14 | class TestMetatile(unittest.TestCase): 15 | 16 | def test_make_metatiles_single(self): 17 | json = "{\"json\":true}" 18 | tiles = [dict(tile=json, coord=Coordinate(0, 0, 0), 19 | format=json_format, layer='all')] 20 | metatiles = make_metatiles(1, tiles) 21 | self.assertEqual(1, len(metatiles)) 22 | self.assertEqual(Coordinate(0, 0, 0), metatiles[0]['coord']) 23 | self.assertEqual('all', metatiles[0]['layer']) 24 | self.assertEqual(zip_format, metatiles[0]['format']) 25 | buf = StringIO.StringIO(metatiles[0]['tile']) 26 | with zipfile.ZipFile(buf, mode='r') as z: 27 | self.assertEqual(json, z.open('0/0/0.json').read()) 28 | 29 | def test_make_metatiles_multiple(self): 30 | json = "{\"json\":true}" 31 | tiles = [ 32 | dict(tile=json, coord=Coordinate(0, 0, 0), 33 | format=json_format, layer='all'), 34 | dict(tile=json, coord=Coordinate(0, 0, 0), 35 | format=topojson_format, layer='all'), 36 | ] 37 | 38 | metatiles = make_metatiles(1, tiles) 39 | self.assertEqual(1, len(metatiles)) 40 | self.assertEqual(Coordinate(0, 0, 0), metatiles[0]['coord']) 41 | self.assertEqual('all', metatiles[0]['layer']) 42 | self.assertEqual(zip_format, metatiles[0]['format']) 43 | buf = StringIO.StringIO(metatiles[0]['tile']) 44 | with zipfile.ZipFile(buf, mode='r') as z: 45 | self.assertEqual(json, z.open('0/0/0.json').read()) 46 | self.assertEqual(json, z.open('0/0/0.topojson').read()) 47 | 48 | def test_make_metatiles_multiple_coordinates(self): 49 | # checks that we can make a single metatile which contains multiple 50 | # coordinates. this is used for "512px" tiles as well as cutting out 51 | # z17+ tiles and storing them in the z16 (z15 if "512px") metatile. 52 | 53 | json = "{\"json\":true}" 54 | tiles = [ 55 | # NOTE: coordinates are (y, x, z), possibly the most confusing 56 | # possible permutation. 57 | dict(tile=json, coord=Coordinate(123, 456, 17), 58 | format=json_format, layer='all'), 59 | dict(tile=json, coord=Coordinate(123, 457, 17), 60 | format=json_format, layer='all'), 61 | ] 62 | 63 | metatiles = make_metatiles(1, tiles) 64 | self.assertEqual(1, len(metatiles)) 65 | meta = metatiles[0] 66 | 67 | # NOTE: Coordinate(y, x, z) 68 | coord = Coordinate(61, 228, 16) 69 | self.assertEqual(meta['coord'], coord) 70 | 71 | self.assertEqual('all', meta['layer']) 72 | self.assertEqual(zip_format, meta['format']) 73 | buf = StringIO.StringIO(meta['tile']) 74 | with zipfile.ZipFile(buf, mode='r') as z: 75 | self.assertEqual(json, z.open('1/0/1.json').read()) 76 | self.assertEqual(json, z.open('1/1/1.json').read()) 77 | 78 | # test extracting as well 79 | self.assertEqual(json, extract_metatile( 80 | buf, json_format, offset=Coordinate(zoom=1, column=0, row=1))) 81 | self.assertEqual(json, extract_metatile( 82 | buf, json_format, offset=Coordinate(zoom=1, column=1, row=1))) 83 | 84 | def test_extract_metatiles_single(self): 85 | json = "{\"json\":true}" 86 | tile = dict(tile=json, coord=Coordinate(0, 0, 0), 87 | format=json_format, layer='all') 88 | metatiles = make_metatiles(1, [tile]) 89 | self.assertEqual(1, len(metatiles)) 90 | buf = StringIO.StringIO(metatiles[0]['tile']) 91 | extracted = extract_metatile(buf, json_format) 92 | self.assertEqual(json, extracted) 93 | 94 | def test_metatile_file_timing(self): 95 | from time import gmtime, time 96 | from tilequeue.metatile import metatiles_are_equal 97 | 98 | # tilequeue's "GET before PUT" optimisation relies on being able to 99 | # fetch a tile from S3 and compare it to the one that was just 100 | # generated. to do this, we should try to make the tiles as similar 101 | # as possible across multiple runs. 102 | 103 | json = "{\"json\":true}" 104 | tiles = [dict(tile=json, coord=Coordinate(0, 0, 0), 105 | format=json_format, layer='all')] 106 | 107 | when_will_then_be_now = 10 108 | t = time() 109 | now = gmtime(t)[0:6] 110 | then = gmtime(t - when_will_then_be_now)[0:6] 111 | 112 | metatile_1 = make_metatiles(1, tiles, then) 113 | metatile_2 = make_metatiles(1, tiles, now) 114 | 115 | self.assertTrue(metatiles_are_equal( 116 | metatile_1[0]['tile'], metatile_2[0]['tile'])) 117 | 118 | def test_metatile_common_parent(self): 119 | from tilequeue.metatile import common_parent 120 | 121 | def tile(z, x, y): 122 | return Coordinate(zoom=z, column=x, row=y) 123 | 124 | self.assertEqual( 125 | tile(0, 0, 0), 126 | common_parent(tile(1, 1, 1), tile(1, 0, 0))) 127 | 128 | self.assertEqual( 129 | tile(0, 0, 0), 130 | common_parent(tile(0, 0, 0), tile(1, 0, 0))) 131 | self.assertEqual( 132 | tile(0, 0, 0), 133 | common_parent(tile(1, 0, 0), tile(0, 0, 0))) 134 | 135 | self.assertEqual( 136 | tile(0, 0, 0), 137 | common_parent(tile(2, 0, 0), tile(2, 3, 3))) 138 | self.assertEqual( 139 | tile(0, 0, 0), 140 | common_parent(tile(3, 0, 0), tile(3, 7, 7))) 141 | self.assertEqual( 142 | tile(0, 0, 0), 143 | common_parent(tile(4, 0, 0), tile(4, 15, 15))) 144 | 145 | self.assertEqual( 146 | tile(0, 0, 0), 147 | common_parent(tile(3, 3, 3), tile(2, 3, 3))) 148 | self.assertEqual( 149 | tile(0, 0, 0), 150 | common_parent(tile(4, 7, 7), tile(3, 7, 7))) 151 | self.assertEqual( 152 | tile(0, 0, 0), 153 | common_parent(tile(5, 15, 15), tile(4, 15, 15))) 154 | 155 | self.assertEqual( 156 | tile(1, 1, 1), 157 | common_parent(tile(3, 4, 4), tile(2, 3, 3))) 158 | self.assertEqual( 159 | tile(1, 1, 1), 160 | common_parent(tile(4, 8, 8), tile(3, 7, 7))) 161 | self.assertEqual( 162 | tile(1, 1, 1), 163 | common_parent(tile(5, 16, 16), tile(4, 15, 15))) 164 | -------------------------------------------------------------------------------- /tests/test_metro_extract.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | 4 | class TestMetroExtractParse(unittest.TestCase): 5 | 6 | def _call_fut(self, fp): 7 | from tilequeue.metro_extract import parse_metro_extract 8 | return parse_metro_extract(fp) 9 | 10 | def test_invalid_json(self): 11 | from cStringIO import StringIO 12 | from tilequeue.metro_extract import MetroExtractParseError 13 | fp = StringIO('{"foo": "bar"}') 14 | try: 15 | self._call_fut(fp) 16 | except MetroExtractParseError: 17 | # expecting error to be raised 18 | pass 19 | else: 20 | self.fail('Expected MetroExtractParseError to be raised') 21 | 22 | def _generate_stub(self): 23 | return dict( 24 | regions=dict( 25 | region1=dict( 26 | cities=dict( 27 | city1=self._city_bounds(1, 1, 2, 2), 28 | city2=self._city_bounds(3, 3, 4, 4), 29 | ) 30 | ) 31 | ) 32 | ) 33 | 34 | def _city_bounds(self, minx, miny, maxx, maxy): 35 | return dict( 36 | bbox=dict( 37 | left=str(minx), 38 | right=str(maxx), 39 | top=str(maxy), 40 | bottom=str(miny), 41 | ) 42 | ) 43 | 44 | def test_valid_parse(self): 45 | from json import dumps 46 | stub = self._generate_stub() 47 | from cStringIO import StringIO 48 | fp = StringIO(dumps(stub)) 49 | results = self._call_fut(fp) 50 | self.assertEqual(2, len(results)) 51 | results.sort(key=lambda x: x.city) 52 | city1, city2 = results 53 | 54 | self.assertEqual('region1', city1.region) 55 | self.assertEqual('city1', city1.city) 56 | self.assertEqual((1, 1, 2, 2), city1.bounds) 57 | 58 | self.assertEqual('region1', city2.region) 59 | self.assertEqual('city2', city2.city) 60 | self.assertEqual((3, 3, 4, 4), city2.bounds) 61 | 62 | def test_city_bounds(self): 63 | from json import dumps 64 | stub = self._generate_stub() 65 | from cStringIO import StringIO 66 | fp = StringIO(dumps(stub)) 67 | results = self._call_fut(fp) 68 | self.assertEqual(2, len(results)) 69 | results.sort(key=lambda x: x.city) 70 | 71 | from tilequeue.metro_extract import city_bounds 72 | bounds = city_bounds(results) 73 | self.assertEqual(2, len(bounds)) 74 | bounds1, bounds2 = bounds 75 | self.assertEqual((1, 1, 2, 2), bounds1) 76 | self.assertEqual((3, 3, 4, 4), bounds2) 77 | -------------------------------------------------------------------------------- /tests/test_mvt.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | 4 | class MapboxVectorTileTest(unittest.TestCase): 5 | 6 | def _make_tiles(self, shape, coord, metatile_zoom): 7 | from tilequeue.format import mvt_format 8 | from tilequeue.process import process_coord 9 | from tilequeue.tile import coord_children_range 10 | from tilequeue.tile import coord_to_mercator_bounds 11 | 12 | db_features = [dict( 13 | __id__=1, 14 | __geometry__=shape.wkb, 15 | __properties__={}, 16 | )] 17 | 18 | nominal_zoom = coord.zoom + metatile_zoom 19 | unpadded_bounds = coord_to_mercator_bounds(coord) 20 | feature_layers = [dict( 21 | layer_datum=dict( 22 | name='fake_layer', 23 | geometry_types=[shape.geom_type], 24 | transform_fn_names=[], 25 | sort_fn_name=None, 26 | is_clipped=False 27 | ), 28 | padded_bounds={shape.geom_type.lower(): unpadded_bounds}, 29 | features=db_features 30 | )] 31 | formats = [mvt_format] 32 | 33 | post_process_data = {} 34 | buffer_cfg = {} 35 | cut_coords = [coord] 36 | if nominal_zoom > coord.zoom: 37 | cut_coords.extend(coord_children_range(coord, nominal_zoom)) 38 | 39 | def _output_fn(shape, props, fid, meta): 40 | return dict(fake='data', min_zoom=0) 41 | 42 | output_calc_mapping = dict(fake_layer=_output_fn) 43 | tiles, extra = process_coord( 44 | coord, nominal_zoom, feature_layers, post_process_data, formats, 45 | unpadded_bounds, cut_coords, buffer_cfg, output_calc_mapping) 46 | 47 | self.assertEqual(len(cut_coords), len(tiles)) 48 | return tiles, cut_coords 49 | 50 | def _check_metatile(self, metatile_size): 51 | from mock import patch 52 | from shapely.geometry import box 53 | from ModestMaps.Core import Coordinate 54 | from tilequeue.tile import coord_to_mercator_bounds 55 | from tilequeue.tile import metatile_zoom_from_size 56 | 57 | name = 'tilequeue.format.mvt.mvt_encode' 58 | with patch(name, return_value='') as encode: 59 | coord = Coordinate(0, 0, 0) 60 | bounds = coord_to_mercator_bounds(coord) 61 | pixel_fraction = 1.0 / 4096.0 62 | box_width = pixel_fraction * (bounds[2] - bounds[0]) 63 | box_height = pixel_fraction * (bounds[3] - bounds[1]) 64 | shape = box(bounds[0], bounds[1], 65 | bounds[0] + box_width, 66 | bounds[1] + box_height) 67 | 68 | metatile_zoom = metatile_zoom_from_size(metatile_size) 69 | tiles, tile_coords = self._make_tiles(shape, coord, metatile_zoom) 70 | 71 | num_tiles = 0 72 | for z in range(0, metatile_zoom + 1): 73 | num_tiles += 4**z 74 | 75 | # resolution should be 4096 at 256px, which is metatile_zoom 76 | # levels down from the extent of the world. 77 | resolution = (bounds[2] - bounds[0]) / (4096 * 2**metatile_zoom) 78 | 79 | self.assertEqual(num_tiles, len(tiles)) 80 | self.assertEqual(num_tiles, encode.call_count) 81 | for (posargs, kwargs), coord in zip(encode.call_args_list, 82 | tile_coords): 83 | self.assertIn('quantize_bounds', kwargs) 84 | # We hardcoded the extent to be 4096 in 85 | # https://github.com/tilezen/tilequeue/pull/404 86 | # thus the extent calculation is thus commented out 87 | # quantize_bounds = kwargs['quantize_bounds'] 88 | # extent = int(round((quantize_bounds[2] - quantize_bounds[0]) / 89 | # resolution)) 90 | extent = 4096 91 | self.assertIn('extents', kwargs) 92 | actual_extent = kwargs['extents'] 93 | self.assertEquals(extent, actual_extent, 94 | 'Expected %r, not %r, for coord %r' % 95 | (extent, actual_extent, coord)) 96 | 97 | def test_single_tile(self): 98 | self._check_metatile(1) 99 | 100 | def test_metatile_size_2(self): 101 | self._check_metatile(2) 102 | 103 | def test_metatile_size_4(self): 104 | self._check_metatile(4) 105 | -------------------------------------------------------------------------------- /tests/test_query_common.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | 4 | class TestCommon(unittest.TestCase): 5 | 6 | def test_parse_shape_types(self): 7 | from tilequeue.query.common import ShapeType 8 | 9 | def _test(expects, inputs): 10 | self.assertEquals(set(expects), ShapeType.parse_set(inputs)) 11 | 12 | # basic types 13 | _test([ShapeType.point], ['point']) 14 | _test([ShapeType.line], ['line']) 15 | _test([ShapeType.polygon], ['polygon']) 16 | 17 | # should be case insensitive 18 | _test([ShapeType.point], ['Point']) 19 | _test([ShapeType.line], ['LINE']) 20 | _test([ShapeType.polygon], ['Polygon']) 21 | 22 | # should handle OGC-style names, including Multi* 23 | _test([ShapeType.point], ['MultiPoint']) 24 | _test([ShapeType.line], ['LineString']) 25 | _test([ShapeType.line], ['MultiLineString']) 26 | _test([ShapeType.polygon], ['MultiPolygon']) 27 | 28 | # should handle multiple, repeated names 29 | _test([ShapeType.point], ['Point', 'MultiPoint']) 30 | _test([ 31 | ShapeType.point, 32 | ShapeType.line, 33 | ShapeType.polygon, 34 | ], [ 35 | 'Point', 'MultiPoint', 36 | 'Line', 'LineString', 'MultiLineString', 37 | 'Polygon', 'MultiPolygon' 38 | ]) 39 | 40 | # should return None rather than an empty set. 41 | self.assertEquals(None, ShapeType.parse_set([])) 42 | 43 | # should throw KeyError if the name isn't recognised 44 | with self.assertRaises(KeyError): 45 | ShapeType.parse_set(['MegaPolygon']) 46 | 47 | def _route_layer_properties(self, route_tags): 48 | from tilequeue.query.common import layer_properties 49 | from shapely.geometry.linestring import LineString 50 | 51 | class FakeOsm(object): 52 | def __init__(self, test, tags): 53 | tag_list = [] 54 | for k, v in tags.items(): 55 | tag_list.append(k) 56 | tag_list.append(v) 57 | 58 | self.test = test 59 | self.tag_list = tag_list 60 | 61 | def relations_using_way(self, way_id): 62 | self.test.assertEqual(1, way_id) 63 | return [2] 64 | 65 | def relation(self, rel_id): 66 | from tilequeue.query.common import Relation 67 | self.test.assertEqual(2, rel_id) 68 | return Relation(dict( 69 | id=2, way_off=0, rel_off=1, 70 | tags=self.tag_list, 71 | parts=[1] 72 | )) 73 | 74 | fid = 1 75 | shape = LineString([(0, 0), (1, 0)]) 76 | props = {} 77 | layer_name = 'roads' 78 | zoom = 16 79 | osm = FakeOsm(self, route_tags) 80 | 81 | layer_props = layer_properties( 82 | fid, shape, props, layer_name, zoom, osm) 83 | 84 | return layer_props 85 | 86 | def test_business_and_spur_routes(self): 87 | # check that we extend the network specifier for US:I into 88 | # US:I:Business when there's a modifier set to business on the 89 | # route relation. 90 | 91 | layer_props = self._route_layer_properties(dict( 92 | type='route', route='road', network='US:I', ref='70', 93 | modifier='business')) 94 | 95 | self.assertEquals(['road', 'US:I:Business', '70'], 96 | layer_props.get('mz_networks')) 97 | 98 | def test_business_and_spur_routes_existing(self): 99 | # check that, if the network is _already_ a US:I:Business, we don't 100 | # duplicate the suffix. 101 | 102 | layer_props = self._route_layer_properties(dict( 103 | type='route', route='road', network='US:I:Business', ref='70', 104 | modifier='business')) 105 | 106 | self.assertEquals(['road', 'US:I:Business', '70'], 107 | layer_props.get('mz_networks')) 108 | 109 | def test_business_not_at_end(self): 110 | # check that, if the network contains 'Business', but it's not at the 111 | # end, then we still don't append it. 112 | 113 | layer_props = self._route_layer_properties(dict( 114 | type='route', route='road', network='US:I:Business:Loop', ref='70', 115 | modifier='business')) 116 | 117 | self.assertEquals(['road', 'US:I:Business:Loop', '70'], 118 | layer_props.get('mz_networks')) 119 | -------------------------------------------------------------------------------- /tests/test_query_split.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | 4 | class _NullFetcher(object): 5 | 6 | def fetch_tiles(self, data): 7 | for datum in data: 8 | yield self, datum 9 | 10 | 11 | def _c(z, x, y): 12 | from ModestMaps.Core import Coordinate 13 | return Coordinate(zoom=z, column=x, row=y) 14 | 15 | 16 | class TestQuerySplit(unittest.TestCase): 17 | 18 | def test_splits_jobs(self): 19 | from tilequeue.query.split import make_split_data_fetcher 20 | 21 | above = _NullFetcher() 22 | below = _NullFetcher() 23 | 24 | above_data = dict(coord=_c(9, 0, 0)) 25 | below_data = dict(coord=_c(10, 0, 0)) 26 | 27 | splitter = make_split_data_fetcher(10, above, below) 28 | 29 | all_data = [above_data, below_data] 30 | result = splitter.fetch_tiles(all_data) 31 | expected = [(above, above_data), (below, below_data)] 32 | 33 | # sadly, dicts aren't hashable and frozendict isn't a thing in the 34 | # standard library, so seems easier to just sort the lists - although 35 | # a defined sort order isn't available on these objects, they should 36 | # be the same objects in memory, so even an id() based sorting should 37 | # work. 38 | self.assertEquals(sorted(expected), sorted(result)) 39 | -------------------------------------------------------------------------------- /tests/test_rawr.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | 4 | class RawrS3SinkTest(unittest.TestCase): 5 | 6 | def _make_stub_s3_client(self): 7 | class stub_s3_client(object): 8 | def put_object(self, **props): 9 | self.put_props = props 10 | return stub_s3_client() 11 | 12 | def _make_stub_rawr_tile(self): 13 | from raw_tiles.tile import Tile 14 | props = dict( 15 | all_formatted_data=[], 16 | tile=Tile(3, 2, 1), 17 | ) 18 | return type('stub-rawr-tile', (), props) 19 | 20 | def test_tags(self): 21 | s3_client = self._make_stub_s3_client() 22 | from tilequeue.rawr import RawrS3Sink 23 | from tilequeue.store import KeyFormatType 24 | from tilequeue.store import S3TileKeyGenerator 25 | tile_key_gen = S3TileKeyGenerator( 26 | key_format_type=KeyFormatType.hash_prefix) 27 | sink = RawrS3Sink( 28 | s3_client, 'bucket', 'prefix', 'extension', tile_key_gen) 29 | rawr_tile = self._make_stub_rawr_tile() 30 | sink(rawr_tile) 31 | self.assertIsNone(sink.s3_client.put_props.get('Tagging')) 32 | sink.tags = dict(prefix='foo', run_id='bar') 33 | sink(rawr_tile) 34 | self.assertEquals('prefix=foo&run_id=bar', 35 | sink.s3_client.put_props.get('Tagging')) 36 | 37 | 38 | class RawrKeyTest(unittest.TestCase): 39 | def test_s3_path(self): 40 | from tilequeue.tile import deserialize_coord 41 | from tilequeue.store import KeyFormatType 42 | from tilequeue.store import S3TileKeyGenerator 43 | coord = deserialize_coord('10/1/2') 44 | prefix = '19851026' 45 | extension = 'zip' 46 | tile_key_gen = S3TileKeyGenerator( 47 | key_format_type=KeyFormatType.hash_prefix) 48 | key = tile_key_gen(prefix, coord, extension) 49 | self.assertEqual('c35b6/19851026/10/1/2.zip', key) 50 | -------------------------------------------------------------------------------- /tests/test_store.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tests for `tilequeue.store`. 3 | """ 4 | import unittest 5 | 6 | 7 | class TestTileDirectory(unittest.TestCase): 8 | 9 | def setUp(self): 10 | import tempfile 11 | self.dir_path = tempfile.mkdtemp() 12 | 13 | def tearDown(self): 14 | import shutil 15 | shutil.rmtree(self.dir_path) 16 | 17 | def test_write_tile(self): 18 | from ModestMaps.Core import Coordinate 19 | from tilequeue import format 20 | from tilequeue import store 21 | import os 22 | # Verify that the `TileDirectory` directory gets created. 23 | tile_dir = store.TileDirectory(self.dir_path) 24 | self.assertTrue( 25 | os.path.isdir(self.dir_path), 26 | 'The directory path passed to `TileDirectory()` wasn\'t created ' 27 | 'during initialization') 28 | 29 | # Verify that tile data is written to the right files. 30 | tiles_to_write = [ 31 | ('tile1', (1, 2, 3), 'json'), 32 | ('tile2', (8, 4, 9), 'mvt'), 33 | ('tile3', (2, 6, 0), 'vtm'), 34 | ('tile4', (2, 6, 1), 'topojson'), 35 | ] 36 | 37 | for tile_data, (z, c, r), fmt in tiles_to_write: 38 | coords_obj = Coordinate(row=r, column=c, zoom=z) 39 | format_obj = format.OutputFormat(fmt, fmt, None, None, None, False) 40 | tile_dir.write_tile(tile_data, coords_obj, format_obj) 41 | 42 | expected_filename = '{0}/{1}/{2}.{3}'.format( 43 | coords_obj.zoom, coords_obj.column, coords_obj.row, fmt) 44 | expected_path = os.path.join(self.dir_path, expected_filename) 45 | self.assertTrue( 46 | os.path.isfile(expected_path), 47 | 'Tile data must not have been written to the right location, ' 48 | 'because the expected file path does not exist') 49 | 50 | with open(expected_path) as tile_fp: 51 | self.assertEqual( 52 | tile_fp.read(), tile_data, 53 | 'Tile data written to file does not match the input data') 54 | 55 | os.remove(expected_path) 56 | 57 | 58 | class TestStoreKey(unittest.TestCase): 59 | 60 | def test_example_coord(self): 61 | from tilequeue.tile import deserialize_coord 62 | from tilequeue.format import json_format 63 | from tilequeue.store import KeyFormatType 64 | from tilequeue.store import S3TileKeyGenerator 65 | coord = deserialize_coord('8/72/105') 66 | prefix = '20160121' 67 | tile_key_gen = S3TileKeyGenerator( 68 | key_format_type=KeyFormatType.hash_prefix) 69 | tile_key = tile_key_gen(prefix, coord, json_format.extension) 70 | self.assertEqual(tile_key, 'b57e9/20160121/8/72/105.json') 71 | 72 | 73 | class WriteTileIfChangedTest(unittest.TestCase): 74 | 75 | def setUp(self): 76 | self._in = None 77 | self._out = None 78 | self.store = type( 79 | 'test-store', 80 | (), 81 | dict(read_tile=self._read_tile, write_tile=self._write_tile) 82 | ) 83 | 84 | def _read_tile(self, coord, format): 85 | return self._in 86 | 87 | def _write_tile(self, tile_data, coord, format): 88 | self._out = tile_data 89 | 90 | def _call_fut(self, tile_data): 91 | from tilequeue.store import write_tile_if_changed 92 | coord = format = None 93 | result = write_tile_if_changed(self.store, tile_data, coord, format) 94 | return result 95 | 96 | def test_no_data(self): 97 | did_write = self._call_fut('data') 98 | self.assertTrue(did_write) 99 | self.assertEquals('data', self._out) 100 | 101 | def test_diff_data(self): 102 | self._in = 'different data' 103 | did_write = self._call_fut('data') 104 | self.assertTrue(did_write) 105 | self.assertEquals('data', self._out) 106 | 107 | def test_same_data(self): 108 | self._in = 'data' 109 | did_write = self._call_fut('data') 110 | self.assertFalse(did_write) 111 | self.assertIsNone(self._out) 112 | 113 | 114 | class S3Test(unittest.TestCase): 115 | 116 | def _make_stub_s3_client(self): 117 | class stub_s3_client(object): 118 | def put_object(self, **props): 119 | self.put_props = props 120 | return stub_s3_client() 121 | 122 | def test_tags(self): 123 | from tilequeue.store import KeyFormatType 124 | from tilequeue.store import S3 125 | from tilequeue.store import S3TileKeyGenerator 126 | s3_client = self._make_stub_s3_client() 127 | tags = None 128 | tile_key_gen = S3TileKeyGenerator( 129 | key_format_type=KeyFormatType.hash_prefix) 130 | store = S3(s3_client, 'bucket', 'prefix', False, 60, None, 131 | 'public-read', tags, tile_key_gen) 132 | tile_data = 'data' 133 | from tilequeue.tile import deserialize_coord 134 | coord = deserialize_coord('14/1/2') 135 | from tilequeue.format import mvt_format 136 | store.write_tile(tile_data, coord, mvt_format) 137 | self.assertIsNone(store.s3_client.put_props.get('Tagging')) 138 | store.tags = dict(prefix='foo', run_id='bar') 139 | store.write_tile(tile_data, coord, mvt_format) 140 | self.assertEquals('prefix=foo&run_id=bar', 141 | store.s3_client.put_props.get('Tagging')) 142 | 143 | 144 | class _LogicalLog(object): 145 | """ 146 | A logical time description of when things happened. Used for recording that 147 | one write to S3 happened before or after another. 148 | """ 149 | 150 | def __init__(self): 151 | self.time = 0 152 | self.items = [] 153 | 154 | def __call__(self, *args): 155 | self.items.append((self.time,) + args) 156 | self.time += 1 157 | 158 | 159 | class _LoggingStore(object): 160 | """ 161 | A mock store which doesn't store tiles, only logs calls to a logical log. 162 | """ 163 | 164 | def __init__(self, name, log): 165 | self.name = name 166 | self.log = log 167 | 168 | def write_tile(self, tile_data, coord, format): 169 | self.log(self.name, 'write_tile', tile_data, coord, format) 170 | 171 | def read_tile(self, coord, format): 172 | self.log(self.name, 'read_tile', coord, format) 173 | return '' 174 | 175 | def delete_tiles(self, coords, format): 176 | self.log(self.name, 'delete_tiles', coords, format) 177 | return 0 178 | 179 | def list_tiles(self, format): 180 | self.log(self.name, 'list_tiles', format) 181 | return iter(()) 182 | 183 | 184 | class MultiStoreTest(unittest.TestCase): 185 | 186 | def test_multi_write(self): 187 | from tilequeue.format import json_format 188 | from tilequeue.store import MultiStore 189 | from ModestMaps.Core import Coordinate 190 | 191 | coord = Coordinate(zoom=0, column=0, row=0) 192 | 193 | log = _LogicalLog() 194 | s0 = _LoggingStore('s0', log) 195 | s1 = _LoggingStore('s1', log) 196 | m = MultiStore([s0, s1]) 197 | 198 | m.write_tile('foo', coord, json_format) 199 | 200 | # multi store should write to both stores. 201 | self.assertEqual( 202 | log.items, 203 | [ 204 | (0, 's0', 'write_tile', 'foo', coord, json_format), 205 | (1, 's1', 'write_tile', 'foo', coord, json_format), 206 | ]) 207 | 208 | def test_multi_read(self): 209 | from tilequeue.format import json_format 210 | from tilequeue.store import MultiStore 211 | from ModestMaps.Core import Coordinate 212 | 213 | coord = Coordinate(zoom=0, column=0, row=0) 214 | 215 | log = _LogicalLog() 216 | s0 = _LoggingStore('s0', log) 217 | s1 = _LoggingStore('s1', log) 218 | m = MultiStore([s0, s1]) 219 | 220 | m.read_tile(coord, json_format) 221 | 222 | # multi store should only read from final store. 223 | self.assertEqual( 224 | log.items, 225 | [ 226 | (0, 's1', 'read_tile', coord, json_format), 227 | ]) 228 | 229 | def test_multi_cfg_list(self): 230 | from tilequeue.store import _make_s3_store 231 | 232 | calls = [] 233 | 234 | def _construct(name): 235 | calls.append(name) 236 | 237 | # check that a list results in multiple calls to construct a store. 238 | _make_s3_store(['foo', 'bar', 'baz'], _construct) 239 | 240 | self.assertEqual(calls, ['foo', 'bar', 'baz']) 241 | 242 | def test_multi_cfg_singleton(self): 243 | from tilequeue.store import _make_s3_store 244 | 245 | calls = [] 246 | 247 | def _construct(name): 248 | calls.append(name) 249 | 250 | # check that a single-item list results in a single call to construct 251 | # a store. 252 | _make_s3_store(['foo'], _construct) 253 | 254 | self.assertEqual(calls, ['foo']) 255 | 256 | def test_multi_cfg_string(self): 257 | from tilequeue.store import _make_s3_store 258 | 259 | calls = [] 260 | 261 | def _construct(name): 262 | calls.append(name) 263 | 264 | # check that a single string results in a single call to construct, 265 | # and isn't iterated over as single characters. 266 | _make_s3_store('foo', _construct) 267 | 268 | self.assertEqual(calls, ['foo']) 269 | -------------------------------------------------------------------------------- /tests/test_tile.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from itertools import groupby 3 | from math import pow 4 | from operator import attrgetter 5 | 6 | from ModestMaps.Core import Coordinate 7 | 8 | 9 | class TestSeedTiles(unittest.TestCase): 10 | 11 | def _call_fut(self, zoom_until): 12 | from tilequeue.tile import seed_tiles 13 | return list(seed_tiles(zoom_until=zoom_until)) 14 | 15 | def _assert_tilelist(self, expected_tiles, actual_tiles): 16 | expected_tiles.sort() 17 | actual_tiles.sort() 18 | self.assertEqual(expected_tiles, actual_tiles) 19 | 20 | def test_zoom_0(self): 21 | tiles = self._call_fut(0) 22 | self.assertEqual([Coordinate(0, 0, 0)], tiles) 23 | 24 | def test_zoom_1(self): 25 | tiles = self._call_fut(1) 26 | self.assertEqual(5, len(tiles)) 27 | expected_tiles = [ 28 | Coordinate(0, 0, 0), 29 | Coordinate(0, 0, 1), 30 | Coordinate(1, 0, 1), 31 | Coordinate(0, 1, 1), 32 | Coordinate(1, 1, 1), 33 | ] 34 | self._assert_tilelist(expected_tiles, tiles) 35 | 36 | def test_zoom_5(self): 37 | tiles = self._call_fut(5) 38 | # sorts by zoom (first), which is why group by zoom will work 39 | tiles.sort() 40 | for zoom, tiles_per_zoom in groupby(tiles, attrgetter('zoom')): 41 | expected_num_tiles = pow(4, zoom) 42 | actual_num_tiles = len(list(tiles_per_zoom)) 43 | self.assertEqual(expected_num_tiles, actual_num_tiles) 44 | 45 | 46 | class TestCoordToBounds(unittest.TestCase): 47 | 48 | def test_convert_coord(self): 49 | from tilequeue.tile import coord_to_bounds 50 | from ModestMaps.Core import Coordinate 51 | coord = Coordinate(zoom=14, column=4824, row=6160) 52 | bounds = coord_to_bounds(coord) 53 | exp_bounds = (-74.00390625, 40.69729900863674, 54 | -73.98193359375, 40.713955826286046) 55 | self.assertEqual(tuple, type(bounds)) 56 | self.assertEqual(len(exp_bounds), len(bounds)) 57 | for i in range(0, len(exp_bounds)): 58 | exp = exp_bounds[i] 59 | act = bounds[i] 60 | self.assertAlmostEqual( 61 | exp, act, msg='Expected %r but got %r at index %d' % 62 | (exp, act, i)) 63 | 64 | 65 | class TestTileGeneration(unittest.TestCase): 66 | 67 | def _is_zoom(self, zoom): 68 | return lambda coord: zoom == coord.zoom 69 | 70 | def test_tiles_for_coord(self): 71 | from ModestMaps.Core import Coordinate 72 | from tilequeue.tile import coord_to_bounds 73 | from tilequeue.tile import tile_generator_for_single_bounds 74 | coord = Coordinate(1, 1, 1) 75 | bounds = coord_to_bounds(coord) 76 | tile_generator = tile_generator_for_single_bounds(bounds, 1, 1) 77 | tiles = list(tile_generator) 78 | self.assertEqual(1, len(tiles)) 79 | 80 | def test_tiles_for_bounds_firsttile_two_zooms(self): 81 | from tilequeue.tile import tile_generator_for_single_bounds 82 | bounds = (-180, 0.1, -0.1, 85) 83 | tile_generator = tile_generator_for_single_bounds(bounds, 1, 2) 84 | tiles = list(tile_generator) 85 | self.assertEqual(5, len(tiles)) 86 | self.assertEqual(1, len(filter(self._is_zoom(1), tiles))) 87 | self.assertEqual(4, len(filter(self._is_zoom(2), tiles))) 88 | 89 | def test_tiles_for_bounds_lasttile_two_zooms(self): 90 | from tilequeue.tile import tile_generator_for_single_bounds 91 | bounds = (0.1, -85, 180, -0.1) 92 | tile_generator = tile_generator_for_single_bounds(bounds, 1, 2) 93 | tiles = list(tile_generator) 94 | self.assertEqual(5, len(tiles)) 95 | self.assertEqual(1, len(filter(self._is_zoom(1), tiles))) 96 | self.assertEqual(4, len(filter(self._is_zoom(2), tiles))) 97 | 98 | def test_tiles_for_two_bounds_two_zooms(self): 99 | from tilequeue.tile import tile_generator_for_multiple_bounds 100 | bounds1 = (-180, 0.1, -0.1, 85) 101 | bounds2 = (0.1, -85, 180, -0.1) 102 | tile_generator = tile_generator_for_multiple_bounds( 103 | (bounds1, bounds2), 1, 2) 104 | tiles = list(tile_generator) 105 | self.assertEqual(10, len(tiles)) 106 | self.assertEqual(2, len(filter(self._is_zoom(1), tiles))) 107 | self.assertEqual(8, len(filter(self._is_zoom(2), tiles))) 108 | 109 | def test_tiles_children(self): 110 | from tilequeue.tile import coord_children 111 | from ModestMaps.Core import Coordinate 112 | coord = Coordinate(0, 0, 0) 113 | children = coord_children(coord) 114 | self.assertEqual(4, len(children)) 115 | self.assertEqual(Coordinate(0, 0, 1), children[0]) 116 | self.assertEqual(Coordinate(1, 0, 1), children[1]) 117 | self.assertEqual(Coordinate(0, 1, 1), children[2]) 118 | self.assertEqual(Coordinate(1, 1, 1), children[3]) 119 | 120 | def test_tiles_children_range(self): 121 | from tilequeue.tile import coord_children 122 | from tilequeue.tile import coord_children_range 123 | from ModestMaps.Core import Coordinate 124 | coord = Coordinate(3, 4, 2) 125 | actual = list(coord_children_range(coord, 4)) 126 | self.assertEqual(20, len(actual)) 127 | children = list(coord_children(coord)) 128 | grandchildren_list = map(coord_children, children) 129 | from itertools import chain 130 | grandchildren = list(chain(*grandchildren_list)) 131 | exp = children + grandchildren 132 | actual.sort() 133 | exp.sort() 134 | for actual_child, exp_child in zip(actual, exp): 135 | self.assertEqual(exp_child, actual_child) 136 | 137 | def test_tiles_children_subrange(self): 138 | from tilequeue.tile import coord_children_subrange as subrange 139 | from tilequeue.tile import coord_children 140 | 141 | def _c(z, x, y): 142 | return Coordinate(zoom=z, column=x, row=y) 143 | 144 | # for any given coord, the subrange from and until the tile's zoom 145 | # should just return the coord itself. 146 | for coord in (_c(0, 0, 0), _c(1, 1, 1), _c(10, 163, 395)): 147 | z = coord.zoom 148 | self.assertEqual(set([coord]), set(subrange(coord, z, z))) 149 | 150 | # when until zoom > coordinate zoom, it should generate the whole 151 | # pyramid. 152 | expect = set([_c(0, 0, 0)]) 153 | zoom = 0 154 | while zoom < 5: 155 | self.assertEqual(expect, set(subrange(_c(0, 0, 0), 0, zoom))) 156 | children = [] 157 | for c in expect: 158 | children.extend(coord_children(c)) 159 | expect |= set(children) 160 | zoom += 1 161 | 162 | # when both start and until are >, then should generate a slice of 163 | # the pyramid. 164 | for z in range(0, 5): 165 | max_coord = 2 ** z 166 | all_tiles = set(_c(z, x, y) 167 | for x in range(0, max_coord) 168 | for y in range(0, max_coord)) 169 | 170 | self.assertEqual(all_tiles, set(subrange(_c(0, 0, 0), z, z))) 171 | 172 | # when start > until, then nothing is generated 173 | for z in range(0, 5): 174 | self.assertEqual(set(), set(subrange(_c(0, 0, 0), z+1, z))) 175 | 176 | def test_tiles_low_zooms(self): 177 | from tilequeue.tile import tile_generator_for_single_bounds 178 | bounds = -1.115, 50.941, 0.895, 51.984 179 | tile_generator = tile_generator_for_single_bounds(bounds, 0, 5) 180 | tiles = list(tile_generator) 181 | self.assertEqual(11, len(tiles)) 182 | 183 | 184 | class TestReproject(unittest.TestCase): 185 | 186 | def test_reproject(self): 187 | from tilequeue.tile import reproject_lnglat_to_mercator 188 | coord = reproject_lnglat_to_mercator(0, 0) 189 | self.assertAlmostEqual(0, coord[0]) 190 | self.assertAlmostEqual(0, coord[1]) 191 | 192 | def test_reproject_with_z(self): 193 | from tilequeue.tile import reproject_lnglat_to_mercator 194 | coord = reproject_lnglat_to_mercator(0, 0, 0) 195 | self.assertAlmostEqual(0, coord[0]) 196 | self.assertAlmostEqual(0, coord[1]) 197 | 198 | 199 | class CoordIntZoomTest(unittest.TestCase): 200 | 201 | def test_verify_low_seed_tiles(self): 202 | from tilequeue.tile import coord_int_zoom_up 203 | from tilequeue.tile import coord_marshall_int 204 | from tilequeue.tile import seed_tiles 205 | seed_coords = seed_tiles(1, 5) 206 | for coord in seed_coords: 207 | coord_int = coord_marshall_int(coord) 208 | parent_coord = coord.zoomTo(coord.zoom - 1).container() 209 | exp_int = coord_marshall_int(parent_coord) 210 | act_int = coord_int_zoom_up(coord_int) 211 | self.assertEquals(exp_int, act_int) 212 | 213 | def test_verify_examples(self): 214 | from ModestMaps.Core import Coordinate 215 | from tilequeue.tile import coord_int_zoom_up 216 | from tilequeue.tile import coord_marshall_int 217 | test_coords = ( 218 | Coordinate(zoom=20, column=1002463, row=312816), 219 | Coordinate(zoom=20, column=(2 ** 20)-1, row=(2 ** 20)-1), 220 | Coordinate(zoom=10, column=(2 ** 10)-1, row=(2 ** 10)-1), 221 | Coordinate(zoom=5, column=20, row=20), 222 | Coordinate(zoom=1, column=0, row=0), 223 | ) 224 | for coord in test_coords: 225 | coord_int = coord_marshall_int(coord) 226 | parent_coord = coord.zoomTo(coord.zoom - 1).container() 227 | exp_int = coord_marshall_int(parent_coord) 228 | act_int = coord_int_zoom_up(coord_int) 229 | self.assertEquals(exp_int, act_int) 230 | 231 | 232 | class TestMetatileZoom(unittest.TestCase): 233 | 234 | def test_zoom_from_size(self): 235 | from tilequeue.tile import metatile_zoom_from_size as func 236 | self.assertEqual(0, func(None)) 237 | self.assertEqual(0, func(1)) 238 | self.assertEqual(1, func(2)) 239 | self.assertEqual(2, func(4)) 240 | 241 | with self.assertRaises(AssertionError): 242 | func(3) 243 | 244 | def test_zoom_from_str(self): 245 | from tilequeue.tile import metatile_zoom_from_str as func 246 | self.assertEqual(0, func(None)) 247 | self.assertEqual(0, func('')) 248 | self.assertEqual(0, func('256')) 249 | self.assertEqual(1, func('512')) 250 | self.assertEqual(2, func('1024')) 251 | -------------------------------------------------------------------------------- /tests/test_toi.py: -------------------------------------------------------------------------------- 1 | import tempfile 2 | import unittest 3 | 4 | from tilequeue.tile import coord_marshall_int 5 | from tilequeue.tile import deserialize_coord 6 | 7 | 8 | class TestToiSet(unittest.TestCase): 9 | def _coord_str_to_int(self, coord_str): 10 | coord = deserialize_coord(coord_str) 11 | coord_int = coord_marshall_int(coord) 12 | return coord_int 13 | 14 | def test_save_set_to_fp(self): 15 | from tilequeue.toi import save_set_to_fp 16 | 17 | toi_set = set() 18 | toi_set.add(self._coord_str_to_int('0/0/0')) 19 | toi_set.add(self._coord_str_to_int('1/0/0')) 20 | toi_set.add(self._coord_str_to_int('1/1/0')) 21 | 22 | with tempfile.TemporaryFile() as fp: 23 | save_set_to_fp(toi_set, fp) 24 | 25 | self.assertEquals(fp.tell(), 18) 26 | 27 | fp.seek(0) 28 | self.assertEquals(fp.read(), '0/0/0\n1/0/0\n1/1/0\n') 29 | 30 | def test_load_set_from_fp(self): 31 | from tilequeue.toi import load_set_from_fp 32 | 33 | with tempfile.TemporaryFile() as fp: 34 | fp.write('0/0/0\n1/0/0\n1/1/0\n') 35 | fp.seek(0) 36 | 37 | actual_toi_set = load_set_from_fp(fp) 38 | expected_toi_set = set() 39 | expected_toi_set.add(self._coord_str_to_int('0/0/0')) 40 | expected_toi_set.add(self._coord_str_to_int('1/0/0')) 41 | expected_toi_set.add(self._coord_str_to_int('1/1/0')) 42 | 43 | self.assertEquals(expected_toi_set, actual_toi_set) 44 | 45 | def test_load_set_from_fp_accidental_dupe(self): 46 | from tilequeue.toi import load_set_from_fp 47 | 48 | with tempfile.TemporaryFile() as fp: 49 | fp.write('0/0/0\n1/0/0\n1/1/0\n1/0/0\n') 50 | fp.seek(0) 51 | 52 | actual_toi_set = load_set_from_fp(fp) 53 | expected_toi_set = set() 54 | expected_toi_set.add(self._coord_str_to_int('0/0/0')) 55 | expected_toi_set.add(self._coord_str_to_int('1/0/0')) 56 | expected_toi_set.add(self._coord_str_to_int('1/1/0')) 57 | 58 | self.assertEquals(expected_toi_set, actual_toi_set) 59 | 60 | def test_save_set_to_gzipped_fp(self): 61 | import gzip 62 | from tilequeue.toi import save_set_to_gzipped_fp 63 | 64 | toi_set = set() 65 | toi_set.add(self._coord_str_to_int('0/0/0')) 66 | toi_set.add(self._coord_str_to_int('1/0/0')) 67 | toi_set.add(self._coord_str_to_int('1/1/0')) 68 | 69 | with tempfile.TemporaryFile() as fp: 70 | save_set_to_gzipped_fp(toi_set, fp) 71 | 72 | self.assertEquals(fp.tell(), 31) 73 | 74 | fp.seek(0) 75 | with gzip.GzipFile(fileobj=fp, mode='r') as gz: 76 | self.assertEquals(gz.read(), '0/0/0\n1/0/0\n1/1/0\n') 77 | 78 | def test_load_set_from_gzipped_fp(self): 79 | import gzip 80 | from tilequeue.toi import load_set_from_gzipped_fp 81 | 82 | with tempfile.TemporaryFile() as fp: 83 | with gzip.GzipFile(fileobj=fp, mode='w') as gz: 84 | gz.write('0/0/0\n1/0/0\n1/1/0\n') 85 | fp.seek(0) 86 | 87 | actual_toi_set = load_set_from_gzipped_fp(fp) 88 | expected_toi_set = set() 89 | expected_toi_set.add(self._coord_str_to_int('0/0/0')) 90 | expected_toi_set.add(self._coord_str_to_int('1/0/0')) 91 | expected_toi_set.add(self._coord_str_to_int('1/1/0')) 92 | 93 | self.assertEquals(expected_toi_set, actual_toi_set) 94 | -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | 4 | class TestCoordsByParent(unittest.TestCase): 5 | 6 | def test_empty(self): 7 | from tilequeue.utils import CoordsByParent 8 | 9 | cbp = CoordsByParent(10) 10 | 11 | count = 0 12 | for key, coords in cbp: 13 | count += 1 14 | 15 | self.assertEquals(0, count) 16 | 17 | def test_lower_zooms_not_grouped(self): 18 | from tilequeue.utils import CoordsByParent 19 | from ModestMaps.Core import Coordinate 20 | 21 | cbp = CoordsByParent(10) 22 | 23 | low_zoom_coords = [(9, 0, 0), (9, 0, 1), (9, 1, 0), (9, 1, 1)] 24 | for z, x, y in low_zoom_coords: 25 | coord = Coordinate(zoom=z, column=x, row=y) 26 | cbp.add(coord) 27 | 28 | count = 0 29 | for key, coords in cbp: 30 | self.assertEquals(1, len(coords)) 31 | count += 1 32 | 33 | self.assertEquals(len(low_zoom_coords), count) 34 | 35 | def test_higher_zooms_grouped(self): 36 | from tilequeue.utils import CoordsByParent 37 | from ModestMaps.Core import Coordinate 38 | 39 | cbp = CoordsByParent(10) 40 | 41 | def _c(z, x, y): 42 | return Coordinate(zoom=z, column=x, row=y) 43 | 44 | groups = { 45 | _c(10, 0, 0): [_c(10, 0, 0), _c(11, 0, 0), _c(11, 0, 1)], 46 | _c(10, 1, 1): [_c(11, 2, 2), _c(11, 3, 3), _c(12, 4, 4)], 47 | } 48 | 49 | for coords in groups.itervalues(): 50 | for coord in coords: 51 | cbp.add(coord) 52 | 53 | count = 0 54 | for key, coords in cbp: 55 | self.assertIn(key, groups) 56 | self.assertEquals(set(groups[key]), set(coords)) 57 | count += 1 58 | 59 | self.assertEquals(len(groups), count) 60 | 61 | def test_with_extra_data(self): 62 | from tilequeue.utils import CoordsByParent 63 | from ModestMaps.Core import Coordinate 64 | 65 | cbp = CoordsByParent(10) 66 | 67 | coord = Coordinate(zoom=10, column=0, row=0) 68 | cbp.add(coord, 'foo', 'bar') 69 | 70 | count = 0 71 | for key, coords in cbp: 72 | self.assertEquals(1, len(coords)) 73 | self.assertEquals((coord, 'foo', 'bar'), coords[0]) 74 | count += 1 75 | 76 | self.assertEquals(1, count) 77 | -------------------------------------------------------------------------------- /tests/test_wof_http.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | try: 3 | # Python 2.x 4 | import BaseHTTPServer as http 5 | except ImportError: 6 | # Python 3.x 7 | from http import server as http 8 | import contextlib 9 | from httptestserver import Server 10 | from tilequeue.wof import make_wof_url_neighbourhood_fetcher, \ 11 | WofProcessor 12 | import datetime 13 | 14 | 15 | # a mock wof model which does nothing - these tests are about 16 | # fetching the data, not parsing it. 17 | class _NullWofModel(object): 18 | 19 | def __init__(self): 20 | self.added = 0 21 | self.updated = 0 22 | self.removed = 0 23 | 24 | def find_previous_neighbourhood_meta(self): 25 | return [] 26 | 27 | def sync_neighbourhoods( 28 | self, neighbourhoods_to_add, neighbourhoods_to_update, 29 | ids_to_remove): 30 | self.added = self.added + len(neighbourhoods_to_add) 31 | self.updated = self.updated + len(neighbourhoods_to_update) 32 | self.removed = self.removed + len(ids_to_remove) 33 | 34 | def insert_neighbourhoods(self, neighbourhoods): 35 | pass 36 | 37 | def update_visible_timestamp(self, zoom, day): 38 | return set() 39 | 40 | 41 | class _WofHandlerContext(object): 42 | 43 | def __init__(self, failure_count=0, content={}, failure_code=500): 44 | self.request_counts = {} 45 | self.failure_count = failure_count 46 | self.content = content 47 | self.failure_code = failure_code 48 | 49 | 50 | class _WofErrorHandler(http.BaseHTTPRequestHandler): 51 | 52 | def __init__(self, context, *args): 53 | self.wof_ctx = context 54 | http.BaseHTTPRequestHandler.__init__(self, *args) 55 | 56 | def do_GET(self): 57 | request_count = self.wof_ctx.request_counts.get(self.path, 0) 58 | 59 | if request_count < self.wof_ctx.failure_count: 60 | self.wof_ctx.request_counts[self.path] = request_count + 1 61 | self.send_response(self.wof_ctx.failure_code) 62 | self.end_headers() 63 | self.wfile.write('') 64 | 65 | else: 66 | self.send_response(200) 67 | content_type, content \ 68 | = self.wof_ctx.content.get(self.path, ('text/plain', '')) 69 | self.send_header('Content-Type', content_type) 70 | self.end_headers() 71 | self.wfile.write(content) 72 | 73 | 74 | # fake Redis object to keep the code happy 75 | class _NullRedisTOI(object): 76 | 77 | def fetch_tiles_of_interest(self): 78 | return [] 79 | 80 | 81 | # guard function to run a test HTTP server on another thread and reap it when 82 | # it goes out of scope. 83 | @contextlib.contextmanager 84 | def _test_http_server(handler): 85 | server = Server('127.0.0.1', 0, 'http', handler) 86 | server.start() 87 | yield server 88 | 89 | 90 | # simple logger that's easy to turn the output on and off. 91 | class _SimpleLogger(object): 92 | 93 | def __init__(self, verbose=True): 94 | self.verbose = verbose 95 | 96 | def info(self, msg): 97 | if self.verbose: 98 | print 'INFO: %s' % msg 99 | 100 | def warn(self, msg): 101 | if self.verbose: 102 | print 'WARN: %s' % msg 103 | 104 | def error(self, msg): 105 | if self.verbose: 106 | print 'ERROR: %s' % msg 107 | 108 | 109 | class TestWofHttp(unittest.TestCase): 110 | 111 | def _simple_test(self, num_failures=0, failure_code=500, max_retries=3): 112 | context = _WofHandlerContext(num_failures, { 113 | '/meta/neighbourhoods.csv': ( 114 | 'text/plain; charset=utf-8', 115 | 'bbox,cessation,deprecated,file_hash,fullname,geom_hash,' 116 | 'geom_latitude,geom_longitude,id,inception,iso,lastmodified,' 117 | 'lbl_latitude,lbl_longitude,name,parent_id,path,placetype,' 118 | 'source,superseded_by,supersedes\n' 119 | "\"0,0,0,0\",u,,00000000000000000000000000000000,," 120 | '00000000000000000000000000000000,0,0,1,u,,0,0,0,Null Island,' 121 | '-1,1/1.geojson,neighbourhood,null,,\n' 122 | ), 123 | '/meta/microhoods.csv': ( 124 | 'text/plain; charset=utf-8', 125 | 'bbox,cessation,deprecated,file_hash,fullname,geom_hash,' 126 | 'geom_latitude,geom_longitude,id,inception,iso,lastmodified,' 127 | 'lbl_latitude,lbl_longitude,name,parent_id,path,placetype,' 128 | 'source,superseded_by,supersedes\n' 129 | ), 130 | '/meta/macrohoods.csv': ( 131 | 'text/plain; charset=utf-8', 132 | 'bbox,cessation,deprecated,file_hash,fullname,geom_hash,' 133 | 'geom_latitude,geom_longitude,id,inception,iso,lastmodified,' 134 | 'lbl_latitude,lbl_longitude,name,parent_id,path,placetype,' 135 | 'source,superseded_by,supersedes\n' 136 | ), 137 | '/data/1/1.geojson': ( 138 | 'application/json; charset=utf-8', 139 | '{"id":1,"type":"Feature","properties":{"wof:id":1,' + 140 | '"wof:name":"Null Island","lbl:latitude":0.0,' + 141 | '"lbl:longitude":0.0,"wof:placetype":"neighbourhood"},' + 142 | '"geometry":{"coordinates":[0,0],"type":"Point"}}' 143 | ) 144 | }, failure_code) 145 | 146 | def handler(*args): 147 | return _WofErrorHandler(context, *args) 148 | 149 | model = _NullWofModel() 150 | 151 | with _test_http_server(handler) as server: 152 | fetcher = make_wof_url_neighbourhood_fetcher( 153 | server.url('/meta/neighbourhoods.csv'), 154 | server.url('/meta/microhoods.csv'), 155 | server.url('/meta/macrohoods.csv'), 156 | server.url('/meta/boroughs.csv'), 157 | server.url('/data'), 158 | 1, max_retries) 159 | redis = _NullRedisTOI() 160 | 161 | def intersector(dummy1, dummy2, dummy3): 162 | return [], None 163 | 164 | def enqueuer(dummy): 165 | pass 166 | 167 | logger = _SimpleLogger(False) 168 | 169 | today = datetime.date.today() 170 | processor = WofProcessor(fetcher, model, redis, intersector, 171 | enqueuer, logger, today) 172 | processor() 173 | 174 | self.assertEqual(model.added, 1) 175 | self.assertEqual(model.updated, 0) 176 | self.assertEqual(model.removed, 0) 177 | 178 | # if there are no failures, then the process should complete correctly. 179 | def test_without_failures(self): 180 | self._simple_test(0, 502) 181 | 182 | # if there are fewer failures than the number of retries, then it should 183 | # process without an error. 184 | def test_with_single_failure(self): 185 | self._simple_test(1, 502) 186 | 187 | # however, if we try to fetch a URL and it's missing then that really 188 | # should be an error - probably indicates that we're not using the right 189 | # logic to form the URLs. 190 | def test_with_missing(self): 191 | with self.assertRaises(AssertionError): 192 | self._simple_test(1, 404) 193 | 194 | # if we try to fetch a URL and it's forbidden then that really should be an 195 | # error - probably indicates a configuration problem with WOF. 196 | def test_with_forbidden(self): 197 | with self.assertRaises(AssertionError): 198 | self._simple_test(1, 403) 199 | -------------------------------------------------------------------------------- /tilequeue/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | -------------------------------------------------------------------------------- /tilequeue/constants.py: -------------------------------------------------------------------------------- 1 | MAX_TILE_ZOOM = 16 # tilezen's default max supported zoom 2 | -------------------------------------------------------------------------------- /tilequeue/format/OSciMap4/StaticKeys/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Sep 13, 2012 3 | 4 | @author: jeff 5 | ''' 6 | 7 | strings = [ 8 | 'access', 9 | 'addr:housename', 10 | 'addr:housenumber', 11 | 'addr:interpolation', 12 | 'admin_level', 13 | 'aerialway', 14 | 'aeroway', 15 | 'amenity', 16 | 'area', 17 | 'barrier', 18 | 'bicycle', 19 | 'brand', 20 | 'bridge', 21 | 'boundary', 22 | 'building', 23 | 'construction', 24 | 'covered', 25 | 'culvert', 26 | 'cutting', 27 | 'denomination', 28 | 'disused', 29 | 'embankment', 30 | 'foot', 31 | 'generator:source', 32 | 'harbour', 33 | 'highway', 34 | 'historic', 35 | 'horse', 36 | 'intermittent', 37 | 'junction', 38 | 'landuse', 39 | 'layer', 40 | 'leisure', 41 | 'lock', 42 | 'man_made', 43 | 'military', 44 | 'motorcar', 45 | 'name', 46 | 'natural', 47 | 'oneway', 48 | 'operator', 49 | 'population', 50 | 'power', 51 | 'power_source', 52 | 'place', 53 | 'railway', 54 | 'ref', 55 | 'religion', 56 | 'route', 57 | 'service', 58 | 'shop', 59 | 'sport', 60 | 'surface', 61 | 'toll', 62 | 'tourism', 63 | 'tower:type', 64 | 'tracktype', 65 | 'tunnel', 66 | 'water', 67 | 'waterway', 68 | 'wetland', 69 | 'width', 70 | 'wood', 71 | 72 | 'height', 73 | 'min_height', 74 | 'roof:shape', 75 | 'roof:height', 76 | 'rank'] 77 | 78 | keys = dict(zip(strings, range(0, len(strings)-1))) 79 | 80 | 81 | def getKeys(): 82 | return keys 83 | -------------------------------------------------------------------------------- /tilequeue/format/OSciMap4/StaticVals/__init__.py: -------------------------------------------------------------------------------- 1 | vals = { 2 | 'yes': 0, 3 | 'residential': 1, 4 | 'service': 2, 5 | 'unclassified': 3, 6 | 'stream': 4, 7 | 'track': 5, 8 | 'water': 6, 9 | 'footway': 7, 10 | 'tertiary': 8, 11 | 'private': 9, 12 | 'tree': 10, 13 | 'path': 11, 14 | 'forest': 12, 15 | 'secondary': 13, 16 | 'house': 14, 17 | 'no': 15, 18 | 'asphalt': 16, 19 | 'wood': 17, 20 | 'grass': 18, 21 | 'paved': 19, 22 | 'primary': 20, 23 | 'unpaved': 21, 24 | 'bus_stop': 22, 25 | 'parking': 23, 26 | 'parking_aisle': 24, 27 | 'rail': 25, 28 | 'driveway': 26, 29 | '8': 27, 30 | 'administrative': 28, 31 | 'locality': 29, 32 | 'turning_circle': 30, 33 | 'crossing': 31, 34 | 'village': 32, 35 | 'fence': 33, 36 | 'grade2': 34, 37 | 'coastline': 35, 38 | 'grade3': 36, 39 | 'farmland': 37, 40 | 'hamlet': 38, 41 | 'hut': 39, 42 | 'meadow': 40, 43 | 'wetland': 41, 44 | 'cycleway': 42, 45 | 'river': 43, 46 | 'school': 44, 47 | 'trunk': 45, 48 | 'gravel': 46, 49 | 'place_of_worship': 47, 50 | 'farm': 48, 51 | 'grade1': 49, 52 | 'traffic_signals': 50, 53 | 'wall': 51, 54 | 'garage': 52, 55 | 'gate': 53, 56 | 'motorway': 54, 57 | 'living_street': 55, 58 | 'pitch': 56, 59 | 'grade4': 57, 60 | 'industrial': 58, 61 | 'road': 59, 62 | 'ground': 60, 63 | 'scrub': 61, 64 | 'motorway_link': 62, 65 | 'steps': 63, 66 | 'ditch': 64, 67 | 'swimming_pool': 65, 68 | 'grade5': 66, 69 | 'park': 67, 70 | 'apartments': 68, 71 | 'restaurant': 69, 72 | 'designated': 70, 73 | 'bench': 71, 74 | 'survey_point': 72, 75 | 'pedestrian': 73, 76 | 'hedge': 74, 77 | 'reservoir': 75, 78 | 'riverbank': 76, 79 | 'alley': 77, 80 | 'farmyard': 78, 81 | 'peak': 79, 82 | 'level_crossing': 80, 83 | 'roof': 81, 84 | 'dirt': 82, 85 | 'drain': 83, 86 | 'garages': 84, 87 | 'entrance': 85, 88 | 'street_lamp': 86, 89 | 'deciduous': 87, 90 | 'fuel': 88, 91 | 'trunk_link': 89, 92 | 'information': 90, 93 | 'playground': 91, 94 | 'supermarket': 92, 95 | 'primary_link': 93, 96 | 'concrete': 94, 97 | 'mixed': 95, 98 | 'permissive': 96, 99 | 'orchard': 97, 100 | 'grave_yard': 98, 101 | 'canal': 99, 102 | 'garden': 100, 103 | 'spur': 101, 104 | 'paving_stones': 102, 105 | 'rock': 103, 106 | 'bollard': 104, 107 | 'convenience': 105, 108 | 'cemetery': 106, 109 | 'post_box': 107, 110 | 'commercial': 108, 111 | 'pier': 109, 112 | 'bank': 110, 113 | 'hotel': 111, 114 | 'cliff': 112, 115 | 'retail': 113, 116 | 'construction': 114, 117 | '-1': 115, 118 | 'fast_food': 116, 119 | 'coniferous': 117, 120 | 'cafe': 118, 121 | '6': 119, 122 | 'kindergarten': 120, 123 | 'tower': 121, 124 | 'hospital': 122, 125 | 'yard': 123, 126 | 'sand': 124, 127 | 'public_building': 125, 128 | 'cobblestone': 126, 129 | 'destination': 127, 130 | 'island': 128, 131 | 'abandoned': 129, 132 | 'vineyard': 130, 133 | 'recycling': 131, 134 | 'agricultural': 132, 135 | 'isolated_dwelling': 133, 136 | 'pharmacy': 134, 137 | 'post_office': 135, 138 | 'motorway_junction': 136, 139 | 'pub': 137, 140 | 'allotments': 138, 141 | 'dam': 139, 142 | 'secondary_link': 140, 143 | 'lift_gate': 141, 144 | 'siding': 142, 145 | 'stop': 143, 146 | 'main': 144, 147 | 'farm_auxiliary': 145, 148 | 'quarry': 146, 149 | '10': 147, 150 | 'station': 148, 151 | 'platform': 149, 152 | 'taxiway': 150, 153 | 'limited': 151, 154 | 'sports_centre': 152, 155 | 'cutline': 153, 156 | 'detached': 154, 157 | 'storage_tank': 155, 158 | 'basin': 156, 159 | 'bicycle_parking': 157, 160 | 'telephone': 158, 161 | 'terrace': 159, 162 | 'town': 160, 163 | 'suburb': 161, 164 | 'bus': 162, 165 | 'compacted': 163, 166 | 'toilets': 164, 167 | 'heath': 165, 168 | 'works': 166, 169 | 'tram': 167, 170 | 'beach': 168, 171 | 'culvert': 169, 172 | 'fire_station': 170, 173 | 'recreation_ground': 171, 174 | 'bakery': 172, 175 | 'police': 173, 176 | 'atm': 174, 177 | 'clothes': 175, 178 | 'tertiary_link': 176, 179 | 'waste_basket': 177, 180 | 'attraction': 178, 181 | 'viewpoint': 179, 182 | 'bicycle': 180, 183 | 'church': 181, 184 | 'shelter': 182, 185 | 'drinking_water': 183, 186 | 'marsh': 184, 187 | 'picnic_site': 185, 188 | 'hairdresser': 186, 189 | 'bridleway': 187, 190 | 'retaining_wall': 188, 191 | 'buffer_stop': 189, 192 | 'nature_reserve': 190, 193 | 'village_green': 191, 194 | 'university': 192, 195 | '1': 193, 196 | 'bar': 194, 197 | 'townhall': 195, 198 | 'mini_roundabout': 196, 199 | 'camp_site': 197, 200 | 'aerodrome': 198, 201 | 'stile': 199, 202 | '9': 200, 203 | 'car_repair': 201, 204 | 'parking_space': 202, 205 | 'library': 203, 206 | 'pipeline': 204, 207 | 'true': 205, 208 | 'cycle_barrier': 206, 209 | '4': 207, 210 | 'museum': 208, 211 | 'spring': 209, 212 | 'hunting_stand': 210, 213 | 'disused': 211, 214 | 'car': 212, 215 | 'tram_stop': 213, 216 | 'land': 214, 217 | 'fountain': 215, 218 | 'hiking': 216, 219 | 'manufacture': 217, 220 | 'vending_machine': 218, 221 | 'kiosk': 219, 222 | 'swamp': 220, 223 | 'unknown': 221, 224 | '7': 222, 225 | 'islet': 223, 226 | 'shed': 224, 227 | 'switch': 225, 228 | 'rapids': 226, 229 | 'office': 227, 230 | 'bay': 228, 231 | 'proposed': 229, 232 | 'common': 230, 233 | 'weir': 231, 234 | 'grassland': 232, 235 | 'customers': 233, 236 | 'social_facility': 234, 237 | 'hangar': 235, 238 | 'doctors': 236, 239 | 'stadium': 237, 240 | 'give_way': 238, 241 | 'greenhouse': 239, 242 | 'guest_house': 240, 243 | 'viaduct': 241, 244 | 'doityourself': 242, 245 | 'runway': 243, 246 | 'bus_station': 244, 247 | 'water_tower': 245, 248 | 'golf_course': 246, 249 | 'conservation': 247, 250 | 'block': 248, 251 | 'college': 249, 252 | 'wastewater_plant': 250, 253 | 'subway': 251, 254 | 'halt': 252, 255 | 'forestry': 253, 256 | 'florist': 254, 257 | 'butcher': 255} 258 | 259 | 260 | def getValues(): 261 | return vals 262 | -------------------------------------------------------------------------------- /tilequeue/format/OSciMap4/TagRewrite/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | # TODO test the lua osm2pgsql for preprocessing ! 4 | # 5 | # fix tags from looking things up in wiki where a value should be used with a specific key, 6 | # i.e. one combination has a wiki page and more use in taginfo and the other does not 7 | # TODO add: 8 | # natural=>meadow 9 | # landuse=>greenhouse,public,scrub 10 | # aeroway=>aerobridge 11 | # leisure=>natural_reserve 12 | 13 | 14 | def fixTag(tag): 15 | drop = False 16 | 17 | if tag[1] is None: 18 | drop = True 19 | 20 | key = tag[0].lower() 21 | 22 | if key == 'highway': 23 | # FIXME remove ; separated part of tags 24 | return (key, tag[1].lower().split(';')[0]) 25 | 26 | # fixed in osm 27 | # if key == 'leisure': 28 | # value = tag[1].lower(); 29 | # if value in ('village_green', 'recreation_ground'): 30 | # return ('landuse', value) 31 | # else: 32 | # return (key, value) 33 | 34 | elif key == 'natural': 35 | value = tag[1].lower() 36 | # if zoomlevel <= 9 and not value in ('water', 'wood'): 37 | # return None 38 | 39 | if value in ('village_green', 'meadow'): 40 | return ('landuse', value) 41 | if value == 'mountain_range': 42 | drop = True 43 | else: 44 | return (key, value) 45 | 46 | elif key == 'landuse': 47 | value = tag[1].lower() 48 | # if zoomlevel <= 9 and not value in ('forest', 'military'): 49 | # return None 50 | 51 | # strange for natural_reserve: more common this way round... 52 | if value in ('park', 'natural_reserve'): 53 | return ('leisure', value) 54 | elif value == 'field': 55 | # wiki: Although this landuse is rendered by Mapnik, it is not an officially 56 | # recognised OpenStreetMap tag. Please use landuse=farmland instead. 57 | return (key, 'farmland') 58 | elif value in ('grassland', 'scrub'): 59 | return ('natural', value) 60 | else: 61 | return (key, value) 62 | 63 | elif key == 'oneway': 64 | value = tag[1].lower() 65 | if value in ('yes', '1', 'true'): 66 | return (key, 'yes') 67 | else: 68 | drop = True 69 | 70 | elif key == 'area': 71 | value = tag[1].lower() 72 | if value in ('yes', '1', 'true'): 73 | return (key, 'yes') 74 | # might be used to indicate that a closed way is not an area 75 | elif value in ('no'): 76 | return (key, 'no') 77 | else: 78 | drop = True 79 | 80 | elif key == 'bridge': 81 | value = tag[1].lower() 82 | if value in ('yes', '1', 'true'): 83 | return (key, 'yes') 84 | elif value in ('no', '-1', '0', 'false'): 85 | drop = True 86 | else: 87 | return (key, value) 88 | 89 | elif key == 'tunnel': 90 | value = tag[1].lower() 91 | if value in ('yes', '1', 'true'): 92 | return (key, 'yes') 93 | elif value in ('no', '-1', '0', 'false'): 94 | drop = True 95 | else: 96 | return (key, value) 97 | 98 | elif key == 'water': 99 | value = tag[1].lower() 100 | if value in ('lake;pond'): 101 | return (key, 'pond') 102 | else: 103 | return (key, value) 104 | 105 | if drop: 106 | logging.debug('drop tag: %s %s' % (tag[0], tag[1])) 107 | return None 108 | 109 | return tag 110 | -------------------------------------------------------------------------------- /tilequeue/format/OSciMap4/TileData_v4.proto: -------------------------------------------------------------------------------- 1 | // Protocol Version 4 2 | 3 | package org.oscim.database.oscimap4; 4 | 5 | message Data { 6 | message Element { 7 | 8 | // number of geometry 'indices' 9 | optional uint32 num_indices = 1 [default = 1]; 10 | 11 | // number of 'tags' 12 | optional uint32 num_tags = 2 [default = 1]; 13 | 14 | // elevation per coordinate 15 | // (pixel relative to ground meters) 16 | // optional bool has_elevation = 3 [default = false]; 17 | 18 | // reference to tile.tags 19 | repeated uint32 tags = 11 [packed = true]; 20 | 21 | // A list of number of coordinates for each geometry. 22 | // - polygons are separated by one '0' index 23 | // - for single points this can be omitted. 24 | // e.g 2,2 for two lines with two points each, or 25 | // 4,3,0,4,3 for two polygons with four points in 26 | // the outer ring and 3 points in the inner. 27 | 28 | repeated uint32 indices = 12 [packed = true]; 29 | 30 | // single delta encoded coordinate x,y pairs scaled 31 | // to a tile size of 4096 32 | // note: geometries start at x,y = tile size / 2 33 | 34 | repeated sint32 coordinates = 13 [packed = true]; 35 | 36 | //---------------- optional items --------------- 37 | // osm layer [-5 .. 5] -> [0 .. 10] 38 | optional uint32 layer = 21 [default = 5]; 39 | 40 | // intended for symbol and label placement, not used 41 | //optional uint32 rank = 32 [packed = true]; 42 | 43 | // elevation per coordinate 44 | // (pixel relative to ground meters) 45 | // repeated sint32 elevation = 33 [packed = true]; 46 | 47 | // building height, precision 1/10m 48 | //repeated sint32 height = 34 [packed = true]; 49 | 50 | // building height, precision 1/10m 51 | //repeated sint32 min_height = 35 [packed = true]; 52 | } 53 | 54 | required uint32 version = 1; 55 | 56 | // tile creation time 57 | optional uint64 timestamp = 2; 58 | 59 | // tile is completely water (not used yet) 60 | optional bool water = 3; 61 | 62 | // number of 'tags' 63 | required uint32 num_tags = 11; 64 | optional uint32 num_keys = 12 [default = 0]; 65 | optional uint32 num_vals = 13 [default = 0]; 66 | 67 | // strings referenced by tags 68 | repeated string keys = 14; 69 | // separate common attributes from label to 70 | // allow 71 | repeated string values = 15; 72 | 73 | // (key[0xfffffffc] | type[0x03]), value pairs 74 | // key: uint32 -> reference to key-strings 75 | // type 0: attribute -> uint32 reference to value-strings 76 | // type 1: string -> uint32 reference to label-strings 77 | // type 2: sint32 78 | // type 3: float 79 | // value: uint32 interpreted according to 'type' 80 | 81 | repeated uint32 tags = 16 [packed = true]; 82 | 83 | 84 | // linestring 85 | repeated Element lines = 21; 86 | 87 | // polygons (MUST be implicitly closed) 88 | repeated Element polygons = 22; 89 | 90 | // points (POIs) 91 | repeated Element points = 23; 92 | } 93 | -------------------------------------------------------------------------------- /tilequeue/format/OSciMap4/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tilezen/tilequeue/7dd1998e2c5c58182c9db49271b70cac4eab5c9d/tilequeue/format/OSciMap4/__init__.py -------------------------------------------------------------------------------- /tilequeue/format/OSciMap4/pbf_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # protoc --python_out=. proto/TileData.proto 4 | # 5 | import sys 6 | 7 | import TileData_v4_pb2 8 | 9 | if __name__ == '__main__': 10 | if len(sys.argv) != 2: 11 | print>>sys.stderr, 'Usage:', sys.argv[0], '' 12 | sys.exit(1) 13 | 14 | tile = TileData_v4_pb2.Data() 15 | 16 | try: 17 | f = open(sys.argv[1], 'rb') 18 | tile.ParseFromString(f.read()[4:]) 19 | f.close() 20 | except IOError: 21 | print sys.argv[1] + ': Could not open file. Creating a new one.' 22 | 23 | print tile 24 | -------------------------------------------------------------------------------- /tilequeue/format/__init__.py: -------------------------------------------------------------------------------- 1 | from tilequeue.format.geojson import encode_multiple_layers as json_encode_multiple_layers 2 | from tilequeue.format.geojson import encode_single_layer as json_encode_single_layer 3 | from tilequeue.format.mvt import encode as mvt_encode 4 | from tilequeue.format.topojson import encode as topojson_encode 5 | from tilequeue.format.vtm import merge as vtm_encode 6 | try: 7 | from coanacatl import encode as coanacatl_encode 8 | except ImportError: 9 | def coanacatl_encode(feature_layers, bounds_merc, extents): 10 | raise RuntimeError('Could not find coanacatl library.') 11 | 12 | 13 | class OutputFormat(object): 14 | 15 | def __init__(self, name, extension, mimetype, format_fn, sort_key, 16 | supports_shapely_geometry): 17 | self.name = name 18 | self.extension = extension 19 | self.mimetype = mimetype 20 | self.format_fn = format_fn 21 | self.sort_key = sort_key 22 | self.supports_shapely_geometry = supports_shapely_geometry 23 | 24 | def __repr__(self): 25 | return 'OutputFormat(%s, %s, %s)' % \ 26 | (self.name, self.extension, self.mimetype) 27 | 28 | def __hash__(self): 29 | return hash(self.extension) 30 | 31 | def __lt__(self, other): 32 | return self.extension < other.extension 33 | 34 | def __eq__(self, other): 35 | return self.extension == other.extension 36 | 37 | def format_tile(self, tile_data_file, feature_layers, zoom, bounds_merc, 38 | bounds_lnglat, extents=4096): 39 | self.format_fn(tile_data_file, feature_layers, zoom, bounds_merc, 40 | bounds_lnglat, extents) 41 | 42 | 43 | def convert_feature_layers_to_dict(feature_layers): 44 | """takes a list of 'feature_layer' objects and converts to a dict 45 | keyed by the layer name""" 46 | features_by_layer = {} 47 | for feature_layer in feature_layers: 48 | layer_name = feature_layer['name'] 49 | features = feature_layer['features'] 50 | features_by_layer[layer_name] = features 51 | return features_by_layer 52 | 53 | 54 | # consistent facade around all formatters that we use 55 | def format_json(fp, feature_layers, zoom, bounds_merc, bounds_lnglat, extents): 56 | if len(feature_layers) == 1: 57 | json_encode_single_layer(fp, feature_layers[0]['features'], zoom) 58 | return 59 | else: 60 | features_by_layer = convert_feature_layers_to_dict(feature_layers) 61 | json_encode_multiple_layers(fp, features_by_layer, zoom) 62 | 63 | 64 | def format_topojson(fp, feature_layers, zoom, bounds_merc, bounds_lnglat, 65 | extents): 66 | features_by_layer = convert_feature_layers_to_dict(feature_layers) 67 | topojson_encode(fp, features_by_layer, bounds_lnglat, extents) 68 | 69 | 70 | def _make_mvt_layers(feature_layers): 71 | mvt_layers = [] 72 | for feature_layer in feature_layers: 73 | mvt_features = [] 74 | for shape, props, feature_id in feature_layer['features']: 75 | mvt_feature = dict( 76 | geometry=shape, 77 | properties=props, 78 | id=feature_id, 79 | ) 80 | mvt_features.append(mvt_feature) 81 | mvt_layer = dict( 82 | name=feature_layer['name'], 83 | features=mvt_features, 84 | ) 85 | mvt_layers.append(mvt_layer) 86 | return mvt_layers 87 | 88 | 89 | def format_mvt(fp, feature_layers, zoom, bounds_merc, bounds_lnglat, extents): 90 | mvt_layers = _make_mvt_layers(feature_layers) 91 | mvt_encode(fp, mvt_layers, bounds_merc, extents) 92 | 93 | 94 | def format_vtm(fp, feature_layers, zoom, bounds_merc, bounds_lnglat): 95 | vtm_encode(fp, feature_layers) 96 | 97 | 98 | def format_coanacatl(fp, feature_layers, zoom, bounds_merc, bounds_lnglat, 99 | extents): 100 | mvt_layers = _make_mvt_layers(feature_layers) 101 | tile = coanacatl_encode(mvt_layers, bounds_merc, extents) 102 | fp.write(tile) 103 | 104 | 105 | supports_shapely_geom = True 106 | json_format = OutputFormat('JSON', 'json', 'application/json', format_json, 1, 107 | supports_shapely_geom) 108 | topojson_format = OutputFormat('TopoJSON', 'topojson', 'application/json', 109 | format_topojson, 2, supports_shapely_geom) 110 | # TODO image/png mimetype? app doesn't work unless image/png? 111 | vtm_format = OutputFormat('OpenScienceMap', 'vtm', 'image/png', format_vtm, 3, 112 | not supports_shapely_geom) 113 | mvt_format = OutputFormat('MVT', 'mvt', 'application/x-protobuf', 114 | format_mvt, 4, supports_shapely_geom) 115 | # buffered mvt - same exact format as mvt, exception for extension and 116 | # also has separate buffer config 117 | mvtb_format = OutputFormat('MVT Buffered', 'mvtb', 'application/x-protobuf', 118 | format_mvt, 4, supports_shapely_geom) 119 | # package of tiles as a metatile zip 120 | zip_format = OutputFormat('ZIP Metatile', 'zip', 'application/zip', 121 | None, None, None) 122 | # MVT, but written out by coanacatl/wagyu 123 | coanacatl_format = OutputFormat('MVT/Coanacatl', 'mvt', 124 | 'application/x-protobuf', format_coanacatl, 125 | 5, supports_shapely_geom) 126 | 127 | extension_to_format = dict( 128 | json=json_format, 129 | topojson=topojson_format, 130 | vtm=vtm_format, 131 | mvt=mvt_format, 132 | mvtb=mvtb_format, 133 | zip=zip_format, 134 | # NOTE: this isn't actually the extension of the format; coanacatl writes 135 | # files ending '.mvt'. this is just to give us something to call this 136 | # format in config files. 137 | coanacatl=coanacatl_format, 138 | ) 139 | 140 | name_to_format = { 141 | 'JSON': json_format, 142 | 'OpenScienceMap': vtm_format, 143 | 'TopoJSON': topojson_format, 144 | 'MVT': mvt_format, 145 | 'MVT Buffered': mvtb_format, 146 | 'ZIP Metatile': zip_format, 147 | 'MVT/Coanacatl': coanacatl_format, 148 | } 149 | 150 | 151 | def lookup_format_by_extension(extension): 152 | return extension_to_format.get(extension) 153 | 154 | 155 | def lookup_format_by_name(name): 156 | return name_to_format.get(name) 157 | -------------------------------------------------------------------------------- /tilequeue/format/geojson.py: -------------------------------------------------------------------------------- 1 | from math import ceil 2 | from math import log 3 | 4 | import shapely.geometry 5 | import shapely.ops 6 | import shapely.wkb 7 | import ujson as json 8 | 9 | precisions = [int(ceil(log(1 << zoom + 8+2) / log(10)) - 2) 10 | for zoom in range(17)] 11 | # at z16, need to be more precise for metatiling 12 | precisions[16] = 8 13 | 14 | 15 | class JsonFeatureCreator(object): 16 | 17 | def __init__(self, precision=None): 18 | self.precision = precision 19 | 20 | def _trim_precision(self, x, y, z=None): 21 | return round(x, self.precision), round(y, self.precision) 22 | 23 | def __call__(self, feature): 24 | assert len(feature) == 3 25 | wkb_or_shape, props, fid = feature 26 | if isinstance(wkb_or_shape, shapely.geometry.base.BaseGeometry): 27 | shape = wkb_or_shape 28 | else: 29 | shape = shapely.wkb.loads(wkb_or_shape) 30 | 31 | if self.precision: 32 | truncated_precision_shape = shapely.ops.transform( 33 | self._trim_precision, shape) 34 | if truncated_precision_shape.is_valid: 35 | shape = truncated_precision_shape 36 | 37 | geometry = shape.__geo_interface__ 38 | result = dict(type='Feature', properties=props, geometry=geometry) 39 | if fid is not None: 40 | result['id'] = fid 41 | return result 42 | 43 | 44 | def create_layer_feature_collection(features, precision): 45 | create_json_feature = JsonFeatureCreator(precision) 46 | fs = map(create_json_feature, features) 47 | feature_collection = dict(type='FeatureCollection', features=fs) 48 | return feature_collection 49 | 50 | 51 | def precision_for_zoom(zoom): 52 | precision_idx = zoom if 0 <= zoom < len(precisions) else -1 53 | precision = precisions[precision_idx] 54 | return precision 55 | 56 | 57 | def encode_single_layer(out, features, zoom): 58 | """ 59 | Encode a list of (WKB|shapely, property dict, id) features into a 60 | GeoJSON stream. 61 | 62 | If no id is available, pass in None 63 | 64 | Geometries in the features list are assumed to be lon, lats. 65 | """ 66 | precision = precision_for_zoom(zoom) 67 | fs = create_layer_feature_collection(features, precision) 68 | json.dump(fs, out) 69 | 70 | 71 | def encode_multiple_layers(out, features_by_layer, zoom): 72 | """ 73 | features_by_layer should be a dict: layer_name -> feature tuples 74 | """ 75 | precision = precision_for_zoom(zoom) 76 | geojson = {} 77 | for layer_name, features in features_by_layer.items(): 78 | fs = create_layer_feature_collection(features, precision) 79 | geojson[layer_name] = fs 80 | json.dump(geojson, out) 81 | -------------------------------------------------------------------------------- /tilequeue/format/mvt.py: -------------------------------------------------------------------------------- 1 | from mapbox_vector_tile import encode as mvt_encode 2 | from mapbox_vector_tile.encoder import on_invalid_geometry_make_valid 3 | 4 | 5 | def encode(fp, feature_layers, bounds_merc, extents=4096): 6 | tile = mvt_encode( 7 | feature_layers, 8 | quantize_bounds=bounds_merc, 9 | on_invalid_geometry=on_invalid_geometry_make_valid, 10 | round_fn=round, 11 | extents=extents, 12 | ) 13 | fp.write(tile) 14 | -------------------------------------------------------------------------------- /tilequeue/format/topojson.py: -------------------------------------------------------------------------------- 1 | import ujson as json 2 | 3 | 4 | def update_arc_indexes(geometry, merged_arcs, old_arcs): 5 | """ Updated geometry arc indexes, and add arcs to merged_arcs along the way. 6 | 7 | Arguments are modified in-place, and nothing is returned. 8 | """ 9 | if geometry['type'] in ('Point', 'MultiPoint'): 10 | return 11 | 12 | elif geometry['type'] == 'LineString': 13 | for arc_index, old_arc in enumerate(geometry['arcs']): 14 | geometry['arcs'][arc_index] = len(merged_arcs) 15 | merged_arcs.append(old_arcs[old_arc]) 16 | 17 | elif geometry['type'] == 'Polygon': 18 | for ring in geometry['arcs']: 19 | for arc_index, old_arc in enumerate(ring): 20 | ring[arc_index] = len(merged_arcs) 21 | merged_arcs.append(old_arcs[old_arc]) 22 | 23 | elif geometry['type'] == 'MultiLineString': 24 | for part in geometry['arcs']: 25 | for arc_index, old_arc in enumerate(part): 26 | part[arc_index] = len(merged_arcs) 27 | merged_arcs.append(old_arcs[old_arc]) 28 | 29 | elif geometry['type'] == 'MultiPolygon': 30 | for part in geometry['arcs']: 31 | for ring in part: 32 | for arc_index, old_arc in enumerate(ring): 33 | ring[arc_index] = len(merged_arcs) 34 | merged_arcs.append(old_arcs[old_arc]) 35 | 36 | else: 37 | raise NotImplementedError("Can't do %s geometries" % geometry['type']) 38 | 39 | 40 | def get_transform(bounds, size=4096): 41 | """ Return a TopoJSON transform dictionary and a point-transforming function. 42 | 43 | Size is the tile size in pixels and sets the implicit output 44 | resolution. 45 | """ 46 | tx, ty = bounds[0], bounds[1] 47 | sx, sy = (bounds[2] - bounds[0]) / size, (bounds[3] - bounds[1]) / size 48 | 49 | def forward(lon, lat): 50 | """ Transform a longitude and latitude to TopoJSON integer space. 51 | """ 52 | return int(round((lon - tx) / sx)), int(round((lat - ty) / sy)) 53 | 54 | return dict(translate=(tx, ty), scale=(sx, sy)), forward 55 | 56 | 57 | def diff_encode(line, transform): 58 | """ Differentially encode a shapely linestring or ring. 59 | """ 60 | coords = [transform(x, y) for (x, y) in line.coords] 61 | 62 | pairs = zip(coords[:], coords[1:]) 63 | diffs = [(x2 - x1, y2 - y1) for ((x1, y1), (x2, y2)) in pairs] 64 | 65 | return coords[:1] + [(x, y) for (x, y) in diffs if (x, y) != (0, 0)] 66 | 67 | 68 | def encode(file, features_by_layer, bounds, size=4096): 69 | """ Encode a dict of layername: (shape, props, id) features into a 70 | TopoJSON stream. 71 | 72 | If no id is available, pass in None 73 | 74 | Geometries in the features list are assumed to be unprojected 75 | lon, lats. Bounds are given in geographic coordinates as 76 | (xmin, ymin, xmax, ymax). 77 | 78 | Size is the number of integer coordinates which span the extent 79 | of the tile. 80 | """ 81 | transform, forward = get_transform(bounds, size=size) 82 | arcs = [] 83 | 84 | geometries_by_layer = {} 85 | 86 | for layer, features in features_by_layer.iteritems(): 87 | geometries = [] 88 | for shape, props, fid in features: 89 | if shape.type == 'GeometryCollection': 90 | continue 91 | 92 | geometry = dict(properties=props) 93 | 94 | if fid is not None: 95 | geometry['id'] = fid 96 | 97 | elif shape.type == 'Point': 98 | geometry.update(dict( 99 | type='Point', 100 | coordinates=forward(shape.x, shape.y))) 101 | 102 | elif shape.type == 'LineString': 103 | geometry.update(dict(type='LineString', arcs=[len(arcs)])) 104 | arcs.append(diff_encode(shape, forward)) 105 | 106 | elif shape.type == 'Polygon': 107 | geometry.update(dict(type='Polygon', arcs=[])) 108 | 109 | rings = [shape.exterior] + list(shape.interiors) 110 | 111 | for ring in rings: 112 | geometry['arcs'].append([len(arcs)]) 113 | arcs.append(diff_encode(ring, forward)) 114 | 115 | elif shape.type == 'MultiPoint': 116 | geometry.update(dict(type='MultiPoint', coordinates=[])) 117 | 118 | for point in shape.geoms: 119 | geometry['coordinates'].append(forward(point.x, point.y)) 120 | 121 | elif shape.type == 'MultiLineString': 122 | geometry.update(dict(type='MultiLineString', arcs=[])) 123 | 124 | for line in shape.geoms: 125 | geometry['arcs'].append([len(arcs)]) 126 | arcs.append(diff_encode(line, forward)) 127 | 128 | elif shape.type == 'MultiPolygon': 129 | geometry.update(dict(type='MultiPolygon', arcs=[])) 130 | 131 | for polygon in shape.geoms: 132 | rings = [polygon.exterior] + list(polygon.interiors) 133 | polygon_arcs = [] 134 | 135 | for ring in rings: 136 | polygon_arcs.append([len(arcs)]) 137 | arcs.append(diff_encode(ring, forward)) 138 | 139 | geometry['arcs'].append(polygon_arcs) 140 | 141 | else: 142 | raise NotImplementedError("Can't do %s geometries" % 143 | shape.type) 144 | 145 | geometries.append(geometry) 146 | 147 | geometries_by_layer[layer] = dict( 148 | type='GeometryCollection', 149 | geometries=geometries, 150 | ) 151 | 152 | result = dict( 153 | type='Topology', 154 | transform=transform, 155 | objects=geometries_by_layer, 156 | arcs=arcs, 157 | ) 158 | 159 | json.dump(result, file) 160 | -------------------------------------------------------------------------------- /tilequeue/format/vtm.py: -------------------------------------------------------------------------------- 1 | # extracted from mapzen tilestache fork 2 | import logging 3 | import struct 4 | 5 | from OSciMap4 import TileData_v4_pb2 6 | from OSciMap4.GeomEncoder import GeomEncoder 7 | from OSciMap4.StaticKeys import getKeys 8 | from OSciMap4.StaticVals import getValues 9 | from OSciMap4.TagRewrite import fixTag 10 | 11 | statickeys = getKeys() 12 | staticvals = getValues() 13 | 14 | # custom keys/values start at attrib_offset 15 | attrib_offset = 256 16 | 17 | # coordindates are scaled to this range within tile 18 | extents = 4096 19 | 20 | # tiles are padded by this number of pixels for the current zoom level 21 | # (OSciMap uses this to cover up seams between tiles) 22 | padding = 5 23 | 24 | 25 | def encode(file, features, layer_name=''): 26 | layer_name = layer_name or '' 27 | tile = VectorTile(extents) 28 | 29 | for feature in features: 30 | tile.addFeature(feature, layer_name) 31 | 32 | tile.complete() 33 | 34 | data = tile.out.SerializeToString() 35 | file.write(struct.pack('>I', len(data))) 36 | file.write(data) 37 | 38 | 39 | def merge(file, feature_layers): 40 | ''' Retrieve a list of OSciMap4 tile responses and merge them into one. 41 | 42 | get_tiles() retrieves data and performs basic integrity checks. 43 | ''' 44 | tile = VectorTile(extents) 45 | 46 | for layer in feature_layers: 47 | tile.addFeatures(layer['features'], layer['name']) 48 | 49 | tile.complete() 50 | 51 | data = tile.out.SerializeToString() 52 | file.write(struct.pack('>I', len(data))) 53 | file.write(data) 54 | 55 | 56 | class VectorTile: 57 | """ 58 | """ 59 | 60 | def __init__(self, extents): 61 | self.geomencoder = GeomEncoder(extents) 62 | 63 | # TODO count to sort by number of occurrences 64 | self.keydict = {} 65 | self.cur_key = attrib_offset 66 | 67 | self.valdict = {} 68 | self.cur_val = attrib_offset 69 | 70 | self.tagdict = {} 71 | self.num_tags = 0 72 | 73 | self.out = TileData_v4_pb2.Data() 74 | self.out.version = 4 75 | 76 | def complete(self): 77 | if self.num_tags == 0: 78 | logging.info('empty tags') 79 | 80 | self.out.num_tags = self.num_tags 81 | 82 | if self.cur_key - attrib_offset > 0: 83 | self.out.num_keys = self.cur_key - attrib_offset 84 | 85 | if self.cur_val - attrib_offset > 0: 86 | self.out.num_vals = self.cur_val - attrib_offset 87 | 88 | def addFeatures(self, features, this_layer): 89 | for feature in features: 90 | self.addFeature(feature, this_layer) 91 | 92 | def addFeature(self, row, this_layer): 93 | geom = self.geomencoder 94 | tags = [] 95 | 96 | # height = None 97 | layer = None 98 | # add layer tag 99 | tags.append(self.getTagId(('layer_name', this_layer))) 100 | for k, v in row[1].iteritems(): 101 | if v is None: 102 | continue 103 | 104 | # the vtm stylesheet expects the heights to be an integer, 105 | # multiplied by 100 106 | if this_layer == 'buildings' and k in ('height', 'min_height'): 107 | try: 108 | v = int(v * 100) 109 | except ValueError: 110 | logging.warning('vtm: Invalid %s value: %s' % (k, v)) 111 | 112 | tag = str(k), str(v) 113 | 114 | # use unsigned int for layer. i.e. map to 0..10 115 | if 'layer' == tag[0]: 116 | layer = self.getLayer(tag[1]) 117 | continue 118 | 119 | tag = fixTag(tag) 120 | 121 | if tag is None: 122 | continue 123 | 124 | tags.append(self.getTagId(tag)) 125 | 126 | if len(tags) == 0: 127 | logging.debug('missing tags') 128 | return 129 | 130 | geom.parseGeometry(row[0]) 131 | feature = None 132 | 133 | geometry_type = None 134 | if geom.isPoint: 135 | geometry_type = 'Point' 136 | feature = self.out.points.add() 137 | # add number of points (for multi-point) 138 | if len(geom.coordinates) > 2: 139 | logging.info('points %s' % len(geom.coordinates)) 140 | feature.indices.append(len(geom.coordinates)/2) 141 | else: 142 | # empty geometry 143 | if len(geom.index) == 0: 144 | logging.debug('empty geom: %s %s' % row[1]) 145 | return 146 | 147 | if geom.isPoly: 148 | geometry_type = 'Polygon' 149 | feature = self.out.polygons.add() 150 | else: 151 | geometry_type = 'LineString' 152 | feature = self.out.lines.add() 153 | 154 | # add coordinate index list (coordinates per geometry) 155 | feature.indices.extend(geom.index) 156 | 157 | # add indice count (number of geometries) 158 | if len(feature.indices) > 1: 159 | feature.num_indices = len(feature.indices) 160 | 161 | # add coordinates 162 | feature.coordinates.extend(geom.coordinates) 163 | 164 | # add geometry type to tags 165 | geometry_type_tag = 'geometry_type', geometry_type 166 | tags.append(self.getTagId(geometry_type_tag)) 167 | 168 | # add tags 169 | feature.tags.extend(tags) 170 | if len(tags) > 1: 171 | feature.num_tags = len(tags) 172 | 173 | # add osm layer 174 | if layer is not None and layer != 5: 175 | feature.layer = layer 176 | 177 | # logging.debug('tags %d, indices %d' %(len(tags),len(feature.indices))) 178 | 179 | def getLayer(self, val): 180 | try: 181 | layer = max(min(10, int(val)) + 5, 0) 182 | if layer != 0: 183 | return layer 184 | except ValueError: 185 | logging.debug('layer invalid %s' % val) 186 | 187 | return None 188 | 189 | def getKeyId(self, key): 190 | if key in statickeys: 191 | return statickeys[key] 192 | 193 | if key in self.keydict: 194 | return self.keydict[key] 195 | 196 | self.out.keys.append(key) 197 | 198 | r = self.cur_key 199 | self.keydict[key] = r 200 | self.cur_key += 1 201 | return r 202 | 203 | def getAttribId(self, var): 204 | if var in staticvals: 205 | return staticvals[var] 206 | 207 | if var in self.valdict: 208 | return self.valdict[var] 209 | 210 | self.out.values.append(var) 211 | 212 | r = self.cur_val 213 | self.valdict[var] = r 214 | self.cur_val += 1 215 | return r 216 | 217 | def getTagId(self, tag): 218 | # logging.debug(tag) 219 | 220 | if tag in self.tagdict: 221 | return self.tagdict[tag] 222 | 223 | key = self.getKeyId(tag[0].decode('utf-8')) 224 | val = self.getAttribId(tag[1].decode('utf-8')) 225 | 226 | self.out.tags.append(key) 227 | self.out.tags.append(val) 228 | # logging.info("add tag %s - %d/%d" %(tag, key, val)) 229 | r = self.num_tags 230 | self.tagdict[tag] = r 231 | self.num_tags += 1 232 | return r 233 | -------------------------------------------------------------------------------- /tilequeue/metatile.py: -------------------------------------------------------------------------------- 1 | import cStringIO as StringIO 2 | import zipfile 3 | from collections import defaultdict 4 | from time import gmtime 5 | 6 | from tilequeue.format import zip_format 7 | 8 | 9 | def make_multi_metatile(parent, tiles, date_time=None): 10 | """ 11 | Make a metatile containing a list of tiles all having the same layer, 12 | with coordinates relative to the given parent. Set date_time to a 6-tuple 13 | of (year, month, day, hour, minute, second) to set the timestamp for 14 | members. Otherwise the current wall clock time is used. 15 | """ 16 | 17 | assert parent is not None, \ 18 | 'Parent tile must be provided and not None to make a metatile.' 19 | 20 | if len(tiles) == 0: 21 | return [] 22 | 23 | if date_time is None: 24 | date_time = gmtime()[0:6] 25 | 26 | layer = tiles[0]['layer'] 27 | 28 | buf = StringIO.StringIO() 29 | with zipfile.ZipFile(buf, mode='w') as z: 30 | for tile in tiles: 31 | assert tile['layer'] == layer 32 | 33 | coord = tile['coord'] 34 | 35 | # change in zoom level from parent to coord. since parent should 36 | # be a parent, its zoom should always be equal or smaller to that 37 | # of coord. 38 | delta_z = coord.zoom - parent.zoom 39 | assert delta_z >= 0, 'Coordinates must be descendents of parent' 40 | 41 | # change in row/col coordinates are relative to the upper left 42 | # coordinate at that zoom. both should be positive. 43 | delta_row = coord.row - (int(parent.row) << delta_z) 44 | delta_column = coord.column - (int(parent.column) << delta_z) 45 | assert delta_row >= 0, \ 46 | 'Coordinates must be contained by their parent, but ' + \ 47 | 'row is not.' 48 | assert delta_column >= 0, \ 49 | 'Coordinates must be contained by their parent, but ' + \ 50 | 'column is not.' 51 | 52 | tile_name = '%d/%d/%d.%s' % \ 53 | (delta_z, delta_column, delta_row, tile['format'].extension) 54 | tile_data = tile['tile'] 55 | info = zipfile.ZipInfo(tile_name, date_time) 56 | z.writestr(info, tile_data, zipfile.ZIP_DEFLATED) 57 | 58 | return [dict(tile=buf.getvalue(), format=zip_format, coord=parent, 59 | layer=layer)] 60 | 61 | 62 | def common_parent(a, b): 63 | """ 64 | Find the common parent tile of both a and b. The common parent is the tile 65 | at the highest zoom which both a and b can be transformed into by lowering 66 | their zoom levels. 67 | """ 68 | 69 | if a.zoom < b.zoom: 70 | b = b.zoomTo(a.zoom).container() 71 | 72 | elif a.zoom > b.zoom: 73 | a = a.zoomTo(b.zoom).container() 74 | 75 | while a.row != b.row or a.column != b.column: 76 | a = a.zoomBy(-1).container() 77 | b = b.zoomBy(-1).container() 78 | 79 | # by this point a == b. 80 | return a 81 | 82 | 83 | def _parent_tile(tiles): 84 | """ 85 | Find the common parent tile for a sequence of tiles. 86 | """ 87 | parent = None 88 | for t in tiles: 89 | if parent is None: 90 | parent = t 91 | 92 | else: 93 | parent = common_parent(parent, t) 94 | 95 | return parent 96 | 97 | 98 | def make_metatiles(size, tiles, date_time=None): 99 | """ 100 | Group by layers, and make metatiles out of all the tiles which share those 101 | properties relative to the "top level" tile which is parent of them all. 102 | Provide a 6-tuple date_time to set the timestamp on each tile within the 103 | metatile, or leave it as None to use the current time. 104 | """ 105 | 106 | groups = defaultdict(list) 107 | for tile in tiles: 108 | key = tile['layer'] 109 | groups[key].append(tile) 110 | 111 | metatiles = [] 112 | for group in groups.itervalues(): 113 | parent = _parent_tile(t['coord'] for t in group) 114 | metatiles.extend(make_multi_metatile(parent, group, date_time)) 115 | 116 | return metatiles 117 | 118 | 119 | def extract_metatile(io, fmt, offset=None): 120 | """ 121 | Extract the tile at the given offset (defaults to 0/0/0) and format from 122 | the metatile in the file-like object io. 123 | """ 124 | 125 | ext = fmt.extension 126 | if offset is None: 127 | tile_name = '0/0/0.%s' % ext 128 | else: 129 | tile_name = '%d/%d/%d.%s' % (offset.zoom, offset.column, offset.row, 130 | ext) 131 | 132 | with zipfile.ZipFile(io, mode='r') as zf: 133 | if tile_name in zf.namelist(): 134 | return zf.read(tile_name) 135 | else: 136 | return None 137 | 138 | 139 | def _metatile_contents_equal(zip_1, zip_2): 140 | """ 141 | Given two open zip files as arguments, this returns True if the zips 142 | both contain the same set of files, having the same names, and each 143 | file within the zip is byte-wise identical to the one with the same 144 | name in the other zip. 145 | """ 146 | 147 | names_1 = set(zip_1.namelist()) 148 | names_2 = set(zip_2.namelist()) 149 | 150 | if names_1 != names_2: 151 | return False 152 | 153 | for n in names_1: 154 | bytes_1 = zip_1.read(n) 155 | bytes_2 = zip_2.read(n) 156 | 157 | if bytes_1 != bytes_2: 158 | return False 159 | 160 | return True 161 | 162 | 163 | def metatiles_are_equal(tile_data_1, tile_data_2): 164 | """ 165 | Return True if the two tiles are both zipped metatiles and contain the 166 | same set of files with the same contents. This ignores the timestamp of 167 | the individual files in the zip files, as well as their order or any 168 | other metadata. 169 | """ 170 | 171 | try: 172 | buf_1 = StringIO.StringIO(tile_data_1) 173 | buf_2 = StringIO.StringIO(tile_data_2) 174 | 175 | with zipfile.ZipFile(buf_1, mode='r') as zip_1: 176 | with zipfile.ZipFile(buf_2, mode='r') as zip_2: 177 | return _metatile_contents_equal(zip_1, zip_2) 178 | 179 | except (StandardError, zipfile.BadZipFile, zipfile.LargeZipFile): 180 | # errors, such as files not being proper zip files, or missing 181 | # some attributes or contents that we expect, are treated as not 182 | # equal. 183 | pass 184 | 185 | return False 186 | -------------------------------------------------------------------------------- /tilequeue/metro_extract.py: -------------------------------------------------------------------------------- 1 | from json import load 2 | 3 | 4 | class MetroExtractCity(object): 5 | 6 | def __init__(self, region, city, bounds): 7 | self.region = region 8 | self.city = city 9 | self.bounds = bounds 10 | 11 | def __repr__(self): 12 | return 'MetroExtractCity(%s, %s, %s)' % \ 13 | (self.region, self.city, self.bounds) 14 | 15 | 16 | class MetroExtractParseError(Exception): 17 | 18 | def __init__(self, cause): 19 | self.cause = cause 20 | 21 | def __repr__(self): 22 | return 'MetroExtractParseError(%s: %s)' % ( 23 | self.cause.__class__.__name__, str(self.cause)) 24 | 25 | 26 | def parse_metro_extract(metro_extract_fp): 27 | json_data = load(metro_extract_fp) 28 | metros = [] 29 | try: 30 | regions = json_data[u'regions'] 31 | for region_name, region_data in regions.iteritems(): 32 | cities = region_data[u'cities'] 33 | for city_name, city_data in cities.iteritems(): 34 | city_json_bounds = city_data[u'bbox'] 35 | minx = float(city_json_bounds[u'left']) 36 | miny = float(city_json_bounds[u'bottom']) 37 | maxx = float(city_json_bounds[u'right']) 38 | maxy = float(city_json_bounds[u'top']) 39 | city_bounds = (minx, miny, maxx, maxy) 40 | metro = MetroExtractCity(region_name, city_name, city_bounds) 41 | metros.append(metro) 42 | except (KeyError, ValueError), e: 43 | raise MetroExtractParseError(e) 44 | return metros 45 | 46 | 47 | def city_bounds(metro_extract_cities): 48 | return [city.bounds for city in metro_extract_cities] 49 | -------------------------------------------------------------------------------- /tilequeue/query/__init__.py: -------------------------------------------------------------------------------- 1 | from tilequeue.process import Source 2 | from tilequeue.query.fixture import make_fixture_data_fetcher 3 | from tilequeue.query.pool import DBConnectionPool 4 | from tilequeue.query.postgres import make_db_data_fetcher 5 | from tilequeue.query.rawr import make_rawr_data_fetcher 6 | from tilequeue.query.split import make_split_data_fetcher 7 | from tilequeue.store import make_s3_tile_key_generator 8 | from tilequeue.utils import AwsSessionHelper 9 | 10 | 11 | __all__ = [ 12 | 'DBConnectionPool', 13 | 'make_db_data_fetcher', 14 | 'make_fixture_data_fetcher', 15 | 'make_data_fetcher', 16 | ] 17 | 18 | 19 | def make_data_fetcher(cfg, layer_data, query_cfg, io_pool, 20 | s3_role_arn=None, 21 | s3_role_session_duration_s=None): 22 | """ Make data fetcher from RAWR store and PostgreSQL database. 23 | When s3_role_arn and s3_role_session_duration_s are available 24 | the RAWR store will use the s3_role_arn to access the RAWR S3 bucket 25 | """ 26 | db_fetcher = make_db_data_fetcher( 27 | cfg.postgresql_conn_info, cfg.template_path, cfg.reload_templates, 28 | query_cfg, io_pool) 29 | 30 | if cfg.yml.get('use-rawr-tiles'): 31 | rawr_fetcher = _make_rawr_fetcher( 32 | cfg, layer_data, s3_role_arn, s3_role_session_duration_s) 33 | 34 | group_by_zoom = cfg.yml.get('rawr').get('group-zoom') 35 | assert group_by_zoom is not None, 'Missing group-zoom rawr config' 36 | return make_split_data_fetcher( 37 | group_by_zoom, db_fetcher, rawr_fetcher) 38 | 39 | else: 40 | return db_fetcher 41 | 42 | 43 | class _NullRawrStorage(object): 44 | 45 | def __init__(self, data_source, table_sources): 46 | self.data_source = data_source 47 | self.table_sources = table_sources 48 | 49 | def __call__(self, tile): 50 | # returns a "tables" object, which responds to __call__(table_name) 51 | # with tuples for that table. 52 | data = {} 53 | for location in self.data_source(tile): 54 | data[location.name] = location.records 55 | 56 | def _tables(table_name): 57 | from tilequeue.query.common import Table 58 | source = self.table_sources[table_name] 59 | return Table(source, data.get(table_name, [])) 60 | 61 | return _tables 62 | 63 | 64 | def _make_rawr_fetcher(cfg, layer_data, 65 | s3_role_arn=None, 66 | s3_role_session_duration_s=None): 67 | """ 68 | When s3_role_arn and s3_role_session_duration_s are available 69 | the RAWR store will use the s3_role_arn to access the RAWR S3 70 | bucket 71 | """ 72 | rawr_yaml = cfg.yml.get('rawr') 73 | assert rawr_yaml is not None, 'Missing rawr configuration in yaml' 74 | 75 | group_by_zoom = rawr_yaml.get('group-zoom') 76 | assert group_by_zoom is not None, 'Missing group-zoom rawr config' 77 | 78 | rawr_source_yaml = rawr_yaml.get('source') 79 | assert rawr_source_yaml, 'Missing rawr source config' 80 | 81 | table_sources = rawr_source_yaml.get('table-sources') 82 | assert table_sources, 'Missing definitions of source per table' 83 | 84 | # map text for table source onto Source objects 85 | for tbl, data in table_sources.items(): 86 | source_name = data['name'] 87 | source_value = data['value'] 88 | table_sources[tbl] = Source(source_name, source_value) 89 | 90 | label_placement_layers = rawr_yaml.get('label-placement-layers', {}) 91 | for geom_type, layers in label_placement_layers.items(): 92 | assert geom_type in ('point', 'polygon', 'linestring'), \ 93 | 'Geom type %r not understood, expecting point, polygon or ' \ 94 | 'linestring.' % (geom_type,) 95 | label_placement_layers[geom_type] = set(layers) 96 | 97 | indexes_cfg = rawr_yaml.get('indexes') 98 | assert indexes_cfg, 'Missing definitions of table indexes.' 99 | 100 | # source types are: 101 | # s3 - to fetch RAWR tiles from S3 102 | # store - to fetch RAWR tiles from any tilequeue tile source 103 | # generate - to generate RAWR tiles directly, rather than trying to load 104 | # them from S3. this can be useful for standalone use and 105 | # testing. provide a postgresql subkey for database connection 106 | # settings. 107 | source_type = rawr_source_yaml.get('type') 108 | 109 | if source_type == 's3': 110 | rawr_source_s3_yaml = rawr_source_yaml.get('s3') 111 | bucket = rawr_source_s3_yaml.get('bucket') 112 | assert bucket, 'Missing rawr source s3 bucket' 113 | region = rawr_source_s3_yaml.get('region') 114 | assert region, 'Missing rawr source s3 region' 115 | prefix = rawr_source_s3_yaml.get('prefix') 116 | assert prefix, 'Missing rawr source s3 prefix' 117 | extension = rawr_source_s3_yaml.get('extension') 118 | assert extension, 'Missing rawr source s3 extension' 119 | allow_missing_tiles = rawr_source_s3_yaml.get( 120 | 'allow-missing-tiles', False) 121 | 122 | import boto3 123 | from tilequeue.rawr import RawrS3Source 124 | if s3_role_arn: 125 | # use provided role to access S3 126 | assert s3_role_session_duration_s, \ 127 | 's3_role_session_duration_s is either None or 0' 128 | aws_helper = AwsSessionHelper('tilequeue_dataaccess', 129 | s3_role_arn, 130 | region, 131 | s3_role_session_duration_s) 132 | s3_client = aws_helper.get_client('s3') 133 | else: 134 | s3_client = boto3.client('s3', region_name=region) 135 | 136 | tile_key_gen = make_s3_tile_key_generator(rawr_source_s3_yaml) 137 | storage = RawrS3Source( 138 | s3_client, bucket, prefix, extension, table_sources, tile_key_gen, 139 | allow_missing_tiles) 140 | 141 | elif source_type == 'generate': 142 | from raw_tiles.source.conn import ConnectionContextManager 143 | from raw_tiles.source.osm import OsmSource 144 | 145 | postgresql_cfg = rawr_source_yaml.get('postgresql') 146 | assert postgresql_cfg, 'Missing rawr postgresql config' 147 | 148 | conn_ctx = ConnectionContextManager(postgresql_cfg) 149 | rawr_osm_source = OsmSource(conn_ctx) 150 | storage = _NullRawrStorage(rawr_osm_source, table_sources) 151 | 152 | elif source_type == 'store': 153 | from tilequeue.store import make_store 154 | from tilequeue.rawr import RawrStoreSource 155 | 156 | store_cfg = rawr_source_yaml.get('store') 157 | store = make_store(store_cfg) 158 | storage = RawrStoreSource(store, table_sources) 159 | 160 | else: 161 | assert False, 'Source type %r not understood. ' \ 162 | 'Options are s3, generate and store.' % (source_type,) 163 | 164 | # TODO: this needs to be configurable, everywhere! this is a long term 165 | # refactor - it's hard-coded in a bunch of places :-( 166 | max_z = 16 167 | 168 | layers = _make_layer_info(layer_data, cfg.process_yaml_cfg) 169 | 170 | return make_rawr_data_fetcher( 171 | group_by_zoom, max_z, storage, layers, indexes_cfg, 172 | label_placement_layers) 173 | 174 | 175 | def _make_layer_info(layer_data, process_yaml_cfg): 176 | from tilequeue.query.common import LayerInfo, ShapeType 177 | 178 | layers = {} 179 | functions = _parse_yaml_functions(process_yaml_cfg) 180 | 181 | for layer_datum in layer_data: 182 | # preprocessed layers are not from the database and not found in the rawr tile 183 | if layer_datum.get('pre_processed_layer_path') is not None: 184 | continue 185 | name = layer_datum['name'] 186 | min_zoom_fn, props_fn = functions[name] 187 | shape_types = ShapeType.parse_set(layer_datum['geometry_types']) 188 | layer_info = LayerInfo(min_zoom_fn, props_fn, shape_types) 189 | layers[name] = layer_info 190 | 191 | return layers 192 | 193 | 194 | def _parse_yaml_functions(process_yaml_cfg): 195 | from tilequeue.command import make_output_calc_mapping 196 | from tilequeue.command import make_min_zoom_calc_mapping 197 | 198 | output_layer_data = make_output_calc_mapping(process_yaml_cfg) 199 | min_zoom_layer_data = make_min_zoom_calc_mapping(process_yaml_cfg) 200 | 201 | keys = set(output_layer_data.keys()) 202 | assert keys == set(min_zoom_layer_data.keys()) 203 | 204 | functions = {} 205 | for key in keys: 206 | min_zoom_fn = min_zoom_layer_data[key] 207 | output_fn = output_layer_data[key] 208 | functions[key] = (min_zoom_fn, output_fn) 209 | 210 | return functions 211 | -------------------------------------------------------------------------------- /tilequeue/query/pool.py: -------------------------------------------------------------------------------- 1 | import random 2 | import threading 3 | from itertools import cycle 4 | from itertools import islice 5 | 6 | import psycopg2 7 | import ujson 8 | from psycopg2.extras import register_hstore 9 | from psycopg2.extras import register_json 10 | 11 | 12 | class ConnectionsContextManager(object): 13 | 14 | """Handle automatically closing connections via with statement""" 15 | 16 | def __init__(self, conns): 17 | self.conns = conns 18 | 19 | def __enter__(self): 20 | return self.conns 21 | 22 | def __exit__(self, exc_type, exc_val, exc_tb): 23 | for conn in self.conns: 24 | try: 25 | conn.close() 26 | except Exception: 27 | pass 28 | suppress_exception = False 29 | return suppress_exception 30 | 31 | 32 | class DBConnectionPool(object): 33 | 34 | """Manage database connections with varying database names""" 35 | 36 | def __init__(self, dbnames, conn_info, readonly=True): 37 | self.dbnames = cycle(dbnames) 38 | self.conn_info = conn_info 39 | self.conn_mapping = {} 40 | self.lock = threading.Lock() 41 | self.readonly = readonly 42 | 43 | def _make_conn(self, conn_info): 44 | # if multiple hosts are provided, select one at random as a kind of 45 | # simple load balancing. 46 | host = conn_info.get('host') 47 | if host and isinstance(host, list): 48 | host = random.choice(host) 49 | conn_info = conn_info.copy() 50 | conn_info['host'] = host 51 | 52 | conn = psycopg2.connect(**conn_info) 53 | conn.set_session(readonly=self.readonly, autocommit=True) 54 | register_hstore(conn) 55 | register_json(conn, loads=ujson.loads) 56 | return conn 57 | 58 | def get_conns(self, n_conn): 59 | with self.lock: 60 | dbnames = list(islice(self.dbnames, n_conn)) 61 | conns = [] 62 | for dbname in dbnames: 63 | conn_info_with_db = dict(self.conn_info, dbname=dbname) 64 | conn = self._make_conn(conn_info_with_db) 65 | conns.append(conn) 66 | conns_ctx_mgr = ConnectionsContextManager(conns) 67 | return conns_ctx_mgr 68 | -------------------------------------------------------------------------------- /tilequeue/query/split.py: -------------------------------------------------------------------------------- 1 | from itertools import chain 2 | 3 | 4 | class DataFetcher(object): 5 | 6 | """ 7 | Splits requests between two data fetchers depending on whether the zoom of 8 | the coordinate is above or below (or equal) the "split zoom". 9 | 10 | This is used to choose between going to the database for low zooms or 11 | fetching a RAWR tile for high zooms. 12 | """ 13 | 14 | def __init__(self, split_zoom, below_fetcher, above_fetcher): 15 | self.split_zoom = split_zoom 16 | self.below_fetcher = below_fetcher 17 | self.above_fetcher = above_fetcher 18 | 19 | def fetch_tiles(self, all_data): 20 | below_data = [] 21 | above_data = [] 22 | 23 | for data in all_data: 24 | coord = data['coord'] 25 | if coord.zoom < self.split_zoom: 26 | below_data.append(data) 27 | else: 28 | above_data.append(data) 29 | 30 | return chain(self.above_fetcher.fetch_tiles(above_data), 31 | self.below_fetcher.fetch_tiles(below_data)) 32 | 33 | 34 | def make_split_data_fetcher(split_zoom, below_fetcher, above_fetcher): 35 | return DataFetcher(split_zoom, below_fetcher, above_fetcher) 36 | -------------------------------------------------------------------------------- /tilequeue/queue/__init__.py: -------------------------------------------------------------------------------- 1 | from message import MessageHandle # noreorder 2 | from file import OutputFileQueue 3 | from memory import MemoryQueue 4 | from redis_queue import make_redis_queue 5 | from sqs import JobProgressException 6 | from sqs import make_sqs_queue 7 | from sqs import make_visibility_manager 8 | from sqs import SqsQueue 9 | 10 | __all__ = [ 11 | JobProgressException, 12 | make_redis_queue, 13 | make_sqs_queue, 14 | make_visibility_manager, 15 | MemoryQueue, 16 | MessageHandle, 17 | OutputFileQueue, 18 | SqsQueue, 19 | ] 20 | -------------------------------------------------------------------------------- /tilequeue/queue/file.py: -------------------------------------------------------------------------------- 1 | import threading 2 | 3 | from tilequeue.queue import MessageHandle 4 | 5 | 6 | class OutputFileQueue(object): 7 | ''' 8 | A local, file-based queue for storing the coordinates of tiles to render. 9 | Can be used as a drop-in replacement for `tilequeue.queue.sqs.SqsQueue`. 10 | Note that it doesn't support reading/writing from multiple `tilequeue` 11 | instances; you *can* `seed` and `process` at the same time, but you *can't* 12 | run more than one `seed` or `write` instance at the same time. This is 13 | primarily meant for development/debugging, so adding multi-process locking 14 | probably isn't worth the complexity. 15 | ''' 16 | 17 | def __init__(self, fp, read_size=10): 18 | self.read_size = read_size 19 | self.fp = fp 20 | self.lock = threading.RLock() 21 | 22 | def enqueue(self, payload): 23 | with self.lock: 24 | self.fp.write(payload + '\n') 25 | 26 | def enqueue_batch(self, payloads): 27 | n = 0 28 | for payload in payloads: 29 | self.enqueue(payload) 30 | n += 1 31 | return n, 0 32 | 33 | def read(self): 34 | with self.lock: 35 | msg_handles = [] 36 | for _ in range(self.read_size): 37 | payload = self.fp.readline().strip() 38 | if payload: 39 | msg_handle = MessageHandle(None, payload) 40 | msg_handles.append(msg_handle) 41 | 42 | return msg_handles 43 | 44 | def job_done(self, msg_handle): 45 | pass 46 | 47 | def job_progress(self, handle): 48 | pass 49 | 50 | def clear(self): 51 | with self.lock: 52 | self.fp.seek(0) 53 | self.fp.truncate() 54 | return -1 55 | 56 | def close(self): 57 | with self.lock: 58 | self.clear() 59 | self.fp.close() 60 | -------------------------------------------------------------------------------- /tilequeue/queue/inflight.py: -------------------------------------------------------------------------------- 1 | from tilequeue.tile import coord_marshall_int 2 | from tilequeue.utils import grouper 3 | 4 | 5 | class RedisInFlightManager(object): 6 | 7 | """ 8 | manage in flight list 9 | 10 | Manage the two operations for the inflight list: 11 | 12 | 1. filter coordinates out that are already in flight 13 | 2. mark coordinates as in flight (presumably these were just enqueued) 14 | """ 15 | 16 | def __init__(self, redis_client, inflight_key, chunk_size=100): 17 | self.redis_client = redis_client 18 | self.inflight_key = inflight_key 19 | self.chunk_size = chunk_size 20 | 21 | def is_inflight(self, coord): 22 | coord_int = coord_marshall_int(coord) 23 | return self.redis_client.sismember(self.inflight_key, coord_int) 24 | 25 | def filter(self, coords): 26 | for coord in coords: 27 | if not self.is_inflight(coord): 28 | yield coord 29 | 30 | def mark_inflight(self, coords): 31 | for coords_chunk in grouper(coords, self.chunk_size): 32 | coord_ints = map(coord_marshall_int, coords_chunk) 33 | self.redis_client.sadd(self.inflight_key, *coord_ints) 34 | 35 | def unmark_inflight(self, coord): 36 | coord_int = coord_marshall_int(coord) 37 | self.redis_client.srem(self.inflight_key, coord_int) 38 | 39 | 40 | class NoopInFlightManager(object): 41 | 42 | def filter(self, coords): 43 | return coords 44 | 45 | def is_inflight(self, coord_int): 46 | return False 47 | 48 | def mark_inflight(self, coords): 49 | pass 50 | 51 | def unmark_inflight(self, coord): 52 | pass 53 | -------------------------------------------------------------------------------- /tilequeue/queue/mapper.py: -------------------------------------------------------------------------------- 1 | from collections import defaultdict 2 | from collections import namedtuple 3 | 4 | from tilequeue.tile import coord_marshall_int 5 | 6 | 7 | # this is what gets returned by the group function 8 | # each group represents what the payload to the queue should be 9 | # the queue_id exists to allow dispatching to different underlying 10 | # tile queue implementations. This can be useful when having several 11 | # priority queues, each dedicated to their own zoom level range 12 | CoordGroup = namedtuple('CoordGroup', 'coords queue_id') 13 | 14 | 15 | class SingleQueueMapper(object): 16 | 17 | def __init__(self, queue_name, tile_queue): 18 | self.queue_name = queue_name 19 | self.tile_queue = tile_queue 20 | 21 | def group(self, coords): 22 | for coord in coords: 23 | yield CoordGroup([coord], self.queue_name) 24 | 25 | def get_queue(self, queue_id): 26 | assert queue_id == self.queue_name, 'Unknown queue_id: %s' % queue_id 27 | return self.tile_queue 28 | 29 | def queues_in_priority_order(self): 30 | return ((self.queue_name, self.tile_queue),) 31 | 32 | 33 | # what gets passed into the zoom range mapper 34 | # pass in None for start/end to add queues that are read from 35 | # but that aren't considered for enqueueing directly when dispatching 36 | # eg a high priority queue 37 | ZoomRangeQueueSpec = namedtuple( 38 | 'ZoomRangeCfgSpec', 39 | 'start end queue_name queue group_by_zoom in_toi') 40 | # set the last parameter to default to None 41 | ZoomRangeQueueSpec.__new__.__defaults__ = (None,) 42 | 43 | # what the mapper uses internally 44 | # these are what will get checked for queue dispatch 45 | ZoomRangeQueueItem = namedtuple( 46 | 'ZoomRangeItem', 47 | 'start end queue_id group_by_zoom in_toi') 48 | 49 | 50 | class ZoomRangeAndZoomGroupQueueMapper(object): 51 | 52 | def __init__(self, zoom_range_specs, toi=None): 53 | # NOTE: zoom_range_specs should be passed in priority order 54 | self.zoom_range_items = [] 55 | self.queue_mapping = [] 56 | for i, zrs in enumerate(zoom_range_specs): 57 | self.queue_mapping.append(zrs.queue) 58 | zri = ZoomRangeQueueItem(zrs.start, zrs.end, i, zrs.group_by_zoom, 59 | zrs.in_toi) 60 | self.zoom_range_items.append(zri) 61 | 62 | # check that if any queue item uses the TOI as part of the check, then 63 | # we have been passed a TOI object to check it against. 64 | uses_toi = any(zri.in_toi is not None for zri in self.zoom_range_items) 65 | if uses_toi: 66 | assert toi is not None, 'If any zoom range item depends on ' \ 67 | 'whether a coordinate is in the TOI then a TOI object must ' \ 68 | 'be provided, but there is only None.' 69 | 70 | # NOTE: this is a one-off operation, so for long-running processes, 71 | # we must either re-create the mapper object, or periodically 72 | # refresh the TOI set. 73 | self.toi_set = toi.fetch_tiles_of_interest() 74 | 75 | def group(self, coords): 76 | """return CoordGroups that can be used to send to queues 77 | 78 | Each CoordGroup represents a message that can be sent to a 79 | particular queue, stamped with the queue_id. The list of 80 | coords, which can be 1, is what should get used for the 81 | payload for each queue message. 82 | """ 83 | 84 | groups = [] 85 | for i in range(len(self.zoom_range_items)): 86 | groups.append([]) 87 | 88 | # first group the coordinates based on their queue 89 | for coord in coords: 90 | for i, zri in enumerate(self.zoom_range_items): 91 | toi_match = zri.in_toi is None or \ 92 | (coord in self.toi_set) == zri.in_toi 93 | if zri.start <= coord.zoom < zri.end and toi_match: 94 | groups[i].append(coord) 95 | break 96 | 97 | # now, we need to just verify that for each particular group, 98 | # should they be further grouped, eg by a particular zoom 10 99 | # tile 100 | for i, zri in enumerate(self.zoom_range_items): 101 | group = groups[i] 102 | if not group: 103 | continue 104 | if zri.group_by_zoom is None: 105 | for coord in group: 106 | yield CoordGroup([coord], zri.queue_id) 107 | else: 108 | by_parent_coords = defaultdict(list) 109 | for coord in group: 110 | if coord.zoom >= zri.group_by_zoom: 111 | group_coord = coord.zoomTo(zri.group_by_zoom) 112 | group_key = coord_marshall_int(group_coord) 113 | by_parent_coords[group_key].append(coord) 114 | else: 115 | # this means that a coordinate belonged to a 116 | # particular queue but the zoom was lower than 117 | # the group by zoom 118 | # this probably shouldn't happen 119 | # should it be an assert instead? 120 | yield CoordGroup([coord], zri.queue_id) 121 | 122 | for group_key, coords in by_parent_coords.iteritems(): 123 | yield CoordGroup(coords, zri.queue_id) 124 | 125 | def get_queue(self, queue_id): 126 | assert 0 <= queue_id < len(self.queue_mapping) 127 | return self.queue_mapping[queue_id] 128 | 129 | def queues_in_priority_order(self): 130 | return enumerate(self.queue_mapping) 131 | -------------------------------------------------------------------------------- /tilequeue/queue/memory.py: -------------------------------------------------------------------------------- 1 | from tilequeue.queue import MessageHandle 2 | 3 | 4 | class MemoryQueue(object): 5 | 6 | def __init__(self): 7 | self.q = [] 8 | 9 | def enqueue(self, payload): 10 | self.q.append(payload) 11 | 12 | def enqueue_batch(self, payloads): 13 | for payload in payloads: 14 | self.enqueue(payload) 15 | 16 | def read(self): 17 | max_to_read = 10 18 | self.q, payloads = self.q[max_to_read:], self.q[:max_to_read] 19 | return [MessageHandle(None, payload) for payload in payloads] 20 | 21 | def job_done(self, msg_handle): 22 | pass 23 | 24 | def job_progress(self, handle): 25 | pass 26 | 27 | def clear(self): 28 | n = len(self.q) 29 | del self.q[:] 30 | return n 31 | 32 | def close(self): 33 | pass 34 | -------------------------------------------------------------------------------- /tilequeue/queue/message.py: -------------------------------------------------------------------------------- 1 | import threading 2 | from collections import namedtuple 3 | 4 | from tilequeue.tile import deserialize_coord 5 | from tilequeue.tile import serialize_coord 6 | 7 | 8 | class MessageHandle(object): 9 | 10 | """ 11 | represents a message read from a queue 12 | 13 | This encapsulates both the payload and an opaque message handle 14 | that's queue specific. When a job is complete, this handle is 15 | given back to the queue, to allow for implementations to mark 16 | completion for those that support it. 17 | """ 18 | 19 | def __init__(self, handle, payload, metadata=None): 20 | # metadata is optional, and can capture information like the 21 | # timestamp and age of the message, which can be useful to log 22 | self.handle = handle 23 | self.payload = payload 24 | self.metadata = metadata 25 | 26 | 27 | class QueueHandle(object): 28 | """ 29 | message handle combined with a queue id 30 | """ 31 | 32 | def __init__(self, queue_id, handle): 33 | self.queue_id = queue_id 34 | self.handle = handle 35 | 36 | 37 | class SingleMessageMarshaller(object): 38 | 39 | """marshall/unmarshall a single coordinate from a queue message""" 40 | 41 | def marshall(self, coords): 42 | assert len(coords) == 1 43 | coord = coords[0] 44 | return serialize_coord(coord) 45 | 46 | def unmarshall(self, payload): 47 | coord = deserialize_coord(payload) 48 | assert coord 49 | return [coord] 50 | 51 | 52 | class CommaSeparatedMarshaller(object): 53 | 54 | """ 55 | marshall/unmarshall coordinates in a comma separated format 56 | 57 | coordinates are represented textually as z/x/y separated by commas 58 | """ 59 | 60 | def marshall(self, coords): 61 | return ','.join(serialize_coord(x) for x in coords) 62 | 63 | def unmarshall(self, payload): 64 | coord_strs = payload.split(',') 65 | coords = [] 66 | for coord_str in coord_strs: 67 | coord_str = coord_str.strip() 68 | if coord_str: 69 | coord = deserialize_coord(coord_str) 70 | assert coord 71 | coords.append(coord) 72 | return coords 73 | 74 | 75 | MessageDoneResult = namedtuple( 76 | 'MessageDoneResult', 77 | ('queue_handle all_done parent_tile')) 78 | 79 | 80 | class SingleMessagePerCoordTracker(object): 81 | 82 | """ 83 | one-to-one mapping between queue handles and coordinates 84 | """ 85 | 86 | def track(self, queue_handle, coords, parent_tile=None): 87 | assert len(coords) == 1 88 | return [queue_handle] 89 | 90 | def done(self, coord_handle): 91 | queue_handle = coord_handle 92 | all_done = True 93 | parent_tile = None 94 | return MessageDoneResult(queue_handle, all_done, parent_tile) 95 | 96 | 97 | class MultipleMessagesPerCoordTracker(object): 98 | 99 | """ 100 | track a mapping for multiple coordinates 101 | 102 | Support tracking a mapping for multiple coordinates to a single 103 | queue handle. 104 | """ 105 | 106 | def __init__(self, msg_tracker_logger): 107 | self.msg_tracker_logger = msg_tracker_logger 108 | self.queue_handle_map = {} 109 | self.coord_ids_map = {} 110 | self.pyramid_map = {} 111 | # TODO we might want to have a way to purge this, or risk 112 | # running out of memory if a coordinate never completes 113 | self.lock = threading.Lock() 114 | 115 | def track(self, queue_handle, coords, parent_tile=None): 116 | is_pyramid = len(coords) > 1 117 | if is_pyramid: 118 | assert parent_tile is not None, 'parent tile was not provided, ' \ 119 | 'but is required for tracking pyramids of tiles.' 120 | 121 | with self.lock: 122 | # rely on the queue handle token as the mapping key 123 | queue_handle_id = queue_handle.handle 124 | self.queue_handle_map[queue_handle_id] = queue_handle 125 | 126 | coord_ids = set() 127 | coord_handles = [] 128 | for coord in coords: 129 | coord_id = (int(coord.zoom), int(coord.column), int(coord.row)) 130 | coord_handle = (coord_id, queue_handle_id) 131 | assert coord_id not in coord_ids 132 | coord_ids.add(coord_id) 133 | coord_handles.append(coord_handle) 134 | 135 | self.coord_ids_map[queue_handle_id] = coord_ids 136 | 137 | if is_pyramid: 138 | self.pyramid_map[queue_handle_id] = parent_tile 139 | 140 | return coord_handles 141 | 142 | def done(self, coord_handle): 143 | queue_handle = None 144 | all_done = False 145 | parent_tile = None 146 | 147 | with self.lock: 148 | coord_id, queue_handle_id = coord_handle 149 | 150 | coord_ids = self.coord_ids_map.get(queue_handle_id) 151 | queue_handle = self.queue_handle_map.get(queue_handle_id) 152 | 153 | if queue_handle is None or coord_ids is None: 154 | self.msg_tracker_logger.unknown_queue_handle_id( 155 | coord_id, queue_handle_id) 156 | return MessageDoneResult(None, False, None) 157 | 158 | if coord_id not in coord_ids: 159 | self.msg_tracker_logger.unknown_coord_id( 160 | coord_id, queue_handle_id) 161 | else: 162 | coord_ids.remove(coord_id) 163 | 164 | if not coord_ids: 165 | # we're done with all coordinates for the queue message 166 | try: 167 | del self.queue_handle_map[queue_handle_id] 168 | except KeyError: 169 | pass 170 | try: 171 | del self.coord_ids_map[queue_handle_id] 172 | except KeyError: 173 | pass 174 | all_done = True 175 | parent_tile = self.pyramid_map.pop(queue_handle_id, None) 176 | 177 | return MessageDoneResult(queue_handle, all_done, parent_tile) 178 | -------------------------------------------------------------------------------- /tilequeue/queue/redis_queue.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from tilequeue.queue import MessageHandle 4 | from tilequeue.utils import grouper 5 | 6 | 7 | class RedisQueue(object): 8 | """ 9 | Redis backed queue implementation 10 | 11 | No attempts are made to guarantee messages are not lost if not 12 | acknowledged. If a worker reads messages from the queue and 13 | crashes, they are lost. 14 | """ 15 | 16 | enqueue_batch_size = 100 17 | sleep_time_seconds_when_empty = 10 18 | 19 | def __init__(self, redis_client, queue_key): 20 | self.redis_client = redis_client 21 | self.queue_key = queue_key 22 | 23 | def enqueue(self, payload): 24 | self.redis_client.rpush(payload) 25 | 26 | def enqueue_batch(self, payloads): 27 | for payloads_chunk in grouper(payloads, self.enqueue_batch_size): 28 | self.redis_client.rpush(self.queue_key, *payloads_chunk) 29 | 30 | def read(self): 31 | read_size = 10 32 | with self.redis_client.pipeline() as pipe: 33 | pipe.lrange(self.queue_key, 0, read_size - 1) 34 | pipe.ltrim(self.queue_key, read_size, -1) 35 | payloads, _ = pipe.execute() 36 | if not payloads: 37 | time.sleep(self.sleep_time_seconds_when_empty) 38 | return [] 39 | msg_handles = [] 40 | for payload in payloads: 41 | msg_handle = MessageHandle(None, payload) 42 | msg_handles.append(msg_handle) 43 | return msg_handles 44 | 45 | def job_progress(self, handle): 46 | pass 47 | 48 | def job_done(self, msg_handle): 49 | pass 50 | 51 | def clear(self): 52 | with self.redis_client.pipeline() as pipe: 53 | pipe.llen(self.queue_key) 54 | pipe.delete(self.queue_key) 55 | n, _ = pipe.execute() 56 | return n 57 | 58 | def close(self): 59 | pass 60 | 61 | 62 | def make_redis_queue(redis_client, queue_key): 63 | return RedisQueue(redis_client, queue_key) 64 | -------------------------------------------------------------------------------- /tilequeue/queue/sqs.py: -------------------------------------------------------------------------------- 1 | import threading 2 | from datetime import datetime 3 | 4 | from tilequeue.queue import MessageHandle 5 | from tilequeue.utils import grouper 6 | 7 | 8 | class VisibilityState(object): 9 | 10 | def __init__(self, last, total): 11 | # the datetime when the message was last extended 12 | self.last = last 13 | # the total amount of time currently extended 14 | self.total = total 15 | 16 | 17 | class VisibilityManager(object): 18 | 19 | def __init__(self, extend_secs, max_extend_secs, timeout_secs): 20 | self.extend_secs = extend_secs 21 | self.max_extend_secs = max_extend_secs 22 | self.timeout_secs = timeout_secs 23 | self.handle_state_map = {} 24 | self.lock = threading.Lock() 25 | 26 | def should_extend(self, handle, now=None): 27 | if now is None: 28 | now = datetime.now() 29 | with self.lock: 30 | state = self.handle_state_map.get(handle) 31 | if not state: 32 | return True 33 | if state.total + self.extend_secs > self.max_extend_secs: 34 | return False 35 | delta = now - state.last 36 | return delta.seconds > self.extend_secs 37 | 38 | def extend(self, handle, now=None): 39 | if now is None: 40 | now = datetime.now() 41 | with self.lock: 42 | state = self.handle_state_map.get(handle) 43 | if state: 44 | state.last = now 45 | state.total += self.extend_secs 46 | else: 47 | state = VisibilityState(now, self.extend_secs) 48 | self.handle_state_map[handle] = state 49 | return state 50 | 51 | def done(self, handle): 52 | try: 53 | with self.lock: 54 | del self.handle_state_map[handle] 55 | except KeyError: 56 | pass 57 | 58 | 59 | class JobProgressException(Exception): 60 | 61 | def __init__(self, msg, cause, err_details): 62 | super(JobProgressException, self).__init__( 63 | msg + ', caused by ' + repr(cause)) 64 | self.err_details = err_details 65 | 66 | 67 | class SqsQueue(object): 68 | 69 | def __init__(self, sqs_client, queue_url, read_size, 70 | recv_wait_time_seconds, visibility_mgr): 71 | self.sqs_client = sqs_client 72 | self.queue_url = queue_url 73 | self.read_size = read_size 74 | self.recv_wait_time_seconds = recv_wait_time_seconds 75 | self.visibility_mgr = visibility_mgr 76 | 77 | def enqueue(self, payload): 78 | return self.sqs_client.send( 79 | QueueUrl=self.queue_url, 80 | MessageBody=payload, 81 | ) 82 | 83 | def enqueue_batch(self, payloads): 84 | # sqs can only send 10 messages at once 85 | for payloads_chunk in grouper(payloads, 10): 86 | msgs = [] 87 | for i, payload in enumerate(payloads_chunk): 88 | msg_id = str(i) 89 | msg = dict( 90 | Id=msg_id, 91 | MessageBody=payload, 92 | ) 93 | msgs.append(msg) 94 | resp = self.sqs_client.send_message_batch( 95 | QueueUrl=self.queue_url, 96 | Entries=msgs, 97 | ) 98 | if resp['ResponseMetadata']['HTTPStatusCode'] != 200: 99 | raise Exception('Invalid status code from sqs: %s' % 100 | resp['ResponseMetadata']['HTTPStatusCode']) 101 | failed_messages = resp.get('Failed') 102 | if failed_messages: 103 | # TODO maybe retry failed messages if not sender's fault? up to 104 | # a certain maximum number of attempts? 105 | # http://boto3.readthedocs.io/en/latest/reference/services/sqs.html#SQS.Client.send_message_batch 106 | raise Exception('Messages failed to send to sqs: %s' % 107 | len(failed_messages)) 108 | 109 | def read(self): 110 | msg_handles = [] 111 | resp = self.sqs_client.receive_message( 112 | QueueUrl=self.queue_url, 113 | MaxNumberOfMessages=self.read_size, 114 | AttributeNames=('SentTimestamp',), 115 | WaitTimeSeconds=self.recv_wait_time_seconds, 116 | VisibilityTimeout=self.visibility_mgr.timeout_secs, 117 | ) 118 | if resp['ResponseMetadata']['HTTPStatusCode'] != 200: 119 | raise Exception('Invalid status code from sqs: %s' % 120 | resp['ResponseMetadata']['HTTPStatusCode']) 121 | 122 | sqs_messages = resp.get('Messages') 123 | if not sqs_messages: 124 | return None 125 | for sqs_message in sqs_messages: 126 | payload = sqs_message['Body'] 127 | try: 128 | timestamp = float(sqs_message['Attributes']['SentTimestamp']) 129 | except (TypeError, ValueError): 130 | timestamp = None 131 | sqs_handle = sqs_message['ReceiptHandle'] 132 | 133 | metadata = dict(timestamp=timestamp) 134 | msg_handle = MessageHandle(sqs_handle, payload, metadata) 135 | msg_handles.append(msg_handle) 136 | 137 | return msg_handles 138 | 139 | def job_done(self, handle): 140 | self.visibility_mgr.done(handle) 141 | self.sqs_client.delete_message( 142 | QueueUrl=self.queue_url, 143 | ReceiptHandle=handle, 144 | ) 145 | 146 | def job_progress(self, handle): 147 | if self.visibility_mgr.should_extend(handle): 148 | visibility_state = self.visibility_mgr.extend(handle) 149 | 150 | try: 151 | self.sqs_client.change_message_visibility( 152 | QueueUrl=self.queue_url, 153 | ReceiptHandle=handle, 154 | VisibilityTimeout=self.visibility_mgr.extend_secs, 155 | ) 156 | except Exception as e: 157 | err_details = dict( 158 | visibility=dict( 159 | last=visibility_state.last.isoformat(), 160 | total=visibility_state.total, 161 | )) 162 | raise JobProgressException( 163 | 'update visibility timeout', e, err_details) 164 | 165 | def clear(self): 166 | n = 0 167 | while True: 168 | msgs = self.read() 169 | if not msgs: 170 | break 171 | for msg in msgs: 172 | self.job_done(msg.handle) 173 | n += len(msgs) 174 | return n 175 | 176 | def close(self): 177 | pass 178 | 179 | 180 | def make_visibility_manager(extend_secs, max_extend_secs, timeout_secs): 181 | visibility_mgr = VisibilityManager(extend_secs, max_extend_secs, 182 | timeout_secs) 183 | return visibility_mgr 184 | 185 | 186 | def make_sqs_queue(name, region, visibility_mgr): 187 | import boto3 188 | sqs_client = boto3.client('sqs', region_name=region) 189 | resp = sqs_client.get_queue_url(QueueName=name) 190 | assert resp['ResponseMetadata']['HTTPStatusCode'] == 200, \ 191 | 'Failed to get queue url for: %s' % name 192 | queue_url = resp['QueueUrl'] 193 | read_size = 10 194 | recv_wait_time_seconds = 20 195 | return SqsQueue(sqs_client, queue_url, read_size, recv_wait_time_seconds, 196 | visibility_mgr) 197 | -------------------------------------------------------------------------------- /tilequeue/queue/writer.py: -------------------------------------------------------------------------------- 1 | # coordinates all the pieces required to enqueue coordinates 2 | from collections import defaultdict 3 | 4 | 5 | class InFlightCounter(object): 6 | """store state while filtering in inflight""" 7 | 8 | def __init__(self, inflight_mgr): 9 | self.n_inflight = 0 10 | self.n_not_inflight = 0 11 | self.inflight_mgr = inflight_mgr 12 | 13 | def filter(self, coords): 14 | for coord in coords: 15 | if self.inflight_mgr.is_inflight(coord): 16 | self.n_inflight += 1 17 | else: 18 | self.n_not_inflight += 1 19 | yield coord 20 | 21 | 22 | class QueueWriter(object): 23 | 24 | def __init__(self, queue_mapper, msg_marshaller, inflight_mgr, 25 | enqueue_batch_size): 26 | self.queue_mapper = queue_mapper 27 | self.msg_marshaller = msg_marshaller 28 | self.inflight_mgr = inflight_mgr 29 | self.enqueue_batch_size = enqueue_batch_size 30 | 31 | def _enqueue_batch(self, queue_id, coords_chunks): 32 | queue = self.queue_mapper.get_queue(queue_id) 33 | assert queue, 'No queue found for: %s' % queue_id 34 | payloads = [] 35 | all_coords = [] 36 | for coords_chunk in coords_chunks: 37 | payload = self.msg_marshaller.marshall(coords_chunk) 38 | payloads.append(payload) 39 | all_coords.extend(coords_chunk) 40 | queue.enqueue_batch(payloads) 41 | self.inflight_mgr.mark_inflight(all_coords) 42 | 43 | def enqueue_batch(self, coords): 44 | inflight_ctr = InFlightCounter(self.inflight_mgr) 45 | coords = inflight_ctr.filter(coords) 46 | coord_groups = self.queue_mapper.group(coords) 47 | 48 | # buffer the coords to send out per queue 49 | queue_send_buffer = defaultdict(list) 50 | 51 | for coord_group in coord_groups: 52 | coords = coord_group.coords 53 | queue_id = coord_group.queue_id 54 | send_data = queue_send_buffer[queue_id] 55 | send_data.append(coords) 56 | if len(send_data) >= self.enqueue_batch_size: 57 | tile_queue = self.queue_mapper.get_queue(queue_id) 58 | assert tile_queue, 'No tile_queue found for: %s' % queue_id 59 | self._enqueue_batch(queue_id, send_data) 60 | del send_data[:] 61 | 62 | for queue_id, send_data in queue_send_buffer.iteritems(): 63 | if send_data: 64 | self._enqueue_batch(queue_id, send_data) 65 | 66 | return inflight_ctr.n_not_inflight, inflight_ctr.n_inflight 67 | -------------------------------------------------------------------------------- /tilequeue/stats.py: -------------------------------------------------------------------------------- 1 | class TileProcessingStatsHandler(object): 2 | 3 | def __init__(self, stats): 4 | self.stats = stats 5 | 6 | def processed_coord(self, coord_proc_data): 7 | with self.stats.pipeline() as pipe: 8 | pipe.timing('process.time.fetch', coord_proc_data.timing['fetch']) 9 | pipe.timing('process.time.process', 10 | coord_proc_data.timing['process']) 11 | pipe.timing('process.time.upload', coord_proc_data.timing['s3']) 12 | pipe.timing('process.time.ack', coord_proc_data.timing['ack']) 13 | pipe.timing('process.time.queue', coord_proc_data.timing['queue']) 14 | 15 | for layer_name, features_size in coord_proc_data.size.items(): 16 | metric_name = 'process.size.%s' % layer_name 17 | pipe.gauge(metric_name, features_size) 18 | 19 | pipe.incr('process.storage.stored', 20 | coord_proc_data.store_info['stored']) 21 | pipe.incr('process.storage.skipped', 22 | coord_proc_data.store_info['not_stored']) 23 | 24 | def processed_pyramid(self, parent_tile, 25 | start_time, stop_time): 26 | duration = stop_time - start_time 27 | self.stats.timing('process.pyramid', duration) 28 | 29 | def fetch_error(self): 30 | self.stats.incr('process.errors.fetch', 1) 31 | 32 | def proc_error(self): 33 | self.stats.incr('process.errors.process', 1) 34 | 35 | 36 | def emit_time_dict(pipe, timing, prefix): 37 | for timing_label, value in timing.items(): 38 | metric_name = '%s.%s' % (prefix, timing_label) 39 | if isinstance(value, dict): 40 | emit_time_dict(pipe, value, metric_name) 41 | else: 42 | pipe.timing(metric_name, value) 43 | 44 | 45 | class RawrTileEnqueueStatsHandler(object): 46 | 47 | def __init__(self, stats): 48 | self.stats = stats 49 | 50 | def __call__(self, n_coords, n_payloads, n_msgs_sent, 51 | intersect_metrics, timing): 52 | 53 | with self.stats.pipeline() as pipe: 54 | pipe.gauge('rawr.enqueue.coords', n_coords) 55 | pipe.gauge('rawr.enqueue.groups', n_payloads) 56 | pipe.gauge('rawr.enqueue.calls', n_msgs_sent) 57 | 58 | pipe.gauge('rawr.enqueue.intersect.toi', 59 | intersect_metrics['n_toi']) 60 | pipe.gauge('rawr.enqueue.intersect.candidates', 61 | intersect_metrics['total']) 62 | pipe.gauge('rawr.enqueue.intersect.hits', 63 | intersect_metrics['hits']) 64 | pipe.gauge('rawr.enqueue.intersect.misses', 65 | intersect_metrics['misses']) 66 | pipe.gauge('rawr.enqueue.intersect.cached', 67 | 1 if intersect_metrics['cached'] else 0) 68 | 69 | prefix = 'rawr.enqueue.toi.time' 70 | emit_time_dict(pipe, timing, prefix) 71 | 72 | 73 | class RawrTilePipelineStatsHandler(object): 74 | 75 | def __init__(self, stats): 76 | self.stats = stats 77 | 78 | def __call__(self, n_enqueued, n_inflight, did_rawr_tile_gen, timing): 79 | with self.stats.pipeline() as pipe: 80 | 81 | pipe.incr('rawr.process.tiles', 1) 82 | 83 | pipe.gauge('rawr.process.enqueued', n_enqueued) 84 | pipe.gauge('rawr.process.inflight', n_inflight) 85 | 86 | rawr_tile_gen_val = 1 if did_rawr_tile_gen else 0 87 | pipe.gauge('rawr.process.rawr_tile_gen', rawr_tile_gen_val) 88 | 89 | prefix = 'rawr.process.time' 90 | emit_time_dict(pipe, timing, prefix) 91 | -------------------------------------------------------------------------------- /tilequeue/toi/__init__.py: -------------------------------------------------------------------------------- 1 | from .file import FileTilesOfInterestSet 2 | from .file import load_set_from_fp 3 | from .file import load_set_from_gzipped_fp 4 | from .file import save_set_to_fp 5 | from .file import save_set_to_gzipped_fp 6 | from .s3 import S3TilesOfInterestSet 7 | 8 | __all__ = [ 9 | FileTilesOfInterestSet, 10 | S3TilesOfInterestSet, 11 | save_set_to_fp, 12 | load_set_from_fp, 13 | save_set_to_gzipped_fp, 14 | load_set_from_gzipped_fp, 15 | ] 16 | -------------------------------------------------------------------------------- /tilequeue/toi/file.py: -------------------------------------------------------------------------------- 1 | import gzip 2 | 3 | from tilequeue.tile import coord_marshall_int 4 | from tilequeue.tile import coord_unmarshall_int 5 | from tilequeue.tile import deserialize_coord 6 | from tilequeue.tile import serialize_coord 7 | 8 | 9 | def save_set_to_fp(the_set, fp): 10 | for coord_int in sorted(the_set): 11 | coord = coord_unmarshall_int(coord_int) 12 | fp.write(serialize_coord(coord)) 13 | fp.write('\n') 14 | 15 | 16 | def load_set_from_fp(fp): 17 | toi_set = set() 18 | 19 | for coord_str in fp: 20 | coord = deserialize_coord(coord_str) 21 | coord_int = coord_marshall_int(coord) 22 | toi_set.add(coord_int) 23 | 24 | return toi_set 25 | 26 | 27 | def load_set_from_gzipped_fp(gzipped_fp): 28 | fp = gzip.GzipFile(fileobj=gzipped_fp, mode='r') 29 | return load_set_from_fp(fp) 30 | 31 | 32 | def save_set_to_gzipped_fp(the_set, fp): 33 | gzipped_fp = gzip.GzipFile(fileobj=fp, mode='w') 34 | save_set_to_fp(the_set, gzipped_fp) 35 | gzipped_fp.close() 36 | 37 | 38 | class FileTilesOfInterestSet(object): 39 | def __init__(self, filename): 40 | self.filename = filename 41 | 42 | def fetch_tiles_of_interest(self): 43 | toi_set = set() 44 | 45 | with open(self.filename, 'r') as toi_data: 46 | toi_set = load_set_from_gzipped_fp(toi_data) 47 | 48 | return toi_set 49 | 50 | def set_tiles_of_interest(self, new_set): 51 | with open(self.filename, 'w') as toi_data: 52 | save_set_to_gzipped_fp(new_set, toi_data) 53 | -------------------------------------------------------------------------------- /tilequeue/toi/s3.py: -------------------------------------------------------------------------------- 1 | from cStringIO import StringIO 2 | 3 | import boto 4 | 5 | from tilequeue.toi import load_set_from_gzipped_fp 6 | from tilequeue.toi import save_set_to_gzipped_fp 7 | 8 | 9 | class S3TilesOfInterestSet(object): 10 | def __init__(self, bucket, key): 11 | s3 = boto.connect_s3() 12 | buk = s3.get_bucket(bucket) 13 | self.key = buk.get_key(key, validate=False) 14 | 15 | def fetch_tiles_of_interest(self): 16 | toi_data_gz = StringIO() 17 | self.key.get_contents_to_file(toi_data_gz) 18 | toi_data_gz.seek(0) 19 | 20 | return load_set_from_gzipped_fp(toi_data_gz) 21 | 22 | def set_tiles_of_interest(self, new_set): 23 | toi_data_gz = StringIO() 24 | save_set_to_gzipped_fp(new_set, toi_data_gz) 25 | self.key.set_contents_from_string(toi_data_gz.getvalue()) 26 | -------------------------------------------------------------------------------- /tilequeue/top_tiles.py: -------------------------------------------------------------------------------- 1 | import csv 2 | 3 | from ModestMaps.Core import Coordinate 4 | 5 | 6 | def parse_top_tiles(fp, zoom_start, zoom_until): 7 | coords = [] 8 | reader = csv.reader(fp) 9 | for row in reader: 10 | try: 11 | zoom = int(row[0]) 12 | column = int(row[1]) 13 | row = int(row[2]) 14 | except (ValueError, IndexError): 15 | continue 16 | else: 17 | if zoom_start <= zoom <= zoom_until: 18 | coord = Coordinate( 19 | zoom=zoom, 20 | column=column, 21 | row=row, 22 | ) 23 | coords.append(coord) 24 | 25 | return coords 26 | -------------------------------------------------------------------------------- /tilequeue/transform.py: -------------------------------------------------------------------------------- 1 | import math 2 | from numbers import Number 3 | 4 | import shapely.errors 5 | from shapely import geometry 6 | from shapely.ops import transform 7 | from shapely.wkb import dumps 8 | 9 | from tilequeue.format import json_format 10 | from tilequeue.format import topojson_format 11 | from tilequeue.format import vtm_format 12 | from tilequeue.tile import bounds_buffer 13 | from tilequeue.tile import normalize_geometry_type 14 | 15 | 16 | half_circumference_meters = 20037508.342789244 17 | 18 | 19 | def mercator_point_to_lnglat(x, y, z=None): 20 | x /= half_circumference_meters 21 | y /= half_circumference_meters 22 | 23 | y = (2 * math.atan(math.exp(y * math.pi)) - (math.pi / 2)) / math.pi 24 | 25 | x *= 180 26 | y *= 180 27 | 28 | return x, y 29 | 30 | 31 | def rescale_point(bounds, scale): 32 | minx, miny, maxx, maxy = bounds 33 | 34 | def fn(x, y, z=None): 35 | xfac = scale / (maxx - minx) 36 | yfac = scale / (maxy - miny) 37 | x = xfac * (x - minx) 38 | y = yfac * (y - miny) 39 | 40 | return round(x), round(y) 41 | 42 | return fn 43 | 44 | 45 | def apply_to_all_coords(fn): 46 | return lambda shape: transform(fn, shape) 47 | 48 | 49 | # returns a geometry which is the given bounds expanded by `factor`. that is, 50 | # if the original shape was a 1x1 box, the new one will be `factor`x`factor` 51 | # box, with the same centroid as the original box. 52 | def calculate_padded_bounds(factor, bounds): 53 | min_x, min_y, max_x, max_y = bounds 54 | dx = 0.5 * (max_x - min_x) * (factor - 1.0) 55 | dy = 0.5 * (max_y - min_y) * (factor - 1.0) 56 | return geometry.box(min_x - dx, min_y - dy, max_x + dx, max_y + dy) 57 | 58 | 59 | # function which returns its argument, used to assign to a function variable 60 | # to use as a null transform. flake8 insists that it be named and not a 61 | # lambda. 62 | def _noop(shape): 63 | return shape 64 | 65 | 66 | def calc_buffered_bounds( 67 | format, bounds, meters_per_pixel_dim, layer_name, geometry_type, 68 | buffer_cfg): 69 | """ 70 | Calculate the buffered bounds per format per layer based on config. 71 | """ 72 | 73 | if not buffer_cfg: 74 | return bounds 75 | 76 | format_buffer_cfg = buffer_cfg.get(format.extension) 77 | if format_buffer_cfg is None: 78 | return bounds 79 | 80 | geometry_type = normalize_geometry_type(geometry_type) 81 | 82 | per_layer_cfg = format_buffer_cfg.get('layer', {}).get(layer_name) 83 | if per_layer_cfg is not None: 84 | layer_geom_pixels = per_layer_cfg.get(geometry_type) 85 | if layer_geom_pixels is not None: 86 | assert isinstance(layer_geom_pixels, Number) 87 | result = bounds_buffer( 88 | bounds, meters_per_pixel_dim * layer_geom_pixels) 89 | return result 90 | 91 | by_geometry_pixels = format_buffer_cfg.get('geometry', {}).get( 92 | geometry_type) 93 | if by_geometry_pixels is not None: 94 | assert isinstance(by_geometry_pixels, Number) 95 | result = bounds_buffer( 96 | bounds, meters_per_pixel_dim * by_geometry_pixels) 97 | return result 98 | 99 | return bounds 100 | 101 | 102 | def calc_max_padded_bounds(bounds, meters_per_pixel_dim, buffer_cfg): 103 | """ 104 | :return: The bounds expanded by the maximum value in buffer_cfg, default = 0 105 | """ 106 | max_buffer = 0 107 | 108 | if buffer_cfg is None: 109 | return bounds 110 | 111 | for _, format_cfg in buffer_cfg.items(): 112 | layer_cfg = format_cfg.get('layer', {}) 113 | if layer_cfg is not None: 114 | for _, value in layer_cfg.items(): 115 | assert isinstance(value, Number) 116 | max_buffer = max(max_buffer, value) 117 | 118 | geometry_cfg = format_cfg.get('geometry', {}) 119 | if geometry_cfg is not None: 120 | for _, value in geometry_cfg.items(): 121 | assert isinstance(value, Number) 122 | max_buffer = max(max_buffer, value) 123 | 124 | return bounds_buffer(bounds, meters_per_pixel_dim * max_buffer) 125 | 126 | 127 | def _intersect_multipolygon(shape, tile_bounds, clip_bounds): 128 | """ 129 | Return the parts of the MultiPolygon shape which overlap the tile_bounds, 130 | each clipped to the clip_bounds. This can be used to extract only the 131 | parts of a multipolygon which are actually visible in the tile, while 132 | keeping those parts which extend beyond the tile clipped to avoid huge 133 | polygons. 134 | """ 135 | 136 | polys = [] 137 | for poly in shape.geoms: 138 | if tile_bounds.intersects(poly): 139 | if not clip_bounds.contains(poly): 140 | try: 141 | poly = clip_bounds.intersection(poly) 142 | except shapely.errors.TopologicalError: 143 | continue 144 | 145 | # the intersection operation can make the resulting polygon 146 | # invalid. including it in a MultiPolygon would make that 147 | # invalid too. instead, we skip it, and hope it wasn't too 148 | # important. 149 | if not poly.is_valid: 150 | continue 151 | 152 | if poly.type == 'Polygon': 153 | polys.append(poly) 154 | elif poly.type == 'MultiPolygon': 155 | polys.extend(poly.geoms) 156 | 157 | return geometry.MultiPolygon(polys) 158 | 159 | 160 | def _clip_shape(shape, buffer_padded_bounds, is_clipped, clip_factor): 161 | """ 162 | Return the shape clipped to a clip_factor expansion of buffer_padded_bounds 163 | if is_clipped is True. Otherwise return the original shape, or None if the 164 | shape does not intersect buffer_padded_bounds at all. 165 | 166 | This is used to reduce the size of the geometries which are encoded in the 167 | tiles by removing things which aren't in the tile, and clipping those which 168 | are to the clip_factor expanded bounding box. 169 | """ 170 | 171 | shape_buf_bounds = geometry.box(*buffer_padded_bounds) 172 | 173 | if not shape_buf_bounds.intersects(shape): 174 | return None 175 | 176 | if is_clipped: 177 | # now we know that we should include the geometry, but 178 | # if the geometry should be clipped, we'll clip to the 179 | # layer-specific padded bounds 180 | layer_padded_bounds = calculate_padded_bounds( 181 | clip_factor, buffer_padded_bounds) 182 | 183 | if shape.type == 'MultiPolygon': 184 | shape = _intersect_multipolygon( 185 | shape, shape_buf_bounds, layer_padded_bounds) 186 | else: 187 | try: 188 | shape = shape.intersection(layer_padded_bounds) 189 | except shapely.errors.TopologicalError: 190 | return None 191 | 192 | return shape 193 | 194 | 195 | def transform_feature_layers_shape( 196 | feature_layers, format, scale, unpadded_bounds, 197 | meters_per_pixel_dim, buffer_cfg): 198 | if format in (json_format, topojson_format): 199 | transform_fn = apply_to_all_coords(mercator_point_to_lnglat) 200 | elif format == vtm_format: 201 | transform_fn = apply_to_all_coords( 202 | rescale_point(unpadded_bounds, scale)) 203 | else: 204 | # mvt and unknown formats get no geometry transformation 205 | transform_fn = _noop 206 | 207 | # shape_unpadded_bounds = geometry.box(*unpadded_bounds) 208 | 209 | transformed_feature_layers = [] 210 | for feature_layer in feature_layers: 211 | layer_name = feature_layer['name'] 212 | transformed_features = [] 213 | layer_datum = feature_layer['layer_datum'] 214 | is_clipped = layer_datum['is_clipped'] 215 | clip_factor = layer_datum.get('clip_factor', 1.0) 216 | 217 | for shape, props, feature_id in feature_layer['features']: 218 | 219 | if shape.is_empty or shape.type == 'GeometryCollection': 220 | continue 221 | 222 | buffer_padded_bounds = calc_buffered_bounds( 223 | format, unpadded_bounds, meters_per_pixel_dim, layer_name, 224 | shape.type, buffer_cfg) 225 | 226 | shape = _clip_shape( 227 | shape, buffer_padded_bounds, is_clipped, clip_factor) 228 | if shape is None or shape.is_empty: 229 | continue 230 | 231 | # perform the format specific geometry transformations 232 | shape = transform_fn(shape) 233 | 234 | if format.supports_shapely_geometry: 235 | geom = shape 236 | else: 237 | geom = dumps(shape) 238 | 239 | transformed_features.append((geom, props, feature_id)) 240 | 241 | transformed_feature_layer = dict( 242 | name=feature_layer['name'], 243 | features=transformed_features, 244 | layer_datum=layer_datum, 245 | ) 246 | transformed_feature_layers.append(transformed_feature_layer) 247 | 248 | return transformed_feature_layers 249 | -------------------------------------------------------------------------------- /tilequeue/utils.py: -------------------------------------------------------------------------------- 1 | import re 2 | import sys 3 | import traceback 4 | from collections import defaultdict 5 | from datetime import datetime 6 | from itertools import islice 7 | from time import time 8 | 9 | import boto3 10 | from botocore.credentials import RefreshableCredentials 11 | from botocore.session import get_session 12 | 13 | from tilequeue.tile import coord_marshall_int 14 | from tilequeue.tile import create_coord 15 | 16 | 17 | def format_stacktrace_one_line(exc_info=None): 18 | # exc_info is expected to be an exception tuple from sys.exc_info() 19 | if exc_info is None: 20 | exc_info = sys.exc_info() 21 | exc_type, exc_value, exc_traceback = exc_info 22 | exception_lines = traceback.format_exception(exc_type, exc_value, 23 | exc_traceback) 24 | stacktrace = ' | '.join([x.replace('\n', '') 25 | for x in exception_lines]) 26 | return stacktrace 27 | 28 | 29 | def grouper(iterable, n): 30 | """Yield n-length chunks of the iterable""" 31 | it = iter(iterable) 32 | while True: 33 | chunk = tuple(islice(it, n)) 34 | if not chunk: 35 | return 36 | yield chunk 37 | 38 | 39 | def parse_log_file(log_file): 40 | ip_pattern = r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' 41 | # didn't match againts explicit date pattern, in case it changes 42 | date_pattern = r'\[([\d\w\s\/:]+)\]' 43 | tile_id_pattern = r'\/([\w]+)\/([\d]+)\/([\d]+)\/([\d]+)\.([\d\w]*)' 44 | 45 | log_pattern = r'%s - - %s "([\w]+) %s.*' % ( 46 | ip_pattern, date_pattern, tile_id_pattern) 47 | 48 | tile_log_records = [] 49 | for log_string in log_file: 50 | match = re.search(log_pattern, log_string) 51 | if match and len(match.groups()) == 8: 52 | tile_log_records.append( 53 | (match.group(1), 54 | datetime.strptime(match.group(2), '%d/%B/%Y %H:%M:%S'), 55 | coord_marshall_int( 56 | create_coord( 57 | match.group(6), match.group(7), match.group(5))))) 58 | 59 | return tile_log_records 60 | 61 | 62 | def encode_utf8(x): 63 | if x is None: 64 | return None 65 | elif isinstance(x, unicode): 66 | return x.encode('utf-8') 67 | elif isinstance(x, dict): 68 | result = {} 69 | for k, v in x.items(): 70 | if isinstance(k, unicode): 71 | k = k.encode('utf-8') 72 | result[k] = encode_utf8(v) 73 | return result 74 | elif isinstance(x, list): 75 | return map(encode_utf8, x) 76 | elif isinstance(x, tuple): 77 | return tuple(encode_utf8(list(x))) 78 | else: 79 | return x 80 | 81 | 82 | class time_block(object): 83 | 84 | """Convenience to capture timing information""" 85 | 86 | def __init__(self, timing_state, key): 87 | # timing_state should be a dictionary 88 | self.timing_state = timing_state 89 | self.key = key 90 | 91 | def __enter__(self): 92 | self.start = time() 93 | 94 | def __exit__(self, exc_type, exc_val, exc_tb): 95 | stop = time() 96 | duration_seconds = stop - self.start 97 | duration_millis = duration_seconds * 1000 98 | self.timing_state[self.key] = duration_millis 99 | 100 | 101 | class CoordsByParent(object): 102 | 103 | def __init__(self, parent_zoom): 104 | self.parent_zoom = parent_zoom 105 | self.groups = defaultdict(list) 106 | 107 | def add(self, coord, *extra): 108 | data = coord 109 | if extra: 110 | data = (coord,) + extra 111 | 112 | # treat tiles as singletons below the parent zoom 113 | if coord.zoom < self.parent_zoom: 114 | self.groups[coord].append(data) 115 | 116 | else: 117 | # otherwise, group by the parent tile at the parent zoom. 118 | parent_coord = coord.zoomTo(self.parent_zoom).container() 119 | self.groups[parent_coord].append(data) 120 | 121 | def __iter__(self): 122 | return self.groups.iteritems() 123 | 124 | 125 | def convert_seconds_to_millis(time_in_seconds): 126 | time_in_millis = int(time_in_seconds * 1000) 127 | return time_in_millis 128 | 129 | 130 | class AwsSessionHelper: 131 | """ The AwsSessionHelper creates a auto-refreshable boto3 session object 132 | and allows for creating clients with those refreshable credentials. 133 | """ 134 | 135 | def __init__(self, session_name, role_arn, region='us-east-1', 136 | s3_role_session_duration_s=3600): 137 | """ session_name: str; The name of the session we are creating 138 | role_arn: str; The ARN of the role we are assuming with STS 139 | region: str; The region for the STS client to be created in 140 | s3_role_session_duration_s: int; the time that session is good for 141 | """ 142 | self.role_arn = role_arn 143 | self.session_name = session_name 144 | self.region = region 145 | self.session_duration_seconds = s3_role_session_duration_s 146 | self.sts_client = boto3.client('sts') 147 | 148 | credentials = self._refresh() 149 | session_credentials = RefreshableCredentials.create_from_metadata( 150 | metadata=credentials, 151 | refresh_using=self._refresh, 152 | method='sts-assume-role' 153 | ) 154 | aws_session = get_session() 155 | aws_session._credentials = session_credentials 156 | aws_session.set_config_variable('region', region) 157 | self.aws_session = boto3.Session(botocore_session=aws_session) 158 | 159 | def get_client(self, service): 160 | """ Returns boto3.client with the refreshable session 161 | 162 | service: str; String of what service to create a client for 163 | (e.g. 'sqs', 's3') 164 | """ 165 | return self.aws_session.client(service) 166 | 167 | def get_session(self): 168 | """ Returns the raw refreshable aws session 169 | """ 170 | return self.aws_session 171 | 172 | def _refresh(self): 173 | params = { 174 | 'RoleArn': self.role_arn, 175 | 'RoleSessionName': self.session_name, 176 | 'DurationSeconds': self.session_duration_seconds, 177 | } 178 | 179 | response = self.sts_client.assume_role(**params).get('Credentials') 180 | credentials = { 181 | 'access_key': response.get('AccessKeyId'), 182 | 'secret_key': response.get('SecretAccessKey'), 183 | 'token': response.get('SessionToken'), 184 | 'expiry_time': response.get('Expiration').isoformat(), 185 | } 186 | return credentials 187 | --------------------------------------------------------------------------------