├── .config.json ├── .gitignore ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── borken-map ├── borken_map │ ├── __init__.py │ ├── cli │ │ ├── __init__.py │ │ └── cli.py │ ├── data │ │ ├── mapbox.png │ │ ├── mb_dg.png │ │ └── mb_osm.png │ └── utils.py └── setup.py ├── data └── world.geojson ├── handler.py ├── package.json └── serverless.yml /.config.json: -------------------------------------------------------------------------------- 1 | { 2 | "consumer_key" : "", 3 | "consumer_secret" : "", 4 | "access_key" : "", 5 | "access_secret" : "", 6 | "mapbox_token" : "", 7 | "output_bucket" : "" 8 | } 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | local_settings.py 56 | 57 | # Flask stuff: 58 | instance/ 59 | .webassets-cache 60 | 61 | # Scrapy stuff: 62 | .scrapy 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | 67 | # PyBuilder 68 | target/ 69 | 70 | # Jupyter Notebook 71 | .ipynb_checkpoints 72 | 73 | # pyenv 74 | .python-version 75 | 76 | # celery beat schedule file 77 | celerybeat-schedule 78 | 79 | # SageMath parsed files 80 | *.sage.py 81 | 82 | # dotenv 83 | .env 84 | 85 | # virtualenv 86 | .venv 87 | venv/ 88 | ENV/ 89 | 90 | # Spyder project settings 91 | .spyderproject 92 | .spyproject 93 | 94 | # Rope project settings 95 | .ropeproject 96 | 97 | # mkdocs documentation 98 | /site 99 | 100 | # mypy 101 | .mypy_cache/ 102 | 103 | .serverless 104 | node_modules/ 105 | package.zip 106 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Use the official amazonlinux AMI image 2 | FROM amazonlinux:latest 3 | 4 | # Install apt dependencies 5 | RUN yum install -y gcc gcc-c++ freetype-devel yum-utils findutils openssl-devel 6 | 7 | RUN yum -y groupinstall development 8 | 9 | RUN curl https://www.python.org/ftp/python/3.6.1/Python-3.6.1.tar.xz | tar -xJ \ 10 | && cd Python-3.6.1 \ 11 | && ./configure --prefix=/usr/local --enable-shared \ 12 | && make \ 13 | && make install \ 14 | && cd .. \ 15 | && rm -rf Python-3.6.1 16 | 17 | ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH 18 | 19 | COPY borken-map borken-map 20 | # Install Python dependencies 21 | RUN pip3 install ./borken-map --no-binary numpy --pre rasterio -t /tmp/vendored -U 22 | 23 | # Reduce Lambda package size to fit the 250Mb limit 24 | # Mostly based on https://github.com/jamesandersen/aws-machine-learning-demo 25 | RUN du -sh /tmp/vendored 26 | 27 | # This is the list of available modules on AWS lambda Python 3 28 | # ['boto3', 'botocore', 'docutils', 'jmespath', 'pip', 'python-dateutil', 's3transfer', 'setuptools', 'six'] 29 | RUN find /tmp/vendored -name "*-info" -type d -exec rm -rdf {} + 30 | RUN rm -rdf /tmp/vendored/boto3/ 31 | RUN rm -rdf /tmp/vendored/botocore/ 32 | RUN rm -rdf /tmp/vendored/docutils/ 33 | RUN rm -rdf /tmp/vendored/dateutil/ 34 | RUN rm -rdf /tmp/vendored/jmespath/ 35 | RUN rm -rdf /tmp/vendored/s3transfer/ 36 | RUN rm -rdf /tmp/vendored/numpy/doc/ 37 | 38 | # Leave module precompiles for faster Lambda startup 39 | RUN find /tmp/vendored -type f -name '*.pyc' | while read f; do n=$(echo $f | sed 's/__pycache__\///' | sed 's/.cpython-36//'); cp $f $n; done; 40 | RUN find /tmp/vendored -type d -a -name '__pycache__' -print0 | xargs -0 rm -rf 41 | RUN find /tmp/vendored -type f -a -name '*.py' -print0 | xargs -0 rm -f 42 | 43 | RUN du -sh /tmp/vendored 44 | 45 | COPY handler.py /tmp/vendored/handler.py 46 | COPY data /tmp/vendored/data 47 | 48 | # Create archive 49 | RUN cd /tmp/vendored && zip -r9q /tmp/package.zip * 50 | 51 | # Cleanup 52 | RUN rm -rf /tmp/vendored/ 53 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2018, Vincent Sarago 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | 2 | SHELL = /bin/bash 3 | 4 | all: build 5 | 6 | build: 7 | docker build --tag lambda:latest . 8 | docker run --name lambda -itd lambda:latest 9 | docker cp lambda:/tmp/package.zip package.zip 10 | docker stop lambda 11 | docker rm lambda 12 | 13 | 14 | shell: 15 | docker run \ 16 | --name lambda \ 17 | --volume $(shell pwd)/:/data \ 18 | --rm \ 19 | -it \ 20 | lambda:latest /bin/bash 21 | 22 | clean: 23 | docker stop lambda 24 | docker rm lambda 25 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # borken-map 2 | 3 | a bot that makes broken maps 4 | 5 | ![2b7381a0-1671-11e8-b7a0-22ed929f7fd1](https://user-images.githubusercontent.com/10407788/36449472-e2643224-1658-11e8-9010-6ab6047e3116.jpg) 6 | 7 | ![35da9ba0-1604-11e8-bbfa-6e6a2137250a](https://user-images.githubusercontent.com/10407788/36449476-e37d0f64-1658-11e8-8fc0-4817aa6bea09.jpg) 8 | 9 | ![970dd786-160c-11e8-b963-a6d222e01b0a](https://user-images.githubusercontent.com/10407788/36449479-e4a64676-1658-11e8-95f6-cf54b15585a6.jpg) 10 | -------------------------------------------------------------------------------- /borken-map/borken_map/__init__.py: -------------------------------------------------------------------------------- 1 | # borken_map 2 | 3 | __version__ = '0.0.1' 4 | -------------------------------------------------------------------------------- /borken-map/borken_map/cli/__init__.py: -------------------------------------------------------------------------------- 1 | """cloud_bot.cli""" 2 | -------------------------------------------------------------------------------- /borken-map/borken_map/cli/cli.py: -------------------------------------------------------------------------------- 1 | """borken_map.cli.cli""" 2 | 3 | import os 4 | import uuid 5 | import click 6 | import cligj 7 | 8 | from supermercado import super_utils 9 | from borken_map.utils import create_img 10 | 11 | 12 | @click.command() 13 | @click.option('--min-zoom', type=int, default=8, help='') 14 | @click.option('--max-zoom', type=int, default=13, help='') 15 | @click.option('--mapid', type=str, default='mapbox.satellite') 16 | @click.option('--col', type=int, default=3, help='') 17 | @click.option('--row', type=int, default=3, help='') 18 | @cligj.features_in_arg 19 | def create(features, min_zoom, max_zoom, mapid, col, row): 20 | """ 21 | """ 22 | access_token = os.environ['MapboxAccessToken'] 23 | features = [f for f in super_utils.filter_polygons(features)] 24 | img = create_img(features, access_token, min_zoom, max_zoom, mapid, nb_col=col, nb_row=row) 25 | renderid = str(uuid.uuid1()) 26 | outfile = f'./{renderid}.jpg' 27 | img.save(outfile) 28 | -------------------------------------------------------------------------------- /borken-map/borken_map/data/mapbox.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vincentsarago/borken-map/64519a8defe1e44fdcac20210846149ca1dc1621/borken-map/borken_map/data/mapbox.png -------------------------------------------------------------------------------- /borken-map/borken_map/data/mb_dg.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vincentsarago/borken-map/64519a8defe1e44fdcac20210846149ca1dc1621/borken-map/borken_map/data/mb_dg.png -------------------------------------------------------------------------------- /borken-map/borken_map/data/mb_osm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vincentsarago/borken-map/64519a8defe1e44fdcac20210846149ca1dc1621/borken-map/borken_map/data/mb_osm.png -------------------------------------------------------------------------------- /borken-map/borken_map/utils.py: -------------------------------------------------------------------------------- 1 | """borken_map.utils""" 2 | 3 | import os 4 | from io import BytesIO 5 | from concurrent import futures 6 | 7 | import requests 8 | from PIL import Image 9 | 10 | from random import randint 11 | from supermercado.burntiles import burn 12 | 13 | 14 | def create_img(features, access_token, min_zoom=6, max_zoom=13, mapid='mapbox.satellite', nb_col=14, nb_row=10): 15 | """ 16 | """ 17 | 18 | img_size_x = nb_col * 512 19 | img_size_y = nb_row * 512 20 | new_im = Image.new('RGB', (img_size_x, img_size_y)) 21 | 22 | ind = [] 23 | for i in range(0, img_size_x, 512): 24 | for j in range(0, img_size_y, 512): 25 | ind.append({'c': i, 'r': j}) 26 | 27 | def worker(idx): 28 | """ 29 | """ 30 | nb_features = len(features) 31 | while True: 32 | zoom = randint(min_zoom, max_zoom) 33 | feature = features[randint(0, nb_features - 1)] 34 | 35 | tiles = burn([feature], zoom) 36 | nb_mctiles = len(tiles) 37 | tile = tiles[randint(0, nb_mctiles - 1)] 38 | tile_x = tile[0] 39 | tile_y = tile[1] 40 | tile_z = tile[2] 41 | 42 | url = f'https://api.mapbox.com/v4/{mapid}/{tile_z}/{tile_x}/{tile_y}@2x.jpg?access_token={access_token}' 43 | response = requests.get(url) 44 | if response.status_code != 200: 45 | continue 46 | 47 | header = response.headers 48 | size = header.get('Content-Length') 49 | if int(size) < 10000: 50 | continue 51 | 52 | new_im.paste(Image.open(BytesIO(response.content)), (idx['c'], idx['r'])) 53 | return True 54 | 55 | with futures.ThreadPoolExecutor(max_workers=5) as executor: 56 | executor.map(worker, ind) 57 | 58 | new_im = new_im.convert('RGBA') 59 | 60 | imageWatermark = Image.new('RGBA', new_im.size, (255, 255, 255, 0)) 61 | img_logo = os.path.join(os.path.dirname(__file__), 'data', 'mapbox.png') 62 | logo = Image.open(img_logo) 63 | imageWatermark.paste(logo, (20, img_size_y - 80)) 64 | new_im = Image.alpha_composite(new_im, imageWatermark).convert('RGB') 65 | 66 | if mapid == 'mapbox.satellite': 67 | img_logo = os.path.join(os.path.dirname(__file__), 'data', 'mb_dg.png') 68 | logo = Image.open(img_logo).convert('RGB') 69 | h = logo.height 70 | w = logo.width 71 | new_im.paste(logo, (img_size_x - w, img_size_y - h)) 72 | else: 73 | img_logo = os.path.join(os.path.dirname(__file__), 'data', 'mb_osm.png') 74 | logo = Image.open(img_logo).convert('RGB') 75 | h = logo.height 76 | w = logo.width 77 | new_im.paste(logo, (img_size_x - w, img_size_y - h)) 78 | 79 | return new_im 80 | -------------------------------------------------------------------------------- /borken-map/setup.py: -------------------------------------------------------------------------------- 1 | 2 | from setuptools import setup 3 | 4 | # Parse the version from the poster module. 5 | with open('borken_map/__init__.py') as f: 6 | for line in f: 7 | if line.find("__version__") >= 0: 8 | version = line.split("=")[1].strip() 9 | version = version.strip('"') 10 | version = version.strip("'") 11 | continue 12 | 13 | 14 | setup(name='borken_map', 15 | version=version, 16 | python_requires='>=3', 17 | description=u"do the job", 18 | long_description=u"do my job", 19 | author=u"", 20 | author_email='', 21 | license='BSD', 22 | packages=['borken_map'], 23 | include_package_data=True, 24 | package_data={'borken_map': [ 25 | 'data/mapbox.png', 26 | 'data/mb_osm.png', 27 | 'data/mb_dg.png']}, 28 | install_requires=[ 29 | 'tweepy', 30 | 'Pillow', 31 | 'requests', 32 | 'supermercado', 33 | 'rasterio[s3]>=1.0a12' 34 | ], 35 | zip_safe=False, 36 | entry_points=""" 37 | [console_scripts] 38 | borken=borken_map.cli.cli:create 39 | """) 40 | -------------------------------------------------------------------------------- /handler.py: -------------------------------------------------------------------------------- 1 | """borken_map.handler""" 2 | 3 | import os 4 | import json 5 | import uuid 6 | from io import BytesIO 7 | from random import randint 8 | 9 | import boto3 10 | import tweepy 11 | from PIL import Image 12 | from borken_map.utils import create_img 13 | 14 | 15 | def handler(event, context): 16 | """Tweet the image 17 | """ 18 | consumer_key = os.environ['C_KEY'] 19 | consumer_secret = os.environ['C_SECRET'] 20 | access_key = os.environ['A_KEY'] 21 | access_secret = os.environ['A_SECRET'] 22 | access_token = os.environ['MapboxAccessToken'] 23 | out_bucket = os.environ['OUTPUT_BUCKET'] 24 | 25 | min_zoom = event.get('min_zoom', 8) 26 | max_zoom = event.get('max_zoom', 13) 27 | mapid = event.get('mapid') 28 | col = event.get('col', 6) 29 | row = event.get('row', 4) 30 | 31 | auth = tweepy.OAuthHandler(consumer_key, consumer_secret) 32 | auth.set_access_token(access_key, access_secret) 33 | api = tweepy.API(auth) 34 | 35 | feat = os.path.join(os.path.dirname(__file__), 'data', 'world.geojson') 36 | with open(feat, 'r') as f: 37 | feats = json.loads(f.read())['features'] 38 | 39 | if not mapid: 40 | mapids = [ 41 | 'mapbox.streets', 42 | 'mapbox.light', 43 | 'mapbox.satellite', 44 | 'mapbox.wheatpaste', 45 | 'mapbox.streets-basic', 46 | 'mapbox.comic', 47 | 'mapbox.outdoors', 48 | 'mapbox.run-bike-hike', 49 | 'mapbox.pencil', 50 | 'mapbox.pirates', 51 | 'mapbox.emerald', 52 | 'mapbox.high-contrast'] 53 | mapid = mapids[randint(0, len(mapids) - 1)] 54 | 55 | img = create_img(feats, access_token, min_zoom=min_zoom, max_zoom=max_zoom, 56 | mapid=mapid, nb_col=col, nb_row=row) 57 | 58 | # Upload high res to s3 59 | uid = str(uuid.uuid1()) 60 | key = f'data/borken_map/{uid}.jpg' 61 | params = { 62 | 'ACL': 'public-read', 63 | 'ContentType': 'image/jpeg'} 64 | 65 | im = BytesIO() 66 | img.save(im, 'jpeg', subsampling=0, quality=100) 67 | im.seek(0) 68 | 69 | client = boto3.client('s3') 70 | client.put_object(Body=im, Bucket=out_bucket, Key=key, **params) 71 | 72 | # Create a thumbnail 73 | img.thumbnail((1024, 1024), Image.ANTIALIAS) 74 | 75 | im = BytesIO() 76 | img.save(im, 'jpeg', subsampling=0, quality=100) 77 | im.seek(0) 78 | 79 | tweet_content = f'Download full resolution at https://s3.amazonaws.com/{out_bucket}/{key} #maptiles' 80 | api.update_with_media('borken_map.jpg', status=tweet_content, file=im) 81 | return True 82 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "borken-map-bot", 3 | "version": "1.0.0", 4 | "description": "", 5 | "devDependencies": { 6 | "serverless": "^1.23.0" 7 | }, 8 | "author": "Vincent Sarago" 9 | } 10 | -------------------------------------------------------------------------------- /serverless.yml: -------------------------------------------------------------------------------- 1 | service: borken-bot 2 | 3 | provider: 4 | name: aws 5 | runtime: python3.6 6 | stage: ${opt:stage, 'production'} 7 | 8 | region: us-east-1 9 | 10 | iamRoleStatements: 11 | - Effect: "Allow" 12 | Action: 13 | - "s3:*" 14 | Resource: 15 | - "arn:aws:s3:::${file(.config.json):output_bucket}/*" 16 | 17 | environment: 18 | C_KEY: ${file(.config.json):consumer_key} 19 | C_SECRET: ${file(.config.json):consumer_secret} 20 | A_KEY: ${file(.config.json):access_key} 21 | A_SECRET: ${file(.config.json):access_secret} 22 | MapboxAccessToken: ${file(.config.json):mapbox_token} 23 | OUTPUT_BUCKET: ${file(.config.json):output_bucket} 24 | 25 | deploymentBucket: ${file(.config.json):output_bucket} 26 | 27 | package: 28 | artifact: package.zip 29 | 30 | functions: 31 | bot: 32 | handler: handler.handler 33 | memorySize: 1536 34 | timeout: 20 35 | events: 36 | - schedule: rate(1 hour) 37 | --------------------------------------------------------------------------------