├── requirements.txt ├── .gitignore ├── .dockerignore ├── bin ├── start_fcgiwarp.sh ├── update_overpass_loop.sh ├── dispatcher_start.sh ├── rules_loop.sh └── update_overpass.sh ├── .pre-commit-config.yaml ├── docker-healthcheck.sh ├── docker-compose.yml ├── LICENSE ├── etc ├── supervisord.conf └── nginx-overpass.conf.template ├── update.py ├── Dockerfile.template ├── docker-entrypoint.sh └── README.md /requirements.txt: -------------------------------------------------------------------------------- 1 | osmium~=3.7.0 2 | requests 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | .DS_Store 3 | */Dockerfile 4 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | # ignore the big files when building yourself 2 | **/db 3 | **/diffs 4 | **/*.log 5 | **/*.bin 6 | **/*.idx 7 | **/*.map 8 | **/*.osm.bz2 9 | README.md 10 | -------------------------------------------------------------------------------- /bin/start_fcgiwarp.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | find /nginx -type s -print0 | xargs -0 --no-run-if-empty rm && fcgiwrap -c "${OVERPASS_FASTCGI_PROCESSES:-4}" -s unix:/nginx/fcgiwrap.socket 4 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/jumanjihouse/pre-commit-hooks 3 | rev: 2.1.6 # or specific git tag 4 | hooks: 5 | - id: shellcheck 6 | - id: shfmt 7 | -------------------------------------------------------------------------------- /bin/update_overpass_loop.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | OVERPASS_UPDATE_SLEEP=${OVERPASS_UPDATE_SLEEP:-60} 4 | set +e 5 | while true; do 6 | if [ -n "$OVERPASS_DIFF_URL" ]; then 7 | /app/bin/update_overpass.sh 8 | fi 9 | sleep "${OVERPASS_UPDATE_SLEEP}" 10 | done 11 | -------------------------------------------------------------------------------- /docker-healthcheck.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e -o pipefail 4 | 5 | NODE_ID=1 6 | 7 | # if we allow duplicate queries, the healthcheck will fail because it always fetches node id 1 8 | # if that is the case (default), we query a random node 9 | if [[ ! -n ${OVERPASS_ALLOW_DUPLICATE_QUERIES} || ${OVERPASS_ALLOW_DUPLICATE_QUERIES} == "no" ]]; then 10 | NODE_ID=$(shuf -i 1-10000000 -n 1) 11 | fi 12 | 13 | OVERPASS_HEALTHCHECK=${OVERPASS_HEALTHCHECK:-'curl --noproxy "*" -qf "http://localhost/api/interpreter?data=\[out:json\];node(${NODE_ID});out;" | jq ".generator" |grep -q Overpass || exit 1'} 14 | 15 | eval "${OVERPASS_HEALTHCHECK}" 16 | -------------------------------------------------------------------------------- /bin/dispatcher_start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e -o pipefail 3 | 4 | DISPATCHER_ARGS=("--osm-base" "--db-dir=/db/db") 5 | if [[ "${OVERPASS_META}" == "attic" ]]; then 6 | DISPATCHER_ARGS+=("--attic") 7 | elif [[ "${OVERPASS_META}" == "yes" ]]; then 8 | DISPATCHER_ARGS+=("--meta") 9 | fi 10 | 11 | if [[ -n ${OVERPASS_RATE_LIMIT} ]]; then 12 | DISPATCHER_ARGS+=("--rate-limit=${OVERPASS_RATE_LIMIT}") 13 | fi 14 | 15 | if [[ -n ${OVERPASS_TIME} ]]; then 16 | DISPATCHER_ARGS+=("--time=${OVERPASS_TIME}") 17 | fi 18 | 19 | if [[ -n ${OVERPASS_SPACE} ]]; then 20 | DISPATCHER_ARGS+=("--space=${OVERPASS_SPACE}") 21 | fi 22 | 23 | if [[ -n ${OVERPASS_ALLOW_DUPLICATE_QUERIES} ]]; then 24 | DISPATCHER_ARGS+=("--allow-duplicate-queries=${OVERPASS_ALLOW_DUPLICATE_QUERIES}") 25 | fi 26 | 27 | find /db/db -type s -print0 | xargs -0 --no-run-if-empty rm && /app/bin/dispatcher "${DISPATCHER_ARGS[@]}" 28 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | volumes: 2 | overpass-db: 3 | 4 | version: '3.8' 5 | services: 6 | overpass: 7 | # specify the image here 8 | image: wiktorn/overpass-api:0.7.62 9 | container_name: overpass 10 | # uncomment if you want to build the image yourself 11 | # build: . 12 | ports: 13 | - 12345:80 14 | volumes: 15 | # use a docker managed volume 16 | - overpass-db:/db 17 | environment: 18 | - OVERPASS_META=no 19 | - OVERPASS_MODE=init 20 | - OVERPASS_PLANET_URL=http://download.geofabrik.de/europe/monaco-latest.osm.bz2 21 | - OVERPASS_DIFF_URL=http://download.openstreetmap.fr/replication/europe/monaco/minute/ 22 | - OVERPASS_UPDATE_SLEEP=3600 23 | - OVERPASS_USE_AREAS=false 24 | healthcheck: 25 | test: curl --noproxy '*' -qfg 'http://localhost/api/interpreter?data=[out:json];node(1);out;' | jq '.generator' |grep -q Overpass || exit 1 26 | start_period: 48h 27 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Wiktor Niesiobędzki 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /etc/supervisord.conf: -------------------------------------------------------------------------------- 1 | [supervisord] 2 | nodaemon=true 3 | 4 | [program:overpass_dispatch] 5 | command=/app/bin/dispatcher_start.sh 6 | user=overpass 7 | priority=1 8 | redirect_stderr=true 9 | stdout_logfile=/dev/stdout 10 | stdout_logfile_maxbytes=0 11 | autorestart=true 12 | 13 | [program:nginx] 14 | command=nginx 15 | priority=2 16 | redirect_stderr=true 17 | stdout_logfile=/dev/stdout 18 | stdout_logfile_maxbytes=0 19 | autorestart=true 20 | 21 | [program:fcgiwrap] 22 | command=/app/bin/start_fcgiwarp.sh 23 | user=nginx 24 | priority=3 25 | redirect_stderr=true 26 | stdout_logfile=/dev/stdout 27 | stdout_logfile_maxbytes=0 28 | autorestart=true 29 | 30 | [program:update_overpass] 31 | command=/app/bin/update_overpass_loop.sh 32 | user=overpass 33 | priority=5 34 | redirect_stderr=true 35 | stdout_logfile=/dev/stdout 36 | stdout_logfile_maxbytes=0 37 | autorestart=true 38 | 39 | [program:dispatcher_areas] 40 | command=nice /app/bin/dispatcher --areas --db-dir="/db/db" --allow-duplicate-queries=%(ENV_OVERPASS_ALLOW_DUPLICATE_QUERIES)s 41 | autostart=%(ENV_OVERPASS_USE_AREAS)s 42 | user=overpass 43 | priority=6 44 | redirect_stderr=true 45 | stdout_logfile=/dev/stdout 46 | stdout_logfile_maxbytes=0 47 | autorestart=true 48 | 49 | [program:areas_rules] 50 | command=nice /app/bin/rules_loop.sh /db/db %(ENV_OVERPASS_RULES_LOAD)s 51 | autostart=%(ENV_OVERPASS_USE_AREAS)s 52 | user=overpass 53 | priority=7 54 | redirect_stderr=true 55 | stdout_logfile=/dev/stdout 56 | stdout_logfile_maxbytes=0 57 | autorestart=true 58 | -------------------------------------------------------------------------------- /etc/nginx-overpass.conf.template: -------------------------------------------------------------------------------- 1 | daemon off; 2 | 3 | user nginx; 4 | worker_processes auto; 5 | 6 | pid /var/run/nginx.pid; 7 | 8 | 9 | events { 10 | worker_connections 1024; 11 | } 12 | 13 | 14 | http { 15 | include /etc/nginx/mime.types; 16 | default_type application/octet-stream; 17 | 18 | log_format main '$remote_addr - $remote_user [$time_local] "$request" ' 19 | '$status $body_bytes_sent "$http_referer" ' 20 | '"$http_user_agent" "$http_x_forwarded_for"'; 21 | 22 | access_log /var/log/nginx/access.log; 23 | error_log /var/log/nginx/error.log; 24 | 25 | sendfile on; 26 | #tcp_nopush on; 27 | 28 | keepalive_timeout 60s; 29 | 30 | gzip on; 31 | gzip_proxied any; 32 | gzip_types *; 33 | 34 | server { 35 | listen 80; 36 | server_name localhost; 37 | 38 | #charset koi8-r; 39 | 40 | location / { 41 | root /usr/share/nginx/html; 42 | index index.html index.htm; 43 | } 44 | 45 | #error_page 404 /404.html; 46 | 47 | # redirect server error pages to the static page /50x.html 48 | # 49 | error_page 500 502 503 504 /50x.html; 50 | location = /50x.html { 51 | root /usr/share/nginx/html; 52 | } 53 | 54 | location /cgi-bin/ { 55 | include fastcgi_params; 56 | fastcgi_param SCRIPT_FILENAME /app/$fastcgi_script_name; 57 | fastcgi_param PATH_INFO $fastcgi_path_info; 58 | fastcgi_pass unix:/nginx/fcgiwrap.socket; 59 | fastcgi_read_timeout ${OVERPASS_MAX_TIMEOUT}; 60 | fastcgi_send_timeout ${OVERPASS_MAX_TIMEOUT}; 61 | } 62 | 63 | location /api/ { 64 | rewrite ^/api/(.+)$ /cgi-bin/$1 last; 65 | } 66 | } 67 | 68 | 69 | } 70 | -------------------------------------------------------------------------------- /bin/rules_loop.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Copyright 2008, 2009, 2010, 2011, 2012 Roland Olbricht 4 | # 5 | # This file is part of Overpass_API. 6 | # 7 | # Overpass_API is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU Affero General Public License as 9 | # published by the Free Software Foundation, either version 3 of the 10 | # License, or (at your option) any later version. 11 | # 12 | # Overpass_API is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU Affero General Public License 18 | # along with Overpass_API. If not, see . 19 | 20 | if [[ -z $1 ]]; then 21 | { 22 | echo "Usage: $0 database_dir [desired_cpu_load]" 23 | exit 0 24 | } 25 | fi 26 | 27 | CPU_LOAD=${2:-100} 28 | DB_DIR="$(pwd)/$1" 29 | 30 | EXEC_DIR="$(dirname "$0")/" 31 | if [[ ! ${EXEC_DIR:0:1} == "/" ]]; then 32 | { 33 | EXEC_DIR="$(pwd)/$EXEC_DIR" 34 | } 35 | fi 36 | 37 | pushd "$EXEC_DIR" || exit 1 38 | 39 | while true; do 40 | { 41 | START=$(date +%s) 42 | echo "$(date '+%F %T'): update started" >>"$DB_DIR/rules_loop.log" 43 | ./osm3s_query --progress --rules <"$DB_DIR/rules/areas.osm3s" 44 | echo "$(date '+%F %T'): update finished" >>"$DB_DIR/rules_loop.log" 45 | WORK_TIME=$(($(date +%s) - START)) 46 | SLEEP_TIME=$((WORK_TIME * 100 / CPU_LOAD - WORK_TIME)) 47 | # let SLEEP_TIME be at least 3 seconds 48 | SLEEP_TIME=$((SLEEP_TIME < 3 ? 3 : SLEEP_TIME)) 49 | echo "It took $WORK_TIME to run the loop. Desired load is: ${CPU_LOAD}%. Sleeping: $SLEEP_TIME" 50 | sleep "$SLEEP_TIME" 51 | } 52 | done 53 | -------------------------------------------------------------------------------- /update.py: -------------------------------------------------------------------------------- 1 | import html.parser 2 | import os 3 | import pathlib 4 | import shutil 5 | import urllib.request 6 | 7 | url = "http://dev.overpass-api.de/releases/" 8 | 9 | 10 | class VersionFinder(html.parser.HTMLParser): 11 | def error(self, message): 12 | raise RuntimeError(message) 13 | 14 | def __init__(self): 15 | super().__init__() 16 | self.versions = [] 17 | 18 | def handle_starttag(self, tag, attrs): 19 | if attrs: 20 | href = dict(attrs).get('href') 21 | if tag == 'a' and href and href.startswith('osm-3s'): 22 | version = href[len('osm-3s_v'):-len('.tar.gz')] 23 | self.versions.append(version) 24 | 25 | 26 | def main(): 27 | parser = VersionFinder() 28 | response = urllib.request.urlopen(url) 29 | data = response.read().decode(response.headers.get_content_charset()) 30 | parser.feed(data) 31 | with open("Dockerfile.template") as f: 32 | template = f.read() 33 | for ver in parser.versions: 34 | if any((ver.startswith(x) for x in ('0.6', 'eta', '0.7.1', '0.7.2', '0.7.3', '0.7.4', '0.7.50', '0.7.52', 35 | '0.7.54.11', # invalid CRC in archive 36 | '0.7.51', # no autoconf 37 | ))) or \ 38 | ver == '0.7': 39 | # ignore old releases 40 | continue 41 | if os.path.exists(ver): 42 | shutil.rmtree(ver) 43 | os.mkdir(ver) 44 | with open(pathlib.Path(ver) / "Dockerfile", "w+") as f: 45 | f.write(template.format(version=ver)) 46 | #for i in ("etc", "bin"): 47 | # shutil.copytree(i, pathlib.Path(ver) / i) 48 | #shutil.copyfile("docker-entrypoint.sh", pathlib.Path(ver) / "docker-entrypoint.sh") 49 | #shutil.copyfile("requirements.txt", pathlib.Path(ver) / "requirements.txt") 50 | 51 | 52 | if __name__ == '__main__': 53 | main() 54 | -------------------------------------------------------------------------------- /bin/update_overpass.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # TODO: split to two files - one with while loop for unattended use 4 | # one for updating to current state - second called by first 5 | 6 | DIFF_FILE=/db/diffs/changes.osc 7 | 8 | OVERPASS_META=${OVERPASS_META:-no} 9 | OVERPASS_COMPRESSION=${OVERPASS_COMPRESSION:-gz} 10 | OVERPASS_FLUSH_SIZE=${OVERPASS_FLUSH_SIZE:-16} 11 | OVERPASS_DIFF_PREPROCESS=${OVERPASS_DIFF_PREPROCESS:-} 12 | 13 | if [ -z "$OVERPASS_DIFF_URL" ]; then 14 | echo "No OVERPASS_DIFF_URL set. Skipping update." 15 | exit 0 16 | fi 17 | 18 | ( 19 | set -e 20 | UPDATE_ARGS=("--flush-size=${OVERPASS_FLUSH_SIZE}") 21 | if [[ "${OVERPASS_META}" == "attic" ]]; then 22 | UPDATE_ARGS+=("--keep-attic") 23 | elif [[ "${OVERPASS_META}" == "yes" ]]; then 24 | UPDATE_ARGS+=("--meta") 25 | fi 26 | 27 | if [[ ! -d /db/diffs ]]; then 28 | mkdir /db/diffs 29 | fi 30 | 31 | if /app/bin/dispatcher --show-dir | grep -q File_Error; then 32 | UPDATE_ARGS+=("--db-dir=/db/db") 33 | fi 34 | 35 | if [[ "${USE_OAUTH_COOKIE_CLIENT}" = "yes" ]]; then 36 | /app/venv/bin/python /app/bin/oauth_cookie_client.py -o /db/cookie.jar -s /secrets/oauth-settings.json --format netscape 37 | fi 38 | 39 | while true; do 40 | # if DIFF_FILE doesn't exit, try fetch new data 41 | if [[ ! -e ${DIFF_FILE} ]]; then 42 | # if /db/replicate_id exists, do not pass $1 arg (which could contain -O arg pointing to planet file 43 | if [[ -s /db/replicate_id ]]; then 44 | cp -f /db/replicate_id /db/replicate_id.backup 45 | set +e 46 | /app/venv/bin/pyosmium-get-changes -vvv --cookie /db/cookie.jar --server "${OVERPASS_DIFF_URL}" -o "${DIFF_FILE}" -f /db/replicate_id 47 | OSMIUM_STATUS=$? 48 | set -e 49 | else 50 | set +e 51 | /app/venv/bin/pyosmium-get-changes -vvv "$@" --cookie /db/cookie.jar --server "${OVERPASS_DIFF_URL}" -o "${DIFF_FILE}" -f /db/replicate_id 52 | OSMIUM_STATUS=$? 53 | set -e 54 | fi 55 | else 56 | echo "${DIFF_FILE} exists. Trying to apply again." 57 | fi 58 | 59 | # if DIFF_FILE is non-empty, try to process it 60 | if [[ -s ${DIFF_FILE} ]]; then 61 | VERSION=$(osmium fileinfo -e -g data.timestamp.last "${DIFF_FILE}" || (cp -f /db/replicate_id.backup /db/replicate_id && echo "Broken file" && cat "${DIFF_FILE}" && rm -f "${DIFF_FILE}" && exit 1)) 62 | if [[ -n "${VERSION// /}" ]]; then 63 | #Check if an Preprocessing command exists and try to execute it 64 | if [[ -n ${OVERPASS_DIFF_PREPROCESS+x} ]]; then 65 | echo "Running preprocessing command: ${OVERPASS_DIFF_PREPROCESS}" 66 | eval "${OVERPASS_DIFF_PREPROCESS}" 67 | fi 68 | echo /app/bin/update_from_dir --osc-dir="$(dirname ${DIFF_FILE})" --version="${VERSION}" "${UPDATE_ARGS[@]}" 69 | /app/bin/update_from_dir --osc-dir="$(dirname ${DIFF_FILE})" --version="${VERSION}" "${UPDATE_ARGS[@]}" 70 | else 71 | echo "Empty version, skipping file" 72 | cat "${DIFF_FILE}" 73 | fi 74 | fi 75 | 76 | # processed successfuly, remove 77 | rm "${DIFF_FILE}" 78 | 79 | if [[ "${OSMIUM_STATUS}" -eq 3 ]]; then 80 | echo "Update finished with status code: ${OSMIUM_STATUS}" 81 | break 82 | else 83 | echo "There are still some updates remaining" 84 | continue 85 | fi 86 | break 87 | done 88 | ) 2>&1 | tee -a /db/changes.log 89 | -------------------------------------------------------------------------------- /Dockerfile.template: -------------------------------------------------------------------------------- 1 | FROM nginx:1.29-bookworm-otel AS builder 2 | 3 | RUN apt-get update \ 4 | && apt-get upgrade -y \ 5 | && apt-get install --no-install-recommends --no-install-suggests -y \ 6 | autoconf \ 7 | automake \ 8 | bash \ 9 | bzip2 \ 10 | ca-certificates \ 11 | curl \ 12 | expat \ 13 | fcgiwrap \ 14 | g++ \ 15 | libexpat1-dev \ 16 | liblz4-1 \ 17 | liblz4-dev \ 18 | libtool \ 19 | m4 \ 20 | make \ 21 | osmium-tool \ 22 | python3 \ 23 | python3-venv \ 24 | supervisor \ 25 | wget \ 26 | zlib1g \ 27 | zlib1g-dev 28 | 29 | ADD http://dev.overpass-api.de/releases/osm-3s_v{version}.tar.gz /app/src.tar.gz 30 | 31 | RUN mkdir -p /app/src \ 32 | && cd /app/src \ 33 | && tar -x -z --strip-components 1 -f ../src.tar.gz \ 34 | && autoscan \ 35 | && aclocal \ 36 | && autoheader \ 37 | && libtoolize \ 38 | && automake --add-missing \ 39 | && autoconf \ 40 | && CXXFLAGS='-O2' CFLAGS='-O2' ./configure --prefix=/app --enable-lz4 \ 41 | && make dist \ 42 | && make -j $(nproc) \ 43 | && make install clean \ 44 | && mkdir -p /db/diffs /app/etc \ 45 | && cp -r /app/src/rules /app/etc/rules \ 46 | && rm -rf /app/src /app/src.tar.gz 47 | 48 | FROM nginx:1.29-bookworm-otel 49 | 50 | RUN apt-get update \ 51 | && apt-get upgrade -y \ 52 | && apt-get install --no-install-recommends --no-install-suggests -y \ 53 | bash \ 54 | bzip2 \ 55 | ca-certificates \ 56 | curl \ 57 | expat \ 58 | fcgiwrap \ 59 | jq \ 60 | liblz4-1 \ 61 | osmium-tool \ 62 | python3 \ 63 | python3-venv \ 64 | supervisor \ 65 | wget \ 66 | zlib1g \ 67 | && rm -rf /var/lib/apt/lists/* 68 | 69 | COPY --from=builder /app /app 70 | 71 | ADD https://raw.githubusercontent.com/geofabrik/sendfile_osm_oauth_protector/master/oauth_cookie_client.py \ 72 | /app/bin/ 73 | RUN sed -i -e 's/allow_read_prefs": "yes"/allow_read_prefs": "1"/g' /app/bin/oauth_cookie_client.py 74 | RUN addgroup overpass && adduser --home /db --disabled-password --gecos overpass --ingroup overpass overpass 75 | 76 | COPY requirements.txt /app/ 77 | 78 | RUN python3 -m venv /app/venv \ 79 | && /app/venv/bin/pip install -r /app/requirements.txt --only-binary osmium 80 | 81 | RUN mkdir /nginx /docker-entrypoint-initdb.d && chown nginx:nginx /nginx && chown -R overpass:overpass /db 82 | 83 | COPY etc/supervisord.conf /etc/supervisor/conf.d/supervisord.conf 84 | 85 | COPY etc/nginx-overpass.conf.template /etc/nginx/nginx.conf.template 86 | 87 | COPY bin/update_overpass.sh bin/update_overpass_loop.sh bin/rules_loop.sh bin/dispatcher_start.sh bin/start_fcgiwarp.sh /app/bin/ 88 | 89 | COPY docker-entrypoint.sh docker-healthcheck.sh /app/ 90 | 91 | RUN chmod a+rx /app/docker-entrypoint.sh /app/bin/update_overpass.sh /app/bin/rules_loop.sh /app/bin/dispatcher_start.sh \ 92 | /app/bin/oauth_cookie_client.py /app/bin/start_fcgiwarp.sh 93 | 94 | ENV OVERPASS_RULES_LOAD=${{OVERPASS_RULES_LOAD:-1}} 95 | ENV OVERPASS_USE_AREAS=${{OVERPASS_USE_AREAS:-true}} 96 | ENV OVERPASS_ALLOW_DUPLICATE_QUERIES=${{OVERPASS_ALLOW_DUPLICATE_QUERIES:-no}} 97 | 98 | EXPOSE 80 99 | 100 | HEALTHCHECK --start-period=48h CMD /app/docker-healthcheck.sh 101 | 102 | CMD ["/app/docker-entrypoint.sh"] 103 | -------------------------------------------------------------------------------- /docker-entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -eo pipefail 4 | shopt -s nullglob 5 | OVERPASS_META=${OVERPASS_META:-no} 6 | OVERPASS_MODE=${OVERPASS_MODE:-clone} 7 | OVERPASS_COMPRESSION=${OVERPASS_COMPRESSION:-gz} 8 | OVERPASS_FLUSH_SIZE=${OVERPASS_FLUSH_SIZE:-16} 9 | OVERPASS_CLONE_SOURCE=${OVERPASS_CLONE_SOURCE:-https://dev.overpass-api.de/api_drolbr/} 10 | 11 | # this is used by other processes, so needs to be exported 12 | export OVERPASS_MAX_TIMEOUT=${OVERPASS_MAX_TIMEOUT:-1000s} 13 | 14 | if [[ "$OVERPASS_META" == "attic" ]]; then 15 | META="--keep-attic" 16 | elif [[ "${OVERPASS_META}" == "yes" ]]; then 17 | META="--meta" 18 | else 19 | META="" 20 | fi 21 | 22 | for f in /docker-entrypoint-initdb.d/*; do 23 | case "$f" in 24 | *.sh) 25 | if [[ -x "$f" ]]; then 26 | echo "$0: running $f" 27 | "$f" 28 | else 29 | echo "$0: sourcing $f" 30 | # shellcheck disable=SC1090 # ignore SC1090 (unable to follow file) because they are dynamically provided 31 | . "$f" 32 | fi 33 | ;; 34 | *) echo "$0: ignoring $f" ;; 35 | esac 36 | echo 37 | done 38 | 39 | if [[ ! -f /db/init_done ]]; then 40 | echo "No database directory. Initializing" 41 | touch /db/changes.log 42 | mkdir -p /db/diffs 43 | 44 | if [[ "${USE_OAUTH_COOKIE_CLIENT}" = "yes" ]]; then 45 | /app/venv/bin/python /app/bin/oauth_cookie_client.py -o /db/cookie.jar -s /secrets/oauth-settings.json --format netscape 46 | # necessary to add newline at the end as oauth_cookie_client doesn't do that 47 | echo >>/db/cookie.jar 48 | else 49 | echo "# Netscape HTTP Cookie File" >/db/cookie.jar 50 | echo "${OVERPASS_COOKIE_JAR_CONTENTS}" >>/db/cookie.jar 51 | fi 52 | chown -R overpass:overpass /db/cookie.jar /db/changes.log /db/diffs 53 | 54 | if [[ "$OVERPASS_MODE" = "clone" ]]; then 55 | ( 56 | mkdir -p /db/db && 57 | /app/bin/download_clone.sh --db-dir=/db/db --source="${OVERPASS_CLONE_SOURCE}" --meta="${OVERPASS_META}" && 58 | cp /db/db/replicate_id /db/replicate_id && 59 | cp -r /app/etc/rules /db/db && 60 | chown -R overpass:overpass /db/* && 61 | touch /db/init_done 62 | ) || ( 63 | echo "Failed to clone overpass repository" 64 | exit 1 65 | ) 66 | if [[ "${OVERPASS_STOP_AFTER_INIT}" == "false" ]]; then 67 | echo "Overpass container ready to receive requests" 68 | else 69 | echo "Overpass container initialization complete. Exiting." 70 | exit 0 71 | fi 72 | fi 73 | 74 | if [[ "$OVERPASS_MODE" = "init" ]]; then 75 | CURL_STATUS_CODE=$(curl -L -b /db/cookie.jar -o /db/planet.osm.bz2 -w "%{http_code}" "${OVERPASS_PLANET_URL}") 76 | # try again until it's allowed 77 | while [ "$CURL_STATUS_CODE" = "429" ]; do 78 | echo "Server responded with 429 Too many requests. Trying again in 5 minutes..." 79 | sleep 300 80 | CURL_STATUS_CODE=$(curl -L -b /db/cookie.jar -o /db/planet.osm.bz2 -w "%{http_code}" "${OVERPASS_PLANET_URL}") 81 | done 82 | # for `file:///` scheme curl returns `000` HTTP status code 83 | if [[ $CURL_STATUS_CODE = "200" || $CURL_STATUS_CODE = "000" ]]; then 84 | ( 85 | if [[ -n "${OVERPASS_PLANET_PREPROCESS:+x}" ]]; then 86 | echo "Running preprocessing command: ${OVERPASS_PLANET_PREPROCESS}" 87 | eval "${OVERPASS_PLANET_PREPROCESS}" 88 | fi && 89 | /app/bin/init_osm3s.sh /db/planet.osm.bz2 /db/db /app "${META}" "--version=$(osmium fileinfo -e -g data.timestamp.last /db/planet.osm.bz2) --compression-method=${OVERPASS_COMPRESSION} --map-compression-method=${OVERPASS_COMPRESSION} --flush-size=${OVERPASS_FLUSH_SIZE}" && 90 | echo "Database created. Now updating it." && 91 | cp -r /app/etc/rules /db/db && 92 | chown -R overpass:overpass /db/* && 93 | echo "Updating" && 94 | /app/bin/update_overpass.sh -O /db/planet.osm.bz2 && 95 | if [[ "${OVERPASS_USE_AREAS}" = "true" ]]; then 96 | echo "Generating areas..." && /app/bin/osm3s_query --progress --rules --db-dir=/db/db /etc/nginx/nginx.conf 125 | 126 | echo "Starting supervisord process" 127 | exec /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf 128 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # How to use this image 2 | 3 | By default, this image will clone an existing Overpass server for the whole planet, and make it available at `http://localhost/api/interpreter`. 4 | 5 | The following enviroment variables can be used to customize the setup: 6 | 7 | * `OVERPASS_MODE` - takes the value of either `init` or `clone`. Defaults to `clone`. 8 | * `OVERPASS_META` - `attic`, `yes`, or `no`. Passed as `--keep-attic`, `--meta`, or nothing to `update_database` and `init`. Defaults to `no`. 9 | * `OVERPASS_PLANET_URL` - (`init` mode only) url to a "planet" file (e.g. https://planet.openstreetmap.org/planet/planet-latest.osm.bz2) 10 | * `OVERPASS_CLONE_SOURCE` - (`clone` mode only) the url to clone a copy of Overpass from. Defaults to https://dev.overpass-api.de/api_drolbr/, which uses minute diffs. 11 | * `OVERPASS_DIFF_URL` - url to a diff directory for updating the instance (e.g. https://planet.openstreetmap.org/replication/minute/). 12 | * `OVERPASS_COMPRESSION` - (`init` mode only) takes values of `no`, `gz` or `lz4`. Specifies compression mode of the Overpass database. Defaults to `gz`. 13 | * `OVERPASS_RULES_LOAD` - integer, desired load from area generation. Controls the ratio of sleep to work. A value of 1 will make the system sleep 99x times longer than it works, a value of 50 will result in sleep and work in equal measure, and a value of 100 will only sleep 3 seconds between each execution. Defaults to 1. 14 | * `OVERPASS_UPDATE_SLEEP` - integer, the delay between updates (seconds). 15 | * `OVERPASS_COOKIE_JAR_CONTENTS` - cookie-jar compatible content to be used when fetching planet.osm files and updates. 16 | * `OVERPASS_PLANET_PREPROCESS` - commands to be run before passing the planet.osm file to `update_database`, e.g. conversion from pbf to osm.bz2 using osmium. 17 | * `OVERPASS_DIFF_PREPROCESS` - commands to be run before passing the diff.osc file to `update_database`, e.g. extracting geo specific regions 18 | * `USE_OAUTH_COOKIE_CLIENT` - set to `yes` if you want to use oauth_cookie_client to update cookies before each update. Settings are read from /secrets/oauth-settings.json. Read the documentation [here](https://github.com/geofabrik/sendfile_osm_oauth_protector/blob/master/doc/client.md). 19 | * `OVERPASS_FASTCGI_PROCESSES` - number of fcgiwarp processes. Defaults to 4. Use higher values if you notice performance problems. 20 | * `OVERPASS_RATE_LIMIT` - set the maximum allowed number of concurrent accesses from a single IP address. 21 | * `OVERPASS_TIME` - set the maximum amount of time units (available time). 22 | * `OVERPASS_SPACE` - set the maximum amount of RAM (available space) in bytes. 23 | * `OVERPASS_MAX_TIMEOUT` - set the maximum timeout for queries (default: 1000s). Translates to send/recv timeout for fastcgi_wrap. 24 | * `OVERPASS_USE_AREAS` - if `false` initial area generation and the area updater process will be disabled. Default `true`. 25 | * `OVERPASS_HEALTHCHECK` - shell commands to execute to verify that image is healthy. `exit 1` in case of failures, `exit 0` when container is healthy. Default healthcheck queries overpass and verifies that there is reponse returned 26 | * `OVERPASS_STOP_AFTER_INIT` - if `false` the container will keep runing after init is complete. Otherwise container will be stopped after initialization process is complete. Default `true` 27 | * `OVERPASS_ALLOW_DUPLICATE_QUERIES` - if `yes`, duplicate queries (same query from the same IP address) will be allowed. Default `no`. 28 | 29 | ### Modes 30 | 31 | Image works in two modes `init` or `clone`. This affects how the instance gets initialized: 32 | 33 | * `init` - OSM data is downloaded from `OVERPASS_PLANET_URL`, which can be a full planet or partial planet dump. 34 | This file will then be indexed by Overpass and later updated using `OVERPASS_DIFF_URL`. 35 | * `clone` - data is copied from an existing server, given by `OVERPASS_CLONE_SOURCE`, and then updated using `OVERPASS_DIFF_URL`. 36 | This mode is faster to set up, as the OSM planet file is already indexed. 37 | The default clone source provides an Overpass instance using minute diffs covering the whole world (hourly or daily diffs will not work with this image). 38 | 39 | ### Running 40 | 41 | To monitor the progress of file downloads, run with the stdin (`-i`) and TTY (`-t`) flags: 42 | `docker run -i -t wiktorn/overpass-api` 43 | 44 | After initialization is finished, the Docker container will stop. Once you start it again (with `docker start` command) it will start downloading diffs, applying them to database, and serving API requests. 45 | 46 | The container exposes port 80. Map it to your host port using `-p`: 47 | `docker run -p 80:80 wiktorn/overpass-api` 48 | 49 | The Overpass API will then be available at `http://localhost:80/api/interpreter`. 50 | 51 | Container includes binaries of pyosmium (in `/app/venv/bin/`) and osmium-tool (in `/usr/bin`) 52 | 53 | All data resides within the `/db` directory in the container. 54 | 55 | For convenience, a [`docker-compose.yml` template](./docker-compose.yml) is included. 56 | 57 | # Examples 58 | 59 | ## Overpass instance covering part of the world 60 | 61 | In this example the Overpass instance will be initialized with a planet file for Monaco downloaded from Geofabrik. 62 | Data will be stored in folder`/big/docker/overpass_db/` on the host machine and will not contain metadata as this example uses public Geofabrik extracts that do not contain metadata (such as changeset and user). 63 | Overpass will be available on port 12345 on the host machine. 64 | 65 | ``` 66 | docker run \ 67 | -e OVERPASS_META=yes \ 68 | -e OVERPASS_MODE=init \ 69 | -e OVERPASS_PLANET_URL=http://download.geofabrik.de/europe/monaco-latest.osm.bz2 \ 70 | -e OVERPASS_DIFF_URL=http://download.openstreetmap.fr/replication/europe/monaco/minute/ \ 71 | -e OVERPASS_RULES_LOAD=10 \ 72 | -v /big/docker/overpass_db/:/db \ 73 | -p 12345:80 \ 74 | -i -t \ 75 | --name overpass_monaco wiktorn/overpass-api 76 | ``` 77 | 78 | ## Overpass clone covering whole world 79 | 80 | In this example Overpass instance will be initialized with data from main Overpass instance and updated with master planet diffs. 81 | Data will be stored in `/big/docker/overpass_clone_db/` on the host machine and the API will be available on port 12346 on the host machine. 82 | 83 | ``` 84 | docker run \ 85 | -e OVERPASS_META=yes \ 86 | -e OVERPASS_MODE=clone \ 87 | -e OVERPASS_DIFF_URL=https://planet.openstreetmap.org/replication/minute/ \ 88 | -v /big/docker/overpass_clone_db/:/db \ 89 | -p 12346:80 \ 90 | -i -t \ 91 | --name overpass_world \ 92 | wiktorn/overpass-api 93 | ``` 94 | 95 | ## Overpass instance covering part of the world using cookie 96 | 97 | In this example Overpass instance will be initialized with planet file for Monaco downloaded from internal Geofabrik server. 98 | Data will be stored in `/big/docker/overpass_db/` on the host machine and the API will be available on port 12347 on the host machine. 99 | 100 | Prepare file with your credentials `/home/osm/oauth-settings.json`: 101 | 102 | ```json 103 | { 104 | "user": "your-username", 105 | "password": "your-secure-password", 106 | "osm_host": "https://www.openstreetmap.org", 107 | "consumer_url": "https://osm-internal.download.geofabrik.de/get_cookie" 108 | } 109 | ``` 110 | 111 | Because Geofabrik provides only PBF extracts with metadata, `osmium` is used in `OVERPASS_PLANET_PREPROCESS` to convert the `pbf` file to `osm.bz2` that's used by Overpass. 112 | 113 | ``` 114 | docker run \ 115 | -e OVERPASS_META=yes \ 116 | -e OVERPASS_MODE=init \ 117 | -e OVERPASS_PLANET_URL=https://osm-internal.download.geofabrik.de/europe/monaco-latest-internal.osm.pbf \ 118 | -e OVERPASS_DIFF_URL=https://osm-internal.download.geofabrik.de/europe/monaco-updates/ \ 119 | -e OVERPASS_RULES_LOAD=10 \ 120 | -e OVERPASS_COMPRESSION=gz \ 121 | -e OVERPASS_UPDATE_SLEEP=3600 \ 122 | -e OVERPASS_PLANET_PREPROCESS='mv /db/planet.osm.bz2 /db/planet.osm.pbf && osmium cat -o /db/planet.osm.bz2 /db/planet.osm.pbf && rm /db/planet.osm.pbf' \ 123 | -e USE_OAUTH_COOKIE_CLIENT=yes \ 124 | --mount type=bind,source=/home/osm/oauth-settings.json,target=/secrets/oauth-settings.json \ 125 | -v /big/docker/overpass_db/:/db \ 126 | -p 12347:80 \ 127 | -i -t \ 128 | --name overpass_monaco wiktorn/overpass-api 129 | ``` 130 | 131 | ## Healthcheck checking that instance is up-to-date 132 | 133 | Using following environment variable: 134 | 135 | ``` 136 | -e OVERPASS_HEALTHCHECK=' 137 | OVERPASS_RESPONSE=$(curl --noproxy "*" -s "http://localhost/api/interpreter?data=\[out:json\];node(1);out;" | jq -r .osm3s.timestamp_osm_base) 138 | OVERPASS_DATE=$(date -d "$OVERPASS_RESPONSE" +%s) 139 | TWO_DAYS_AGO=$(($(date +%s) - 2*86400)) ; 140 | if [ ${OVERPASS_DATE} -lt ${TWO_DAYS_AGO} ] ; then 141 | echo "Overpass out of date. Overpass date: ${OVERPASS_RESPONSE}" 142 | exit 1; 143 | fi 144 | echo "Overpass date: ${OVERPASS_RESPONSE}" 145 | ' 146 | ``` 147 | 148 | healthcheck will verify the date of last update of Overpass instance and if data in instance are earlier than two days ago, healthcheck will fail. 149 | 150 | # How to use Overpass after deploying using above examples 151 | 152 | The Overpass API will be exposed on the port exposed by `docker run` - for example `http://localhost:12346/api/interpreter`. 153 | 154 | You may then use this directly as an Overpass API url, or use it within [Overpass Turbo](http://overpass-turbo.eu/). 155 | 156 | Try a direct query with `http://localhost:12346/api/interpreter?data=node(3470507586);out geom;`, which should return a pub in Dublin. 157 | 158 | To use the API in Overpass Turbo, go to settings and set Server to`http://localhost:12346/api/`. Now you will use your local Overpass instance for your queries. 159 | --------------------------------------------------------------------------------