├── .gitignore ├── pyproject.toml ├── README.md ├── .github └── workflows │ └── deploy.yaml ├── LICENSE ├── parser.py └── burgerbot.py /.gitignore: -------------------------------------------------------------------------------- 1 | .env 2 | poetry.lock 3 | chats.json 4 | chats_tmp.json 5 | test.py 6 | __pycache__ 7 | auslander.py 8 | burgerbot_dev.py 9 | log.txt 10 | .vscode -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "burgerbot" 3 | version = "0.1.0" 4 | description = "" 5 | authors = ["sonac "] 6 | 7 | [tool.poetry.dependencies] 8 | python = "^3.8" 9 | requests = {extras = ["socks"], version = "^2.26.0"} 10 | beautifulsoup4 = "^4.9.3" 11 | python-telegram-bot = "^13.7" 12 | 13 | [tool.poetry.dev-dependencies] 14 | 15 | [build-system] 16 | requires = ["poetry-core>=1.0.0"] 17 | build-backend = "poetry.core.masonry.api" 18 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## BurgerBot 2 | 3 | I was frustrated with the lack of available slots in Burgeramt, so I've created this bot for myself, to catch one that available. It's pretty straightforward, once per 30 seconds it parses page with all appointments in all Berlin Burgeramts and if there is available slot for current and next month - it notifies in telegram with the link to registration. 4 | 5 | # Update 6 | 7 | I've shut down burgerbot in 2023 due to changes in the burgeramt website and it's lowered limits. https://allaboutberlin.com/ has good implementation of bot and I suggest either to use that or to fork this repo and continue yourself. 8 | -------------------------------------------------------------------------------- /.github/workflows/deploy.yaml: -------------------------------------------------------------------------------- 1 | name: deploy 2 | on: 3 | push: 4 | branches: 5 | - master 6 | 7 | jobs: 8 | deploy: 9 | name: Deploying 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v2 13 | - name: Deploy to DigitalOcean 14 | uses: appleboy/ssh-action@master 15 | with: 16 | host: ${{ secrets.DROPLET_IP }} 17 | key: ${{ secrets.SSH_KEY }} 18 | username: ${{ secrets.SSH_USER }} 19 | script: | 20 | cd burgerbot 21 | git pull 22 | systemctl restart burgerbot 23 | echo 'done' 24 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 sonac 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /parser.py: -------------------------------------------------------------------------------- 1 | import time 2 | import logging 3 | from dataclasses import dataclass 4 | from typing import List 5 | from re import S 6 | 7 | import requests 8 | from bs4 import BeautifulSoup 9 | 10 | default_url = "https://service.berlin.de/terminvereinbarung/termin/tag.php?termin=0&anliegen[]={}&dienstleisterlist=122210,122217,327316,122219,327312,122227,327314,122231,327346,122243,327348,122252,329742,122260,329745,122262,329748,122254,329751,122271,327278,122273,327274,122277,327276,330436,122280,327294,122282,327290,122284,327292,327539,122291,327270,122285,327266,122286,327264,122296,327268,150230,329760,122301,327282,122297,327286,122294,327284,122312,329763,122314,329775,122304,327330,122311,327334,122309,327332,122281,327352,122279,329772,122276,327324,122274,327326,122267,329766,122246,327318,122251,327320,122257,327322,122208,327298,122226,327300,121362,121364&herkunft=http%3A%2F%2Fservice.berlin.de%2Fdienstleistung%2F120686%2F" 11 | 12 | naturalization_url = "https://service.berlin.de/terminvereinbarung/termin/tag.php?termin=1&dienstleister=324261&anliegen[]=318998&herkunft=1" 13 | 14 | 15 | def build_url(id: int) -> str: 16 | if id == 318998: 17 | return naturalization_url.format(id) 18 | return default_url.format(id) 19 | 20 | 21 | @dataclass 22 | class Slot: 23 | msg: str 24 | service_id: int 25 | 26 | 27 | class Parser: 28 | def __init__(self, services: List[int]) -> None: 29 | self.services = services 30 | self.proxy_on: bool = False 31 | self.parse() 32 | 33 | def __get_url(self, url) -> requests.Response: 34 | logging.debug(url) 35 | try: 36 | if self.proxy_on: 37 | return requests.get(url, proxies={"https": "socks5://127.0.0.1:9050"}) 38 | return requests.get(url) 39 | except Exception as err: 40 | logging.warn( 41 | "received an error from the server, waiting for 1 minute before retry" 42 | ) 43 | logging.warn(err) 44 | time.sleep(60) 45 | return self.__get_url(url) 46 | 47 | def __toggle_proxy(self) -> None: 48 | self.proxy_on = not self.proxy_on 49 | 50 | def __parse_page(self, page, service_id) -> List[str]: 51 | try: 52 | if page.status_code == 428 or page.status_code == 429: 53 | logging.info("exceeded rate limit. Sleeping for a while") 54 | time.sleep(299) 55 | self.__toggle_proxy() 56 | return [] 57 | soup = BeautifulSoup(page.content, "html.parser") 58 | slots = soup.find_all("td", class_="buchbar") 59 | is_valid = soup.find_all("td", class_="nichtbuchbar") 60 | if len(is_valid) > 0: 61 | logging.info("page is valid") 62 | else: 63 | logging.debug(page) 64 | if len(slots) == 0: 65 | logging.info("no luck yet") 66 | return [Slot(slot.a["href"], service_id) for slot in slots] 67 | except Exception as e: ## sometimes shit happens 68 | logging.error(f"error occured during page parsing, {e}") 69 | self.__toggle_proxy() 70 | 71 | def add_service(self, service_id: int) -> None: 72 | self.services.append(service_id) 73 | 74 | def parse(self) -> List[str]: 75 | slots = [] 76 | logging.info("services are: " + str(self.services)) 77 | for svc in self.services: 78 | page = self.__get_url(build_url(svc)) 79 | slots += self.__parse_page(page, svc) 80 | return slots 81 | -------------------------------------------------------------------------------- /burgerbot.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import time 4 | import os 5 | import json 6 | import threading 7 | import logging 8 | import sys 9 | from dataclasses import dataclass, asdict 10 | from typing import List 11 | from datetime import datetime 12 | 13 | from telegram import ParseMode 14 | from telegram.ext import CommandHandler, Updater 15 | from telegram.ext.callbackcontext import CallbackContext 16 | from telegram.update import Update 17 | 18 | from parser import Parser, Slot, build_url 19 | 20 | 21 | CHATS_FILE = "chats.json" 22 | ua_url = "https://service.berlin.de/terminvereinbarung/termin/tag.php?termin=1&dienstleister=330857&anliegen[]=330869&herkunft=1" 23 | register_prefix = "https://service.berlin.de" 24 | 25 | service_map = { 26 | 120335: "Abmeldung einer Wohnung", 27 | 120686: "Anmeldung", 28 | 120691: "Verpflichtungserklärung für einen kurzen Aufenthalt", 29 | 120697: "Änderung/Wechsel der Hauptwohnung", 30 | 120701: "Personalausweis beantragen", 31 | 120702: "Meldebescheinigung", 32 | 120703: "Reisepass beantragen", 33 | 120914: "Zulassung eines Fahrzeuges mit auswärtigem Kennzeichen mit Halterwechsel", 34 | 121469: "Kinderreisepass beantragen / verlängern / aktualisieren", 35 | 121598: "Fahrerlaubnis Umschreibung einer ausländischen Fahrerlaubnis aus einem EU-/EWR-Staat", 36 | 121616: "Führerschein Kartenführerschein umtauschen", 37 | 121627: "Fahrerlaubnis Ersterteilung beantragen", 38 | 121701: "Beglaubigung von Kopien", 39 | 121921: "Gewerbeanmeldung", 40 | 305244: "Aufenthaltserlaubnis zum Studium", 41 | 318998: "Einbürgerung Verleihung der deutschen Staatsangehörigkeit beantragen", 42 | 324269: "Aufenthaltserlaubnis für im Bundesgebiet geborene Kinder - Erteilung", 43 | 324280: "Niederlassungserlaubnis oder Erlaubnis zum Daueraufenthalt-EU auf einen neuen Pass übertragen", 44 | 326556: "Niederlassungserlaubnis für Inhaber einer Blauen Karte EU", 45 | 326798: "Blaue Karte EU auf einen neuen Pass übertragen", 46 | 327537: "Fahrerlaubnis Umschreibung einer ausländischen Fahrerlaubnis aus einem Nicht-EU/EWR-Land (Drittstaat/Anlage 11)", 47 | 329328: "Aufenthaltserlaubnis für Fachkräfte mit akademischer Ausbildung", 48 | } 49 | 50 | 51 | @dataclass 52 | class Message: 53 | message: str 54 | ts: int # timestamp of adding msg to cache in seconds 55 | 56 | 57 | @dataclass 58 | class User: 59 | chat_id: int 60 | services: List[int] 61 | 62 | def __init__(self, chat_id, services=[120686]): 63 | self.chat_id = chat_id 64 | self.services = services if len(services) > 0 else [120686] 65 | 66 | def marshall_user(self) -> str: 67 | self.services = list( 68 | set([s for s in self.services if s in list(service_map.keys())]) 69 | ) 70 | return asdict(self) 71 | 72 | 73 | class Bot: 74 | def __init__(self) -> None: 75 | self.updater = Updater(os.environ["TELEGRAM_API_KEY"]) 76 | self.__init_chats() 77 | self.users = self.__get_chats() 78 | self.services = self.__get_uq_services() 79 | self.parser = Parser(self.services) 80 | self.dispatcher = self.updater.dispatcher 81 | self.dispatcher.add_handler(CommandHandler("help", self.__help)) 82 | self.dispatcher.add_handler(CommandHandler("start", self.__start)) 83 | self.dispatcher.add_handler(CommandHandler("stop", self.__stop)) 84 | self.dispatcher.add_handler(CommandHandler("add_service", self.__add_service)) 85 | self.dispatcher.add_handler( 86 | CommandHandler("remove_service", self.__remove_service) 87 | ) 88 | self.dispatcher.add_handler(CommandHandler("my_services", self.__my_services)) 89 | self.dispatcher.add_handler(CommandHandler("services", self.__services)) 90 | self.cache: List[Message] = [] 91 | 92 | def __get_uq_services(self) -> List[int]: 93 | services = [] 94 | for u in self.users: 95 | services.extend(u.services) 96 | services = filter(lambda x: x in service_map.keys(), services) 97 | return list(set(services)) 98 | 99 | def __init_chats(self) -> None: 100 | if not os.path.exists(CHATS_FILE): 101 | with open(CHATS_FILE, "w") as f: 102 | f.write("[]") 103 | 104 | def __get_chats(self) -> List[User]: 105 | with open(CHATS_FILE, "r") as f: 106 | users = [User(u["chat_id"], u["services"]) for u in json.load(f)] 107 | f.close() 108 | print(users) 109 | return users 110 | 111 | def __persist_chats(self) -> None: 112 | with open(CHATS_FILE, "w") as f: 113 | json.dump([u.marshall_user() for u in self.users], f) 114 | f.close() 115 | 116 | def __add_chat(self, chat_id: int) -> None: 117 | if chat_id not in [u.chat_id for u in self.users]: 118 | logging.info("adding new user") 119 | self.users.append(User(chat_id)) 120 | self.__persist_chats() 121 | 122 | def __remove_chat(self, chat_id: int) -> None: 123 | logging.info("removing the chat " + str(chat_id)) 124 | self.users = [u for u in self.users if u.chat_id != chat_id] 125 | self.__persist_chats() 126 | 127 | def __services(self, update: Update, _: CallbackContext) -> None: 128 | services_text = "" 129 | for k, v in service_map.items(): 130 | services_text += f"{k} - {v}\n" 131 | update.message.reply_text("Available services:\n" + services_text) 132 | 133 | def __help(self, update: Update, _: CallbackContext) -> None: 134 | try: 135 | update.message.reply_text( 136 | """ 137 | /start - start the bot 138 | /stop - stop the bot 139 | /add_service - add service to your list 140 | /remove_service - remove service from your list 141 | /my_services - view services on your list 142 | /services - list of available services 143 | """ 144 | ) 145 | except Exception as e: 146 | logging.error(f"error occured during help reply, {e}") 147 | 148 | def __start(self, update: Update, _: CallbackContext) -> None: 149 | self.__add_chat(update.message.chat_id) 150 | logging.info(f"got new user with id {update.message.chat_id}") 151 | update.message.reply_text( 152 | "Welcome to BurgerBot. When there will be slot - you will receive notification. To get information about usage - type /help. To stop it - just type /stop" 153 | ) 154 | 155 | def __stop(self, update: Update, _: CallbackContext) -> None: 156 | self.__remove_chat(update.message.chat_id) 157 | update.message.reply_text("Thanks for using me! Bye!") 158 | 159 | def __my_services(self, update: Update, _: CallbackContext) -> None: 160 | try: 161 | service_ids = set( 162 | service_id 163 | for u in self.users 164 | for service_id in u.services 165 | if u.chat_id == update.message.chat_id 166 | ) 167 | msg = ( 168 | "\n".join([f" - {service_id}" for service_id in service_ids]) 169 | or " - (none)" 170 | ) 171 | update.message.reply_text( 172 | "The following services are on your list:\n" + msg 173 | ) 174 | except Exception as e: 175 | logging.error(f"error occured when listing user services, {e}") 176 | 177 | def __add_service(self, update: Update, _: CallbackContext) -> None: 178 | logging.info(f"adding service {update.message}") 179 | try: 180 | service_id = int(update.message.text.split(" ")[1]) 181 | for u in self.users: 182 | if u.chat_id == update.message.chat_id: 183 | u.services.append(int(service_id)) 184 | self.__persist_chats() 185 | break 186 | update.message.reply_text("Service added") 187 | except Exception as e: 188 | update.message.reply_text( 189 | "Failed to add service, have you specified the service id?" 190 | ) 191 | logging.error(f"error occured when adding new service {e}") 192 | 193 | def __remove_service(self, update: Update, _: CallbackContext) -> None: 194 | logging.info(f"removing service {update.message}") 195 | try: 196 | service_id = int(update.message.text.split(" ")[1]) 197 | for u in self.users: 198 | if u.chat_id == update.message.chat_id: 199 | u.services.remove(int(service_id)) 200 | self.__persist_chats() 201 | break 202 | update.message.reply_text("Service removed") 203 | except IndexError: 204 | update.message.reply_text( 205 | "Wrong usage. Please type '/remove_service 123456'" 206 | ) 207 | 208 | def __poll(self) -> None: 209 | try: 210 | self.updater.start_polling() 211 | except Exception as e: 212 | logging.warn(e) 213 | logging.warn("got error during polling, retying") 214 | return self.__poll() 215 | 216 | def __parse(self) -> None: 217 | while True: 218 | slots = self.parser.parse() 219 | for slot in slots: 220 | self.__send_message(slot) 221 | time.sleep(30) 222 | 223 | def __send_message(self, slot: Slot) -> None: 224 | if self.__msg_in_cache(slot.msg): 225 | logging.info("Notification is cached already. Do not repeat sending") 226 | return 227 | self.__add_msg_to_cache(slot.msg) 228 | md_msg = f"There are slots on {self.__date_from_msg(slot.msg)} available for booking for {service_map[slot.service_id]}, click [here]({build_url(slot.service_id)}) to check it out" 229 | users = [u for u in self.users if slot.service_id in u.services] 230 | for u in users: 231 | logging.debug(f"sending msg to {str(u.chat_id)}") 232 | try: 233 | self.updater.bot.send_message( 234 | chat_id=u.chat_id, text=md_msg, parse_mode=ParseMode.MARKDOWN_V2 235 | ) 236 | except Exception as e: 237 | if ( 238 | "bot was blocked by the user" in e.__str__() 239 | or "user is deactivated" in e.__str__() 240 | ): 241 | logging.info( 242 | "removing since user blocked bot or user was deactivated" 243 | ) 244 | self.__remove_chat(u.chat_id) 245 | else: 246 | logging.error("error occured when sending message {md_msg}, \n{e}") 247 | self.__clear_cache() 248 | 249 | def __msg_in_cache(self, msg: str) -> bool: 250 | for m in self.cache: 251 | if m.message == msg: 252 | return True 253 | return False 254 | 255 | def __add_msg_to_cache(self, msg: str) -> None: 256 | self.cache.append(Message(msg, int(time.time()))) 257 | 258 | def __clear_cache(self) -> None: 259 | cur_ts = int(time.time()) 260 | if len(self.cache) > 0: 261 | logging.info("clearing some messages from cache") 262 | self.cache = [m for m in self.cache if (cur_ts - m.ts) < 300] 263 | 264 | def __date_from_msg(self, msg: str) -> str: 265 | msg_arr = msg.split("/") 266 | logging.info(msg) 267 | ts = ( 268 | int(msg_arr[len(msg_arr) - 2]) + 7200 269 | ) # adding two hours to match Berlin TZ with UTC 270 | return datetime.fromtimestamp(ts).strftime("%d %B") 271 | 272 | def start(self) -> None: 273 | logging.info("starting bot") 274 | poll_task = threading.Thread(target=self.__poll) 275 | parse_task = threading.Thread(target=self.__parse) 276 | parse_task.start() 277 | poll_task.start() 278 | parse_task.join() 279 | poll_task.join() 280 | 281 | 282 | def main() -> None: 283 | bot = Bot() 284 | bot.start() 285 | 286 | 287 | if __name__ == "__main__": 288 | log_level = os.getenv("LOG_LEVEL", "INFO") 289 | logging.basicConfig( 290 | level=log_level, 291 | format="%(asctime)s [%(levelname)-5.5s] %(message)s", 292 | handlers=[logging.StreamHandler(sys.stdout)], 293 | ) 294 | main() 295 | --------------------------------------------------------------------------------