├── .gitignore ├── Makefile ├── credentials.yml.example ├── Pipfile ├── rules.yml.example ├── README.md ├── LICENCE ├── Pipfile.lock ├── examples └── post_tripadvisor_email_to_booking_server └── inboxbot.py /.gitignore: -------------------------------------------------------------------------------- 1 | rules.yml 2 | credentials.yml 3 | *.eml 4 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: run 2 | run: 3 | pipenv sync 4 | pipenv run ./inboxbot.py 5 | -------------------------------------------------------------------------------- /credentials.yml.example: -------------------------------------------------------------------------------- 1 | --- 2 | hostname: "imap.fastmail.com" 3 | username: "you@fastmail.com" 4 | password: "your-password" 5 | -------------------------------------------------------------------------------- /Pipfile: -------------------------------------------------------------------------------- 1 | [[source]] 2 | url = "https://pypi.org/simple" 3 | verify_ssl = true 4 | name = "pypi" 5 | 6 | [dev-packages] 7 | 8 | [packages] 9 | pyyaml = "*" 10 | 11 | [requires] 12 | python_version = "3.6" 13 | -------------------------------------------------------------------------------- /rules.yml.example: -------------------------------------------------------------------------------- 1 | rules: 2 | 3 | - search: 4 | folder: "INBOX" 5 | from: "alert@updown.io" 6 | older_than_days: 3 7 | subject: "[updown alert]" 8 | action: "delete" 9 | 10 | - search: 11 | folder: "Archive" 12 | from: "alert@updown.io" 13 | older_than_days: 3 14 | subject: "[updown alert]" 15 | action: "delete" 16 | 17 | - search: 18 | folder: "INBOX" 19 | from: "bank-statements@example.com" 20 | subject: "Your latest statement" 21 | action: 22 | name: "run_script" 23 | script_path: "/usr/local/bin/parse_statement_from_email" 24 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Inboxbot 2 | 3 | Manage your inbox with YAML-defined rules, for example: 4 | 5 | ```yaml 6 | rules: 7 | 8 | - search: 9 | folder: "INBOX" 10 | from: "alert@updown.io" 11 | older_than_days: 3 12 | subject: "[updown alert]" 13 | action: "delete" 14 | ``` 15 | 16 | See [`rules.yml.example`](https://github.com/paulfurley/inboxbot/blob/master/rules.yml.example) for more examples. 17 | 18 | ## Install 19 | 20 | You'll need to run in a virtualenv: 21 | 22 | ``` 23 | virtualenv -p $(which python3) venv 24 | . venv/bin/activate 25 | 26 | pip install -r requirements.txt 27 | ``` 28 | 29 | ## Configure 30 | 31 | Copy and edit `rules.yml.example` and `credentials.yml.example` 32 | 33 | ``` 34 | cp credentials.yml.example credentials.yml 35 | cp rules.yml.example rules.yml 36 | ``` 37 | 38 | 39 | ## Run 40 | 41 | ``` 42 | ./run.py 43 | ``` 44 | -------------------------------------------------------------------------------- /LICENCE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (C) 2017 Paul M Furley 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 9 | of the Software, and to permit persons to whom the Software is furnished to do 10 | so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Pipfile.lock: -------------------------------------------------------------------------------- 1 | { 2 | "_meta": { 3 | "hash": { 4 | "sha256": "d518a36ed441568acff15b0a3c4b536738a55fb68801cdd682045be04d29954a" 5 | }, 6 | "pipfile-spec": 6, 7 | "requires": { 8 | "python_version": "3.6" 9 | }, 10 | "sources": [ 11 | { 12 | "name": "pypi", 13 | "url": "https://pypi.org/simple", 14 | "verify_ssl": true 15 | } 16 | ] 17 | }, 18 | "default": { 19 | "pyyaml": { 20 | "hashes": [ 21 | "sha256:0e7f69397d53155e55d10ff68fdfb2cf630a35e6daf65cf0bdeaf04f127c09dc", 22 | "sha256:2e9f0b7c5914367b0916c3c104a024bb68f269a486b9d04a2e8ac6f6597b7803", 23 | "sha256:35ace9b4147848cafac3db142795ee42deebe9d0dad885ce643928e88daebdcc", 24 | "sha256:38a4f0d114101c58c0f3a88aeaa44d63efd588845c5a2df5290b73db8f246d15", 25 | "sha256:483eb6a33b671408c8529106df3707270bfacb2447bf8ad856a4b4f57f6e3075", 26 | "sha256:4b6be5edb9f6bb73680f5bf4ee08ff25416d1400fbd4535fe0069b2994da07cd", 27 | "sha256:7f38e35c00e160db592091751d385cd7b3046d6d51f578b29943225178257b31", 28 | "sha256:8100c896ecb361794d8bfdb9c11fce618c7cf83d624d73d5ab38aef3bc82d43f", 29 | "sha256:c0ee8eca2c582d29c3c2ec6e2c4f703d1b7f1fb10bc72317355a746057e7346c", 30 | "sha256:e4c015484ff0ff197564917b4b4246ca03f411b9bd7f16e02a2f586eb48b6d04", 31 | "sha256:ebc4ed52dcc93eeebeae5cf5deb2ae4347b3a81c3fa12b0b8c976544829396a4" 32 | ], 33 | "index": "pypi", 34 | "version": "==5.2" 35 | } 36 | }, 37 | "develop": {} 38 | } 39 | -------------------------------------------------------------------------------- /examples/post_tripadvisor_email_to_booking_server: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | import email.policy 4 | import re 5 | import sys 6 | 7 | import lxml.html 8 | 9 | from datetime import datetime 10 | from email.parser import BytesParser 11 | 12 | 13 | def main(): 14 | parser = BytesParser(policy=email.policy.default) 15 | 16 | email_bytes = sys.stdin.buffer.read() 17 | 18 | eml = parser.parsebytes(text=email_bytes) 19 | print("Got email!") 20 | print(f"from: {eml['from']}") 21 | print(f"to: {eml['to']}") 22 | print(f"subject: {eml['subject']}") 23 | 24 | html_body = eml.get_body(preferencelist=('html',)) 25 | if html_body is None: 26 | raise RuntimeError("no html body") 27 | 28 | body = html_body.get_content() 29 | # print(f"\n{body}...") 30 | 31 | parsed = parse_html_body(body) 32 | print("\n----------") 33 | from pprint import pprint 34 | pprint(parsed) 35 | print("\n----------") 36 | 37 | 38 | def parse_html_body(html_body): 39 | root = lxml.html.fromstring(html_body) 40 | 41 | return { 42 | "name": parse_name(root), 43 | "phone": parse_phone(root), 44 | "date": parse_date(root), 45 | "game_slug": parse_game_slug(root), 46 | "net_price": parse_net_price(root), 47 | } 48 | 49 | 50 | def parse_name(root): 51 | """ 52 | 53 | Lead Traveler Name: John Doe 54 | 55 | """ 56 | 57 | spans = root.xpath('//td[contains(text(), "Lead Traveler Name:")]/span') 58 | assert len(spans) == 1 59 | 60 | return spans[0].text_content().strip() 61 | 62 | 63 | def parse_date(root): 64 | """ 65 | 66 | Travel Date: Sat, Nov 09, 2019 67 | 68 | """ 69 | 70 | spans = root.xpath('//td[contains(text(), "Travel Date:")]/span') 71 | assert len(spans) == 1 72 | 73 | return datetime.strptime(spans[0].text_content(), "%a, %b %d, %Y").date() 74 | 75 | 76 | def parse_phone(root): 77 | """ 78 | 79 | Phone: (Alternate Phone)0776000123 80 | Send the customer a message. 82 | 83 | 84 | OR: 85 | (Alternate Phone)CH+41 70000007 86 | """ 87 | 88 | tds = root.xpath('//td[contains(text(), "Phone:")]') 89 | assert len(tds) == 1 90 | 91 | line = tds[0].text_content().strip() 92 | pattern = re.compile(r'(?P\+?\d+ ?\d+)') 93 | match = pattern.search(line) 94 | if match is None: 95 | raise RuntimeError(f"can't parse phone number from {line}") 96 | 97 | return match.group('phone').replace(" ", "") 98 | 99 | 100 | def parse_game_slug(root): 101 | """ 102 | 103 | Product Code: 158392P1 104 | 105 | """ 106 | 107 | spans = root.xpath('//td[contains(text(), "Product Code:")]/span') 108 | assert len(spans) == 1 109 | 110 | product_code = spans[0].text_content().strip() 111 | 112 | lookup = { 113 | "158392P1": "liverpool-the-grand-voyage", 114 | "158392P2": "liverpool-the-two-cathedrals", 115 | } 116 | 117 | try: 118 | return lookup[product_code] 119 | except KeyError: 120 | raise ValueError(f"Don't know product code {product_code}") 121 | 122 | 123 | def parse_net_price(root): 124 | """ 125 | 126 | Net Rate: GBP £7.99 127 | 128 | 129 | """ 130 | 131 | spans = root.xpath('//td[contains(text(), "Net Rate:")]/span') 132 | assert len(spans) == 1 133 | 134 | line = spans[0].text_content() 135 | 136 | pattern = re.compile(r'GBP £(?P\d+\.\d\d)') 137 | match = pattern.search(line) 138 | if match is None: 139 | raise RuntimeError(f"can't parse price from {line}") 140 | 141 | return match.group('amount') 142 | 143 | 144 | if __name__ == "__main__": 145 | main() 146 | -------------------------------------------------------------------------------- /inboxbot.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Thanks https://gist.github.com/robulouski/7441883 4 | 5 | import datetime 6 | import io 7 | import email 8 | import email.policy 9 | import imaplib 10 | import logging 11 | import os 12 | import shlex 13 | import subprocess 14 | import sys 15 | import smtplib 16 | 17 | from email.message import EmailMessage 18 | from email.parser import BytesParser 19 | 20 | from pathlib import Path, PurePath 21 | 22 | import yaml 23 | 24 | 25 | class MessageSet(): 26 | def __init__(self, folder, message_numbers): 27 | self.folder = folder 28 | self.message_numbers = message_numbers 29 | 30 | def __len__(self): 31 | return len(self.message_numbers) 32 | 33 | def __str__(self): 34 | return '{} - {}'.format(self.folder, ' '.join(self.message_numbers)) 35 | 36 | 37 | class SearchStringBuilder(): 38 | def __init__(self, search_conditions): 39 | 40 | search_parts = [] 41 | 42 | for key, value in search_conditions.items(): 43 | if key == 'from': 44 | search_parts.append('FROM "{}"'.format(value)) 45 | 46 | elif key == 'older_than_days': 47 | date_before = datetime.date.today() - datetime.timedelta( 48 | days=value 49 | ) 50 | search_parts.append('BEFORE "{}"'.format( 51 | date_before.strftime('%d-%b-%Y')) 52 | ) 53 | 54 | elif key == 'subject': 55 | search_parts.append('SUBJECT "{}"'.format(value)) 56 | 57 | elif key == 'to': 58 | search_parts.append('TO "{}"'.format(value)) 59 | 60 | elif key == 'is_unread': 61 | if value is True: 62 | search_parts.append('UNSEEN') 63 | else: 64 | search_parts.append('SEEN') 65 | 66 | elif key == 'has_header': 67 | search_parts.append('HEADER {} ""'.format(value)) 68 | 69 | else: 70 | raise NotImplementedError( 71 | "Don't understand search condition `{}`".format(key) 72 | ) 73 | 74 | self._imap_string = '({})'.format(' '.join(search_parts)) 75 | 76 | def __str__(self): 77 | return self._imap_string 78 | 79 | 80 | class Mailbox(): 81 | def __init__(self, imap_hostname, smtp_hostname, username, password): 82 | self.c = imaplib.IMAP4_SSL(imap_hostname) 83 | 84 | self.c.login(username, password) 85 | status, folders = self.c.list() 86 | 87 | if status != "OK": 88 | raise RuntimeError(status) 89 | 90 | for folder in folders: 91 | logging.debug(f"folder: {folder}") 92 | 93 | if smtp_hostname: 94 | self.smtp = smtplib.SMTP_SSL(smtp_hostname) 95 | self.smtp.login(username, password) 96 | else: 97 | self.smtp = None 98 | 99 | def delete(self, message_set): 100 | self.c.select(message_set.folder) 101 | 102 | for num in message_set.message_numbers: 103 | logging.info("DELETE {}: {}".format(message_set.folder, num)) 104 | self.c.store(num, '+FLAGS', '\\Deleted') 105 | 106 | self.c.expunge() 107 | 108 | def mark_read(self, message_set): 109 | self.c.select(message_set.folder) 110 | 111 | for num in message_set.message_numbers: 112 | logging.info("SEEN {}: {}".format(message_set.folder, num)) 113 | self.c.store(num, '+FLAGS', '\\Seen') 114 | 115 | def echo(self, message_set): 116 | count = 0 117 | for email_message in self.load_email_messages(message_set): 118 | count += 1 119 | print("----------") 120 | print(f"From: {email_message['from']}") 121 | print(f"To: {email_message['to']}") 122 | print(f"Subject: {email_message['subject']}") 123 | 124 | body_email_message = email_message.get_body(preferencelist=('plain',)) 125 | if body_email_message is not None: 126 | body = body_email_message.get_content() 127 | print(f"\n{body}...") 128 | else: 129 | print("[no text body found]") 130 | 131 | logging.info(f"{count} emails echoed") 132 | 133 | def dump(self, message_set): 134 | count = 0 135 | for email_bytes in self.load_raw_emails(message_set): 136 | count += 1 137 | 138 | filename = f"email_{count}.eml" 139 | logging.info(f"writing {filename}") 140 | 141 | with io.open(filename, "wb") as f: 142 | f.write(email_bytes) 143 | 144 | logging.info(f"{count} emails dumped") 145 | 146 | def run_script(self, message_set, script_path): 147 | # TODO: move these out of Mailbox: they don't belong here 148 | 149 | count = 0 150 | for email_bytes in self.load_raw_emails(message_set): 151 | count += 1 152 | 153 | logging.info(f"sending email to {script_path}") 154 | p = subprocess.Popen( 155 | shlex.split(script_path), 156 | stdin=subprocess.PIPE, 157 | stdout=subprocess.PIPE, 158 | stderr=subprocess.PIPE, 159 | ) 160 | 161 | try: 162 | stdout, stderr = p.communicate(email_bytes, timeout=15) 163 | except subprocess.TimeoutExpired: 164 | p.kill() 165 | stdout, stderr = p.communicate() 166 | 167 | for line in stdout.decode("utf-8").splitlines(): 168 | logging.info(f"stdout> {line}") 169 | 170 | for line in stderr.decode("utf-8").splitlines(): 171 | logging.info(f"stderr> {line}") 172 | 173 | if p.returncode != 0: 174 | raise RuntimeError(f"failed with exit code {p.returncode}") 175 | 176 | logging.info(f"{count} emails sent to {script_path}") 177 | 178 | def forward(self, message_set, to, from_address): 179 | count = 0 180 | for email_message in self.load_email_messages(message_set): 181 | count += 1 182 | 183 | msg = EmailMessage() 184 | msg["subject"] = f"Fwd: {email_message['subject']}" 185 | msg["to"] = to 186 | msg["from"] = from_address 187 | 188 | msg.set_content("Please see attached email.") 189 | 190 | msg.add_attachment( 191 | email_message.as_bytes(), 192 | maintype="message", 193 | subtype="rfc822", 194 | filename=f"{email_message['subject']}.eml", 195 | ) 196 | 197 | logging.info(f"forwarding email to {to}: email_message['subject']") 198 | self.smtp.send_message(msg) 199 | 200 | logging.info(f"{count} emails forwarded to {to}") 201 | 202 | def search(self, search_conditions): 203 | """ 204 | See https://tools.ietf.org/html/rfc3501#section-6.4.4 205 | """ 206 | 207 | folder = search_conditions.pop('folder') 208 | status, other = self.c.select(folder) 209 | if status != "OK": 210 | raise RuntimeError(f"{status} {other}") 211 | 212 | search_string = str(SearchStringBuilder(search_conditions)) 213 | 214 | logging.debug(f"search string: {search_string}") 215 | 216 | typ, msgnums = self.c.search(None, search_string) 217 | assert typ == 'OK', typ 218 | assert len(msgnums) == 1, msgnums 219 | 220 | message_numbers = msgnums[0].decode('ascii').split() 221 | 222 | message_set = MessageSet(folder, message_numbers) 223 | 224 | logging.debug("Got {} messages: {} ".format( 225 | len(message_set), message_set)) 226 | return message_set 227 | 228 | def load_email_messages(self, message_set): 229 | """ 230 | load_email_messages yields an EmailMessage for each email defined in message_set 231 | """ 232 | parser = BytesParser(policy=email.policy.default) 233 | 234 | for email_bytes in self.load_raw_emails(message_set): 235 | yield parser.parsebytes(text=email_bytes) 236 | 237 | def load_raw_emails(self, message_set): 238 | """ 239 | load_raw_emails yields a slice of bytes for the raw content of each email defined 240 | in message_set. 241 | """ 242 | 243 | self.c.select(message_set.folder) 244 | 245 | for message_number in message_set.message_numbers: 246 | 247 | type_, crappy_data = self.c.fetch(message_number, '(RFC822)') 248 | assert type_ == 'OK', type_ 249 | 250 | yield crappy_data[0][1] 251 | 252 | 253 | def main(): 254 | logging.basicConfig(level=logging.DEBUG) 255 | 256 | config_dir = get_config_path() 257 | 258 | account_dirs = list(get_account_dirs(config_dir)) 259 | if not account_dirs: 260 | print("ERROR: no config found.") 261 | print(f"create a subdirectory in {config_dir} containing rules.yml, credentials.yml") 262 | sys.exit(1) 263 | 264 | for account_dir in account_dirs: 265 | logging.info(f"running rules from {account_dir}") 266 | 267 | with io.open(account_dir.joinpath("credentials.yml"), "rb") as f: 268 | credentials = yaml.load(f) 269 | 270 | with io.open(account_dir.joinpath("rules.yml"), "rb") as f: 271 | rules = yaml.load(f) 272 | logging.debug(f"rules: {rules}") 273 | 274 | mailbox = Mailbox(**credentials) 275 | 276 | run_rules(mailbox, rules) 277 | 278 | 279 | def get_config_path(): 280 | """ 281 | returns a Path from the environment variable XDG_CONFIG_HOME or ~/.config if unset 282 | """ 283 | 284 | try: 285 | return PurePath(os.environ["XDG_CONFIG_HOME"]) 286 | except KeyError: 287 | return Path.home().joinpath(".config", "inboxbot", "accounts.d") 288 | 289 | 290 | def get_account_dirs(config_dir): 291 | """ 292 | get_account_dirs returns subdirectories of ${XDG_CONFIG_HOME}/inboxbot/accounts.d 293 | (defaulting to ~/.config/) 294 | """ 295 | 296 | try: 297 | for fn in os.listdir(config_dir): 298 | if os.path.isdir(config_dir.joinpath(fn)) and ".disabled" not in fn: 299 | yield config_dir.joinpath(fn) 300 | 301 | except FileNotFoundError: 302 | return 303 | 304 | 305 | def run_rules(mailbox, rules): 306 | ACTIONS = { 307 | 'delete': mailbox.delete, 308 | 'mark_read': mailbox.mark_read, 309 | 'unsubscribe': attempt_unsubscribe, 310 | 'echo': mailbox.echo, 311 | 'dump': mailbox.dump, 312 | 'run_script': mailbox.run_script, 313 | 'forward': mailbox.forward, 314 | } 315 | 316 | for rule in rules['rules']: 317 | logging.debug(f"running rule: {rule}") 318 | 319 | search_results = mailbox.search(rule['search']) 320 | 321 | if "action" in rule: 322 | actions = [rule["action"]] 323 | elif "actions" in rule: 324 | actions = rule["actions"] 325 | else: 326 | raise ValueError("every rule needs `action` or `actions`") 327 | 328 | for action in actions: 329 | if isinstance(action, str): 330 | action_name = action 331 | action_kwargs = {} 332 | 333 | elif isinstance(action, dict): 334 | action_name = action.pop("name") 335 | action_kwargs = action 336 | 337 | try: 338 | action_func = ACTIONS[action_name] 339 | except KeyError: 340 | raise NotImplementedError(f"action isn't implemented: {action_name}") 341 | else: 342 | action_func(search_results, **action_kwargs) 343 | 344 | 345 | def attempt_unsubscribe(message_set): 346 | logging.debug("Attempt unsubscribe: {}".format(message_set)) 347 | raise NotImplementedError 348 | 349 | 350 | if __name__ == '__main__': 351 | main() 352 | --------------------------------------------------------------------------------