├── .gitignore ├── .travis.yml ├── LICENSE ├── MANIFEST.in ├── README.rst ├── ad_notifier ├── __init__.py ├── cli.py ├── mailer.py └── pinkbike.py ├── pytest.ini ├── requirements.txt ├── setup.cfg ├── setup.py └── tox.ini /.gitignore: -------------------------------------------------------------------------------- 1 | *.sw[a-z] 2 | *.py[cox] 3 | 4 | .coverage 5 | .*.cache 6 | .env* 7 | 8 | .cache/ 9 | .tox/ 10 | *.egg-info/ 11 | 12 | htmlcov/ 13 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | 3 | env: 4 | matrix: 5 | - TOX_ENV=py27 6 | - TOX_ENV=py34 7 | 8 | before_script: 9 | - pip install tox 10 | 11 | script: 12 | - tox -e ${TOX_ENV} 13 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Sebastian Vetter 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.rst LICENSE 2 | prune *.pyc 3 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | pycon-talk-click-example 2 | ############################# 3 | 4 | 5 | .. image:: https://travis-ci.org/elbaschid/pycon-talk-click-example.svg?branch=master 6 | :target: https://travis-ci.org/elbaschid/pycon-talk-click-example 7 | 8 | 9 | License 10 | ------- 11 | 12 | This code is licensed under the `MIT License`_. 13 | 14 | .. _`MIT License`: https://github.com/elbaschid/pycon-talk-click-example/blob/master/LICENSE 15 | -------------------------------------------------------------------------------- /ad_notifier/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roadsideseb/pycon-talk-click-example/4b9c208a8138892b0b7e7b8e72eecfa4552c1e12/ad_notifier/__init__.py -------------------------------------------------------------------------------- /ad_notifier/cli.py: -------------------------------------------------------------------------------- 1 | import time 2 | import click 3 | import schedule 4 | 5 | from .pinkbike import find_new_ads 6 | from .mailer import send_email, ads_as_list 7 | 8 | 9 | def process_url(url, email, reset_cache): 10 | click.echo('Processing URL: {}'.format(url)) 11 | 12 | if reset_cache: 13 | click.secho('Resetting ad cache 🔥.', fg='yellow') 14 | 15 | new_ads = find_new_ads(url, reset_cache) 16 | 17 | if new_ads: 18 | click.secho('Found {} new ads 🎉'.format(len(new_ads)), fg='green') 19 | else: 20 | click.secho('No new ads 😢.', fg='red') 21 | 22 | if email and new_ads: 23 | send_email(email, 24 | subject='New ads available', 25 | content=ads_as_list(new_ads)) 26 | 27 | click.secho('Sent email to {} 📮 '.format(email), fg='green') 28 | else: 29 | click.secho('No notification sent 💤.', fg='red') 30 | 31 | 32 | @click.group() 33 | @click.argument('url') 34 | @click.option('--email', help='email to send notifications to') 35 | @click.option('--reset-cache', default=False, is_flag=True, 36 | help='reset the internal ads cache') 37 | @click.pass_context 38 | def main(context, url, email, reset_cache): 39 | """ 40 | Check pinkbike ads for URL and (optionally) send email notification. 41 | """ 42 | context.obj = {'url': url, 43 | 'email': email, 44 | 'reset_cache': reset_cache} 45 | 46 | 47 | @main.command() 48 | @click.pass_context 49 | def run_once(context): 50 | """ 51 | Run check for new ads once. 52 | """ 53 | process_url(**context.obj) 54 | 55 | 56 | @main.command() 57 | @click.option('--run-every', default=5, type=int, 58 | help='run the ad check every X minutes') 59 | @click.pass_context 60 | def run_periodically(context, run_every): 61 | """ 62 | Run scheduler to check for new ads periodically. 63 | """ 64 | click.echo('Running as scheduled job, every {} minute(s)'.format(run_every)) 65 | 66 | schedule.every(run_every).minutes.do(process_url, **context.obj) 67 | 68 | while True: 69 | schedule.run_pending() 70 | time.sleep(1) 71 | -------------------------------------------------------------------------------- /ad_notifier/mailer.py: -------------------------------------------------------------------------------- 1 | import dj_email_url 2 | 3 | from envelopes import Envelope 4 | 5 | 6 | def ads_as_list(ads): 7 | content = [] 8 | for ad in ads: 9 | content.append('* {ad.id} - {ad.title} - {ad.url}\n\n'.format(ad=ad)) 10 | return ''.join(content) 11 | 12 | 13 | def send_email(to, subject, content): 14 | email_config = dj_email_url.config() 15 | 16 | if not email_config: 17 | return 18 | 19 | smtp_config = {'host': email_config['EMAIL_HOST'], 20 | 'port': email_config['EMAIL_PORT'], 21 | 'login': email_config['EMAIL_HOST_USER'], 22 | 'password': email_config['EMAIL_HOST_PASSWORD'], 23 | 'tls': email_config['EMAIL_USE_TLS']} 24 | 25 | envelope = Envelope( 26 | from_addr='seb@roadsi.de', 27 | to_addr=[to], 28 | subject=subject, 29 | text_body=content) 30 | 31 | envelope.send(**smtp_config) 32 | -------------------------------------------------------------------------------- /ad_notifier/pinkbike.py: -------------------------------------------------------------------------------- 1 | import os 2 | import click 3 | import requests 4 | 5 | from hashlib import sha1 6 | from collections import namedtuple 7 | 8 | from bs4 import BeautifulSoup 9 | 10 | 11 | Ad = namedtuple('Ad', ['id', 'title', 'url', 'image']) 12 | 13 | 14 | def get_cache_filename(url, reset=False): 15 | filename = '.{}.cache'.format(sha1(url.encode('utf-8')).hexdigest()) 16 | 17 | if reset and os.path.exists(filename): 18 | os.remove(filename) 19 | 20 | return filename 21 | 22 | 23 | def find_new_ads(url, reset_cache=False): 24 | """ 25 | Extracts a list of new ads at the URL in *url*. 26 | """ 27 | response = requests.get(url) 28 | 29 | if not response.ok: 30 | raise click.ClickException("couldn't get search result page") 31 | 32 | cache_file = get_cache_filename(url, reset=reset_cache) 33 | 34 | soup = BeautifulSoup(response.content, 'html.parser') 35 | 36 | old_ads = [] 37 | if os.path.exists(cache_file): 38 | with open(cache_file, 'r') as cache: 39 | old_ads = [l.strip() for l in cache] 40 | 41 | new_ads = [] 42 | with open(cache_file, 'a+') as cache: 43 | for bsitem in soup.select('.bsitem'): 44 | item_id = bsitem['id'] 45 | 46 | if item_id in old_ads: 47 | continue 48 | 49 | elem = bsitem.select('td > div > a')[0] 50 | 51 | ad = Ad(id=item_id, 52 | title=elem.text.strip(), 53 | url=elem.get('href', ''), 54 | image='') 55 | 56 | new_ads.append(ad) 57 | cache.write('{}\n'.format(item_id)) 58 | 59 | return new_ads 60 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/roadsideseb/pycon-talk-click-example/4b9c208a8138892b0b7e7b8e72eecfa4552c1e12/pytest.ini -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | tox 2 | pytest 3 | pytest-cache 4 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [wheel] 2 | universal = 1 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import sys 3 | 4 | from setuptools import setup 5 | from setuptools.command.test import test as TestCommand 6 | 7 | requires = ['click', 8 | 'requests', 9 | 'beautifulsoup4', 10 | 'envelopes', 11 | 'dj-email-url', 12 | 'schedule'] 13 | 14 | tests_require = ['pytest', 'pytest-cache', 'pytest-cov'] 15 | 16 | 17 | class PyTest(TestCommand): 18 | def finalize_options(self): 19 | TestCommand.finalize_options(self) 20 | self.test_args = [] 21 | self.test_suite = True 22 | 23 | def run_tests(self): 24 | # import here, cause outside the eggs aren't loaded 25 | import pytest 26 | errno = pytest.main(self.test_args) 27 | sys.exit(errno) 28 | 29 | 30 | setup( 31 | name="pycon-talk-click-example", 32 | version='0.0.0', 33 | description="", 34 | long_description="\n\n".join([open("README.rst").read()]), 35 | license=open('LICENSE').read(), 36 | author="Sebastian Vetter", 37 | author_email="seb@roadsi.de", 38 | url="https://pycon-talk-click-example.readthedocs.org", 39 | packages=['ad_notifier'], 40 | install_requires=requires, 41 | entry_points={'console_scripts': [ 42 | 'ad_notifier = ad_notifier.cli:main' 43 | ]}, 44 | classifiers=[ 45 | 'Development Status :: 1 - Planning', 46 | 'License :: OSI Approved :: MIT License', 47 | 'Intended Audience :: Developers', 48 | 'Programming Language :: Python', 49 | 'Programming Language :: Python :: 2', 50 | 'Programming Language :: Python :: 2.7', 51 | 'Programming Language :: Python :: 3', 52 | 'Programming Language :: Python :: 3.4', 53 | 'Programming Language :: Python :: Implementation :: CPython'], 54 | extras_require={'test': tests_require}, 55 | cmdclass={'test': PyTest}) 56 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = {py27,py34} 3 | 4 | [testenv] 5 | deps = 6 | -rrequirements.txt 7 | commands = 8 | pip install -e . 9 | py.test --cov ad_notifier {posargs} 10 | --------------------------------------------------------------------------------