├── hn ├── __init__.py ├── version.py ├── hn.conf ├── hnctl.py ├── compat.py ├── defaults.py ├── endpoints.py ├── reltime.py ├── hnd.py ├── commands.py └── workers.py ├── requirements.txt ├── .gitignore ├── test ├── test_commands.py └── test_workers.py ├── .travis.yml ├── Makefile ├── LICENSE ├── setup.py └── README.md /hn/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | oi 2 | requests -------------------------------------------------------------------------------- /hn/version.py: -------------------------------------------------------------------------------- 1 | VERSION = '0.3.3' 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | 3 | *.py[co] 4 | *.egg-info 5 | 6 | /build 7 | /dist 8 | -------------------------------------------------------------------------------- /hn/hn.conf: -------------------------------------------------------------------------------- 1 | # Configuration example for the daemon part 2 | 3 | [settings] 4 | interval = 200 5 | 6 | [watch.worker] 7 | watch = true 8 | regexes = elixir, erlang, python, rust, golang 9 | 10 | [notify.worker] 11 | notify = true 12 | -------------------------------------------------------------------------------- /hn/hnctl.py: -------------------------------------------------------------------------------- 1 | # Command line interface to the program 2 | 3 | import oi 4 | 5 | 6 | def main(): 7 | ctl = oi.CtlProgram('ctl program', 'ipc:///tmp/oi-qixdlkfuep.sock') 8 | ctl.run() 9 | 10 | if __name__ == '__main__': 11 | main() 12 | -------------------------------------------------------------------------------- /hn/compat.py: -------------------------------------------------------------------------------- 1 | # Import certain modules based on python version 2 | # so we're compatible with multiple versions of python 3 | 4 | try: 5 | from urlparse import urlparse 6 | except ImportError: 7 | from urllib.parse import urlparse 8 | 9 | try: 10 | from Queue import Queue 11 | except ImportError: 12 | from queue import Queue 13 | -------------------------------------------------------------------------------- /hn/defaults.py: -------------------------------------------------------------------------------- 1 | # Default config options when no config file 2 | # is provided to the daemon 3 | 4 | DEFAULTS = { 5 | 'settings': { 6 | 'interval': 60*5 7 | }, 8 | 'watch.worker': { 9 | 'watch': False, 10 | 'regexex': '' 11 | }, 12 | 'notify.worker': { 13 | 'notify': False 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /test/test_commands.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import oi 3 | 4 | from hn import commands 5 | 6 | 7 | class TestCommands(unittest.TestCase): 8 | 9 | def setUp(self): 10 | self.p = oi.Program('test program', None) 11 | self.c = commands.Commands(self.p) 12 | 13 | def test_init(self): 14 | self.assertIsNotNone(self.c) 15 | -------------------------------------------------------------------------------- /test/test_workers.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import oi 3 | 4 | from hn import workers 5 | 6 | 7 | class TestWorkers(unittest.TestCase): 8 | 9 | def test_init(self): 10 | p = oi.Program('test program', None) 11 | a = workers.HNWorker(p) 12 | b = workers.WatchWorker(p) 13 | c = workers.NotifyWorker(p) 14 | self.assertTrue(None not in [a, b, c]) 15 | -------------------------------------------------------------------------------- /hn/endpoints.py: -------------------------------------------------------------------------------- 1 | # These are the endpoints for fetching HN data 2 | 3 | NEW = 'https://hacker-news.firebaseio.com/v0/newstories.json' 4 | TOP = 'https://hacker-news.firebaseio.com/v0/topstories.json' 5 | STORY = 'https://hacker-news.firebaseio.com/v0/item/{}.json' 6 | USER = 'https://hacker-news.firebaseio.com/v0/user/{}.json' 7 | ASK = 'https://hacker-news.firebaseio.com/v0/askstories.json' 8 | JOBS = 'https://hacker-news.firebaseio.com/v0/jobstories.json' 9 | SHOW = 'https://hacker-news.firebaseio.com/v0/showstories.json' 10 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - "2.7" 4 | - "3.2" 5 | - "3.3" 6 | - "3.4" 7 | - "nightly" 8 | 9 | git: 10 | submodules: false 11 | 12 | # install libnanomsg 13 | install: 14 | - git clone --quiet --depth=100 "https://github.com/nanomsg/nanomsg.git" ~/builds/nanomsg 15 | && pushd ~/builds/nanomsg 16 | && ./autogen.sh 17 | && ./configure 18 | && make 19 | && sudo make install 20 | && popd; 21 | 22 | before_script: make install 23 | script: LD_LIBRARY_PATH=/lib:/usr/lib:/usr/local/lib make test -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: help test 2 | 3 | help: 4 | @echo 5 | @echo "USAGE: make [target]" 6 | @echo 7 | @echo "TARGETS:" 8 | @echo 9 | @echo " install - install python package" 10 | @echo " clean - cleanup" 11 | @echo " test - run tests" 12 | @echo " distribute - upload to PyPI" 13 | @echo 14 | 15 | install: 16 | @python setup.py install 17 | 18 | test: 19 | @nosetests test 20 | 21 | clean: 22 | @rm -rf build dist *.egg-info 23 | 24 | distribute: 25 | @python setup.py register -r pypi && python setup.py sdist upload -r pypi 26 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Tony Walker 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | try: 4 | import setuptools 5 | from setuptools import setup 6 | except ImportError: 7 | setuptools = None 8 | from distutils.core import setup 9 | 10 | 11 | readme_file = 'README.md' 12 | try: 13 | import pypandoc 14 | long_description = pypandoc.convert(readme_file, 'rst') 15 | except (ImportError, OSError) as e: 16 | print('No pypandoc or pandoc: %s' % (e,)) 17 | with open(readme_file) as fh: 18 | long_description = fh.read() 19 | 20 | with open('./hn/version.py') as fh: 21 | for line in fh: 22 | if line.startswith('VERSION'): 23 | version = line.split('=')[1].strip().strip("'") 24 | 25 | setup( 26 | name='hn', 27 | version=version, 28 | packages=['hn'], 29 | author='Tony Walkr', 30 | author_email='walkr.walkr@gmail.com', 31 | url='https://github.com/walkr/hn', 32 | license='MIT', 33 | description='', 34 | long_description=long_description, 35 | install_requires=[ 36 | 'oi', 37 | 'nose', 38 | 'requests', 39 | ], 40 | classifiers=[ 41 | 'License :: OSI Approved :: MIT License', 42 | 'Programming Language :: Python', 43 | ], 44 | 45 | entry_points={ 46 | 'console_scripts': [ 47 | 'hnd = hn.hnd:main', 48 | 'hnctl = hn.hnctl:main', 49 | ], 50 | }, 51 | 52 | ) 53 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | hn 2 | ========= 3 | read HN from the command line 4 | 5 | [![Build Status](https://travis-ci.org/walkr/hn.svg)](https://travis-ci.org/walkr/hn) 6 | 7 | ![screenshot](http://i.imgur.com/xxWGfKu.png) 8 | 9 | **Features** 10 | 11 | * Modular, "daemon" + ctl 12 | * Daemon can watch for regexes in stories and trigger notifications (currently broken on OSX) or do any other type of work 13 | 14 | ### Install 15 | 16 | ```shell 17 | $ make install 18 | ``` 19 | 20 | ### Usage 21 | 22 | 1. Start the daemon 23 | 2. Use the command line interface 24 | 25 | ```shell 26 | # Start the daemon 27 | $ hnd 28 | 29 | # OR start the daemon with a config file (see example config file) 30 | $ hnd --config hn/hn.conf 31 | 32 | # Use the command line interface 33 | $ hnctl 34 | ctl > 35 | ctl > top # show top stories 36 | ctl > new # show new stories 37 | ctl > ask # show ask stories 38 | ctl > jobs # show jobs stories 39 | ctl > show # show "show hn" stories 40 | 41 | ctl > ping # ping HN site 42 | ctl > open 1 # open story <1> in browser 43 | ctl > user pg # show user 44 | ctl > help # show help 45 | ctl > help user # show help for command 46 | ctl > quit # quit 47 | 48 | # You can also invoke a command directly w/o the loop 49 | $ hnctl top 50 | $ hnctl open 1 51 | ``` 52 | 53 | MIT Licensed -------------------------------------------------------------------------------- /hn/reltime.py: -------------------------------------------------------------------------------- 1 | # A module for transforming timestamps to relative time 2 | 3 | import time 4 | from collections import OrderedDict 5 | 6 | 7 | durations = OrderedDict([ 8 | ('second', 1.0), 9 | ('minute', 60.0), 10 | ('hour', 3600.0), 11 | ('day', 3600.0 * 24), 12 | ('week', 3600.0 * 24 * 7), 13 | ('month', 3600.0 * 24 * 30), 14 | ('year', 3600.0 * 24 * 356), 15 | ]) 16 | 17 | 18 | def _since_now(timestamp): 19 | now = int(time.time()) 20 | diff = now - timestamp 21 | chosen_unit = None 22 | for unit, duration in reversed(list(durations.items())): 23 | if diff < duration: 24 | continue 25 | else: 26 | chosen_unit = unit 27 | break 28 | value = round(diff/duration, 2) 29 | return value, chosen_unit 30 | 31 | 32 | def since_now(timestamp, roundoff=True): 33 | """ Return the diff from since timestamp to now as relative time. 34 | `roundoff=True` round the numerical value to nearest integer. also return 35 | return second in a approximate format (a few seconds ago) """ 36 | 37 | value, unit = _since_now(timestamp) 38 | 39 | if roundoff: 40 | value = int(round(value)) 41 | if unit in ('second', 'seconds'): 42 | return 'a few seconds' 43 | elif value > 1: 44 | unit += 's' 45 | else: 46 | if value > 1.0: 47 | unit += 's' 48 | 49 | return '{} {}'.format(value, unit) 50 | -------------------------------------------------------------------------------- /hn/hnd.py: -------------------------------------------------------------------------------- 1 | # Daemon program fetching HN stories 2 | 3 | import os 4 | import oi 5 | 6 | from .commands import Commands 7 | from .workers import HNWorker 8 | from .workers import WatchWorker 9 | from .workers import NotifyWorker 10 | 11 | 12 | def notify_linux(story): 13 | """ Show notification on linux """ 14 | pass 15 | 16 | 17 | def notify_osx(story): 18 | """ Show notification on OS X """ 19 | cmd = u'terminal-notifier -title "New HN Story" -message "{}" -open {}' 20 | cmd = cmd.format(story['title'], story['url']) 21 | code = os.system(cmd) 22 | assert code == 0 23 | 24 | 25 | def main(): 26 | program = oi.Program('my program', 'ipc:///tmp/oi-qixdlkfuep.sock') 27 | 28 | hn_worker = HNWorker(program) 29 | watch_worker = WatchWorker(program) 30 | 31 | # Show notifications when new stories are found 32 | notify_worker = NotifyWorker(program) 33 | notify_worker.do(lambda story: notify_osx(story)) 34 | 35 | # Add workers to our program 36 | program.workers.append(hn_worker) 37 | program.workers.append(watch_worker) 38 | program.workers.append(notify_worker) 39 | 40 | # Register commands on the program 41 | c = Commands(program) 42 | program.add_command('top', lambda: c.which('top'), 'show top stories') 43 | program.add_command('new', lambda: c.which('new'), 'show new stories') 44 | program.add_command('ask', lambda: c.which('ask'), 'show ask stories') 45 | program.add_command('jobs', lambda: c.which('jobs'), 'show jobs stories') 46 | program.add_command('show', lambda: c.which('show'), 'show(show) stories') 47 | 48 | program.add_command('ping', c.ping, 'ping HN') 49 | program.add_command('user', c.user, 'show user profile') 50 | program.add_command('open', c.open, 'show in browser') 51 | program.run() 52 | 53 | if __name__ == '__main__': 54 | main() 55 | -------------------------------------------------------------------------------- /hn/commands.py: -------------------------------------------------------------------------------- 1 | # This modules implents the logic for the commands 2 | # supported by the daemon program 3 | 4 | import requests 5 | import webbrowser 6 | 7 | from . import endpoints 8 | 9 | 10 | class Commands(object): 11 | """ The commands supported by the daemon """ 12 | 13 | def __init__(self, program): 14 | self.program = program 15 | 16 | def _render(self, index, storyid=None): 17 | """ Render a story """ 18 | 19 | layout = u""" 20 | {index:2n}. {title} - ({hostname}) 21 | {score} points by {by} {time} ago | {descendants} comments 22 | """ 23 | data = self.program.state.stories[storyid] 24 | return layout.format(index=index, **data) 25 | 26 | # ---------------------------------------------- 27 | 28 | def which(self, section): 29 | """ Render stories from `section` (top, new, ask, show, etc) """ 30 | 31 | self.program.state.last_viewed = section 32 | ids = getattr(self.program.state, section) 33 | return ''.join([ 34 | self._render(i+1, s) for i, s in enumerate(ids) 35 | ]) 36 | 37 | def user(self, username='None'): 38 | """ Render a user's profile """ 39 | 40 | layout = u""" 41 | user: {id} 42 | created: {created} 43 | karma: {karma} 44 | about: {about} 45 | """ 46 | userdata = requests.get(endpoints.USER.format(username)).json() 47 | return layout.format(**userdata) if userdata else 'user not found' 48 | 49 | def ping(self): 50 | """ Check HN site status """ 51 | 52 | res = requests.get('https://news.ycombinator.com/news') 53 | return res.status_code 54 | 55 | def open(self, index): 56 | """ Open in browser a story with `index` 57 | from the latest viewed section """ 58 | 59 | index = int(index.strip()) 60 | index -= 1 61 | section = self.program.state.last_viewed 62 | storyid = getattr(self.program.state, section)[index] 63 | data = self.program.state.stories[storyid] 64 | webbrowser.open(data['url']) 65 | -------------------------------------------------------------------------------- /hn/workers.py: -------------------------------------------------------------------------------- 1 | # Workers are threads which are performing the heavy work 2 | 3 | import oi 4 | import re 5 | import time 6 | import functools 7 | import requests 8 | import logging 9 | 10 | from . import compat 11 | from . import reltime 12 | from . import endpoints 13 | from . import defaults 14 | 15 | logging.getLogger("requests").setLevel(logging.WARNING) 16 | 17 | 18 | class BaseWorker(oi.worker.Worker): 19 | """ Subclass this worker """ 20 | 21 | def __init__(self, program, **kwargs): 22 | super(BaseWorker, self).__init__(**kwargs) 23 | self.program = program 24 | 25 | 26 | class HNWorker(BaseWorker): 27 | """ A worker to check for new stories on HN periodically 28 | and store them in the program state """ 29 | 30 | def __init__(self, program, **kwargs): 31 | super(HNWorker, self).__init__(program, **kwargs) 32 | 33 | # Story indices per category 34 | self.program.state.top = [] 35 | self.program.state.new = [] 36 | self.program.state.ask = [] 37 | self.program.state.jobs = [] 38 | self.program.state.show = [] 39 | self.program.state.notified = [] 40 | 41 | # Stories collection 42 | self.program.state.stories = {} 43 | 44 | def put_stories(self, kind, ids, limit): 45 | 46 | def enhance(story, kind): 47 | story['via'] = kind 48 | story['hostname'] = compat.urlparse(story.get('url', '')).hostname 49 | story['time'] = reltime.since_now(int(story['time'])) 50 | story['descendants'] = story.get('descendants', 0) 51 | story.pop('kids', None) 52 | return story 53 | 54 | # Clear old stories so that the story collection will not grow forever 55 | self.program.state.stories = { 56 | key: story for key, story in self.program.state.stories.items() 57 | if story['via'] != kind 58 | } 59 | 60 | # Fetch each story in the id list and add it to the collection 61 | for i in ids[:limit]: 62 | story = requests.get(endpoints.STORY.format(i)).json() 63 | story = enhance(story, kind) 64 | self.program.state.stories[i] = story 65 | 66 | def run(self): 67 | """ Get data from endpoints, store it, then wait. Repeat. """ 68 | 69 | urls = { 70 | 'top': endpoints.TOP, 71 | 'new': endpoints.NEW, 72 | 'ask': endpoints.ASK, 73 | 'jobs': endpoints.JOBS, 74 | 'show': endpoints.SHOW, 75 | } 76 | 77 | while True: 78 | # Get stories for each category and store them 79 | limit = 15 80 | for name, url in urls.items(): 81 | logging.debug('Getting stories for section {}'.format(name)) 82 | ids = requests.get(url).json() 83 | setattr(self.program.state, name, ids[:limit]) 84 | self.put_stories(name, ids, limit) 85 | 86 | # Sleep for a little bit 87 | try: 88 | interval = int( 89 | self.program.config.getint('settings', 'interval')) 90 | except: 91 | interval = defaults.DEFAULTS['settings']['interval'] 92 | 93 | logging.debug('HNWorker will sleep for {}s'.format(interval)) 94 | time.sleep(interval) 95 | 96 | 97 | class WatchWorker(BaseWorker): 98 | """ Watch for certain patterns in the data then put those stories 99 | in the watch_queue so other threads can to do something with them """ 100 | 101 | class Watch(object): 102 | """ An object to keep track of new and seen stories """ 103 | NOTIFIED_LIMIT = 1000 104 | 105 | def __init__(self): 106 | self.notified = [] # stories for which notifis were trigere 107 | self.new_queue = compat.Queue() 108 | 109 | def get(self): 110 | """ Get a story from the queue """ 111 | return self.new_queue.get() 112 | 113 | def put(self, story): 114 | """ Add a story to the queue only if a notification has not 115 | been sent for it """ 116 | 117 | if story['id'] not in self.notified: 118 | return self.new_queue.put(story) 119 | 120 | def mark_notified(self, story): 121 | """ Mark that a notifcation was sent for this story """ 122 | 123 | if story['id'] not in self.notified: 124 | self.notified.append(story['id']) 125 | 126 | # Don't overgrow list 127 | if len(self.notified) > self.NOTIFIED_LIMIT: 128 | self.notified = self.notified[-self.NOTIFIED_LIMIT:] 129 | 130 | def was_notified(self, story): 131 | """ Was a notifcation already sent for this story """ 132 | return story['id'] in self.notified 133 | 134 | # -------- 135 | 136 | def __init__(self, program, **kwargs): 137 | super(WatchWorker, self).__init__(program, **kwargs) 138 | self.program.state.watch = self.Watch() 139 | 140 | def run(self): 141 | 142 | # Dont enter loop if watch is false 143 | try: 144 | watch = self.program.config.getboolean('watch.worker', 'watch') 145 | except: 146 | watch = defaults.DEFAULTS['watch.worker']['watch'] 147 | 148 | if not watch: 149 | logging.debug('Nothing to watch. Exit thread.') 150 | return 151 | 152 | while True: 153 | logging.debug('WatchWorker will look for patterns') 154 | # Trim notifications 155 | if len(self.program.state.notified) > 300: 156 | self.program.state.notified = self.program.state.notified[-300:] 157 | 158 | # Read patterns to look for in the story titles 159 | try: 160 | patterns = self.program.config.get('watch.worker', 'regexes') 161 | except: 162 | patterns = defaults.DEFAULTS['watch.worker']['regexes'] 163 | 164 | patterns = [p.strip() for p in patterns.split(',')] 165 | patterns = [p for p in patterns if p] 166 | logging.debug('Watch patterns: {}'.format(patterns)) 167 | 168 | # When a pattern is found, add the story to the watch_new_queue 169 | for pat in patterns: 170 | for story in self.program.state.stories.values(): 171 | 172 | if self.program.state.watch.was_notified(story): 173 | continue 174 | 175 | if re.search(pat, story['title'], flags=re.I): 176 | logging.debug('Found watch pattern {}'.format(pat)) 177 | self.program.state.watch.put(story) 178 | 179 | # Wait a little 180 | interval = 5 181 | logging.debug('WatchWorker will sleep for {}s'.format(interval)) 182 | time.sleep(interval) 183 | 184 | 185 | class NotifyWorker(BaseWorker): 186 | """ Show notifications """ 187 | 188 | def __init__(self, program, **kwargs): 189 | super(NotifyWorker, self).__init__(program, **kwargs) 190 | self.registered = [] 191 | 192 | def do(self, function): 193 | """ Do something when data comes in """ 194 | 195 | def make_new_fun(fun): 196 | """ Make a new function which catches all errors """ 197 | @functools.wraps(fun) 198 | def wrapper(*args, **kwargs): 199 | try: 200 | return fun(*args, **kwargs) 201 | except Exception as e: 202 | logging.debug('NotifyWorker.do error: {}'.format(e)) 203 | return wrapper 204 | 205 | self.registered.append(make_new_fun(function)) 206 | 207 | def run(self): 208 | """ Take items from the watch queue then process them """ 209 | 210 | try: 211 | notify = self.program.config.getboolean('notify.worker', 'notify') 212 | except: 213 | notify = defaults.DEFAULTS['notify.worker']['notify'] 214 | 215 | if not notify: 216 | logging.debug('No notify. Exit thread.') 217 | return 218 | 219 | while True: 220 | logging.debug('Getting a new story from queue') 221 | 222 | # Take new stories, mark them as seen and do some work 223 | story = self.program.state.watch.get() 224 | self.program.state.watch.mark_notified(story) 225 | 226 | logging.debug('Do notifications for story {}'.format(story['id'])) 227 | 228 | for function in self.registered: 229 | function(story) 230 | 231 | # Wait a little 232 | interval = 5 233 | logging.debug('NotifyWorker will sleep for {}s'.format(interval)) 234 | time.sleep(interval) 235 | --------------------------------------------------------------------------------