├── .gitignore ├── feedo ├── __init__.py ├── formatter.py └── feedo.py ├── MANIFEST.in ├── requirements.txt ├── setup.py └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.egg-info/ 3 | dist/ 4 | -------------------------------------------------------------------------------- /feedo/__init__.py: -------------------------------------------------------------------------------- 1 | from feedo import main 2 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include *.txt 2 | include README.md 3 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | feedparser==5.2.1 2 | python-slugify==1.2.4 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from setuptools import setup 4 | 5 | setup(name="feedo", 6 | version="1.1.1", 7 | description="Read, format and output an RSS stream.", 8 | url='https://github.com/kamikat/feedo', 9 | author="Kamikat", 10 | author_email="kamikat@banana.moe", 11 | install_requires=['feedparser', 'python-slugify'], 12 | packages=["feedo"], 13 | entry_points={ 14 | 'console_scripts': [ 'feedo = feedo:main' ] }, 15 | license="MIT") 16 | -------------------------------------------------------------------------------- /feedo/formatter.py: -------------------------------------------------------------------------------- 1 | import string 2 | import hashlib 3 | 4 | def sha1_hexdigest(value): 5 | return hashlib.sha1(value).hexdigest() 6 | 7 | def sha256_hexdigest(value): 8 | return hashlib.sha256(value).hexdigest() 9 | 10 | def escape_file_name(value): 11 | return value.replace(u'/', u'\u2215') 12 | 13 | from slugify import slugify 14 | 15 | class Formatter(string.Formatter): 16 | 17 | def __init__(self): 18 | self.conversions = { 19 | 'x': sha1_hexdigest, 20 | 'X': sha256_hexdigest, 21 | 'f': escape_file_name, 22 | 'g': slugify 23 | } 24 | pass 25 | 26 | def convert_field(self, value, conversion): 27 | if conversion in self.conversions: 28 | return self.conversions[conversion](value) 29 | return super(Formatter, self).convert_field(value, conversion) 30 | 31 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # feedo 2 | 3 | [![PyPI version](https://badge.fury.io/py/feedo.svg)](https://badge.fury.io/py/feedo) 4 | 5 | Read, format and output an RSS stream. 6 | 7 | feedo is inspired by [rsstail](https://github.com/flok99/rsstail) and it's python copy [rsstail.py](https://github.com/gvalkov/rsstail.py) 8 | 9 | ## Usage 10 | 11 | To fetch an RSS stream: 12 | 13 | ```sh 14 | feedo http://lorem-rss.herokuapp.com/feed --format '{title}' 15 | ``` 16 | 17 | Add `--follow` option to watch updates on feed. 18 | 19 | Supported format arguments: 20 | 21 | - `{id}` 22 | - `{link}` 23 | - `{title}` 24 | - `{author}` 25 | - `{content}` 26 | - `{enclosures[i].href}` 27 | - `{enclosures[i].length}` 28 | - `{enclosures[i].type}` 29 | - `{published}` 30 | - `{created}` 31 | - `{updated}` 32 | - `{expired}` 33 | 34 | (See [feedparser Documentation](https://pythonhosted.org/feedparser/search.html?q=entry) for more available format arguments) 35 | 36 | Values can be styled in python 3 [string formatting](https://docs.python.org/3/library/string.html#string-formatting) syntax. For example, to format `published` value in RFC 3339: 37 | 38 | ```sh 39 | feedo http://lorem-rss.herokuapp.com/feed --format '{title:.11} ({link}) -- {published:%Y-%m-%dT%H:%MZ%z}' 40 | ``` 41 | 42 | (See [strftime (3)](https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior) for date format arguments) 43 | 44 | ### Value conversion 45 | 46 | feedo extends [value conversion](https://pyformat.info/#conversion_flags) syntax and support following extra conversion flags: 47 | 48 | - `!x` converts value to a SHA-1 checksum string 49 | - `!X` converts value to a SHA-256 checksum string 50 | - `!f` escapes value to make it safe for a file name 51 | - `!g` slugifys the value with [python-slugify](https://github.com/un33k/python-slugify) 52 | 53 | For example, to get the sha1 hash of a `link` value and get first 12 characters: 54 | 55 | ```sh 56 | feedo http://lorem-rss.herokuapp.com/feed --format '{link} => {link!x} => {link!x:.12}' 57 | ``` 58 | 59 | ## Example 60 | 61 | feedo can be used in implementing some automatic job: 62 | 63 | ```sh 64 | feedo "http://bt.byr.cn/torrentrss.php?rows=10&linktype=dl&passkey=XXXXXXXXXX" \ 65 | --format '[ ! -f /media/.abt/{id} ] && abt add --uri "{enclosures[0].href}" --dir="/media/Downloads/{title!f}" && touch /media/.abt/{id}' \ 66 | --follow | sh 67 | ``` 68 | 69 | This snippet watches an RSS stream of a RSS subscription and add new tasks to aria2 using [abt](https://github.com/kamikat/abt). 70 | 71 | **WARNING** take a trust-worthy source before pipe anything generated into a shell!!! 72 | 73 | ## License 74 | 75 | (The MIT License) 76 | 77 | -------------------------------------------------------------------------------- /feedo/feedo.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | 3 | import argparse 4 | import feedparser 5 | import sys 6 | 7 | from time import sleep 8 | from formatter import Formatter 9 | from datetime import datetime 10 | 11 | def main(): 12 | stdout, stderr = get_encoding_safe_stdio() 13 | 14 | parser = argparse.ArgumentParser(description="Read, format and output an RSS stream.") 15 | parser.add_argument('uri', action='store', help="load feed from uri") 16 | parser.add_argument('-o', '--format', action='store', required=True, help="python 3.x styled format string (see https://pyformat.info/)") 17 | parser.add_argument('-f', '--follow', action='store_true', help="follow feed stream updates") 18 | parser.add_argument('-i', '--interval', action='store', type=float, default=None, help="time between each fetch for updates (in seconds)") 19 | args = parser.parse_args() 20 | 21 | last_item_link = None 22 | formatter = Formatter() 23 | format_string = args.format.decode('utf-8') 24 | 25 | while True: 26 | try: 27 | d = feedparser.parse(args.uri) 28 | except Exception: 29 | print >>stderr, "Failed to parse '{}'".format(args.uri) 30 | continue 31 | interval = args.interval if args.interval else float(d.feed.ttl) if hasattr(d.feed, 'ttl') else 60 32 | newlines = [] 33 | for entry in d.entries: 34 | if entry.link == last_item_link: 35 | break 36 | item = dict(entry) 37 | item['enclosures'] = entry.enclosures 38 | item['published'] = datetime(*entry.published_parsed[:6]) if hasattr(entry, 'published_parsed') else None 39 | item['created'] = datetime(*entry.created_parsed[:6]) if hasattr(entry, 'created_parsed') else None 40 | item['updated'] = datetime(*entry.updated_parsed[:6]) if hasattr(entry, 'updated_parsed') else None 41 | item['expired'] = datetime(*entry.expired_parsed[:6]) if hasattr(entry, 'expired_parsed') else None 42 | newlines.append(formatter.format(format_string, **item)) 43 | if len(d.entries): 44 | last_item_link = d.entries[0].link 45 | for line in reversed(newlines): 46 | print >>stdout, line 47 | stdout.flush() 48 | if args.follow: 49 | sleep(interval) 50 | continue 51 | break 52 | 53 | def get_encoding_safe_stdio(encoding='utf-8'): 54 | import codecs 55 | stdout = sys.stdout if sys.stdout.encoding else codecs.getwriter(encoding)(sys.stdout) 56 | stderr = sys.stderr if sys.stderr.encoding else codecs.getwriter(encoding)(sys.stderr) 57 | return stdout, stderr 58 | 59 | --------------------------------------------------------------------------------