37 |

45 | 46 | 47 | 48 | %s 49 | 50 | 51 | 52 | 53 |

%s

54 | """ % (feed_title, feed_title) 55 | 56 | feed = feedparser.parse(RSS_URL) 57 | 58 | # feedparser has no error return! One way is to check len(feed.feed). 59 | if len(feed.feed) == 0: 60 | print("Couldn't fetch RSS from", RSS_URL, file=sys.stderr) 61 | return 62 | 63 | articleno = 0 64 | for item in feed.entries: 65 | if 'links' in item: 66 | href = [str(i['href']) for i in item.links 67 | if 'rel' in i and 'href' in i 68 | and i['rel'] == 'alternate'] 69 | else: 70 | href = [] 71 | 72 | # Get an item ID 73 | if 'id' in item: 74 | item_id = str(item.id) 75 | if verbose: 76 | print("\nID %s" % item_id, file=sys.stderr) 77 | elif href: 78 | item_id = str(href[0]) 79 | if verbose: 80 | print("Using URL '%s' for ID" % item_id, file=sys.stderr) 81 | else: 82 | if verbose: 83 | print("Item in %s had no ID or URL." % str(href[0]), 84 | file=sys.stderr) 85 | continue # or return? 86 | 87 | # Get the published date. 88 | # item.pubDate is a unicode string, supposed to be in format 89 | # Thu, 11 Aug 2016 14:46:50 GMT (or +0000) 90 | # email.utils.parsedate returns a tuple. 91 | # Pass it to time.mktime() to get seconds since epoch, 92 | # XXX feedparser now has published_parsed which is 93 | # a time.struct_time. Can we count on that and not 94 | # have to do parsing here? 95 | try: 96 | pub_date = time.mktime(parsedate(item.published)) 97 | except: 98 | pub_date = None 99 | 100 | if verbose: 101 | print("pub_date", pub_date) 102 | 103 | if 'author' in item: 104 | author = str(item.author) 105 | else: 106 | author = None 107 | if verbose: 108 | print("Author:", author) 109 | 110 | item_link = str(item.link) 111 | if verbose: 112 | print("Link:", item_link) 113 | print(item.summary) 114 | 115 | # Now fetch the item to a local file, if there's a driver 116 | if not sbrowser or not savedir: 117 | return 118 | 119 | # On the first article, make the directory if necessary 120 | try: 121 | os.makedirs(savedir) 122 | except FileExistsError: 123 | pass 124 | 125 | fullhtml = fetch_article_selenium(item_link, sbrowser) 126 | if not fullhtml: 127 | print("Couldn't fetch", item_link) 128 | return 129 | continue 130 | 131 | soup = BeautifulSoup(fullhtml, "lxml") 132 | 133 | # Try several possible containers 134 | article = soup.find("section", {"name": "articleBody"}) 135 | 136 | if not article: 137 | print("No articleBody") 138 | article = soup.find(class_="live-blog-post") 139 | else: 140 | print("yay, found articleBody") 141 | 142 | if not article: 143 | print("No live-blog-post either") 144 | article = soup 145 | fullfile = os.path.join(savedir, "%d-full.html" % articleno) 146 | print("Couldn't find any containers: saving %s" % fullfile) 147 | with open(fullfile, "w") as fp: 148 | fp.write(fullhtml) 149 | 150 | # Remove ads, story-ad-*-wrapper 151 | for ad in article.find_all(class_=adpat): 152 | ad.decompose() 153 | 154 | # Remove images, for now, until this is folded into the 155 | # regular feedme/pageparser code to fetch images locally. 156 | for img in article.find_all("img"): 157 | img.decompose() 158 | # NYT has huge SVG images that use the "svg" tag 159 | for img in article.find_all("svg"): 160 | img.decompose() 161 | 162 | # Done with processing, ready to write it! 163 | htmlfile = os.path.join(savedir, "%d.html" % articleno) 164 | with open(htmlfile, "wb") as outfp: 165 | outfp.write(article.prettify(encoding='utf-8')) 166 | print("Wrote", htmlfile) 167 | 168 | try: 169 | item_title = str(item.title) 170 | except: 171 | item_title = "(no title)" 172 | 173 | # Add this article to the index 174 | index += """

%s 175 |

176 | %s 177 |

188 | 189 |

(Downloaded by FeedMe 1.0 NYT fetcher) 190 | """ 191 | 192 | with open(os.path.join(savedir, "index.html"), "w") as indexfp: 193 | indexfp.write(index) 194 | 195 | print("Wrote", articleno, "files plus index") 196 | 197 | 198 | def find_firefox_profile_dir(): 199 | """Return the first profile in ~/.mozilla/firefox/ 200 | that has "selenium" in its name. 201 | """ 202 | mozdir = os.path.expanduser("~/.mozilla/firefox/") 203 | for pdir in os.listdir(mozdir): 204 | if "selenium" in pdir: 205 | return os.path.join(mozdir, pdir) 206 | raise RuntimeError("Can't find a selenium profile in %s" % pdir) 207 | 208 | 209 | def fetch_article_selenium(article_url, sbrowser): 210 | """Fetch a list of articles taken from the RSS. 211 | Takes a list of urlsand a directory to store the result. 212 | Saves each article as savedir/basename(url). 213 | """ 214 | try: 215 | sbrowser.get(article_url) 216 | return sbrowser.page_source 217 | 218 | except Exception as e: 219 | print("*** Exception:", e) 220 | traceback.print_exc() 221 | return None 222 | 223 | """ 224 | # Try to get the article body. 225 | article = None 226 | for container in nytcontainers: 227 | try: 228 | article = sbrowser.find_element_by_name(container) 229 | break 230 | except selenium_exceptions.NoSuchElementException: 231 | pass 232 | 233 | if not article: 234 | article = sbrowser.page_source 235 | print(article_url, ": Writing full page source to /tmp/pagesource") 236 | with open("/tmp/pagesource", "w") as outfp: 237 | outfp.write(article) 238 | 239 | # or tag name, sbrowser.find_element_by_tag_name("section") 240 | # for matching multiple conditions, xpath might be useful: 241 | # email_input = sbrowser.find_element_by_xpath("//input[@name='email']") 242 | # However, selenium can't actually modify the content of anything, 243 | # so it's probably best to punt and do the rest in BeautifulSoup. 244 | 245 | return article.get_attribute('innerHTML') 246 | """ 247 | 248 | 249 | def init_selenium(): 250 | """Initialize selenium, returning the web driver object.""" 251 | 252 | foxprofiledir = find_firefox_profile_dir() 253 | 254 | # Deprecated, but no one seems to know the new way: 255 | options = Options() 256 | options.headless = True 257 | print("Creating headless browser...") 258 | sbrowser = webdriver.Firefox(firefox_profile=foxprofiledir, 259 | options=options) 260 | 261 | # Another way, also deprecated: 262 | # profile = webdriver.FirefoxProfile(profile_directory=foxprofiledir) 263 | # sbrowser = webdriver.firefox.webdriver.WebDriver(firefox_profile=profile) 264 | 265 | # sbrowser.get('https://nytimes.com/') 266 | # sbrowser.get('https://www.nytimes.com/2021/09/30/opinion/andrew-yang.html') 267 | 268 | return sbrowser 269 | 270 | 271 | if __name__ == '__main__': 272 | sbrowser = None 273 | sbrowser = init_selenium() 274 | read_rss(sbrowser, savedir=os.path.expanduser("~/feeds/nyt")) 275 | -------------------------------------------------------------------------------- /experiments/nytcookies.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import os 4 | import pathlib 5 | import requests 6 | from http.cookiejar import MozillaCookieJar 7 | 8 | ####################################### 9 | # https://stackoverflow.com/a/62112043 10 | 11 | def cookies_from_cookies_dot_txt(): 12 | cookiesFile = str(pathlib.Path(__file__).parent.absolute() / "cookies.txt") # Places "cookies.txt" next to the script file. 13 | cj = MozillaCookieJar(cookiesFile) 14 | if os.path.exists(cookiesFile): # Only attempt to load if the cookie file exists. 15 | cj.load(ignore_discard=True, ignore_expires=True) # Loads session cookies too (expirydate=0). 16 | 17 | s = requests.Session() 18 | s.headers = { 19 | "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36", 20 | "Accept-Language": "en-US,en" 21 | } 22 | s.cookies = cj # Tell Requests session to use the cookiejar. 23 | 24 | # DO STUFF HERE WHICH REQUIRES THE PERSISTENT COOKIES... 25 | s.get("https://www.somewebsite.com/") 26 | 27 | cj.save(ignore_discard=True, ignore_expires=True) # Saves session cookies too (expirydate=0). 28 | 29 | 30 | ####################################### 31 | # https://blog.mithis.net/archives/python/90-firefox3-cookies-in-python 32 | 33 | def sqlite2cookie_mithis(filename): 34 | from cStringIO import StringIO 35 | from pysqlite2 import dbapi2 as sqlite 36 | 37 | con = sqlite.connect(filename) 38 | 39 | cur = con.cursor() 40 | cur.execute("select host, path, isSecure, expiry, name, value from moz_cookies") 41 | 42 | ftstr = ["FALSE","TRUE"] 43 | 44 | s = StringIO() 45 | s.write("""\ 46 | # Netscape HTTP Cookie File 47 | # http://www.netscape.com/newsref/std/cookie_spec.html 48 | # This is a generated file! Do not edit. 49 | """) 50 | for item in cur.fetchall(): 51 | s.write("%s\t%s\t%s\t%s\t%s\t%s\t%s\n" % ( 52 | item[0], ftstr[item[0].startswith('.')], item[1], 53 | ftstr[item[2]], item[3], item[4], item[5])) 54 | 55 | s.seek(0) 56 | 57 | cookie_jar = cookielib.MozillaCookieJar() 58 | cookie_jar._really_load(s, '', True, True) 59 | return cookie_jar 60 | 61 | 62 | ####################################### 63 | # https://stackoverflow.com/a/33078599 64 | 65 | COOKIE_FILE = '/home/akkana/.mozilla/firefox/a6sezjo1.selenium/cookies.sqlite' 66 | 67 | def get_cookie_jar(filename): 68 | """ 69 | Protocol implementation for handling gsocmentors.com transactions 70 | Author: Noah Fontes nfontes AT cynigram DOT com 71 | License: MIT 72 | Original: http://blog.mithis.net/archives/python/90-firefox3-cookies-in-python 73 | 74 | Ported to Python 3 by Dotan Cohen 75 | """ 76 | 77 | from io import StringIO 78 | import http.cookiejar 79 | import sqlite3 80 | 81 | con = sqlite3.connect(filename) 82 | cur = con.cursor() 83 | cur.execute("SELECT host, path, isSecure, expiry, name, value FROM moz_cookies") 84 | 85 | ftstr = ["FALSE","TRUE"] 86 | 87 | s = StringIO() 88 | s.write("""\ 89 | # Netscape HTTP Cookie File 90 | # http://www.netscape.com/newsref/std/cookie_spec.html 91 | # This is a generated file! Do not edit. 92 | """) 93 | 94 | for item in cur.fetchall(): 95 | s.write("%s\t%s\t%s\t%s\t%s\t%s\t%s\n" % ( 96 | item[0], ftstr[item[0].startswith('.')], item[1], 97 | ftstr[item[2]], item[3], item[4], item[5])) 98 | 99 | s.seek(0) 100 | cookie_jar = http.cookiejar.MozillaCookieJar() 101 | cookie_jar._really_load(s, '', True, True) 102 | 103 | return cookie_jar 104 | 105 | 106 | cj = get_cookie_jar(COOKIE_FILE) 107 | # print(cj) 108 | 109 | url = 'https://www.nytimes.com/2021/11/06/us/dark-sky-parks-us.html' 110 | response = requests.get(url, cookies=cj) 111 | html = response.text 112 | print("Got", len(html), "bytes") 113 | outfile = '/tmp/nytfoo.html' 114 | with open(outfile, 'w') as fp: 115 | print(html, file=fp) 116 | print("Saved", outfile) 117 | 118 | -------------------------------------------------------------------------------- /experiments/pluralistic.conf: -------------------------------------------------------------------------------- 1 | [Pluralistic] 2 | url = https://pluralistic.net/feed/ 3 | levels = 1 4 | 5 | # This is ignored because feedme can't handle images in the RSS yet: 6 | # skip_images = false 7 | -------------------------------------------------------------------------------- /experiments/python-ninja.conf: -------------------------------------------------------------------------------- 1 | # Python Ninja 2 | [Python Ninja] 3 | url = https://pythonninja.xyz/feed 4 | skip_images = false 5 | # page_start = 6 | # page_end = 7 | -------------------------------------------------------------------------------- /experiments/risks.conf: -------------------------------------------------------------------------------- 1 | [Risks Digest] 2 | url = https://catless.ncl.ac.uk/risksrss2.xml 3 | -------------------------------------------------------------------------------- /experiments/zdnet.conf: -------------------------------------------------------------------------------- 1 | # Doesn't work: Wired does have hidden RSS feeds, 2 | # https://www.wired.com/about/rss-feeds/ 3 | # but they require subscription to get more than a few stories. 4 | 5 | [ZDNet] 6 | url = https://www.zdnet.com/news/rss.xml 7 | skip_images = False 8 | page_start =

10 | -------------------------------------------------------------------------------- /feedfetcher/README: -------------------------------------------------------------------------------- 1 | 2 | How to set up feedme so that it runs remotely on a web server, 3 | triggered from a phone, laptop or other web-enabled device: 4 | 5 | (Yes, these instructions are somewhat complex. But fortunately 6 | you should only need to do it once!) 7 | 8 | On the web server, create $SERVERDIR/feedme and $SERVERDIR/feeds 9 | ($SERVERDIR will probably be /var/www/ for simple setups). 10 | They must both be writable by the web server's user (probably www-data). 11 | Once pageparser.py has been created, $SERVERDIR/feedme probably 12 | no longer needs to be writable, but $SERVERDIR/feeds and 13 | /$SERVERDIR/feedme/.cache/feedme still will need to remain 14 | writable by www-data forever. 15 | 16 | Create $SERVERDIR/feedme/.config with an appropriate configuration 17 | (feedme.conf plus optional links to other .conf files). 18 | In feedme.conf, make sure that dir = points to a directory writable by 19 | the web server. If you were using something like ~/feeds for your own 20 | homedir, that may cause feedme not to start (and there will be no 21 | obvious error message in that case, since it will have no place to log 22 | the error). I suggest using $SERVERDIR/feeds where $SERVERDIR is the 23 | same path you'd use from a login shell, e.g. /var/www/feeds. 24 | 25 | Copy or link the binaries you need -- 26 | feedme, pageparser.py, and urlrss.cgi -- 27 | to $SERVERDIR/feedme/urlrss.cgi and make sure they're executable. 28 | (Your web server will also need to be able to execute CGI scripts 29 | in that directory. If not, mvoe urlrss.cgi to wherever CGI scripts 30 | need to live. You may also need to move feedme and pageparser.py, 31 | but I'm not sure since I allow CGIs anywhere.) 32 | 33 | Now you should be able to initiate a feed by visiting the URL 34 | pointing to urlrss.cgi, also specifying any "Extra URLs" you want 35 | fetched if you use them. Extra URLs must be URL encoded and connected 36 | by the string '%0a', for instance, 37 | urlrss.cgi?xtraurls=http%3A%2F%2Fblog.arduino.cc%2F2013%2F07%2F10%2Fsend-in-the-clones%2F%0ahttp%3A%2F%2Fread.bi%2F10Lbfh9%0ahttp%3A%2F%2Fwww.popsci.com%2Ftechnology%2Farticle%2F2013-07%2Fdrones-civil-war%0ahttp%3A%2F%2Fwww.thisamericanlife.org%2Fblog%2F2015%2F05%2Fcanvassers-study-in-episode-555-has-been-retracted 38 | If you don't need any extra URLs, it's safest to call 39 | urlrss.cgi?xtraurls=none since in the past I've sometimes seen bugs 40 | with Python's CGI module having trouble if the argument isn't present. 41 | 42 | If everything is set up correctly, urlrss.cgi should save off the xtraurls 43 | and then start feedme. You can then check on the progress of the feeding 44 | by periodically checking your feeds directory (the one you specified 45 | in feedme.conf with dir =). The last file to be written will be MANIFEST, 46 | so when you see that, wait a few seconds (to make sure the manifest is 47 | completely written) then fetch the MANIFEST, which will contain a list 48 | of filenames fetched in the feed. Those are the files you need to download. 49 | HTML files are first, image files last, so if you're on a bad connection 50 | and have trouble fetching all the images, you can kill the fetch at 51 | that point without losing any HTML content. 52 | 53 | 54 | TESTING/DEBUGGING: 55 | 56 | Testing feedme alone is easy: you can run it directly. 57 | Make a backup of .cache/feedme/feedme.dat (or, if you've already run 58 | feedme that day, make a backup of .cache/feedme/feedme.YYMMDDday.dat 59 | and copy that repeatedly to feedme.dat) or use -n (--nocache). 60 | 61 | But when you need to test the whole setup, including feedfetcher, 62 | testing can be more difficult. 63 | Start by saving the cache file, as noted in the previous paragraph. 64 | With an appropriate cache in place, you can start feedme by using 65 | curl or wget to call the special url from anywhere: 66 | wget "http://example.com/path/to/urlrss.cgi?xtraurls=none" 67 | 68 | So a typical debug cycle might be: 69 | 70 | On the server, as root or www-data or whoever: 71 | rm -rf /var/www/feeds/$(date +'%m-%d-%a') 72 | cp feedme.dat.sav /var/www/feedme/.cache/feedme.dat 73 | From anywhere: 74 | wget --content-disposition 'http://example.com/feedme/urlrss.cgi?xtraurls=none' 75 | or 76 | wget -O output.txt 'http://example.com/feedme/urlrss.cgi?xtraurls=none' 77 | 78 | and then, on the server, 79 | tail -f /var/www/feeds/LOG 80 | to see feedme's progress and check for error messages. 81 | If LOG never appears, check /var/www/feeds/log-urlrss 82 | which might explain why. 83 | 84 | If you don't have shell access to the server, load 85 | http://example.com/feeds/ 86 | in a browser and watch for log-urlrss, LOG and MANIFEST. 87 | 88 | If you're tailing LOG rather than watching for MANIFEST, 89 | feedme is finished when you see a line like: 90 | Renaming /var/www/feeds/LOG to /var/www/feeds/10-28-Sat/LOG 91 | 92 | 93 | -------------------------------------------------------------------------------- /feedfetcher/feedfetcher.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Initiate feedme on a remote web server; 4 | # wait for it to finish, then download the feeds to the local filesystem. 5 | # Can run either on a local Linux machine or on Android under SL4A. 6 | 7 | import urllib, urllib2 8 | 9 | import BeautifulSoup 10 | # http://www.crummy.com/software/BeautifulSoup/bs3/documentation.html 11 | import HTMLParser # Needed for the exceptions used by BeautifulSoup 12 | 13 | import sys, os 14 | import time 15 | 16 | # import traceback 17 | 18 | ############# CONFIGURATION ######################################## 19 | 20 | # Put your server base URL here, the dir that will contain 21 | # both feedme/ and feeds/ directories. It must end with a slash. 22 | serverurl = 'http://localhost/' 23 | 24 | # Where to download feeds if running locally. 25 | # This may include ~ for home directory. 26 | localdir = '~/feeds' 27 | 28 | # Where to download feeds if running on Android. 29 | # Should be an absolute path, probably starting with /mnd/sdcard. 30 | android_localdir = '/mnt/sdcard/external_sd/feeds/' 31 | 32 | ############# END CONFIGURATION #################################### 33 | 34 | errstr = '' 35 | 36 | # Are we on Android? Make it optional, for easier testing. 37 | try: 38 | import android 39 | print "Running on Android" 40 | is_android = True 41 | droid = android.Android() 42 | #droid.makeToast("Running on Android!") 43 | except ImportError: 44 | print "Not running on Android" 45 | is_android = False 46 | 47 | def perror(s): 48 | global errstr 49 | # makeToast is cool but takes way too long and might still not be seen. 50 | #if is_android: 51 | # droid.makeToast(s) 52 | print s 53 | errstr += '\n' + s 54 | 55 | def fetch_url_to(url, outfile): 56 | if os.path.exists(outfile): 57 | print os.path.basename(outfile), "already exists -- not re-fetching" 58 | return 59 | 60 | # If there's a named anchor appended, strip it from both url and filename. 61 | # Pro Publica in particular has a lot of these. 62 | if '#' in url: 63 | print "Stripping named anchor from url", url 64 | url = url[:url.find('#')] 65 | if '#' in outfile: 66 | print "Stripping named anchor from outfile", outfile 67 | outfile = outfile[:outfile.find('#')] 68 | 69 | # If there's a % in the url, e.g. if there's a filename with a space 70 | # like file%20name.jpg (Los Alamos Daily Post loves to do this), 71 | # apache on the server end will urldecode it. 72 | # Tried encoding the % to %25 to prevent that from happening, 73 | # and that prevents throwing error but the photos still don't download. 74 | # It's best to make sure feedme doesn't save any URLs like that 75 | # in the first place. 76 | url = url.replace('%', '%25') 77 | 78 | print "Fetching", url, "to", outfile 79 | 80 | # Read the URL. It may fail: not all referenced links 81 | # were successfully downloaded to the server. 82 | # But sometimes it fails for other reasons too, 83 | # so we need to distinguish a "not found" from other causes. 84 | try: 85 | infile = urllib2.urlopen(url) 86 | contents = infile.read() 87 | infile.close() 88 | except urllib2.HTTPError, e: 89 | perror("Couldn't fetch %s: HTTPError code %s" % (url, str(e.code))) 90 | return 91 | except urllib2.URLError, e: 92 | perror("Couldn't fetch %s: URLError args %s" % (url, str(e.args))) 93 | return 94 | except ValueError, e: 95 | perror("Couldn't fetch %s: ValueError, %s" % (url, str(e))) 96 | return 97 | 98 | # Copy to the output file 99 | outf = open(outfile, 'w') 100 | outf.write(contents) 101 | outf.close() 102 | 103 | if not url.lower().endswith('.html'): 104 | return 105 | 106 | # If html, go through the contents and recursively fetch 107 | # any local links. 108 | try: 109 | soup = BeautifulSoup.BeautifulSoup(contents) 110 | except HTMLParser.HTMLParseError, e: 111 | perror("Parse error on " + url + "! " + str(e)) 112 | print "First part of file was:", contents[:256] 113 | return 114 | 115 | def not_a_local_link(l): 116 | '''If a link doesn't have a schema, consists only of an 117 | absolute path or a named anchor like #, it's local. 118 | ''' 119 | if not link or ':' in link or link.startswith('#') or link[0] == '/' \ 120 | or link.startswith('../'): 121 | # I don't know why we see ../ links, but we do. 122 | return True 123 | return False 124 | 125 | dirurl = os.path.dirname(url) + '/' 126 | outdir = os.path.dirname(outfile) 127 | 128 | for tag in soup.findAll('a'): 129 | # BeautifulSoup doesn't support calls like 'href' in tag 130 | try: 131 | link = tag['href'] 132 | except KeyError: 133 | continue 134 | if not_a_local_link(link): 135 | continue 136 | 137 | # Lots of pages have recursive links to themselves. 138 | # Obviously no point in fetching those. 139 | if os.path.join(outdir, link) != outfile: 140 | fetch_url_to(dirurl + link, os.path.join(outdir, link)) 141 | 142 | for tag in soup.findAll('img'): 143 | # BeautifulSoup doesn't support calls like 'href' in tag 144 | #print "Checking image", tag 145 | try: 146 | link = tag['src'] 147 | except KeyError: 148 | continue 149 | if not_a_local_link(link): 150 | continue 151 | fetch_url_to(dirurl + link, os.path.join(outdir, link)) 152 | 153 | def fetch_feed_dir(dirurl, outdir): 154 | '''Fetch index.html inside the given url, plus everything it points to.''' 155 | 156 | if not dirurl.endswith('/'): 157 | dirurl += '/' 158 | 159 | print "Fetch_feed_dir", dirurl, outdir 160 | 161 | # Make sure the directory exists 162 | if not os.access(outdir, os.W_OK): 163 | try: 164 | os.makedirs(outdir, 0755) 165 | except Exception as e: 166 | perror("Couldn't make directory %s" % outdir) 167 | raise e 168 | 169 | index = fetch_url_to(dirurl + 'index.html', 170 | os.path.join(outdir, 'index.html')) 171 | 172 | def parse_directory_page(urldir): 173 | '''Parse a directory page from the server, returning a list of subdirs. 174 | Return None implies there was a problem reaching the urldir. 175 | Return of [] means the urldir was there, but contains no subdirs. 176 | ''' 177 | if not urldir.endswith('/'): 178 | urldir += '/' 179 | 180 | try: 181 | f = urllib2.urlopen(urldir) 182 | dirpage = f.read() 183 | # dirlines = dirpage.split('\n') 184 | f.close() 185 | except urllib2.HTTPError, e: 186 | # This always happens the first time, but if it happens 187 | # repeatedly then it indicates a problem. 188 | perror("HTTP error parsing directory page %s: code is %d" \ 189 | % (urldir, e.code)) 190 | return None 191 | 192 | # Parse the directory contents to get the list of feeds 193 | # It's in a table tag, where valid entries will look like: 194 | # [DIR]

BBC_News_Science/28-Jun-2013 18:42 - 195 | feeddirs = [] 196 | try: 197 | soup = BeautifulSoup.BeautifulSoup(dirpage) 198 | except HTMLParser.HTMLParseError, e: 199 | perror("Couldn't parse directory page " + urldir + str(e)) 200 | print "Parsed HTML began:", dirpage[:256] 201 | return feeddirs 202 | 203 | for tag in soup.findAll('a'): 204 | try: 205 | link = tag['href'] 206 | except KeyError: 207 | continue 208 | if link[0] != '?' and link[0] != '/': 209 | feeddirs.append(link) 210 | 211 | return feeddirs 212 | 213 | def fetch_feeds_dir_recursive(urldir, outdir): 214 | feeddirs = parse_directory_page(urldir) 215 | if feeddirs == None: 216 | err = "Couldn't find %s on server" % os.path.basename(urldir) 217 | perror(err) 218 | if is_android: 219 | droid.vibrate() 220 | droid.notify(err) 221 | return 222 | 223 | # now feeddirs[] should contain the subdirs we want to fetch. 224 | print "Will try to fetch feed dirs:", feeddirs 225 | print 226 | 227 | for subdir in feeddirs: 228 | # Don't fetch the log file 229 | if subdir == 'LOG': 230 | continue 231 | fetch_feed_dir(urldir + subdir, os.path.join(outdir, subdir)) 232 | 233 | def run_feed(serverurl, outdir): 234 | # Hit the CGI URL on the server to tell it to run feedme. 235 | # First build up the URL with any extra URLs we've collected: 236 | url = os.path.join(serverurl + 'feedme/urlrss.cgi?xtraurls=') 237 | savedpath = os.path.join(outdir, 'saved-urls') 238 | saved_urls = [] 239 | try: 240 | saved = open(savedpath) 241 | for line in saved: 242 | print line, 243 | saved_urls.append(urllib.quote_plus(line.strip())) 244 | saved.close() 245 | # Now rename the saved file so we won't get those urls again. 246 | bakpath = savedpath + '.bak' 247 | if os.path.exists(bakpath): 248 | os.unlink(bakpath) 249 | os.rename(savedpath, bakpath) 250 | except: 251 | print 'No saved urls' 252 | 253 | if saved_urls: 254 | #url += '?xtraurls=' + '%0a'.join(saved_urls) 255 | url += '%0a'.join(saved_urls) 256 | else: 257 | url += "none" 258 | 259 | print 'Requesting URL:', url 260 | 261 | # Now hit the URL. We don't actually care about what it returns, 262 | # though we do care if it throws an error. 263 | # This may throw various HTTP errors. 264 | infile = urllib2.urlopen(url) 265 | contents = infile.read() 266 | infile.close() 267 | print "Read:" 268 | print contents 269 | 270 | # Now, supposedly, feedme is running on the server. 271 | 272 | def url_exists(url): 273 | '''Does the URL exist? Return True or False.''' 274 | try: 275 | urlfile = urllib2.urlopen(url) 276 | urlfile.close() 277 | return True 278 | except urllib2.HTTPError, e: 279 | if e.code == 404: 280 | return False 281 | perror("HTTP error checking whether URL %s exists! code %d" % (url, e.code)) 282 | raise(e) 283 | 284 | except urllib2.URLError, e: 285 | # There is NO documentation on how to handle URLErrors 286 | # or how to find out the code, 287 | # but from fiddling around, it's a a socket.gaierror, 288 | # which can be treated as a tuple where the 0th element is the errno. 289 | perror("URL error checking whether URL %s exists! errno %d" \ 290 | % (url, e.reason[0])) 291 | # Was it because we don't have a network at all? 292 | # 2 = failure in address resolution, -2 = name or service not known. 293 | # If it's one of these, give the user a chance to notice it and 294 | # restart the network. 295 | if e.reason[0] == 2 or e.reason[0] == -2: 296 | droid.vibrate() 297 | droid.makeToast("Network may be down!") 298 | return False 299 | raise(e) 300 | 301 | # We can also get various other errors, such as httplib.BadStatusLine 302 | except Exception, e: 303 | perror("Problem checking whether URL %s exists!\nException: %s" \ 304 | % (url, str(e))) 305 | raise(e) 306 | 307 | def wait_for_feeds(baseurl): 308 | # When the server is done running feedme, it should create a file 309 | # inside the date directory called LOG. 310 | # So look for that: 311 | logurl = baseurl + 'LOG' 312 | print "Waiting for LOG to appear at", logurl, '...' 313 | save_feeddirs = [] 314 | while not url_exists(logurl): 315 | print '.', 316 | sys.stdout.flush() 317 | 318 | # Check for new directories appearing in the feeds dir, 319 | # and print them out as they appear. 320 | feeddirs = parse_directory_page(baseurl) 321 | # Can't do a simple set(feeddirs) - set(save_feeddirs) here: 322 | # one of them might be None and that throws an error. 323 | if feeddirs: 324 | newdirs = set(feeddirs) 325 | if save_feeddirs: 326 | newdirs -= set(save_feeddirs) 327 | for newfeed in newdirs: 328 | print newfeed, 329 | save_feeddirs = feeddirs 330 | else: 331 | print "(no feeddirs yet)", 332 | 333 | time.sleep(10) 334 | 335 | def download_feeds(baseurl, outdir): 336 | fetch_feeds_dir_recursive(baseurl, outdir) 337 | 338 | if is_android: 339 | droid.makeToast("Feeds downloaded") 340 | droid.vibrate() 341 | droid.notify('Feed Fetcher', 'Feeds downloaded to ' + outdir) 342 | 343 | print 'Feeds downloaded to ' + outdir 344 | 345 | def check_if_feedme_run(feedurl, dateurl): 346 | '''Only initiate a feed if there isn't already a log file there, 347 | either in the base dir (meaning a feed is still running) 348 | or in any subdir (meaning it ran and has finished). 349 | feedurl and dateurl are both expected to end with / already. 350 | We might have aborted some earlier attempt, or even kicked 351 | off feeds from some other machine, but now need to download feeds. 352 | Return 0 if we think feedme has not yet run, 1 if we think it 353 | is still running, 2 if we think it has finished. 354 | ''' 355 | 356 | if url_exists(feedurl + 'LOG'): 357 | return 1 # still running 358 | 359 | if url_exists(dateurl + 'LOG'): 360 | return 2 # ran and is now finished 361 | 362 | return 0 # has not yet run 363 | 364 | if __name__ == '__main__': 365 | if is_android: 366 | outdir = android_localdir 367 | else: 368 | outdir = os.path.expanduser(localdir) 369 | dirdate = time.strftime("%m-%d-%a") 370 | 371 | feedurl = serverurl + 'feeds/' 372 | baseurl = feedurl + dirdate + '/' 373 | 374 | already_ran = check_if_feedme_run(feedurl, baseurl) 375 | if already_ran == 0: 376 | print "Feedme has not yet run" 377 | elif already_ran == 1: 378 | print "Feedme is running already" 379 | else: 380 | print "Feedme already ran to completion" 381 | sys.stdout.flush() 382 | 383 | try: 384 | if already_ran == 0: 385 | print "Initiating feedme:", serverurl 386 | run_feed(serverurl, outdir) 387 | 388 | if already_ran < 2: 389 | wait_for_feeds(baseurl) 390 | 391 | download_feeds(baseurl, os.path.join(outdir, dirdate)) 392 | 393 | except KeyboardInterrupt: 394 | print "KeyboardInterrupt" 395 | except urllib2.URLError, e: 396 | print "Couldn't access server: " + str(e) 397 | 398 | if errstr: 399 | print "\n\n====== ERRORS ============" 400 | print errstr 401 | -------------------------------------------------------------------------------- /feeds.css: -------------------------------------------------------------------------------- 1 | /* Stylesheet for feedme output read on Android phones */ 2 | 3 | body { 4 | color: white !important; 5 | background: #000 !important; 6 | /* font-size: 1.35em; */ 7 | font-size: 14.5px; 8 | } 9 | 10 | a:link, a:hover, a:active { color: #ffff00 !important; } 11 | 12 | a:visited { color: #ffdd00 !important; } 13 | 14 | /* Space out the feeds list page so there's space for a finger */ 15 | div.index { margin-top: 1em; margin-bottom: 1.45em; } 16 | 17 | /* Android's default preformatted font is tiny in 1.6. 18 | * But it's fine in 2.3 and later (until the Oct 2017 WebView bustage 19 | * after which no fix seems to work). 20 | tt, pre { font-size: 1.2em; } 21 | */ 22 | 23 | /* Limit images to viewport width. High Country News wraps their img 24 | * tags inside a

so handle figure too. 25 | */ 26 | img, figure { max-width: 98.5%; height: auto; !important; } 27 | 28 | /* Hack to get XKCD What-If images to display. 29 | * They're transparent PNG, so we'd otherwise see them as black on black. 30 | */ 31 | .illustration { background-color: white; max-width: initial; } 32 | 33 | /* Similar for XKCD */ 34 | #comic { background-color: white; } 35 | #comic img { background-color: white; max-width: initial; } 36 | 37 | /* Longreads has an annoying modal blocker on some stories */ 38 | div[class="modal"] { 39 | display: none !important; 40 | } 41 | 42 | /* Show inserted and deleted sections, for losalamosmeetings */ 43 | ins { background: #079; } 44 | del { background: #b00; } 45 | -------------------------------------------------------------------------------- /feedviewer/README.md: -------------------------------------------------------------------------------- 1 | This is FeedViewer, a program for reading RSS feeds fetched by FeedMe. 2 | 3 | This FeedViewer is a start at writing a Python version of the 4 | [Android program FeedViewer](http://shallowsky.com/software/FeedViewer/). 5 | It's written out of frustration over all the bugs and limitations of 6 | the Android WebView library. 7 | Unfortunately, it doesn't really help that since there's no Python 8 | that Android can use for GUI apps. 9 | -------------------------------------------------------------------------------- /feedviewer/feedviewer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright (C) 2018 by Akkana Peck. 4 | # Share and enjoy under the GPL v2 or later. 5 | 6 | """View RSS feeds pulled by FeedMe.""" 7 | 8 | from PyQt5.QtWidgets import QApplication, QMainWindow, QToolBar, QTabWidget, \ 9 | QAction, QStatusBar, QProgressBar, QShortcut 10 | 11 | import sys, os 12 | import posixpath 13 | import shutil 14 | 15 | import quickbrowse 16 | 17 | class DummyURLbar: 18 | def setText(self, s): 19 | pass 20 | 21 | class FeedViewerWindow(quickbrowse.BrowserWindow): 22 | def __init__(self, *args, **kwargs): 23 | # Strip out arguments we handle that are different from QMainWindow: 24 | if 'feed_dir' in kwargs: 25 | self.feed_dir = kwargs['feed_dir'] 26 | del kwargs['width'] 27 | else: 28 | self.feed_dir = os.path.expanduser("~/feeds") 29 | 30 | super(FeedViewerWindow, self).__init__(*args, **kwargs) 31 | 32 | self.cur_feed = None 33 | self.cur_url = None 34 | self.last_url = None 35 | 36 | self.urlbar = DummyURLbar() 37 | 38 | # FeedViewer should remember where it was, but for now, 39 | # always start on the feed page. 40 | # We need to load something here, so quickbrowse will call new_tab 41 | # and give us self.webviews[0] so we can set up events. 42 | self.feed_page() 43 | 44 | self.webviews[0].urlChanged.connect(self.url_changed) 45 | self.webviews[0].loadFinished.connect(self.load_finished) 46 | 47 | # To get scroll position we need a QWebEnginePage. 48 | webpage = self.webviews[0].page() 49 | webpage.scrollPositionChanged.connect(self.scroll_position_changed) 50 | 51 | def init_chrome(self): 52 | self.setWindowTitle("FeedViewer") 53 | 54 | # Set up the minimal FeedViewer chrome 55 | toolbar = QToolBar("Toolbar") 56 | self.addToolBar(toolbar) 57 | 58 | btn_act = QAction("<<", self) 59 | # for an icon: QAction(QIcon("bug.png"), "Your button", self) 60 | btn_act.setStatusTip("Go back") 61 | btn_act.triggered.connect(self.go_back) 62 | toolbar.addAction(btn_act) 63 | 64 | btn_act = QAction("Feeds", self) 65 | btn_act.setStatusTip("Go forward") 66 | btn_act.triggered.connect(self.feed_page) 67 | toolbar.addAction(btn_act) 68 | 69 | btn_act = QAction("Del", self) 70 | btn_act.setStatusTip("Go forward") 71 | btn_act.triggered.connect(self.delete_feed) 72 | toolbar.addAction(btn_act) 73 | 74 | btn_act = QAction("ToC", self) 75 | btn_act.setStatusTip("Go forward") 76 | btn_act.triggered.connect(self.table_of_contents) 77 | toolbar.addAction(btn_act) 78 | 79 | btn_act = QAction(">>", self) 80 | btn_act.setStatusTip("Go forward") 81 | btn_act.triggered.connect(self.go_forward) 82 | toolbar.addAction(btn_act) 83 | 84 | self.webviews = [] 85 | 86 | self.tabwidget = QTabWidget() 87 | self.tabwidget.setTabBarAutoHide(True) 88 | self.setCentralWidget(self.tabwidget) 89 | self.tabwidget.tabBar().installEventFilter(self) 90 | self.prev_middle = -1 91 | self.active_tab = 0 92 | 93 | self.setStatusBar(QStatusBar(self)) 94 | self.progress = QProgressBar() 95 | self.statusBar().addPermanentWidget(self.progress) 96 | 97 | # Key bindings. 98 | # For keys like function keys, use QtGui.QKeySequence("F12") 99 | QShortcut("Ctrl+Q", self, activated=self.close) 100 | QShortcut("Ctrl+L", self, activated=self.select_urlbar) 101 | QShortcut("Ctrl+T", self, activated=self.new_tab) 102 | QShortcut("Ctrl+R", self, activated=self.reload) 103 | 104 | QShortcut("Alt+Left", self, activated=self.go_back) 105 | QShortcut("Alt+Right", self, activated=self.go_forward) 106 | 107 | def go_back(self): 108 | self.webviews[self.active_tab].back() 109 | 110 | def go_forward(self): 111 | self.webviews[self.active_tab].forward() 112 | 113 | def feed_page(self): 114 | '''Load the top-level page showing feeds available.''' 115 | 116 | self.cur_feed = None 117 | self.cur_url = None 118 | 119 | html = '' 120 | 121 | # Loop over directories inside the feed dir, which should mostly be days 122 | for d in os.listdir(self.feed_dir): 123 | full_d = os.path.join(self.feed_dir, d) 124 | if not os.path.isdir(full_d): 125 | continue 126 | day_feeds_html = '' 127 | 128 | # Loop over directories in the day dir, which should be feeds: 129 | dayfeeds = os.listdir(full_d) 130 | dayfeeds.sort() 131 | for f in dayfeeds: 132 | full_f = os.path.join(full_d, f) 133 | if not os.path.isdir(full_f): 134 | continue 135 | index = os.path.join(full_f, "index.html") 136 | if not os.path.exists(index): 137 | continue 138 | day_feeds_html += '

%s' % (index, f) 139 | 140 | # Did we get any valid feeds? 141 | if day_feeds_html: 142 | html += '

\n%s:\n%s' % (d, day_feeds_html) 143 | 144 | if not html: 145 | html = "

\nNo feeds found\n" 146 | 147 | # QtWebEngine ignores 150 | 151 | Feeds 152 | 153 | 154 | 155 | %s 156 | 157 | 158 | ''' % (self.feed_dir, html) 159 | 160 | # print("html:", html) 161 | # print("Feed dir:", self.feed_dir) 162 | # print("base:", 'file://' + self.feed_dir) 163 | self.load_html(html, 'file://' + self.feed_dir) 164 | 165 | def delete_feed(self): 166 | thisfeeddir = posixpath.join(self.feed_dir, self.cur_feed) 167 | print("deleting", thisfeeddir) 168 | shutil.rmtree(thisfeeddir) 169 | self.feed_page() 170 | 171 | def table_of_contents(self): 172 | self.load_url('file://' + posixpath.join(self.feed_dir, 173 | self.cur_feed, 174 | "index.html")) 175 | 176 | def reload(self): 177 | self.webviews[self.active_tab].reload() 178 | 179 | def whichfeed(self, url): 180 | '''Figure out which feed we're on from the URL''' 181 | if url.scheme() != 'file': 182 | return None 183 | 184 | path = url.path() 185 | print("whichfeed", path) 186 | if not path.startswith(self.feed_dir): 187 | return None 188 | path = path[len(self.feed_dir) + 1:] 189 | if not path: 190 | return None 191 | 192 | splitpath = path.split('/') 193 | if len(splitpath) < 2: 194 | print("Yikes, splitpath has too few components:", splitpath) 195 | return splitpath[0] 196 | return splitpath[0] + '/' + splitpath[1] 197 | 198 | # 199 | # Slots, in addition to what the parent BrowserWindow class registers: 200 | # 201 | 202 | def url_changed(self, url): 203 | # print("FeedmeView URL changed", url) 204 | self.last_url = url 205 | 206 | def load_finished(self, ok): 207 | # print("FeedmeView load finished", ok) 208 | if ok and self.last_url: 209 | self.cur_url = self.last_url 210 | self.cur_feed = self.whichfeed(self.cur_url) 211 | self.last_url = None 212 | 213 | def scroll_position_changed(self, position): 214 | print("Scroll position changed:", position) 215 | print("Page says", self.webviews[0].page().scrollPosition()) 216 | # Happily, these are the same. They seem to be in pixels. 217 | print("Contents size", self.webviews[0].page().contentsSize()) 218 | 219 | # If this doesn't work out, there are also apparently 220 | # ways of getting scroll pos from JS. See _update_pos() in 221 | # qutebrowser/qutebrowser/browser/webengine/webenginetab.py 222 | 223 | if __name__ == '__main__': 224 | # # Return control to the shell before creating the window: 225 | # rc = os.fork() 226 | # if rc: 227 | # sys.exit(0) 228 | 229 | app = QApplication(sys.argv) 230 | 231 | win = FeedViewerWindow(width=500, height=700) 232 | 233 | win.show() 234 | 235 | app.exec() 236 | -------------------------------------------------------------------------------- /helpers/README-selenium.md: -------------------------------------------------------------------------------- 1 | # Configuring Selenium with Headless Firefox on Linux without a Desktop 2 | 3 | Specifically, this is what I needed to do on our Debian server, 4 | which did not previously have X or any related programs. 5 | 6 | 7 | ## Install Selenium 8 | 9 | ``` 10 | apt install --no-install-recommends --no-install-suggests python3-selenium 11 | ``` 12 | 13 | ## Install Firefox 14 | 15 | Download some version of Firefox from mozilla.org. 16 | I opted for the 17 | [extended release version](https://www.mozilla.org/en-US/firefox/all/#firefox-desktop-esr) . 18 | I installed it in *~/firefox-esr*. 19 | 20 | You also need geckodriver, which doesn't come with Firefox. 21 | Despite Mozilla having its own fileservers and not using git as 22 | its version control system, for some reason geckodriver only seems 23 | to be available from GitHub: 24 | [github.com/mozilla/geckodriver/releases](https://github.com/mozilla/geckodriver/releases) . 25 | The tarballs there extract into a single executable. 26 | I moved that executable into *~/firefox-esr*, where I'd installed Firefox, 27 | so I'd only need to add one directory to my PATH. Selenium lets you 28 | specify the path to the geckodriver executable, but then how will 29 | geckodriver find Firefox if it's not in your PATH? 30 | 31 | 32 | ## Set up a virtual X environment 33 | 34 | Our server didn't have X installed since it's never used as a desktop 35 | machine. 36 | Unfortunately, browsers (both Firefox and Chromium) 37 | require libraries for X11 and a GUI toolkit (like GTK), 38 | even when they're running in headless mode. 39 | 40 | For Firefox, 41 | 42 | ``` 43 | apt install --no-install-recommends --no-install-suggests \ 44 | xvfb python3-xvfbwrapper libgtk-3-0 libdbus-glib-1-2 45 | ``` 46 | 47 | This pulls in a total of 61 packages, 51 of which are from libgtk-3.0. 48 | 49 | Now, from somewhere where you can display 50 | X clients -- for instance, log in to your remote server with 51 | `ssh -X servername` 52 | and verify that Firefox runs. 53 | 54 | ``` 55 | localhost% ssh -X servername 56 | servername% cd firefox-esr 57 | servername% ./firefox -p 58 | ``` 59 | 60 | The -p tells firefox to start the profile manager so you can create a 61 | new profile. (Copying the profile from another machine may not be an 62 | option if the firefox versions are significantly different.) 63 | I named my profile "selenium"; the nyt-selenium helper will 64 | look for a profile of that name. After that, you can start firefox 65 | with `./firefox -P selenium`. 66 | 67 | 68 | 69 | 70 | FeedMe dependencies: 71 | python3 python3-feedparser python3-lxml python3-bs4 72 | 73 | 74 | 75 | 76 | ## Copying Securely to the Public Server 77 | 78 | Now I had it working on our home fileserver. But now I needed a way 79 | to copy it to the actual web server, so my regular feedfetcher run 80 | will pick it up there whether I'm at home or away. 81 | 82 | For copying files between machines, I favor rsync over ssh. 83 | But since this will be running automatically from cron, it needs 84 | to run without a password -- I need to set up an ssh key. 85 | It's easy enough to 86 | [set up an ssh key for passwordless file 87 | copies](https://shallowsky.com/blog/linux/ssh-keys-passwordless.html), 88 | but you might not want to allow passwordless key access to any 89 | important accounts, like your personal user account or the web server's user. 90 | 91 | In that case, one option is to create a new user without a password 92 | or a login shell: 93 | set the login shell to something like */bin/false*, then set up the 94 | key in the new user's *~/.ssh*. You won't be able to do the normal 95 | 96 | 97 | ## Problems with this 98 | 99 | Apparently rsync over ssh requires shell access: it fails silently. 100 | 101 | To try to get error messages, try (from moon) 102 | 103 | rsync -v -e 'ssh -vvvv' urls feeb:feeds/ 104 | 105 | But the error isn't clear, just 106 | 107 | debug2: channel 0: chan_shutdown_write (i0 o1 sock -1 wfd 5 efd 6 [write]) 108 | rsync: connection unexpectedly closed (0 bytes received so far) [sender] 109 | debug2: channel 0: output drain -> closed 110 | debug3: receive packet: type 98 111 | rsync error: error in rsync protocol data stream (code 12) at io.c(228) [sender=3.2.3] 112 | 113 | Trying scp instead of rsync gets a 114 | 115 | lost connection 116 | 117 | instead of the error in protocol data stream, but still no help. 118 | 119 | I found a few pages recommending using something called rssh as the 120 | login shell, but that's not in either Debian or Ubuntu repositories, 121 | and it apparently has quite a history of security holes. 122 | 123 | However, you may be able to restrict the user to a predefined set 124 | of commands: 125 | https://stackoverflow.com/questions/402615/how-to-restrict-ssh-users-to-a-predefined-set-of-commands-after-login 126 | 127 | A chrooted user might also be something to pursue. 128 | 129 | 130 | 131 | ## Older Crap 132 | 133 | (If you don't want to install xvfbwrapper, see other approaches in 134 | https://stackoverflow.com/questions/6183276/how-do-i-run-selenium-in-xvfb/6300672#6300672 ) 135 | 136 | In theory now you can do: 137 | 138 | ``` 139 | from selenium import webdriver 140 | from selenium.webdriver.firefox.options import Options 141 | from xvfbwrapper import Xvfb 142 | import os 143 | 144 | foxprofiledir = os.path.expanduser("~/.mozilla/firefox/someprofile/") 145 | 146 | options = Options() 147 | options.headless = True 148 | 149 | executable_dir = os.path.expanduser("~/firefox-esr") 150 | os.environ["PATH"] = "%s:%s" % (executable_dir, os.environ["PATH"]) 151 | 152 | vdisplay = Xvfb() 153 | vdisplay.start() 154 | 155 | sbrowser = webdriver.Firefox(firefox_profile=foxprofiledir, 156 | options=options) 157 | ``` 158 | 159 | Except no, that didn't work: 160 | 161 | ``` 162 | :1: DeprecationWarning: firefox_profile has been deprecated, please pass in a Service object 163 | Traceback (most recent call last): 164 | File "", line 1, in 165 | File "/usr/lib/python3/dist-packages/selenium/webdriver/firefox/webdriver.py", line 170, in __init__ 166 | RemoteWebDriver.__init__( 167 | File "/usr/lib/python3/dist-packages/selenium/webdriver/remote/webdriver.py", line 152, in __init__ 168 | self.start_session(capabilities, browser_profile) 169 | File "/usr/lib/python3/dist-packages/selenium/webdriver/remote/webdriver.py", line 249, in start_session 170 | response = self.execute(Command.NEW_SESSION, parameters) 171 | File "/usr/lib/python3/dist-packages/selenium/webdriver/remote/webdriver.py", line 318, in execute 172 | self.error_handler.check_response(response) 173 | File "/usr/lib/python3/dist-packages/selenium/webdriver/remote/errorhandler.py", line 242, in check_response 174 | raise exception_class(message, screen, stacktrace) 175 | selenium.common.exceptions.WebDriverException: Message: Process unexpectedly closed with status 255 176 | 177 | ``` 178 | 179 | It turns out that if you ssh -X into the server and try running firefox, 180 | you get: 181 | 182 | ``` 183 | XPCOMGlueLoad error for file /home/akkana/firefox-esr/libmozgtk.so: 184 | libgtk-3.so.0: cannot open shared object file: No such file or directory 185 | Couldn't load XPCOM. 186 | ``` 187 | 188 | https://bugzilla.mozilla.org/show_bug.cgi?id=1372998 189 | 190 | The issue is that headless mode isn't really headless, it still requires 191 | all the GTK-related libraries, and there are no plans to fix that 192 | in the forseeable future. 193 | -------------------------------------------------------------------------------- /helpers/README.md: -------------------------------------------------------------------------------- 1 | 2 | # FeedMe Helpers 3 | 4 | Some sites make fetching feeds too complicated to handle through 5 | the normal FeedMe process. 6 | 7 | For instance, a site may not have RSS, so you need to write code to 8 | parse an HTML page. Or you may need to fetch pages on a machine other 9 | than the one that's normally assembling your feeds. 10 | 11 | Enter feedme helpers. There are two types of helpers: ```page_helpers``` 12 | and ```feed_helpers```. 13 | 14 | Helpers will be imported as Python modules, so they must either 15 | be in the *helpers/* directory wherever feedme is installed, or 16 | somewhere in your PYTHONPATH. They also must conform to Python 17 | module naming rules, e.g. nyt_selenium.py, not nyt-selenium.py. 18 | 19 | 20 | ## Passing Arguments to Helpers 21 | 22 | Both types of helpers can take arguments. Any argument beginning with 23 | `helper_` in the site file will be passed to the helper. 24 | For instance, *nyt_selenium* takes two arguments: 25 | 26 | ``` 27 | helper_executable_path = ~/firefox-esr 28 | helper_log_file = $d/NYT_selenium.log 29 | ``` 30 | 31 | A $d in the arg string will be expanded to the current day's 32 | feeds directory, e.g. `~/feeds/10-07-Wed`, 33 | before being passed to the helper. 34 | $f will be expanded to the feed name inside $d, e.g. 35 | `~/feeds/10-07-Wed/NY_Times/`. 36 | If you need a literal $d or $f, use a backslash escape, `\$d`. 37 | 38 | For page helpers, you can also use the *url* in the site file to 39 | pass information. url must be set to something anyway, even if it's 40 | not used: feedme uses the presence of url to determine if a site file 41 | is valid. 42 | 43 | When specifying directories, you'll probably need to specify an 44 | absolute path (in place of ~/) if you call feedme from a web server, 45 | since it isn't running as your user. Tilde expansion is done (or not) 46 | inside the helper, since helper_args may not always be filenames. 47 | Any expansion such as tilde (expanduser) expansion is left to 48 | the helper modules, since feedme doesn't know how the helper_arg 49 | will be used by the helper. $d and $f expansion are done in feedme 50 | before passing them to the helper. 51 | 52 | 53 | ## Page Helpers: 54 | 55 | A ```page_helper``` can fetch single pages 56 | one at a time. The HTML will be passed back to feedme, which can 57 | then do its normal operations and store the result in the feeds directory. 58 | This assumes that the site has a usable RSS feed, and help is only 59 | needed for fetching the individual articles. 60 | 61 | Let's say you want to feed the New York Times. You're a subscriber, 62 | and you have a Firefox profile with the appropriate cookies. 63 | 64 | In your site file, add a line like 65 | 66 | ``` 67 | page_helper = nyt_selenium 68 | ``` 69 | 70 | That tells feedme to import the helper module nyt_selenium.py. 71 | 72 | nyt_selenium.py must define the following functions: 73 | 74 | ``` 75 | initialize(helper_arg) 76 | # can be a no-op if no persistent state is needed 77 | 78 | fetch_article(url) 79 | # Returns the desired html as a string (not bytes). 80 | # This can be further processed by feedme 81 | # if your site file has directives like page_start, 82 | # skip_pat etc. 83 | ``` 84 | 85 | 86 | ## Feed Helpers 87 | 88 | A ```feed_helper``` fetches a whole feed at once. 89 | For instance, a site that has no RSS would need a feed helper, 90 | since feedme can only get the list of stories from RSS. 91 | With a feed helper you would have to manage last-seen dates 92 | on your own. 93 | 94 | For instance, to use the copyfeed helper, add a line like 95 | 96 | ``` 97 | feed_helper = copyfeed 98 | helper_arg = ~/feeds/$d/New_York_Times/ 99 | ``` 100 | 101 | But see the comment above under *Passing Arguments to Helpers* 102 | regarding ~. 103 | 104 | To write a feed helper, you must implement one call: 105 | 106 | ``` 107 | fetch_feed(target_dir, helper_arg) 108 | ``` 109 | 110 | The ```target_dir``` is the target feed directory, 111 | e.g. ```.../feeds/10-06-Wed/``` into which the final files should be written. 112 | The helper is responsible for creating that directory, 113 | if it's successful in getting feed files. 114 | 115 | All files in the target directory will be added to the day's ```MANIFEST```. 116 | -------------------------------------------------------------------------------- /helpers/copyfeed.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # A FeedMe helper module that copies files from a fixed location. 4 | 5 | 6 | import time 7 | import shutil 8 | import os, sys 9 | 10 | from utils import expanduser 11 | 12 | 13 | def fetch_feed(target_dir, helper_args): 14 | """Copy a full feed, including index.html and all stories and images, 15 | to the target directory. Return a list of files copied 16 | (relative to the target directory, not full paths). 17 | If the source dir contains index.html, all files in it will be copied. 18 | Returns a list of the files copied. 19 | """ 20 | 21 | if "srcdir" in helper_args: 22 | srcdir = expanduser(helper_args["srcdir"]) 23 | 24 | if os.path.exists(os.path.join(srcdir, "index.html")): 25 | return copy_files(srcdir, target_dir) 26 | 27 | print("Couldn't find a feed for today under", srcdir, file=sys.stderr) 28 | return None 29 | 30 | 31 | def copy_files(srcdir, dstdir): 32 | children = os.listdir(srcdir) 33 | copied = [] 34 | 35 | try: 36 | os.mkdir(dstdir) 37 | except FileExistsError: 38 | pass 39 | if not os.path.isdir(dstdir): 40 | print("Couldn't make directory", dstdir, file=sys.stderr) 41 | return None 42 | 43 | for child in children: 44 | if os.path.isdir(child): 45 | continue 46 | shutil.copy(os.path.join(srcdir, child), dstdir) 47 | copied.append(child) 48 | 49 | print("Copied", len(copied), "files", file=sys.stderr) 50 | return copied 51 | 52 | 53 | if __name__ == '__main__': 54 | if len(sys.argv) < 3: 55 | print("Usage: %s source_dir target_dir" % os.path.basename(sys.argv[0])) 56 | sys.exit(1) 57 | 58 | copied = fetch_feed(sys.argv[2], sys.argv[1]) 59 | 60 | print("Copied %d files:" % len(copied)) 61 | for f in copied: 62 | print(" ", f) 63 | 64 | -------------------------------------------------------------------------------- /helpers/nyt_selenium.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # A FeedMe helper that can fetch articles from the NYTimes 4 | # using a logged-in subscriber profile and selenium. 5 | # As currently written, it will use the first profile in ~/.mozilla/firefox 6 | # that has "selenium" in the name. 7 | # 8 | # If geckodriver isn't in your path, pass the path to it 9 | # as the helper_arg. 10 | 11 | # XXX Ctrl-C seems to kill the selenium webdriver, 12 | # so all subsequent fetches will fail. Is there a solution? 13 | # Sometimes it hangs forever and ctrl-C is the only solution. 14 | 15 | from selenium import webdriver 16 | from selenium.webdriver.firefox.options import Options 17 | 18 | from urllib3.exceptions import MaxRetryError, NewConnectionError 19 | from selenium.common.exceptions import TimeoutException 20 | 21 | from bs4 import BeautifulSoup 22 | 23 | import os, sys 24 | 25 | import re 26 | import time 27 | import tempfile 28 | 29 | import traceback 30 | 31 | 32 | verbose = True 33 | 34 | # The selenium browser driver will be set by initialize() 35 | sbrowser = None 36 | 37 | # On the NYT, once anything times out, all subsequent stories will also 38 | # time out, and none of the ways that supposedly control selenium 39 | # timeouts actually work, so they'll all wait for several minutes. 40 | # Returning None or an empty string will keep the URL out of the 41 | # cache, which isn't good because it means the same URL will keep 42 | # being tried and timing out every day forever. 43 | # Instead, for NYT timeouts, let's return an HTML snippet 44 | # indicating a timeout, so feedme will mark it as cached 45 | # and won't keep retrying. 46 | num_timeouts = 0 47 | 48 | # How many timeouts will be tolerated before giving up? 49 | MAX_TIMEOUTS = 3 50 | 51 | 52 | adpat = re.compile("story-ad-[0-9]*-wrapper") 53 | 54 | 55 | def initialize(helper_args=None): 56 | """Initialize selenium, returning the web driver object.""" 57 | 58 | global sbrowser 59 | 60 | foxprofiledir = find_firefox_profile_dir() 61 | 62 | # Deprecated, but no one seems to know the new way: 63 | options = Options() 64 | options.headless = True 65 | 66 | # "eager": Don't wait for full page load including images etc., 67 | # but return as soon as the page would normally be usable. 68 | # "none": return as soon as the base page has been loaded. 69 | options.page_load_strategy = "none" 70 | 71 | print("Creating headless browser...", file=sys.stderr) 72 | 73 | # Unpack the helper_args. Currently support executable_path 74 | # (which has already been $d expanded, but not ~ expanded) 75 | # and log_file. 76 | # With the default of "geckodriver", selenium will search $PATH. 77 | executable_path = "geckodriver" 78 | log_file = None 79 | if helper_args: 80 | if "executable_path" in helper_args: 81 | executable_path = os.path.expanduser(helper_args["executable_path"]) 82 | 83 | if "log_file" in helper_args: 84 | log_file = os.path.expanduser(helper_args["log_file"]) 85 | # $d or $f in log_file could fail because the feeds/datedir 86 | # hasn't yet been created. So create it if needed. 87 | logdir = os.path.dirname(log_file) 88 | if not os.path.exists(logdir): 89 | os.makedirs(logdir) 90 | print("Created", logdir, "for log file", file=sys.stderr) 91 | if not os.path.exists(logdir): 92 | print("Couldn't create %s: storing log in /tmp" % logdir, 93 | file=sys.stderr) 94 | log_file = None 95 | 96 | if not log_file: 97 | log_file = tempfile.mkstemp(prefix="nyt_geckodriver", suffix=".log") 98 | 99 | if executable_path.startswith('/'): 100 | # Did this point to the actual geckodriver executable? 101 | # If so, pass it as executable_path. 102 | if executable_path.endswith("geckodriver") \ 103 | and os.path.exists(executable_path) \ 104 | and os.path.isfile(executable_path): 105 | executable_path = executable_path 106 | elif os.path.isdir(executable_path) and \ 107 | os.path.isfile(os.path.join(executable_path, "geckodriver")): 108 | # It's a directory. Add it to the beginning of $PATH. 109 | os.environ["PATH"] = "%s:%s" % (executable_path, 110 | os.environ["PATH"]) 111 | # Reset the executable path we'll pass in to webdriver.Firefox 112 | # back to the default, since it only needs to search PATH 113 | executable_path = "geckodriver" 114 | # XXX No way (yet) to support adding two paths if firefox and 115 | # geckodriver are in two different places. 116 | # Should allow dir1:dir2, but then we'd have to check all 117 | # of them to see if geckodriver exists. 118 | 119 | if verbose: 120 | print("nyt_selenium: executable_path '%s', log_file '%s'" 121 | % (executable_path, log_file), file=sys.stderr) 122 | 123 | sbrowser = webdriver.Firefox(firefox_profile=foxprofiledir, 124 | executable_path=executable_path, 125 | service_log_path=log_file, 126 | options=options) 127 | 128 | # Attempt to limit the timeout. 129 | # None of these reliably limits the timeout, however: 130 | # sometimes selenium will wait minutes for each story 131 | # and I haven't found any way around that. 132 | sbrowser.set_page_load_timeout(25) 133 | sbrowser.implicitly_wait(20); 134 | sbrowser.set_script_timeout(20); 135 | 136 | 137 | def timeout_boilerplate(url, errstr): 138 | """If there have been too many timeouts, return an HTML page saying so. 139 | """ 140 | return """ 141 | Timeout on %s 142 | 143 |

Timeout

144 | 145 |

146 | %s
147 |

148 | 149 |

150 | URL was: %s 151 | 152 | 153 | """ % (errstr, url, url) 154 | 155 | 156 | def fetch_article(url): 157 | """Fetch the given article using the already initialized 158 | selenium browser driver. 159 | Filter it down using BeautifulSoup so feedme doesn't have to. 160 | Return html source as a string, or None. 161 | """ 162 | global num_timeouts 163 | 164 | # Was there a timeout earlier? Then everything subsequent will fail, 165 | # so don't even bother trying to get it. 166 | if num_timeouts >= MAX_TIMEOUTS: 167 | return timeout_boilerplate(url, "Gave up after earlier timeout") 168 | 169 | # While debugging: keep track of how long each article takes. 170 | t0 = time.time() 171 | 172 | try: 173 | sbrowser.get(url) 174 | except TimeoutException as e: 175 | num_timeouts += 1 176 | # Supposedly this sometimes helps in recovering from timeouts. 177 | # But in practice, nothing helps: once anything times out, 178 | # every subsequent story also times out. 179 | # sbrowser.back() 180 | print("EEK! TimeoutException", e, file=sys.stderr) 181 | return timeout_boilerplate(url, "TimeoutException") 182 | except (ConnectionRefusedError, MaxRetryError, NewConnectionError) as e: 183 | # MaxRetryError and NewConnectionError come from urllib3.exceptions 184 | # ConnectionRefusedError is a Python builtin. 185 | num_timeouts += 1 186 | print("EEK! Connection error", e, file=sys.stderr) 187 | traceback.print_exc(file=sys.stderr) 188 | return timeout_boilerplate(url, str(e)) 189 | except Exception as e: 190 | num_timeouts += 1 191 | errstr = "Unexpected exception in webdriver.get: " + str(e) 192 | print(errstr, file=sys.stderr) 193 | return timeout_boilerplate(url, errstr) 194 | 195 | # Hitting ^C will mess up the webdriver and cause all subsequent 196 | # stories to fail. Not particularly recommended, but sometimes 197 | # there's no alternative. 198 | except KeyboardInterrupt: 199 | # sbrowser.back() 200 | num_timeouts += 1 201 | return timeout_boilerplate(url, "Keyboard Interrupt") 202 | 203 | t1 = time.time() 204 | print("%.1f seconds for %s" % (t1 - t0, url), file=sys.stderr) 205 | 206 | try: 207 | fullhtml = sbrowser.page_source 208 | except Exception as e: 209 | num_timeouts += 1 210 | errstr = "Fetched page but couldn't get html: " + str(e) 211 | print(errstr, file=sys.stderr) 212 | return timeout_boilerplate(url, errstr) 213 | print("%.1f seconds to get page_source" % (time.time() - t1), 214 | file=sys.stderr) 215 | 216 | if not fullhtml: 217 | print("nyt_selenium: couldn't fetch", url) 218 | return None 219 | 220 | soup = BeautifulSoup(fullhtml, "lxml") 221 | 222 | # Look for several possible containers 223 | article = soup.find("section", {"name": "articleBody"}) 224 | if not article: 225 | if verbose: 226 | print("No articleBody", file=sys.stderr) 227 | article = soup.find(class_="live-blog-post") 228 | 229 | if not article: 230 | if verbose: 231 | print("No live-blog-post either.", file=sys.stderr) 232 | article = soup 233 | fullfile = os.path.join("/tmp/%d-full.html" % articleno) 234 | print("Couldn't find any containers: saving %s" % fullfile, 235 | file=sys.stderr) 236 | with open(fullfile, "w") as fp: 237 | fp.write(fullhtml) 238 | 239 | # Remove ads, story-ad-*-wrapper 240 | for ad in article.find_all(class_=adpat): 241 | ad.decompose() 242 | 243 | # Remove images, for now, until this is folded into the 244 | # regular feedme/pagemeparser code to fetch images locally. 245 | for img in article.find_all("img"): 246 | img.decompose() 247 | # NYT has huge SVG images that use the "svg" tag 248 | for img in article.find_all("svg"): 249 | img.decompose() 250 | # Plus inline videos that we definitely don't want to download: 251 | for img in article.find_all("video"): 252 | img.decompose() 253 | 254 | # Done with processing. 255 | # BS randomly sometimes returns str, sometimes bytes when encoding 256 | # is specified. Supposedly if no encoding is specified, it will 257 | # always return str, which is what's wanted here. 258 | # return article.prettify(encoding='utf-8') 259 | return article.prettify() 260 | 261 | 262 | def find_firefox_profile_dir(): 263 | """Return the first profile in ~/.mozilla/firefox/ 264 | that has "selenium" in its name. 265 | """ 266 | mozdir = os.path.expanduser("~/.mozilla/firefox/") 267 | for pdir in os.listdir(mozdir): 268 | if "selenium" in pdir: 269 | return os.path.join(mozdir, pdir) 270 | raise RuntimeError("Can't find a selenium profile in %s" % pdir) 271 | 272 | 273 | if __name__ == '__main__': 274 | import feedparser 275 | import sys 276 | 277 | initialize() 278 | 279 | if len(sys.argv) > 1: 280 | RSS_URL = sys.argv[1] 281 | else: 282 | RSS_URL = 'https://rss.nytimes.com/services/xml/rss/nyt/HomePage.xml' 283 | 284 | feed = feedparser.parse(RSS_URL) 285 | 286 | # feedparser has no error return! One way is to check len(feed.feed). 287 | if len(feed.feed) == 0: 288 | print("Couldn't fetch RSS from", RSS_URL, file=sys.stderr) 289 | sys.exit(1) 290 | 291 | for item in feed.entries: 292 | if 'links' not in item: 293 | print("Item with no links! Continuing") 294 | continue 295 | 296 | lasttime = time.time() 297 | 298 | # href = [str(link['href']) for link in item.links 299 | # if 'rel' in link and 'href' in link 300 | # and link['rel'] == 'alternate'] 301 | 302 | item_link = str(item.link) 303 | sys.stdout.flush() 304 | print("\n==========================================") 305 | print("Link:", item_link) 306 | print(item.summary) 307 | 308 | fullhtml = fetch_article(item_link) 309 | if not fullhtml: 310 | print("Couldn't fetch", item_link) 311 | continue 312 | 313 | print("full html had", len(fullhtml), "characters") 314 | 315 | thistime = time.time() 316 | print("Took", thistime - lasttime, "seconds") 317 | lasttime = thistime 318 | -------------------------------------------------------------------------------- /htmlindex.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """For sites that don't have RSS, this module pulls links from an 4 | HTML page according to the specification in html_index_links, 5 | which is a string indicating which HTML tag and attributes to look for, 6 | e.g. 'div class="layout-homepage__lite"' 7 | Multiple attributes are allowed. 8 | """ 9 | 10 | from datetime import datetime, timedelta 11 | import urllib.request, urllib.parse 12 | import shlex 13 | import ast 14 | from bs4 import BeautifulSoup 15 | import feedparser 16 | import email.utils as email_utils 17 | from types import SimpleNamespace 18 | import sys 19 | 20 | # feedme modules 21 | import pageparser 22 | import utils 23 | 24 | 25 | def parse(feedname, html_index_links, verbose=True): 26 | """Parse an HTML page and try to extract all the links matching 27 | the string html_index_links. 28 | Return a feed object, with structure similar to what 29 | feedparser returns. 30 | """ 31 | identifiers = shlex.split(html_index_links) 32 | tagname = None 33 | attrs = {} 34 | for ident in identifiers: 35 | if '=' in ident: 36 | name, val = ident.split('=') 37 | try: 38 | val = ast.literal_eval(val) 39 | except ValueError: 40 | pass 41 | attrs[name] = val 42 | else: 43 | tagname = ident 44 | 45 | downloader = pageparser.FeedmeURLDownloader(feedname) 46 | feedurl = utils.g_config.get(feedname, 'url') 47 | print("Trying to download", feedurl) 48 | html = downloader.download_url(feedurl) 49 | print("Fetched", len(html), "characters") 50 | soup = BeautifulSoup(html, 'lxml') 51 | print("Made soup") 52 | 53 | # May need to splice feedurl onto the beginning of other URLs later 54 | feedurlparts = urllib.parse.urlparse(feedurl) 55 | 56 | # feed needs to be accessible via dot notation as feed.feed 57 | # to match expectations based on feedparser. 58 | feedret = SimpleNamespace( 59 | encoding='utf-8', 60 | feed=SimpleNamespace(title=feedname), 61 | entries=[] 62 | ) 63 | 64 | # In feed.entries, feedme uses: link links id title author published 65 | # summary|content|description 66 | # Generally it uses them like: if 'link' in entry: link = entry.link 67 | # so the object has to handle both "in" and dot notation. 68 | # 69 | # What is the difference between item["link"] and item["links"] ? 70 | # link is a single URL; 71 | # links is a list of dictionaries with 'href', 72 | # 'rel'(='alternate'), 'type'(='text/html') 73 | # item.pubDate is a unicode string, supposed to be in format 74 | # Thu, 11 Aug 2016 14:46:50 GMT (or +0000) 75 | # but feedme actually uses published, 76 | # pub_date = time.mktime(email_utils.parsedate(item.published)) 77 | # feedparser has added published_parsed which is a time.struct_time, 78 | # but feedme doesn't count on that. Probably best to provide all three. 79 | 80 | # I haven't found a BS find syntax that lets the tag name 81 | # be optional. 82 | if tagname: 83 | finder = soup.find_all(tagname, attrs=attrs) 84 | else: 85 | finder = soup.find_all(attrs=attrs) 86 | for container in finder: 87 | for link in container.find_all('a', href=True): 88 | linktext = link.text.strip() 89 | linkhref = link.get('href') 90 | 91 | urlparts = urllib.parse.urlparse(linkhref) 92 | if not urlparts.scheme: 93 | linkhref = urllib.parse.urljoin(feedurl, linkhref) 94 | 95 | # IMPORTANT: With FeedParserDict, you can set either 96 | # fpd.id = 123 or fpd['id'] = 123, but YOU SHOULD ONLY USE THE 97 | # SECOND FORM. If you use dot notation to set an attribute, 98 | # that attribute will NOT create a corresponding dict-style 99 | # attribute, so 'id' in fpd will still be false. 100 | thisentry = feedparser.util.FeedParserDict() 101 | thisentry['id'] = linkhref 102 | thisentry['link'] = linkhref 103 | thisentry['links'] = [ linkhref ] 104 | thisentry['title'] = linktext 105 | 106 | # Summary has to be some sort of object where summary.value 107 | # is the linktext, so feedme can look at feed.feed.summary.value 108 | thisentry['summary'] = SimpleNamespace(value=linktext) 109 | 110 | # Guard against trashing the whole feed just because one 111 | # link didn't work: 112 | try: 113 | # Try to get the last modified date. Some websites have 114 | # last-modified, CNN has X-Last-Modified, possibly this 115 | # list will have to grow. 116 | lastmodheaders = [ 'last-modified', 'X-Last-Modified' ] 117 | conn = urllib.request.urlopen(linkhref) 118 | lastmodstr = None 119 | lastmod = None 120 | for lmh in lastmodheaders: 121 | try: 122 | lastmodstr = conn.headers.get(lmh) 123 | if lastmodstr: 124 | # print("Last mod string", lastmodstr, "for", linkhref) 125 | break 126 | except: 127 | pass 128 | except Exception as e: 129 | print("Couldn't fetch", linkhref, "because", e) 130 | continue 131 | if lastmodstr: 132 | # sites mostly seem to use format 133 | # 'Sun, 11 Feb 2024 20:34:41 GMT' 134 | # XXX for now, let's hope that's always true. 135 | try: 136 | lastmod = datetime.strptime( 137 | lastmodstr, '%a, %d %b %Y %H:%M:%S %Z').astimezone() 138 | except Exception as e: 139 | print("htmlindex: couldn't parse '%s': %s" 140 | % (lastmodstr, e), file=sys.stderr) 141 | if not lastmod: 142 | # Set it to a bit under a week ago 143 | lastmod = datetime.now() - timedelta(days=5) 144 | if not lastmodstr: 145 | lastmodstr = lastmod.strftime('%a, %d %b %Y %H:%M:%S %Z') 146 | 147 | # Now both lastmod and lastmodstr are set. 148 | thisentry['published'] = lastmodstr 149 | thisentry['published_parsed'] = lastmod.timetuple() 150 | 151 | # feedret["entries"].append(thisentry) 152 | feedret.entries.append(thisentry) 153 | 154 | return feedret 155 | 156 | 157 | if __name__ == '__main__': 158 | utils.read_config_file() 159 | 160 | from pprint import pprint 161 | pprint(parse('CNN', 'li class="card--lite"')) 162 | 163 | -------------------------------------------------------------------------------- /msglog.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import sys 4 | 5 | 6 | ################################################################## 7 | # MsgLog: Print messages and also batch them up to print at the end: 8 | # 9 | class MsgLog: 10 | def __init__(self): 11 | self.msgstr = "" 12 | self.errstr = "" 13 | 14 | def msg(self, s): 15 | self.msgstr += "\n" + s 16 | print("MESSAGE:", s, file=sys.stderr) 17 | 18 | def warn(self, s): 19 | self.msgstr += "\n" + s 20 | print("WARNING:", s, file=sys.stderr) 21 | 22 | def err(self, s): 23 | self.errstr += "\n" + s 24 | print("ERROR:", s, file=sys.stderr) 25 | 26 | def get_msgs(self): 27 | return self.msgstr 28 | 29 | def get_errs(self): 30 | return self.errstr 31 | 32 | 33 | gMsglog = MsgLog() 34 | 35 | def msg(s): 36 | gMsglog.msg(s) 37 | 38 | def warn(s): 39 | gMsglog.warn(s) 40 | 41 | def err(s): 42 | gMsglog.err(s) 43 | 44 | def get_msgs(): 45 | return gMsglog.get_msgs() 46 | 47 | def get_errs(): 48 | return gMsglog.get_errs() 49 | -------------------------------------------------------------------------------- /output_fmt.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | ################################################################## 4 | # OUTPUT GENERATING FUNCTIONS FOR FEEDME 5 | # Define functions for each output format you need to support. 6 | # 7 | 8 | def run_conversion_cmd(appargs): 9 | if verbose: 10 | cmd = " ".join(appargs) 11 | print("Running:", cmd, file=sys.stderr) 12 | sys.stdout.flush() 13 | 14 | retval = os.spawnvp(os.P_WAIT, appargs[0], appargs) 15 | #retval = os.system(cmd) 16 | if retval != 0: 17 | raise OSError(retval, "Couldn't run: " + ' '.join(appargs)) 18 | 19 | # 20 | # Generate a Plucker file 21 | # 22 | def make_plucker_file(indexfile, feedname, levels, ascii): 23 | day = time.strftime("%a") 24 | docname = day + " " + feedname 25 | cleanfilename = day + "_" + feedname.replace(" ", "_") 26 | 27 | # Make sure the plucker directory exists: 28 | pluckerdir = os.path.join(expanduser("~/.plucker"), "feedme") 29 | if not os.path.exists(pluckerdir): 30 | os.makedirs(pluckerdir) 31 | 32 | # Run plucker. This should eventually be configurable -- 33 | # but how, with arguments like these? 34 | 35 | # Plucker mysteriously creates unbeamable files if the 36 | # document name has a colons in it. 37 | # So use the less pretty but safer underscored docname. 38 | #docname = cleanfilename 39 | appargs = [ "plucker-build", "-N", docname, 40 | "-f", os.path.join("feedme", cleanfilename), 41 | "--stayonhost", "--noimages", 42 | "--maxdepth", str(levels), 43 | "--zlib-compression", "--beamable", 44 | "-H", "file://" + indexfile ] 45 | if not ascii: 46 | appargs.append("--charset=utf-8") 47 | 48 | run_conversion_cmd(appargs) 49 | 50 | # 51 | # http://calibre-ebook.com/user_manual/conversion.html 52 | # 53 | def make_calibre_file(indexfile, feedname, extension, levels, ascii, 54 | author, flags): 55 | day = time.strftime("%a") 56 | # Prepend daynum to the filename because fbreader can only sort by filename 57 | #daynum = time.strftime("%w") 58 | cleanfilename = day + "_" + feedname.replace(" ", "_") 59 | outdir = os.path.join(utils.g_config.get('DEFAULT', 'dir'), extension[1:]) 60 | if not os.access(outdir, os.W_OK): 61 | os.makedirs(outdir) 62 | 63 | appargs = [ "ebook-convert", 64 | indexfile, 65 | #os.path.join(expanduser("~/feeds"), 66 | # cleanfilename + extension), 67 | # directory should be configurable too, probably 68 | os.path.join(outdir, cleanfilename + extension), 69 | "--authors", author ] 70 | for flag in flags: 71 | appargs.append(flag) 72 | if verbose: 73 | cmd = " ".join(appargs) 74 | print("Running:", cmd, file=sys.stderr) 75 | sys.stdout.flush() 76 | 77 | run_conversion_cmd(appargs) 78 | 79 | # 80 | # Generate a fictionbook2 file 81 | # 82 | def make_fb2_file(indexfile, feedname, levels, ascii): 83 | make_calibre_file(indexfile, feedname, ".fb2", levels, ascii, 84 | "feedme", flags = [ "--disable-font-rescaling" ] ) 85 | 86 | # 87 | # Generate an ePub file 88 | # http://calibre-ebook.com/user_manual/cli/ebook-convert-3.html#html-input-to-epub-output 89 | # XXX Would be nice to have a way to do this without needing calibre, 90 | # so it could run on servers that don't have X/Qt libraries installed. 91 | # 92 | def make_epub_file(indexfile, feedname, levels, ascii): 93 | make_calibre_file(indexfile, feedname, ".epub", levels, ascii, 94 | time.strftime("%m-%d %a") + " feeds", 95 | flags = [ '--no-default-epub-cover', 96 | '--dont-split-on-page-breaks' ]) 97 | 98 | # END OUTPUT GENERATING FUNCTIONS 99 | ################################################################## 100 | 101 | -------------------------------------------------------------------------------- /siteconf/404media.conf: -------------------------------------------------------------------------------- 1 | [404 Media] 2 | url = https://www.404media.co/rss/ 3 | # Level 2 requires a subscription. Level 1 or 1.5 is a good preview 4 | # to see if you like the feed enough to want a subscription. 5 | levels = 1.5 6 | rss_entry_size = 1600 7 | skip_images = false 8 | skip_nodes = div class="subscribe__content" 9 | div class="kg-bookmark-content" 10 | 11 | 12 | -------------------------------------------------------------------------------- /siteconf/LWN.conf: -------------------------------------------------------------------------------- 1 | [Linux Weekly News] 2 | url = https://lwn.net/headlines/newrss 3 | levels = 1 4 | -------------------------------------------------------------------------------- /siteconf/a-word-a-day.conf: -------------------------------------------------------------------------------- 1 | [A Word A Day] 2 | url = http://wordsmith.org/awad/rss1.xml 3 | page_start = 4 | page_end =

5 | skip_pats = style=".*?" 6 | Follow us on.*?

7 | skip_images = false 8 | 9 | -------------------------------------------------------------------------------- /siteconf/ars-technica.conf: -------------------------------------------------------------------------------- 1 | [Ars Technica] 2 | url = https://feeds.arstechnica.com/arstechnica/index 3 | skip_images = false 4 | page_start = 5 | # page_end was getting skipped because the html gets reformatted at some point, 6 | # with the class on a different line from the div 7 | page_end = ]*> 5 |

7 |

8 | Show more 9 | Related Topics 10 |

.*?End Quote 12 |

13 |

.*?

14 |

15 |

17 | 18 | -------------------------------------------------------------------------------- /siteconf/bespacific.conf: -------------------------------------------------------------------------------- 1 | [Be Spacific] 2 | url = https://www.bespacific.com/feed/ 3 | # individual pages just duplicate what was on the feed 4 | levels = 1 5 | # page_start =

]*> 6 | # page_end =

]*> 8 | -------------------------------------------------------------------------------- /siteconf/boomtownla.conf: -------------------------------------------------------------------------------- 1 | [Boomtown Los Alamos] 2 | url = https://www.boomtownlosalamos.org/feed 3 | levels = 1.5 4 | # Rhe initial image seems to take up a lot of characters, so use 5 | # a large entry_size 6 | rss_entry_size = 2800 7 | skip_images = false 8 | nonlocal_images = true 9 | -------------------------------------------------------------------------------- /siteconf/cartalk.conf: -------------------------------------------------------------------------------- 1 | [Car Talk] 2 | url = http://www.cartalk.com/sites/default/files/rss/tomray.xml 3 | page_start =

6 | page_end =

7 | levels = 1 8 | -------------------------------------------------------------------------------- /siteconf/comics/dinosaur-comics.conf: -------------------------------------------------------------------------------- 1 | [Dinosaur Comics] 2 | url = https://www.comicsrss.com/rss/dinosaur-comics.rss 3 | skip_images = false 4 | nonlocal_images = true 5 | levels = 1 6 | -------------------------------------------------------------------------------- /siteconf/comics/foxtrot.conf: -------------------------------------------------------------------------------- 1 | [Foxtrot] 2 | url = https://www.comicsrss.com/rss/foxtrot.rss 3 | skip_images = false 4 | nonlocal_images = true 5 | levels = 1 6 | -------------------------------------------------------------------------------- /siteconf/comics/smbc.conf: -------------------------------------------------------------------------------- 1 | [Saturday Morning Breakfast Cereal] 2 | url = https://www.smbc-comics.com/comic/rss 3 | skip_images = false 4 | levels = 1 5 | -------------------------------------------------------------------------------- /siteconf/comics/theoatmeal.conf: -------------------------------------------------------------------------------- 1 | [The Oatmeal] 2 | url = https://theoatmeal.com/feed/rss 3 | skip_images = false 4 | nonlocal_images = true 5 | page_start =

6 | page_end =

7 |

14 | page_end =

6 | page_end = 7 |

]*> 4 | page_end =

5 | rss_entry_size = 1300 6 | skip_images = false 7 | -------------------------------------------------------------------------------- /siteconf/feedme.conf: -------------------------------------------------------------------------------- 1 | # Default configuration file for Feedme 1.0. 2 | 3 | # Mandatory for all feeds: name [section name], url. 4 | # dir is also mandatory but typically inherited from the defaults. 5 | 6 | # page_start and page_end will drop anything before and after those patterns 7 | # (the patterns themselves will be kept). Normal Python re syntax. 8 | 9 | # If a URL matching single_page_pat is found in the story page and 10 | # can be loaded, it will be subsituted for the normal story page. 11 | 12 | # All skip_pats can include multiple patterns -- separate with newlines/indents. 13 | 14 | # You can put site entries in this file, but it's better to keep 15 | # separate sitename.conf files, which makes it easier to update, 16 | # enable or disable specific sites. 17 | 18 | [DEFAULT] 19 | # Defaults for all feeds: may be overridden for specific feeds. 20 | 21 | # Where to save output 22 | dir = ~/feeds 23 | 24 | # Print verbose messages to stderr and to the log file? 25 | verbose = True 26 | 27 | # How many levels to fetch. Level 1 means only fetch the main RSS file. 28 | # Level 2 means follow links to stories. There is no level 3. 29 | levels = 2 30 | 31 | # Skip images in stories and show only text? 32 | skip_images = true 33 | 34 | # If not skipping images, should you include images that aren't from 35 | # the same site as the RSS file? On most sites, setting this to true 36 | # means you'll see a lot of advertising images. But in some cases 37 | # you may want to allow images from everywhere. 38 | # For a site that always serves images from a few domains that aren't 39 | # the same as the domain that hosts the RSS, 40 | # you can specify those domains with alt_domains. 41 | nonlocal_images = false 42 | 43 | # If we're not allowing nonlocal images, should we replace nonlocal image 44 | # links with something bogus, to prevent unexpected data use? 45 | # You'll probably want this to be false if you have an unlimited data plan, 46 | # true if you have to pay for data use. 47 | block_nonlocal = false 48 | 49 | # On sites that use srcset for images, what's the maximum pixel width 50 | # we should pretend to have? 51 | max_srcset_size = 800 52 | 53 | # Some sites have crazy-big images. If you're reading on a phone, 54 | # there's no need to wait for downloading something that big: 55 | # resize it locally. This is in pixels. Set to 0 if you want big images. 56 | max_image_size = 1200 57 | 58 | # Skip all links, showing only the text? 59 | # Most people will want this to be false. 60 | skip_links = false 61 | 62 | # Convert special/accented characters to plain ASCII (requires ununicode.py). 63 | ascii = false 64 | 65 | # Log file 66 | logfile = ~/feedme.out 67 | 68 | # formats could be plucker, epub or fbreader. 69 | # Use none if you only need HTML. 70 | formats = none 71 | 72 | # How long to save fetched files before cleaning up. 73 | save_days = 7 74 | 75 | # Use gzip-compresseed http if the site supports it? 76 | # (Some sites have broken compression.) 77 | allow_gzip = true 78 | 79 | -------------------------------------------------------------------------------- /siteconf/freakonomics.conf: -------------------------------------------------------------------------------- 1 | [Freakonomics] 2 | url = https://freakonomics.com/feed/ 3 | page_start =

4 | page_end =

5 | rss_entry_size = 1300 6 | -------------------------------------------------------------------------------- /siteconf/freedom-to-tinker.conf: -------------------------------------------------------------------------------- 1 | [Freedom to Tinker] 2 | url = https://freedom-to-tinker.com/feed/ 3 | page_end = 7 | -------------------------------------------------------------------------------- /siteconf/high-country-news.conf: -------------------------------------------------------------------------------- 1 | [High Country News] 2 | # url = https://www.hcn.org/feed/ 3 | url = https://www.hcn.org/feed/ 4 | page_start =

5 | page_end =

6 | skip_images = false 7 | alt_domains = i0.wp.com 8 | skip_nodes = div class="newspack-popup-container" 9 | -------------------------------------------------------------------------------- /siteconf/jemezplants.conf: -------------------------------------------------------------------------------- 1 | [Jemez Mountain Plants] 2 | url = https://jemezmountainplants.wordpress.com/rss 3 | skip_images = false 4 | page_start =

5 | page_end =

6 | rss_entry_size = 1200 7 | alt_domains = jemezmountainplants.files.wordpress.com 8 | 9 | -------------------------------------------------------------------------------- /siteconf/justins-gis.conf: -------------------------------------------------------------------------------- 1 | [Justin's GIS Blog] 2 | url = https://justincolegis.com/feed/ 3 | skip_images = false 4 | page_start =

5 | page_end =

4 | page_end =

5 | -------------------------------------------------------------------------------- /siteconf/letters-american.conf: -------------------------------------------------------------------------------- 1 | [Letters From an American] 2 | url = https://heathercoxrichardson.substack.com/feed 3 | levels = 1.5 4 | rss_entry_size = 1600 5 | skip_images = true 6 | allow_dup_titles = true 7 | 8 | 9 | -------------------------------------------------------------------------------- /siteconf/longreads.conf: -------------------------------------------------------------------------------- 1 | [Longreads] 2 | url = http://longreads.com/rss 3 | skip_images = false 4 | nonlocal_images = true 5 | continue_on_timeout = true 6 | page_start =

8 | page_end =

%s

Timeout

Feedme URLs %s

FeedMe: a Lightweight Feed/RSS reader

About RSS

Download Feedme

The feedme.conf file

General options

Defining a new feed

Basic options for specific feeds

Trickier Site Options

Helper Modules

Output Formats

Maintenance:

The Cache, and Fixing Bad Runs

License

Reading Your Feeds