###TITLE###

├── .gitignore ├── templates ├── partial_url.html ├── partial_search_link.html ├── partial_user.html ├── partial_menu_item.html ├── partial_link_selftext.html ├── partial_subreddit_pager_link.html ├── partial_index_subreddit.html ├── partial_comment.html ├── partial_link.html ├── partial_user_link.html ├── index.html ├── user.html ├── search.html ├── subreddit.html └── link.html ├── screenshots ├── sub.jpg └── post.jpg ├── r └── static │ ├── fonts │ ├── S6uyw4BMUTPHjx4wXg.woff2 │ ├── S6uyw4BMUTPHjxAwXjeu.woff2 │ ├── S6u9w4BMUTPHh6UVSwiPGQ.woff2 │ ├── S6u9w4BMUTPHh7USSwiPGQ.woff2 │ └── S6u9w4BMUTPHh6UVSwaPGR_p.woff2 │ ├── js │ ├── archive-comments.js │ ├── bootstrap.min.js │ └── jquery-3.3.1.slim.min.js │ └── css │ ├── lato.css │ └── archive.css ├── LICENSE ├── README.md ├── fetch_links.py └── write_html.py /.gitignore: -------------------------------------------------------------------------------- 1 | /data 2 | /r 3 | !/r/static -------------------------------------------------------------------------------- /templates/partial_url.html: -------------------------------------------------------------------------------- 1 | #INNER_HTML# -------------------------------------------------------------------------------- /templates/partial_search_link.html: -------------------------------------------------------------------------------- 1 | ###TITLE### -------------------------------------------------------------------------------- /templates/partial_user.html: -------------------------------------------------------------------------------- 1 | ###AUTHOR### -------------------------------------------------------------------------------- /templates/partial_menu_item.html: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /screenshots/sub.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/libertysoft3/reddit-html-archiver/HEAD/screenshots/sub.jpg -------------------------------------------------------------------------------- /screenshots/post.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/libertysoft3/reddit-html-archiver/HEAD/screenshots/post.jpg -------------------------------------------------------------------------------- /templates/partial_link_selftext.html: -------------------------------------------------------------------------------- 1 |

###SELFTEXT###

-------------------------------------------------------------------------------- /templates/partial_subreddit_pager_link.html: -------------------------------------------------------------------------------- 1 |

#TEXT#

-------------------------------------------------------------------------------- /r/static/fonts/S6uyw4BMUTPHjx4wXg.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/libertysoft3/reddit-html-archiver/HEAD/r/static/fonts/S6uyw4BMUTPHjx4wXg.woff2 -------------------------------------------------------------------------------- /r/static/fonts/S6uyw4BMUTPHjxAwXjeu.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/libertysoft3/reddit-html-archiver/HEAD/r/static/fonts/S6uyw4BMUTPHjxAwXjeu.woff2 -------------------------------------------------------------------------------- /templates/partial_index_subreddit.html: -------------------------------------------------------------------------------- 1 |

#SUB# #NUM_LINKS#

-------------------------------------------------------------------------------- /r/static/fonts/S6u9w4BMUTPHh6UVSwiPGQ.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/libertysoft3/reddit-html-archiver/HEAD/r/static/fonts/S6u9w4BMUTPHh6UVSwiPGQ.woff2 -------------------------------------------------------------------------------- /r/static/fonts/S6u9w4BMUTPHh7USSwiPGQ.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/libertysoft3/reddit-html-archiver/HEAD/r/static/fonts/S6u9w4BMUTPHh7USSwiPGQ.woff2 -------------------------------------------------------------------------------- /r/static/fonts/S6u9w4BMUTPHh6UVSwaPGR_p.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/libertysoft3/reddit-html-archiver/HEAD/r/static/fonts/S6u9w4BMUTPHh6UVSwaPGR_p.woff2 -------------------------------------------------------------------------------- /templates/partial_comment.html: -------------------------------------------------------------------------------- 1 |

2 |

###SCORE### ###HTML_AUTHOR_URL### ###DATE###

3 |

###BODY###

4 |

-------------------------------------------------------------------------------- /templates/partial_link.html: -------------------------------------------------------------------------------- 1 |

2 |

###TITLE###

3 | ###SCORE### ###NUM_COMMENTS### comments ###DATE### ###HTML_AUTHOR_URL### ###LINK_DOMAIN### 4 |

-------------------------------------------------------------------------------- /templates/partial_user_link.html: -------------------------------------------------------------------------------- 1 |

2 |

###TITLE###

3 | ###SCORE### ###NUM_COMMENTS### comments ###DATE### ###HTML_AUTHOR_URL### in r/###SUB### 4 |

-------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 libertysoft3 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /r/static/js/archive-comments.js: -------------------------------------------------------------------------------- 1 | $(document).ready(function() { 2 | $('a.score').click(function(){ 3 | var $comment = $(this).closest('.comment'); 4 | var depth = parseInt($comment.data('depth')); 5 | if ($comment.hasClass('collapsed')) { 6 | $comment.removeClass('collapsed'); 7 | $check_comment = $comment.next('.comment'); 8 | depth_sibling = false; 9 | while (!depth_sibling) { 10 | if ($check_comment.length == 0) { 11 | depth_sibling = true; 12 | } 13 | if ($check_comment.data('depth') == '' || parseInt($check_comment.data('depth')) <= depth) { 14 | depth_sibling = true; 15 | } else { 16 | $check_comment.removeClass('hidden'); 17 | } 18 | $check_comment = $check_comment.next('.comment'); 19 | } 20 | } else { 21 | $comment.addClass('collapsed'); 22 | $check_comment = $comment.next('.comment'); 23 | depth_sibling = false; 24 | while (!depth_sibling) { 25 | if ($check_comment.length == 0) { 26 | depth_sibling = true; 27 | } 28 | if ($check_comment.data('depth') == '' || parseInt($check_comment.data('depth')) <= depth) { 29 | depth_sibling = true; 30 | } else { 31 | $check_comment.addClass('hidden'); 32 | } 33 | $check_comment = $check_comment.next('.comment'); 34 | } 35 | } 36 | }); 37 | }); -------------------------------------------------------------------------------- /templates/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | ###TITLE### 10 | 11 | 12 |

13 | 30 |

31 |

32 |

33 | ###HTML_LINKS### 34 |

35 |

36 | 39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /templates/user.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | ###TITLE### 10 | 11 | 12 |

13 | 30 |

31 |

32 |

33 | ###HTML_LINKS### 34 |

35 |

36 | 39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /r/static/css/lato.css: -------------------------------------------------------------------------------- 1 | /* latin-ext */ 2 | @font-face { 3 | font-family: 'Lato'; 4 | font-style: normal; 5 | font-weight: 300; 6 | src: local('Lato Light'), local('Lato-Light'), url(../fonts/S6u9w4BMUTPHh7USSwaPGR_p.woff2) format('woff2'); 7 | unicode-range: U+0100-024F, U+0259, U+1E00-1EFF, U+2020, U+20A0-20AB, U+20AD-20CF, U+2113, U+2C60-2C7F, U+A720-A7FF; 8 | } 9 | /* latin */ 10 | @font-face { 11 | font-family: 'Lato'; 12 | font-style: normal; 13 | font-weight: 300; 14 | src: local('Lato Light'), local('Lato-Light'), url(../fonts/S6u9w4BMUTPHh7USSwiPGQ.woff2) format('woff2'); 15 | unicode-range: U+0000-00FF, U+0131, U+0152-0153, U+02BB-02BC, U+02C6, U+02DA, U+02DC, U+2000-206F, U+2074, U+20AC, U+2122, U+2191, U+2193, U+2212, U+2215, U+FEFF, U+FFFD; 16 | } 17 | /* latin-ext */ 18 | @font-face { 19 | font-family: 'Lato'; 20 | font-style: normal; 21 | font-weight: 400; 22 | src: local('Lato Regular'), local('Lato-Regular'), url(../fonts/S6uyw4BMUTPHjxAwXjeu.woff2) format('woff2'); 23 | unicode-range: U+0100-024F, U+0259, U+1E00-1EFF, U+2020, U+20A0-20AB, U+20AD-20CF, U+2113, U+2C60-2C7F, U+A720-A7FF; 24 | } 25 | /* latin */ 26 | @font-face { 27 | font-family: 'Lato'; 28 | font-style: normal; 29 | font-weight: 400; 30 | src: local('Lato Regular'), local('Lato-Regular'), url(../fonts/S6uyw4BMUTPHjx4wXg.woff2) format('woff2'); 31 | unicode-range: U+0000-00FF, U+0131, U+0152-0153, U+02BB-02BC, U+02C6, U+02DA, U+02DC, U+2000-206F, U+2074, U+20AC, U+2122, U+2191, U+2193, U+2212, U+2215, U+FEFF, U+FFFD; 32 | } 33 | /* latin-ext */ 34 | @font-face { 35 | font-family: 'Lato'; 36 | font-style: normal; 37 | font-weight: 700; 38 | src: local('Lato Bold'), local('Lato-Bold'), url(../fonts/S6u9w4BMUTPHh6UVSwaPGR_p.woff2) format('woff2'); 39 | unicode-range: U+0100-024F, U+0259, U+1E00-1EFF, U+2020, U+20A0-20AB, U+20AD-20CF, U+2113, U+2C60-2C7F, U+A720-A7FF; 40 | } 41 | /* latin */ 42 | @font-face { 43 | font-family: 'Lato'; 44 | font-style: normal; 45 | font-weight: 700; 46 | src: local('Lato Bold'), local('Lato-Bold'), url(../fonts/S6u9w4BMUTPHh6UVSwiPGQ.woff2) format('woff2'); 47 | unicode-range: U+0000-00FF, U+0131, U+0152-0153, U+02BB-02BC, U+02C6, U+02DA, U+02DC, U+2000-206F, U+2074, U+20AC, U+2122, U+2191, U+2193, U+2212, U+2215, U+FEFF, U+FFFD; 48 | } 49 | -------------------------------------------------------------------------------- /r/static/css/archive.css: -------------------------------------------------------------------------------- 1 | footer { 2 | text-align: center; 3 | } 4 | 5 | /* markdown */ 6 | .md { 7 | word-wrap: break-word; 8 | overflow-wrap: break-word; 9 | } 10 | .md p, .md ol, .md ul, .md blockquote { 11 | margin: 3px 0; 12 | } 13 | .md blockquote { 14 | border-left: 2px solid rgba(255, 255, 255, 0.4); /* text-muted */ 15 | padding-left: 0.5rem; 16 | } 17 | .md blockquote, .md del { 18 | color: rgba(255, 255, 255, 0.4); /* text-muted */ 19 | } 20 | .md code, .md pre { 21 | border: 1px solid #4E5D6C; /* alert-secondary */ 22 | background: #4E5D6C; /* disabled form input color */ 23 | } 24 | .md h1 { 25 | font-size: 1.5rem; 26 | } 27 | .md h2 { 28 | font-size: 1.4rem; 29 | } 30 | .md h3 { 31 | font-size: 1.3rem; 32 | } 33 | .md h4 { 34 | font-size: 1.2rem; 35 | } 36 | .md h5 { 37 | font-size: 1.1rem; 38 | } 39 | .md h6 { 40 | font-size: 1rem; 41 | font-weight: bold; 42 | } 43 | 44 | /* subreddit links */ 45 | .links .title { 46 | line-height: 1.25; 47 | } 48 | .links .title a, .submission .title a { 49 | color: inherit; 50 | } 51 | .search .title { 52 | color: inherit; 53 | display: block; 54 | } 55 | 56 | /* link/post page */ 57 | .op .author, .submission .author { 58 | color: #5bc0de; /* match bootstrap link color / badge-primary */ 59 | } 60 | .submission .card-body { 61 | padding: 1rem; 62 | } 63 | 64 | /* comments */ 65 | .comment { 66 | margin-bottom: 5px; /*1rem;*/ 67 | } 68 | .comment .byline, .comment .byline a { 69 | line-height: 1; 70 | } 71 | .collapsed .md { 72 | display: none; 73 | } 74 | .hidden { 75 | display: none; 76 | } 77 | .to-top { 78 | display: block; 79 | } 80 | .comments .ml-1 { 81 | margin-left: 1rem !important; 82 | } 83 | .comments .ml-2 { 84 | margin-left: 2rem !important; 85 | } 86 | .comments .ml-3 { 87 | margin-left: 3rem !important; 88 | } 89 | .comments .ml-4 { 90 | margin-left: 4rem !important; 91 | } 92 | .comments .ml-5 { 93 | margin-left: 5rem !important; 94 | } 95 | .comments .ml-6 { 96 | margin-left: 6rem !important; 97 | } 98 | .comments .ml-7 { 99 | margin-left: 7rem !important; 100 | } 101 | .comments .ml-8 { 102 | margin-left: 8rem !important; 103 | } -------------------------------------------------------------------------------- /templates/search.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | r/###SUB### ###TITLE### 10 | 11 | 12 |

13 |

14 | 15 | 18 | 41 |

42 |

43 |

44 |

45 | ###HTML_LINKS### 46 |

47 |

48 | 51 | 52 | 53 | 54 | -------------------------------------------------------------------------------- /templates/subreddit.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | r/###SUB### ###TITLE### 10 | 11 | 12 |

13 |

14 | 15 | 18 | 41 |

42 |

43 |

44 |

45 | ###HTML_PAGER### 46 | 47 |

48 | ###HTML_LINKS### 49 |

50 |

51 | ###HTML_PAGER### 52 | 53 |

54 | 57 | 58 | 59 | 60 | -------------------------------------------------------------------------------- /templates/link.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | r/###SUB###: ###TITLE### 10 | 11 | 12 |

13 |

14 | 15 | 18 | 41 |

42 |

43 |

44 |

45 |

###HTML_TITLE###

46 |

###SCORE### ###DATE### by ###HTML_AUTHOR_URL###

47 | ###HTML_SELFTEXT### 48 |

49 |

50 |

###NUM_COMMENTS### comments

51 | ###HTML_COMMENTS### 52 |

53 |

54 | 58 | 59 | 60 | 61 | 62 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## reddit html archiver 2 | 3 | pulls reddit data from the [pushshift](https://github.com/pushshift/api) api and renders offline compatible html pages. uses the reddit markdown renderer. 4 | 5 | ### install 6 | 7 | requires python 3 on linux, OSX, or Windows. 8 | 9 | **warning:** if `$ python --version` outputs a python 2 version on your system, then you need to replace all occurances of `python` with `python3` in the commands below. 10 | 11 | $ sudo apt-get install pip 12 | $ pip install psaw -U 13 | $ git clone https://github.com/chid/snudown 14 | $ cd snudown 15 | $ sudo python setup.py install 16 | $ cd .. 17 | $ git clone [this repo] 18 | $ cd reddit-html-archiver 19 | $ chmod u+x *.py 20 | 21 | Windows users may need to run 22 | 23 | > chcp 65001 24 | > set PYTHONIOENCODING=utf-8 25 | 26 | before running `fetch_links.py` or `write_html.py` to resolve encoding errors such as 'codec can't encode character'. 27 | 28 | ### fetch reddit data 29 | 30 | fetch data by subreddit and date range, writing to csv files in `data`: 31 | 32 | $ python ./fetch_links.py politics 2017-1-1 2017-2-1 33 | 34 | or you can filter links/posts to download less data: 35 | 36 | $ python ./fetch_links.py --self_only --score "> 2000" politics 2015-1-1 2016-1-1 37 | 38 | to show all available options and filters run: 39 | 40 | $ python ./fetch_links.py -h 41 | 42 | decrease your date range or adjust `pushshift_rate_limit_per_minute` in `fetch_links.py` if you are getting connection errors. 43 | 44 | ### write web pages 45 | 46 | write html files for all subreddits to `r`: 47 | 48 | $ python ./write_html.py 49 | 50 | you can add some output filtering to have less empty postssmaller archive size 51 | 52 | $ python ./write_html.py --min-score 100 --min-comments 100 --hide-deleted-comments 53 | 54 | to show all available filters run: 55 | 56 | $ python ./write_html.py -h 57 | 58 | your html archive has been written to `r`. once you are satisfied with your archive feel free to copy/move the contents of `r` to elsewhere and to delete the git repos you have created. everything in `r` is fully self contained. 59 | 60 | to update an html archive, delete everything in `r` aside from `r/static` and re-run `write_html.py` to regenerate everything. 61 | 62 | ### hosting the archived pages 63 | 64 | copy the contents of the `r` directory to a web root or appropriately served git repo. 65 | 66 | ### potential improvements 67 | 68 | * fetch_links 69 | * num_comments filtering 70 | * thumbnails or thumbnail urls 71 | * media posts 72 | * score update 73 | * scores from reddit with [praw](https://github.com/praw-dev/praw) 74 | * real templating 75 | * choose [Bootswatch](https://bootswatch.com/) theme 76 | * specify subreddits to output 77 | * show link domain/post type 78 | * user pages 79 | * add pagination, posts sorted by score, comments, date, sub 80 | * too many files in one directory 81 | * view on reddit.com 82 | * js powered search page, show no links by default 83 | * js inline media embeds/expandos 84 | * archive.org links 85 | 86 | ### see also 87 | 88 | * [pushshift](https://github.com/pushshift/api), [r/pushshift](https://www.reddit.com/r/pushshift/) 89 | * [psaw](https://github.com/dmarx/psaw) 90 | * [snudown](https://github.com/reddit/snudown) 91 | * [redditsearch.io](https://redditsearch.io/) 92 | * [reddit post archiver](https://github.com/sJohnsonStoever/redditPostArchiver) 93 | * [reddit downloader](https://github.com/shadowmoose/RedditDownloader) 94 | 95 | ### screenshots 96 | 97 | ![](screenshots/sub.jpg) 98 | ![](screenshots/post.jpg) 99 | -------------------------------------------------------------------------------- /fetch_links.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | import time 3 | from time import mktime 4 | from datetime import datetime, timedelta 5 | import argparse 6 | from pprint import pprint 7 | import json 8 | import csv 9 | import os 10 | from psaw import PushshiftAPI 11 | 12 | pushshift_rate_limit_per_minute = 20 13 | max_comments_per_query = 150 14 | write_every = 10 15 | 16 | link_fields = ['author', 'created_utc', 'domain', 'id', 'is_self', 17 | 'num_comments', 'over_18', 'permalink', 'retrieved_on', 'score', 18 | 'selftext', 'stickied', 'subreddit_id', 'title', 'url'] 19 | comment_fields = ['author', 'body', 'created_utc', 'id', 'link_id', 20 | 'parent_id', 'score', 'stickied', 'subreddit_id'] 21 | 22 | def fetch_links(subreddit=None, date_start=None, date_stop=None, limit=None, score=None, self_only=False): 23 | if subreddit is None or date_start is None or date_stop is None: 24 | print('ERROR: missing required arguments') 25 | exit() 26 | 27 | api = PushshiftAPI(rate_limit_per_minute=pushshift_rate_limit_per_minute, detect_local_tz=False) 28 | 29 | # get links 30 | links = [] 31 | print('fetching submissions %s to %s...' % (time.strftime('%Y-%m-%d', date_start), time.strftime('%Y-%m-%d', date_stop))) 32 | params = { 33 | 'after': int(mktime(date_start)) - 86400, # make date inclusive, adjust for UTC 34 | 'before': int(mktime(date_stop)) + 86400, 35 | 'subreddit': subreddit, 36 | 'filter': link_fields, 37 | 'sort': 'asc', 38 | 'sort_type': 'created_utc', 39 | } 40 | if limit: 41 | params['limit'] = int(limit) 42 | if score: 43 | params['score'] = score 44 | if self_only: 45 | params['is_self'] = True 46 | link_results = list(api.search_submissions(**params)) 47 | print('processing %s links' % len(link_results)) 48 | for s in link_results: 49 | # print('%s %s' % (datetime.utcfromtimestamp(int(s.d_['created_utc'])), s.d_['title'])) 50 | # pprint(s) 51 | 52 | # get comment ids 53 | comments = [] 54 | if s.d_['num_comments'] > 0 and not comment_data_exists(subreddit, s.d_['created_utc'], s.d_['id']): 55 | comment_ids = list(api._get_submission_comment_ids(s.d_['id'])) 56 | # print('%s comment_ids: %s' % (data['id'], comment_ids)) 57 | 58 | # get comments 59 | if (len(comment_ids) > 0): 60 | mychunks = [] 61 | if len(comment_ids) > max_comments_per_query: 62 | mychunks = chunks(comment_ids, max_comments_per_query) 63 | else: 64 | mychunks = [comment_ids] 65 | for chunk in mychunks: 66 | comment_params = { 67 | 'filter': comment_fields, 68 | 'ids': ','.join(chunk), 69 | 'limit': max_comments_per_query, 70 | } 71 | comments_results = list(api.search_comments(**comment_params)) 72 | print('%s fetch link %s comments %s/%s' % (datetime.utcfromtimestamp(int(s.d_['created_utc'])), s.d_['id'], len(comments_results), len(comment_ids))) 73 | for c in comments_results: 74 | comments.append(c.d_) 75 | 76 | s.d_['comments'] = comments 77 | links.append(s.d_) 78 | 79 | # write results 80 | if len(links) >= write_every: 81 | success = write_links(subreddit, links) 82 | if success: 83 | links = [] 84 | 85 | # write remining results 86 | if len(links): 87 | write_links(subreddit, links) 88 | 89 | # csvs are not guaranteed to be sorted by date but you can resume broken runs 90 | # and change sort criteria later to add more posts without getting duplicates. 91 | # delete csvs and re-run to update existing posts 92 | def write_links(subreddit, links): 93 | if links and len(links) > 0: 94 | writing_day = None 95 | file = None 96 | writer = None 97 | existing_link_ids = [] 98 | wrote_links = 0 99 | wrote_comments = 0 100 | 101 | for r in links: 102 | # print('%s link %s' % (r['id'], r['title'])) 103 | 104 | # grab link comments 105 | existing_comment_ids = [] 106 | comments = r['comments'] 107 | # print('%s comments %s' % (r['id'], comments)) 108 | 109 | created_ts = int(r['created_utc']) 110 | created = datetime.utcfromtimestamp(created_ts).strftime('%Y-%m-%d') 111 | created_path = datetime.utcfromtimestamp(created_ts).strftime('%Y/%m/%d') 112 | if created != writing_day: 113 | if file: 114 | file.close() 115 | writing_day = created 116 | path = 'data/' + subreddit + '/' + created_path 117 | os.makedirs(path, exist_ok=True) 118 | 119 | # create and parse existing links 120 | filename = 'links.csv' 121 | filepath = path + '/' + filename 122 | if not os.path.isfile(filepath): 123 | file = open(filepath, 'a', encoding='utf-8') 124 | writer = csv.DictWriter(file, fieldnames=link_fields) 125 | writer.writeheader() 126 | # print('created %s' % filepath) 127 | else: 128 | with open(filepath, 'r', encoding='utf-8') as file: 129 | reader = csv.DictReader(file) 130 | for row in reader: 131 | existing_link_ids.append(row['id']) 132 | 133 | file = open(filepath, 'a', encoding='utf-8') 134 | writer = csv.DictWriter(file, fieldnames=link_fields) 135 | 136 | # create and parse existing comments 137 | # writing empty comments csvs resuming and comment_data_exists() 138 | filename = r['id'] + '.csv' 139 | filepath = path + '/' + filename 140 | if not os.path.isfile(filepath): 141 | comments_file = open(filepath, 'a', encoding='utf-8') 142 | comments_writer = csv.DictWriter(comments_file, fieldnames=comment_fields) 143 | comments_writer.writeheader() 144 | # print('created %s' % filepath) 145 | else: 146 | with open(filepath, 'r', encoding='utf-8') as comments_file: 147 | reader = csv.DictReader(comments_file) 148 | for row in reader: 149 | existing_comment_ids.append(row['id']) 150 | 151 | comments_file = open(filepath, 'a', encoding='utf-8') 152 | comments_writer = csv.DictWriter(comments_file, fieldnames=comment_fields) 153 | 154 | # write link row 155 | if r['id'] not in existing_link_ids: 156 | for field in list(r): 157 | if field not in link_fields: 158 | del r[field] 159 | 160 | writer.writerow(r) 161 | wrote_links += 1 162 | 163 | # write comments 164 | for c in comments: 165 | if c['id'] not in existing_comment_ids: 166 | for field in list(c): 167 | if field not in comment_fields: 168 | del c[field] 169 | comments_writer.writerow(c) 170 | wrote_comments += 1 171 | comments_file.close() 172 | 173 | 174 | print('got %s links, wrote %s and %s comments' % (len(links), wrote_links, wrote_comments)) 175 | return True 176 | 177 | def link_data_exists(subreddit, date): 178 | created_path = time.strftime('%Y/%m/%d', date) 179 | path = 'data/' + subreddit + '/' + created_path + '/links.csv' 180 | if not os.path.isfile(path): 181 | return False 182 | return True 183 | 184 | def comment_data_exists(subreddit, link_created_utc, link_id): 185 | created_ts = int(link_created_utc) 186 | created_path = datetime.utcfromtimestamp(created_ts).strftime('%Y/%m/%d') 187 | path = 'data/' + subreddit + '/' + created_path + '/' + link_id + '.csv' 188 | if os.path.isfile(path): 189 | return True 190 | return False 191 | 192 | def chunks(l, n): 193 | """Yield successive n-sized chunks from l.""" 194 | for i in range(0, len(l), n): 195 | yield l[i:i + n] 196 | 197 | def mkdate(datestr): 198 | try: 199 | return time.strptime(datestr, '%Y-%m-%d') 200 | except ValueError: 201 | raise argparse.ArgumentTypeError(datestr + ' is not a proper date string') 202 | 203 | if __name__ == '__main__': 204 | parser=argparse.ArgumentParser() 205 | parser.add_argument('subreddit', help='subreddit to archive') 206 | parser.add_argument('date_start', type=mkdate, help='start archiving at date, e.g. 2005-1-1') 207 | parser.add_argument('date_stop', type=mkdate, help='stop archiving at date, inclusive, cannot be date_start') 208 | parser.add_argument('--limit', default=None, help='pushshift api limit param, default None') 209 | parser.add_argument('--score', default=None, help='pushshift api score param, e.g. "> 10", default None') 210 | parser.add_argument('--self_only', action="store_true", help='only fetch selftext submissions, default False') 211 | args=parser.parse_args() 212 | 213 | self_only = False 214 | if args.self_only: 215 | self_only = True 216 | 217 | args.subreddit = args.subreddit.lower() 218 | 219 | fetch_links(args.subreddit, args.date_start, args.date_stop, args.limit, args.score, self_only) 220 | -------------------------------------------------------------------------------- /write_html.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | from datetime import datetime, date, timedelta 3 | import argparse 4 | import csv 5 | import os 6 | import re 7 | import snudown 8 | import psutil 9 | 10 | url_project = 'https://github.com/libertysoft3/reddit-html-archiver' 11 | links_per_page = 30 12 | pager_skip = 10 13 | pager_skip_long = 100 14 | start_date = date(2005, 1, 1) 15 | end_date = datetime.today().date() + timedelta(days=1) 16 | source_data_links = 'links.csv' 17 | max_comment_depth = 8 # mostly for mobile, which might be silly 18 | removed_content_identifiers = ['[deleted]','deleted','[removed]','removed'] 19 | default_sort = 'score' 20 | sort_indexes = { 21 | 'score': { 22 | 'default': 1, 23 | 'slug': 'score' 24 | }, 25 | 'num_comments': { 26 | 'default': 0, 27 | 'slug': 'comments', 28 | }, 29 | 'created_utc': { 30 | 'default': 1000198000, 31 | 'slug': 'date', 32 | } 33 | } 34 | missing_comment_score_label = 'n/a' 35 | 36 | template_index = '' 37 | with open('templates/index.html', 'r', encoding='utf-8') as file: 38 | template_index = file.read() 39 | 40 | template_subreddit = '' 41 | with open('templates/subreddit.html', 'r', encoding='utf-8') as file: 42 | template_subreddit = file.read() 43 | 44 | template_link = '' 45 | with open('templates/link.html', 'r', encoding='utf-8') as file: 46 | template_link = file.read() 47 | 48 | template_comment = '' 49 | with open('templates/partial_comment.html', 'r', encoding='utf-8') as file: 50 | template_comment = file.read() 51 | 52 | template_search = '' 53 | with open('templates/search.html', 'r', encoding='utf-8') as file: 54 | template_search = file.read() 55 | 56 | template_user = '' 57 | with open('templates/user.html', 'r', encoding='utf-8') as file: 58 | template_user = file.read() 59 | 60 | template_sub_link = '' 61 | with open('templates/partial_menu_item.html', 'r', encoding='utf-8') as file: 62 | template_sub_link = file.read() 63 | 64 | template_user_url = '' 65 | with open('templates/partial_user.html', 'r', encoding='utf-8') as file: 66 | template_user_url = file.read() 67 | 68 | template_link_url = '' 69 | with open('templates/partial_link.html', 'r', encoding='utf-8') as file: 70 | template_link_url = file.read() 71 | 72 | template_search_link = '' 73 | with open('templates/partial_search_link.html', 'r', encoding='utf-8') as file: 74 | template_search_link = file.read() 75 | 76 | template_index_sub = '' 77 | with open('templates/partial_index_subreddit.html', 'r', encoding='utf-8') as file: 78 | template_index_sub = file.read() 79 | 80 | template_index_pager_link = '' 81 | with open('templates/partial_subreddit_pager_link.html', 'r', encoding='utf-8') as file: 82 | template_index_pager_link = file.read() 83 | 84 | template_selftext = '' 85 | with open('templates/partial_link_selftext.html', 'r', encoding='utf-8') as file: 86 | template_selftext = file.read() 87 | 88 | template_user_page_link = '' 89 | with open('templates/partial_user_link.html', 'r', encoding='utf-8') as file: 90 | template_user_page_link = file.read() 91 | 92 | teplate_url = '' 93 | with open('templates/partial_url.html', 'r', encoding='utf-8') as file: 94 | template_url = file.read() 95 | 96 | process = psutil.Process(os.getpid()) 97 | 98 | def generate_html(min_score=0, min_comments=0, hide_deleted_comments=False): 99 | delta = timedelta(days=1) 100 | subs = get_subs() 101 | user_index = {} 102 | processed_subs = [] 103 | stat_links = 0 104 | stat_filtered_links = 0 105 | 106 | for sub in subs: 107 | # write link pages 108 | # print('generate_html() processing %s %s kb' % (sub, int(int(process.memory_info().rss) / 1024))) 109 | stat_sub_links = 0 110 | stat_sub_filtered_links = 0 111 | stat_sub_comments = 0 112 | d = start_date 113 | while d <= end_date: 114 | raw_links = load_links(d, sub, True) 115 | stat_links += len(raw_links) 116 | stat_sub_links += len(raw_links) 117 | for l in raw_links: 118 | if validate_link(l, min_score, min_comments): 119 | write_link_page(subs, l, sub, hide_deleted_comments) 120 | stat_filtered_links += 1 121 | stat_sub_filtered_links += 1 122 | if 'comments' in l: 123 | stat_sub_comments += len(l['comments']) 124 | d += delta 125 | if stat_sub_filtered_links > 0: 126 | processed_subs.append({'name': sub, 'num_links': stat_sub_filtered_links}) 127 | print('%s: %s links filtered to %s' % (sub, stat_sub_links, stat_sub_filtered_links)) 128 | 129 | # write subreddit pages 130 | valid_sub_links = [] 131 | d = start_date 132 | while d <= end_date: 133 | raw_links = load_links(d, sub) 134 | for l in raw_links: 135 | if validate_link(l, min_score, min_comments): 136 | valid_sub_links.append(l) 137 | 138 | # collect links for user pages 139 | # TODO: this is the least performant bit. load and generate user pages user by user instead. 140 | l['subreddit'] = sub 141 | if l['author'] not in user_index.keys(): 142 | user_index[l['author']] = [] 143 | user_index[l['author']].append(l) 144 | d += delta 145 | write_subreddit_pages(sub, subs, valid_sub_links, stat_sub_filtered_links, stat_sub_comments) 146 | write_subreddit_search_page(sub, subs, valid_sub_links, stat_sub_filtered_links, stat_sub_comments) 147 | 148 | # write user pages 149 | write_user_page(processed_subs, user_index) 150 | 151 | # write index page 152 | write_index(processed_subs) 153 | print('all done. %s links filtered to %s' % (stat_links, stat_filtered_links)) 154 | 155 | def write_subreddit_pages(subreddit, subs, link_index, stat_sub_filtered_links, stat_sub_comments): 156 | if len(link_index) == 0: 157 | return True 158 | 159 | for sort in sort_indexes.keys(): 160 | links = sorted(link_index, key=lambda k: (int(k[sort]) if k[sort] != '' else sort_indexes[sort]['default']), reverse=True) 161 | pages = list(chunks(links, links_per_page)) 162 | page_num = 0 163 | 164 | sort_based_prefix = '../' 165 | if sort == default_sort: 166 | sort_based_prefix = '' 167 | 168 | # render subreddits list 169 | subs_menu_html = '' 170 | for sub in subs: 171 | sub_url = sort_based_prefix + '../' + sub + '/index.html' 172 | subs_menu_html += template_sub_link.replace('###URL_SUB###', sub_url).replace('###SUB###', sub) 173 | 174 | for page in pages: 175 | page_num += 1 176 | # print('%s page' % (page)) 177 | 178 | links_html = '' 179 | for l in page: 180 | author_link_html = template_user_url 181 | author_url = sort_based_prefix + '../user/' + l['author'] + '.html' 182 | author_link_html = author_link_html.replace('###URL_AUTHOR###', author_url).replace('###AUTHOR###', l['author']) 183 | 184 | link_url = l['url'] 185 | link_comments_url = sort_based_prefix + l['permalink'].lower().strip('/') 186 | link_comments_url = link_comments_url.replace('r/' + subreddit + '/', '') 187 | idpath = '/'.join(list(l['id'])) 188 | link_comments_url = link_comments_url.replace(l['id'], idpath) 189 | link_comments_url += '.html' 190 | if l['is_self'] is True or l['is_self'] == 'True': 191 | link_url = link_comments_url 192 | 193 | index_link_data_map = { 194 | '###TITLE###': l['title'], 195 | '###URL###': link_url, 196 | '###URL_COMMENTS###': link_comments_url, 197 | '###SCORE###': str(l['score']), 198 | '###NUM_COMMENTS###': l['num_comments'] if int(l['num_comments']) > 0 else str(0), 199 | '###DATE###': datetime.utcfromtimestamp(int(l['created_utc'])).strftime('%Y-%m-%d'), 200 | '###LINK_DOMAIN###': '(self.' + subreddit + ')' if l['is_self'] is True or l['is_self'] == 'True' else '', 201 | '###HTML_AUTHOR_URL###': author_link_html, 202 | } 203 | link_html = template_link_url 204 | for key, value in index_link_data_map.items(): 205 | link_html = link_html.replace(key, value) 206 | links_html += link_html + '\n' 207 | 208 | index_page_data_map = { 209 | '###INCLUDE_PATH###': sort_based_prefix + '../', 210 | '###TITLE###': 'by ' + sort_indexes[sort]['slug'] + ' page ' + str(page_num) + ' of ' + str(len(pages)), 211 | '###SUB###': subreddit, 212 | '###ARCH_NUM_POSTS###': str(stat_sub_filtered_links), 213 | '###ARCH_NUM_COMMENTS###': str(stat_sub_comments), 214 | '###URL_SUBS###': sort_based_prefix + '../index.html', 215 | '###URL_PROJECT###': url_project, 216 | '###URL_IDX_SCORE###': sort_based_prefix + 'index.html', 217 | '###URL_IDX_CMNT###': sort_based_prefix + 'index-' + sort_indexes['num_comments']['slug'] + '/index.html', 218 | '###URL_IDX_DATE###': sort_based_prefix + 'index-' + sort_indexes['created_utc']['slug'] + '/index.html', 219 | '###URL_SEARCH###': sort_based_prefix + 'search.html', 220 | '###URL_IDX_SCORE_CSS###': 'active' if sort == 'score' else '', 221 | '###URL_IDX_CMNT_CSS###': 'active' if sort == 'num_comments' else '', 222 | '###URL_IDX_DATE_CSS###': 'active' if sort == 'created_utc' else '', 223 | '###URL_SEARCH_CSS###': '', 224 | '###HTML_LINKS###': links_html, 225 | '###HTML_SUBS_MENU###': subs_menu_html, 226 | '###HTML_PAGER###': get_pager_html(page_num, len(pages)), 227 | } 228 | page_html = template_subreddit 229 | for key, value in index_page_data_map.items(): 230 | page_html = page_html.replace(key, value) 231 | 232 | 233 | # write file 234 | suffix = '-' + str(page_num) + '.html' 235 | if page_num == 1: 236 | suffix = '.html' 237 | filename = 'index' + suffix 238 | if sort == default_sort: 239 | filepath = 'r/' + subreddit + '/' + filename 240 | else: 241 | filepath = 'r/' + subreddit + '/index-' + sort_indexes[sort]['slug'] + '/' + filename 242 | if not os.path.isfile(filepath): 243 | os.makedirs(os.path.dirname(filepath), exist_ok=True) 244 | with open(filepath, 'w', encoding='utf-8') as file: 245 | file.write(page_html) 246 | # print('wrote %s %s, %s links' % (sort, filepath, len(page))) 247 | 248 | return True 249 | 250 | def write_link_page(subreddits, link, subreddit='', hide_deleted_comments=False): 251 | # reddit: https://www.reddit.com/r/conspiracy/comments/8742iv/happening_now_classmate_former_friend_of/ 252 | # archive: r/conspiracy/comments/8/7/4/2/i/v/happening_now_classmate_former_friend_of.html 253 | idpath = '/'.join(list(link['id'])) 254 | filepath = link['permalink'].lower().strip('/') + '.html' 255 | filepath = filepath.replace(link['id'], idpath) 256 | if os.path.isfile(filepath): 257 | return True 258 | 259 | created = datetime.utcfromtimestamp(int(link['created_utc'])) 260 | sorted_comments = [] 261 | if len(link['comments']) > 0: 262 | sorted_comments = sort_comments(link['comments'], hide_deleted_comments) 263 | 264 | # traverse up to root dir, depends on id length 265 | static_include_path = '' 266 | for i in range(len(link['id']) + 2): 267 | static_include_path += '../' 268 | 269 | # render comments 270 | comments_html = '' 271 | for c in sorted_comments: 272 | css_classes = 'ml-' + (str(c['depth']) if int(c['depth']) <= max_comment_depth else str(max_comment_depth)) 273 | if c['author'] == link['author'] and c['author'] not in removed_content_identifiers: 274 | css_classes += ' op' 275 | if c['stickied'].lower() == 'true' or c['stickied'] is True: 276 | css_classes += ' stickied' 277 | 278 | # author link 279 | url = static_include_path + 'user/' + c['author'] + '.html' 280 | author_link_html = template_user_url.replace('###URL_AUTHOR###', url).replace('###AUTHOR###', c['author']) 281 | 282 | comment_data_map = { 283 | '###ID###': c['id'], 284 | '###PARENT_ID###': c['parent_id'], 285 | '###DEPTH###': str(c['depth']), 286 | '###DATE###': created.strftime('%Y-%m-%d'), 287 | '###SCORE###': str(c['score']) if len(str(c['score'])) > 0 else missing_comment_score_label, 288 | '###BODY###': snudown.markdown(c['body'].replace('>','>')), 289 | '###CSS_CLASSES###': css_classes, 290 | '###CLASS_SCORE###': 'badge-danger' if len(c['score']) > 0 and int(c['score']) < 1 else 'badge-secondary', 291 | '###HTML_AUTHOR_URL###': author_link_html, 292 | } 293 | comment_html = template_comment 294 | for key, value in comment_data_map.items(): 295 | comment_html = comment_html.replace(key, value) 296 | comments_html += comment_html + '\n' 297 | 298 | # render subreddits list 299 | subs_menu_html = '' 300 | for sub in subreddits: 301 | sub_url = static_include_path + sub + '/index.html' 302 | subs_menu_html += template_sub_link.replace('###URL_SUB###', sub_url).replace('###SUB###', sub) 303 | 304 | # render selftext 305 | selftext_html = '' 306 | if len(link['selftext']) > 0: 307 | selftext_html = template_selftext.replace('###SELFTEXT###', snudown.markdown(link['selftext'].replace('>','>'))) 308 | 309 | # author link 310 | url = static_include_path + 'user/' + link['author'] + '.html' 311 | author_link_html = template_user_url.replace('###URL_AUTHOR###', url).replace('###AUTHOR###', link['author']) 312 | 313 | html_title = template_url.replace('#HREF#', link['url']).replace('#INNER_HTML#', link['title']) 314 | if link['is_self'] is True or link['is_self'].lower() == 'true': 315 | html_title = link['title'] 316 | 317 | # render link page 318 | link_data_map = { 319 | '###INCLUDE_PATH###': static_include_path, 320 | '###SUB###': subreddit, 321 | '###TITLE###': link['title'], 322 | '###ID###': link['id'], 323 | '###DATE###': created.strftime('%Y-%m-%d'), 324 | '###ARCHIVE_DATE###': datetime.utcfromtimestamp(int(link['retrieved_on'])).strftime('%Y-%m-%d') if link['retrieved_on'] != '' else 'n/a', 325 | '###SCORE###': str(link['score']), 326 | '###NUM_COMMENTS###': str(link['num_comments']), 327 | '###URL_PROJECT###': url_project, 328 | '###URL_SUBS###': static_include_path + 'index.html', 329 | '###URL_SUB###': static_include_path + subreddit + '/index.html', 330 | '###URL_SUB_CMNT###': static_include_path + subreddit + '/index-' + sort_indexes['num_comments']['slug'] + '/index.html', 331 | '###URL_SUB_DATE###': static_include_path + subreddit + '/index-' + sort_indexes['created_utc']['slug'] + '/index.html', 332 | '###URL_SEARCH###': static_include_path + subreddit + '/search.html', 333 | '###HTML_SUBS_MENU###': subs_menu_html, 334 | '###HTML_SELFTEXT###': selftext_html, 335 | '###HTML_COMMENTS###': comments_html, 336 | '###HTML_AUTHOR_URL###': author_link_html, 337 | '###HTML_TITLE###': html_title, 338 | } 339 | html = template_link 340 | for key, value in link_data_map.items(): 341 | html = html.replace(key, value) 342 | 343 | # write html 344 | # reddit: https://www.reddit.com/r/conspiracy/comments/8742iv/happening_now_classmate_former_friend_of/ 345 | # archive: r/conspiracy/comments/8/7/4/2/i/v/happening_now_classmate_former_friend_of.html 346 | idpath = '/'.join(list(link['id'])) 347 | filepath = link['permalink'].lower().strip('/') + '.html' 348 | filepath = filepath.replace(link['id'], idpath) 349 | if not os.path.isfile(filepath): 350 | os.makedirs(os.path.dirname(filepath), exist_ok=True) 351 | with open(filepath, 'w', encoding='utf-8') as file: 352 | file.write(html) 353 | # print('wrote %s %s' % (created.strftime('%Y-%m-%d'), filepath)) 354 | 355 | return True 356 | 357 | def write_subreddit_search_page(subreddit, subs, link_index, stat_sub_filtered_links, stat_sub_comments): 358 | if len(link_index) == 0: 359 | return True 360 | 361 | # name sort? 362 | links = sorted(link_index, key=lambda k: re.sub(r'\W+', '', k['title']).lower()) 363 | 364 | # render subreddits list 365 | subs_menu_html = '' 366 | for sub in subs: 367 | sub_url = '../' + sub + '/index.html' 368 | subs_menu_html += template_sub_link.replace('###URL_SUB###', sub_url).replace('###SUB###', sub) 369 | 370 | links_html = '' 371 | for l in links: 372 | link_comments_url = l['permalink'].lower().strip('/').replace('r/' + subreddit + '/', '') 373 | idpath = '/'.join(list(l['id'])) 374 | link_comments_url = link_comments_url.replace(l['id'], idpath) 375 | link_comments_url += '.html' 376 | index_link_data_map = { 377 | '###TITLE###': l['title'], 378 | '###URL###': link_comments_url, 379 | } 380 | link_html = template_search_link 381 | for key, value in index_link_data_map.items(): 382 | link_html = link_html.replace(key, value) 383 | links_html += link_html + '\n' 384 | 385 | index_page_data_map = { 386 | '###INCLUDE_PATH###': '../', 387 | '###TITLE###': 'search', 388 | '###SUB###': subreddit, 389 | '###ARCH_NUM_POSTS###': str(stat_sub_filtered_links), 390 | '###ARCH_NUM_COMMENTS###': str(stat_sub_comments), 391 | '###URL_SUBS###': '../index.html', 392 | '###URL_PROJECT###': url_project, 393 | '###URL_IDX_SCORE###': 'index.html', 394 | '###URL_IDX_CMNT###': 'index-' + sort_indexes['num_comments']['slug'] + '/index.html', 395 | '###URL_IDX_DATE###': 'index-' + sort_indexes['created_utc']['slug'] + '/index.html', 396 | '###URL_SEARCH###': 'search.html', 397 | '###URL_IDX_SCORE_CSS###': '', 398 | '###URL_IDX_CMNT_CSS###': '', 399 | '###URL_IDX_DATE_CSS###': '', 400 | '###URL_SEARCH_CSS###': 'active', 401 | '###HTML_LINKS###': links_html, 402 | '###HTML_SUBS_MENU###': subs_menu_html, 403 | } 404 | page_html = template_search 405 | for key, value in index_page_data_map.items(): 406 | page_html = page_html.replace(key, value) 407 | 408 | # write file 409 | filename = 'search.html' 410 | filepath = 'r/' + subreddit + '/' + filename 411 | if not os.path.isfile(filepath): 412 | os.makedirs(os.path.dirname(filepath), exist_ok=True) 413 | with open(filepath, 'w', encoding='utf-8') as file: 414 | file.write(page_html) 415 | # print('wrote %s, %s links' % (filepath, len(links))) 416 | return True 417 | 418 | def write_user_page(subs, user_index): 419 | if len(user_index.keys()) == 0: 420 | return False 421 | 422 | # subreddits list 423 | subs_menu_html = '' 424 | for sub in subs: 425 | sub_url = '../' + sub['name'] + '/index.html' 426 | subs_menu_html += template_sub_link.replace('###URL_SUB###', sub_url).replace('###SUB###', sub['name']) 427 | 428 | for user in user_index.keys(): 429 | links = user_index[user] 430 | links.sort(key=lambda k: (int(k['score']) if k['score'] != '' else sort_indexes['score']['default']), reverse=True) 431 | 432 | links_html = '' 433 | for l in links: 434 | 435 | author_link_html = template_user_url 436 | author_url = l['author'] + '.html' 437 | author_link_html = author_link_html.replace('###URL_AUTHOR###', author_url).replace('###AUTHOR###', l['author']) 438 | 439 | link_comments_url = l['permalink'].lower().replace('/r/', '').strip('/') 440 | link_comments_url = '../' + link_comments_url 441 | idpath = '/'.join(list(l['id'])) 442 | link_comments_url = link_comments_url.replace(l['id'], idpath) 443 | link_comments_url += '.html' 444 | link_url = l['url'] 445 | if l['is_self'] is True or l['is_self'] == 'True': 446 | link_url = link_comments_url 447 | 448 | link_data_map = { 449 | '###TITLE###': l['title'], 450 | '###URL###': link_url, 451 | '###URL_COMMENTS###': link_comments_url, 452 | '###SCORE###': str(l['score']), 453 | '###NUM_COMMENTS###': str(l['num_comments']) if int(l['num_comments']) > 0 else str(0), 454 | '###DATE###': datetime.utcfromtimestamp(int(l['created_utc'])).strftime('%Y-%m-%d'), 455 | '###SUB###': l['subreddit'], 456 | '###SUB_URL###': '../' + l['subreddit'] + '/index.html', 457 | '###HTML_AUTHOR_URL###': author_link_html, 458 | } 459 | link_html = template_user_page_link 460 | for key, value in link_data_map.items(): 461 | link_html = link_html.replace(key, value) 462 | links_html += link_html + '\n' 463 | 464 | page_data_map = { 465 | '###INCLUDE_PATH###': '../', 466 | '###TITLE###': 'user/' + user, 467 | '###ARCH_NUM_POSTS###': str(len(links)), 468 | '###URL_USER###': user + '.html', 469 | '###URL_SUBS###': '../index.html', 470 | '###URL_PROJECT###': url_project, 471 | '###HTML_LINKS###': links_html, 472 | '###HTML_SUBS_MENU###': subs_menu_html, 473 | } 474 | page_html = template_user 475 | for key, value in page_data_map.items(): 476 | page_html = page_html.replace(key, value) 477 | 478 | filepath = 'r/user/' + user + '.html' 479 | if not os.path.isfile(filepath): 480 | os.makedirs(os.path.dirname(filepath), exist_ok=True) 481 | with open(filepath, 'w', encoding='utf-8') as file: 482 | file.write(page_html) 483 | # print('wrote %s' % (filepath)) 484 | 485 | return True 486 | 487 | def write_index(subs): 488 | if len(subs) == 0: 489 | return False 490 | subs.sort(key=lambda k: k['name'].casefold()) 491 | 492 | stat_num_links = 0 493 | links_html = '' 494 | subs_menu_html = '' 495 | for sub in subs: 496 | sub_url = sub['name'] + '/index.html' 497 | links_html += template_index_sub.replace('#URL_SUB#', sub_url).replace('#SUB#', sub['name']).replace('#NUM_LINKS#', str(sub['num_links'])) 498 | subs_menu_html += template_sub_link.replace('###URL_SUB###', sub_url).replace('###SUB###', sub['name']) 499 | stat_num_links += sub['num_links'] 500 | 501 | index_page_data_map = { 502 | '###INCLUDE_PATH###': '', 503 | '###TITLE###': 'subreddits', 504 | '###URL_SUBS###': 'index.html', 505 | '###URL_PROJECT###': url_project, 506 | '###ARCH_NUM_POSTS###': str(stat_num_links), 507 | '###HTML_LINKS###': links_html, 508 | '###HTML_SUBS_MENU###': subs_menu_html, 509 | } 510 | page_html = template_index 511 | for key, value in index_page_data_map.items(): 512 | page_html = page_html.replace(key, value) 513 | 514 | filepath = 'r/index.html' 515 | if not os.path.isfile(filepath): 516 | os.makedirs(os.path.dirname(filepath), exist_ok=True) 517 | with open(filepath, 'w', encoding='utf-8') as file: 518 | file.write(page_html) 519 | # print('wrote %s' % (filepath)) 520 | 521 | return True 522 | 523 | # a 'top' comments sort with orphaned comments (incomplete data) rendered last 524 | def sort_comments(comments, hide_deleted_comments=False): 525 | sorted_comments = [] 526 | if len(comments) == 0: 527 | return sorted_comments 528 | parent_map = {} 529 | id_map = {} 530 | top_level_comments = [] 531 | link_id = comments[0]['link_id'] 532 | depth = 0 533 | 534 | for c in comments: 535 | c['depth'] = depth 536 | id_map[c['id']] = c 537 | parent_map[c['id']] = c['parent_id'] 538 | # add stickied comments 539 | if c['stickied'].lower() == 'true': 540 | sorted_comments.append(c) 541 | # store top level comments 542 | elif c['parent_id'] == c['link_id']: 543 | top_level_comments.append(c) 544 | 545 | # sort non stickied top level comments 546 | if len(top_level_comments) > 0: 547 | top_level_comments = sorted(top_level_comments, key=lambda k: (int(k['score']) if k['score'] != '' else 1), reverse=True) 548 | sorted_comments += top_level_comments 549 | 550 | # add each top level comment's child comments 551 | sorted_linear_comments = [] 552 | for c in sorted_comments: 553 | # only remove deleted comments if no children 554 | if hide_deleted_comments and c['body'] in removed_content_identifiers and 't1_' + c['id'] not in parent_map.values(): 555 | pass 556 | else: 557 | sorted_linear_comments.append(c) 558 | child_comments = get_comment_tree_list([], depth + 1, c, id_map, parent_map, hide_deleted_comments) 559 | if len(child_comments) > 0: 560 | sorted_linear_comments += child_comments 561 | 562 | # add orphaned comments 563 | for c in comments: 564 | if c['parent_id'] != link_id and c['parent_id'].replace('t1_', '') not in id_map.keys(): 565 | if hide_deleted_comments and c['body'] in removed_content_identifiers: 566 | continue 567 | sorted_linear_comments.append(c) 568 | 569 | # print('sort_comments() in %s out %s show deleted: %s' % (len(comments), len(sorted_comments), hide_deleted_comments)) 570 | return sorted_linear_comments 571 | 572 | def get_comment_tree_list(tree, depth, parent_comment, id_map, parent_map, hide_deleted_comments): 573 | parent_id = 't1_' + parent_comment['id'] 574 | child_comments = [] 575 | for key, value in parent_map.items(): 576 | if value == parent_id: 577 | if hide_deleted_comments and id_map[key]['body'] in removed_content_identifiers and 't1_' + key not in parent_map.values(): 578 | pass 579 | else: 580 | child_comments.append(id_map[key]) 581 | 582 | # sort children by score 583 | # TODO: sort by score and # of child comments 584 | if len(child_comments) > 0: 585 | child_comments = sorted(child_comments, key=lambda k: (int(k['score']) if k['score'] != '' else 1), reverse=True) 586 | for child_comment in child_comments: 587 | child_comment['depth'] = depth 588 | tree.append(child_comment) 589 | tree = get_comment_tree_list(tree, depth + 1, child_comment, id_map, parent_map, hide_deleted_comments) 590 | return tree 591 | 592 | def validate_link(link, min_score=0, min_comments=0): 593 | if not link: 594 | return False 595 | elif not 'id' in link.keys(): 596 | return False 597 | # apply multiple conditions as an OR, keep high score low comments and high comment low score links/posts 598 | # TODO this should be configurable 599 | if min_score > 0 and min_comments > 0: 600 | if int(link['score']) < min_score and int(link['num_comments']) < min_comments: 601 | return False 602 | else: 603 | if min_score > 0 and int(link['score']) < min_score: 604 | return False 605 | if min_comments > 0 and int(link['num_comments']) < min_comments: 606 | return False 607 | 608 | return True 609 | 610 | def load_links(date, subreddit, with_comments=False): 611 | links = [] 612 | if not date or not subreddit: 613 | return links 614 | 615 | date_path = date.strftime("%Y/%m/%d") 616 | daily_path = 'data/' + subreddit + '/' + date_path 617 | daily_links_path = daily_path + '/' + source_data_links 618 | if os.path.isfile(daily_links_path): 619 | links = [] 620 | with open(daily_links_path, 'r', encoding='utf-8') as links_file: 621 | reader = csv.DictReader(links_file) 622 | for link_row in reader: 623 | if with_comments and 'id' in link_row.keys(): 624 | comments = [] 625 | comments_file_path = daily_path + '/' + link_row['id'] + '.csv' 626 | if os.path.isfile(comments_file_path): 627 | with open(comments_file_path, 'r', encoding='utf-8') as comments_file: 628 | reader = csv.DictReader(comments_file) 629 | for comment_row in reader: 630 | comments.append(comment_row) 631 | link_row['comments'] = comments 632 | links.append(link_row) 633 | return links 634 | 635 | def get_subs(): 636 | subs = [] 637 | if not os.path.isdir('data'): 638 | print('ERROR: no data, run fetch_links.py first') 639 | return subs 640 | for d in os.listdir('data'): 641 | if os.path.isdir('data' + '/' + d): 642 | subs.append(d.lower()) 643 | return subs 644 | 645 | def get_pager_html(page_num=1, pages=1): 646 | html_pager = '' 647 | 648 | # previous 649 | css = '' 650 | if page_num == 1: 651 | css = 'disabled' 652 | url = 'index' 653 | if page_num - 1 > 1: 654 | url += '-' + str(page_num - 1) 655 | url += '.html' 656 | html_pager += template_index_pager_link.replace('#URL#', url).replace('#TEXT#', '‹').replace('#CSS_CLASS#', css) 657 | 658 | # skip back 659 | css = '' 660 | prev_skip = page_num - pager_skip 661 | if prev_skip < 1: 662 | prev_skip = 1 663 | if page_num == 1: 664 | css = 'disabled' 665 | url = 'index' 666 | if prev_skip > 1: 667 | url += '-' + str(prev_skip) 668 | url += '.html' 669 | html_pager += template_index_pager_link.replace('#URL#', url).replace('#TEXT#', '‹‹').replace('#CSS_CLASS#', css) 670 | 671 | # skip back far 672 | css = '' 673 | prev_skip = page_num - pager_skip_long 674 | if prev_skip < 1: 675 | prev_skip = 1 676 | if page_num == 1: 677 | css += ' disabled' 678 | url = 'index' 679 | if prev_skip > 1: 680 | url += '-' + str(prev_skip) 681 | url += '.html' 682 | html_pager += template_index_pager_link.replace('#URL#', url).replace('#TEXT#', '‹‹‹').replace('#CSS_CLASS#', css) 683 | 684 | # n-1 685 | start = -2 686 | if page_num + 1 > pages: 687 | start -= 1 688 | if page_num + 2 > pages: 689 | start -= 1 690 | for prev_page_num in range(start,0): 691 | if page_num + prev_page_num > 0: 692 | css = '' 693 | url = 'index' 694 | if page_num + prev_page_num > 1: 695 | url += '-' + str(page_num + prev_page_num) 696 | url += '.html' 697 | if prev_page_num < -1: 698 | css = 'd-none d-sm-block' 699 | html_pager += template_index_pager_link.replace('#URL#', url).replace('#TEXT#', str(page_num + prev_page_num)).replace('#CSS_CLASS#', css) 700 | # n 701 | url = 'index' 702 | if page_num > 1: 703 | url += '-' + str(page_num) 704 | url += '.html' 705 | html_pager += template_index_pager_link.replace('#URL#', url).replace('#TEXT#', str(page_num)).replace('#CSS_CLASS#', 'active') 706 | # n + 1 707 | css = '' 708 | end = 3 709 | if page_num -1 < 1: 710 | end += 1 711 | if page_num - 2 < 1: 712 | end += 1 713 | for next_page_num in range(1,end): 714 | if page_num + next_page_num <= pages: 715 | if next_page_num > 1: 716 | css = 'd-none d-sm-block' 717 | html_pager += template_index_pager_link.replace('#URL#', 'index' + '-' + str(page_num + next_page_num) + '.html').replace('#TEXT#', str(page_num + next_page_num)).replace('#CSS_CLASS#', css) 718 | 719 | # skip forward far 720 | next_skip = page_num + pager_skip_long 721 | css = '' 722 | if page_num == pages: 723 | css += ' disabled' 724 | if next_skip > pages: 725 | next_skip = pages 726 | url = 'index' 727 | if next_skip > 1: 728 | url += '-' + str(next_skip) 729 | url += '.html' 730 | html_pager += template_index_pager_link.replace('#URL#', url).replace('#TEXT#', '›››').replace('#CSS_CLASS#', css) 731 | 732 | # skip forward 733 | next_skip = page_num + pager_skip 734 | css = '' 735 | if page_num == pages: 736 | css = 'disabled' 737 | if next_skip > pages: 738 | next_skip = pages 739 | url = 'index' 740 | if next_skip > 1: 741 | url += '-' + str(next_skip) 742 | url += '.html' 743 | html_pager += template_index_pager_link.replace('#URL#', url).replace('#TEXT#', '››').replace('#CSS_CLASS#', css) 744 | 745 | # next 746 | css = '' 747 | next_num = page_num + 1 748 | if page_num == pages: 749 | css = 'disabled' 750 | next_num = pages 751 | html_pager += template_index_pager_link.replace('#URL#', 'index' + '-' + str(next_num) + '.html').replace('#TEXT#', '›').replace('#CSS_CLASS#', css) 752 | 753 | return html_pager 754 | 755 | def chunks(l, n): 756 | """Yield successive n-sized chunks from l.""" 757 | for i in range(0, len(l), n): 758 | yield l[i:i + n] 759 | 760 | if __name__ == '__main__': 761 | parser=argparse.ArgumentParser() 762 | parser.add_argument('--min-score', default=0, help='limit post rendering, default 0') 763 | parser.add_argument('--min-comments', default=0, help='limit post rendering, default 0') 764 | parser.add_argument('--hide-deleted-comments', action='store_true', help='exclude deleted and removed comments where possible') 765 | args=parser.parse_args() 766 | 767 | hide_deleted_comments = False 768 | if args.hide_deleted_comments: 769 | hide_deleted_comments = True 770 | 771 | args.min_score = int(args.min_score) 772 | args.min_comments = int(args.min_comments) 773 | 774 | generate_html(args.min_score, args.min_comments, hide_deleted_comments) 775 | -------------------------------------------------------------------------------- /r/static/js/bootstrap.min.js: -------------------------------------------------------------------------------- 1 | /*! 2 | * Bootstrap v4.1.3 (https://getbootstrap.com/) 3 | * Copyright 2011-2018 The Bootstrap Authors (https://github.com/twbs/bootstrap/graphs/contributors) 4 | * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) 5 | */ 6 | !function(t,e){"object"==typeof exports&&"undefined"!=typeof module?e(exports,require("jquery"),require("popper.js")):"function"==typeof define&&define.amd?define(["exports","jquery","popper.js"],e):e(t.bootstrap={},t.jQuery,t.Popper)}(this,function(t,e,h){"use strict";function i(t,e){for(var n=0;nthis._items.length-1||t<0))if(this._isSliding)P(this._element).one(Q.SLID,function(){return e.to(t)});else{if(n===t)return this.pause(),void this.cycle();var i=ndocument.documentElement.clientHeight;!this._isBodyOverflowing&&t&&(this._element.style.paddingLeft=this._scrollbarWidth+"px"),this._isBodyOverflowing&&!t&&(this._element.style.paddingRight=this._scrollbarWidth+"px")},t._resetAdjustments=function(){this._element.style.paddingLeft="",this._element.style.paddingRight=""},t._checkScrollbar=function(){var t=document.body.getBoundingClientRect();this._isBodyOverflowing=t.left+t.right