├── .coveragerc ├── .github ├── CONTRIBUTING.md └── PULL_REQUEST_TEMPLATE.md ├── .gitignore ├── .travis.yml ├── CHANGES.md ├── Dockerfile ├── Procfile ├── README.md ├── blaggregator ├── __init__.py ├── settings │ ├── __init__.py │ ├── base.py │ ├── docker.py │ ├── heroku.py │ ├── production.py │ ├── staging.py │ └── travis.py ├── urls.py └── wsgi.py ├── docker-compose.pg.yml ├── docker-compose.yml ├── docs ├── development.md └── server-maintenance-and-deploys.md ├── home ├── __init__.py ├── admin.py ├── context_processors.py ├── feedergrabber27.py ├── feeds.py ├── management │ ├── __init__.py │ └── commands │ │ ├── __init__.py │ │ ├── crawlposts.py │ │ ├── de-dup-posts.py │ │ ├── delete_medium_comments.py │ │ ├── notify_uncrawlable_blogs.py │ │ └── update_user_details.py ├── middleware.py ├── migrations │ ├── 0001_initial.py │ ├── 0002_auto_20161201_2313.py │ ├── 0003_auto_20180313_1026.py │ ├── 0004_auto_20180316_0351.py │ ├── 0005_blog_skip_crawl.py │ ├── 0006_auto_20180317_0559.py │ ├── 0007_auto_20180318_0236.py │ ├── 0008_auto_20191224_0511.py │ ├── 0009_hacker_zulip_id.py │ └── __init__.py ├── models.py ├── oauth.py ├── static │ └── js │ │ └── update_images.js ├── templates │ ├── 404.html │ └── home │ │ ├── about.html │ │ ├── add_blog.html │ │ ├── base.html │ │ ├── confirm_delete.html │ │ ├── disabling-crawling.md │ │ ├── edit_blog.html │ │ ├── feed_item.tmpl │ │ ├── log_in_oauth.html │ │ ├── login_error.html │ │ ├── most_viewed.html │ │ ├── new.html │ │ ├── postlist.html │ │ ├── profile.html │ │ ├── readme.html │ │ └── search.html ├── templatetags │ ├── __init__.py │ └── customtags.py ├── tests │ ├── __init__.py │ ├── test_crawlposts.py │ ├── test_feed_parser.py │ ├── test_oauth.py │ ├── test_views.py │ └── utils.py ├── token_auth.py ├── urls.py ├── utils.py ├── views.py └── zulip_helpers.py ├── manage.py ├── poetry.lock ├── pyproject.toml ├── requirements.txt ├── runtime.txt ├── scripts ├── delete_messages └── readme_to_about └── web-variables.env /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | source = home,blaggregator 3 | omit = 4 | home/migrations/* 5 | blaggregator/wsgi.py 6 | 7 | [report] 8 | sort=Miss 9 | -------------------------------------------------------------------------------- /.github/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to Blaggregator! 2 | 3 | Blaggregator is a blog aggregator for the Recurse Center, built and maintained 4 | by the community. Thank you for considering contributing to it, and taking the 5 | time to read this document! :heart: :heart: :heart: 6 | 7 | If you enjoy the tool or are interested in practicing web programming on a 8 | "real app" that's used at least by a few hundred people, quite regularly, this 9 | is a great way to get some experience. Please feel free to contribute features 10 | and fixes. If you have any questions or project ideas, get in touch with 11 | Puneeth(@punchagan) or Sasha(@sursh). 12 | 13 | ## Code of conduct 14 | 15 | RC's social rules apply, while you are contributing to this Blaggregator! 16 | 17 | ## Commit-messages 18 | 19 | - Use the present tense ("Add feature" not "Added feature") 20 | - Use the imperative mood ("Move cursor to..." not "Moves cursor to...") 21 | - Limit the first line to 72 characters or less 22 | - Reference issues and pull requests liberally 23 | - When only changing documentation, include `[ci skip]` in the commit 24 | description 25 | 26 | ## Pull-requests 27 | 28 | - Adding tests for the code you submit is highly encouraged! 29 | - If you are making a UI change, try and include a screenshot. 30 | - Before starting work on a big feature/contribution, you are encouraged to let 31 | us know in a related issue. 32 | - :+1: for referring to issue numbers in your pull-requests 33 | 34 | ## Some general ideas/areas to help in 35 | 36 | - More stuff should happen "async". Currently, most things happen 37 | synchronously, not giving the best possible user experience 38 | 39 | - Anything that makes it easier for alum (who usually find it hard to keep up 40 | with Zulip) to follow Blaggregator posts 41 | 42 | - Make it easier to search for old posts - by author, content, topic, etc. 43 | 44 | - Testing/security - Security audit of the app or adding more tests to the app 45 | would go a long way 46 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | - [ ] Test 2 | - [ ] Screenshot 3 | - [ ] CHANGES.md entry 4 | - [ ] Documentation 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | build/ 2 | *.pyc 3 | .DS_Store 4 | bin/ 5 | lib/ 6 | include/ 7 | .Python 8 | blaggregator/static-collected 9 | web-variables.env 10 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | 3 | cache: pip 4 | 5 | services: 6 | - postgresql 7 | 8 | python: 9 | - "3.7" 10 | 11 | # command to install dependencies 12 | install: 13 | - "pip install -r requirements.txt" 14 | 15 | before_script: 16 | - psql -c "CREATE DATABASE travisdb;" -U postgres 17 | - # Random tokens that aren't really used 18 | - export SOCIAL_AUTH_HS_KEY=897316929176464ebc9ad085f31e7284 19 | - export SOCIAL_AUTH_HS_SECRET=26ab0db90d72e28ad0ba1e22ee510510 20 | - export HS_PERSONAL_TOKEN=b026324c6904b2a9cb4b88d6d61c81d1 21 | 22 | 23 | # command to run tests 24 | script: 25 | # ensure README.md changes all synced to about.html 26 | - ./scripts/readme_to_about && git diff --exit-code 27 | - coverage run manage.py test home 28 | - flake8 --ignore=E501 --exclude=migrations,wsgi.py,blaggregator/settings/__init__.py 29 | 30 | after_script: 31 | - coverage report -m 32 | -------------------------------------------------------------------------------- /CHANGES.md: -------------------------------------------------------------------------------- 1 | # Change Log 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | The format is based on [Keep a Changelog](http://keepachangelog.com/). Though 6 | this project doesn't really have releases (or even versioning, until now), it 7 | is a project that effects Recursers in "real ways". It would be nice to have a 8 | place to quickly see the latest changes that the project has undergone. 9 | 10 | 11 | ## 2021-09 12 | 13 | ### Added 14 | 15 | - [#165] Mention post authors when announcing posts on Zulip -- @punchagan 16 | 17 | ### Changed 18 | 19 | - [0095aef45af9e906259b48f1e6757ec9cef86870] Switch to using Poetry to manage 20 | dependencies. The requirements.txt file was quite unmanageable. Also, updated 21 | all the dependencies to the latest versions. 22 | 23 | - [421af8fe46b74715a8d2f4020d6253607568821e] Upgrade to the latest version of 24 | Django -- @punchagan 25 | 26 | ## 2019-12 27 | 28 | ### Changed 29 | 30 | - [cd957294e7f75f5b9ec2c9668325a3abc40ccca9] Upgrade from Python2 to Python3 -- @punchagan 31 | 32 | ## 2019-10 33 | 34 | ### Fixed 35 | 36 | - [90d681383d840b913e8f53bbd4b0e5de70461391] Fixed problem with feed generation 37 | when a post title/content contains control characters 38 | 39 | ## 2018-03 40 | 41 | ### Added 42 | 43 | - [#155,#156] Added command to update all user details -- @punchagan 44 | 45 | - [#151] Added command to delete posts with duplicate titles -- @punchagan 46 | 47 | - [#147] Added a skip_crawl flag on blogs to ignore them during crawls. Also, 48 | added a command to mark offending blogs and notify their owners. -- @punchagan 49 | 50 | ### Fixed 51 | 52 | - [#143,#144] Don't add medium comments as posts -- @punchagan 53 | 54 | ### Changed 55 | 56 | - [#107] - Use gevent to make crawls asynchronous -- @punchagan 57 | 58 | - [2d6d3986f019b10f67616054c0a2fb8287c9f345] Remove the add_blog page, and move 59 | the form to the profile page -- @punchagan 60 | 61 | - [#145] Change Django to 1.11.11 (LTS) -- @punchagan 62 | 63 | - [#148] Don't add posts when a new blog is added. Also clean up how feed URL 64 | suggestions are done -- @punchagan 65 | 66 | - [#154] Upgraded to the latest version of bootstrap 4.0.0 and switched to CDN. 67 | Also merged the edit_blog page into the profile page. -- @punchagan 68 | 69 | - [#155] Use a personal token to update user details, and old hairy code that 70 | fetched backend and tried to get user's new details -- @punchagan 71 | 72 | ### Removed 73 | 74 | - [#157] Removed URL attribute for blogs -- @punchagan 75 | 76 | - [#156] Removed code for authenticating legacy users -- @punchagan 77 | 78 | ## 2017-02 79 | 80 | ### Added 81 | 82 | - [#138] Added simple post title based search -- @stanzheng 83 | - [#137] Added a Docker based development environment -- @stanzheng 84 | 85 | ## 2016-12 86 | 87 | ### Fixed 88 | 89 | - [#132] OAuth account creation fails if user does not have 'twitter' or 90 | 'github' details in their profile -- @punchagan 91 | 92 | ## 2016-11 93 | 94 | ### Changed 95 | 96 | - [#106] Upgraded to Django 1.10 -- @punchagan 97 | 98 | ## 2016-10 99 | 100 | ### Added 101 | 102 | - [#116] Added an AGPL license to the project -- @punchagan 103 | 104 | - [#128] Added token based authentication for atom feed -- @punchagan 105 | 106 | ### Changed 107 | 108 | - [#127] Use Django's feed generation mechanism instead of a custom template 109 | for Atom feeds -- @punchagan 110 | 111 | - [#127] Also, set-up travis integration to start running Django tests -- @punchagan 112 | 113 | - [#124] Post content is also saved to the DB for each crawled blog post -- @punchagan 114 | 115 | - [#118] Switched over the live site to https://blaggregator.recurse.com -- @punchagan 116 | 117 | ### Removed 118 | 119 | - [#121] The comments feature has been removed because it didn't see many 120 | people using it. Instead, links to the Zulip thread have been added -- 121 | @punchagan 122 | 123 | ### Fixed 124 | 125 | - [#115] Fixed a bug introduced when allowing crawls of posts without a date -- 126 | they always showed up at the top because the `date_updated` was being reset 127 | on every crawl -- @punchagan 128 | 129 | ## 2016-09 130 | 131 | ### Fixed 132 | 133 | - [#113] Allow adding feeds that may not have published dates for posts -- 134 | @punchagan 135 | 136 | ## 2016-08 137 | 138 | ### Changed 139 | 140 | - [#112] Change the Zulip API endpoint to the new Zulip domain -- @punchagan 141 | 142 | - [#112] Stop using runscope for making API calls to Zulip to post 143 | notifications -- @punchagan 144 | 145 | ## 2016-05 146 | 147 | ### Changed 148 | 149 | - [#110] Improve user profiles to display all posts made by an author on a page 150 | -- @punchagan 151 | 152 | - [#107] Add a header row to the most viewed posts table -- @sursh 153 | 154 | ### Fixed 155 | 156 | - [#111] Ignore `CharacterEncodingOverride` exception is really a warning, but 157 | `feedparser` treats it as an error causing some blogs to not be parsed 158 | correctly. -- @punchagan 159 | 160 | - [#100] Verify if a url is valid before adding it as a feed url to parse -- 161 | @punchagan 162 | 163 | ## 2015-12 164 | 165 | ### Changed 166 | - [#105] Allow specifying the number of days for which to generate the most 167 | viewed post stats 168 | 169 | ### Fixed 170 | - [#105] most viewed posts stats generation failed when post titles had unicode 171 | -- @punchagan 172 | 173 | ## 2015-06 174 | 175 | - [#83] Add a view to see most viewed posts during the last week -- @punchagan 176 | 177 | ## 2015-05 178 | 179 | ### Changed 180 | 181 | - [#92] Blaggregator previously checked only for the URL of posts to see if a 182 | post is different from another. A bug in Medium's RSS generation caused Zulip 183 | to be spammed with tens or hundreds of notifications. This change added a 184 | check to check for titles of posts, first, before checking URLs -- @punchagan 185 | 186 | - [#81] Fix naive datetime warnings when new blogs are added. -- @punchagan 187 | 188 | ## 2015-03 189 | 190 | ### Changed 191 | 192 | - [#88] Switch to using OAuth API from recurse.com instead of hackerschool.com 193 | -- @zachallaun 194 | 195 | ### Fixed 196 | 197 | - [#85] First crawl of the blog of a user with (an)other blog(s) already added 198 | would fail silently -- @punchagan 199 | 200 | ## 2015-03 201 | 202 | ### Added 203 | 204 | - [#69] Add a button to the header to allow users to add a blog, if they 205 | already didn't add one. -- @punchagan 206 | 207 | - [#67] Log blog post link clicks/visits -- @punchagan 208 | 209 | ### Changed 210 | 211 | - [#68] Instead of ignoring posts older than 2 days, and not announcing them on 212 | Zulip, limit the number of posts announced per crawl per blog to 2 -- @nnja 213 | 214 | ### Removed 215 | 216 | - [#67] Get rid of html frames in the site -- @punchagan 217 | 218 | ### Fixed 219 | 220 | - [#71] Return a 404 instead of a 500 for unknown slugs, and improve the 404 221 | page. -- @punchagan 222 | 223 | ## 2014-08 224 | 225 | ### Fixed 226 | 227 | - [#65] Fix broken profile images because of stale user information obtained 228 | from Hacker School's OAuth end-point -- @punchagan 229 | 230 | ### Changed 231 | 232 | - [#64] Allow user to pick stream & edit their own blogs -- @sursh and @punchagan 233 | 234 | ## 2014-03 235 | 236 | ### Added 237 | 238 | - [#55] Add authentication using Hacker School OAuth -- @davidbalbert 239 | 240 | ### Changed 241 | 242 | - [#59] Use /people/me instead of /people/me.json -- @davidbalbert 243 | 244 | ## 2014-02 245 | 246 | ### Changed 247 | 248 | - [#52] Simplify the topic names, since the notification stream on Zulip was 249 | changed to 'blogging' -- @graue 250 | 251 | - Changed the notification stream to blogging -- @sursh 252 | 253 | ## 2014-02 254 | 255 | ### Fixed 256 | 257 | - [#50] Get runscope bucket information from an environment variable -- @pnf 258 | 259 | ## 2013-07 260 | 261 | ### Added 262 | 263 | - [#33] Add menu to navbar to let users add their blog, and logout functionality -- @porterjamesj 264 | 265 | ## 2013-10 266 | 267 | ### Fixed 268 | 269 | - [#30] Changed CharFields to Textfields -- @PuercoPop 270 | 271 | ## 2013-07 272 | 273 | ### Added 274 | 275 | - [#24] Added pagination in the home page "/new". -- @santialbo 276 | 277 | ### Fixed 278 | 279 | - [#28] Secret key should be loaded from an environment variable -- @PuercoPop 280 | 281 | ## 2013-06 282 | 283 | ### Changed 284 | 285 | - [#17] Made the layout for post discussion page clearer, trying to make the 286 | blog post info the focal point -- @alliejones 287 | 288 | ## 2013-04 289 | 290 | ### Added 291 | 292 | - [#6] Added a bot to post new blog-posts to humbug -- @einashaddad and 293 | @kenyavs 294 | 295 | - [#2] Added an (authenticated) atom feed for all the posts -- @santialbo 296 | 297 | ## Changed 298 | 299 | - [#4] Don't require S3 in local environment -- @thomasboyt 300 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.7 2 | ENV PYTHONUNBUFFERED 1 3 | RUN mkdir /code 4 | WORKDIR /code 5 | ADD requirements.txt /code/ 6 | RUN pip install -r requirements.txt 7 | ADD . /code/ 8 | -------------------------------------------------------------------------------- /Procfile: -------------------------------------------------------------------------------- 1 | web: gunicorn blaggregator.wsgi 2 | 3 | crawlposts: python manage.py crawlposts 4 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Blaggregator 2 | 3 | Blog post aggregator for the Recurse Center community. Live version runs at 4 | (RC login required). 5 | 6 |
7 | 8 |

What is this?

9 | 10 |

During her batch, Sasha (W '13) noticed that her peers were all blogging about 11 | their work at the Recurse Center on their individual blogs. It was really cool 12 | to see what they were working on and thinking about.

13 | 14 |

Some folks would post about their new posts in Zulip, some on Twitter, and some 15 | wouldn't spread the word at all. There was all this great work going on, but it 16 | was scattered across the internet.

17 | 18 |

Blaggregator puts all this awesome content in one place, and provides a place 19 | for the members of the community to read and discuss it. This has the nice 20 | side effect of providing a friendly audience for those who may have just 21 | launched their blog.

22 | 23 |
24 | 25 | [These awesome people](https://github.com/recursecenter/blaggregator/graphs/contributors) have 26 | helped make Blaggregator better! 27 | 28 | ## License 29 | 30 | Copyright © 2013-2017 Sasha Laundy and others. 31 | 32 | This software is licensed under the terms of the AGPL, Version 3. The complete 33 | license can be found at https://www.gnu.org/licenses/agpl-3.0.html. 34 | 35 | ## FAQ 36 | 37 | ### How does it work? 38 | 39 | Once an hour, the crawler checks all registered blogs for new posts. New posts 40 | are displayed on the main page and a message is sent to Zulip. 41 | 42 | "New" is defined as a post having a new URL and a new title. So you can tweak 43 | the title, change the content or the timestamp to your heart's content. 44 | 45 | ### Why do I have to log in? 46 | 47 | The Recurse Center staff wishes to keep the list of people attending the 48 | Recurse Center private. So you are required to authenticate with your Recurse 49 | Center credentials 50 | 51 | If that ever changes (for instance, to surface the best posts that are coming 52 | out of the Recurse Center to show off what we're working on) you will be given 53 | lots and lots of warning to decide how you want to participate. 54 | 55 | ### Who's behind it? 56 | 57 | [Sasha](https://github.com/sursh) is the main author, with a bunch of 58 | other 59 | [recursers](https://github.com/recursecenter/blaggregator/graphs/contributors) 60 | contributing. You are welcome to contribute as well! 61 | 62 | [Puneeth](https://github.com/punchagan) is the primary maintainer, currently. 63 | 64 | ### Can I contribute fixes and features? 65 | 66 | Yes, that would be great! This is a project by and for the Recurse Center 67 | community. Help make it more awesome! 68 | 69 | There's a very generic and high level list 70 | of 71 | [areas that could use some help](https://github.com/recursecenter/blaggregator/blob/master/.github/CONTRIBUTING.md) and 72 | a bunch of specific 73 | [open issues](https://github.com/recursecenter/blaggregator/issues). 74 | 75 | Look at 76 | the 77 | [developer documentation](https://github.com/recursecenter/blaggregator/blob/master/docs/development.md) for 78 | help with setting up your development environment. 79 | 80 | Before implementing a major contribution, it's wise to get in touch with the 81 | maintainers by creating an issue (or using an existing issue) to discuss it. 82 | 83 | ### What's the stack? 84 | 85 | It's a Django (Python) app, with some Bootstrap on the frontend. It's deployed 86 | on Heroku using their Postgres and Scheduler add-ons. Check out the 87 | code [here](https://github.com/recursecenter/blaggregator). 88 | 89 | ### I don't see my blog post. 90 | 91 | If you published it less than an hour ago, hang tight: it'll show up when the 92 | crawler next checks the registered blogs. If Blaggregator finds many new posts 93 | on your blog, it will only post the two most recent posts to Zulip. All of your 94 | posts will still be available on the [site](https://blaggregator.recurse.com) 95 | 96 | ### My blog post appears on blaggregator but no Zulip notification sent. 97 | 98 | For every crawl a maximum of 2 notifications are sent for posts that haven't 99 | already been seen by blaggregator. So, if you published more than 2 posts 100 | between consecutive (hourly) crawls by blaggregator, only the last two posts 101 | will be notified on Zulip. 102 | 103 | ### My blog is multi-purpose and not wholly code/Recurse Center specific 104 | 105 | No problem! RCers usually enjoy reading non-technical posts by others. 106 | 107 | But, if you like, you could tag or categorize your relevant posts and make a 108 | separate RSS/Atom feed for them. Most blogging software has separate feeds for 109 | different categories/tags. 110 | 111 | If you use Wordpress, for instance, categorize all your code-related posts 112 | "code", say. Then your code-specific RSS feed that you should publish to 113 | Blaggregator is: http://yoururl.com/blog/category/code/feed/. 114 | 115 | ### Can I lurk? 116 | 117 | Sure, just make an account and don't add your blog. But you shouldn't lurk. You 118 | should blog. 119 | 120 | ### Why should I blog? 121 | 122 | - It provides a record of your thinking and work for your future self. 123 | - It gives prospective employers insight into the way you think. 124 | - If you do a project that doesn't go as planned, you can still 'finish' the 125 | project and explain what you learned, even if you don't want to put the code 126 | on Github. 127 | - It helps more people hear about and respect the Recurse Center, which in turn 128 | means more people will want to work with you. 129 | - Writing helps you practice communicating, which is critical if you plan on 130 | working on a team of more than one person. 131 | - It helps the developer community at large. 132 | 133 | ### But blogging takes too long! 134 | 135 | Don't let the perfect be the enemy of the good. Your posts don't have to be 136 | long, groundbreaking, perfect, or full of citations. Short, imperfect, and 137 | *published* always beats unpublished. 138 | 139 | ### But I haven't found the perfect blogging tool 140 | 141 | Don't let the perfect be the enemy of the good. Just get something up and 142 | resist the urge to fiddle with it. Use Tumblr if you have to. Just start 143 | writing. 144 | 145 | ### I need some more inspiration. 146 | 147 | - [You Should Write 148 | Blogs](https://sites.google.com/site/steveyegge2/you-should-write-blogs) 149 | (Steve Yegge) 150 | - [How to blog about code and give zero 151 | fucks](http://www.garann.com/dev/2013/how-to-blog-about-code-and-give-zero-fucks/). 152 | (Garann Means) 153 | - Please add your fave inspiration with a [pull 154 | request](https://github.com/recursecenter/blaggregator/pulls). 155 | 156 | ### I need some accountability! 157 | 158 | Consider starting your own Iron Blogger challenge. Participants commit to 159 | writing one blog post a week, and are on the hook for $5 if they don't 160 | post. The pot can be used for a party, donated to charity, or donated to a 161 | charity the group hates (added incentive to hit the publish button!). 162 | 163 | The Fall 2013 batch ran a very successful Iron Blogger program. Mike 164 | Walker 165 | [wrote a very nice article on how it worked](http://blog.lazerwalker.com/2013/12/24/one-post-a-week-running-an-iron-blogger-challenge). 166 | -------------------------------------------------------------------------------- /blaggregator/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/recursecenter/blaggregator/d5f3da939fcb16480be049c027e141b7aa171baa/blaggregator/__init__.py -------------------------------------------------------------------------------- /blaggregator/settings/__init__.py: -------------------------------------------------------------------------------- 1 | from .base import * 2 | 3 | if "PROD" in os.environ: 4 | print("** DETECTED PRODUCTION ENVIRONMENT") 5 | from .production import * 6 | 7 | elif "STAGING" in os.environ: 8 | print("** DETECTED STAGING ENVIRONMENT") 9 | from .staging import * 10 | 11 | elif "TRAVIS" in os.environ: 12 | print("** DETECTED TRAVIS ENVIRONMENT") 13 | from .travis import * 14 | 15 | elif "DOCKER_ENV" in os.environ: 16 | print("** DETECTED DOCKER ENVIRONMENT") 17 | from .docker import * 18 | 19 | else: 20 | print("** DETECTED LOCAL ENVIRONMENT") 21 | -------------------------------------------------------------------------------- /blaggregator/settings/base.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | SITE_ROOT = os.path.dirname(os.path.realpath(__file__)) 4 | 5 | # True: heroku config:set DJANGO_DEBUG=True 6 | # False: heroku config:unset DJANGO_DEBUG 7 | DEBUG = "DJANGO_DEBUG" in os.environ 8 | 9 | STATIC_URL = "/static/" 10 | ROOT_URL = "http://localhost:8000/" 11 | 12 | ADMINS = ( 13 | # ('Your Name', 'your_email@example.com'), 14 | ("Sasha Laundy", "sasha.laundy@gmail.com"), 15 | ("Puneeth Chaganti", "punchagan@muse-amuse.in"), 16 | ) 17 | 18 | MANAGERS = ADMINS 19 | 20 | DATABASES = { 21 | "default": { 22 | # Add 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'. 23 | "ENGINE": "django.db.backends.postgresql_psycopg2", 24 | "NAME": "blaggregator_dev", 25 | # The following settings are not used with sqlite3: 26 | "USER": "sasha", 27 | "PASSWORD": "sasha", 28 | "HOST": "127.0.0.1", 29 | "PORT": "5432", 30 | } 31 | } 32 | 33 | DEFAULT_AUTO_FIELD = "django.db.models.AutoField" 34 | 35 | # Hosts/domain names that are valid for this site; required if DEBUG is False 36 | # See https://docs.djangoproject.com/en/1.5/ref/settings/#allowed-hosts 37 | ALLOWED_HOSTS = [ 38 | "localhost", 39 | ] 40 | 41 | # Use HTTP_X_FORWARDED_HOST header to construct absolute URIs 42 | USE_X_FORWARDED_HOST = True 43 | 44 | # Local time zone for this installation. Choices can be found here: 45 | # https://en.wikipedia.org/wiki/List_of_tz_zones_by_name 46 | # although not all choices may be available on all operating systems. 47 | # In a Windows environment this must be set to your system time zone. 48 | TIME_ZONE = "America/New_York" 49 | 50 | # Language code for this installation. All choices can be found here: 51 | # http://www.i18nguy.com/unicode/language-identifiers.html 52 | LANGUAGE_CODE = "en-us" 53 | 54 | # If you set this to False, Django will make some optimizations so as not 55 | # to load the internationalization machinery. 56 | USE_I18N = True 57 | 58 | # If you set this to False, Django will not format dates, numbers and 59 | # calendars according to the current locale. 60 | USE_L10N = True 61 | 62 | # If you set this to False, Django will not use timezone-aware datetimes. 63 | USE_TZ = True 64 | 65 | # Absolute filesystem path to the directory that will hold user-uploaded files. 66 | # Example: "/var/www/example.com/media/" 67 | MEDIA_ROOT = "" 68 | 69 | # URL that handles the media served from MEDIA_ROOT. Make sure to use a 70 | # trailing slash. 71 | # Examples: "http://example.com/media/", "http://media.example.com/" 72 | MEDIA_URL = "" 73 | 74 | # Absolute path to the directory static files should be collected to. 75 | # Don't put anything in this directory yourself; store your static files 76 | # in apps' "static/" subdirectories and in STATICFILES_DIRS. 77 | # Example: "/var/www/example.com/static/" 78 | STATIC_ROOT = os.path.join(SITE_ROOT, "static-collected") 79 | 80 | # Additional locations of static files 81 | STATICFILES_DIRS = ( 82 | # Put strings here, like "/home/html/static" or "C:/www/django/static". 83 | # Always use forward slashes, even on Windows. 84 | # Don't forget to use absolute paths, not relative paths. 85 | ) 86 | 87 | # List of finder classes that know how to find static files in 88 | # various locations. 89 | STATICFILES_FINDERS = ( 90 | "django.contrib.staticfiles.finders.FileSystemFinder", 91 | "django.contrib.staticfiles.finders.AppDirectoriesFinder", 92 | ) 93 | 94 | # Make this unique, and don't share it with anybody. 95 | SECRET_KEY = os.environ.get( 96 | "BLAGGREGATOR_SECRET_KEY", "%dut3)!1f(nm0x8bm@tuj!*!2oe=+3+bsw2lf0)%(4l8d2^z8s" 97 | ) 98 | 99 | MIDDLEWARE = ( 100 | "home.middleware.RecurseSubdomainMiddleware", 101 | "django.middleware.common.CommonMiddleware", 102 | "django.contrib.sessions.middleware.SessionMiddleware", 103 | "django.middleware.csrf.CsrfViewMiddleware", 104 | "django.contrib.auth.middleware.AuthenticationMiddleware", 105 | "django.contrib.messages.middleware.MessageMiddleware", 106 | "social_django.middleware.SocialAuthExceptionMiddleware", 107 | ) 108 | 109 | ROOT_URLCONF = "blaggregator.urls" 110 | 111 | # Python dotted path to the WSGI application used by Django's runserver. 112 | WSGI_APPLICATION = "blaggregator.wsgi.application" 113 | 114 | TEMPLATES = [ 115 | { 116 | "BACKEND": "django.template.backends.django.DjangoTemplates", 117 | "APP_DIRS": True, 118 | "OPTIONS": { 119 | "debug": DEBUG, 120 | "context_processors": [ 121 | "django.template.context_processors.request", 122 | "django.template.context_processors.debug", 123 | "django.contrib.auth.context_processors.auth", 124 | "django.template.context_processors.i18n", 125 | "django.template.context_processors.media", 126 | "django.template.context_processors.csrf", 127 | "django.contrib.messages.context_processors.messages", 128 | "social_django.context_processors.backends", 129 | "social_django.context_processors.login_redirect", 130 | "home.context_processors.primary_blog", 131 | ], 132 | }, 133 | }, 134 | ] 135 | 136 | MESSAGE_STORAGE = "django.contrib.messages.storage.session.SessionStorage" 137 | 138 | INSTALLED_APPS = ( 139 | "django.contrib.auth", 140 | "django.contrib.contenttypes", 141 | "django.contrib.sessions", 142 | "django.contrib.messages", 143 | "django.contrib.staticfiles", 144 | "django.contrib.postgres", 145 | "home", 146 | "django.contrib.admin", 147 | "storages", 148 | "django.contrib.humanize", 149 | "social_django", 150 | ) 151 | 152 | AUTHENTICATION_BACKENDS = ( 153 | "home.oauth.HackerSchoolOAuth2", 154 | "home.token_auth.TokenAuthBackend", 155 | ) 156 | 157 | LOGIN_URL = "/login/" 158 | HS_PERSONAL_TOKEN = os.environ["HS_PERSONAL_TOKEN"] 159 | SOCIAL_AUTH_HACKERSCHOOL_KEY = os.environ["SOCIAL_AUTH_HS_KEY"] 160 | SOCIAL_AUTH_HACKERSCHOOL_SECRET = os.environ["SOCIAL_AUTH_HS_SECRET"] 161 | SOCIAL_AUTH_HACKERSCHOOL_LOGIN_URL = "/login" 162 | SOCIAL_AUTH_LOGIN_REDIRECT_URL = "/new/" 163 | SOCIAL_AUTH_LOGIN_ERROR_URL = "/login-error/" 164 | 165 | SOCIAL_AUTH_HACKERSCHOOL_REDIRECT_URL = "http://localhost:8000/complete/hackerschool" 166 | 167 | SOCIAL_AUTH_PIPELINE = ( 168 | "social_core.pipeline.social_auth.social_details", 169 | "social_core.pipeline.social_auth.social_uid", 170 | "social_core.pipeline.social_auth.auth_allowed", 171 | "social_core.pipeline.social_auth.social_user", 172 | "social_core.pipeline.user.get_username", 173 | "home.oauth.create_user", 174 | "social_core.pipeline.social_auth.associate_user", 175 | "social_core.pipeline.social_auth.load_extra_data", 176 | "social_core.pipeline.user.user_details", 177 | "home.oauth.create_or_update_hacker", 178 | ) 179 | 180 | # A sample logging configuration. The only tangible logging 181 | # performed by this configuration is to send an email to 182 | # the site admins on every HTTP 500 error when DEBUG=False. 183 | # See http://docs.djangoproject.com/en/dev/topics/logging for 184 | # more details on how to customize your logging configuration. 185 | LOGGING = { 186 | "version": 1, 187 | "disable_existing_loggers": False, 188 | "filters": {"require_debug_false": {"()": "django.utils.log.RequireDebugFalse"}}, 189 | "handlers": { 190 | "mail_admins": { 191 | "level": "ERROR", 192 | "filters": ["require_debug_false"], 193 | "class": "django.utils.log.AdminEmailHandler", 194 | }, 195 | "console": {"level": "DEBUG", "class": "logging.StreamHandler"}, 196 | }, 197 | "loggers": { 198 | "django.request": { 199 | "handlers": ["console"], 200 | "level": "ERROR", 201 | "propagate": True, 202 | }, 203 | "blaggregator": { 204 | "handlers": ["console"], 205 | "level": "DEBUG", 206 | "propagate": True, 207 | }, 208 | }, 209 | } 210 | 211 | ADMIN_MEDIA_PREFIX = STATIC_URL + "admin/" 212 | 213 | # Feed configuration 214 | 215 | # Maximum number of entries to have in the feed. To balance between sending a 216 | # huge feed and forcing users to update feed super-frequently, a good value for 217 | # this would be around twice the average number of posts in a week. 218 | MAX_FEED_ENTRIES = 100 219 | 220 | # Max number of posts per blog to announce on Zulip per crawl 221 | MAX_POST_ANNOUNCE = 2 222 | -------------------------------------------------------------------------------- /blaggregator/settings/docker.py: -------------------------------------------------------------------------------- 1 | DATABASES = { 2 | "default": { 3 | "ENGINE": "django.db.backends.postgresql_psycopg2", 4 | "NAME": "postgres", 5 | "USER": "postgres", 6 | "PASSWORD": "postgres", 7 | "HOST": "db", 8 | "PORT": "5432", 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /blaggregator/settings/heroku.py: -------------------------------------------------------------------------------- 1 | import os 2 | import dj_database_url 3 | from .base import DATABASES 4 | 5 | # Parse database configuration from $DATABASE_URL 6 | DATABASES["default"] = dj_database_url.config() 7 | # Honor the 'X-Forwarded-Proto' header for request.is_secure() 8 | SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") 9 | # S3 10 | AWS_ACCESS_KEY_ID = os.environ["AWS_ACCESS_KEY_ID"] 11 | AWS_SECRET_ACCESS_KEY = os.environ["AWS_SECRET_ACCESS_KEY"] 12 | STATICFILES_STORAGE = "storages.backends.s3boto3.S3Boto3Storage" 13 | DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage" 14 | -------------------------------------------------------------------------------- /blaggregator/settings/production.py: -------------------------------------------------------------------------------- 1 | from .base import ALLOWED_HOSTS 2 | from .heroku import * # noqa 3 | 4 | ROOT_URL = "https://blaggregator.herokuapp.com/" 5 | ALLOWED_HOSTS += [ 6 | "blaggregator.herokuapp.com", 7 | "www.blaggregator.us", 8 | "blaggregator.recurse.com", 9 | ] 10 | AWS_STORAGE_BUCKET_NAME = "blaggregator" 11 | STATIC_URL = "http://" + AWS_STORAGE_BUCKET_NAME + ".s3.amazonaws.com/" 12 | SOCIAL_AUTH_HACKERSCHOOL_REDIRECT_URL = "http://www.blaggregator.us/complete/hackerschool" 13 | -------------------------------------------------------------------------------- /blaggregator/settings/staging.py: -------------------------------------------------------------------------------- 1 | from .base import ALLOWED_HOSTS 2 | from .heroku import * # noqa 3 | 4 | ROOT_URL = "https://blag.recurse.com/" 5 | ALLOWED_HOSTS += ["blaggregator-staging.herokuapp.com", "blag.recurse.com"] 6 | AWS_STORAGE_BUCKET_NAME = "blaggregator-staging" 7 | STATIC_URL = "http://" + AWS_STORAGE_BUCKET_NAME + ".s3.amazonaws.com/" 8 | SOCIAL_AUTH_HACKERSCHOOL_REDIRECT_URL = ( 9 | "http://blaggregator-staging.herokuapp.com/complete/hackerschool" 10 | ) 11 | -------------------------------------------------------------------------------- /blaggregator/settings/travis.py: -------------------------------------------------------------------------------- 1 | DATABASES = { 2 | "default": { 3 | "ENGINE": "django.db.backends.postgresql_psycopg2", 4 | "NAME": "travisdb", # Must match travis.yml setting 5 | "USER": "postgres", 6 | "PASSWORD": "", 7 | "HOST": "localhost", 8 | "PORT": "", 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /blaggregator/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from django.views.static import serve 3 | from django.urls import include, path, re_path 4 | 5 | from django.contrib import admin 6 | 7 | admin.autodiscover() 8 | 9 | urlpatterns = [ 10 | path("", include("home.urls")), 11 | path("admin/", admin.site.urls), 12 | ] 13 | 14 | if not settings.DEBUG: 15 | urlpatterns += [ 16 | re_path(r"^static/(?P.*)$", serve, {"document_root": settings.STATIC_ROOT}), 17 | ] 18 | -------------------------------------------------------------------------------- /blaggregator/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for blaggregator project. 3 | 4 | This module contains the WSGI application used by Django's development server 5 | and any production WSGI deployments. It should expose a module-level variable 6 | named ``application``. Django's ``runserver`` and ``runfcgi`` commands discover 7 | this application via the ``WSGI_APPLICATION`` setting. 8 | 9 | Usually you will have the standard Django WSGI application here, but it also 10 | might make sense to replace the whole Django WSGI application with a custom one 11 | that later delegates to the Django one. For example, you could introduce WSGI 12 | middleware here, or combine a Django application with an application of another 13 | framework. 14 | 15 | """ 16 | import os 17 | 18 | # We defer to a DJANGO_SETTINGS_MODULE already in the environment. This breaks 19 | # if running multiple sites in the same mod_wsgi process. To fix this, use 20 | # mod_wsgi daemon mode with each site in its own daemon process, or use 21 | # os.environ["DJANGO_SETTINGS_MODULE"] = "blaggregator.settings" 22 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "blaggregator.settings") 23 | 24 | # This application object is used by any WSGI server configured to use this 25 | # file. This includes Django's development server, if the WSGI_APPLICATION 26 | # setting points here. 27 | from django.core.wsgi import get_wsgi_application 28 | 29 | application = get_wsgi_application() 30 | 31 | # Apply WSGI middleware here. 32 | # from helloworld.wsgi import HelloWorldApplication 33 | # application = HelloWorldApplication(application) 34 | -------------------------------------------------------------------------------- /docker-compose.pg.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | services: 3 | db: 4 | image: postgres 5 | command: ["postgres", "-c", "log_statement=all"] 6 | ports: 7 | - 5432:5432 8 | environment: 9 | - POSTGRES_USER=sasha 10 | - POSTGRES_PASSWORD=sasha 11 | - POSTGRES_DB=blaggregator_dev 12 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | services: 3 | db: 4 | image: postgres 5 | web: 6 | build: . 7 | command: python3 manage.py runserver 0.0.0.0:8000 8 | depends_on: 9 | - db 10 | - migration 11 | env_file: 12 | - web-variables.env 13 | ports: 14 | - "8000:8000" 15 | volumes: 16 | - .:/code 17 | migration: 18 | build: . 19 | image: app 20 | command: python manage.py migrate 21 | env_file: 22 | - web-variables.env 23 | volumes: 24 | - .:/code 25 | links: 26 | - db 27 | depends_on: 28 | - db 29 | -------------------------------------------------------------------------------- /docs/development.md: -------------------------------------------------------------------------------- 1 | Blaggregator is a Django app with a Bootstrap frontend, that is deployed on 2 | Heroku and uses their Postgres and Scheduler add-ons. 3 | 4 | This document aims to help you setup a development environment and to help you 5 | get started with the code. 6 | 7 | ## Installation: 8 | 9 | ### Docker Setup 10 | Requires Docker and [Docker-compose](https://docs.docker.com/compose/install/) 11 | to simplify setup of development environment. See 12 | the [traditional setup section](#traditional-setup) if you don't wish to use 13 | docker. 14 | 15 | In `web-variables.env` fill out your credentials. See 16 | [Credentials](#credentials) for instructions on generating. 17 | 18 | ```bash 19 | HS_PERSONAL_TOKEN= 20 | SOCIAL_AUTH_HS_KEY= 21 | SOCIAL_AUTH_HS_SECRET= 22 | ``` 23 | 24 | ```bash 25 | docker-compose up 26 | # Runs migrations and the app using runserver 27 | # You can ssh into box to run other commands 28 | ``` 29 | 30 | #### Using Docker for the DB 31 | 32 | It is possible to use Docker just for the DB and use the traditional setup for 33 | setting up your Python development environment. 34 | 35 | ```bash 36 | docker-compose -f docker-compose.pg.yml up --build 37 | ``` 38 | 39 | This will start a Postgres server that the server running on your local host 40 | machine can connect to, with no additional changes to the configuration. 41 | 42 | ### Traditional Setup 43 | - Set up your virtual environment 44 | 45 | - Install dependencies: 46 | 47 | ```bash 48 | $ pip install -r requirements.txt 49 | ``` 50 | 51 | - Install Postgres (it's easy on OSX 52 | with [postgres.app](http://postgresapp.com/)) and `pip install 53 | psycopg2`. Open the app to start your database server running locally. 54 | 55 | It is possible to use Docker for running the DB, if you prefer. See section 56 | Using Docker for the DB above. 57 | 58 | - Open a Postgres shell: 59 | 60 | ```bash 61 | $ psql 62 | ``` 63 | 64 | - Create your database: 65 | 66 | ```sql 67 | CREATE DATABASE blaggregator_dev; 68 | ``` 69 | 70 | The semicolon is critical. IMPORTANT: when you are creating your admin 71 | account on the db, *don't* use the same email address as your Recurse 72 | Center account or you won't be able to create a user account for 73 | yourself. Do username+root@example.com or something. 74 | 75 | - Create the user for Django to use. As the `postgres` user, you can run the following: 76 | 77 | ```shell 78 | createuser --interactive --pwprompt 79 | ``` 80 | 81 | Set username and password to `sasha`. Let the new user create and delete databases. 82 | 83 | - Alternatively, you could use Sqlite instead of Postgres to get off the 84 | blocks, quickly. Change the value of `'ENGINE'` in the 85 | `DATABASES['default']` dictionary to `'django.db.backends.sqlite3'`. 86 | 87 | - Set up initial tables: 88 | 89 | ```bash 90 | $ python manage.py migrate 91 | ``` 92 | 93 | - Bring the tables up to date with the latest South migrations: 94 | 95 | ```bash 96 | $ python manage.py migrate 97 | ``` 98 | 99 | If you get this error: 100 | 101 | ``` 102 | OperationalError: could not connect to server: No such file or directory 103 | Is the server running locally and accepting 104 | connections on Unix domain socket "/var/pgsql_socket/.s.PGSQL.5432"? 105 | ``` 106 | then your server isn't running. Go fiddle with Postgres.app. 107 | 108 | - Turn on debugging in your environment so you can get useful error messages: 109 | 110 | ```bash 111 | $ export DJANGO_DEBUG=True 112 | ``` 113 | 114 | - Blaggregator uses oauth2 to log in users against recurse.com and tokens to update user details. Store `SOCIAL_AUTH_HS_KEY`, `SOCIAL_AUTH_HS_SECRET`, and HS_PERSONAL_TOKEN in your environment. See [Credentials](#credentials) for instructions on generating these. 115 | 116 | 117 | - Then run a local server: 118 | 119 | ```bash 120 | $ python manage.py runserver 121 | ``` 122 | 123 | You can administer your app through 124 | the [handy-dandy admin interface](http://localhost:8000/admin). To see this 125 | page, you'll need to give your user account superuser privileges: 126 | 127 | 1. go to http://localhost:8000/ and auth in through HS's oauth 128 | 2. `$ python manage.py shell` to open Django's shell and use its ORM 129 | 3. `>>> from django.contrib.auth.models import User` import the User model 130 | (should you need the other models defined in `models.py`, import from 131 | `home.models`. User uses Django's built-in User model) 132 | 4. `>>> u = User.objects.get(first_name="Sasha")` or whatever your first name 133 | is. Grab your user object from the db. 134 | 5. `>>> u.is_superuser = True` make your account a superuser so you can access 135 | the admin 136 | 6. `>>> u.save()` Save these changes to the db. 137 | 7. You should now be able to access localhost:8000/admin while logged in! 138 | 139 | ## Setup 140 | 141 | ### Credentials 142 | 143 | Go to [your settings on recurse.com](https://www.recurse.com/settings/apps), make a new app. Name it something like "blaggregator-local" and the url should be http://localhost:8000/complete/hackerschool/ (WITH trailing slash). There you can get `SOCIAL_AUTH_HS_KEY` and `SOCIAL_AUTH_HS_SECRET`. 144 | 145 | Also create a personal access token and give it a name. This will generate the token to use for `HS_PERSONAL_TOKEN`. 146 | 147 | 148 | ## Code overview 149 | 150 | Key files: 151 | 152 | - `home/views.py`: the heart of the app. all of the views ("controllers" if 153 | you're coming from Ruby) 154 | - `blaggregator/settings.py`: app settings 155 | - `home/management/commands/crawlposts.py`: background crawler script 156 | - `home/feedergrabber27.py`: feed parser 157 | - `home/templates/home`: all templates live here 158 | -------------------------------------------------------------------------------- /docs/server-maintenance-and-deploys.md: -------------------------------------------------------------------------------- 1 | # Server maintenance and Deploys 2 | 3 | The deploy process is currently pretty manual. 4 | 5 | - Test stuff locally first. "Testing" in this case involves clicking around, 6 | creating new entries, deleting them, and generally manually trying to break 7 | it. 8 | 9 | - Push it to staging, and try again to break it. 10 | 11 | ```bash 12 | git push staging yourbranch:master 13 | ``` 14 | 15 | - If that all looks good, deploy to production! Traffic is generally very low 16 | at night in the US. People could be notified on Zulip about downtime, but 17 | weight it against the spam cost of notifying people of downtime they will 18 | never experience. 19 | 20 | ```bash 21 | git push heroku master:master 22 | ``` 23 | 24 | ## Rules of thumb 25 | 26 | - Avoid pushing things that may break. 27 | - QA really hard, before anything goes out. 28 | 29 | Users' trust is a hard earned thing. Safeguard it! 30 | 31 | # Staging environment 32 | 33 | To develop and test features, we have a staging environment at 34 | `blaggregator-staging` 35 | 36 | Instructions for configuring git locally: 37 | https://devcenter.heroku.com/articles/multiple-environments 38 | 39 | It's got a partial copy of the production database. It can be 40 | manually [updated manually](DB-dump.md). Heroku lets you do 41 | [db-to-db copies](https://devcenter.heroku.com/articles/heroku-postgres-backups#direct-database-to-database-copies) 42 | 43 | It **doesn't** have Zulip keys, so no worries about accidents there. 44 | 45 | It has an env var STAGING which determines some things in settings.py (mostly 46 | which DB and S3 bucket to use) 47 | 48 | # Heroku Database backups 49 | 50 | Blaggregator is hosted on Heroku and here is how to get the latest db dump 51 | 52 | ## Download backup 53 | 54 | Create new backup and download it. 55 | 56 | ```bash 57 | $ heroku pg:backups:capture HEROKU_POSTGRESQL_GREEN --app blaggregator 58 | $ curl -o latest.dump `heroku pg:backups public-url --app blaggregator` 59 | ``` 60 | 61 | ## Restore backup (locally) 62 | 63 | Loads the dump into your local database using the pg_restore tool. 64 | 65 | ```bash 66 | $ pg_restore --verbose --clean --no-acl --no-owner -h localhost -U sasha -d blaggregator_dev latest.dump 67 | ``` 68 | 69 | ## Restore backup on heroku (staging) 70 | 71 | ```bash 72 | $ heroku pg:backups public-url --app blaggregator # Get backup url 73 | $ heroku pg:backups restore $BACKUP_URL DATABSE_URL # $BACKUP_URL is obtained above 74 | ``` 75 | -------------------------------------------------------------------------------- /home/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/recursecenter/blaggregator/d5f3da939fcb16480be049c027e141b7aa171baa/home/__init__.py -------------------------------------------------------------------------------- /home/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | from django.contrib.auth.admin import UserAdmin 3 | from django.contrib.auth.models import User 4 | from home.models import Blog, Hacker, Post 5 | 6 | 7 | class HackerInline(admin.StackedInline): 8 | model = Hacker 9 | can_delete = False 10 | 11 | 12 | class BlogInline(admin.StackedInline): 13 | model = Blog 14 | can_delete = False 15 | 16 | 17 | class UserAdmin(UserAdmin): 18 | inlines = (HackerInline, BlogInline) 19 | 20 | 21 | admin.site.unregister(User) 22 | admin.site.register(User, UserAdmin) 23 | admin.site.register(Post) 24 | -------------------------------------------------------------------------------- /home/context_processors.py: -------------------------------------------------------------------------------- 1 | from home.models import Blog 2 | 3 | 4 | def primary_blog(request): 5 | count = Blog.objects.filter(user=request.user.id).count() 6 | return {"primary_blog_set": True if count > 0 else False} 7 | -------------------------------------------------------------------------------- /home/feedergrabber27.py: -------------------------------------------------------------------------------- 1 | """Retrieves the links and titles of recent posts from blog feeds.""" 2 | 3 | import datetime 4 | import html 5 | import socket 6 | import urllib.request 7 | import urllib.error 8 | import urllib.parse 9 | 10 | import feedparser 11 | 12 | from .utils import is_medium_comment 13 | 14 | 15 | CharacterEncodingOverride = feedparser.CharacterEncodingOverride 16 | # Set a timeout of 60 seconds for sockets - useful when crawling some blogs 17 | socket.setdefaulttimeout(60) 18 | 19 | 20 | def retrieve_file_contents(url): 21 | """Retrieve file contents from a given URL and log any errors.""" 22 | errors = [] 23 | try: 24 | file_contents = feedparser.parse(url) 25 | except (urllib.error.URLError, urllib.error.HTTPError) as e: 26 | errors.append("Fetching content for {} failed: {}".format(url, e)) 27 | file_contents = None 28 | return file_contents, errors 29 | 30 | 31 | def find_feed_url(parsed_content): 32 | """Try to find the feed url from parsed content.""" 33 | try: 34 | links = parsed_content.feed.links 35 | except AttributeError: 36 | links = [] 37 | for link in links: 38 | if link.get("type", "") in ( 39 | "application/atom+xml", 40 | "application/rss+xml", 41 | ): 42 | return link.href 43 | 44 | 45 | def feedergrabber(url): 46 | """The main function of the module.""" 47 | # Initialize some variables 48 | post_links_and_titles = [] 49 | # Get file contents 50 | file_contents, errors = retrieve_file_contents(url) 51 | if file_contents is None: 52 | return None, errors 53 | 54 | # Gather links, titles, dates and content 55 | for entry in file_contents.entries: 56 | # Link 57 | link = getattr(entry, "link", "") 58 | if not link: 59 | errors.append("No link was found for post: {}".format(url)) 60 | continue 61 | 62 | elif is_medium_comment(entry): 63 | errors.append("A medium comment was skipped: {}".format(link)) 64 | continue 65 | 66 | # Title 67 | title = getattr(entry, "title", "") 68 | if not title: 69 | errors.append("No title was returned for post: {}.".format(link)) 70 | continue 71 | 72 | title = html.unescape(title).replace("\n", " ") 73 | # Date 74 | post_date = getattr(entry, "published_parsed", getattr(entry, "updated_parsed", None)) 75 | now = datetime.datetime.now() 76 | if post_date is None: 77 | # No date posts are marked as crawled now 78 | post_date = now 79 | else: 80 | post_date = datetime.datetime(*post_date[:6]) 81 | # future dated posts are marked as crawled now 82 | if post_date > now: 83 | post_date = now 84 | # posts dated 0001-01-01 are ignored -- common for _pages_ in hugo feeds 85 | elif post_date == datetime.datetime.min: 86 | errors.append("Has min date - hugo page?: {}".format(link)) 87 | continue 88 | 89 | # Post content 90 | content = getattr(entry, "summary", "") 91 | # Append 92 | post_links_and_titles.append((link, title, post_date, content)) 93 | if len(post_links_and_titles) == 0: 94 | post_links_and_titles = None 95 | errors.append(url + ": Parsing methods not successful.") 96 | return post_links_and_titles, errors 97 | -------------------------------------------------------------------------------- /home/feeds.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | from django.conf import settings 4 | from django.contrib.syndication.views import Feed 5 | from django.utils import feedgenerator 6 | 7 | from home.models import Post 8 | 9 | 10 | # This code has been taken from 11 | # https://salsa.debian.org/qa/distro-tracker/blob/d22c25f84c92cd34ce63f131f0f2fb604453dc16/distro_tracker/core/news_feed.py#L27 12 | def filter_control_chars(method): 13 | # We have to filter out control chars otherwise the FeedGenerator 14 | # raises UnserializableContentError (see django/utils/xmlutils.py) 15 | def wrapped(self, obj): 16 | result = method(self, obj) 17 | return re.sub(r"[\x00-\x08\x0B-\x0C\x0E-\x1F]", "", result) 18 | 19 | return wrapped 20 | 21 | 22 | class LatestEntriesFeed(Feed): 23 | feed_type = feedgenerator.Atom1Feed 24 | title = "Blaggregator" 25 | link = "/atom.xml" 26 | description = "Syndicated feed for blaggregator." 27 | description_template = "home/feed_item.tmpl" 28 | 29 | def items(self): 30 | return Post.objects.all()[: settings.MAX_FEED_ENTRIES] 31 | 32 | @filter_control_chars 33 | def item_title(self, item): 34 | return item.title 35 | 36 | def item_link(self, item): 37 | return item.url 38 | 39 | def item_author_name(self, item): 40 | user = item.blog.user 41 | return user.first_name + " " + user.last_name 42 | 43 | def item_pubdate(self, item): 44 | return item.posted_at 45 | -------------------------------------------------------------------------------- /home/management/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/recursecenter/blaggregator/d5f3da939fcb16480be049c027e141b7aa171baa/home/management/__init__.py -------------------------------------------------------------------------------- /home/management/commands/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/recursecenter/blaggregator/d5f3da939fcb16480be049c027e141b7aa171baa/home/management/commands/__init__.py -------------------------------------------------------------------------------- /home/management/commands/crawlposts.py: -------------------------------------------------------------------------------- 1 | # Standard library 2 | from collections import deque # noqa 3 | import logging # noqa 4 | 5 | # 3rd-party library 6 | from django.core.management.base import BaseCommand # noqa 7 | from django.conf import settings # noqa 8 | from django.utils import timezone # noqa 9 | from gevent import pool, wait # noqa 10 | 11 | # Local library 12 | from home import feedergrabber27 # noqa 13 | from home.models import Blog, Post # noqa 14 | from home.zulip_helpers import announce_posts # noqa 15 | 16 | log = logging.getLogger("blaggregator") 17 | 18 | 19 | class Command(BaseCommand): 20 | help = "Periodically crawls all blogs for new posts." 21 | # Queue up the messages for Zulip so they aren't sent until after the 22 | # blog post instance is created in the database 23 | zulip_queue = deque() 24 | 25 | def crawlblog(self, blog): 26 | # Feedergrabber returns ( [(link, title, date, content)], [errors]) 27 | print(f"Crawling {blog.feed_url} ...") 28 | crawled, errors = feedergrabber27.feedergrabber(blog.feed_url) 29 | if not crawled: 30 | log.debug("\n".join(errors)) 31 | return 32 | 33 | log.debug("Crawled %s posts from %s", len(crawled), blog.feed_url) 34 | if errors: 35 | log.debug("\n".join(errors)) 36 | blog.last_crawled = timezone.now() 37 | blog.save(update_fields=["last_crawled"]) 38 | created_count = 0 39 | for link, title, date, content in crawled: 40 | date = timezone.make_aware(date, timezone.get_default_timezone()) 41 | # create the post instance if it doesn't already exist 42 | post, created = get_or_create_post(blog, title, link, date, content) 43 | if created: 44 | created_count += 1 45 | log.debug("Created '%s' from blog '%s'", title, blog.feed_url) 46 | # Throttle the amount of new posts that can be announced per 47 | # user per crawl. 48 | if created_count <= settings.MAX_POST_ANNOUNCE: 49 | self.zulip_queue.append(post) 50 | else: 51 | update_post(post, title, link, content) 52 | 53 | def handle(self, **options): 54 | p = pool.Pool(20) 55 | jobs = [p.spawn(self.crawlblog, blog) for blog in Blog.objects.filter(skip_crawl=False)] 56 | wait(jobs) 57 | announce_posts(self.zulip_queue, debug=settings.DEBUG) 58 | 59 | 60 | def get_or_create_post(blog, title, link, date, content): 61 | try: 62 | # The previous code checked only for url, and therefore, the db can 63 | # have posts with duplicate titles. So, we check if there is atleast 64 | # one post with the title -- using `filter.latest` instead of `get`. 65 | post = Post.objects.filter(blog=blog, title=title).latest("posted_at") 66 | return post, False 67 | 68 | except Post.DoesNotExist: 69 | pass 70 | post, created = Post.objects.get_or_create( 71 | blog=blog, 72 | url=link, 73 | defaults={"title": title, "posted_at": date, "content": content}, 74 | ) 75 | return post, created 76 | 77 | 78 | def update_post(post, title, link, content): 79 | """Update a post with the new content. 80 | 81 | Updates if title, link or content has changed. Date is ignored since it may 82 | not always be parsed correctly, and we sometimes just use datetime.now() 83 | when parsing feeds. 84 | 85 | """ 86 | if post.title == title and post.url == link and post.content == content: 87 | return 88 | 89 | post.title = title 90 | post.url = link 91 | post.content = content 92 | post.save() 93 | log.debug("Updated %s - %s.", title, link) 94 | -------------------------------------------------------------------------------- /home/management/commands/de-dup-posts.py: -------------------------------------------------------------------------------- 1 | # Standard library 2 | import logging 3 | from urllib.parse import urlparse 4 | 5 | # 3rd-party library 6 | from django.core.management.base import BaseCommand 7 | from django.db.models import Count 8 | import grequests 9 | 10 | # Local library 11 | from home.models import Blog, Post 12 | 13 | log = logging.getLogger("blaggregator") 14 | 15 | 16 | class Command(BaseCommand): 17 | help = "Remove posts with duplicated titles." 18 | 19 | def handle(self, **options): 20 | delete_duplicate_title_posts() 21 | 22 | 23 | def delete_duplicate_title_posts(): 24 | for blog, titles in iter_blogs_with_duplicate_titles(): 25 | # Get the base_url for the blog 26 | parsed_url = urlparse(blog.feed_url) 27 | base_url = "{}://{}".format(parsed_url.scheme, parsed_url.netloc) 28 | print("Fetching posts for {}".format(base_url)) 29 | urls = set() 30 | # Delete posts if we can uniquify with base name otherwise collect urls 31 | for title, posts in iter_posts_with_duplicate_titles(blog, titles): 32 | base_url_matches = posts.filter(url__startswith=base_url) 33 | if base_url_matches.count() == 1: 34 | # Delete all other posts, if there's only post starting with base_url 35 | keep = base_url_matches.first().id 36 | posts.exclude(id=keep).delete() 37 | else: 38 | urls = urls.union(set(posts.values_list("url", flat=True))) 39 | if not urls: 40 | continue 41 | 42 | # Do web requests to figure out which URLs still work 43 | urls = list(urls) 44 | print("Requesting {} urls".format(len(urls))) 45 | requests = (grequests.get(u, allow_redirects=True, timeout=30) for u in urls) 46 | responses = grequests.map(requests) 47 | successful = dict(list(filter(filter_successful, list(zip(urls, responses))))) 48 | # Delete duplicate posts based on successful get or keep latest post #### 49 | for title, posts in iter_posts_with_duplicate_titles(blog, titles): 50 | for post in posts: 51 | if post.url in successful: 52 | posts.exclude(id=post.id).delete() 53 | break 54 | 55 | else: 56 | posts.exclude(id=posts.first().id).delete() 57 | 58 | 59 | def filter_successful(pair): 60 | (url, response) = pair 61 | return response is not None and response.status_code == 200 62 | 63 | 64 | def iter_blogs_with_duplicate_titles(): 65 | for blog in Blog.objects.all(): 66 | # Collect all duplicate titles 67 | posts = Post.objects.filter(blog=blog) 68 | duplicate_titles = ( 69 | posts.values("title") 70 | .annotate(Count("id")) 71 | .order_by() 72 | .filter(id__count__gt=1) 73 | ) 74 | if not duplicate_titles.exists(): 75 | continue 76 | 77 | yield blog, duplicate_titles 78 | 79 | 80 | def iter_posts_with_duplicate_titles(blog, titles): 81 | for title in titles: 82 | duplicate_posts = Post.objects.filter( 83 | blog=blog, title=title["title"] 84 | ).distinct() 85 | if duplicate_posts.count() <= 1: 86 | continue 87 | 88 | yield title, duplicate_posts 89 | -------------------------------------------------------------------------------- /home/management/commands/delete_medium_comments.py: -------------------------------------------------------------------------------- 1 | # Standard library 2 | import logging 3 | import re 4 | 5 | # 3rd-party library 6 | from django.core.management.base import BaseCommand 7 | 8 | # Local library 9 | from home.models import Post 10 | from home.utils import is_medium_comment 11 | from home.zulip_helpers import delete_message, get_stream_messages 12 | 13 | log = logging.getLogger("blaggregator") 14 | EMAIL_PREFIX = "blaggregator-bot@" 15 | 16 | 17 | class Command(BaseCommand): 18 | help = "Delete all medium comments and zulip announcements for them." 19 | 20 | def add_arguments(self, parser): 21 | parser.add_argument( 22 | "--stream", 23 | dest="stream", 24 | type=str, 25 | default="blogging", 26 | help="The stream to clean-up Zulip announcements from.", 27 | ) 28 | 29 | def handle(self, **options): 30 | slugs = [ 31 | post.slug 32 | for post in Post.objects.filter(url__icontains="medium.com") 33 | if is_medium_comment(post) 34 | ] 35 | delete_messages_with_slugs(slugs, options["stream"]) 36 | Post.objects.filter(slug__in=slugs).delete() 37 | 38 | 39 | def delete_messages_with_slugs(slugs, stream): 40 | messages = get_stream_messages(stream) 41 | bot_messages = [ 42 | message 43 | for message in messages 44 | if message["sender_email"].startswith(EMAIL_PREFIX) 45 | ] 46 | slug_re = re.compile("/post/({})/view".format("|".join(slugs))) 47 | for message in bot_messages: 48 | if slug_re.search(message["content"]): 49 | delete_message(message["id"], "(medium comment deleted)") 50 | -------------------------------------------------------------------------------- /home/management/commands/notify_uncrawlable_blogs.py: -------------------------------------------------------------------------------- 1 | # Standard library 2 | 3 | from datetime import timedelta 4 | import logging 5 | 6 | # 3rd-party library 7 | from django.core.management.base import BaseCommand 8 | from django.conf import settings 9 | from django.db.models import Q 10 | from django.utils import timezone 11 | from django.utils.text import get_text_list 12 | 13 | # Local library 14 | from home.models import Blog, User 15 | from home import zulip_helpers as Z 16 | 17 | log = logging.getLogger("blaggregator") 18 | 19 | 20 | class Command(BaseCommand): 21 | help = "Notify owners of blogs with failing crawls." 22 | 23 | def handle(self, **options): 24 | last_week = timezone.now() - timedelta(days=7) 25 | flagging_filter = (Q(last_crawled=None) | Q(last_crawled__lt=last_week)) & Q( 26 | skip_crawl=False 27 | ) 28 | flagged_blogs = Blog.objects.filter(flagging_filter).distinct() 29 | log.debug( 30 | "Notifying %s blog owners about blog crawling errors", 31 | flagged_blogs.count(), 32 | ) 33 | if not settings.DEBUG: 34 | yes_or_no = input("Are you sure you want to continue? [y/N]: ") 35 | if yes_or_no.lower().strip()[:1] != "y": 36 | return 37 | 38 | # Skip crawls on these blogs in future, until the feed_url changes 39 | notified = set() 40 | notify_failed = set() 41 | ids = ( 42 | flagged_blogs.order_by("user") 43 | .distinct("user") 44 | .values_list("user", flat=True) 45 | ) 46 | users = User.objects.filter(id__in=ids) 47 | zulip_members = Z.get_members() 48 | admins = User.objects.filter(is_staff=True).exclude(hacker=None) 49 | links = [Z.get_pm_link(admin, zulip_members) for admin in admins] 50 | links = get_text_list(links, "or") 51 | debug = settings.DEBUG 52 | for user in Z.guess_zulip_emails(users, zulip_members): 53 | blogs = flagged_blogs.filter(user=user) 54 | if Z.notify_uncrawlable_blogs(user, blogs, links, debug=debug): 55 | notified.add(user.email) 56 | else: 57 | notify_failed.add(user.email) 58 | # Logging notification success/failure 59 | if notified: 60 | flagged_blogs.filter(user__email__in=notified).update(skip_crawl=True) 61 | log.debug( 62 | "Notified %s blog owners and turned off crawling", 63 | len(notified), 64 | ) 65 | if notify_failed: 66 | log.debug("Failed to notify %s users:", len(notify_failed)) 67 | log.debug("\n".join(notify_failed)) 68 | -------------------------------------------------------------------------------- /home/management/commands/update_user_details.py: -------------------------------------------------------------------------------- 1 | # Standard library 2 | 3 | import logging 4 | 5 | # 3rd-party library 6 | from django.core.management.base import BaseCommand 7 | 8 | # Local library 9 | from home.models import User 10 | from home.oauth import update_user_details 11 | 12 | log = logging.getLogger("blaggregator") 13 | 14 | 15 | class Command(BaseCommand): 16 | help = "Update user details from RC API." 17 | 18 | def handle(self, **options): 19 | users = User.objects.exclude(hacker=None) 20 | log.debug("Updating %s users", users.count()) 21 | for user_id in users.values_list("id", flat=True): 22 | log.debug("Updating user: %s", user_id) 23 | update_user_details(user_id) 24 | -------------------------------------------------------------------------------- /home/middleware.py: -------------------------------------------------------------------------------- 1 | from django import http 2 | 3 | 4 | class RecurseSubdomainMiddleware: 5 | """Middleware to redirect all users to recurse subdomain.""" 6 | 7 | HTTPS_REDIRECTS = { 8 | "blaggregator.us": "blaggregator.recurse.com", 9 | "www.blaggregator.us": "blaggregator.recurse.com", 10 | } 11 | 12 | def __init__(self, get_response): 13 | self.get_response = get_response 14 | 15 | def __call__(self, request): 16 | """Check for old HTTP_HOST and redirect to recurse subdomain.""" 17 | 18 | host = request.get_host() 19 | if host in self.HTTPS_REDIRECTS: 20 | request.META["HTTP_HOST"] = self.HTTPS_REDIRECTS[host] 21 | newurl = request.build_absolute_uri().replace("http://", "https://") 22 | return http.HttpResponsePermanentRedirect(newurl) 23 | 24 | return self.get_response(request) 25 | -------------------------------------------------------------------------------- /home/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.10 on 2016-12-02 04:10 3 | 4 | 5 | from django.conf import settings 6 | from django.db import migrations, models 7 | import django.db.models.deletion 8 | import home.models 9 | 10 | 11 | class Migration(migrations.Migration): 12 | 13 | initial = True 14 | 15 | dependencies = [ 16 | migrations.swappable_dependency(settings.AUTH_USER_MODEL), 17 | ] 18 | 19 | operations = [ 20 | migrations.CreateModel( 21 | name="Blog", 22 | fields=[ 23 | ( 24 | "id", 25 | models.AutoField( 26 | auto_created=True, 27 | primary_key=True, 28 | serialize=False, 29 | verbose_name="ID", 30 | ), 31 | ), 32 | ("url", models.URLField()), 33 | ("feed_url", models.URLField()), 34 | ( 35 | "last_crawled", 36 | models.DateTimeField( 37 | blank=True, null=True, verbose_name=b"last crawled" 38 | ), 39 | ), 40 | ( 41 | "created", 42 | models.DateTimeField( 43 | auto_now_add=True, verbose_name=b"date created" 44 | ), 45 | ), 46 | ( 47 | "stream", 48 | models.CharField( 49 | choices=[(b"BLOGGING", b"blogging"), (b"LOGS", b"Daily Logs")], 50 | default=b"BLOGGING", 51 | max_length=100, 52 | ), 53 | ), 54 | ( 55 | "user", 56 | models.ForeignKey( 57 | on_delete=django.db.models.deletion.CASCADE, 58 | to=settings.AUTH_USER_MODEL, 59 | ), 60 | ), 61 | ], 62 | ), 63 | migrations.CreateModel( 64 | name="Hacker", 65 | fields=[ 66 | ( 67 | "id", 68 | models.AutoField( 69 | auto_created=True, 70 | primary_key=True, 71 | serialize=False, 72 | verbose_name="ID", 73 | ), 74 | ), 75 | ("avatar_url", models.TextField(blank=True)), 76 | ("github", models.TextField(blank=True)), 77 | ("twitter", models.TextField(blank=True)), 78 | ( 79 | "token", 80 | models.SlugField( 81 | default=home.models.token_default, max_length=40, unique=True 82 | ), 83 | ), 84 | ( 85 | "user", 86 | models.OneToOneField( 87 | on_delete=django.db.models.deletion.CASCADE, 88 | to=settings.AUTH_USER_MODEL, 89 | ), 90 | ), 91 | ], 92 | ), 93 | migrations.CreateModel( 94 | name="LogEntry", 95 | fields=[ 96 | ( 97 | "id", 98 | models.AutoField( 99 | auto_created=True, 100 | primary_key=True, 101 | serialize=False, 102 | verbose_name="ID", 103 | ), 104 | ), 105 | ("date", models.DateTimeField()), 106 | ("referer", models.URLField(blank=True, null=True)), 107 | ("remote_addr", models.GenericIPAddressField(blank=True, null=True)), 108 | ("user_agent", models.TextField(blank=True, null=True)), 109 | ], 110 | ), 111 | migrations.CreateModel( 112 | name="Post", 113 | fields=[ 114 | ( 115 | "id", 116 | models.AutoField( 117 | auto_created=True, 118 | primary_key=True, 119 | serialize=False, 120 | verbose_name="ID", 121 | ), 122 | ), 123 | ("url", models.TextField()), 124 | ("title", models.TextField(blank=True)), 125 | ("content", models.TextField()), 126 | ( 127 | "date_posted_or_crawled", 128 | models.DateTimeField(verbose_name=b"date updated"), 129 | ), 130 | ( 131 | "slug", 132 | models.CharField( 133 | default=home.models.generate_random_id, 134 | max_length=6, 135 | unique=True, 136 | ), 137 | ), 138 | ( 139 | "blog", 140 | models.ForeignKey( 141 | on_delete=django.db.models.deletion.CASCADE, to="home.Blog" 142 | ), 143 | ), 144 | ], 145 | ), 146 | migrations.AddField( 147 | model_name="logentry", 148 | name="post", 149 | field=models.ForeignKey( 150 | on_delete=django.db.models.deletion.CASCADE, to="home.Post" 151 | ), 152 | ), 153 | ] 154 | -------------------------------------------------------------------------------- /home/migrations/0002_auto_20161201_2313.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.10 on 2016-12-02 04:13 3 | 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ("home", "0001_initial"), 12 | ] 13 | 14 | operations = [ 15 | migrations.AlterField( 16 | model_name="hacker", 17 | name="github", 18 | field=models.TextField(blank=True, null=True), 19 | ), 20 | migrations.AlterField( 21 | model_name="hacker", 22 | name="twitter", 23 | field=models.TextField(blank=True, null=True), 24 | ), 25 | ] 26 | -------------------------------------------------------------------------------- /home/migrations/0003_auto_20180313_1026.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.11 on 2018-03-13 14:26 3 | 4 | 5 | from django.db import migrations 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ("home", "0002_auto_20161201_2313"), 12 | ] 13 | 14 | operations = [ 15 | migrations.AlterModelOptions( 16 | name="post", 17 | options={"ordering": ["-date_posted_or_crawled"]}, 18 | ), 19 | ] 20 | -------------------------------------------------------------------------------- /home/migrations/0004_auto_20180316_0351.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.11 on 2018-03-16 07:51 3 | 4 | 5 | from django.db import migrations, models 6 | import django.utils.timezone 7 | 8 | 9 | class Migration(migrations.Migration): 10 | 11 | dependencies = [ 12 | ("home", "0003_auto_20180313_1026"), 13 | ] 14 | 15 | operations = [ 16 | migrations.RenameField( 17 | model_name="post", 18 | old_name="date_posted_or_crawled", 19 | new_name="created_at", 20 | ), 21 | migrations.AlterField( 22 | model_name="post", 23 | name="created_at", 24 | field=models.DateTimeField( 25 | auto_now_add=True, verbose_name=b"creation timestamp" 26 | ), 27 | ), 28 | migrations.AlterModelOptions( 29 | name="post", 30 | options={"ordering": ["-created_at"]}, 31 | ), 32 | ] 33 | -------------------------------------------------------------------------------- /home/migrations/0005_blog_skip_crawl.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.11 on 2018-03-17 04:18 3 | 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ("home", "0004_auto_20180316_0351"), 12 | ] 13 | 14 | operations = [ 15 | migrations.AddField( 16 | model_name="blog", 17 | name="skip_crawl", 18 | field=models.BooleanField( 19 | default=False, verbose_name=b"skip crawling this blog" 20 | ), 21 | ), 22 | ] 23 | -------------------------------------------------------------------------------- /home/migrations/0006_auto_20180317_0559.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.11 on 2018-03-17 09:59 3 | 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | def copy_created_at(apps, schema_editor): 9 | Post = apps.get_model("home", "Post") 10 | Post.objects.all().update(posted_at=models.F("created_at")) 11 | 12 | 13 | class Migration(migrations.Migration): 14 | 15 | dependencies = [ 16 | ("home", "0005_blog_skip_crawl"), 17 | ] 18 | 19 | operations = [ 20 | migrations.AddField( 21 | model_name="post", 22 | name="posted_at", 23 | field=models.DateTimeField(null=True, verbose_name=b"posted at"), 24 | ), 25 | migrations.RunPython(copy_created_at, lambda x, y: None), 26 | migrations.AlterField( 27 | model_name="post", 28 | name="posted_at", 29 | field=models.DateTimeField(auto_now_add=True, verbose_name=b"posted at"), 30 | preserve_default=False, 31 | ), 32 | migrations.AlterModelOptions( 33 | name="post", 34 | options={"ordering": ["-posted_at"]}, 35 | ), 36 | ] 37 | -------------------------------------------------------------------------------- /home/migrations/0007_auto_20180318_0236.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.11 on 2018-03-18 06:36 3 | 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ("home", "0006_auto_20180317_0559"), 12 | ] 13 | 14 | operations = [ 15 | migrations.RemoveField( 16 | model_name="blog", 17 | name="url", 18 | ), 19 | migrations.AlterField( 20 | model_name="post", 21 | name="posted_at", 22 | field=models.DateTimeField(verbose_name=b"posted at"), 23 | ), 24 | ] 25 | -------------------------------------------------------------------------------- /home/migrations/0008_auto_20191224_0511.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.11.23 on 2019-12-24 10:11 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ("home", "0007_auto_20180318_0236"), 12 | ] 13 | 14 | operations = [ 15 | migrations.AlterField( 16 | model_name="blog", 17 | name="created", 18 | field=models.DateTimeField(auto_now_add=True, verbose_name="date created"), 19 | ), 20 | migrations.AlterField( 21 | model_name="blog", 22 | name="last_crawled", 23 | field=models.DateTimeField( 24 | blank=True, null=True, verbose_name="last crawled" 25 | ), 26 | ), 27 | migrations.AlterField( 28 | model_name="blog", 29 | name="skip_crawl", 30 | field=models.BooleanField( 31 | default=False, verbose_name="skip crawling this blog" 32 | ), 33 | ), 34 | migrations.AlterField( 35 | model_name="blog", 36 | name="stream", 37 | field=models.CharField( 38 | choices=[("BLOGGING", "blogging"), ("LOGS", "Daily Logs")], 39 | default="BLOGGING", 40 | max_length=100, 41 | ), 42 | ), 43 | migrations.AlterField( 44 | model_name="post", 45 | name="created_at", 46 | field=models.DateTimeField( 47 | auto_now_add=True, verbose_name="creation timestamp" 48 | ), 49 | ), 50 | migrations.AlterField( 51 | model_name="post", 52 | name="posted_at", 53 | field=models.DateTimeField(verbose_name="posted at"), 54 | ), 55 | ] 56 | -------------------------------------------------------------------------------- /home/migrations/0009_hacker_zulip_id.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.2.7 on 2021-09-17 16:20 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('home', '0008_auto_20191224_0511'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AddField( 14 | model_name='hacker', 15 | name='zulip_id', 16 | field=models.PositiveIntegerField(blank=True, null=True), 17 | ), 18 | ] 19 | -------------------------------------------------------------------------------- /home/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/recursecenter/blaggregator/d5f3da939fcb16480be049c027e141b7aa171baa/home/migrations/__init__.py -------------------------------------------------------------------------------- /home/models.py: -------------------------------------------------------------------------------- 1 | import random 2 | import string 3 | import uuid 4 | 5 | from django.contrib.auth.models import User 6 | from django.db import models 7 | 8 | 9 | def generate_random_id(): 10 | return "".join( 11 | random.choice(string.ascii_uppercase + string.digits + string.ascii_lowercase) 12 | for x in range(6) 13 | ) 14 | 15 | 16 | STREAM_CHOICES = (("BLOGGING", "blogging"), ("LOGS", "Daily Logs")) 17 | 18 | 19 | def token_default(): 20 | return uuid.uuid4().hex 21 | 22 | 23 | class Hacker(models.Model): 24 | user = models.OneToOneField(User, on_delete=models.CASCADE) 25 | avatar_url = models.TextField(blank=True) 26 | github = models.TextField(blank=True, null=True) 27 | twitter = models.TextField(blank=True, null=True) 28 | zulip_id = models.PositiveIntegerField(blank=True, null=True) 29 | token = models.SlugField(max_length=40, default=token_default, unique=True) 30 | 31 | @property 32 | def full_name(self): 33 | return self.user.get_full_name() 34 | 35 | 36 | class Blog(models.Model): 37 | def __unicode__(self): 38 | return self.feed_url 39 | 40 | user = models.ForeignKey(User, on_delete=models.CASCADE) 41 | feed_url = models.URLField() 42 | last_crawled = models.DateTimeField("last crawled", blank=True, null=True) 43 | created = models.DateTimeField("date created", auto_now_add=True) 44 | stream = models.CharField(max_length=100, default=STREAM_CHOICES[0][0], choices=STREAM_CHOICES) 45 | skip_crawl = models.BooleanField("skip crawling this blog", default=False) 46 | 47 | @property 48 | def author(self): 49 | return self.user.get_full_name() 50 | 51 | @property 52 | def post_count(self): 53 | return Post.objects.filter(blog=self).count() 54 | 55 | 56 | class Post(models.Model): 57 | def __unicode__(self): 58 | return self.title 59 | 60 | blog = models.ForeignKey(Blog, on_delete=models.CASCADE) 61 | url = models.TextField() 62 | title = models.TextField(blank=True) 63 | content = models.TextField() 64 | slug = models.CharField(max_length=6, default=generate_random_id, unique=True) 65 | posted_at = models.DateTimeField("posted at") 66 | created_at = models.DateTimeField("creation timestamp", auto_now_add=True) 67 | 68 | @property 69 | def author(self): 70 | return self.blog.author 71 | 72 | @property 73 | def authorid(self): 74 | return self.blog.user.id 75 | 76 | @property 77 | def avatar(self): 78 | return self.blog.user.hacker.avatar_url 79 | 80 | @property 81 | def stream(self): 82 | return self.blog.get_stream_display() 83 | 84 | class Meta: 85 | ordering = ["-posted_at"] 86 | 87 | 88 | class LogEntry(models.Model): 89 | def __unicode__(self): 90 | return "%s %s" % (self.date, self.post) 91 | 92 | post = models.ForeignKey(Post, on_delete=models.CASCADE) 93 | date = models.DateTimeField() 94 | referer = models.URLField(blank=True, null=True) 95 | remote_addr = models.GenericIPAddressField(blank=True, null=True) 96 | user_agent = models.TextField(blank=True, null=True) 97 | -------------------------------------------------------------------------------- /home/oauth.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | from urllib.parse import urlencode 4 | 5 | import requests 6 | from django.conf import settings 7 | from social_core.backends.oauth import BaseOAuth2 8 | 9 | from .models import User, Hacker 10 | 11 | HACKER_ATTRIBUTES = ("avatar_url", "twitter", "github", "zulip_id") 12 | USER_FIELDS = ("username", "email") 13 | USER_EXTRA_FIELDS = ("first_name", "last_name", "username") 14 | log = logging.getLogger("blaggregator") 15 | 16 | 17 | def create_user(strategy, details, response, uid, user=None, *args, **kwargs): 18 | if user: 19 | return 20 | 21 | fields = dict( 22 | (name, kwargs.get(name) or details.get(name)) 23 | for name in strategy.setting("USER_FIELDS", USER_FIELDS) 24 | ) 25 | if not fields: 26 | return 27 | 28 | # The new user ID should be the same as their ID on hackerschool.com 29 | fields["id"] = details.get("id") 30 | return {"is_new": True, "user": strategy.create_user(**fields)} 31 | 32 | 33 | def create_or_update_hacker(strategy, details, response, user, *args, **kwargs): 34 | defaults = {attribute: details[attribute] for attribute in HACKER_ATTRIBUTES} 35 | hacker, created = Hacker.objects.get_or_create(user=user, defaults=defaults) 36 | if not created: 37 | for attribute, value in list(defaults.items()): 38 | setattr(hacker, attribute, value) 39 | hacker.save() 40 | 41 | 42 | def update_user(user, details): 43 | user_changed = False 44 | for field in USER_FIELDS + USER_EXTRA_FIELDS: 45 | if details[field] != getattr(user, field, None): 46 | setattr(user, field, details[field]) 47 | user_changed = True 48 | if user_changed: 49 | user.save() 50 | 51 | 52 | def update_user_details(user_id): 53 | params = urlencode({"access_token": settings.HS_PERSONAL_TOKEN}) 54 | path = "/api/v1/profiles/{}".format(user_id) 55 | base_url = HackerSchoolOAuth2.HACKER_SCHOOL_ROOT 56 | url = "{}{}?{}".format(base_url, path, params) 57 | try: 58 | response = requests.get(url) 59 | if response.status_code != 200: 60 | raise ValueError("Could not fetch data for {}".format(user_id)) 61 | 62 | user = User.objects.get(id=user_id) 63 | hacker_data = HackerSchoolOAuth2.get_user_details(response.json()) 64 | create_or_update_hacker(None, hacker_data, None, user) 65 | update_user(user, hacker_data) 66 | except Exception as e: 67 | log.debug("Failed to update user data for hacker %s", user_id) 68 | log.error(e) 69 | 70 | 71 | class HackerSchoolOAuth2(BaseOAuth2): 72 | """HackerSchool.com OAuth2 authentication backend""" 73 | 74 | name = "hackerschool" 75 | HACKER_SCHOOL_ROOT = "https://www.recurse.com" 76 | AUTHORIZATION_URL = HACKER_SCHOOL_ROOT + "/oauth/authorize" 77 | ACCESS_TOKEN_URL = HACKER_SCHOOL_ROOT + "/oauth/token" 78 | ACCESS_TOKEN_METHOD = "POST" 79 | REFRESH_TOKEN_URL = ACCESS_TOKEN_URL 80 | SCOPE_SEPARATOR = "," 81 | EXTRA_DATA = [ 82 | ("id", "id"), 83 | ("expires_in", "expires_in"), 84 | ("refresh_token", "refresh_token"), 85 | ] 86 | 87 | def auth_params(self, state=None): 88 | """Override to allow manually setting redirect_uri.""" 89 | params = super(HackerSchoolOAuth2, self).auth_params(state) 90 | redirect_uri = os.environ.get("SOCIAL_AUTH_REDIRECT_URI") 91 | if redirect_uri: 92 | params["redirect_uri"] = redirect_uri 93 | return params 94 | 95 | def auth_complete_params(self, state=None): 96 | """Override to allow manually setting redirect_uri.""" 97 | params = super(HackerSchoolOAuth2, self).auth_complete_params(state) 98 | redirect_uri = os.environ.get("SOCIAL_AUTH_REDIRECT_URI") 99 | if redirect_uri: 100 | params["redirect_uri"] = redirect_uri 101 | return params 102 | 103 | @staticmethod 104 | def get_user_details(details): 105 | """Return user details.""" 106 | fields = USER_FIELDS + USER_EXTRA_FIELDS + HACKER_ATTRIBUTES 107 | for field in fields: 108 | details.setdefault(field, "") 109 | details["username"] = details["name"] 110 | details["avatar_url"] = details.get("image_path") 111 | return details 112 | 113 | def get_user_id(self, details, response): 114 | """Return a unique ID for the current user, by default from server 115 | response.""" 116 | return response.get("id") 117 | 118 | def user_data(self, access_token, *args, **kwargs): 119 | """Loads user data.""" 120 | url = "{}/api/v1/profiles/me?{}".format( 121 | self.HACKER_SCHOOL_ROOT, urlencode({"access_token": access_token}) 122 | ) 123 | try: 124 | request = self.request(url, method="GET") 125 | return request.json() 126 | 127 | except ValueError: 128 | return None 129 | -------------------------------------------------------------------------------- /home/static/js/update_images.js: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | This file contains the javascript required to update broken profile images 4 | of users. The code detects any profile images that are broken, and requests 5 | the server for new images. The server inturn requests Hacker School API for 6 | the update profile image urls and returns them. 7 | 8 | */ 9 | 10 | function update_avatar_url(img) { 11 | var author_id = $(img).attr('data-author-id'); 12 | if (author_id) { 13 | var xhr = $.get('/updated_avatar/' + author_id +'/') 14 | .done(function(data){ 15 | img.src = data; 16 | }) 17 | } 18 | } 19 | 20 | function update_broken_images(){ 21 | $('img').each(function(index){ 22 | if (this.complete) { 23 | // check if broken and update 24 | if (this.naturalWidth === 0) { 25 | update_avatar_url(this); 26 | } 27 | } else { 28 | // or attach an error handler, to update if broken on load. 29 | $(this).error( 30 | function(){ 31 | // Unbind the error handler, to prevent it from being 32 | // called recursively called, if the update image loading 33 | // fails too. 34 | $(this).unbind('error'); 35 | update_avatar_url(this); 36 | } 37 | ); 38 | } 39 | }); 40 | } 41 | 42 | $(document).ready(function(){ 43 | update_broken_images(); 44 | }); 45 | -------------------------------------------------------------------------------- /home/templates/404.html: -------------------------------------------------------------------------------- 1 | {% extends "home/base.html" %} 2 | 3 | {% block content %} 4 | 5 |
6 |

Not Found

7 |
8 | Oops! The page you are looking for could not be found. 9 |
10 |
11 | 12 | {% endblock %} 13 | -------------------------------------------------------------------------------- /home/templates/home/about.html: -------------------------------------------------------------------------------- 1 | {% extends "home/base.html" %} 2 | 3 | {% block pagetitle %}Blaggregator{% endblock %} 4 | 5 | {% block title %} 6 | 7 | {% endblock %} 8 | 9 | {% block content %} 10 | 11 | {% include "home/readme.html" %} 12 | 13 | {% endblock %} 14 | -------------------------------------------------------------------------------- /home/templates/home/add_blog.html: -------------------------------------------------------------------------------- 1 | {% load customtags %} 2 | {% if forms %} 3 |
Consider adding RC Scout to your blog to help drive traffic to RC.
4 | {% endif %} 5 |
6 | {% if forms %} 7 | 13 | {% else %} 14 |
15 |
16 |

Add your blog

17 |
18 |
19 | {% endif %} 20 |
21 |

Please add your blog's (Atom/RSS) feed url

22 |
    23 |
  • All your blog posts (in this feed) will show up on 24 | Blaggregator
    • 25 |
  • All your future blog posts (in this feed) will be notified on Zulip in 26 | #blogging 27 |
    • 28 |
  • Consider adding RC Scout to your blog to help drive traffic to RC.
    • 29 |
30 |
31 |
32 |
33 | {% csrf_token %} 34 | {% if messages %} 35 | {% for message in messages %} 36 |
37 | {% if 'safe' in message.tags %} 38 | {{ message | safe }} 39 | {% else %} 40 | {{ message }} 41 | {% endif %} 42 |
43 | {% endfor %} 44 | {% endif %} 45 | 46 |
47 |
48 |
49 | 50 |
51 |
52 | 53 |
54 |
55 |
56 |
57 |
58 |
59 | -------------------------------------------------------------------------------- /home/templates/home/base.html: -------------------------------------------------------------------------------- 1 | {% load static %} 2 | 3 | 4 | 5 | 6 | 7 | 8 | 19 | {% block pagetitle %}{% endblock %} 20 | 29 | 30 | 31 | 32 | 58 | 59 |
60 | 61 |
62 |

{% block title %}{% endblock %}

63 |
64 | 65 |
66 | {% block content %}{% endblock %} 67 |
68 | 69 |
70 | 71 |
72 |
73 |
    74 |
  • 75 | About 76 |
    • 77 |
  • 78 | Contribute 79 |
    • 80 | {% if user.is_authenticated %} 81 |
  • 82 | 83 | 84 | 85 |
    • 86 | {% endif %} 87 |
88 | {% if user.is_authenticated %} 89 | 95 | {% endif %} 96 |
97 |
98 | 99 | 100 | {% block scripts %} 101 | 102 | 103 | 104 | {% endblock %} 105 | 106 | 107 | 108 | -------------------------------------------------------------------------------- /home/templates/home/confirm_delete.html: -------------------------------------------------------------------------------- 1 | 22 | -------------------------------------------------------------------------------- /home/templates/home/disabling-crawling.md: -------------------------------------------------------------------------------- 1 | {% load customtags %} 2 | Hi {{user.get_full_name}}, 3 | 4 | Blaggregator bot is unable to parse the following blogs that you own: 5 | 6 | {% for blog in blogs %} 7 | - {{blog.feed_url}} 8 | {% endfor %} 9 | 10 | {% stripnewlines %} 11 | Crawling has been **disabled** for these blogs. You could re-enable crawling by changing 12 | the feed url in your [profile]({{base_url}}{% url 'profile' user.id %}). 13 | {% endstripnewlines %} 14 | 15 | {% stripnewlines %} 16 | If this appears to be a problem with Blaggregator, or you need help fixing 17 | this, please 18 | {% if admins %} 19 | get in touch with {{admins}}. 20 | {% else %} 21 | {% spaceless %} 22 | file an [issue](https://github.com/recursecenter/blaggregator/issues). 23 | {% endspaceless %} 24 | {% endif %} 25 | {% endstripnewlines %} 26 | 27 | Happy Blogging! 28 | -------------------------------------------------------------------------------- /home/templates/home/edit_blog.html: -------------------------------------------------------------------------------- 1 |
2 | {% csrf_token %} 3 |
4 | {% for field in form %} 5 |
6 | {{ field }} 7 |
8 | {% endfor %} 9 | 10 |
11 |
12 | -------------------------------------------------------------------------------- /home/templates/home/feed_item.tmpl: -------------------------------------------------------------------------------- 1 | {% load customtags %} 2 | 3 | {{ obj.content|filter_control_chars|safe }} 4 | 5 |

6 | This post was originally published here and 7 | you are reading it in the 8 | Blaggregator feed. 9 | Join the discussion on Zulip!. 10 |

11 | -------------------------------------------------------------------------------- /home/templates/home/log_in_oauth.html: -------------------------------------------------------------------------------- 1 | {% extends "home/base.html" %} 2 | 3 | {% block pagetitle %}Blaggregator{% endblock %} 4 | 5 | {% block title %} 6 | 7 | {% endblock %} 8 | 9 | {% block content %} 10 | 11 |
12 |

Welcome to Blaggregator!

13 | 14 |

Blaggregator is the central place to read and discuss technical work for 15 | the entire Recurse Center community, including alums. Read 16 | more.

17 | 18 |

Recurse Center OAuth Login

19 | 20 |
21 | 22 | {% endblock %} 23 | -------------------------------------------------------------------------------- /home/templates/home/login_error.html: -------------------------------------------------------------------------------- 1 | {% extends "home/base.html" %} 2 | 3 | {% block pagetitle %}Blaggregator{% endblock %} 4 | 5 | {% block title %} 6 | 7 | {% endblock %} 8 | 9 | {% block content %} 10 | 11 |
12 |

Login Error

13 | 14 |

Error: {{ request.GET.message }}.

15 | 16 |

You should fix the error and try logging in again.

17 |
18 | 19 | {% endblock %} 20 | -------------------------------------------------------------------------------- /home/templates/home/most_viewed.html: -------------------------------------------------------------------------------- 1 | {% extends "home/base.html" %} 2 | {% load customtags %} 3 | 4 | {% block pagetitle %}Blaggregator -- Most Viewed Posts{% endblock %} 5 | 6 | {% block content %} 7 |

Most viewed posts: {{ from }} -- {{ to }}

8 | {% include 'home/postlist.html' %} 9 | {% endblock %} 10 | -------------------------------------------------------------------------------- /home/templates/home/new.html: -------------------------------------------------------------------------------- 1 | {% extends "home/base.html" %} 2 | {% load customtags %} 3 | 4 | {% block pagetitle %}Blaggregator{% endblock %} 5 | 6 | {% block title %} 7 | 8 | {% endblock %} 9 | 10 | {% block content %} 11 | 12 | {% include 'home/postlist.html' %} 13 | 14 | {% endblock %} 15 | -------------------------------------------------------------------------------- /home/templates/home/postlist.html: -------------------------------------------------------------------------------- 1 | {% load customtags %} 2 | 3 | 22 | 23 | {% if post_list.paginator %} 24 | 49 | {% endif %} 50 | -------------------------------------------------------------------------------- /home/templates/home/profile.html: -------------------------------------------------------------------------------- 1 | {% extends "home/base.html" %} 2 | {% block pagetitle %}Blaggregator{% endblock %} 3 | 4 | {% block content %} 5 | {% if not owner %} 6 |
7 |
8 |

{{ hacker.user.get_full_name }}

9 | 10 | 11 | 12 | 13 | 14 | 15 | {% if hacker.github %} 16 | 17 | 18 | 19 | {% endif %} 20 | {% if hacker.twitter %} 21 | 22 | 23 | 24 | {% endif %} 25 | 26 |
27 | 28 | 29 | 30 |
31 | {% endif %} 32 | 33 | 34 |
35 | {% if forms %} 36 |

Blogs

37 | 38 | 39 | 40 | {% if owner %} 41 | 42 | {% endif %} 43 | 44 | {% for form in forms %} 45 | {% with blog=form.instance %} 46 | 47 | 51 | 58 | 59 | {% if owner %} 60 | 74 | {% endif %} 75 | 76 | {% include 'home/confirm_delete.html' %} 77 | 78 | 81 | 82 | {% endwith %} 83 | {% endfor %} 84 |
Feed URL Crawling? Stream 
48 | {{blog.feed_url}} 49 | ({{blog.post_count}} posts) 50 | 52 | {% if blog.skip_crawl %} 53 | Disabled 54 | {% else %} 55 | Active 56 | {% endif %} 57 | {{blog.get_stream_display}} 61 | 67 | 73 |
79 | {% include 'home/edit_blog.html' %} 80 |
85 | {% endif %} 86 | {% if owner %} 87 | {% include 'home/add_blog.html' %} 88 | {% endif %} 89 |
90 | 91 | {% if owner %} 92 |
93 |
94 |

Blaggregator RSS feed

95 |

96 | Blaggregator provides an authenticated 97 | RSS feed. Please keep 98 | your feed URL private. In case you accidentally share the URL, you 99 | can reset it using the button below. 100 |

101 | 102 |
103 |
104 | 105 |
106 |
107 | 108 |
109 |
110 |
111 |
112 | {% endif %} 113 | 114 |
115 | {% if post_list %} 116 |

Blog Posts

117 | {{ post_list.count }} posts 118 | {% include 'home/postlist.html' %} 119 | {% endif %} 120 |
121 | 122 | {% endblock %} 123 | -------------------------------------------------------------------------------- /home/templates/home/readme.html: -------------------------------------------------------------------------------- 1 |

Blaggregator

2 |

Blog post aggregator for the Recurse Center community. Live version runs at 3 | https://blaggregator.recurse.com (RC login required).

4 |
5 | 6 |

What is this?

7 | 8 |

During her batch, Sasha (W '13) noticed that her peers were all blogging about 9 | their work at the Recurse Center on their individual blogs. It was really cool 10 | to see what they were working on and thinking about.

11 | 12 |

Some folks would post about their new posts in Zulip, some on Twitter, and some 13 | wouldn't spread the word at all. There was all this great work going on, but it 14 | was scattered across the internet.

15 | 16 |

Blaggregator puts all this awesome content in one place, and provides a place 17 | for the members of the community to read and discuss it. This has the nice 18 | side effect of providing a friendly audience for those who may have just 19 | launched their blog.

20 | 21 |
22 | 23 |

These awesome people have 24 | helped make Blaggregator better!

25 |

License

26 |

Copyright © 2013-2017 Sasha Laundy and others.

27 |

This software is licensed under the terms of the AGPL, Version 3. The complete 28 | license can be found at https://www.gnu.org/licenses/agpl-3.0.html.

29 |

FAQ

30 |

How does it work?

31 |

Once an hour, the crawler checks all registered blogs for new posts. New posts 32 | are displayed on the main page and a message is sent to Zulip.

33 |

"New" is defined as a post having a new URL and a new title. So you can tweak 34 | the title, change the content or the timestamp to your heart's content.

35 |

Why do I have to log in?

36 |

The Recurse Center staff wishes to keep the list of people attending the 37 | Recurse Center private. So you are required to authenticate with your Recurse 38 | Center credentials

39 |

If that ever changes (for instance, to surface the best posts that are coming 40 | out of the Recurse Center to show off what we're working on) you will be given 41 | lots and lots of warning to decide how you want to participate.

42 |

Who's behind it?

43 |

Sasha is the main author, with a bunch of 44 | other 45 | recursers 46 | contributing. You are welcome to contribute as well!

47 |

Puneeth is the primary maintainer, currently.

48 |

Can I contribute fixes and features?

49 |

Yes, that would be great! This is a project by and for the Recurse Center 50 | community. Help make it more awesome!

51 |

There's a very generic and high level list 52 | of 53 | areas that could use some help and 54 | a bunch of specific 55 | open issues.

56 |

Look at 57 | the 58 | developer documentation for 59 | help with setting up your development environment.

60 |

Before implementing a major contribution, it's wise to get in touch with the 61 | maintainers by creating an issue (or using an existing issue) to discuss it.

62 |

What's the stack?

63 |

It's a Django (Python) app, with some Bootstrap on the frontend. It's deployed 64 | on Heroku using their Postgres and Scheduler add-ons. Check out the 65 | code here.

66 |

I don't see my blog post.

67 |

If you published it less than an hour ago, hang tight: it'll show up when the 68 | crawler next checks the registered blogs. If Blaggregator finds many new posts 69 | on your blog, it will only post the two most recent posts to Zulip. All of your 70 | posts will still be available on the site

71 |

My blog post appears on blaggregator but no Zulip notification sent.

72 |

For every crawl a maximum of 2 notifications are sent for posts that haven't 73 | already been seen by blaggregator. So, if you published more than 2 posts 74 | between consecutive (hourly) crawls by blaggregator, only the last two posts 75 | will be notified on Zulip.

76 |

My blog is multi-purpose and not wholly code/Recurse Center specific

77 |

No problem! RCers usually enjoy reading non-technical posts by others.

78 |

But, if you like, you could tag or categorize your relevant posts and make a 79 | separate RSS/Atom feed for them. Most blogging software has separate feeds for 80 | different categories/tags.

81 |

If you use Wordpress, for instance, categorize all your code-related posts 82 | "code", say. Then your code-specific RSS feed that you should publish to 83 | Blaggregator is: http://yoururl.com/blog/category/code/feed/.

84 |

Can I lurk?

85 |

Sure, just make an account and don't add your blog. But you shouldn't lurk. You 86 | should blog.

87 |

Why should I blog?

88 |
    89 |
  • It provides a record of your thinking and work for your future self.
    • 90 |
  • It gives prospective employers insight into the way you think.
    • 91 |
  • If you do a project that doesn't go as planned, you can still 'finish' the 92 | project and explain what you learned, even if you don't want to put the code 93 | on Github.
    • 94 |
  • It helps more people hear about and respect the Recurse Center, which in turn 95 | means more people will want to work with you.
    • 96 |
  • Writing helps you practice communicating, which is critical if you plan on 97 | working on a team of more than one person.
    • 98 |
  • It helps the developer community at large.
    • 99 |
100 |

But blogging takes too long!

101 |

Don't let the perfect be the enemy of the good. Your posts don't have to be 102 | long, groundbreaking, perfect, or full of citations. Short, imperfect, and 103 | published always beats unpublished.

104 |

But I haven't found the perfect blogging tool

105 |

Don't let the perfect be the enemy of the good. Just get something up and 106 | resist the urge to fiddle with it. Use Tumblr if you have to. Just start 107 | writing.

108 |

I need some more inspiration.

109 | 119 |

I need some accountability!

120 |

Consider starting your own Iron Blogger challenge. Participants commit to 121 | writing one blog post a week, and are on the hook for $5 if they don't 122 | post. The pot can be used for a party, donated to charity, or donated to a 123 | charity the group hates (added incentive to hit the publish button!).

124 |

The Fall 2013 batch ran a very successful Iron Blogger program. Mike 125 | Walker 126 | wrote a very nice article on how it worked.

-------------------------------------------------------------------------------- /home/templates/home/search.html: -------------------------------------------------------------------------------- 1 | {% extends "home/base.html" %} 2 | {% load customtags %} 3 | 4 | {% block pagetitle %}Blaggregator{% endblock %} 5 | 6 | {% block title %} 7 | 8 | {% endblock %} 9 | 10 | {% block content %} 11 | 12 |

Search results for '{{ query }}'

13 | {% if not post_list %} 14 | No results found! 15 | {% else %} 16 |

Found {{ count }} result(s)

17 | 18 | {% endif %} 19 | 20 | 21 | {% include 'home/postlist.html' %} 22 | 23 | {% endblock %} 24 | -------------------------------------------------------------------------------- /home/templatetags/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/recursecenter/blaggregator/d5f3da939fcb16480be049c027e141b7aa171baa/home/templatetags/__init__.py -------------------------------------------------------------------------------- /home/templatetags/customtags.py: -------------------------------------------------------------------------------- 1 | import re 2 | from urllib.parse import quote 3 | 4 | from django.template import Library, Node 5 | 6 | register = Library() 7 | 8 | 9 | @register.tag(name="stripnewlines") 10 | def strip_newlines(parser, token): 11 | nodelist = parser.parse(("endstripnewlines",)) 12 | parser.delete_first_token() 13 | return StripNewlinesNode(nodelist) 14 | 15 | 16 | class StripNewlinesNode(Node): 17 | def __init__(self, nodelist): 18 | self.nodelist = nodelist 19 | 20 | def render(self, context): 21 | output = self.nodelist.render(context) 22 | return re.sub(r"\s+", " ", output.strip()) 23 | 24 | 25 | @register.filter 26 | def zulip_url(title, stream): 27 | """Return the Zulip url given the title.""" 28 | 29 | # We just replicate how Zulip creates/manages urls. 30 | # https://github.com/zulip/zulip/blob/33295180a918fcd420428d9aa2fb737b864cacaf/zerver/lib/notifications.py#L34 31 | # Some browsers zealously URI-decode the contents of window.location.hash. 32 | # So Zulip hides the URI-encoding by replacing '%' with '.' 33 | def replace(x): 34 | return quote(x.encode("utf-8"), safe="").replace(".", "%2E").replace("%", ".") 35 | 36 | if len(title) > 60: 37 | title = title[:57] + "..." 38 | hash_path = "narrow/stream/%s/topic/%s" % (replace(stream), replace(title)) 39 | return "https://recurse.zulipchat.com/#%s" % hash_path 40 | 41 | 42 | @register.filter 43 | def filter_control_chars(text): 44 | """Filter control characters from a given text.""" 45 | return re.sub(r"[\x00-\x08\x0B-\x0C\x0E-\x1F]", "", text) 46 | -------------------------------------------------------------------------------- /home/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # HACK: Tests fail with ImportError: cannot import name range This is probably 2 | # because of some six madness between one of the test dependencies and django's 3 | # internal six stuff. 4 | from django.contrib.staticfiles.storage import staticfiles_storage # noqa 5 | -------------------------------------------------------------------------------- /home/tests/test_crawlposts.py: -------------------------------------------------------------------------------- 1 | from django.core.management import execute_from_command_line 2 | from django.conf import settings 3 | from django.test import TransactionTestCase 4 | from mock import patch 5 | 6 | from home.models import Blog, Post, User 7 | from .utils import BlogFactory, generate_full_feed, fake_response 8 | 9 | 10 | def random_feed(url=None, data=None, timeout=None): 11 | feed = generate_full_feed(min_items=5, max_items=20) 12 | xml = feed.writeString("utf8") 13 | return fake_response(xml.encode("utf8"))() 14 | 15 | 16 | @patch("home.zulip_helpers.update_user_details", new=lambda x: None) 17 | @patch("home.zulip_helpers.get_members", new=lambda: {"by_id": {}}) 18 | @patch("urllib.request.OpenerDirector.open", new=random_feed) 19 | @patch("requests.post") 20 | class CrawlPostsTestCase(TransactionTestCase): 21 | def setUp(self): 22 | # Setup the db with blogs 23 | BlogFactory.create_batch(2) 24 | self.blogs = Blog.objects.all() 25 | super(CrawlPostsTestCase, self).setUp() 26 | 27 | def tearDown(self): 28 | self.clear_db() 29 | super(CrawlPostsTestCase, self).tearDown() 30 | 31 | def clear_db(self): 32 | User.objects.all().delete() 33 | 34 | def test_crawling_posts(self, mock): 35 | # When 36 | execute_from_command_line(["./manage.py", "crawlposts"]) 37 | # Then 38 | for blog in self.blogs: 39 | # at least one post per blog 40 | self.assertGreater(Post.objects.filter(blog=blog).count(), 0) 41 | # posts are unique by blog and title 42 | post_titles = Post.objects.filter(blog=blog).values_list("title", flat=True) 43 | self.assertEqual(len(set(post_titles)), len(post_titles)) 44 | # Number of announcements are correctly throttled 45 | self.assertLessEqual(mock.call_count, settings.MAX_POST_ANNOUNCE * self.blogs.count()) 46 | 47 | def test_crawling_skips_flagged_blogs(self, mock): 48 | # Given 49 | blog1, blog2 = list(self.blogs) 50 | blog1.skip_crawl = True 51 | blog1.save() 52 | # When 53 | execute_from_command_line(["./manage.py", "crawlposts"]) 54 | # Then 55 | self.assertEqual(0, Post.objects.filter(blog=blog1).count()) 56 | self.assertLess(0, Post.objects.filter(blog=blog2).count()) 57 | -------------------------------------------------------------------------------- /home/tests/test_feed_parser.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | from functools import partial 3 | 4 | from django.test import TestCase 5 | from mock import patch 6 | 7 | from home.feedergrabber27 import feedergrabber 8 | from home.tests.utils import generate_full_feed, fake_response 9 | 10 | MIN_DATE_FEED = b""" 11 | 12 | 13 | 14 | Max.Computer 15 | http://max.computer/ 16 | en-US 17 | 18 | 19 | About 20 | http://max.computer/about/ 21 | Mon, 01 Jan 0001 00:00:00 +0000 22 | http://max.computer/about/ 23 | About me 24 | 25 | 26 | 27 | """ 28 | 29 | 30 | class FeedParserTestCase(TestCase): 31 | def test_parsing_valid_feeds(self): 32 | feed = generate_full_feed() 33 | with patch( 34 | "urllib.request.OpenerDirector.open", 35 | new=partial(self.patch_open, feed), 36 | ): 37 | contents, errors = feedergrabber(feed.feed["link"]) 38 | if contents is None: 39 | self.assertEqual(0, len(feed.items)) 40 | self.assertEqual(1, len(errors)) 41 | self.assertIn("Parsing methods not successful", errors[0]) 42 | else: 43 | for i, (link, title, date, content) in enumerate(contents): 44 | item = feed.items[i] 45 | self.assertEqual(link, item["link"]) 46 | self.assertIsNotNone(date) 47 | self.assertGreaterEqual( 48 | datetime.datetime.now().utctimetuple(), 49 | date.utctimetuple(), 50 | ) 51 | 52 | def test_parsing_broken_feeds(self): 53 | feed = generate_full_feed() 54 | with patch( 55 | "urllib.request.OpenerDirector.open", 56 | new=partial(self.patch_open_broken_feed, feed), 57 | ): 58 | contents, errors = feedergrabber(feed.feed["link"]) 59 | self.assertIsNone(contents) 60 | self.assertEqual(len(feed.items) + 1, len(errors)) 61 | self.assertIn("Parsing methods not successful", errors[-1]) 62 | 63 | @staticmethod 64 | def patch_open(feed, url, data=None, timeout=None): 65 | return fake_response(feed.writeString("utf8").encode("utf8"))() 66 | 67 | @staticmethod 68 | def patch_open_broken_feed(feed, url, data=None, timeout=None): 69 | xml = FeedParserTestCase.patch_open(feed, url, data, timeout) 70 | text = xml.read().decode("utf8") 71 | text = text.replace('encoding="utf8"', "") 72 | # feedgenerator makes title and link mandatory, hence we remove from 73 | # generated xml. 74 | if len(feed.items) % 2 == 0: 75 | feed.feed["title"] = None 76 | for item in feed.items: 77 | item["title"] = None 78 | else: 79 | feed.feed["link"] = None 80 | feed.feed["id"] = None 81 | for item in feed.items: 82 | item["link"] = None 83 | item["id"] = None 84 | return fake_response(feed.writeString("utf8").encode("utf8"))() 85 | 86 | 87 | class FeedParserHelpersTestCase(TestCase): 88 | def test_parsing_feeds_with_min_dates(self): 89 | with patch("urllib.request.OpenerDirector.open", new=fake_response(MIN_DATE_FEED)): 90 | contents, errors = feedergrabber("http://max.computer/index.html") 91 | self.assertIsNone(contents) 92 | self.assertEqual(2, len(errors)) 93 | self.assertIn("Parsing methods not successful", errors[-1]) 94 | self.assertIn("hugo page", errors[0]) 95 | -------------------------------------------------------------------------------- /home/tests/test_oauth.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from django.test import TestCase 4 | from mock import patch 5 | from requests import Response 6 | 7 | from home.models import User 8 | from home.oauth import HackerSchoolOAuth2, update_user_details 9 | 10 | 11 | def request(self, url, method="GET", *args, **kwargs): 12 | if url.endswith("/oauth/token"): 13 | data = { 14 | "access_token": "x", 15 | "created_at": 1478432443, 16 | "expires_in": 7200, 17 | "token_type": "bearer", 18 | "scope": "public", 19 | "refresh_token": "y", 20 | } 21 | else: 22 | data = OAuthTestCase.USER_DATA 23 | 24 | response = Response() 25 | response._content = json.dumps(data).encode("utf8") 26 | response.status_code = 200 27 | return response 28 | 29 | 30 | @patch("home.oauth.HackerSchoolOAuth2.validate_state", new=lambda x: True) 31 | @patch("home.oauth.HackerSchoolOAuth2.request", new=request) 32 | class OAuthTestCase(TestCase): 33 | 34 | USER_DATA = { 35 | "id": 1729, 36 | "email": "johndoe@foo-bar.com", 37 | "image_path": "https://x.cloudfront.net/assets/people/y.jpg", 38 | "first_name": "John", 39 | "last_name": "Doe", 40 | "name": "John Doe", 41 | "github": "johndoe", 42 | "zulip_id": 1729, 43 | } 44 | 45 | def setUp(self): 46 | pass 47 | 48 | def tearDown(self): 49 | self.clear_db() 50 | 51 | def clear_db(self): 52 | User.objects.all().delete() 53 | 54 | def test_login_redirects_to_authorization_url(self): 55 | # When 56 | response = self.client.get("/login/hackerschool/") 57 | 58 | # Then 59 | self.assertEqual(302, response.status_code) 60 | self.assertTrue(response["Location"].startswith(HackerSchoolOAuth2.AUTHORIZATION_URL)) 61 | 62 | def test_authorization_completes(self): 63 | # When 64 | response = self.client.get("/complete/hackerschool/") 65 | 66 | # Then 67 | self.assertEqual(302, response.status_code) 68 | self.assertTrue(response["Location"].endswith("/new/")) 69 | self.assertEqual(1, User.objects.count()) 70 | user = User.objects.get(email=self.USER_DATA["email"]) 71 | self.assertEqual(user.hacker.avatar_url, self.USER_DATA["image_path"]) 72 | self.assertEqual(user.hacker.github, self.USER_DATA.get("github", "")) 73 | self.assertEqual(user.hacker.twitter, self.USER_DATA.get("twitter", "")) 74 | 75 | def test_authenticates_existing_user(self): 76 | # When 77 | self.client.get("/complete/hackerschool/") 78 | self.client.get("/logout/", follow=True) 79 | response = self.client.get("/complete/hackerschool/") 80 | 81 | # Then 82 | self.assertEqual(302, response.status_code) 83 | self.assertTrue(response["Location"].endswith("/new/")) 84 | self.assertEqual(1, User.objects.count()) 85 | user = User.objects.get(email=self.USER_DATA["email"]) 86 | self.assertEqual(user.hacker.avatar_url, self.USER_DATA["image_path"]) 87 | self.assertEqual(user.hacker.github, self.USER_DATA.get("github", "")) 88 | self.assertEqual(user.hacker.twitter, self.USER_DATA.get("twitter", "")) 89 | 90 | def test_updates_user_details(self): 91 | # When 92 | self.client.get("/complete/hackerschool/") 93 | user = User.objects.get(email=self.USER_DATA["email"]) 94 | self.USER_DATA["twitter"] = "johndoe" 95 | 96 | # When 97 | with patch("requests.get", new=lambda url: request(None, url)): 98 | update_user_details(self.USER_DATA["id"]) 99 | 100 | # Then 101 | user = User.objects.get(email=self.USER_DATA["email"]) 102 | self.assertEqual(user.hacker.twitter, "johndoe") 103 | -------------------------------------------------------------------------------- /home/tests/test_views.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from django.test import TestCase 3 | from django.test.utils import override_settings 4 | from django.utils import timezone 5 | import feedparser 6 | from mock import patch 7 | 8 | from home.models import Hacker, Blog, LogEntry, Post, User 9 | from .utils import create_posts, fake_response 10 | 11 | N_MAX = settings.MAX_FEED_ENTRIES 12 | 13 | 14 | @override_settings( 15 | AUTHENTICATION_BACKENDS=( 16 | "django.contrib.auth.backends.ModelBackend", 17 | "home.token_auth.TokenAuthBackend", 18 | ) 19 | ) 20 | class BaseViewTestCase(TestCase): 21 | def setUp(self): 22 | self.setup_test_user() 23 | 24 | def tearDown(self): 25 | self.clear_db() 26 | 27 | # Helper methods #### 28 | def clear_db(self): 29 | User.objects.all().delete() 30 | 31 | def create_posts(self, n, **kwargs): 32 | create_posts(n, **kwargs) 33 | # Blog.objects.update(user=self.user) 34 | 35 | def setup_test_user(self): 36 | self.username = self.password = "test" 37 | self.user = User.objects.create_user(self.username) 38 | self.user.set_password(self.password) 39 | self.user.save() 40 | self.hacker = Hacker.objects.create(user_id=self.user.id) 41 | 42 | def login(self): 43 | """Login as test user.""" 44 | self.client.login(username=self.username, password=self.password) 45 | 46 | 47 | class FeedsViewTestCase(BaseViewTestCase): 48 | def test_should_enforce_authentication(self): 49 | response = self.client.get("/atom.xml") 50 | self.assertEqual(response.status_code, 403) 51 | response = self.client.get("/atom.xml", data={"token": ""}) 52 | self.assertEqual(response.status_code, 403) 53 | 54 | def test_should_enforce_token(self): 55 | self.login() 56 | response = self.client.get("/new/") 57 | self.assertEqual(response.status_code, 200) 58 | response = self.client.get("/atom.xml") 59 | self.assertEqual(response.status_code, 403) 60 | response = self.client.get("/atom.xml", data={"token": ""}) 61 | self.assertEqual(response.status_code, 403) 62 | response = self.client.get("/atom.xml", data={"token": "BOGUS-TOKEN"}) 63 | self.assertEqual(response.status_code, 403) 64 | 65 | def test_feed_with_no_posts(self): 66 | self.verify_feed_generation(0) 67 | 68 | def test_feed_with_posts_less_than_max_feed_size(self): 69 | self.verify_feed_generation(N_MAX - 1) 70 | 71 | def test_feed_with_posts_more_than_max_feed_size(self): 72 | self.verify_feed_generation(N_MAX * 5) 73 | 74 | def test_feed_with_control_characters(self): 75 | # Given 76 | blog = Blog.objects.create(user=self.user) 77 | Post.objects.create( 78 | blog=blog, 79 | title="This is a title with control chars \x01", 80 | content="This is content with control chars \x01", 81 | posted_at=timezone.now(), 82 | ) 83 | 84 | # When 85 | response = self.client.get("/atom.xml", data={"token": self.hacker.token}) 86 | 87 | # Then 88 | self.assertEqual(1, Post.objects.count()) 89 | text = response.content.decode("utf8") 90 | self.assertIn( 91 | "This is a title with control chars", 92 | text, 93 | ) 94 | self.assertIn( 95 | "This is content with control chars", 96 | text, 97 | ) 98 | 99 | # Helper methods #### 100 | def parse_feed(self, content): 101 | """Parse feed content and return entries.""" 102 | # FIXME: Would it be a good idea to use feedergrabber? 103 | return feedparser.parse(content) 104 | 105 | def get_included_excluded_posts(self, posts, entries): 106 | """Returns the set of included and excluded posts.""" 107 | entry_links = {(entry.title, entry.link) for entry in entries} 108 | included = [] 109 | excluded = [] 110 | for post in posts: 111 | if (post.title, post.url) in entry_links: 112 | included.append(post) 113 | else: 114 | excluded.append(post) 115 | return included, excluded 116 | 117 | def verify_feed_generation(self, n): 118 | # Given 119 | self.clear_db() 120 | self.login() 121 | self.setup_test_user() 122 | posts = create_posts(n) 123 | # When 124 | response = self.client.get("/atom.xml", data={"token": self.hacker.token}) 125 | # Then 126 | self.assertEqual(n, Post.objects.count()) 127 | self.assertEqual(n, len(posts)) 128 | self.assertEqual(200, response.status_code) 129 | feed = self.parse_feed(response.content) 130 | entries = feed.entries 131 | self.assertEqual(min(n, N_MAX), len(entries)) 132 | if n < 1: 133 | return 134 | 135 | self.assertGreaterEqual(entries[0].updated_parsed, entries[-1].updated_parsed) 136 | included, excluded = self.get_included_excluded_posts(posts, entries) 137 | self.assertEqual(len(included), len(entries)) 138 | if not excluded: 139 | return 140 | 141 | max_excluded_date = max(excluded, key=lambda x: x.posted_at).posted_at 142 | min_included_date = min(included, key=lambda x: x.posted_at).posted_at 143 | self.assertGreaterEqual(min_included_date, max_excluded_date) 144 | 145 | 146 | EMPTY_FEED = b""" 147 | 148 | 149 | 150 | My Site 151 | http://localhost:1313/ 152 | en-us 153 | 154 | 155 | 156 | """ 157 | HTML_PAGE = b""" 158 | 159 | 160 | 161 | 162 | My Site 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 | """ 171 | 172 | 173 | html_page = fake_response(HTML_PAGE) 174 | empty_feed = fake_response(EMPTY_FEED) 175 | 176 | 177 | @patch("urllib.request.OpenerDirector.open", new=empty_feed) 178 | class AddBlogViewTestCase(BaseViewTestCase): 179 | def test_get_add_blog_requires_login(self): 180 | # When 181 | response = self.client.post("/add_blog/", follow=True) 182 | # Then 183 | self.assertRedirects(response, "/login/?next=/add_blog/") 184 | 185 | def test_post_add_blog_without_blog_url_barfs(self): 186 | # Given 187 | self.login() 188 | # When 189 | response = self.client.post("/add_blog/", follow=True) 190 | # Then 191 | self.assertEqual(response.status_code, 200) 192 | self.assertContains(response, "No feed URL provided") 193 | 194 | def test_post_add_blog_adds_blog(self): 195 | # Given 196 | self.login() 197 | data = {"feed_url": "https://jvns.ca/atom.xml"} 198 | # When 199 | response = self.client.post("/add_blog/", data=data, follow=True) 200 | # Then 201 | self.assertRedirects(response, "/profile/{}/".format(self.user.id)) 202 | self.assertEqual(response.status_code, 200) 203 | self.assertIsNotNone(Blog.objects.get(feed_url=data["feed_url"])) 204 | self.assertContains(response, "has been added successfully") 205 | 206 | def test_post_add_blog_adds_blog_without_schema(self): 207 | # Given 208 | self.login() 209 | data = {"feed_url": "jvns.ca/atom.xml"} 210 | # When 211 | response = self.client.post("/add_blog/", data=data, follow=True) 212 | # Then 213 | self.assertRedirects(response, "/profile/{}/".format(self.user.id)) 214 | self.assertEqual(response.status_code, 200) 215 | self.assertIsNotNone(Blog.objects.get(feed_url="http://{}".format(data["feed_url"]))) 216 | 217 | def test_post_add_blog_adds_only_once(self): 218 | # Given 219 | self.login() 220 | data = {"feed_url": "https://jvns.ca/atom.xml"} 221 | self.client.post("/add_blog/", data=data, follow=True) 222 | data_ = {"feed_url": "https://jvns.ca/atom.xml"} 223 | # When 224 | response = self.client.post("/add_blog/", data=data_, follow=True) 225 | # Then 226 | self.assertRedirects(response, "/profile/{}/".format(self.user.id)) 227 | self.assertEqual(response.status_code, 200) 228 | self.assertEqual(1, Blog.objects.count()) 229 | 230 | def test_post_add_blog_existing_unsets_skip_crawl(self): 231 | # Given 232 | self.login() 233 | data = {"feed_url": "https://jvns.ca/atom.xml"} 234 | self.client.post("/add_blog/", data=data, follow=True) 235 | blog = Blog.objects.get(feed_url=data["feed_url"]) 236 | blog.skip_crawl = True 237 | blog.save() 238 | data_ = {"feed_url": "https://jvns.ca/atom.xml"} 239 | # When 240 | response = self.client.post("/add_blog/", data=data_, follow=True) 241 | # Then 242 | self.assertRedirects(response, "/profile/{}/".format(self.user.id)) 243 | self.assertEqual(response.status_code, 200) 244 | self.assertEqual(1, Blog.objects.count()) 245 | blog = Blog.objects.get(feed_url=data["feed_url"]) 246 | self.assertFalse(blog.skip_crawl) 247 | 248 | def test_post_add_blog_adds_different_feeds(self): 249 | # Given 250 | self.login() 251 | data = {"feed_url": "https://jvns.ca/atom.xml"} 252 | self.client.post("/add_blog/", data=data, follow=True) 253 | data_ = {"feed_url": "https://jvns.ca/tags/blaggregator.xml"} 254 | # When 255 | response = self.client.post("/add_blog/", data=data_, follow=True) 256 | # Then 257 | self.assertRedirects(response, "/profile/{}/".format(self.user.id)) 258 | self.assertEqual(response.status_code, 200) 259 | self.assertEqual(2, Blog.objects.count()) 260 | self.assertIsNotNone(Blog.objects.get(feed_url=data["feed_url"])) 261 | 262 | def test_post_add_blog_suggests_feed_url(self): 263 | # Given 264 | self.login() 265 | data = {"feed_url": "https://jvns.ca/"} 266 | # When 267 | with patch("urllib.request.OpenerDirector.open", new=html_page): 268 | response = self.client.post("/add_blog/", data=data, follow=True) 269 | # Then 270 | self.assertEqual(0, Blog.objects.count()) 271 | self.assertRedirects(response, "/profile/{}/".format(self.user.id)) 272 | self.assertEqual(response.status_code, 200) 273 | self.assertContains(response, "Please use your blog's feed url") 274 | self.assertContains(response, "It could be -- ") 275 | self.assertContains(response, "https://jvns.ca/atom.xml") 276 | 277 | 278 | class DeleteBlogViewTestCase(BaseViewTestCase): 279 | def test_should_not_delete_blog_not_logged_in(self): 280 | # Given 281 | feed_url = "https://jvns.ca/atom.xml" 282 | blog = Blog.objects.create(user=self.user, feed_url=feed_url) 283 | # When 284 | self.client.get("/delete_blog/%s/" % blog.id) 285 | # Then 286 | self.assertEqual(1, Blog.objects.count()) 287 | 288 | def test_should_delete_blog(self): 289 | # Given 290 | self.login() 291 | feed_url = "https://jvns.ca/atom.xml" 292 | blog = Blog.objects.create(user=self.user, feed_url=feed_url) 293 | # When 294 | self.client.get("/delete_blog/%s/" % blog.id) 295 | # Then 296 | self.assertEqual(0, Blog.objects.count()) 297 | with self.assertRaises(Blog.DoesNotExist): 298 | Blog.objects.get(feed_url=feed_url) 299 | 300 | def test_should_not_delete_unknown_blog(self): 301 | # Given 302 | self.login() 303 | feed_url = "https://jvns.ca/atom.xml" 304 | blog = Blog.objects.create(user=self.user, feed_url=feed_url) 305 | self.client.get("/delete_blog/%s/" % blog.id) 306 | # When 307 | response = self.client.get("/delete_blog/%s/" % blog.id) 308 | # Then 309 | self.assertEqual(404, response.status_code) 310 | 311 | 312 | class EditBlogViewTestCase(BaseViewTestCase): 313 | def test_should_not_edit_blog_not_logged_in(self): 314 | # Given 315 | feed_url = "https://jvns.ca/atom.xml" 316 | blog = Blog.objects.create(user=self.user, feed_url=feed_url) 317 | # When 318 | response = self.client.get("/edit_blog/%s/" % blog.id, follow=True) 319 | # Then 320 | self.assertRedirects(response, "/login/?next=/edit_blog/%s/" % blog.id) 321 | 322 | def test_should_edit_blog(self): 323 | # Given 324 | self.login() 325 | feed_url = "https://jvns.ca/atom.xml" 326 | blog = Blog.objects.create(user=self.user, feed_url=feed_url) 327 | data = {"feed_url": "https://jvns.ca/rss", "stream": "BLOGGING"} 328 | # When 329 | response = self.client.post("/edit_blog/%s/" % blog.id, data=data, follow=True) 330 | # Then 331 | self.assertEqual(200, response.status_code) 332 | with self.assertRaises(Blog.DoesNotExist): 333 | Blog.objects.get(feed_url=feed_url) 334 | self.assertIsNotNone(Blog.objects.get(feed_url=data["feed_url"])) 335 | 336 | def test_should_unset_skip_crawl_on_edit_blog(self): 337 | # Given 338 | self.login() 339 | feed_url = "https://jvns.ca/atom.xml" 340 | blog = Blog.objects.create(user=self.user, feed_url=feed_url, skip_crawl=True) 341 | data = {"feed_url": "https://jvns.ca/rss", "stream": "BLOGGING"} 342 | assert blog.skip_crawl, "Blog skip crawl should be True" 343 | # When 344 | response = self.client.post("/edit_blog/%s/" % blog.id, data=data, follow=True) 345 | # Then 346 | self.assertEqual(200, response.status_code) 347 | with self.assertRaises(Blog.DoesNotExist): 348 | Blog.objects.get(feed_url=feed_url) 349 | blog = Blog.objects.get(feed_url=data["feed_url"]) 350 | self.assertIsNotNone(blog) 351 | self.assertFalse(blog.skip_crawl) 352 | 353 | def test_should_not_edit_unknown_blog(self): 354 | # Given 355 | self.login() 356 | data = {"feed_url": "https://jvns.ca/rss", "stream": "BLOGGING"} 357 | # When 358 | response = self.client.post("/edit_blog/%s/" % 200, data=data, follow=True) 359 | # Then 360 | self.assertEqual(404, response.status_code) 361 | 362 | 363 | class UpdatedAvatarViewTestCase(BaseViewTestCase): 364 | def test_should_update_avatar_url(self): 365 | # Given 366 | self.login() 367 | expected_url = "foo.bar" 368 | 369 | def update_user_details(user_id): 370 | self.user.hacker.avatar_url = expected_url 371 | self.user.hacker.save() 372 | 373 | # When 374 | with patch("home.views.update_user_details", new=update_user_details): 375 | response = self.client.get("/updated_avatar/%s/" % self.user.id, follow=True) 376 | self.assertEqual(200, response.status_code) 377 | self.assertEqual(expected_url, response.content.decode("utf8")) 378 | 379 | def test_should_not_update_unknown_hacker_avatar_url(self): 380 | # Given 381 | self.login() 382 | # When 383 | with patch("home.views.update_user_details", new=lambda x, y: None): 384 | response = self.client.get("/updated_avatar/200/", follow=True) 385 | self.assertEqual(404, response.status_code) 386 | 387 | 388 | class ViewPostViewTestCase(BaseViewTestCase): 389 | def test_should_redirect_to_post(self): 390 | # Given 391 | self.create_posts(1) 392 | post = Post.objects.filter()[0] 393 | post_url = post.url 394 | # When 395 | response = self.client.get("/post/{}/view/".format(post.slug)) 396 | # Then 397 | self.assertEqual(response["Location"], post_url) 398 | self.assertEqual(response.status_code, 302) 399 | self.assertEqual(1, LogEntry.objects.filter(post=post).count()) 400 | 401 | def test_does_not_redirect_to_bogus_post(self): 402 | # When 403 | response = self.client.get("/post/BOGUS/view/") 404 | # Then 405 | self.assertEqual(response.status_code, 404) 406 | 407 | 408 | class MostViewedViewTestCase(BaseViewTestCase): 409 | def test_should_enforce_authentication(self): 410 | # When 411 | response = self.client.get("/most_viewed/", follow=True) 412 | # Then 413 | self.assertRedirects(response, "/login/?next=/most_viewed/") 414 | 415 | def test_should_show_most_viewed_posts(self): 416 | # Given 417 | self.login() 418 | self.create_posts(10) 419 | post = Post.objects.filter()[0] 420 | self.client.get("/post/{}/view/".format(post.slug)) 421 | # When 422 | response = self.client.get("/most_viewed/", follow=True) 423 | # Then 424 | self.assertContains(response, post.title) 425 | 426 | def test_should_show_most_viewed_posts_n_days(self): 427 | # Given 428 | self.login() 429 | self.create_posts(10) 430 | post = Post.objects.filter()[0] 431 | self.client.get("/post/{}/view/".format(post.slug)) 432 | # When 433 | response = self.client.get("/most_viewed/30/", follow=True) 434 | # Then 435 | self.assertContains(response, post.title) 436 | 437 | def test_should_show_most_viewed_posts_tsv(self): 438 | # Given 439 | self.login() 440 | self.create_posts(10) 441 | post = Post.objects.filter()[0] 442 | self.client.get("/post/{}/view/".format(post.slug)) 443 | # When 444 | response = self.client.get("/most_viewed/?tsv=1", follow=True) 445 | # Then 446 | self.assertEqual(response["Content-Type"], "text/tab-separated-values") 447 | self.assertContains(response, post.title) 448 | 449 | def test_should_show_most_viewed_posts_tsv_n_days(self): 450 | # Given 451 | self.login() 452 | self.create_posts(10) 453 | post = Post.objects.filter()[0] 454 | self.client.get("/post/{}/view/".format(post.slug)) 455 | # When 456 | response = self.client.get("/most_viewed/30/?tsv=1", follow=True) 457 | # Then 458 | self.assertEqual(response["Content-Type"], "text/tab-separated-values") 459 | self.assertContains(response, post.title) 460 | 461 | 462 | class NewViewTestCase(BaseViewTestCase): 463 | def test_should_show_new_posts(self): 464 | # Given 465 | self.login() 466 | self.create_posts(5) 467 | # When 468 | response = self.client.get("/new/", follow=True) 469 | # Then 470 | for post in Post.objects.all(): 471 | self.assertContains(response, post.title) 472 | self.assertContains(response, post.slug) 473 | 474 | def test_should_paginate_new_posts(self): 475 | # Given 476 | self.login() 477 | self.create_posts(35) 478 | # When 479 | response_1 = self.client.get("/new/", follow=True) 480 | response_2 = self.client.get("/new/?page=2", follow=True) 481 | # Then 482 | for post in Post.objects.all(): 483 | if post.title in response_1.content.decode("utf8"): 484 | self.assertContains(response_1, post.slug) 485 | self.assertNotContains(response_2, post.title) 486 | else: 487 | self.assertContains(response_2, post.title) 488 | self.assertContains(response_2, post.slug) 489 | 490 | 491 | class SearchViewTestCase(BaseViewTestCase): 492 | def test_should_show_matching_posts(self): 493 | # Given 494 | self.login() 495 | query = "python" 496 | n = 5 497 | matching_title = "Python is awesome" 498 | non_matching_title = "Django rocks" 499 | self.create_posts(n, title=matching_title) 500 | self.create_posts(n, title=non_matching_title) 501 | # When 502 | response = self.client.get("/search/?q={}".format(query), follow=True) 503 | # Then 504 | for post in Post.objects.all(): 505 | if post.title == matching_title: 506 | self.assertContains(response, post.title) 507 | self.assertContains(response, post.slug) 508 | else: 509 | self.assertNotContains(response, post.slug) 510 | -------------------------------------------------------------------------------- /home/tests/utils.py: -------------------------------------------------------------------------------- 1 | """Utilities for tests.""" 2 | from django.utils import feedgenerator, timezone 3 | import factory 4 | from factory.django import DjangoModelFactory 5 | from faker import Faker 6 | from mock import Mock 7 | 8 | from home.models import Blog, Post, User 9 | 10 | tzinfo = timezone.get_default_timezone() 11 | alphabet = "".join([chr(i) for i in range(32, 2 ** 10)]) 12 | 13 | fake = Faker(["it_IT", "en_US", "ja_JP"]) 14 | 15 | 16 | def _generate_feed(atom=False): 17 | link = fake.url() 18 | feed = { 19 | "title": fake.sentence(), 20 | "link": link, 21 | "description": fake.sentence(), 22 | } 23 | optional = { 24 | "language": fake.locale(), 25 | "author_email": fake.email(), 26 | "author_name": fake.name(), 27 | "author_link": fake.url(), 28 | "subtitle": fake.sentence(), 29 | "categories": fake.pylist(nb_elements=5, value_types=[fake.word]), 30 | "feed_url": fake.url(), 31 | "feed_copyright": fake.sentence(), 32 | "id": link if atom else fake.uuid4(), 33 | "ttl": fake.pyint(), 34 | } 35 | feed.update([(k, v) for (k, v) in optional.items() if fake.pybool()]) 36 | return feed 37 | 38 | 39 | def _generate_item(atom=False): 40 | link = fake.url() 41 | item = { 42 | "title": fake.sentence(), 43 | "link": link, 44 | "description": fake.sentence(), 45 | } 46 | optional = { 47 | "content": fake.paragraph(), 48 | "author_email": fake.email(), 49 | "author_name": fake.name(), 50 | "author_link": fake.url(), 51 | "pubdate": fake.date_time(), 52 | "updateddate": fake.date_time(), 53 | "unique_id": link if atom else fake.uuid4(), 54 | "categories": fake.pylist(nb_elements=5, value_types=[fake.word]), 55 | "item_copyright": fake.sentence(), 56 | "ttl": fake.pyint(), 57 | } 58 | item.update([(k, v) for (k, v) in optional.items() if fake.pybool()]) 59 | return item 60 | 61 | 62 | def generate_full_feed(min_items=0, max_items=20): 63 | atom = fake.boolean() 64 | generator = feedgenerator.Atom1Feed if atom else feedgenerator.Rss201rev2Feed 65 | feed = generator(**_generate_feed(atom=atom)) 66 | items = [ 67 | _generate_item(atom=atom) 68 | for _ in range(fake.pyint(min_value=min_items, max_value=max_items)) 69 | ] 70 | for item in items: 71 | feed.add_item(**item) 72 | 73 | return feed 74 | 75 | 76 | def create_posts(n, **kwargs): 77 | """Create the specified number of posts.""" 78 | return PostFactory.create_batch(n, **kwargs) 79 | 80 | 81 | def fake_response(text): 82 | def wrapped(*args, **kwargs): 83 | response = Mock() 84 | response.headers = {} 85 | response.read = Mock(return_value=text.strip()) 86 | response.url = "" 87 | return response 88 | 89 | return wrapped 90 | 91 | 92 | class UserFactory(DjangoModelFactory): 93 | class Meta: 94 | model = User 95 | 96 | username = factory.Faker("email") 97 | first_name = factory.Faker("first_name") 98 | last_name = factory.Faker("last_name") 99 | 100 | 101 | class BlogFactory(DjangoModelFactory): 102 | class Meta: 103 | model = Blog 104 | 105 | user = factory.SubFactory(UserFactory) 106 | feed_url = factory.Faker("uri") 107 | 108 | 109 | class PostFactory(DjangoModelFactory): 110 | class Meta: 111 | model = Post 112 | 113 | url = factory.Faker("uri") 114 | posted_at = factory.Faker("date_time_this_decade", after_now=True, tzinfo=tzinfo) 115 | title = factory.Faker("sentence") 116 | content = factory.Faker("text") 117 | blog = factory.SubFactory(BlogFactory) 118 | -------------------------------------------------------------------------------- /home/token_auth.py: -------------------------------------------------------------------------------- 1 | """Define a new backend for authenticating with a token.""" 2 | 3 | from django.contrib.auth import get_user_model 4 | from django.contrib.auth.backends import ModelBackend 5 | 6 | 7 | class TokenAuthBackend(ModelBackend): 8 | """Allows users to authenticate using a token.""" 9 | 10 | def authenticate(self, request, token): 11 | # If a user with no related hacker exists in the db, token=None will 12 | # fetch that user! 13 | if not token: 14 | return None 15 | 16 | UserModel = get_user_model() 17 | try: 18 | return UserModel.objects.get(hacker__token=token) 19 | 20 | except UserModel.DoesNotExist: 21 | return None 22 | -------------------------------------------------------------------------------- /home/urls.py: -------------------------------------------------------------------------------- 1 | from django.http import HttpResponseRedirect 2 | from django.urls import include, path, re_path, reverse 3 | 4 | from home import views 5 | 6 | urlpatterns = [ 7 | re_path(r"^$", lambda x: HttpResponseRedirect(reverse("new"))), 8 | re_path(r"^login/$", views.log_in_oauth), 9 | re_path(r"^profile/$", views.own_profile), 10 | re_path(r"^profile/(?P\d+)/$", views.profile, name="profile"), 11 | re_path(r"^new/$", views.new, name="new"), 12 | re_path(r"^add_blog/$", views.add_blog, name="add_blog"), 13 | re_path(r"^edit_blog/(?P\d+)/$", views.edit_blog, name="edit_blog"), 14 | re_path(r"^delete_blog/(?P\d+)/$", views.delete_blog, name="delete_blog"), 15 | re_path(r"^atom\.xml$", views.feed, name="feed"), 16 | re_path(r"^refresh_token/$", views.refresh_token, name="refresh_token"), 17 | re_path(r"^post/(?P\w+)/view", views.view_post, name="view_post"), 18 | re_path(r"^logout/$", views.log_out), 19 | re_path(r"^log_out/$", views.log_out, name="log_out"), 20 | re_path(r"^login-error/$", views.login_error, name="login_error"), 21 | re_path(r"^about/$", views.about, name="about"), 22 | re_path(r"^most_viewed/$", views.most_viewed, name="most_viewed"), 23 | re_path(r"^most_viewed/(?P\d+)/$", views.most_viewed, name="most_viewed_days"), 24 | re_path(r"^updated_avatar/(?P\d+)/$", views.updated_avatar, name="updated_avatar"), 25 | re_path(r"^search/$", views.search, name="search"), 26 | path("", include("social_django.urls", namespace="social")), 27 | ] 28 | -------------------------------------------------------------------------------- /home/utils.py: -------------------------------------------------------------------------------- 1 | import bs4 2 | 3 | 4 | def is_medium_comment(entry): 5 | """Check if a link is a medium comment.""" 6 | 7 | link = getattr(entry, "link", getattr(entry, "url", "")) 8 | if "medium.com" not in link: 9 | return False 10 | 11 | content = getattr(entry, "summary", getattr(entry, "content", "")) 12 | title = entry.title or "" 13 | soup = bs4.BeautifulSoup(content, "lxml") 14 | # Medium comments set their title from the content of the comment. So, we 15 | # verify if the content starts with the content. We convert the string to 16 | # ascii, before comparing, because the content seems to have additional 17 | # unicode characters which are not present in the title. 18 | text_content = soup.text.strip().encode("ascii", "replace").replace(b"?", b" ") 19 | title = title.strip().encode("ascii", "replace").replace(b"?", b" ") 20 | is_comment = text_content.startswith(title) 21 | return is_comment 22 | -------------------------------------------------------------------------------- /home/views.py: -------------------------------------------------------------------------------- 1 | from collections import namedtuple 2 | import datetime 3 | from functools import wraps 4 | import uuid 5 | 6 | from django.contrib import messages 7 | from django.contrib.auth import authenticate, logout 8 | from django.contrib.auth.models import User 9 | from django.contrib.auth.decorators import login_required 10 | from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger 11 | from django.db.models import Count 12 | from django.forms import Select, TextInput 13 | from django.forms.models import modelform_factory 14 | from django.http import HttpResponse, HttpResponseRedirect, Http404, HttpResponseForbidden 15 | from django.shortcuts import render 16 | from django.urls import reverse 17 | from django.utils import timezone 18 | from django.views.decorators.http import require_POST 19 | 20 | from home.models import Blog, Hacker, LogEntry, Post, STREAM_CHOICES 21 | from home.oauth import update_user_details 22 | from home.feeds import LatestEntriesFeed 23 | from . import feedergrabber27 24 | 25 | EXISTING_FEED_MESSAGE = "This feed has already been added!" 26 | NO_CONTENT_MESSAGE = 'Could not fetch feed from {url}. Is the website up?' 27 | INVALID_FEED_MESSAGE = ( 28 | "This does not seem to be a valid feed. " "Please use your blog's feed url (not the web url)!" 29 | ) 30 | SUGGEST_FEED_URL_MESSAGE = INVALID_FEED_MESSAGE + (' It could be -- {url}') 31 | SUCCESS_MESSAGE = ( 32 | 'Your blog ({url}) has been added successfully. ' 33 | "The next crawl (hourly) will fetch your posts." 34 | ) 35 | 36 | 37 | def ensure_blog_exists(f): 38 | @wraps(f) 39 | def wrapper(request, blog_id): 40 | try: 41 | blog = Blog.objects.get(id=blog_id, user=request.user) 42 | request.blog = blog 43 | except Blog.DoesNotExist: 44 | raise Http404 45 | 46 | return f(request, blog_id) 47 | 48 | return wrapper 49 | 50 | 51 | def ensure_hacker_exists(f): 52 | @wraps(f) 53 | def wrapper(request, user_id): 54 | try: 55 | hacker = Hacker.objects.get(user=user_id) 56 | except Hacker.DoesNotExist: 57 | raise Http404 58 | 59 | request.hacker = hacker 60 | return f(request, user_id) 61 | 62 | return wrapper 63 | 64 | 65 | def paginator(queryset, page_number, page_size=30): 66 | paginator = Paginator(queryset, page_size) 67 | try: 68 | items = paginator.page(page_number) 69 | except PageNotAnInteger: 70 | # If page is not an integer, deliver first page. 71 | items = paginator.page(1) 72 | except EmptyPage: 73 | # If page is out of range (e.g. 9999), deliver last page of results. 74 | items = paginator.page(paginator.num_pages) 75 | return items 76 | 77 | 78 | def view_post(request, slug): 79 | """Redirect to the original link. 80 | 81 | We use a redirect, so that we can collect stats if we decide to, and do 82 | useful things with it. 83 | 84 | """ 85 | try: 86 | post = Post.objects.get(slug=slug) 87 | except Post.DoesNotExist: 88 | raise Http404("Post does not exist.") 89 | 90 | LogEntry.objects.create( 91 | post=post, 92 | date=timezone.now(), 93 | referer=request.META.get("HTTP_REFERER", None), 94 | remote_addr=request.META.get("REMOTE_ADDR", None), 95 | user_agent=request.META.get("HTTP_USER_AGENT", None), 96 | ) 97 | return HttpResponseRedirect(post.url) 98 | 99 | 100 | def log_in_oauth(request): 101 | if request.user.is_authenticated: 102 | return HttpResponseRedirect(reverse("new")) 103 | 104 | else: 105 | return render(request, "home/log_in_oauth.html") 106 | 107 | 108 | @login_required 109 | def log_out(request): 110 | """Log out a logged in user.""" 111 | logout(request) 112 | return HttpResponseRedirect("/") 113 | 114 | 115 | @login_required 116 | def add_blog(request): 117 | """Adds a new blog to a user's profile.""" 118 | user_id = request.user.id 119 | # /add_blog in linked to in recurse.com, etc. 120 | if request.method == "GET": 121 | return HttpResponseRedirect(reverse("profile", kwargs={"user_id": user_id})) 122 | 123 | feed_url = request.POST.get("feed_url", None) 124 | if feed_url: 125 | feed_url = feed_url.strip() 126 | # add http:// prefix if missing 127 | if feed_url[:4] != "http": 128 | feed_url = "http://" + feed_url 129 | # Check if the feed has already been added, and bail out 130 | user_blogs = Blog.objects.filter(user=user_id, feed_url=feed_url) 131 | if user_blogs.exists(): 132 | user_blogs.update(skip_crawl=False) 133 | messages.info(request, EXISTING_FEED_MESSAGE) 134 | return HttpResponseRedirect(reverse("profile", kwargs={"user_id": user_id})) 135 | 136 | contents, errors = feedergrabber27.retrieve_file_contents(feed_url) 137 | if contents is None: # URLError or HTTPError 138 | messages.error( 139 | request, 140 | NO_CONTENT_MESSAGE.format(url=feed_url), 141 | extra_tags="safe", 142 | ) 143 | elif contents.bozo and not isinstance( 144 | contents.bozo_exception, 145 | feedergrabber27.CharacterEncodingOverride, 146 | ): # Content failed to parse 147 | guessed_url = feedergrabber27.find_feed_url(contents) 148 | message = SUGGEST_FEED_URL_MESSAGE if guessed_url is not None else INVALID_FEED_MESSAGE 149 | messages.error(request, message.format(url=guessed_url), extra_tags="safe") 150 | else: 151 | # create new blog record in db 152 | Blog.objects.create(user=User.objects.get(id=user_id), feed_url=feed_url) 153 | messages.success( 154 | request, 155 | SUCCESS_MESSAGE.format(url=feed_url), 156 | extra_tags="safe", 157 | ) 158 | else: 159 | messages.error(request, "No feed URL provided.") 160 | return HttpResponseRedirect(reverse("profile", kwargs={"user_id": user_id})) 161 | 162 | 163 | @login_required 164 | @ensure_blog_exists 165 | def delete_blog(request, blog_id): 166 | blog = request.blog 167 | user = request.user 168 | blog.delete() 169 | return HttpResponseRedirect(reverse("profile", kwargs={"user_id": user.id})) 170 | 171 | 172 | @login_required 173 | @ensure_blog_exists 174 | @require_POST 175 | def edit_blog(request, blog_id): 176 | blog = request.blog 177 | user = request.user 178 | form = BlogForm(request.POST, instance=blog) 179 | if form.is_valid(): 180 | form.instance.skip_crawl = False 181 | form.save() 182 | return HttpResponseRedirect(reverse("profile", kwargs={"user_id": user.id})) 183 | 184 | 185 | @login_required 186 | @ensure_hacker_exists 187 | def profile(request, user_id): 188 | added_blogs = Blog.objects.filter(user=user_id) 189 | owner = int(user_id) == request.user.id 190 | post_list = Post.objects.filter(blog__user=user_id) 191 | feed_path = reverse("feed") 192 | feed_url = request.build_absolute_uri(f"{feed_path}?token={request.hacker.token}") 193 | context = { 194 | "hacker": request.hacker, 195 | "owner": owner, 196 | "post_list": post_list, 197 | "show_avatars": False, 198 | "forms": [BlogForm(instance=blog) for blog in added_blogs], 199 | "feed_url": feed_url, 200 | } 201 | return render(request, "home/profile.html", context) 202 | 203 | 204 | @login_required 205 | def own_profile(request): 206 | return HttpResponseRedirect(reverse("profile", kwargs={"user_id": request.user.id})) 207 | 208 | 209 | @login_required 210 | def new(request): 211 | """Newest blog posts - main app view.""" 212 | posts = Post.objects.all() 213 | page = request.GET.get("page", 1) 214 | post_list = paginator(posts, page) 215 | context = {"post_list": post_list, "show_avatars": True, "page_view": "new"} 216 | return render(request, "home/new.html", context) 217 | 218 | 219 | @login_required 220 | def search(request): 221 | """Search blog posts based on query - main app view.""" 222 | query = request.GET.get("q", "") 223 | posts = Post.objects.filter(title__search=query) 224 | count = posts.count() 225 | page = request.GET.get("page", 1) 226 | post_list = paginator(posts, page) 227 | context = { 228 | "count": count, 229 | "query": query, 230 | "post_list": post_list, 231 | "show_avatars": True, 232 | "page_view": "search", 233 | } 234 | return render(request, "home/search.html", context) 235 | 236 | 237 | @login_required 238 | @ensure_hacker_exists 239 | def updated_avatar(request, user_id): 240 | update_user_details(user_id) 241 | hacker = Hacker.objects.get(user=user_id) 242 | return HttpResponse(hacker.avatar_url) 243 | 244 | 245 | # FIXME: This view could be cached, with cache cleared on crawls or Blog 246 | # create/delete signals. Probably most other views could be cached. 247 | def feed(request): 248 | """Atom feed of new posts.""" 249 | token = request.GET.get("token") 250 | if authenticate(token=token) is None: 251 | return HttpResponseForbidden() 252 | 253 | return LatestEntriesFeed()(request) 254 | 255 | 256 | @login_required 257 | def refresh_token(request): 258 | """Refresh a users' auth token.""" 259 | hacker = Hacker.objects.get(user=request.user) 260 | hacker.token = uuid.uuid4().hex 261 | hacker.save() 262 | profile_url = reverse("profile", kwargs={"user_id": request.user.id}) 263 | return HttpResponseRedirect(profile_url) 264 | 265 | 266 | @login_required 267 | def login_error(request): 268 | """OAuth error page""" 269 | return render(request, "home/login_error.html") 270 | 271 | 272 | # login NOT required 273 | def about(request): 274 | """About page with more info on Blaggregator.""" 275 | return render(request, "home/about.html") 276 | 277 | 278 | @login_required 279 | def most_viewed(request, ndays="7"): 280 | now = timezone.now() 281 | ndays = int(ndays) 282 | since = now - datetime.timedelta(days=ndays) 283 | entries = _get_most_viewed_entries(since=since) 284 | streams = dict(STREAM_CHOICES) 285 | # Return a tab separated values file, if requested 286 | if request.GET.get("tsv") == "1": 287 | header = "post_id\ttitle\turl\tcount\n" 288 | text = "\n".join(_get_tsv(entry) for entry in entries) 289 | text = header + text 290 | response = HttpResponse(text, content_type="text/tab-separated-values") 291 | else: 292 | Post = namedtuple("Post", ("authorid", "avatar", "slug", "title", "stream")) 293 | context = { 294 | "post_list": [ 295 | Post( 296 | slug=entry["post__slug"], 297 | authorid=entry["post__blog__user__id"], 298 | avatar=entry["post__blog__user__hacker__avatar_url"], 299 | title=entry["post__title"], 300 | stream=streams[entry["post__blog__stream"]], 301 | ) 302 | for entry in entries 303 | ], 304 | "from": since.date(), 305 | "to": now.date(), 306 | "show_avatars": True, 307 | } 308 | response = render(request, "home/most_viewed.html", context) 309 | return response 310 | 311 | 312 | def _get_most_viewed_entries(since, n=20): 313 | # Get posts visited during the last week 314 | entries = LogEntry.objects.filter(date__gte=since) 315 | # Get post url and title 316 | entries = entries.values( 317 | "post__id", 318 | "post__title", 319 | "post__url", 320 | "post__slug", 321 | "post__blog__user__id", 322 | "post__blog__user__hacker__avatar_url", 323 | "post__blog__stream", 324 | ) 325 | # Count the visits 326 | entries = entries.annotate(total=Count("post__id")) 327 | # Get top 'n' posts 328 | entries = entries.order_by("total").reverse()[:n] 329 | return entries 330 | 331 | 332 | def _get_tsv(entry): 333 | return "{post__id}\t{post__title}\t{post__url}\t{total}".format(**entry) 334 | 335 | 336 | BlogForm = modelform_factory( 337 | Blog, 338 | fields=("feed_url", "stream"), 339 | widgets={ 340 | "feed_url": TextInput(attrs={"class": "form-control", "type": "url"}), 341 | "stream": Select(attrs={"class": "custom-select"}), 342 | }, 343 | ) 344 | -------------------------------------------------------------------------------- /home/zulip_helpers.py: -------------------------------------------------------------------------------- 1 | # Standard library 2 | import json 3 | import logging 4 | import os 5 | import re 6 | 7 | # 3rd-party library 8 | from django.conf import settings 9 | from django.template.loader import get_template 10 | from django.urls import reverse 11 | import requests 12 | 13 | from home.models import Hacker, Post 14 | from home.oauth import update_user_details 15 | 16 | ZULIP_KEY = os.environ.get("ZULIP_KEY") 17 | ZULIP_EMAIL = os.environ.get("ZULIP_EMAIL") 18 | MESSAGES_URL = "https://recurse.zulipchat.com/api/v1/messages" 19 | MEMBERS_URL = "https://recurse.zulipchat.com/api/v1/users" 20 | ANNOUNCE_MESSAGE = "{} has a new blog post: [{}]({})" 21 | log = logging.getLogger("blaggregator") 22 | 23 | 24 | def announce_posts(posts, debug=True): 25 | """Announce new posts on the correct stream. 26 | 27 | *NOTE*: If DEBUG mode is on, all messages are sent to the bot-test stream. 28 | 29 | """ 30 | 31 | if not posts: 32 | log.debug("No posts to announce") 33 | return 34 | 35 | author_zulip_ids = get_author_zulip_ids(posts) 36 | zulip_members = get_members()["by_id"] 37 | 38 | for post in posts: 39 | author_zulip_id = author_zulip_ids.get(post.id) 40 | author_name = zulip_members.get(author_zulip_id, {}).get("full_name") 41 | author = f"@**{author_name}**" if author_name else f"**{post.author}**" 42 | to = post.blog.get_stream_display() if not debug else "bot-test" 43 | title = post.title 44 | subject = title if len(title) <= 60 else title[:57] + "..." 45 | path = reverse("view_post", kwargs={"slug": post.slug}) 46 | url = "{}/{}".format(settings.ROOT_URL.rstrip("/"), path.lstrip("/")) 47 | content = ANNOUNCE_MESSAGE.format(author, title, url) 48 | send_message_zulip(to, subject, content, type_="stream") 49 | 50 | 51 | def delete_message(message_id, content="(deleted)"): 52 | message_url = "{}/{}".format(MESSAGES_URL, message_id) 53 | params = {"content": content, "subject": content} 54 | try: 55 | response = requests.patch(message_url, params=params, auth=(ZULIP_EMAIL, ZULIP_KEY)) 56 | assert response["result"] == "success" 57 | except Exception as e: 58 | log.error("Could not delete Zulip message %s: %s", message_id, e) 59 | 60 | 61 | def get_author_zulip_ids(posts): 62 | """Return mapping of post ID to author Zulip ID. 63 | 64 | NOTE: The function also tries to update the Zulip IDs of users that are not 65 | in our DB using data from the RC API end-point: /api/v1/profiles/:id 66 | 67 | """ 68 | 69 | post_ids = {post.id for post in posts} 70 | author_zulip_ids = dict( 71 | Post.objects.filter(pk__in=post_ids).values_list( 72 | "blog__user", "blog__user__hacker__zulip_id" 73 | ) 74 | ) 75 | 76 | for user_id, zulip_id in author_zulip_ids.items(): 77 | if zulip_id is None: 78 | update_user_details(user_id) 79 | hacker = Hacker.objects.filter(user_id=user_id).first() 80 | if hacker is not None and hacker.zulip_id is not None: 81 | author_zulip_ids[user_id] = hacker.zulip_id 82 | log.debug("Updated Zulip ID for hacker %s", user_id) 83 | else: 84 | log.error("Failed to update Zulip ID for hacker %s", user_id) 85 | 86 | return {post.id: author_zulip_ids.get(post.blog.user_id) for post in posts} 87 | 88 | 89 | def get_members(): 90 | """Returns info of all the Zulip users. 91 | 92 | Returns a mapping with three keys - by_name, by_email and by_id. 93 | 94 | """ 95 | try: 96 | log.debug("Fetching all Zulip members") 97 | response = requests.get(MEMBERS_URL, auth=(ZULIP_EMAIL, ZULIP_KEY)) 98 | members = response.json()["members"] 99 | except Exception as e: 100 | log.error("Could not fetch zulip users: %s", e) 101 | members = [] 102 | by_name = { 103 | strip_batch(member["full_name"]): member 104 | for member in members 105 | if not member["is_bot"] and member["is_active"] 106 | } 107 | by_email = {member["email"]: member for member in by_name.values()} 108 | by_id = {member["user_id"]: member for member in by_name.values()} 109 | return dict(by_email=by_email, by_name=by_name, by_id=by_id) 110 | 111 | 112 | def get_pm_link(user, members): 113 | """Returns a zulip link for PM with a user.""" 114 | name = user.get_full_name() 115 | first_name = user.first_name.lower() 116 | uid = members["by_name"][name]["user_id"] 117 | return "[{name}](#narrow/pm-with/{uid}-{first_name})".format( 118 | name=name, uid=uid, first_name=first_name 119 | ) 120 | 121 | 122 | def get_stream_messages(stream): 123 | request = { 124 | "anchor": 10000000000000000, 125 | "num_before": 5000, 126 | "num_after": 0, 127 | "narrow": json.dumps([{"operator": "stream", "operand": stream}]), 128 | } 129 | try: 130 | log.debug("Fetching Zulip messages") 131 | response = requests.get(MESSAGES_URL, params=request, auth=(ZULIP_EMAIL, ZULIP_KEY)) 132 | messages = response.json()["messages"] 133 | except Exception as e: 134 | log.error("Could not fetch Zulip messages: %s", e) 135 | messages = [] 136 | 137 | return messages 138 | 139 | 140 | def guess_zulip_emails(users, members): 141 | """Get zulip emails for users 142 | 143 | Some users may not have the same email ids on zulip and recurse.com, in 144 | which case sending private messages will fail. This function tries to 145 | detect the zulip email based on the full name of the user. 146 | 147 | *NOTE*: The email in our DB is not changed. The email is temporarily set as 148 | an attribute on the user, so that notifications can be sent to the user. 149 | 150 | """ 151 | EMAILS = members["by_email"] 152 | NAMES = members["by_name"] 153 | for user in users: 154 | if user.email not in EMAILS and user.get_full_name() in NAMES: 155 | user.zulip_email = NAMES[user.get_full_name()] 156 | else: 157 | # Either the email is correct OR 158 | # Both name and email have changed or account deleted! 159 | pass 160 | return users 161 | 162 | 163 | def notify_uncrawlable_blogs(user, blogs, admins, debug=True): 164 | """Notify blog owner about blogs that are failing crawls.""" 165 | subject = "Blaggregator: Action required!" 166 | context = dict( 167 | user=user, 168 | blogs=blogs, 169 | base_url=settings.ROOT_URL.rstrip("/"), 170 | admins=admins, 171 | ) 172 | content = get_template("home/disabling-crawling.md").render(context) 173 | to = getattr(user, "zulip_email", user.email) 174 | type_ = "private" 175 | if debug: 176 | log.debug("Sending message \n\n%s\n\n to %s (%s)", content, to, type_) 177 | return False 178 | 179 | else: 180 | return send_message_zulip(to, subject, content, type_=type_) 181 | 182 | 183 | def send_message_zulip(to, subject, content, type_="private"): 184 | """Send a message to Zulip.""" 185 | data = {"type": type_, "to": to, "subject": subject, "content": content} 186 | try: 187 | log.debug('Sending message "%s" to %s (%s)', content, to, type_) 188 | response = requests.post(MESSAGES_URL, data=data, auth=(ZULIP_EMAIL, ZULIP_KEY)) 189 | log.debug( 190 | "Post returned with %s: %s", 191 | response.status_code, 192 | response.content, 193 | ) 194 | return response.status_code == 200 195 | 196 | except Exception as e: 197 | log.exception(e) 198 | return False 199 | 200 | 201 | def strip_batch(name): 202 | """Strip parenthesized batch from a name""" 203 | return re.sub(r"\(.*\)", "", name).strip() 204 | -------------------------------------------------------------------------------- /manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | if __name__ == "__main__": 6 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "blaggregator.settings") 7 | 8 | from django.core.management import execute_from_command_line 9 | 10 | execute_from_command_line(sys.argv) 11 | -------------------------------------------------------------------------------- /poetry.lock: -------------------------------------------------------------------------------- 1 | [[package]] 2 | name = "asgiref" 3 | version = "3.4.1" 4 | description = "ASGI specs, helper code, and adapters" 5 | category = "main" 6 | optional = false 7 | python-versions = ">=3.6" 8 | 9 | [package.extras] 10 | tests = ["pytest", "pytest-asyncio", "mypy (>=0.800)"] 11 | 12 | [[package]] 13 | name = "beautifulsoup4" 14 | version = "4.10.0" 15 | description = "Screen-scraping library" 16 | category = "main" 17 | optional = false 18 | python-versions = ">3.0.0" 19 | 20 | [package.dependencies] 21 | soupsieve = ">1.2" 22 | 23 | [package.extras] 24 | html5lib = ["html5lib"] 25 | lxml = ["lxml"] 26 | 27 | [[package]] 28 | name = "boto3" 29 | version = "1.18.44" 30 | description = "The AWS SDK for Python" 31 | category = "main" 32 | optional = false 33 | python-versions = ">= 3.6" 34 | 35 | [package.dependencies] 36 | botocore = ">=1.21.44,<1.22.0" 37 | jmespath = ">=0.7.1,<1.0.0" 38 | s3transfer = ">=0.5.0,<0.6.0" 39 | 40 | [package.extras] 41 | crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] 42 | 43 | [[package]] 44 | name = "botocore" 45 | version = "1.21.44" 46 | description = "Low-level, data-driven core of boto 3." 47 | category = "main" 48 | optional = false 49 | python-versions = ">= 3.6" 50 | 51 | [package.dependencies] 52 | jmespath = ">=0.7.1,<1.0.0" 53 | python-dateutil = ">=2.1,<3.0.0" 54 | urllib3 = ">=1.25.4,<1.27" 55 | 56 | [package.extras] 57 | crt = ["awscrt (==0.11.24)"] 58 | 59 | [[package]] 60 | name = "certifi" 61 | version = "2021.5.30" 62 | description = "Python package for providing Mozilla's CA Bundle." 63 | category = "main" 64 | optional = false 65 | python-versions = "*" 66 | 67 | [[package]] 68 | name = "cffi" 69 | version = "1.14.6" 70 | description = "Foreign Function Interface for Python calling C code." 71 | category = "main" 72 | optional = false 73 | python-versions = "*" 74 | 75 | [package.dependencies] 76 | pycparser = "*" 77 | 78 | [[package]] 79 | name = "charset-normalizer" 80 | version = "2.0.6" 81 | description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." 82 | category = "main" 83 | optional = false 84 | python-versions = ">=3.5.0" 85 | 86 | [package.extras] 87 | unicode_backport = ["unicodedata2"] 88 | 89 | [[package]] 90 | name = "cryptography" 91 | version = "3.4.8" 92 | description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." 93 | category = "main" 94 | optional = false 95 | python-versions = ">=3.6" 96 | 97 | [package.dependencies] 98 | cffi = ">=1.12" 99 | 100 | [package.extras] 101 | docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1)", "sphinx-rtd-theme"] 102 | docstest = ["doc8", "pyenchant (>=1.6.11)", "twine (>=1.12.0)", "sphinxcontrib-spelling (>=4.0.1)"] 103 | pep8test = ["black", "flake8", "flake8-import-order", "pep8-naming"] 104 | sdist = ["setuptools-rust (>=0.11.4)"] 105 | ssh = ["bcrypt (>=3.1.5)"] 106 | test = ["pytest (>=6.0)", "pytest-cov", "pytest-subtests", "pytest-xdist", "pretend", "iso8601", "pytz", "hypothesis (>=1.11.4,!=3.79.2)"] 107 | 108 | [[package]] 109 | name = "defusedxml" 110 | version = "0.7.1" 111 | description = "XML bomb protection for Python stdlib modules" 112 | category = "main" 113 | optional = false 114 | python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" 115 | 116 | [[package]] 117 | name = "dj-database-url" 118 | version = "0.5.0" 119 | description = "Use Database URLs in your Django Application." 120 | category = "main" 121 | optional = false 122 | python-versions = "*" 123 | 124 | [[package]] 125 | name = "django" 126 | version = "3.2.7" 127 | description = "A high-level Python Web framework that encourages rapid development and clean, pragmatic design." 128 | category = "main" 129 | optional = false 130 | python-versions = ">=3.6" 131 | 132 | [package.dependencies] 133 | asgiref = ">=3.3.2,<4" 134 | pytz = "*" 135 | sqlparse = ">=0.2.2" 136 | 137 | [package.extras] 138 | argon2 = ["argon2-cffi (>=19.1.0)"] 139 | bcrypt = ["bcrypt"] 140 | 141 | [[package]] 142 | name = "django-storages" 143 | version = "1.11.1" 144 | description = "Support for many storage backends in Django" 145 | category = "main" 146 | optional = false 147 | python-versions = ">=3.5" 148 | 149 | [package.dependencies] 150 | Django = ">=2.2" 151 | 152 | [package.extras] 153 | azure = ["azure-storage-blob (>=1.3.1,<12.0.0)"] 154 | boto3 = ["boto3 (>=1.4.4)"] 155 | dropbox = ["dropbox (>=7.2.1)"] 156 | google = ["google-cloud-storage (>=1.15.0)"] 157 | libcloud = ["apache-libcloud"] 158 | sftp = ["paramiko"] 159 | 160 | [[package]] 161 | name = "factory-boy" 162 | version = "3.2.0" 163 | description = "A versatile test fixtures replacement based on thoughtbot's factory_bot for Ruby." 164 | category = "dev" 165 | optional = false 166 | python-versions = ">=3.6" 167 | 168 | [package.dependencies] 169 | Faker = ">=0.7.0" 170 | 171 | [package.extras] 172 | dev = ["coverage", "django", "flake8", "isort", "pillow", "sqlalchemy", "mongoengine", "wheel (>=0.32.0)", "tox", "zest.releaser"] 173 | doc = ["sphinx", "sphinx-rtd-theme", "sphinxcontrib-spelling"] 174 | 175 | [[package]] 176 | name = "faker" 177 | version = "8.13.2" 178 | description = "Faker is a Python package that generates fake data for you." 179 | category = "dev" 180 | optional = false 181 | python-versions = ">=3.6" 182 | 183 | [package.dependencies] 184 | python-dateutil = ">=2.4" 185 | text-unidecode = "1.3" 186 | 187 | [[package]] 188 | name = "feedparser" 189 | version = "6.0.8" 190 | description = "Universal feed parser, handles RSS 0.9x, RSS 1.0, RSS 2.0, CDF, Atom 0.3, and Atom 1.0 feeds" 191 | category = "main" 192 | optional = false 193 | python-versions = ">=3.6" 194 | 195 | [package.dependencies] 196 | sgmllib3k = "*" 197 | 198 | [[package]] 199 | name = "flake8" 200 | version = "3.9.2" 201 | description = "the modular source code checker: pep8 pyflakes and co" 202 | category = "dev" 203 | optional = false 204 | python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" 205 | 206 | [package.dependencies] 207 | mccabe = ">=0.6.0,<0.7.0" 208 | pycodestyle = ">=2.7.0,<2.8.0" 209 | pyflakes = ">=2.3.0,<2.4.0" 210 | 211 | [[package]] 212 | name = "gevent" 213 | version = "21.8.0" 214 | description = "Coroutine-based network library" 215 | category = "main" 216 | optional = false 217 | python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5" 218 | 219 | [package.dependencies] 220 | cffi = {version = ">=1.12.2", markers = "platform_python_implementation == \"CPython\" and sys_platform == \"win32\""} 221 | greenlet = {version = ">=1.1.0,<2.0", markers = "platform_python_implementation == \"CPython\""} 222 | "zope.event" = "*" 223 | "zope.interface" = "*" 224 | 225 | [package.extras] 226 | dnspython = ["dnspython (>=1.16.0,<2.0)", "idna"] 227 | docs = ["repoze.sphinx.autointerface", "sphinxcontrib-programoutput", "zope.schema"] 228 | monitor = ["psutil (>=5.7.0)"] 229 | recommended = ["cffi (>=1.12.2)", "dnspython (>=1.16.0,<2.0)", "idna", "selectors2", "backports.socketpair", "psutil (>=5.7.0)"] 230 | test = ["requests", "objgraph", "cffi (>=1.12.2)", "dnspython (>=1.16.0,<2.0)", "idna", "selectors2", "futures", "mock", "backports.socketpair", "contextvars (==2.4)", "coverage (>=5.0)", "coveralls (>=1.7.0)", "psutil (>=5.7.0)"] 231 | 232 | [[package]] 233 | name = "greenlet" 234 | version = "1.1.1" 235 | description = "Lightweight in-process concurrent programming" 236 | category = "main" 237 | optional = false 238 | python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*" 239 | 240 | [package.extras] 241 | docs = ["sphinx"] 242 | 243 | [[package]] 244 | name = "gunicorn" 245 | version = "20.1.0" 246 | description = "WSGI HTTP Server for UNIX" 247 | category = "main" 248 | optional = false 249 | python-versions = ">=3.5" 250 | 251 | [package.extras] 252 | eventlet = ["eventlet (>=0.24.1)"] 253 | gevent = ["gevent (>=1.4.0)"] 254 | setproctitle = ["setproctitle"] 255 | tornado = ["tornado (>=0.2)"] 256 | 257 | [[package]] 258 | name = "idna" 259 | version = "3.2" 260 | description = "Internationalized Domain Names in Applications (IDNA)" 261 | category = "main" 262 | optional = false 263 | python-versions = ">=3.5" 264 | 265 | [[package]] 266 | name = "jmespath" 267 | version = "0.10.0" 268 | description = "JSON Matching Expressions" 269 | category = "main" 270 | optional = false 271 | python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" 272 | 273 | [[package]] 274 | name = "lxml" 275 | version = "4.9.2" 276 | description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." 277 | category = "main" 278 | optional = false 279 | python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, != 3.4.*" 280 | 281 | [package.extras] 282 | cssselect = ["cssselect (>=0.7)"] 283 | html5 = ["html5lib"] 284 | htmlsoup = ["beautifulsoup4"] 285 | source = ["Cython (>=0.29.7)"] 286 | 287 | [[package]] 288 | name = "mccabe" 289 | version = "0.6.1" 290 | description = "McCabe checker, plugin for flake8" 291 | category = "dev" 292 | optional = false 293 | python-versions = "*" 294 | 295 | [[package]] 296 | name = "mock" 297 | version = "4.0.3" 298 | description = "Rolling backport of unittest.mock for all Pythons" 299 | category = "dev" 300 | optional = false 301 | python-versions = ">=3.6" 302 | 303 | [package.extras] 304 | build = ["twine", "wheel", "blurb"] 305 | docs = ["sphinx"] 306 | test = ["pytest (<5.4)", "pytest-cov"] 307 | 308 | [[package]] 309 | name = "oauthlib" 310 | version = "3.1.1" 311 | description = "A generic, spec-compliant, thorough implementation of the OAuth request-signing logic" 312 | category = "main" 313 | optional = false 314 | python-versions = ">=3.6" 315 | 316 | [package.extras] 317 | rsa = ["cryptography (>=3.0.0,<4)"] 318 | signals = ["blinker (>=1.4.0)"] 319 | signedtoken = ["cryptography (>=3.0.0,<4)", "pyjwt (>=2.0.0,<3)"] 320 | 321 | [[package]] 322 | name = "psycopg2" 323 | version = "2.9.1" 324 | description = "psycopg2 - Python-PostgreSQL Database Adapter" 325 | category = "main" 326 | optional = false 327 | python-versions = ">=3.6" 328 | 329 | [[package]] 330 | name = "pycodestyle" 331 | version = "2.7.0" 332 | description = "Python style guide checker" 333 | category = "dev" 334 | optional = false 335 | python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" 336 | 337 | [[package]] 338 | name = "pycparser" 339 | version = "2.20" 340 | description = "C parser in Python" 341 | category = "main" 342 | optional = false 343 | python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" 344 | 345 | [[package]] 346 | name = "pyflakes" 347 | version = "2.3.1" 348 | description = "passive checker of Python programs" 349 | category = "dev" 350 | optional = false 351 | python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" 352 | 353 | [[package]] 354 | name = "pyjwt" 355 | version = "2.1.0" 356 | description = "JSON Web Token implementation in Python" 357 | category = "main" 358 | optional = false 359 | python-versions = ">=3.6" 360 | 361 | [package.extras] 362 | crypto = ["cryptography (>=3.3.1,<4.0.0)"] 363 | dev = ["sphinx", "sphinx-rtd-theme", "zope.interface", "cryptography (>=3.3.1,<4.0.0)", "pytest (>=6.0.0,<7.0.0)", "coverage[toml] (==5.0.4)", "mypy", "pre-commit"] 364 | docs = ["sphinx", "sphinx-rtd-theme", "zope.interface"] 365 | tests = ["pytest (>=6.0.0,<7.0.0)", "coverage[toml] (==5.0.4)"] 366 | 367 | [[package]] 368 | name = "python-dateutil" 369 | version = "2.8.2" 370 | description = "Extensions to the standard Python datetime module" 371 | category = "main" 372 | optional = false 373 | python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" 374 | 375 | [package.dependencies] 376 | six = ">=1.5" 377 | 378 | [[package]] 379 | name = "python3-openid" 380 | version = "3.2.0" 381 | description = "OpenID support for modern servers and consumers." 382 | category = "main" 383 | optional = false 384 | python-versions = "*" 385 | 386 | [package.dependencies] 387 | defusedxml = "*" 388 | 389 | [package.extras] 390 | mysql = ["mysql-connector-python"] 391 | postgresql = ["psycopg2"] 392 | 393 | [[package]] 394 | name = "pytz" 395 | version = "2021.1" 396 | description = "World timezone definitions, modern and historical" 397 | category = "main" 398 | optional = false 399 | python-versions = "*" 400 | 401 | [[package]] 402 | name = "requests" 403 | version = "2.26.0" 404 | description = "Python HTTP for Humans." 405 | category = "main" 406 | optional = false 407 | python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" 408 | 409 | [package.dependencies] 410 | certifi = ">=2017.4.17" 411 | charset-normalizer = {version = ">=2.0.0,<2.1.0", markers = "python_version >= \"3\""} 412 | idna = {version = ">=2.5,<4", markers = "python_version >= \"3\""} 413 | urllib3 = ">=1.21.1,<1.27" 414 | 415 | [package.extras] 416 | socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"] 417 | use_chardet_on_py3 = ["chardet (>=3.0.2,<5)"] 418 | 419 | [[package]] 420 | name = "requests-oauthlib" 421 | version = "1.3.0" 422 | description = "OAuthlib authentication support for Requests." 423 | category = "main" 424 | optional = false 425 | python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" 426 | 427 | [package.dependencies] 428 | oauthlib = ">=3.0.0" 429 | requests = ">=2.0.0" 430 | 431 | [package.extras] 432 | rsa = ["oauthlib[signedtoken] (>=3.0.0)"] 433 | 434 | [[package]] 435 | name = "s3transfer" 436 | version = "0.5.0" 437 | description = "An Amazon S3 Transfer Manager" 438 | category = "main" 439 | optional = false 440 | python-versions = ">= 3.6" 441 | 442 | [package.dependencies] 443 | botocore = ">=1.12.36,<2.0a.0" 444 | 445 | [package.extras] 446 | crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] 447 | 448 | [[package]] 449 | name = "sgmllib3k" 450 | version = "1.0.0" 451 | description = "Py3k port of sgmllib." 452 | category = "main" 453 | optional = false 454 | python-versions = "*" 455 | 456 | [[package]] 457 | name = "six" 458 | version = "1.16.0" 459 | description = "Python 2 and 3 compatibility utilities" 460 | category = "main" 461 | optional = false 462 | python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" 463 | 464 | [[package]] 465 | name = "social-auth-app-django" 466 | version = "5.0.0" 467 | description = "Python Social Authentication, Django integration." 468 | category = "main" 469 | optional = false 470 | python-versions = ">=3.6" 471 | 472 | [package.dependencies] 473 | social-auth-core = ">=4.1.0" 474 | 475 | [[package]] 476 | name = "social-auth-core" 477 | version = "4.1.0" 478 | description = "Python social authentication made simple." 479 | category = "main" 480 | optional = false 481 | python-versions = ">=3.6" 482 | 483 | [package.dependencies] 484 | cryptography = ">=1.4" 485 | defusedxml = ">=0.5.0rc1" 486 | oauthlib = ">=1.0.3" 487 | PyJWT = ">=2.0.0" 488 | python3-openid = ">=3.0.10" 489 | requests = ">=2.9.1" 490 | requests-oauthlib = ">=0.6.1" 491 | 492 | [package.extras] 493 | all = ["python-jose (>=3.0.0)", "python3-saml (>=1.2.1)", "cryptography (>=2.1.1)"] 494 | allpy3 = ["python-jose (>=3.0.0)", "python3-saml (>=1.2.1)", "cryptography (>=2.1.1)"] 495 | azuread = ["cryptography (>=2.1.1)"] 496 | openidconnect = ["python-jose (>=3.0.0)"] 497 | saml = ["python3-saml (>=1.2.1)"] 498 | 499 | [[package]] 500 | name = "soupsieve" 501 | version = "2.2.1" 502 | description = "A modern CSS selector implementation for Beautiful Soup." 503 | category = "main" 504 | optional = false 505 | python-versions = ">=3.6" 506 | 507 | [[package]] 508 | name = "sqlparse" 509 | version = "0.4.2" 510 | description = "A non-validating SQL parser." 511 | category = "main" 512 | optional = false 513 | python-versions = ">=3.5" 514 | 515 | [[package]] 516 | name = "text-unidecode" 517 | version = "1.3" 518 | description = "The most basic Text::Unidecode port" 519 | category = "dev" 520 | optional = false 521 | python-versions = "*" 522 | 523 | [[package]] 524 | name = "urllib3" 525 | version = "1.26.6" 526 | description = "HTTP library with thread-safe connection pooling, file post, and more." 527 | category = "main" 528 | optional = false 529 | python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" 530 | 531 | [package.extras] 532 | brotli = ["brotlipy (>=0.6.0)"] 533 | secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"] 534 | socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] 535 | 536 | [[package]] 537 | name = "zope.event" 538 | version = "4.5.0" 539 | description = "Very basic event publishing system" 540 | category = "main" 541 | optional = false 542 | python-versions = "*" 543 | 544 | [package.extras] 545 | docs = ["sphinx"] 546 | test = ["zope.testrunner"] 547 | 548 | [[package]] 549 | name = "zope.interface" 550 | version = "5.4.0" 551 | description = "Interfaces for Python" 552 | category = "main" 553 | optional = false 554 | python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" 555 | 556 | [package.extras] 557 | docs = ["sphinx", "repoze.sphinx.autointerface"] 558 | test = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] 559 | testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] 560 | 561 | [metadata] 562 | lock-version = "1.1" 563 | python-versions = "^3.9" 564 | content-hash = "0b5496f639b360618826d1c47512b8acfda927c6db5141998b719ca5307294fc" 565 | 566 | [metadata.files] 567 | asgiref = [] 568 | beautifulsoup4 = [ 569 | {file = "beautifulsoup4-4.10.0-py3-none-any.whl", hash = "sha256:9a315ce70049920ea4572a4055bc4bd700c940521d36fc858205ad4fcde149bf"}, 570 | {file = "beautifulsoup4-4.10.0.tar.gz", hash = "sha256:c23ad23c521d818955a4151a67d81580319d4bf548d3d49f4223ae041ff98891"}, 571 | ] 572 | boto3 = [] 573 | botocore = [] 574 | certifi = [] 575 | cffi = [] 576 | charset-normalizer = [] 577 | cryptography = [] 578 | defusedxml = [] 579 | dj-database-url = [] 580 | django = [] 581 | django-storages = [] 582 | factory-boy = [] 583 | faker = [] 584 | feedparser = [] 585 | flake8 = [] 586 | gevent = [] 587 | greenlet = [] 588 | gunicorn = [ 589 | {file = "gunicorn-20.1.0-py3-none-any.whl", hash = "sha256:9dcc4547dbb1cb284accfb15ab5667a0e5d1881cc443e0677b4882a4067a807e"}, 590 | {file = "gunicorn-20.1.0.tar.gz", hash = "sha256:e0a968b5ba15f8a328fdfd7ab1fcb5af4470c28aaf7e55df02a99bc13138e6e8"}, 591 | ] 592 | idna = [] 593 | jmespath = [] 594 | lxml = [] 595 | mccabe = [] 596 | mock = [] 597 | oauthlib = [ 598 | {file = "oauthlib-3.1.1-py2.py3-none-any.whl", hash = "sha256:42bf6354c2ed8c6acb54d971fce6f88193d97297e18602a3a886603f9d7730cc"}, 599 | {file = "oauthlib-3.1.1.tar.gz", hash = "sha256:8f0215fcc533dd8dd1bee6f4c412d4f0cd7297307d43ac61666389e3bc3198a3"}, 600 | ] 601 | psycopg2 = [] 602 | pycodestyle = [] 603 | pycparser = [] 604 | pyflakes = [] 605 | pyjwt = [] 606 | python-dateutil = [ 607 | {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, 608 | {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, 609 | ] 610 | python3-openid = [] 611 | pytz = [] 612 | requests = [ 613 | {file = "requests-2.26.0-py2.py3-none-any.whl", hash = "sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24"}, 614 | {file = "requests-2.26.0.tar.gz", hash = "sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7"}, 615 | ] 616 | requests-oauthlib = [ 617 | {file = "requests-oauthlib-1.3.0.tar.gz", hash = "sha256:b4261601a71fd721a8bd6d7aa1cc1d6a8a93b4a9f5e96626f8e4d91e8beeaa6a"}, 618 | {file = "requests_oauthlib-1.3.0-py2.py3-none-any.whl", hash = "sha256:7f71572defaecd16372f9006f33c2ec8c077c3cfa6f5911a9a90202beb513f3d"}, 619 | {file = "requests_oauthlib-1.3.0-py3.7.egg", hash = "sha256:fa6c47b933f01060936d87ae9327fead68768b69c6c9ea2109c48be30f2d4dbc"}, 620 | ] 621 | s3transfer = [] 622 | sgmllib3k = [] 623 | six = [ 624 | {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, 625 | {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, 626 | ] 627 | social-auth-app-django = [] 628 | social-auth-core = [] 629 | soupsieve = [] 630 | sqlparse = [] 631 | text-unidecode = [] 632 | urllib3 = [] 633 | "zope.event" = [] 634 | "zope.interface" = [] 635 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "blaggregator" 3 | version = "0.1.0" 4 | description = "A blog aggregator for the Recurse Center community" 5 | authors = ["Sasha Laundy ", "Puneeth Chaganti "] 6 | license = "AGPL v3" 7 | 8 | [tool.poetry.dependencies] 9 | python = "^3.9" 10 | Django = "^3.2.7" 11 | requests = "^2.26.0" 12 | feedparser = "^6.0.8" 13 | psycopg2 = "^2.9.1" 14 | django-storages = "^1.11.1" 15 | social-auth-app-django = "^5.0.0" 16 | beautifulsoup4 = "^4.10.0" 17 | gevent = "^21.8.0" 18 | dj-database-url = "^0.5.0" 19 | boto3 = "^1.18.44" 20 | gunicorn = "^20.1.0" 21 | lxml = "^4.6.3" 22 | 23 | [tool.poetry.dev-dependencies] 24 | mock = "^4.0.3" 25 | factory-boy = "^3.2.0" 26 | flake8 = "^3.9.2" 27 | 28 | [build-system] 29 | requires = ["poetry-core>=1.0.0"] 30 | build-backend = "poetry.core.masonry.api" 31 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | asgiref==3.4.1; python_version >= "3.6" 2 | beautifulsoup4==4.10.0; python_full_version > "3.0.0" 3 | boto3==1.18.44; python_version >= "3.6" 4 | botocore==1.21.44; python_version >= "3.6" 5 | certifi==2021.5.30; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" 6 | cffi==1.14.6; platform_python_implementation == "CPython" and sys_platform == "win32" and (python_version >= "2.7" and python_full_version < "3.0.0" or python_version > "3.5") and python_version >= "3.6" 7 | charset-normalizer==2.0.6; python_full_version >= "3.6.0" and python_version >= "3.6" 8 | cryptography==3.4.8; python_version >= "3.6" 9 | defusedxml==0.7.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" 10 | dj-database-url==0.5.0 11 | django-storages==1.11.1; python_version >= "3.5" 12 | django==3.2.7; python_version >= "3.6" 13 | factory-boy==3.2.0; python_version >= "3.6" 14 | faker==8.13.2; python_version >= "3.6" 15 | feedparser==6.0.8; python_version >= "3.6" 16 | flake8==3.9.2; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0") 17 | gevent==21.8.0; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_version > "3.5") 18 | greenlet==1.1.1; python_version >= "2.7" and python_full_version < "3.0.0" and platform_python_implementation == "CPython" or python_version > "3.5" and python_full_version < "3.0.0" and platform_python_implementation == "CPython" or python_version > "3.5" and platform_python_implementation == "CPython" and python_full_version >= "3.5.0" 19 | gunicorn==20.1.0; python_version >= "3.5" 20 | idna==3.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" 21 | jmespath==0.10.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" 22 | lxml==4.9.2; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0") 23 | mccabe==0.6.1; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" 24 | mock==4.0.3; python_version >= "3.6" 25 | oauthlib==3.1.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" 26 | psycopg2==2.9.1; python_version >= "3.6" 27 | pycodestyle==2.7.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" 28 | pycparser==2.20; python_version >= "2.7" and python_full_version < "3.0.0" and platform_python_implementation == "CPython" and sys_platform == "win32" and (python_version >= "2.7" and python_full_version < "3.0.0" or python_version > "3.5") or platform_python_implementation == "CPython" and sys_platform == "win32" and (python_version >= "2.7" and python_full_version < "3.0.0" or python_version > "3.5") and python_full_version >= "3.4.0" 29 | pyflakes==2.3.1; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" 30 | pyjwt==2.1.0; python_version >= "3.6" 31 | python-dateutil==2.8.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" 32 | python3-openid==3.2.0; python_version >= "3.6" 33 | pytz==2021.1; python_version >= "3.6" 34 | requests-oauthlib==1.3.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" 35 | requests==2.26.0; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.6.0") 36 | s3transfer==0.5.0; python_version >= "3.6" 37 | sgmllib3k==1.0.0; python_version >= "3.6" 38 | six==1.16.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" 39 | social-auth-app-django==5.0.0; python_version >= "3.6" 40 | social-auth-core==4.1.0; python_version >= "3.6" 41 | soupsieve==2.2.1; python_version >= "3.6" and python_full_version > "3.0.0" 42 | sqlparse==0.4.2; python_version >= "3.6" 43 | text-unidecode==1.3; python_version >= "3.6" 44 | urllib3==1.26.6; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.6" 45 | zope.event==4.5.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_version > "3.5" 46 | zope.interface==5.4.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_version > "3.5" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version > "3.5" 47 | -------------------------------------------------------------------------------- /runtime.txt: -------------------------------------------------------------------------------- 1 | python-3.9.16 2 | -------------------------------------------------------------------------------- /scripts/delete_messages: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """Script to delete spammy messages sent by the blaggregator bot. 4 | 5 | We update the messages and set the topic and content of the messages to empty. 6 | See https://zulipchat.com/api/update-message for the API documentation. 7 | 8 | """ 9 | 10 | import os 11 | 12 | import requests 13 | 14 | # NOTE: For each run, manually populate this list 15 | MESSAGE_IDS = [] 16 | EMAIL = os.environ["ZULIP_EMAIL"] 17 | KEY = os.environ["ZULIP_KEY"] 18 | 19 | session = requests.Session() 20 | auth = (EMAIL, KEY) 21 | 22 | 23 | for message_id in MESSAGE_IDS: 24 | url = f"https://recurse.zulipchat.com/api/v1/messages/{message_id}" 25 | data = {"topic": "(deleted)", "content": ""} 26 | response = session.patch(url, data=data, auth=auth) 27 | print(response.status_code) 28 | print(response.json()) 29 | -------------------------------------------------------------------------------- /scripts/readme_to_about: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ROOT=$( dirname "${BASH_SOURCE[0]}" )/.. 3 | markdown_py $ROOT/README.md -f $ROOT/home/templates/home/readme.html -e utf-8 4 | -------------------------------------------------------------------------------- /web-variables.env: -------------------------------------------------------------------------------- 1 | HS_PERSONAL_TOKEN= 2 | SOCIAL_AUTH_HS_KEY= 3 | SOCIAL_AUTH_HS_SECRET= 4 | DJANGO_DEBUG=1 5 | DOCKER_ENV=1 6 | --------------------------------------------------------------------------------