├── .gitignore ├── .tool-versions ├── Procfile ├── .hound.yml ├── bin ├── setup └── vet ├── .circleci └── config.yml ├── vendor └── vendor.json ├── LICENSE ├── rss_test.go ├── README.md ├── CONTRIBUTING.md ├── integration_test.go └── main.go /.gitignore: -------------------------------------------------------------------------------- 1 | vendor/*/ 2 | -------------------------------------------------------------------------------- /.tool-versions: -------------------------------------------------------------------------------- 1 | golang 1.12.6 2 | -------------------------------------------------------------------------------- /Procfile: -------------------------------------------------------------------------------- 1 | web: rss -port=$PORT 2 | -------------------------------------------------------------------------------- /.hound.yml: -------------------------------------------------------------------------------- 1 | go: 2 | enabled: true 3 | -------------------------------------------------------------------------------- /bin/setup: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | set -e 4 | 5 | # Install and initialize govendor 6 | go get -u github.com/kardianos/govendor 7 | govendor init 8 | 9 | # Update Go dependencies 10 | govendor sync 11 | 12 | # Only if this isn't CI 13 | if [ -z "$CI" ]; then 14 | # Set up deploys 15 | if ! command -v heroku > /dev/null; then 16 | printf 'Heroku Toolbelt is not installed.\n' 17 | printf 'See https://toolbelt.heroku.com/ for install instructions.\n' 18 | exit 1 19 | fi 20 | 21 | heroku git:remote -r production -a thoughtbot-rss 22 | fi 23 | -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2.0 2 | jobs: 3 | build: 4 | working_directory: /go/src/github.com/thoughtbot/rss 5 | 6 | docker: 7 | - image: circleci/golang:1.12.6 8 | 9 | steps: 10 | - checkout 11 | 12 | - restore_cache: 13 | keys: 14 | - v1-thoughtbot-rss-cache 15 | 16 | - run: 17 | name: Go get 18 | command: 'go get' 19 | 20 | - run: 21 | name: Go test 22 | command: 'go test -v' 23 | 24 | - save_cache: 25 | key: v1-thoughtbot-rss-cache 26 | paths: 27 | - "/go/pkg" 28 | -------------------------------------------------------------------------------- /vendor/vendor.json: -------------------------------------------------------------------------------- 1 | { 2 | "comment": "", 3 | "ignore": "test", 4 | "package": [ 5 | { 6 | "checksumSHA1": "eghrgRs1Yvx1kWHj1vqqydnVmiU=", 7 | "path": "github.com/gorilla/feeds", 8 | "revision": "fa8f5548eedbe306d748925237a48d70ae4150de", 9 | "revisionTime": "2017-06-11T03:22:06Z" 10 | }, 11 | { 12 | "checksumSHA1": "M9g4Eqb2f5R56JI8lraVHWcbRfg=", 13 | "path": "github.com/mattn/go-pkg-rss", 14 | "revision": "cb3c88c0b82e6cb1691c2620beda2a44f43d4217", 15 | "revisionTime": "2018-02-08T00:54:46Z" 16 | }, 17 | { 18 | "checksumSHA1": "AGOlEDLFdG0YNEE+3aWu3DPEYz8=", 19 | "path": "github.com/mattn/go-pkg-xmlx", 20 | "revision": "db53493302d1dfcc6b9f16016e3c43cedaf5213d", 21 | "revisionTime": "2018-02-08T00:52:05Z" 22 | } 23 | ], 24 | "rootPath": "github.com/thoughtbot/rss", 25 | "heroku": { 26 | "goVersion": "go1.12.6" 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /bin/vet: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # validates that the project is clean of formatting and vet errors. 4 | # symlink as .git/hooks/pre-commit to use as a pre-commit check. 5 | # 6 | 7 | if [[ -n "${GIT_INDEX_FILE}" ]]; then 8 | gofiles=$(git diff --cached --name-only --diff-filter=ACM | grep '.go$') 9 | else 10 | gofiles=$(find . ! -path "*/_*" -name "*.go") 11 | fi 12 | 13 | [ -z "$gofiles" ] && exit 0 14 | 15 | function checkfmt() { 16 | unformatted=$(gofmt -l $*) 17 | [ -z "$unformatted" ] && return 0 18 | 19 | echo >&2 "Go files must be formatted with gofmt. Please run:" 20 | for fn in $unformatted; do 21 | echo >&2 " gofmt -w $PWD/$fn" 22 | done 23 | 24 | return 1 25 | } 26 | 27 | function checkvet() { 28 | unvetted=$(go vet ./... 2>&1 | grep -v "exit status") 29 | [ -z "$unvetted" ] && return 0 30 | 31 | echo >&2 "Go files must be vetted. Check these problems:" 32 | IFS=$'\n' 33 | for line in $unvetted; do 34 | echo >&2 " $line" 35 | done 36 | unset IFS 37 | 38 | return 1 39 | } 40 | 41 | checkfmt $gofiles || fail=yes 42 | checkvet $gofiles || fail=yes 43 | 44 | [ -z "$fail" ] || exit 1 45 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015-2016 Dan Croak and thoughtbot, inc. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /rss_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "testing" 5 | 6 | rss "github.com/mattn/go-pkg-rss" 7 | ) 8 | 9 | func TestStripPodcastEpisodePrefix(t *testing.T) { 10 | for _, tt := range []struct{ in, want string }{ 11 | {"Better Commit Messages", "Better Commit Messages"}, 12 | {"10: Minisode 0.1.1", "Minisode 0.1.1"}, 13 | {"", ""}, 14 | } { 15 | got := stripPodcastEpisodePrefix(tt.in) 16 | 17 | if got != tt.want { 18 | t.Errorf("stripPodcastEpisodePrefix(%q) = %q; want %q", tt.in, got, tt.want) 19 | } 20 | } 21 | } 22 | 23 | func TestGetDescription(t *testing.T) { 24 | want := "from description" 25 | item := &rss.Item{ 26 | Description: want, 27 | } 28 | got := getDescription(item) 29 | if got != want { 30 | t.Errorf("getDescription(%v) = %q; want %q", item, got, want) 31 | } 32 | 33 | want = "from itunes" 34 | item = &rss.Item{Extensions: map[string]map[string][]rss.Extension{ 35 | "http://www.itunes.com/dtds/podcast-1.0.dtd": map[string][]rss.Extension{ 36 | "subtitle": []rss.Extension{ 37 | { 38 | Value: want, 39 | }, 40 | }, 41 | }, 42 | }, 43 | } 44 | got = getDescription(item) 45 | if got != want { 46 | t.Errorf("getDescription(%v) = %q; want %q", item, got, want) 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | > [!IMPORTANT] 2 | > This application has been replaced with a Zapier Zap. 3 | 4 | # RSS 5 | 6 | > All the thoughts fit to bot. 7 | 8 | An RSS feed at [rss.thoughtbot.com] which combines our blog, podcast, and open 9 | source software release RSS feeds into one feed for the past week's content. 10 | 11 | Used as the data source for our weekly newsletter. 12 | 13 | ## Contributing 14 | 15 | See the [CONTRIBUTING] document. Thank you, [contributors]! 16 | 17 | ## License 18 | 19 | RSS is Copyright (c) 2015-2019 thoughtbot, inc. It is free software, and may be 20 | redistributed under the terms specified in the [LICENSE] file. 21 | 22 | ## About 23 | 24 | [![thoughtbot](http://presskit.thoughtbot.com/images/thoughtbot-logo-for-readmes.svg)][go] 25 | 26 | RSS is maintained and funded by thoughtbot, inc. The names and logos for 27 | thoughtbot are trademarks of thoughtbot, inc. 28 | 29 | We love open source software! See [our other Go projects][go] or [hire us][hire] 30 | to help build your product. 31 | 32 | [contributing]: CONTRIBUTING.md 33 | [contributors]: https://github.com/thoughtbot/rss/graphs/contributors 34 | [go]: https://thoughtbot.com/services/go?utm_source=github 35 | [hire]: https://thoughtbot.com/hire-us?utm_source=github 36 | [license]: /LICENSE 37 | [rss.thoughtbot.com]: https://rss.thoughtbot.com 38 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Contributing 2 | ============ 3 | 4 | We love pull requests from everyone. 5 | By participating in this project, 6 | you agree to abide by the thoughtbot [code of conduct]. 7 | 8 | [code of conduct]: https://thoughtbot.com/open-source-code-of-conduct 9 | 10 | We expect everyone to follow the code of conduct 11 | anywhere in thoughtbot's project codebases, 12 | issue trackers, chatrooms, and mailing lists. 13 | 14 | Fork the repo. 15 | 16 | Get a working [Go installation], 17 | and clone the project into your [Go work environment] 18 | (that is, `$GOPATH/src/github.com/thoughtbot/rss`). 19 | 20 | [Go installation]: http://golang.org/doc/install 21 | [Go work environment]: http://golang.org/doc/code.html 22 | 23 | Run `./bin/setup` to install the project's dependencies. 24 | 25 | If you add or update a dependency, 26 | run `./bin/setup` again to vendor the changes. 27 | 28 | To test the `rss` package, run `govendor test +local`. 29 | 30 | Make your change, with new passing tests. 31 | 32 | Run `go run main.go` to see the change at `localhost:8080` in a web browser. 33 | 34 | Push to your fork. Write a [good commit message][commit]. Submit a pull request. 35 | 36 | [commit]: http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html 37 | 38 | Others will give constructive feedback. 39 | This is a time for discussion and improvements, 40 | and making the necessary changes will be required before we can 41 | merge the contribution. 42 | 43 | The master branch on GitHub is automatically deployed 44 | to the `thoughtbot-rss` app on Heroku 45 | after the CI build passes. 46 | -------------------------------------------------------------------------------- /integration_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | "net/http/httptest" 7 | "testing" 8 | "time" 9 | 10 | "github.com/gorilla/feeds" 11 | rss "github.com/mattn/go-pkg-rss" 12 | ) 13 | 14 | func TestRSSHandler(t *testing.T) { 15 | var ( 16 | recentPost = newFeedItem("Recent Post", 0) 17 | olderPost = newFeedItem("Older Post", 3) 18 | oldPost = newFeedItem("Old Post", 30) 19 | 20 | blogFeed = newFeed("Blog", recentPost, olderPost, oldPost) 21 | 22 | // podcast titles are prefixed with an episode number 23 | podcastTitle = "We Record Things" 24 | podcastEpisodeTitle = "12: " + podcastTitle 25 | recentPodcast = newFeedItem(podcastEpisodeTitle, 1) 26 | 27 | podcastFeed = newFeed("Podcast", recentPodcast) 28 | ) 29 | 30 | feedServer := newFeedServer(map[string]*feeds.Feed{ 31 | "/blog": blogFeed, 32 | "/podcast": podcastFeed, 33 | }) 34 | defer feedServer.Close() 35 | 36 | server := httptest.NewServer(rssHandler([]sourceFeed{ 37 | {uri: feedServer.URL + "/blog", name: "Blog"}, 38 | {uri: feedServer.URL + "/podcast", name: "Podcast"}, 39 | })) 40 | defer server.Close() 41 | 42 | feed := rss.NewWithHandlers(0, false, nil, nil) 43 | if err := feed.Fetch(server.URL, nil); err != nil { 44 | t.Fatalf("failed to fetch feed: %s", err) 45 | } 46 | 47 | if got, want := len(feed.Channels), 1; got != want { 48 | t.Fatalf("len(feed.Channels) = %d, want %d", got, want) 49 | } 50 | 51 | channel := feed.Channels[0] 52 | 53 | if got, want := len(channel.Items), 3; got != want { 54 | t.Fatalf("len(channel.Items) = %d, want %d", got, want) 55 | } 56 | 57 | if got, want := channel.Title, "thoughtbot"; got != want { 58 | t.Errorf("channel.Title = %q, want %q", got, want) 59 | } 60 | 61 | if got, want := channel.Items[0].Title, recentPost.Title; got != want { 62 | t.Errorf("channel.Items[0].Title = %q, want %q", got, want) 63 | } 64 | 65 | if got, want := channel.Items[1].Title, podcastTitle; got != want { 66 | t.Errorf("channel.Items[1].Title = %q, want %q", got, want) 67 | } 68 | 69 | if got, want := channel.Items[2].Title, olderPost.Title; got != want { 70 | t.Errorf("channel.Items[2].Title = %q, want %q", got, want) 71 | } 72 | } 73 | 74 | func newFeed(title string, items ...*feeds.Item) *feeds.Feed { 75 | return &feeds.Feed{ 76 | Title: title, 77 | Link: &feeds.Link{Href: "http://example.com/feed"}, 78 | Items: items, 79 | } 80 | } 81 | 82 | func newFeedItem(title string, ageInDays int) *feeds.Item { 83 | return &feeds.Item{ 84 | Title: title, 85 | Link: &feeds.Link{Href: "http://example.com"}, 86 | Description: title, 87 | Updated: time.Now().AddDate(0, 0, -ageInDays), 88 | } 89 | } 90 | 91 | func newFeedServer(routes map[string]*feeds.Feed) *httptest.Server { 92 | return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 93 | feed, ok := routes[r.URL.Path] 94 | if !ok { 95 | w.WriteHeader(http.StatusNotFound) 96 | return 97 | } 98 | 99 | err := feed.WriteRss(w) 100 | if err != nil { 101 | log.Fatal(err) 102 | } 103 | })) 104 | } 105 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "log" 7 | "net/http" 8 | "regexp" 9 | "sort" 10 | "time" 11 | 12 | "github.com/gorilla/feeds" 13 | rss "github.com/mattn/go-pkg-rss" 14 | ) 15 | 16 | var podcastEpisodePrefix = regexp.MustCompile(`^\d+: `) 17 | 18 | var sourceFeeds = []sourceFeed{ 19 | {uri: "https://robots.thoughtbot.com/summaries.xml", name: "Giant Robots blog"}, 20 | {uri: "https://feeds.simplecast.com/KARThxOK", name: "Giant Robots podcast"}, 21 | {uri: "https://feeds.simplecast.com/ky3kewHN", name: "The Bike Shed podcast"}, 22 | {uri: "https://feeds.simplecast.com/ZBfsoMJW", name: "Tentative podcast"}, 23 | {uri: "https://thoughtbot.com/upcase/the-weekly-iteration.rss", name: "The Weekly Iteration videos"}, 24 | {uri: "https://hub.thoughtbot.com/releases.atom", name: "Open source software releases"}, 25 | } 26 | 27 | func main() { 28 | port := flag.String("port", "8080", "HTTP Port to listen on") 29 | flag.Parse() 30 | http.Handle("/", rssHandler(sourceFeeds)) 31 | log.Fatal(http.ListenAndServe(":"+*port, nil)) 32 | } 33 | 34 | func rssHandler(sourceFeeds []sourceFeed) http.Handler { 35 | return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { 36 | if req.Header.Get("X-Forwarded-Proto") == "http" { 37 | destination := *req.URL 38 | destination.Host = req.Host 39 | destination.Scheme = "https" 40 | http.Redirect(w, req, destination.String(), http.StatusFound) 41 | return 42 | } 43 | 44 | master := &feeds.Feed{ 45 | Title: "thoughtbot", 46 | Link: &feeds.Link{Href: "https://rss.thoughtbot.com"}, 47 | Description: "All the thoughts fit to bot.", 48 | Author: &feeds.Author{Name: "thoughtbot", Email: "hello@thoughtbot.com"}, 49 | Created: time.Now(), 50 | } 51 | 52 | for _, feed := range sourceFeeds { 53 | fetch(feed, master) 54 | } 55 | 56 | sort.Sort(byCreated(master.Items)) 57 | 58 | result, err := master.ToAtom() 59 | if err != nil { 60 | log.Printf("error generating feed: %v", err) 61 | http.Error(w, "error generating feed", http.StatusInternalServerError) 62 | return 63 | } 64 | 65 | _, err = fmt.Fprintln(w, result) 66 | if err != nil { 67 | log.Printf("error printing feed: %v", err) 68 | http.Error(w, "error printing feed", http.StatusInternalServerError) 69 | return 70 | } 71 | }) 72 | } 73 | 74 | func fetch(feed sourceFeed, master *feeds.Feed) { 75 | fetcher := rss.New(5, true, chanHandler, makeHandler(master, feed.name)) 76 | client := &http.Client{ 77 | Timeout: time.Second, 78 | } 79 | 80 | err := fetcher.FetchClient(feed.uri, client, nil) 81 | if err != nil { 82 | log.Printf("error fetching feed: %v", err) 83 | } 84 | } 85 | 86 | func chanHandler(feed *rss.Feed, newchannels []*rss.Channel) { 87 | // no need to do anything... 88 | } 89 | 90 | func makeHandler(master *feeds.Feed, sourceName string) rss.ItemHandlerFunc { 91 | return func(feed *rss.Feed, ch *rss.Channel, items []*rss.Item) { 92 | for i := 0; i < len(items); i++ { 93 | published, err := items[i].ParsedPubDate() 94 | if err != nil { 95 | log.Printf("error parsing publication date: %v", err) 96 | continue 97 | } 98 | 99 | weekAgo := time.Now().AddDate(0, 0, -7) 100 | 101 | if published.After(weekAgo) { 102 | item := &feeds.Item{ 103 | Title: stripPodcastEpisodePrefix(items[i].Title), 104 | Link: &feeds.Link{Href: items[i].Links[0].Href}, 105 | Description: getDescription(items[i]), 106 | Author: &feeds.Author{Name: sourceName}, 107 | Created: published, 108 | } 109 | master.Add(item) 110 | } 111 | } 112 | } 113 | } 114 | 115 | type byCreated []*feeds.Item 116 | 117 | func (s byCreated) Len() int { 118 | return len(s) 119 | } 120 | 121 | func (s byCreated) Swap(i, j int) { 122 | s[i], s[j] = s[j], s[i] 123 | } 124 | 125 | func (s byCreated) Less(i, j int) bool { 126 | return s[j].Created.Before(s[i].Created) 127 | } 128 | 129 | func stripPodcastEpisodePrefix(s string) string { 130 | return podcastEpisodePrefix.ReplaceAllString(s, "") 131 | } 132 | 133 | func getDescription(item *rss.Item) string { 134 | if ext, ok := item.Extensions["http://www.itunes.com/dtds/podcast-1.0.dtd"]; ok { 135 | return ext["subtitle"][0].Value 136 | } 137 | 138 | return item.Description 139 | } 140 | 141 | type sourceFeed struct { 142 | uri string 143 | name string 144 | } 145 | --------------------------------------------------------------------------------