├── assets ├── rss_feeds.png └── rss_items.png ├── main.go ├── .gitignore ├── .github └── workflows │ ├── cron.yml │ ├── release.yml │ └── testing.yml ├── go.mod ├── utils.go ├── tasks.go ├── utils_test.go ├── README.md ├── notion_dao_integ_test.go ├── rss.go ├── go.sum ├── notion_dao_test.go └── notion_dao.go /assets/rss_feeds.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Jeadie/notion-rss/HEAD/assets/rss_feeds.png -------------------------------------------------------------------------------- /assets/rss_items.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Jeadie/notion-rss/HEAD/assets/rss_items.png -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | func main() { 8 | 9 | nDao, err := ConstructNotionDaoFromEnv() 10 | if err != nil { 11 | panic(fmt.Errorf("configuration error: %w", err)) 12 | } 13 | 14 | tasks := GetAllTasks() 15 | errs := make([]error, len(tasks)) 16 | for i, t := range tasks { 17 | errs[i] = t.Run(nDao) 18 | } 19 | 20 | PanicOnErrors(errs) 21 | } 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Dependency directories (remove the comment below to include it) 15 | # vendor/ 16 | 17 | # Local environment variables 18 | .env.local 19 | .env.integ 20 | 21 | # IDE 22 | .idea/ 23 | -------------------------------------------------------------------------------- /.github/workflows/cron.yml: -------------------------------------------------------------------------------- 1 | name: Get Feed 2 | 3 | on: 4 | workflow_dispatch: 5 | schedule: 6 | - cron: "0 14 * * *" # 12am AEST 7 | 8 | jobs: 9 | get-rss-feed: 10 | runs-on: ubuntu-latest 11 | env: 12 | NOTION_RSS_KEY: ${{ secrets.NOTION_API_TOKEN }} 13 | NOTION_RSS_CONTENT_DATABASE_ID: ${{ secrets.NOTION_RSS_CONTENT_DATABASE_ID }} 14 | NOTION_RSS_FEEDS_DATABASE_ID: ${{ secrets.NOTION_RSS_FEEDS_DATABASE_ID }} 15 | steps: 16 | - name: checkout 17 | uses: actions/checkout@v3 18 | with: 19 | fetch-depth: 0 20 | - name: run cron job 21 | run: go run . 22 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/Jeadie/notion-rss 2 | 3 | go 1.18 4 | 5 | require ( 6 | github.com/jomei/notionapi v1.8.5 7 | github.com/mmcdole/gofeed v1.1.3 8 | ) 9 | 10 | require ( 11 | github.com/PuerkitoBio/goquery v1.5.1 // indirect 12 | github.com/andybalholm/cascadia v1.1.0 // indirect 13 | github.com/json-iterator/go v1.1.10 // indirect 14 | github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf // indirect 15 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect 16 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 // indirect 17 | github.com/pkg/errors v0.9.1 // indirect 18 | golang.org/x/net v0.0.0-20200301022130-244492dfa37a // indirect 19 | golang.org/x/text v0.3.2 // indirect 20 | ) 21 | -------------------------------------------------------------------------------- /utils.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "fmt" 4 | 5 | // PanicOnErrors prints all non-nil err in errors and panics if there is at least one non-nil 6 | // error in errors. Otherwise, return normally. 7 | func PanicOnErrors(errors []error) { 8 | // Only used if one error (for better error handling). 9 | var firstErr error 10 | errN := 0 11 | 12 | // Print all non-nil errors. 13 | for _, err := range errors { 14 | if err != nil { 15 | fmt.Printf("%s\n", err.Error()) 16 | errN++ 17 | firstErr = err 18 | } 19 | } 20 | 21 | // Multiple errors, panic with generic message. 22 | if errN > 1 { 23 | panic(fmt.Errorf("Multiple errors occured. Check output for details")) 24 | } 25 | 26 | if errN == 1 { 27 | panic(firstErr) 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | on: 2 | release: 3 | types: [created] 4 | 5 | jobs: 6 | test-code: 7 | name: checkout 8 | runs-on: ubuntu-latest 9 | steps: 10 | - name: checkout 11 | uses: actions/checkout@v2 12 | with: 13 | fetch-depth: 0 14 | - name: check-go-fmt 15 | run: test -z `go fmt ./...` 16 | - name: test 17 | run: go test ./... 18 | 19 | release-binaries: 20 | name: Release Go Binary 21 | runs-on: ubuntu-latest 22 | needs: test-code 23 | strategy: 24 | matrix: 25 | goos: [linux] 26 | goarch: [arm64, amd64] 27 | steps: 28 | - name: checkout 29 | uses: actions/checkout@v3 30 | with: 31 | fetch-depth: 0 32 | - uses: wangyoucao577/go-release-action@v1.25 33 | with: 34 | build_command: make build 35 | goos: ${{ matrix.goos }} 36 | goarch: ${{ matrix.goarch }} 37 | github_token: ${{ secrets.GITHUB_TOKEN }} 38 | -------------------------------------------------------------------------------- /.github/workflows/testing.yml: -------------------------------------------------------------------------------- 1 | on: [push, pull_request] 2 | 3 | name: Testing 4 | 5 | env: 6 | NOTION_RSS_KEY: ${{ secrets.NOTION_API_TOKEN }} 7 | 8 | # Required: Manually share database with the notion integration with api key `NOTION_RSS_KEY`. 9 | NOTION_RSS_CONTENT_DATABASE_ID: "7ce7588a89734b8081cbb4869ba36460" 10 | NOTION_RSS_FEEDS_DATABASE_ID: "44d719ce002943deada2cf7d91d32274" 11 | 12 | jobs: 13 | test-code: 14 | runs-on: ubuntu-latest 15 | steps: 16 | - name: checkout 17 | uses: actions/checkout@v3 18 | with: 19 | fetch-depth: 0 20 | - name: check-go-fmt 21 | run: test -z `go fmt ./...` 22 | - name: test 23 | run: go test ./... 24 | 25 | integration-tests: 26 | name: checkout 27 | runs-on: ubuntu-latest 28 | steps: 29 | - name: checkout 30 | uses: actions/checkout@v2 31 | with: 32 | fetch-depth: 0 33 | - name: integration-tests 34 | run: echo "Write some integration tests" 35 | -------------------------------------------------------------------------------- /tasks.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | ) 7 | 8 | // NotionTask represents an independent unit of work to perform in Notion.so 9 | type NotionTask struct { 10 | Run func(*NotionDao) error 11 | } 12 | 13 | // GetAllTasks that should be run. 14 | func GetAllTasks() []NotionTask { 15 | return []NotionTask{ 16 | {Run: ArchiveOldUnstarredContent}, 17 | {Run: AddNewContent}, 18 | } 19 | } 20 | 21 | // ArchiveOldUnstarredContent from the content database that is older than 30 days and is not starred. 22 | func ArchiveOldUnstarredContent(nDao *NotionDao) error { 23 | pageIds := nDao.GetOldUnstarredRSSItemIds(time.Now().Add(-30 * time.Hour * time.Duration(24))) 24 | return nDao.ArchivePages(pageIds) 25 | } 26 | 27 | // AddNewContent from all enabled RSS Feeds that have been published within the last 24 hours. 28 | func AddNewContent(nDao *NotionDao) error { 29 | rssFeeds := nDao.GetEnabledRssFeeds() 30 | last24Hours := time.Now().Add(-1 * time.Hour * time.Duration(24)) 31 | rssContent := GetRssContent(rssFeeds, last24Hours) 32 | 33 | failedCount := 0 34 | for item := range rssContent { 35 | err := nDao.AddRssItem(item) 36 | if err != nil { 37 | fmt.Printf("Could not create page for %s, URL: %s. Error: %s\n", item.title, item.link.String(), err.Error()) 38 | failedCount++ 39 | } 40 | } 41 | 42 | // Fail after all RSS items are processed to minimise impact. 43 | if failedCount > 0 { 44 | return fmt.Errorf("%d Rss item/s failed to be created in the notion database. See errors above", failedCount) 45 | } 46 | return nil 47 | } 48 | -------------------------------------------------------------------------------- /utils_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | // ShouldPanic 9 | func ShouldPanic(t *testing.T, f func(), expectedErrorMessage string, errMsg string) { 10 | t.Helper() 11 | defer func() { 12 | err := recover() 13 | if err.(error).Error() != expectedErrorMessage { 14 | t.Errorf(errMsg) 15 | } 16 | }() 17 | f() 18 | t.Errorf(errMsg) 19 | } 20 | 21 | // ShouldNotPanic 22 | func ShouldNotPanic(t *testing.T, f func(), errMsg string) { 23 | t.Helper() 24 | defer func() { 25 | err := recover() 26 | if err != nil { 27 | t.Errorf(errMsg) 28 | } 29 | }() 30 | f() 31 | } 32 | 33 | func TestPanicOnErrors(t *testing.T) { 34 | type TestCase struct { 35 | Errors []error 36 | IsPanicExpected bool 37 | PanicErrMsg string 38 | FailMsg string 39 | } 40 | tests := []TestCase{ 41 | { 42 | Errors: []error{}, 43 | IsPanicExpected: false, 44 | FailMsg: "should not panic when no errors provided", 45 | }, 46 | { 47 | Errors: []error{fmt.Errorf("single Error")}, 48 | IsPanicExpected: true, 49 | PanicErrMsg: "single Error", 50 | FailMsg: "For a single error, should panic with specific message", 51 | }, 52 | { 53 | Errors: []error{nil, fmt.Errorf("single Error")}, 54 | IsPanicExpected: true, 55 | PanicErrMsg: "single Error", 56 | FailMsg: "should panic if input contains nil errors", 57 | }, 58 | { 59 | Errors: []error{fmt.Errorf("single Error"), fmt.Errorf("one more")}, 60 | IsPanicExpected: true, 61 | PanicErrMsg: "Multiple errors occured. Check output for details", 62 | FailMsg: "For multiple errors, should provide generic message", 63 | }, 64 | } 65 | 66 | fmt.Println("The following output may contain error messages. These are expected") 67 | for _, test := range tests { 68 | if test.IsPanicExpected { 69 | ShouldPanic(t, func() { PanicOnErrors(test.Errors) }, test.PanicErrMsg, test.FailMsg) 70 | } else { 71 | ShouldNotPanic(t, func() { PanicOnErrors(test.Errors) }, test.FailMsg) 72 | } 73 | } 74 | // Separate out expected error messages from rest of testing output 75 | fmt.Printf("End of expected errors\n\n") 76 | } 77 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # notion-rss 2 | Get RSS feeds in notion.so daily 3 | 4 | ## Overview 5 | notion-rss lets you manage and retrieve all your RSS feeds in Notion.so. 6 | 7 | One can add rss feeds, and every day, a cron Github Action will trawl your RSS feeds for new content. 8 | ![rss-feeds](assets/rss_feeds.png) 9 | 10 | Then you can find them all in Notion.so 11 | ![rss-items](assets/rss_items.png) 12 | 13 | ## Usage 14 | 1. Fork the repo (it sets up Github Actions, etc) 15 | 2. Create two databases in Notion with the properties defined below. 16 | 3. Create a notion connection and get an API token. 17 | 4. Add Github action secrets defined [below](#github-action-secrets) 18 | 5. Populate your Feeds Database in Notion 19 | 6. Wait until tomorrow for new RSS content (or manually run the Github action .github/workflow/cron.yml) 20 | 21 | ## Database Interface 22 | This project uses two notion databases: to store RSS links (to subscribe to), to store the RSS content. 23 | 24 | ### Feeds Database 25 | 26 | | Property Name | Property Type | 27 | | --- | :-- | 28 | | Title | title | 29 | | Link | url | 30 | | Enabled | checkbox | 31 | 32 | ### Content Database 33 | 34 | | Property Name | Property Type | 35 | | --- | :-- | 36 | | Title | title | 37 | | Link | Url | 38 | | Description | Text | 39 | | Enabled | Checkbox | 40 | | From | Select | 41 | | Categories | MultiSelect | 42 | | Published | Date | 43 | | Starred | Checkbox | 44 | | Created | Date | 45 | 46 | ## Github Action Secrets 47 | Github Secrets needed in the repository for the workflow actions to work: 48 | - `NOTION_API_TOKEN`: notion.so api token for a specific integration. Integration must have access to `NOTION_RSS_CONTENT_DATABASE_ID` and `NOTION_RSS_FEEDS_DATABASE_ID`. 49 | - `NOTION_RSS_CONTENT_DATABASE_ID`: notion.so database id that stores RSS content (see Database Interface / Content Database). 50 | - `NOTION_RSS_FEEDS_DATABASE_ID`: notion.so database id that stores RSS feed details (see Database Interface / Feeds Database). 51 | 52 | ## Nice to haves 53 | 1. Use rss.Item.Content into notion blocks so the content can be viewed in Notion. 54 | 2. Convert combined RSS feeds into single feed: https://github.com/gorilla/feeds 55 | 56 | ### Improve Code Quality 57 | 1. Use release binary in `.github/workflows/release.yml`. 58 | 2. Write unit tests 59 | 3. Write integration tests 60 | 4. Finish Github action to run unit and integration tests. 61 | 5. Add in Precommit 62 | 6. Add in badging on README. 63 | 7. Replicate Github actions with local scripts -------------------------------------------------------------------------------- /notion_dao_integ_test.go: -------------------------------------------------------------------------------- 1 | //go:build integration 2 | // +build integration 3 | 4 | package main 5 | 6 | import ( 7 | "context" 8 | "fmt" 9 | "github.com/jomei/notionapi" 10 | "net/url" 11 | "testing" 12 | "time" 13 | ) 14 | 15 | func TestGetOldUnstarredRSSItems(t *testing.T) { 16 | nDao, err := ConstructNotionDaoFromEnv() 17 | if err != nil { 18 | t.Fatalf(err.Error()) 19 | } 20 | 21 | itemUrl, _ := url.Parse("https://github.com/Jeadie/notion-rss") 22 | published := time.Now().Add(-1 * time.Hour) 23 | 24 | t.Run("Should use created time, not published time", func(t *testing.T) { 25 | 26 | // Setup 27 | err := nDao.AddRssItem(RssItem{ 28 | title: "notion-rss integration test", 29 | link: *itemUrl, 30 | content: []string{}, 31 | categories: []string{"Integration testing", "notion", "rss"}, 32 | feedName: "Integration Test", 33 | published: &published, 34 | }) 35 | if err != nil { 36 | t.Fatalf(err.Error()) 37 | } 38 | 39 | pages := nDao.GetOldUnstarredRSSItems(time.Now().Add(-1 * time.Minute)) 40 | if len(pages) > 0 { 41 | fmt.Println(pages) 42 | t.Errorf("No pages are expected to exist that are this old") 43 | } 44 | }) 45 | t.Run("Should return items olderThan", func(t *testing.T) { 46 | 47 | // Setup 48 | err := nDao.AddRssItem(RssItem{ 49 | title: "notion-rss integration test", 50 | link: *itemUrl, 51 | content: []string{}, 52 | categories: []string{"Integration testing", "notion", "rss"}, 53 | feedName: "Integration Test", 54 | published: &published, 55 | }) 56 | if err != nil { 57 | t.Fatalf(err.Error()) 58 | } 59 | 60 | pageIds := nDao.GetOldUnstarredRSSItems(time.Now().Add(1 * time.Hour)) 61 | if len(pageIds) == 0 { 62 | t.Errorf("Expected GetOldUnstarredRSSItems to return an item") 63 | } 64 | }) 65 | cleanupContentDatabase(nDao) 66 | } 67 | 68 | func cleanupContentDatabase(nDao *NotionDao) { 69 | resp, _ := nDao.client.Database.Query(context.Background(), nDao.contentDatabaseId, ¬ionapi.DatabaseQueryRequest{}) 70 | for _, p := range resp.Results { 71 | _, err := nDao.client.Page.Update( 72 | context.TODO(), 73 | notionapi.PageID(p.ID), 74 | ¬ionapi.PageUpdateRequest{ 75 | Archived: true, 76 | Properties: notionapi.Properties{}, 77 | }) 78 | if err != nil { 79 | fmt.Printf(err.Error()) 80 | } 81 | } 82 | } 83 | 84 | // 85 | //func TestArchivePages(t *testing.T) { 86 | // 87 | //} 88 | // 89 | //func TestGetEnabledRssFeeds(t *testing.T) { 90 | // 91 | //} 92 | // 93 | //func TestAddRssItem(t *testing.T) { 94 | // 95 | //} 96 | -------------------------------------------------------------------------------- /rss.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "net/url" 6 | "time" 7 | 8 | "github.com/mmcdole/gofeed" 9 | ) 10 | 11 | type RssItem struct { 12 | title string 13 | link url.URL 14 | content []string 15 | categories []string 16 | feedName string 17 | published *time.Time 18 | description *string 19 | } 20 | 21 | type FeedDatabaseItem struct { 22 | FeedLink *url.URL 23 | Name string 24 | Created time.Time 25 | LastModified time.Time 26 | } 27 | 28 | // GetRssContent from a channel of RSS urls, parses new RSS items (that are from the lastNHours), 29 | // and sends them to an output channel. 30 | func GetRssContent(feedDatabaseItems chan *FeedDatabaseItem, since time.Time) chan RssItem { 31 | result := make(chan RssItem) 32 | 33 | go func(feeds chan *FeedDatabaseItem, since time.Time, rssContent chan RssItem) { 34 | defer close(result) 35 | 36 | for f := range feeds { 37 | for _, item := range GetRssContentFrom(f, since) { 38 | rssContent <- *item 39 | } 40 | } 41 | }(feedDatabaseItems, since, result) 42 | 43 | return result 44 | } 45 | 46 | // GetRssContentFrom since afterTime from the RSS feed found at url. 47 | func GetRssContentFrom(feed *FeedDatabaseItem, afterTime time.Time) []*RssItem { 48 | feedUrl := feed.FeedLink 49 | feedContent, err := gofeed.NewParser().ParseURL(feed.FeedLink.String()) 50 | if err != nil { 51 | fmt.Println(fmt.Errorf("could not get content from rss url: %s. Error occurred %w", feedUrl, err).Error()) 52 | return []*RssItem{} 53 | } 54 | 55 | // If Feed entry is new, publish all the content from it. 56 | publishAllItems := feed.Created.After(afterTime) 57 | return ExtractRssContentFeed(feedContent, afterTime, publishAllItems, feed.Name) 58 | } 59 | 60 | // ExtractRssContentFeed Extract RSS content from an RSS feed 61 | func ExtractRssContentFeed(f *gofeed.Feed, afterTime time.Time, publishAllItems bool, databaseFeedName string) []*RssItem { 62 | result := make([]*RssItem, len(f.Items)) 63 | count := 0 64 | for _, item := range f.Items { 65 | if publishAllItems || item.PublishedParsed.After(afterTime) { 66 | result[count] = convert(item, databaseFeedName) 67 | count++ 68 | } 69 | } 70 | fmt.Printf("Feed %s has %d items. %d are eligible to be uploaded\n", f.Title, len(f.Items), count) 71 | return result[:count] 72 | } 73 | 74 | // convert gofeed.Item into an internal RSSItem model. 75 | func convert(item *gofeed.Item, itemFeedName string) *RssItem { 76 | link, _ := url.Parse(item.Link) 77 | return &RssItem{ 78 | title: item.Title, 79 | link: *link, 80 | content: []string{item.Content}, 81 | categories: item.Categories, 82 | feedName: itemFeedName, 83 | published: item.PublishedParsed, 84 | description: &item.Description, 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 2 | github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE= 3 | github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= 4 | github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo= 5 | github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= 6 | github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= 7 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 8 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 9 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 10 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 11 | github.com/jomei/notionapi v1.8.5 h1://+CPjxKFutpQ4jwVYtqpF/B//MqnTOoTVsYwY264Gg= 12 | github.com/jomei/notionapi v1.8.5/go.mod h1:wgxFlmxL+oIfxclWkt8jta0PkcBepajish2uCxzBxTo= 13 | github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= 14 | github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 15 | github.com/mmcdole/gofeed v1.1.3 h1:pdrvMb18jMSLidGp8j0pLvc9IGziX4vbmvVqmLH6z8o= 16 | github.com/mmcdole/gofeed v1.1.3/go.mod h1:QQO3maftbOu+hiVOGOZDRLymqGQCos4zxbA4j89gMrE= 17 | github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf h1:sWGE2v+hO0Nd4yFU/S/mDBM5plIU8v/Qhfz41hkDIAI= 18 | github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf/go.mod h1:pasqhqstspkosTneA62Nc+2p9SOBBYAPbnmRRWPQ0V8= 19 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= 20 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 21 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= 22 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 23 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 24 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 25 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 26 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 27 | github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 28 | github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= 29 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 30 | github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= 31 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 32 | github.com/urfave/cli v1.22.3/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= 33 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 34 | golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 35 | golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 36 | golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0= 37 | golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 38 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 39 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 40 | golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= 41 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 42 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 43 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 44 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 45 | -------------------------------------------------------------------------------- /notion_dao_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/jomei/notionapi" 6 | "net/url" 7 | "os" 8 | "testing" 9 | "time" 10 | ) 11 | 12 | func runConstructNotionDaoFromEnvWith(rsskey string, content_database_id string, content_feeds_id string) (*NotionDao, error) { 13 | if len(rsskey) > 0 { 14 | os.Setenv("NOTION_RSS_KEY", rsskey) 15 | } 16 | if len(content_database_id) > 0 { 17 | os.Setenv("NOTION_RSS_CONTENT_DATABASE_ID", content_database_id) 18 | } 19 | if len(content_feeds_id) > 0 { 20 | os.Setenv("NOTION_RSS_FEEDS_DATABASE_ID", content_feeds_id) 21 | } 22 | 23 | nDao, err := ConstructNotionDaoFromEnv() 24 | 25 | if len(rsskey) > 0 { 26 | os.Unsetenv("NOTION_RSS_KEY") 27 | } 28 | if len(content_database_id) > 0 { 29 | os.Unsetenv("NOTION_RSS_CONTENT_DATABASE_ID") 30 | } 31 | if len(content_feeds_id) > 0 { 32 | os.Unsetenv("NOTION_RSS_FEEDS_DATABASE_ID") 33 | } 34 | 35 | return nDao, err 36 | } 37 | 38 | // getEnvWithDefault returns the environment variable (string), whether it existed (boolean), and 39 | // the value to use (either the defaultValue, or the environment variable). 40 | func getEnvWithDefault(key string, defaultValue string) (string, bool, string) { 41 | value, exists := os.LookupEnv(key) 42 | if exists { 43 | return value, exists, value 44 | } else { 45 | return value, exists, defaultValue 46 | } 47 | } 48 | 49 | func TestConstructNotionDaoFromEnv(t *testing.T) { 50 | // Store environment variables, and calculate values (possibly with defaults). 51 | PRIOR_NOTION_RSS_KEY, PNRK_exists, NOTION_RSS_KEY := getEnvWithDefault("NOTION_RSS_KEY", "NOTION_RSS_KEY") 52 | PRIOR_NOTION_RSS_CONTENT_DATABASE_ID, PNRCDI_exists, NOTION_RSS_CONTENT_DATABASE_ID := getEnvWithDefault("NOTION_RSS_CONTENT_DATABASE_ID", "NOTION_RSS_CONTENT_DATABASE_ID") 53 | PRIOR_NOTION_RSS_FEEDS_DATABASE_ID, PNRFDI_exists, NOTION_RSS_FEEDS_DATABASE_ID := getEnvWithDefault("NOTION_RSS_FEEDS_DATABASE_ID", "NOTION_RSS_FEEDS_DATABASE_ID") 54 | 55 | os.Unsetenv("NOTION_RSS_KEY") 56 | os.Unsetenv("NOTION_RSS_CONTENT_DATABASE_ID") 57 | os.Unsetenv("NOTION_RSS_FEEDS_DATABASE_ID") 58 | 59 | _, err := runConstructNotionDaoFromEnvWith("", NOTION_RSS_CONTENT_DATABASE_ID, NOTION_RSS_FEEDS_DATABASE_ID) 60 | if err == nil { 61 | t.Errorf("ConstructNotionDaoFromEnvWith should return error if `NOTION_RSS_KEY` is not set") 62 | } 63 | 64 | _, err = runConstructNotionDaoFromEnvWith(NOTION_RSS_KEY, "", NOTION_RSS_FEEDS_DATABASE_ID) 65 | if err == nil { 66 | t.Errorf("ConstructNotionDaoFromEnvWith should return error if `NOTION_RSS_KEY` is not set") 67 | } 68 | 69 | _, err = runConstructNotionDaoFromEnvWith(NOTION_RSS_KEY, NOTION_RSS_CONTENT_DATABASE_ID, "") 70 | if err == nil { 71 | t.Errorf("ConstructNotionDaoFromEnvWith should return error if `NOTION_RSS_KEY` is not set") 72 | } 73 | 74 | nDao, err := runConstructNotionDaoFromEnvWith(NOTION_RSS_KEY, NOTION_RSS_CONTENT_DATABASE_ID, NOTION_RSS_FEEDS_DATABASE_ID) 75 | if err != nil { 76 | t.Errorf("unexpected error: %s", err.Error()) 77 | } 78 | if nDao.client == nil { 79 | t.Errorf("notion client was not constructed") 80 | } 81 | 82 | // Reset environment variables 83 | if PNRK_exists { 84 | os.Setenv("NOTION_RSS_KEY", PRIOR_NOTION_RSS_KEY) 85 | } 86 | if PNRCDI_exists { 87 | os.Setenv("NOTION_RSS_CONTENT_DATABASE_ID", PRIOR_NOTION_RSS_CONTENT_DATABASE_ID) 88 | } 89 | if PNRFDI_exists { 90 | os.Setenv("NOTION_RSS_FEEDS_DATABASE_ID", PRIOR_NOTION_RSS_FEEDS_DATABASE_ID) 91 | } 92 | } 93 | 94 | // GetRssFeedFromDatabaseObject(p *notionapi.Page) (*FeedDatabaseItem, error) { 95 | // urlProperty := p.Properties["Link"].(*notionapi.URLProperty).URL 96 | // rssUrl, err := url.Parse(urlProperty) 97 | // if err != nil { 98 | // return &FeedDatabaseItem{}, err 99 | // } 100 | // 101 | // nameRichTexts := p.Properties["Title"].(*notionapi.TitleProperty).Title 102 | // if len(nameRichTexts) == 0 { 103 | // return &FeedDatabaseItem{}, fmt.Errorf("RSS Feed database entry does not have any Title in 'Title' field") 104 | // } 105 | // 106 | // return &FeedDatabaseItem{ 107 | // FeedLink: rssUrl, 108 | // Created: p.CreatedTime, 109 | // LastModified: p.LastEditedTime, 110 | // Name: nameRichTexts[0].PlainText, 111 | // }, nil 112 | // } 113 | func TestGetRssFeedFromDatabaseObject(t *testing.T) { 114 | type TestCase struct { 115 | page *notionapi.Page 116 | expectedDbItem *FeedDatabaseItem 117 | expectedErr error 118 | subTestName string 119 | } 120 | 121 | editedTime := time.Now() 122 | repoUrl, _ := url.Parse("https://github.com/Jeadie/notion-rss") 123 | tests := []TestCase{ 124 | { 125 | page: ¬ionapi.Page{ 126 | LastEditedTime: editedTime, 127 | Properties: map[string]notionapi.Property{ 128 | "Title": ¬ionapi.TitleProperty{Title: []notionapi.RichText{{PlainText: "TestTitle"}}}, 129 | "Link": ¬ionapi.URLProperty{URL: repoUrl.String()}, 130 | "Enabled": ¬ionapi.CheckboxProperty{Checkbox: true}, 131 | }, 132 | }, 133 | expectedDbItem: &FeedDatabaseItem{ 134 | FeedLink: repoUrl, 135 | Name: "TestTitle", 136 | LastModified: editedTime, 137 | }, 138 | expectedErr: nil, 139 | subTestName: "valid parsing", 140 | }, 141 | { 142 | page: ¬ionapi.Page{ 143 | LastEditedTime: editedTime, 144 | Properties: map[string]notionapi.Property{ 145 | "Title": ¬ionapi.TitleProperty{}, 146 | "Link": ¬ionapi.URLProperty{URL: repoUrl.String()}, 147 | "Enabled": ¬ionapi.CheckboxProperty{Checkbox: true}, 148 | }, 149 | }, 150 | expectedDbItem: &FeedDatabaseItem{}, 151 | expectedErr: fmt.Errorf("failed"), 152 | subTestName: "no Title element in TitleProperty", 153 | }, 154 | { 155 | page: ¬ionapi.Page{ 156 | LastEditedTime: editedTime, 157 | Properties: map[string]notionapi.Property{ 158 | "Link": ¬ionapi.URLProperty{URL: repoUrl.String()}, 159 | "Enabled": ¬ionapi.CheckboxProperty{Checkbox: true}, 160 | }, 161 | }, 162 | expectedDbItem: &FeedDatabaseItem{}, 163 | expectedErr: fmt.Errorf("failed"), 164 | subTestName: "Missing TitleProperty", 165 | }, 166 | } 167 | 168 | for _, test := range tests { 169 | t.Run(test.subTestName, func(t *testing.T) { 170 | item, err := GetRssFeedFromDatabaseObject(test.page) 171 | if (err != nil) != (test.expectedErr != nil) { 172 | if err != nil { 173 | t.Errorf("Unexpected error occurred. Error: %s \n", err.Error()) 174 | } else { 175 | t.Errorf("Error was expected, but none returned. Expected error: %s \n", test.expectedErr.Error()) 176 | } 177 | } 178 | 179 | if test.expectedErr == nil { 180 | expectedItem := test.expectedDbItem 181 | if item.Name != expectedItem.Name { 182 | t.Errorf("Incorrect name of item. Expected %s, returned %s", expectedItem.Name, item.Name) 183 | } 184 | if item.FeedLink.String() != expectedItem.FeedLink.String() { 185 | t.Errorf("Incorrect RSS feed url. Expected %s, returned %s", expectedItem.FeedLink, item.FeedLink) 186 | } 187 | if item.Created != expectedItem.Created { 188 | t.Errorf("Incorrect created timestamp. Expected %s, returned %s", expectedItem.Created, item.Created) 189 | } 190 | if item.LastModified != expectedItem.LastModified { 191 | t.Errorf("Incorrect last modified timestamp. Expected %s, returned %s", expectedItem.LastModified, item.LastModified) 192 | } 193 | } 194 | 195 | }) 196 | } 197 | 198 | } 199 | -------------------------------------------------------------------------------- /notion_dao.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/url" 7 | "os" 8 | "regexp" 9 | "strings" 10 | "time" 11 | 12 | "github.com/jomei/notionapi" 13 | ) 14 | 15 | type NotionDao struct { 16 | feedDatabaseId notionapi.DatabaseID 17 | contentDatabaseId notionapi.DatabaseID 18 | client *notionapi.Client 19 | } 20 | 21 | // ConstructNotionDaoFromEnv given environment variables: NOTION_RSS_KEY, 22 | // NOTION_RSS_CONTENT_DATABASE_ID, NOTION_RSS_FEEDS_DATABASE_ID 23 | func ConstructNotionDaoFromEnv() (*NotionDao, error) { 24 | integrationKey, exists := os.LookupEnv("NOTION_RSS_KEY") 25 | if !exists { 26 | return &NotionDao{}, fmt.Errorf("`NOTION_RSS_KEY` not set") 27 | } 28 | 29 | contentDatabaseId, exists := os.LookupEnv("NOTION_RSS_CONTENT_DATABASE_ID") 30 | if !exists { 31 | return &NotionDao{}, fmt.Errorf("`NOTION_RSS_CONTENT_DATABASE_ID` not set") 32 | } 33 | 34 | feedDatabaseId, exists := os.LookupEnv("NOTION_RSS_FEEDS_DATABASE_ID") 35 | if !exists { 36 | return &NotionDao{}, fmt.Errorf("`NOTION_RSS_FEEDS_DATABASE_ID` not set") 37 | } 38 | 39 | return ConstructNotionDao(feedDatabaseId, contentDatabaseId, integrationKey), nil 40 | } 41 | 42 | func ConstructNotionDao(feedDatabaseId string, contentDatabaseId string, integrationKey string) *NotionDao { 43 | return &NotionDao{ 44 | feedDatabaseId: notionapi.DatabaseID(feedDatabaseId), 45 | contentDatabaseId: notionapi.DatabaseID(contentDatabaseId), 46 | client: notionapi.NewClient(notionapi.Token(integrationKey)), 47 | } 48 | } 49 | 50 | // GetOldUnstarredRSSItems that were created strictly before olderThan and are not starred. 51 | func (dao NotionDao) GetOldUnstarredRSSItems(olderThan time.Time) []notionapi.Page { 52 | resp, err := dao.client.Database.Query(context.TODO(), dao.contentDatabaseId, ¬ionapi.DatabaseQueryRequest{ 53 | Filter: (notionapi.AndCompoundFilter)([]notionapi.Filter{ 54 | 55 | // Use `Created`, not `Published` as to avoid deleting cold-started RSS feeds. 56 | notionapi.PropertyFilter{ 57 | Property: "Created", 58 | Date: ¬ionapi.DateFilterCondition{ 59 | Before: (*notionapi.Date)(&olderThan), 60 | }, 61 | }, 62 | notionapi.PropertyFilter{ 63 | Property: "Starred", 64 | Checkbox: ¬ionapi.CheckboxFilterCondition{ 65 | Equals: false, 66 | DoesNotEqual: true, 67 | }, 68 | }, 69 | }), 70 | // TODO: pagination 71 | //StartCursor: "", 72 | //PageSize: 0, 73 | }) 74 | if err != nil { 75 | fmt.Printf("error occurred in GetOldUnstarredRSSItems. Error: %s\n", err.Error()) 76 | return []notionapi.Page{} 77 | } 78 | return resp.Results 79 | } 80 | 81 | func (dao NotionDao) GetOldUnstarredRSSItemIds(olderThan time.Time) []notionapi.PageID { 82 | pages := dao.GetOldUnstarredRSSItems(olderThan) 83 | result := make([]notionapi.PageID, len(pages)) 84 | for i, page := range pages { 85 | result[i] = notionapi.PageID(page.ID) 86 | } 87 | return result 88 | } 89 | 90 | // ArchivePages for a list of pageIds. Will archive each page even if other pages fail. 91 | func (dao *NotionDao) ArchivePages(pageIds []notionapi.PageID) error { 92 | failedCount := 0 93 | for _, p := range pageIds { 94 | _, err := dao.client.Page.Update( 95 | context.TODO(), 96 | p, 97 | ¬ionapi.PageUpdateRequest{ 98 | Archived: true, 99 | Properties: notionapi.Properties{}, // Must be provided, even if empty 100 | }, 101 | ) 102 | if err != nil { 103 | fmt.Printf("Failed to archive page: %s. Error: %s\n", p.String(), err.Error()) 104 | failedCount++ 105 | } 106 | } 107 | if failedCount > 0 { 108 | return fmt.Errorf("failed to archive %d pages", failedCount) 109 | } 110 | return nil 111 | } 112 | 113 | // GetEnabledRssFeeds from the Feed Database. Results filtered on property "Enabled"=true 114 | func (dao *NotionDao) GetEnabledRssFeeds() chan *FeedDatabaseItem { 115 | rssFeeds := make(chan *FeedDatabaseItem) 116 | 117 | go func(dao *NotionDao, output chan *FeedDatabaseItem) { 118 | defer close(output) 119 | 120 | req := ¬ionapi.DatabaseQueryRequest{ 121 | Filter: notionapi.PropertyFilter{ 122 | Property: "Enabled", 123 | Checkbox: ¬ionapi.CheckboxFilterCondition{ 124 | Equals: true, 125 | }, 126 | }, 127 | } 128 | 129 | //TODO: Get multi-page pagination results from resp.HasMore 130 | resp, err := dao.client.Database.Query(context.Background(), dao.feedDatabaseId, req) 131 | if err != nil { 132 | return 133 | } 134 | for _, r := range resp.Results { 135 | feed, err := GetRssFeedFromDatabaseObject(&r) 136 | if err == nil { 137 | rssFeeds <- feed 138 | } 139 | } 140 | }(dao, rssFeeds) 141 | return rssFeeds 142 | } 143 | 144 | func GetRssFeedFromDatabaseObject(p *notionapi.Page) (*FeedDatabaseItem, error) { 145 | if p.Properties["Link"] == nil || p.Properties["Title"] == nil { 146 | return &FeedDatabaseItem{}, fmt.Errorf("notion page is expected to have `Link` and `Title` properties. Properties: %s", p.Properties) 147 | } 148 | urlProperty := p.Properties["Link"].(*notionapi.URLProperty).URL 149 | rssUrl, err := url.Parse(urlProperty) 150 | if err != nil { 151 | return &FeedDatabaseItem{}, err 152 | } 153 | 154 | nameRichTexts := p.Properties["Title"].(*notionapi.TitleProperty).Title 155 | if len(nameRichTexts) == 0 { 156 | return &FeedDatabaseItem{}, fmt.Errorf("RSS Feed database entry does not have any Title in 'Title' field") 157 | } 158 | 159 | return &FeedDatabaseItem{ 160 | FeedLink: rssUrl, 161 | Created: p.CreatedTime, 162 | LastModified: p.LastEditedTime, 163 | Name: nameRichTexts[0].PlainText, 164 | }, nil 165 | } 166 | 167 | func GetImageUrl(x string) *string { 168 | // Extract the first image src from the document to use as cover 169 | re := regexp.MustCompile(`(?m)]+?src\s*=\s*['"]?([^\s'"?#>]+)`) 170 | match := re.FindSubmatch([]byte(x)) 171 | if match != nil { 172 | v := string(match[1]) 173 | if strings.HasPrefix(v, "http") { 174 | return &v 175 | } else { 176 | fmt.Printf("[ERROR]: Invalid image url found in url=%s\n", string(match[1])) 177 | return nil 178 | } 179 | } 180 | return nil 181 | } 182 | 183 | // AddRssItem to Notion database as a single new page with Block content. On failure, no retry is attempted. 184 | func (dao NotionDao) AddRssItem(item RssItem) error { 185 | categories := make([]notionapi.Option, len(item.categories)) 186 | for i, c := range item.categories { 187 | categories[i] = notionapi.Option{ 188 | Name: c, 189 | } 190 | } 191 | var imageProp *notionapi.Image 192 | // TODO: Currently notionapi.URLProperty is not nullable, which is needed 193 | // to use thumbnail properly (i.e. handle the case when no image in RSS item). 194 | //thumbnailProp := ¬ionapi.URLProperty{ 195 | // Type: "url", 196 | // URL: , 197 | //} 198 | 199 | image := GetImageUrl(strings.Join(item.content, " ")) 200 | if image != nil { 201 | imageProp = ¬ionapi.Image{ 202 | Type: "external", 203 | External: ¬ionapi.FileObject{ 204 | URL: *image, 205 | }, 206 | } 207 | } 208 | 209 | _, err := dao.client.Page.Create(context.Background(), ¬ionapi.PageCreateRequest{ 210 | Parent: notionapi.Parent{ 211 | Type: "database_id", 212 | DatabaseID: dao.contentDatabaseId, 213 | }, 214 | Properties: map[string]notionapi.Property{ 215 | "Title": notionapi.TitleProperty{ 216 | Type: "title", 217 | Title: []notionapi.RichText{{ 218 | Type: "text", 219 | Text: notionapi.Text{ 220 | Content: item.title, 221 | }, 222 | }}, 223 | }, 224 | "Description": notionapi.RichTextProperty{ 225 | Type: "rich_text", 226 | RichText: []notionapi.RichText{{ 227 | Type: notionapi.ObjectTypeText, 228 | Text: notionapi.Text{ 229 | Content: *item.description, 230 | }, 231 | PlainText: *item.description, 232 | }, 233 | }, 234 | }, 235 | "Link": notionapi.URLProperty{ 236 | Type: "url", 237 | URL: item.link.String(), 238 | }, 239 | "Categories": notionapi.MultiSelectProperty{ 240 | MultiSelect: categories, 241 | }, 242 | "From": notionapi.SelectProperty{Select: notionapi.Option{Name: item.feedName}}, 243 | "Published": notionapi.DateProperty{Date: ¬ionapi.DateObject{Start: (*notionapi.Date)(item.published)}}, 244 | }, 245 | Children: RssContentToBlocks(item), 246 | Cover: imageProp, 247 | }) 248 | return err 249 | } 250 | 251 | func RssContentToBlocks(item RssItem) []notionapi.Block { 252 | // TODO: implement when we know RssItem struct better 253 | return []notionapi.Block{} 254 | } 255 | --------------------------------------------------------------------------------