├── .editorconfig ├── .gitattributes ├── .github ├── dependabot.yml └── workflows │ ├── go.yml │ └── release.yml ├── .gitignore ├── .goreleaser.yml ├── CHANGELOG.md ├── Dockerfile ├── LICENSE ├── README.md ├── config.yml.example ├── go.mod ├── go.sum ├── internal ├── feed │ ├── cache │ │ ├── cache.go │ │ ├── state.go │ │ ├── v1.go │ │ └── v2.go │ ├── feed.go │ ├── filter │ │ └── filter.go │ ├── item.go │ ├── mail.go │ ├── parse.go │ └── template │ │ ├── funcs.go │ │ ├── funcs_test.go │ │ ├── html.tpl │ │ ├── template.go │ │ ├── template_test.go │ │ └── text.tpl ├── http │ └── client.go ├── imap │ ├── client.go │ ├── cmds.go │ ├── commando.go │ ├── connection.go │ ├── folder.go │ ├── imap.go │ └── mailboxes.go └── msg │ └── msg.go ├── main.go ├── pkg ├── config │ ├── body.go │ ├── config.go │ ├── deprecated.go │ ├── feed.go │ ├── url.go │ ├── url_test.go │ ├── yaml.go │ └── yaml_test.go ├── log │ └── log.go ├── rfc822 │ ├── writer.go │ └── writer_test.go ├── util │ └── util.go └── version │ └── version.go └── tools ├── README └── print-cache └── print-cache.go /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | end_of_line = lf 5 | 6 | [*.go] 7 | indent_style = tab 8 | indent_size = 4 9 | 10 | [*.tpl] 11 | indent_style = space 12 | indent_size = 2 13 | end_of_line = crlf 14 | trim_trailing_whitespace = false -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # Default 2 | * text=auto 3 | 4 | # Templates should have CRLF 5 | *.tpl text eol=crlf -linguist-detectable 6 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: github-actions 4 | directory: "/" 5 | schedule: 6 | interval: weekly 7 | day: friday 8 | - package-ecosystem: gomod 9 | directory: "/" 10 | schedule: 11 | interval: weekly 12 | day: friday 13 | -------------------------------------------------------------------------------- /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | jobs: 10 | 11 | build: 12 | name: Build 13 | runs-on: ubuntu-latest 14 | steps: 15 | 16 | - name: Checkout 17 | uses: actions/checkout@v4 18 | 19 | - name: Set up Go 20 | uses: actions/setup-go@v5 21 | with: 22 | go-version-file: 'go.mod' 23 | 24 | - name: Get dependencies 25 | run: | 26 | go get -v -t -d ./... 27 | 28 | - name: Build 29 | run: go build -v ./... 30 | 31 | - name: Test 32 | run: go test -v ./... 33 | 34 | - name: Vet 35 | run: go vet ./... 36 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: release 2 | 3 | on: 4 | push: 5 | tags: 6 | - 'v*' 7 | 8 | jobs: 9 | goreleaser: 10 | name: Release 11 | runs-on: ubuntu-latest 12 | steps: 13 | 14 | - name: Checkout 15 | uses: actions/checkout@v4 16 | with: 17 | fetch-depth: 0 18 | 19 | - name: Set up Go 20 | uses: actions/setup-go@v5 21 | with: 22 | go-version-file: 'go.mod' 23 | 24 | - name: Get version from tag 25 | uses: little-core-labs/get-git-tag@v3.0.2 26 | id: tag_name 27 | with: 28 | tagRegex: "v(.*)" 29 | 30 | - name: Get Changelog Entry 31 | id: changelog_reader 32 | uses: mindsers/changelog-reader-action@v2 33 | with: 34 | version: ${{ steps.tag_name.outputs.tag }} 35 | path: ./CHANGELOG.md 36 | 37 | - name: Safe Changelog Text 38 | id: changelog_text 39 | run: | 40 | echo '${{ steps.changelog_reader.outputs.changes }}' >> $HOME/changelog_entry 41 | echo ::set-output name=clfile::$HOME/changelog_entry 42 | 43 | - name: Docker Login 44 | uses: azure/docker-login@v2 45 | with: 46 | login-server: 'docker.pkg.github.com' 47 | username: ${{ github.repository_owner }} 48 | password: ${{ secrets.GITHUB_TOKEN }} 49 | 50 | - name: Docker Login 51 | uses: azure/docker-login@v2 52 | with: 53 | login-server: 'https://index.docker.io/v1/' 54 | username: ${{ secrets.DOCKER_USERNAME }} 55 | password: ${{ secrets.DOCKER_TOKEN }} 56 | 57 | - name: Run GoReleaser 58 | uses: goreleaser/goreleaser-action@v6 59 | with: 60 | version: latest 61 | args: --release-notes ${{ steps.changelog_text.outputs.clfile }} 62 | env: 63 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 64 | 65 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # editor stuff 2 | .idea 3 | 4 | # binaries 5 | /feed2imap-go 6 | /print-cache 7 | 8 | # local setup 9 | config*.yml 10 | *.cache 11 | /dist/ 12 | *.old 13 | /*.tpl 14 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | project_name: feed2imap-go 3 | 4 | before: 5 | hooks: 6 | - go mod download 7 | builds: 8 | - 9 | id: standalone 10 | binary: feed2imap-go 11 | ldflags: 12 | - -s -w -X github.com/Necoro/feed2imap-go/pkg/version.version={{.Version}} -X github.com/Necoro/feed2imap-go/pkg/version.commit={{.ShortCommit}} 13 | goos: 14 | - windows 15 | - linux 16 | - darwin 17 | goarch: 18 | - amd64 19 | 20 | # for DOCKER we explicitly disable CGO 21 | # we keep it enabled for the standalone version, b/c it might have advantages on 22 | # more complicated setups, where the go internal implementations call out to glibc 23 | - 24 | id: docker 25 | binary: feed2imap-go 26 | env: 27 | - CGO_ENABLED=0 28 | ldflags: 29 | - -s -w -X github.com/Necoro/feed2imap-go/pkg/version.version={{.Version}} -X github.com/Necoro/feed2imap-go/pkg/version.commit={{.ShortCommit}} 30 | goos: 31 | - linux 32 | goarch: 33 | - amd64 34 | 35 | dockers: 36 | - 37 | ids: 38 | - docker 39 | goos: linux 40 | goarch: amd64 41 | image_templates: 42 | - "necorodm/feed2imap-go:latest" 43 | - "necorodm/feed2imap-go:{{ .Version }}" 44 | - "docker.pkg.github.com/necoro/feed2imap-go/feed2imap-go:latest" 45 | - "docker.pkg.github.com/necoro/feed2imap-go/feed2imap-go:{{ .Version }}" 46 | 47 | archives: 48 | - 49 | builds: 50 | - standalone 51 | 52 | format: tar.gz 53 | format_overrides: 54 | - goos: windows 55 | format: zip 56 | files: 57 | - LICENSE 58 | - README.md 59 | - CHANGELOG.md 60 | - config.yml.example 61 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | All notable changes to this project will be documented in this file. 3 | 4 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 5 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 6 | 7 | ## [Unreleased] 8 | 9 | ## [1.7.2] - 2024-07-10 10 | ### Fixed 11 | - Cache loading problem introduced by v1.7.1 12 | 13 | ## [1.7.1] - 2024-07-10 14 | - Upgraded dependencies 15 | 16 | ## [1.7.0] - 2023-06-13 17 | ### Fixed 18 | - [Issue #97](https://github.com/Necoro/feed2imap-go/issues/97): Fix panic when an IMAP connection is tried to established when the program has already entered the `Disconnect` state. 19 | ### Added 20 | - New global config variable `max-imap-connections`, with a default of `5`. 21 | ### Changed 22 | - Change go-readability back to track upstream. This now incorporates an overhaul of the readability engine, so changes are to be expected when using `body: fetch`. 23 | - Improve strictness regarding config parsing, especially in the differentiation between feeds and groups of feeds. 24 | - [Issue #95](https://github.com/Necoro/feed2imap-go/issues/95): Ensure cache is not world-readable. Add a warning when the config is world-readable. 25 | 26 | ## [1.6.0] - 2023-03-28 27 | - Upgrade dependencies 28 | ### Fixed 29 | - [Issue #91](https://github.com/Necoro/feed2imap-go/issues/91): Panic on using per-item targets 30 | 31 | ## [1.5.2] - 2023-01-31 32 | - Upgrade dependencies 33 | 34 | ## [1.5.1] - 2022-06-11 35 | - Upgrade dependencies 36 | - Minor lifting to Go 1.18 37 | 38 | ## [1.5.0] - 2022-01-11 39 | ### Added 40 | - [Issue #66](https://github.com/Necoro/feed2imap-go/issues/66): Allow specifying custom template files for HTML/Text output (configuration: `html-template`/`text-template`). 41 | - [Issue #67](https://github.com/Necoro/feed2imap-go/issues/67): Support for fetching the linked article/website instead of using the body in the feed (configuration: `body: fetch`). This is especially useful when the feed only supplies links/teaser and not the full content. 42 | ### Fixed 43 | - Panic on setting `body` on feed-level. 44 | 45 | ## [1.4.0] - 2021-12-21 46 | ### Added 47 | - Make links absolute: relative links inside a feed cannot be resolved outside a webbrowser (Example/culprit: https://go.dev/blog/go1.18beta1) 48 | 49 | ## [1.3.0] - 2021-11-01 50 | ### Added 51 | - [Issue #64](https://github.com/Necoro/feed2imap-go/issues/64): Set filename for included images. 52 | 53 | ## [1.2.0] - 2021-10-20 54 | ### Added 55 | - Location of the cache can now be specified in the config.yml. Rationale: Easier way of dealing with multiple configurations, as each also requires a distinct cache. 56 | - [Issue #6](https://github.com/Necoro/feed2imap-go/issues/6): Support old-style configurations with imap targets on each feed. Restriction: Servers must be equal over all connection strings! 57 | ### Fixed 58 | - [Issue #62](https://github.com/Necoro/feed2imap-go/issues/62): Allow empty root folder. 59 | 60 | ## [1.1.1] - 2021-07-24 61 | ### Fixed 62 | - Correctly log out from the imap server and do not harshly disconnect. 63 | 64 | ## [1.1.0] - 2021-06-18 65 | ### Fixed 66 | - Do not try to download already embedded images (i.e. `data:image/`). 67 | ### Changed 68 | - Updated dependencies. Most notable: Upgrade of yaml.v3, which entails changes of nil-handling. 69 | 70 | ## [1.0.0] - 2021-05-19 71 | ### Fixed 72 | - [Issue #47](https://github.com/Necoro/feed2imap-go/issues/47): Fixed occassional deadlocks. Reason was unilateral updates from the server which were ill-handled by go-imap. 73 | 74 | ## [0.8.0] - 2021-03-06 75 | ### Added 76 | - New cache format v2 that uses gzip compression 77 | - Support for JSON v1.1 feed (via [gofeed](https://github.com/mmcdole/gofeed/pull/169)) 78 | ### Changed 79 | - Caches store now 1000 old entries (i.e., not included in the last fetch) at maximum. This will clean obsolete cruft and drastically reduce cache size. 80 | - Feeds not updated (and not part of the config) for 180 days are now automatically removed from the cache. 81 | - Connecting to the IMAP server now happens in the background and in parallel, using connections directly as they are established. This should yield a speed-up with slow IMAP servers. 82 | - IMAP connections now can time out during establishment. 83 | 84 | ## [0.7.0] - 2021-02-21 85 | ### Changed 86 | - Remove `srcset` attribute of `img` tags when including images in mail 87 | - Strip whitespaces from folder names 88 | 89 | ### Fixed 90 | - [Issue #39](https://github.com/Necoro/feed2imap-go/issues/39): Do not re-introduce deleted mails, even though `reupload-if-updated` is false. 91 | - [Issue #25](https://github.com/Necoro/feed2imap-go/issues/25): Normalize folder names, so `foo` and `foo/` are not seen as different folders. 92 | 93 | ## [0.6.0] - 2021-02-14 94 | ### Fixed 95 | - [Issue #46](https://github.com/Necoro/feed2imap-go/issues/46): Fixed line endings in templates, thereby pleasing Cyrus IMAP server. 96 | 97 | ## [0.5.2] - 2020-11-23 98 | - Update of libraries 99 | - This now also includes the updated gofeed dependency, that was promised with the last version but not included... 100 | 101 | ## [0.5.1] - 2020-09-11 102 | - Update of gofeed dependency: Now supports json feeds 103 | - Make sure, cache locks are deleted on shutdown (looks cleaner) 104 | 105 | ## [0.5.0] - 2020-08-22 106 | ### Added 107 | - Cache files are now explicitly locked. This avoids two instances of feed2imap-go running at the same time. 108 | - New header `X-Feed2Imap-Create-Date` holding the date of creation of that mail. Mostly needed for debugging issues. 109 | - Updated to Go 1.15. 110 | - New global option `auto-target` to change the default behavior of omitted `target` fields. 111 | When set to `false`, a missing `target` is identical to specifying `null` or `""` for the target. 112 | When set to `true` (the default), the standard behavior of falling back onto the name is used. 113 | 114 | ### Fixed 115 | - [Issue #24](https://github.com/Necoro/feed2imap-go/issues/24): Patched gofeed to support atom tags in RSS feeds 116 | 117 | ## [0.4.1] - 2020-06-20 118 | ### Fixed 119 | - Fix a bug, where cached items get deleted when a feed runs dry. 120 | This resulted in duplicate entries as soon as the feed contained (possibly older) entries again. 121 | 122 | ## [0.4.0] - 2020-05-25 123 | ### Added 124 | - Verbose variant of 'target' in config: Do not hassle with urlencoded passwords anymore! 125 | - New feed option 'item-filter' for filtering out specific items from feed. 126 | - New feed option 'exec', allowing to specify a command to execute instead of a Url to fetch from. 127 | 128 | ## [0.3.1] - 2020-05-12 129 | - Docker Setup 130 | 131 | ## [0.3.0] - 2020-05-10 132 | ### Added 133 | - Options are now also allowed on group-level (closes #12) 134 | - Render text parts in mails (closes #7) 135 | 136 | ## [0.2.0] - 2020-05-10 137 | ### Added 138 | - New header X-Feed2Imap-Guid 139 | 140 | ### Changed 141 | - Default for `min-frequency` is now 0 instead of 1. 142 | - Fixed date parsing from feed. _Changes cache format!_ 143 | - Do not assume items to be new when their published date is newer than the last run. Some feeds just lie... 144 | - Misc bug fixes in template rendering. 145 | 146 | ## [0.1.1] - 2020-05-04 147 | 148 | ### Added 149 | - Automatic releasing via goreleaser 150 | 151 | ### Changed 152 | - Improved version output 153 | 154 | ## [0.1.0] - 2020-05-04 155 | 156 | Initial release 157 | 158 | [Unreleased]: https://github.com/Necoro/feed2imap-go/compare/v1.7.2...HEAD 159 | [1.7.2]: https://github.com/Necoro/feed2imap-go/compare/v1.7.1...v1.7.2 160 | [1.7.1]: https://github.com/Necoro/feed2imap-go/compare/v1.7.0...v1.7.1 161 | [1.7.0]: https://github.com/Necoro/feed2imap-go/compare/v1.6.0...v1.7.0 162 | [1.6.0]: https://github.com/Necoro/feed2imap-go/compare/v1.5.2...v1.6.0 163 | [1.5.2]: https://github.com/Necoro/feed2imap-go/compare/v1.5.1...v1.5.2 164 | [1.5.1]: https://github.com/Necoro/feed2imap-go/compare/v1.5.0...v1.5.1 165 | [1.5.0]: https://github.com/Necoro/feed2imap-go/compare/v1.4.0...v1.5.0 166 | [1.4.0]: https://github.com/Necoro/feed2imap-go/compare/v1.3.0...v1.4.0 167 | [1.3.0]: https://github.com/Necoro/feed2imap-go/compare/v1.2.0...v1.3.0 168 | [1.2.0]: https://github.com/Necoro/feed2imap-go/compare/v1.1.1...v1.2.0 169 | [1.1.1]: https://github.com/Necoro/feed2imap-go/compare/v1.1.0...v1.1.1 170 | [1.1.0]: https://github.com/Necoro/feed2imap-go/compare/v1.0.0...v1.1.0 171 | [1.0.0]: https://github.com/Necoro/feed2imap-go/compare/v0.8.0...v1.0.0 172 | [0.8.0]: https://github.com/Necoro/feed2imap-go/compare/v0.7.0...v0.8.0 173 | [0.7.0]: https://github.com/Necoro/feed2imap-go/compare/v0.6.0...v0.7.0 174 | [0.6.0]: https://github.com/Necoro/feed2imap-go/compare/v0.5.2...v0.6.0 175 | [0.5.2]: https://github.com/Necoro/feed2imap-go/compare/v0.5.1...v0.5.2 176 | [0.5.1]: https://github.com/Necoro/feed2imap-go/compare/v0.5.0...v0.5.1 177 | [0.5.0]: https://github.com/Necoro/feed2imap-go/compare/v0.4.1...v0.5.0 178 | [0.4.1]: https://github.com/Necoro/feed2imap-go/compare/v0.4.0...v0.4.1 179 | [0.4.0]: https://github.com/Necoro/feed2imap-go/compare/v0.3.1...v0.4.0 180 | [0.3.1]: https://github.com/Necoro/feed2imap-go/compare/v0.3.0...v0.3.1 181 | [0.3.0]: https://github.com/Necoro/feed2imap-go/compare/v0.2.0...v0.3.0 182 | [0.2.0]: https://github.com/Necoro/feed2imap-go/compare/v0.1.1...v0.2.0 183 | [0.1.1]: https://github.com/Necoro/feed2imap-go/compare/v0.1.0...v0.1.1 184 | [0.1.0]: https://github.com/Necoro/feed2imap-go/releases/tag/v0.1.0 185 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine 2 | 3 | RUN mkdir /app 4 | COPY feed2imap-go /app 5 | 6 | ENTRYPOINT ["/app/feed2imap-go"] 7 | CMD ["-c", "/app/data/feed.cache", "-f", "/app/data/config.yml"] 8 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 2, June 1991 3 | 4 | Copyright (C) 1989, 1991 Free Software Foundation, Inc., 5 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 6 | Everyone is permitted to copy and distribute verbatim copies 7 | of this license document, but changing it is not allowed. 8 | 9 | Preamble 10 | 11 | The licenses for most software are designed to take away your 12 | freedom to share and change it. By contrast, the GNU General Public 13 | License is intended to guarantee your freedom to share and change free 14 | software--to make sure the software is free for all its users. This 15 | General Public License applies to most of the Free Software 16 | Foundation's software and to any other program whose authors commit to 17 | using it. (Some other Free Software Foundation software is covered by 18 | the GNU Lesser General Public License instead.) You can apply it to 19 | your programs, too. 20 | 21 | When we speak of free software, we are referring to freedom, not 22 | price. Our General Public Licenses are designed to make sure that you 23 | have the freedom to distribute copies of free software (and charge for 24 | this service if you wish), that you receive source code or can get it 25 | if you want it, that you can change the software or use pieces of it 26 | in new free programs; and that you know you can do these things. 27 | 28 | To protect your rights, we need to make restrictions that forbid 29 | anyone to deny you these rights or to ask you to surrender the rights. 30 | These restrictions translate to certain responsibilities for you if you 31 | distribute copies of the software, or if you modify it. 32 | 33 | For example, if you distribute copies of such a program, whether 34 | gratis or for a fee, you must give the recipients all the rights that 35 | you have. You must make sure that they, too, receive or can get the 36 | source code. And you must show them these terms so they know their 37 | rights. 38 | 39 | We protect your rights with two steps: (1) copyright the software, and 40 | (2) offer you this license which gives you legal permission to copy, 41 | distribute and/or modify the software. 42 | 43 | Also, for each author's protection and ours, we want to make certain 44 | that everyone understands that there is no warranty for this free 45 | software. If the software is modified by someone else and passed on, we 46 | want its recipients to know that what they have is not the original, so 47 | that any problems introduced by others will not reflect on the original 48 | authors' reputations. 49 | 50 | Finally, any free program is threatened constantly by software 51 | patents. We wish to avoid the danger that redistributors of a free 52 | program will individually obtain patent licenses, in effect making the 53 | program proprietary. To prevent this, we have made it clear that any 54 | patent must be licensed for everyone's free use or not licensed at all. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | GNU GENERAL PUBLIC LICENSE 60 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 61 | 62 | 0. This License applies to any program or other work which contains 63 | a notice placed by the copyright holder saying it may be distributed 64 | under the terms of this General Public License. The "Program", below, 65 | refers to any such program or work, and a "work based on the Program" 66 | means either the Program or any derivative work under copyright law: 67 | that is to say, a work containing the Program or a portion of it, 68 | either verbatim or with modifications and/or translated into another 69 | language. (Hereinafter, translation is included without limitation in 70 | the term "modification".) Each licensee is addressed as "you". 71 | 72 | Activities other than copying, distribution and modification are not 73 | covered by this License; they are outside its scope. The act of 74 | running the Program is not restricted, and the output from the Program 75 | is covered only if its contents constitute a work based on the 76 | Program (independent of having been made by running the Program). 77 | Whether that is true depends on what the Program does. 78 | 79 | 1. You may copy and distribute verbatim copies of the Program's 80 | source code as you receive it, in any medium, provided that you 81 | conspicuously and appropriately publish on each copy an appropriate 82 | copyright notice and disclaimer of warranty; keep intact all the 83 | notices that refer to this License and to the absence of any warranty; 84 | and give any other recipients of the Program a copy of this License 85 | along with the Program. 86 | 87 | You may charge a fee for the physical act of transferring a copy, and 88 | you may at your option offer warranty protection in exchange for a fee. 89 | 90 | 2. You may modify your copy or copies of the Program or any portion 91 | of it, thus forming a work based on the Program, and copy and 92 | distribute such modifications or work under the terms of Section 1 93 | above, provided that you also meet all of these conditions: 94 | 95 | a) You must cause the modified files to carry prominent notices 96 | stating that you changed the files and the date of any change. 97 | 98 | b) You must cause any work that you distribute or publish, that in 99 | whole or in part contains or is derived from the Program or any 100 | part thereof, to be licensed as a whole at no charge to all third 101 | parties under the terms of this License. 102 | 103 | c) If the modified program normally reads commands interactively 104 | when run, you must cause it, when started running for such 105 | interactive use in the most ordinary way, to print or display an 106 | announcement including an appropriate copyright notice and a 107 | notice that there is no warranty (or else, saying that you provide 108 | a warranty) and that users may redistribute the program under 109 | these conditions, and telling the user how to view a copy of this 110 | License. (Exception: if the Program itself is interactive but 111 | does not normally print such an announcement, your work based on 112 | the Program is not required to print an announcement.) 113 | 114 | These requirements apply to the modified work as a whole. If 115 | identifiable sections of that work are not derived from the Program, 116 | and can be reasonably considered independent and separate works in 117 | themselves, then this License, and its terms, do not apply to those 118 | sections when you distribute them as separate works. But when you 119 | distribute the same sections as part of a whole which is a work based 120 | on the Program, the distribution of the whole must be on the terms of 121 | this License, whose permissions for other licensees extend to the 122 | entire whole, and thus to each and every part regardless of who wrote it. 123 | 124 | Thus, it is not the intent of this section to claim rights or contest 125 | your rights to work written entirely by you; rather, the intent is to 126 | exercise the right to control the distribution of derivative or 127 | collective works based on the Program. 128 | 129 | In addition, mere aggregation of another work not based on the Program 130 | with the Program (or with a work based on the Program) on a volume of 131 | a storage or distribution medium does not bring the other work under 132 | the scope of this License. 133 | 134 | 3. You may copy and distribute the Program (or a work based on it, 135 | under Section 2) in object code or executable form under the terms of 136 | Sections 1 and 2 above provided that you also do one of the following: 137 | 138 | a) Accompany it with the complete corresponding machine-readable 139 | source code, which must be distributed under the terms of Sections 140 | 1 and 2 above on a medium customarily used for software interchange; or, 141 | 142 | b) Accompany it with a written offer, valid for at least three 143 | years, to give any third party, for a charge no more than your 144 | cost of physically performing source distribution, a complete 145 | machine-readable copy of the corresponding source code, to be 146 | distributed under the terms of Sections 1 and 2 above on a medium 147 | customarily used for software interchange; or, 148 | 149 | c) Accompany it with the information you received as to the offer 150 | to distribute corresponding source code. (This alternative is 151 | allowed only for noncommercial distribution and only if you 152 | received the program in object code or executable form with such 153 | an offer, in accord with Subsection b above.) 154 | 155 | The source code for a work means the preferred form of the work for 156 | making modifications to it. For an executable work, complete source 157 | code means all the source code for all modules it contains, plus any 158 | associated interface definition files, plus the scripts used to 159 | control compilation and installation of the executable. However, as a 160 | special exception, the source code distributed need not include 161 | anything that is normally distributed (in either source or binary 162 | form) with the major components (compiler, kernel, and so on) of the 163 | operating system on which the executable runs, unless that component 164 | itself accompanies the executable. 165 | 166 | If distribution of executable or object code is made by offering 167 | access to copy from a designated place, then offering equivalent 168 | access to copy the source code from the same place counts as 169 | distribution of the source code, even though third parties are not 170 | compelled to copy the source along with the object code. 171 | 172 | 4. You may not copy, modify, sublicense, or distribute the Program 173 | except as expressly provided under this License. Any attempt 174 | otherwise to copy, modify, sublicense or distribute the Program is 175 | void, and will automatically terminate your rights under this License. 176 | However, parties who have received copies, or rights, from you under 177 | this License will not have their licenses terminated so long as such 178 | parties remain in full compliance. 179 | 180 | 5. You are not required to accept this License, since you have not 181 | signed it. However, nothing else grants you permission to modify or 182 | distribute the Program or its derivative works. These actions are 183 | prohibited by law if you do not accept this License. Therefore, by 184 | modifying or distributing the Program (or any work based on the 185 | Program), you indicate your acceptance of this License to do so, and 186 | all its terms and conditions for copying, distributing or modifying 187 | the Program or works based on it. 188 | 189 | 6. Each time you redistribute the Program (or any work based on the 190 | Program), the recipient automatically receives a license from the 191 | original licensor to copy, distribute or modify the Program subject to 192 | these terms and conditions. You may not impose any further 193 | restrictions on the recipients' exercise of the rights granted herein. 194 | You are not responsible for enforcing compliance by third parties to 195 | this License. 196 | 197 | 7. If, as a consequence of a court judgment or allegation of patent 198 | infringement or for any other reason (not limited to patent issues), 199 | conditions are imposed on you (whether by court order, agreement or 200 | otherwise) that contradict the conditions of this License, they do not 201 | excuse you from the conditions of this License. If you cannot 202 | distribute so as to satisfy simultaneously your obligations under this 203 | License and any other pertinent obligations, then as a consequence you 204 | may not distribute the Program at all. For example, if a patent 205 | license would not permit royalty-free redistribution of the Program by 206 | all those who receive copies directly or indirectly through you, then 207 | the only way you could satisfy both it and this License would be to 208 | refrain entirely from distribution of the Program. 209 | 210 | If any portion of this section is held invalid or unenforceable under 211 | any particular circumstance, the balance of the section is intended to 212 | apply and the section as a whole is intended to apply in other 213 | circumstances. 214 | 215 | It is not the purpose of this section to induce you to infringe any 216 | patents or other property right claims or to contest validity of any 217 | such claims; this section has the sole purpose of protecting the 218 | integrity of the free software distribution system, which is 219 | implemented by public license practices. Many people have made 220 | generous contributions to the wide range of software distributed 221 | through that system in reliance on consistent application of that 222 | system; it is up to the author/donor to decide if he or she is willing 223 | to distribute software through any other system and a licensee cannot 224 | impose that choice. 225 | 226 | This section is intended to make thoroughly clear what is believed to 227 | be a consequence of the rest of this License. 228 | 229 | 8. If the distribution and/or use of the Program is restricted in 230 | certain countries either by patents or by copyrighted interfaces, the 231 | original copyright holder who places the Program under this License 232 | may add an explicit geographical distribution limitation excluding 233 | those countries, so that distribution is permitted only in or among 234 | countries not thus excluded. In such case, this License incorporates 235 | the limitation as if written in the body of this License. 236 | 237 | 9. The Free Software Foundation may publish revised and/or new versions 238 | of the General Public License from time to time. Such new versions will 239 | be similar in spirit to the present version, but may differ in detail to 240 | address new problems or concerns. 241 | 242 | Each version is given a distinguishing version number. If the Program 243 | specifies a version number of this License which applies to it and "any 244 | later version", you have the option of following the terms and conditions 245 | either of that version or of any later version published by the Free 246 | Software Foundation. If the Program does not specify a version number of 247 | this License, you may choose any version ever published by the Free Software 248 | Foundation. 249 | 250 | 10. If you wish to incorporate parts of the Program into other free 251 | programs whose distribution conditions are different, write to the author 252 | to ask for permission. For software which is copyrighted by the Free 253 | Software Foundation, write to the Free Software Foundation; we sometimes 254 | make exceptions for this. Our decision will be guided by the two goals 255 | of preserving the free status of all derivatives of our free software and 256 | of promoting the sharing and reuse of software generally. 257 | 258 | NO WARRANTY 259 | 260 | 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY 261 | FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN 262 | OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES 263 | PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED 264 | OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 265 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS 266 | TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE 267 | PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, 268 | REPAIR OR CORRECTION. 269 | 270 | 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 271 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR 272 | REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, 273 | INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING 274 | OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED 275 | TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY 276 | YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER 277 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE 278 | POSSIBILITY OF SUCH DAMAGES. 279 | 280 | END OF TERMS AND CONDITIONS 281 | 282 | How to Apply These Terms to Your New Programs 283 | 284 | If you develop a new program, and you want it to be of the greatest 285 | possible use to the public, the best way to achieve this is to make it 286 | free software which everyone can redistribute and change under these terms. 287 | 288 | To do so, attach the following notices to the program. It is safest 289 | to attach them to the start of each source file to most effectively 290 | convey the exclusion of warranty; and each file should have at least 291 | the "copyright" line and a pointer to where the full notice is found. 292 | 293 | 294 | Copyright (C) 295 | 296 | This program is free software; you can redistribute it and/or modify 297 | it under the terms of the GNU General Public License as published by 298 | the Free Software Foundation; either version 2 of the License, or 299 | (at your option) any later version. 300 | 301 | This program is distributed in the hope that it will be useful, 302 | but WITHOUT ANY WARRANTY; without even the implied warranty of 303 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 304 | GNU General Public License for more details. 305 | 306 | You should have received a copy of the GNU General Public License along 307 | with this program; if not, write to the Free Software Foundation, Inc., 308 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 309 | 310 | Also add information on how to contact you by electronic and paper mail. 311 | 312 | If the program is interactive, make it output a short notice like this 313 | when it starts in an interactive mode: 314 | 315 | Gnomovision version 69, Copyright (C) year name of author 316 | Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 317 | This is free software, and you are welcome to redistribute it 318 | under certain conditions; type `show c' for details. 319 | 320 | The hypothetical commands `show w' and `show c' should show the appropriate 321 | parts of the General Public License. Of course, the commands you use may 322 | be called something other than `show w' and `show c'; they could even be 323 | mouse-clicks or menu items--whatever suits your program. 324 | 325 | You should also get your employer (if you work as a programmer) or your 326 | school, if any, to sign a "copyright disclaimer" for the program, if 327 | necessary. Here is a sample; alter the names: 328 | 329 | Yoyodyne, Inc., hereby disclaims all copyright interest in the program 330 | `Gnomovision' (which makes passes at compilers) written by James Hacker. 331 | 332 | , 1 April 1989 333 | Ty Coon, President of Vice 334 | 335 | This General Public License does not permit incorporating your program into 336 | proprietary programs. If your program is a subroutine library, you may 337 | consider it more useful to permit linking proprietary applications with the 338 | library. If this is what you want to do, use the GNU Lesser General 339 | Public License instead of this License. 340 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Go Report Card](https://goreportcard.com/badge/github.com/Necoro/feed2imap-go)](https://goreportcard.com/report/github.com/Necoro/feed2imap-go) 2 | 3 | # feed2imap-go 4 | 5 | A software to convert rss feeds into mails. feed2imap-go acts as an RSS/Atom feed aggregator. After downloading feeds 6 | (over HTTP or HTTPS), it uploads them to a specified folder of an IMAP mail server. The user can then access the feeds 7 | using their preferred client (Mutt, Evolution, Mozilla Thunderbird, webmail,...). 8 | 9 | It is a rewrite in Go of the wonderful, but unfortunately now unmaintained, [feed2imap](https://github.com/feed2imap/feed2imap). 10 | It also includes the features that up to now only lived on [my own branch][nec]. 11 | 12 | It aims to be compatible in functionality and configuration, and should mostly work as a drop-in replacement 13 | (but see [Changes](#changes)). 14 | 15 | An example configuration can be found [here](config.yml.example) with additional information in the [wiki](https://github.com/Necoro/feed2imap-go/wiki/Detailed-Options). 16 | 17 | See [the Installation section](#installation) on how to install feed2imap-go. (Spoiler: It's easy ;)). 18 | 19 | ## Features 20 | 21 | * Support for most feed formats. See [gofeed documentation](https://github.com/mmcdole/gofeed/blob/master/README.md#features) 22 | for details. Feeds need not be supplied via URL but can also be yielded by an executable. 23 | * Connection to any IMAP server, using IMAP, IMAP+STARTTLS, or IMAPS. 24 | * Detection of duplicates: Heuristics what feed items have already been uploaded. 25 | * Update mechanism: When a feed item is updated, so is the mail. 26 | * Detailed configuration options per feed (fetch frequency, should images be included, tune change heuristics, ...) 27 | * Support for custom filters on feeds 28 | * [Readability support][i67], i.e. fetching and presenting the linked content instead of the teaser/link included in the feed itself. 29 | 30 | ## Changes 31 | 32 | ### Additions to feed2imap 33 | 34 | * Groups: Have the ability to group feeds that share characteristics, most often the same parent folder in the hiearchy. 35 | It also allows sharing options between feeds. Usages: Categories ("News", "Linux") and merging different feeds of the same origin. 36 | * Heavier use of parallel processing (it's Go after all ;)). Also, it is way faster. 37 | * Global `target` and each feed only specifies the folder relative to that target. 38 | (feature contained also in [fork of the original][nec]) 39 | * Fix `include-images` option: It now includes images as mime-parts. An additional `embed-images` option serves the images 40 | as inline base64-encoded data (the old default behavior of feed2imap). 41 | * Improved image inclusion: Support any relative URLs, including `//example.com/foo.png` 42 | * Use an HTML parser instead of regular expressions for modifying the HTML content. 43 | * STARTTLS support. As it turned out only in testing, the old feed2imap never supported it... 44 | * `item-filter` option that allows to specify an inline filter expression on the items of a feed. 45 | * Readability support: Fetch and present the linked article. 46 | * Mail templates can be customized. 47 | 48 | ### Subtle differences 49 | 50 | * **Feed rendering**: Unfortunately, semantics of RSS and Atom tags are very broad. As we use a different feed parser 51 | ibrary than the original, the interpretation (e.g., what tag is "the author") can differ. 52 | * **Caching**: We do not implement the caching algorithm of feed2imap point by point. In general, we opted for fewer 53 | heuristics and more optimism (belief that GUID is filled correctly). If this results in a problem, file a bug and include the `X-Feed2Imap-Reason` header of the mail. 54 | * **Configuration**: We took the liberty to restructure the configuration options. Old configs are supported, but a 55 | warning is issued when an option should now be in another place or is no longer supported (i.e., the option is without function). 56 | 57 | ### Unsupported features of feed2imap 58 | 59 | * Maildir ([issue #4][i4]) 60 | * Different IMAP servers in the same configuration file. Please use multiple config files, if this is needed (see also [issue #6][i6]). 61 | 62 | ## Installation 63 | 64 | The easiest way of installation is to head over to [the releases page](https://github.com/Necoro/feed2imap-go/releases/latest) 65 | and get the appropriate download package. Go is all about static linking, thus for all platforms the result is a single 66 | binary which can be placed whereever you need. 67 | 68 | Please open an issue if you are missing your platform. 69 | 70 | ### Use your package manager 71 | 72 | #### Arch Linux 73 | 74 | feed2imap-go is present in the [AUR](https://aur.archlinux.org/packages/feed2imap-go). 75 | 76 | #### Nix 77 | 78 | feed2imap-go is present in [nixpkgs](https://github.com/NixOS/nixpkgs/tree/master/pkgs/applications/networking/feedreaders/feed2imap-go). 79 | 80 | ### Install from source 81 | 82 | Clone the repository and, optionally, switch to the tag you want: 83 | ````bash 84 | git clone https://github.com/Necoro/feed2imap-go 85 | git checkout v1.7.2 86 | ```` 87 | 88 | The official way of building feed2imap-go is using [goreleaser](https://github.com/goreleaser/goreleaser): 89 | ````bash 90 | goreleaser build --single-target --snapshot --clean 91 | ```` 92 | The built binary is then inside the corresponding arch folder in `dist`. 93 | 94 | In case you do not want to install yet another build tool, doing 95 | ````bash 96 | go build 97 | ```` 98 | should also suffice, but does not embed version information in the binary (and the result is slightly larger). 99 | 100 | If you are only interested in getting the latest build out of the `master` branch, do 101 | ````bash 102 | go install github.com/Necoro/feed2imap-go@master 103 | ```` 104 | Using `@latest` instead of `@master` gives you the latest stable version. 105 | 106 | ### Run in docker 107 | 108 | Most times, putting feed2imap-go somewhere and adding a cron job does everything you need. For the times when it isn't, we provide docker containers for your convenience at [Github Packages](https://github.com/Necoro/feed2imap-go/packages) and at [Docker Hub](https://hub.docker.com/r/necorodm/feed2imap-go). 109 | 110 | The container is configured to expect both config file and cache under `/app/data/`, thus needs it mounted there. 111 | When both are stored in `~/feed`, you can do: 112 | ````bash 113 | docker run -v ~/feed:/app/data necorodm/feed2imap-go:latest 114 | ```` 115 | 116 | Alternatively, build the docker image yourself (requires the `feed2imap-go` binary at toplevel): 117 | ````bash 118 | docker build -t feed2imap-go . 119 | ```` 120 | Note that the supplied binary must not be linked to glibc, i.e. has to be built with `CGO_ENABLED=0`. When using `goreleaser`, you'll find this in `dist/docker_linux_amd64`. 121 | 122 | Or you can roll your own Dockerfile, supplying a glibc... 123 | 124 | **NB**: feed2imap-go employs no server-mode. Thus, each run terminates directly after a couple seconds. Therefore, the docker container in itself is not that useful, and you have to have a mechanism in place to spin up the container regularly. 125 | 126 | [i6]: https://github.com/Necoro/feed2imap-go/issues/6 127 | [i4]: https://github.com/Necoro/feed2imap-go/issues/4 128 | [i9]: https://github.com/Necoro/feed2imap-go/issues/9 129 | [i67]: https://github.com/Necoro/feed2imap-go/issues/67 130 | [nec]: https://github.com/Necoro/feed2imap 131 | [jb]: https://www.jetbrains.com/?from=feed2imap-go 132 | -------------------------------------------------------------------------------- /config.yml.example: -------------------------------------------------------------------------------- 1 | # Example configuration. Each configuration option presented shows its default, so sensible. 2 | # NB: THIS FILE *WILL* INCLUDE IMAP CREDENTIALS. ENSURE SENSIBLE ACCESS RIGHTS! 3 | 4 | ## Target 5 | target: 6 | # scheme: either imap or imaps; if omitted, it is deduced from the port 7 | scheme: imap 8 | # user 9 | user: test@example.com 10 | # password 11 | password: passw0rd 12 | # host, without the port 13 | host: mail.example.com 14 | # port; optional if scheme is given 15 | port: 143 16 | # root denotes the root of the hierarchy for all feeds. Probably should start with INBOX, but may also be empty. 17 | # The allowed delimiter in the path is either '/' as a default, or the one used by your mailserver, if so known (for 18 | # example: '.'). 19 | root: INBOX/Feeds 20 | # Instead of the verbose target, specifiying a URI is also legitimate. Be sure to properly url-encode user and password. 21 | # The example from above would read: 22 | # target: imap://test%40example.com:passw0rd@mail.example.com:143/INBOX/Feeds 23 | 24 | # NB: Instead of specifying the target on the global level, for compatibility with old configurations, it is also allowed 25 | # to specify the full URL string for each feed. 26 | # When doing so, _all_ target strings must point to the same server, else an error will be thrown. 27 | # See https://github.com/Necoro/feed2imap-go/issues/6 for more details. 28 | 29 | ## Global Options 30 | # Location of the cache. Can also be overwritten on the command line. 31 | cache: "feed.cache" 32 | # Timeout in seconds for fetching feeds. 33 | timeout: 30 34 | # Maximum number of failures allowed before they are reported in normal mode. 35 | # By default, failures are only visible in verbose mode. Most feeds tend to suffer from temporary failures. 36 | max-failures: 10 37 | # Maximum number of concurrent IMAP connections opened. 38 | max-imap-connections: 5 39 | # Parts to generate in the resulting emails. 40 | # Valid parts are "text" and "html" 41 | parts: ["text", "html"] 42 | # Overwrite the default template for text/html. 43 | # See https://github.com/Necoro/feed2imap-go/wiki/Detailed-Options for more information. 44 | html-template: html.tpl 45 | text-template: text.tpl 46 | # Email-Address to use in 'From' and 'To' when not specified in feed. 47 | # Default uses 'current user'@hostname 48 | default-email: username@hostname 49 | # Whether the target folder determination for each feed shall fallback on the feed's/group's name. 50 | # When `true`, omitting a target is identical to specifying the name as the target. 51 | # When `false`, omitting a target is identical to specifying `null` or `""` as the target. 52 | auto-target: true 53 | 54 | ## Per-Feed Options 55 | # Defaults for options per feed, overridable in each feed and group 56 | # NB: For compatibility with old feed2imap, options existing in feed2imap can also be specified at the toplevel, 57 | # i.e. not beneath 'options'. Triggers a warning though :) 58 | options: 59 | # Frequency in hours for checking. 0 = on each run. 60 | min-frequency: 0 61 | # Include images referenced in the item per URL in the mail. 62 | # For instance, when a feed item includes , this image is fetched 63 | # and included in the mail. 64 | include-images: true 65 | # By default, images are added as an additional part to the email and referred to in the HTML part. 66 | # If you, for some reason, prefer the images to be directly encoded in the HTML part, set this option to true. 67 | # Without function when `include-images` is false. 68 | embed-images: false 69 | # Specify what type of a feed item determines the message's body. 70 | # Values: 71 | # - default: default heuristics 72 | # - content: Use the 'content' tag 73 | # - description: Use the 'description' tag 74 | # - both: Use both 75 | # - fetch: Ignore the body delivered by the feed and instead fetch the linked website. 76 | # It may be advisable to set `include-images` to false in that mode to avoid unexpected large mails. 77 | body: default 78 | # Disable a feed. Beats commenting ;) 79 | disable: false 80 | # Disable certificate verification for HTTPS connections. 81 | # This is sometimes needed, when a site delivers broken certificate (chains). 82 | tls-no-verify: false 83 | # Some feeds change the content of their items all the time, so we detect that they have been updated at each run. 84 | # When this option is enabled, the content of an item is ignored when determining whether this item is already known. 85 | ignore-hash: false 86 | # We employ a clever algorithm to determine whether an item is new or has been updated. This does not always work 87 | # perfectly. When this flag is enabled, all items which don't match exactly any previously downloaded item are 88 | # considered as new items. 89 | always-new: false 90 | # If an item is updated, but has been deleted on the server already, it is re-uploaded when this option is true. 91 | # Else it is ignored. 92 | reupload-if-updated: false 93 | # Items of a feed may be filtered. In general there is no real use in specifying this globally. 94 | # For full information about this feature, visit https://github.com/Necoro/feed2imap-go/wiki/Detailed-Options. 95 | item-filter: 'Author.Name != "Weirdo"' 96 | 97 | ## Feeds 98 | # Each feed must have a name, and a URL or Exec argument. The name must be unique. 99 | # The name also determines the folder to use for that feed, which can be overwritten with an explicit target. 100 | # This behavior can be changed by toggling the global option `auto-target` (see above). 101 | # Groups can be used to build a hierarchy, with arbitrary nesting. 102 | feeds: 103 | - name: XKCD 104 | url: http://xkcd.com/rss.xml 105 | # specify any per feed option to overwrite it for this feed 106 | min-frequency: 12 107 | # No target has been defined, so it falls back onto the feed's name. 108 | # Combined with the global `target`, the final folder will be: 109 | # INBOX/Feeds/XKCD 110 | # Would the global option `auto-target` been set to `false`, this fallback would not occur, 111 | # and the final folder would be: 112 | # INBOX/Feeds 113 | # Groups can be used for, well, grouping. 114 | - group: Linux 115 | # You can specify options on group level that are then used for all feeds contained 116 | min-frequency: 6 117 | feeds: 118 | - name: Arch Linux 119 | # Use `target` to specify the folder name. 120 | # Together with the group folder this now spells 'Linux/Arch'. 121 | # Considering the global `target` the final folder will be: 122 | # INBOX/Feeds/Linux/Arch 123 | target: Arch 124 | # Use `exec` instead of `url` when fetching is not enough and script magic is needed. 125 | # See https://github.com/Necoro/feed2imap-go/wiki/Detailed-Options for details. 126 | exec: ["wget", "https://www.archlinux.org/feeds/news/", "-O", "-"] 127 | # Groups can be nested... 128 | - group: Gentoo 129 | # and also specify a target (which is superfluous here, because it is identical to the group name) 130 | target: Gentoo 131 | feeds: 132 | - name: Planet Gentoo 133 | # An empty target omits the creation of a folder for this feed and uses the one from the level above. 134 | # Thus "Planet Gentoo" and "Gentoo News" will finally reside in 'Linux/Gentoo' 135 | target: 136 | url: https://planet.gentoo.org/atom.xml 137 | min-frequency: 24 138 | - name: Gentoo News 139 | target: 140 | url: https://gentoo.org/feeds/news.xml 141 | min-frequency: 24 142 | - group: News 143 | feeds: 144 | - name: Heise 145 | url: http://www.heise.de/newsticker/heise-atom.xml 146 | ignore-hash: true 147 | - name: Spiegel 148 | url: http://www.spiegel.de/schlagzeilen/index.rss 149 | - group: Süddeutsche 150 | target: SZ 151 | feeds: 152 | - name: Bayern 153 | url: http://rssfeed.sueddeutsche.de/c/795/f/448243/index.rss 154 | target: 155 | - name: München 156 | url: http://rssfeed.sueddeutsche.de/c/795/f/448324/index.rss 157 | target: 158 | - group: ZEIT Online 159 | target: Zeit 160 | feeds: 161 | - name: Digital 162 | url: http://newsfeed.zeit.de/digital/index 163 | target: 164 | 165 | # vim: ft=yaml:sts=2:expandtab 166 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/Necoro/feed2imap-go 2 | 3 | go 1.24 4 | 5 | toolchain go1.24.3 6 | 7 | require ( 8 | github.com/Necoro/gofeed v1.2.2-0.20240710191639-dd0fdcc70756 9 | github.com/Necoro/html2text v0.0.0-20250130175213-09177e7b4534 10 | github.com/PuerkitoBio/goquery v1.10.3 11 | github.com/emersion/go-imap v1.2.1 12 | github.com/emersion/go-imap-uidplus v0.0.0-20200503180755-e75854c361e9 13 | github.com/emersion/go-message v0.18.2 14 | github.com/expr-lang/expr v1.17.4 15 | github.com/gabriel-vasile/mimetype v1.4.9 16 | github.com/go-shiori/go-readability v0.0.0-20250217085726-9f5bf5ca7612 17 | github.com/google/go-cmp v0.6.0 18 | github.com/google/uuid v1.6.0 19 | github.com/mitchellh/mapstructure v1.5.0 20 | github.com/nightlyone/lockfile v1.0.0 21 | golang.org/x/net v0.40.0 22 | gopkg.in/yaml.v3 v3.0.1 23 | ) 24 | 25 | require ( 26 | github.com/andybalholm/cascadia v1.3.3 // indirect 27 | github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de // indirect 28 | github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 // indirect 29 | github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c // indirect 30 | github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect 31 | github.com/json-iterator/go v1.1.12 // indirect 32 | github.com/mattn/go-runewidth v0.0.16 // indirect 33 | github.com/mmcdole/goxpp v1.1.1 // indirect 34 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 35 | github.com/modern-go/reflect2 v1.0.2 // indirect 36 | github.com/olekukonko/tablewriter v0.0.5 // indirect 37 | github.com/rivo/uniseg v0.4.7 // indirect 38 | github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect 39 | golang.org/x/text v0.25.0 // indirect 40 | ) 41 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/Necoro/gofeed v1.2.2-0.20240710191639-dd0fdcc70756 h1:urWpnwBPDmeXQNqi2L/qQq+sTCvQKN3dlCkj3eh8ebE= 2 | github.com/Necoro/gofeed v1.2.2-0.20240710191639-dd0fdcc70756/go.mod h1:CNNjbGuYoEU5UWnzd8+59gWF1lsL/sZpLId0/xDVwCY= 3 | github.com/Necoro/html2text v0.0.0-20250130175213-09177e7b4534 h1:FKv9tkuFxs92RHSvsRRrZcGNOiJd9j3Lg/nbXltu7Cc= 4 | github.com/Necoro/html2text v0.0.0-20250130175213-09177e7b4534/go.mod h1:2H7CH1htfqSXRNUdqd+jAfwDmpKYaRSg59iJfb5yWRk= 5 | github.com/PuerkitoBio/goquery v1.10.3 h1:pFYcNSqHxBD06Fpj/KsbStFRsgRATgnf3LeXiUkhzPo= 6 | github.com/PuerkitoBio/goquery v1.10.3/go.mod h1:tMUX0zDMHXYlAQk6p35XxQMqMweEKB7iK7iLNd4RH4Y= 7 | github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM= 8 | github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= 9 | github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhPwqqXc4/vE0f7GvRjuAsbW+HOIe8KnA= 10 | github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw= 11 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 12 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 13 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 14 | github.com/emersion/go-imap v1.2.1 h1:+s9ZjMEjOB8NzZMVTM3cCenz2JrQIGGo5j1df19WjTA= 15 | github.com/emersion/go-imap v1.2.1/go.mod h1:Qlx1FSx2FTxjnjWpIlVNEuX+ylerZQNFE5NsmKFSejY= 16 | github.com/emersion/go-imap-uidplus v0.0.0-20200503180755-e75854c361e9 h1:2Kbw3iu7fFeSso6RWIArVNUj1VGG2PvjetnPUW7bnis= 17 | github.com/emersion/go-imap-uidplus v0.0.0-20200503180755-e75854c361e9/go.mod h1:GfiSiw/du0221I3Cf4F0DqX3Bv5Xe580gIIATrQtnJg= 18 | github.com/emersion/go-message v0.15.0/go.mod h1:wQUEfE+38+7EW8p8aZ96ptg6bAb1iwdgej19uXASlE4= 19 | github.com/emersion/go-message v0.18.2 h1:rl55SQdjd9oJcIoQNhubD2Acs1E6IzlZISRTK7x/Lpg= 20 | github.com/emersion/go-message v0.18.2/go.mod h1:XpJyL70LwRvq2a8rVbHXikPgKj8+aI0kGdHlg16ibYA= 21 | github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21/go.mod h1:iL2twTeMvZnrg54ZoPDNfJaJaqy0xIQFuBdrLsmspwQ= 22 | github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 h1:oP4q0fw+fOSWn3DfFi4EXdT+B+gTtzx8GC9xsc26Znk= 23 | github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6/go.mod h1:iL2twTeMvZnrg54ZoPDNfJaJaqy0xIQFuBdrLsmspwQ= 24 | github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594/go.mod h1:aqO8z8wPrjkscevZJFVE1wXJrLpC5LtJG7fqLOsPb2U= 25 | github.com/expr-lang/expr v1.17.4 h1:qhTVftZ2Z3WpOEXRHWErEl2xf1Kq011MnQmWgLq06CY= 26 | github.com/expr-lang/expr v1.17.4/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4= 27 | github.com/gabriel-vasile/mimetype v1.4.9 h1:5k+WDwEsD9eTLL8Tz3L0VnmVh9QxGjRmjBvAG7U/oYY= 28 | github.com/gabriel-vasile/mimetype v1.4.9/go.mod h1:WnSQhFKJuBlRyLiKohA/2DtIlPFAbguNaG7QCHcyGok= 29 | github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c h1:wpkoddUomPfHiOziHZixGO5ZBS73cKqVzZipfrLmO1w= 30 | github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c/go.mod h1:oVDCh3qjJMLVUSILBRwrm+Bc6RNXGZYtoh9xdvf1ffM= 31 | github.com/go-shiori/go-readability v0.0.0-20250217085726-9f5bf5ca7612 h1:BYLNYdZaepitbZreRIa9xeCQZocWmy/wj4cGIH0qyw0= 32 | github.com/go-shiori/go-readability v0.0.0-20250217085726-9f5bf5ca7612/go.mod h1:wgqthQa8SAYs0yyljVeCOQlZ027VW5CmLsbi9jWC08c= 33 | github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f h1:3BSP1Tbs2djlpprl7wCLuiqMaUh5SJkkzI2gDs+FgLs= 34 | github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f/go.mod h1:Pcatq5tYkCW2Q6yrR2VRHlbHpZ/R4/7qyL1TCF7vl14= 35 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 36 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 37 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 38 | github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= 39 | github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 40 | github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= 41 | github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= 42 | github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= 43 | github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= 44 | github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= 45 | github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= 46 | github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= 47 | github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= 48 | github.com/mmcdole/goxpp v1.1.1 h1:RGIX+D6iQRIunGHrKqnA2+700XMCnNv0bAOOv5MUhx8= 49 | github.com/mmcdole/goxpp v1.1.1/go.mod h1:v+25+lT2ViuQ7mVxcncQ8ch1URund48oH+jhjiwEgS8= 50 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 51 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 52 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 53 | github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= 54 | github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= 55 | github.com/nightlyone/lockfile v1.0.0 h1:RHep2cFKK4PonZJDdEl4GmkabuhbsRMgk/k3uAmxBiA= 56 | github.com/nightlyone/lockfile v1.0.0/go.mod h1:rywoIealpdNse2r832aiD9jRk8ErCatROs6LzC841CI= 57 | github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= 58 | github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= 59 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 60 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 61 | github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= 62 | github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= 63 | github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= 64 | github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= 65 | github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg= 66 | github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= 67 | github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= 68 | github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf h1:pvbZ0lM0XWPBqUKqFU8cmavspvIl9nulOYwdy6IFRRo= 69 | github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf/go.mod h1:RJID2RhlZKId02nZ62WenDCkgHFerpIOmW0iT7GKmXM= 70 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 71 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 72 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 73 | github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= 74 | github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= 75 | github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= 76 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 77 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 78 | golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= 79 | golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= 80 | golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= 81 | golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= 82 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= 83 | golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= 84 | golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= 85 | golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= 86 | golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= 87 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 88 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 89 | golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= 90 | golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= 91 | golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= 92 | golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= 93 | golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= 94 | golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= 95 | golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= 96 | golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY= 97 | golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= 98 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 99 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 100 | golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 101 | golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= 102 | golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= 103 | golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= 104 | golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= 105 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 106 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 107 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 108 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 109 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 110 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 111 | golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 112 | golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 113 | golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 114 | golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 115 | golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 116 | golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= 117 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 118 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 119 | golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= 120 | golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= 121 | golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= 122 | golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= 123 | golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= 124 | golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= 125 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 126 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 127 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 128 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 129 | golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= 130 | golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= 131 | golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= 132 | golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= 133 | golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= 134 | golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= 135 | golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= 136 | golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= 137 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 138 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 139 | golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= 140 | golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= 141 | golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= 142 | golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= 143 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 144 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 145 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 146 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 147 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 148 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 149 | -------------------------------------------------------------------------------- /internal/feed/cache/cache.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "bufio" 5 | "errors" 6 | "fmt" 7 | "io" 8 | "os" 9 | "path/filepath" 10 | "time" 11 | 12 | "github.com/nightlyone/lockfile" 13 | 14 | "github.com/Necoro/feed2imap-go/internal/feed" 15 | "github.com/Necoro/feed2imap-go/pkg/log" 16 | ) 17 | 18 | type Version byte 19 | 20 | const ( 21 | currentVersion Version = v2Version 22 | ) 23 | 24 | type Impl interface { 25 | cachedFeed(*feed.Feed) CachedFeed 26 | transformTo(Version) (Impl, error) 27 | cleanup(knownDescriptors map[feed.Descriptor]struct{}) 28 | load(io.Reader) error 29 | store(io.Writer) error 30 | Version() Version 31 | Info() string 32 | SpecificInfo(any) string 33 | } 34 | 35 | type Cache struct { 36 | Impl 37 | lock lockfile.Lockfile 38 | locked bool 39 | } 40 | 41 | type CachedFeed interface { 42 | // Checked marks the feed as being a failure or a success on last check. 43 | Checked(withFailure bool) 44 | // Failures of this feed up to now. 45 | Failures() int 46 | // The Last time, this feed has been checked 47 | Last() time.Time 48 | // Filter the given items against the cached items. 49 | Filter(items []feed.Item, ignoreHash bool, alwaysNew bool) []feed.Item 50 | // Commit any changes done to the cache state. 51 | Commit() 52 | // The Feed, that is cached. 53 | Feed() *feed.Feed 54 | } 55 | 56 | func forVersion(version Version) (Impl, error) { 57 | switch version { 58 | case v1Version: 59 | return newV1Cache(), nil 60 | case v2Version: 61 | return newV2Cache(), nil 62 | default: 63 | return nil, fmt.Errorf("unknown cache version '%d'", version) 64 | } 65 | } 66 | 67 | func lockName(fileName string) (string, error) { 68 | return filepath.Abs(fileName + ".lck") 69 | } 70 | 71 | func lock(fileName string) (lock lockfile.Lockfile, err error) { 72 | var lockFile string 73 | 74 | if lockFile, err = lockName(fileName); err != nil { 75 | return 76 | } 77 | log.Debugf("Handling lock file '%s'", lockFile) 78 | 79 | if lock, err = lockfile.New(lockFile); err != nil { 80 | err = fmt.Errorf("Creating lock file: %w", err) 81 | return 82 | } 83 | 84 | if err = lock.TryLock(); err != nil { 85 | err = fmt.Errorf("Locking cache: %w", err) 86 | return 87 | } 88 | 89 | return 90 | } 91 | 92 | func (cache *Cache) store(fileName string) error { 93 | if cache.Impl == nil { 94 | return fmt.Errorf("trying to store nil cache") 95 | } 96 | if cache.Version() != currentVersion { 97 | return fmt.Errorf("trying to store cache with unsupported version '%d' (current: '%d')", cache.Version(), currentVersion) 98 | } 99 | 100 | f, err := os.Create(fileName) 101 | if err != nil { 102 | return fmt.Errorf("trying to store cache to '%s': %w", fileName, err) 103 | } 104 | defer f.Close() 105 | 106 | if err = os.Chmod(fileName, 0600); err != nil { 107 | return fmt.Errorf("changing access rights of '%s': %w", fileName, err) 108 | } 109 | 110 | writer := bufio.NewWriter(f) 111 | if err = writer.WriteByte(byte(currentVersion)); err != nil { 112 | return fmt.Errorf("writing to '%s': %w", fileName, err) 113 | } 114 | 115 | if err = cache.Impl.store(writer); err != nil { 116 | return fmt.Errorf("encoding cache: %w", err) 117 | } 118 | 119 | writer.Flush() 120 | log.Printf("Stored cache to '%s'.", fileName) 121 | 122 | return cache.Unlock() 123 | } 124 | 125 | func (cache *Cache) Unlock() error { 126 | if cache.locked { 127 | if err := cache.lock.Unlock(); err != nil { 128 | return fmt.Errorf("Unlocking cache: %w", err) 129 | } 130 | } 131 | cache.locked = false 132 | return nil 133 | } 134 | 135 | func create() (Cache, error) { 136 | cache, err := forVersion(currentVersion) 137 | if err != nil { 138 | return Cache{}, err 139 | } 140 | return Cache{ 141 | Impl: cache, 142 | locked: false, 143 | }, nil 144 | } 145 | 146 | func Load(fileName string, upgrade bool) (Cache, error) { 147 | f, err := os.Open(fileName) 148 | if err != nil { 149 | if errors.Is(err, os.ErrNotExist) { 150 | // no cache there yet -- make new 151 | return create() 152 | } 153 | return Cache{}, fmt.Errorf("opening cache at '%s': %w", fileName, err) 154 | } 155 | defer f.Close() 156 | 157 | lock, err := lock(fileName) 158 | if err != nil { 159 | return Cache{}, err 160 | } 161 | 162 | log.Printf("Loading cache from '%s'", fileName) 163 | 164 | reader := bufio.NewReader(f) 165 | version, err := reader.ReadByte() 166 | if err != nil { 167 | return Cache{}, fmt.Errorf("reading from '%s': %w", fileName, err) 168 | } 169 | 170 | cache, err := forVersion(Version(version)) 171 | if err != nil { 172 | return Cache{}, err 173 | } 174 | 175 | if err = cache.load(reader); err != nil { 176 | return Cache{}, fmt.Errorf("decoding for version '%d' from '%s': %w", version, fileName, err) 177 | } 178 | 179 | if upgrade && currentVersion != cache.Version() { 180 | if cache, err = cache.transformTo(currentVersion); err != nil { 181 | return Cache{}, fmt.Errorf("cannot transform from version %d to %d: %w", version, currentVersion, err) 182 | } 183 | 184 | log.Printf("Loaded cache (version %d), transformed to version %d.", version, currentVersion) 185 | } else { 186 | log.Printf("Loaded cache (version %d)", version) 187 | } 188 | 189 | return Cache{cache, lock, true}, nil 190 | } 191 | -------------------------------------------------------------------------------- /internal/feed/cache/state.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "sync" 5 | 6 | "github.com/Necoro/feed2imap-go/internal/feed" 7 | "github.com/Necoro/feed2imap-go/pkg/config" 8 | "github.com/Necoro/feed2imap-go/pkg/log" 9 | ) 10 | 11 | type State struct { 12 | feeds map[string]*feed.Feed 13 | cachedFeeds map[string]CachedFeed 14 | knownFeeds map[feed.Descriptor]struct{} 15 | cache Cache 16 | cfg *config.Config 17 | } 18 | 19 | func (state *State) Foreach(f func(CachedFeed)) { 20 | for _, feed := range state.cachedFeeds { 21 | f(feed) 22 | } 23 | } 24 | 25 | func (state *State) ForeachGo(goFunc func(CachedFeed)) { 26 | var wg sync.WaitGroup 27 | wg.Add(len(state.cachedFeeds)) 28 | 29 | f := func(feed CachedFeed, wg *sync.WaitGroup) { 30 | goFunc(feed) 31 | wg.Done() 32 | } 33 | 34 | for _, feed := range state.cachedFeeds { 35 | go f(feed, &wg) 36 | } 37 | wg.Wait() 38 | } 39 | 40 | func (state *State) LoadCache(fileName string, forceNew bool) error { 41 | var ( 42 | cache Cache 43 | err error 44 | ) 45 | 46 | if forceNew { 47 | cache, err = create() 48 | } else { 49 | cache, err = Load(fileName, true) 50 | } 51 | 52 | if err != nil { 53 | return err 54 | } 55 | state.cache = cache 56 | 57 | for name, feed := range state.feeds { 58 | state.cachedFeeds[name] = cache.cachedFeed(feed) 59 | state.knownFeeds[feed.Descriptor()] = struct{}{} 60 | } 61 | 62 | // state.feeds should not be used after loading the cache --> enforce a panic 63 | state.feeds = nil 64 | 65 | return nil 66 | } 67 | 68 | func (state *State) StoreCache(fileName string) error { 69 | state.cache.cleanup(state.knownFeeds) 70 | return state.cache.store(fileName) 71 | } 72 | 73 | func (state *State) UnlockCache() { 74 | _ = state.cache.Unlock() 75 | } 76 | 77 | func (state *State) Fetch() int { 78 | state.ForeachGo(handleFeed) 79 | 80 | ctr := 0 81 | for _, cf := range state.cachedFeeds { 82 | success := cf.Feed().FetchSuccessful() 83 | cf.Checked(!success) 84 | 85 | if success { 86 | ctr++ 87 | } 88 | } 89 | 90 | return ctr 91 | } 92 | 93 | func handleFeed(cf CachedFeed) { 94 | feed := cf.Feed() 95 | log.Printf("Fetching %s from %s", feed.Name, feed.Url) 96 | 97 | err := feed.Parse() 98 | if err != nil { 99 | if feed.Url == "" || cf.Failures() >= feed.Global.MaxFailures { 100 | log.Error(err) 101 | } else { 102 | log.Print(err) 103 | } 104 | } 105 | } 106 | 107 | func filterFeed(cf CachedFeed) { 108 | cf.Feed().Filter(cf.Filter) 109 | } 110 | 111 | func (state *State) Filter() { 112 | if log.IsDebug() { 113 | // single threaded for better output 114 | state.Foreach(filterFeed) 115 | } else { 116 | state.ForeachGo(filterFeed) 117 | } 118 | } 119 | 120 | func NewState(cfg *config.Config) (*State, error) { 121 | numFeeds := len(cfg.Feeds) 122 | state := State{ 123 | feeds: make(map[string]*feed.Feed, numFeeds), 124 | cachedFeeds: make(map[string]CachedFeed, numFeeds), 125 | knownFeeds: make(map[feed.Descriptor]struct{}, numFeeds), 126 | cache: Cache{}, // loaded later on 127 | cfg: cfg, 128 | } 129 | 130 | for name, parsedFeed := range cfg.Feeds { 131 | feed, err := feed.Create(parsedFeed, cfg.GlobalOptions) 132 | if err != nil { 133 | return nil, err 134 | } 135 | state.feeds[name] = feed 136 | } 137 | 138 | return &state, nil 139 | } 140 | 141 | func (state *State) RemoveUndue() { 142 | for name, feed := range state.cachedFeeds { 143 | if feed.Feed().Disable || !feed.Feed().NeedsUpdate(feed.Last()) { 144 | delete(state.cachedFeeds, name) 145 | } 146 | } 147 | } 148 | 149 | func (state *State) NumFeeds() int { 150 | return len(state.cachedFeeds) 151 | } 152 | -------------------------------------------------------------------------------- /internal/feed/cache/v1.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "crypto/sha256" 5 | "encoding/base64" 6 | "encoding/gob" 7 | "encoding/hex" 8 | "fmt" 9 | "io" 10 | "slices" 11 | "sort" 12 | "strconv" 13 | "strings" 14 | "time" 15 | 16 | "github.com/google/uuid" 17 | 18 | "github.com/Necoro/feed2imap-go/internal/feed" 19 | "github.com/Necoro/feed2imap-go/pkg/log" 20 | "github.com/Necoro/feed2imap-go/pkg/util" 21 | ) 22 | 23 | const ( 24 | v1Version Version = 1 25 | startFeedId uint64 = 1 26 | maxCacheSize = 1000 27 | maxCacheDays = 180 28 | ) 29 | 30 | type feedId uint64 31 | 32 | func (id feedId) String() string { 33 | return strconv.FormatUint(uint64(id), 16) 34 | } 35 | 36 | func idFromString(s string) feedId { 37 | id, _ := strconv.ParseUint(s, 16, 64) 38 | return feedId(id) 39 | } 40 | 41 | type v1Cache struct { 42 | Ids map[feed.Descriptor]feedId 43 | NextId uint64 44 | Feeds map[feedId]*cachedFeed 45 | } 46 | 47 | type cachedFeed struct { 48 | feed *feed.Feed 49 | id feedId // not saved, has to be set on loading 50 | LastCheck time.Time 51 | currentCheck time.Time 52 | NumFailures int // can't be named `Failures` b/c it'll collide with the interface 53 | Items []cachedItem 54 | newItems []cachedItem 55 | } 56 | 57 | type itemHash [sha256.Size]byte 58 | 59 | func (h itemHash) String() string { 60 | return hex.EncodeToString(h[:]) 61 | } 62 | 63 | type cachedItem struct { 64 | Guid string 65 | Title string 66 | Link string 67 | Date time.Time 68 | UpdatedCache time.Time 69 | Hash itemHash 70 | ID uuid.UUID 71 | deleted bool 72 | } 73 | 74 | func (item cachedItem) String() string { 75 | return fmt.Sprintf(`{ 76 | ID: %s 77 | Title: %q 78 | Guid: %q 79 | Link: %q 80 | Date: %s 81 | Hash: %s 82 | }`, 83 | base64.RawURLEncoding.EncodeToString(item.ID[:]), 84 | item.Title, item.Guid, item.Link, util.TimeFormat(item.Date), item.Hash) 85 | } 86 | 87 | func (cf *cachedFeed) Checked(withFailure bool) { 88 | cf.currentCheck = time.Now() 89 | if withFailure { 90 | cf.NumFailures++ 91 | } else { 92 | cf.NumFailures = 0 93 | } 94 | } 95 | 96 | func (cf *cachedFeed) Commit() { 97 | if cf.newItems != nil { 98 | cf.Items = cf.newItems 99 | cf.newItems = nil 100 | } 101 | cf.LastCheck = cf.currentCheck 102 | } 103 | 104 | func (cf *cachedFeed) Failures() int { 105 | return cf.NumFailures 106 | } 107 | 108 | func (cf *cachedFeed) Last() time.Time { 109 | return cf.LastCheck 110 | } 111 | 112 | func (cf *cachedFeed) Feed() *feed.Feed { 113 | return cf.feed 114 | } 115 | 116 | func (cache *v1Cache) Version() Version { 117 | return v1Version 118 | } 119 | 120 | func (cache *v1Cache) Info() string { 121 | descriptors := make([]feed.Descriptor, len(cache.Ids)) 122 | i := 0 123 | for descr := range cache.Ids { 124 | descriptors[i] = descr 125 | i++ 126 | } 127 | 128 | sort.Slice(descriptors, func(i, j int) bool { 129 | return descriptors[i].Name < descriptors[j].Name 130 | }) 131 | 132 | b := strings.Builder{} 133 | for _, descr := range descriptors { 134 | id := cache.Ids[descr] 135 | feed := cache.Feeds[id] 136 | b.WriteString(fmt.Sprintf("%3s: %s (%s) (%d items)\n", id.String(), descr.Name, descr.Url, len(feed.Items))) 137 | } 138 | return b.String() 139 | } 140 | 141 | func (cache *v1Cache) SpecificInfo(i any) string { 142 | id := idFromString(i.(string)) 143 | 144 | b := strings.Builder{} 145 | feed := cache.Feeds[id] 146 | 147 | for descr, fId := range cache.Ids { 148 | if id == fId { 149 | b.WriteString(descr.Name) 150 | b.WriteString(" -- ") 151 | b.WriteString(descr.Url) 152 | b.WriteByte('\n') 153 | break 154 | } 155 | } 156 | 157 | b.WriteString(fmt.Sprintf(` 158 | Last Check: %s 159 | Num Failures: %d 160 | Num Items: %d 161 | `, 162 | util.TimeFormat(feed.LastCheck), 163 | feed.NumFailures, 164 | len(feed.Items))) 165 | 166 | for _, item := range feed.Items { 167 | b.WriteString("\n--------------------\n") 168 | b.WriteString(item.String()) 169 | } 170 | return b.String() 171 | } 172 | 173 | func newV1Cache() *v1Cache { 174 | cache := v1Cache{ 175 | Ids: map[feed.Descriptor]feedId{}, 176 | Feeds: map[feedId]*cachedFeed{}, 177 | NextId: startFeedId, 178 | } 179 | return &cache 180 | } 181 | 182 | func (cache *v1Cache) transformTo(v Version) (Impl, error) { 183 | switch v { 184 | case v2Version: 185 | return (*v2Cache)(cache), nil 186 | default: 187 | return nil, fmt.Errorf("Transformation not supported") 188 | } 189 | } 190 | 191 | func (cache *v1Cache) getFeed(id feedId) *cachedFeed { 192 | feed, ok := cache.Feeds[id] 193 | if !ok { 194 | feed = &cachedFeed{} 195 | cache.Feeds[id] = feed 196 | } 197 | feed.id = id 198 | return feed 199 | } 200 | 201 | func (cache *v1Cache) cachedFeed(f *feed.Feed) CachedFeed { 202 | fDescr := f.Descriptor() 203 | id, ok := cache.Ids[fDescr] 204 | if !ok { 205 | var otherId feed.Descriptor 206 | changed := false 207 | for otherId, id = range cache.Ids { 208 | if otherId.Name == fDescr.Name { 209 | log.Warnf("Feed %s seems to have changed URLs: new '%s', old '%s'. Updating.", 210 | fDescr.Name, fDescr.Url, otherId.Url) 211 | changed = true 212 | break 213 | } else if otherId.Url == fDescr.Url { 214 | log.Warnf("Feed with URL '%s' seems to have changed its name: new '%s', old '%s'. Updating.", 215 | fDescr.Url, fDescr.Name, otherId.Name) 216 | changed = true 217 | break 218 | } 219 | } 220 | if changed { 221 | delete(cache.Ids, otherId) 222 | } else { 223 | id = feedId(cache.NextId) 224 | cache.NextId++ 225 | } 226 | 227 | cache.Ids[fDescr] = id 228 | } 229 | 230 | cf := cache.getFeed(id) 231 | cf.feed = f 232 | f.SetExtID(id) 233 | return cf 234 | } 235 | 236 | func (cf *cachedFeed) buildCachedItem(item *feed.Item) cachedItem { 237 | var ci cachedItem 238 | 239 | ci.ID = uuid.UUID(item.ID) 240 | ci.Title = item.Item.Title 241 | ci.Link = item.Item.Link 242 | if item.DateParsed() != nil { 243 | ci.Date = *item.DateParsed() 244 | } 245 | ci.Guid = item.Item.GUID 246 | 247 | contentByte := []byte(item.Item.Description + item.Item.Content) 248 | ci.Hash = sha256.Sum256(contentByte) 249 | 250 | return ci 251 | } 252 | 253 | func (item *cachedItem) similarTo(other *cachedItem, ignoreHash bool) bool { 254 | return other.Title == item.Title && 255 | other.Link == item.Link && 256 | other.Date.Equal(item.Date) && 257 | (ignoreHash || other.Hash == item.Hash) 258 | } 259 | 260 | func (cf *cachedFeed) markItemDeleted(index int) { 261 | cf.Items[index].deleted = true 262 | } 263 | 264 | func (cf *cachedFeed) Filter(items []feed.Item, ignoreHash, alwaysNew bool) []feed.Item { 265 | if len(items) == 0 { 266 | return items 267 | } 268 | 269 | cacheItems := make(map[cachedItem]*feed.Item, len(items)) 270 | for idx := range items { 271 | i := &items[idx] 272 | ci := cf.buildCachedItem(i) 273 | 274 | // remove complete duplicates on the go 275 | cacheItems[ci] = i 276 | } 277 | log.Debugf("%d items after deduplication", len(cacheItems)) 278 | 279 | filtered := make([]feed.Item, 0, len(items)) 280 | cacheadd := make([]cachedItem, 0, len(items)) 281 | app := func(item *feed.Item, ci cachedItem, oldIdx int) { 282 | if oldIdx > -1 { 283 | item.UpdateOnly = true 284 | prevId := cf.Items[oldIdx].ID 285 | ci.ID = prevId 286 | item.ID = feed.ItemID(prevId) 287 | log.Debugf("oldIdx: %d, prevId: %s, item.id: %s", oldIdx, prevId, item.Id()) 288 | cf.markItemDeleted(oldIdx) 289 | } 290 | filtered = append(filtered, *item) 291 | cacheadd = append(cacheadd, ci) 292 | } 293 | 294 | seen := func(oldIdx int) { 295 | ci := cf.Items[oldIdx] 296 | cf.markItemDeleted(oldIdx) 297 | cacheadd = append(cacheadd, ci) 298 | } 299 | 300 | CACHE_ITEMS: 301 | for ci, item := range cacheItems { 302 | log.Debugf("Now checking %s", ci) 303 | 304 | if ci.Guid != "" { 305 | for idx, oldItem := range cf.Items { 306 | if oldItem.Guid == ci.Guid { 307 | log.Debugf("Guid matches with: %s", oldItem) 308 | if !oldItem.similarTo(&ci, ignoreHash) { 309 | item.AddReason("guid (upd)") 310 | app(item, ci, idx) 311 | } else { 312 | log.Debugf("Similar, ignoring item %s", base64.RawURLEncoding.EncodeToString(oldItem.ID[:])) 313 | seen(idx) 314 | } 315 | 316 | continue CACHE_ITEMS 317 | } 318 | } 319 | 320 | log.Debug("Found no matching GUID, including.") 321 | item.AddReason("guid") 322 | app(item, ci, -1) 323 | continue 324 | } 325 | 326 | for idx, oldItem := range cf.Items { 327 | if oldItem.similarTo(&ci, ignoreHash) { 328 | log.Debugf("Similarity matches, ignoring: %s", oldItem) 329 | seen(idx) 330 | continue CACHE_ITEMS 331 | } 332 | 333 | if oldItem.Link == ci.Link { 334 | if alwaysNew { 335 | log.Debugf("Link matches, but `always-new`.") 336 | item.AddReason("always-new") 337 | continue 338 | } 339 | log.Debugf("Link matches, updating: %s", oldItem) 340 | item.AddReason("link (upd)") 341 | app(item, ci, idx) 342 | 343 | continue CACHE_ITEMS 344 | } 345 | } 346 | 347 | log.Debugf("No match found, inserting.") 348 | item.AddReason("new") 349 | app(item, ci, -1) 350 | } 351 | 352 | log.Debugf("%d items after filtering", len(filtered)) 353 | 354 | // only the old items (cf.Items) is filtered and trimmed 355 | // this is to ensure that really all new additions are part of the cache 356 | cf.newItems = slices.Concat(cacheadd, filterItems(cf.Items)) 357 | 358 | return filtered 359 | } 360 | 361 | func filterItems(items []cachedItem) []cachedItem { 362 | n := min(len(items), maxCacheSize) 363 | 364 | copiedItems := make([]cachedItem, 0, n) 365 | for _, item := range items { 366 | if !item.deleted { 367 | copiedItems = append(copiedItems, item) 368 | if len(copiedItems) >= n { 369 | break 370 | } 371 | } 372 | } 373 | 374 | return copiedItems 375 | } 376 | 377 | func (cache *v1Cache) cleanup(knownDescriptors map[feed.Descriptor]struct{}) { 378 | for descr, id := range cache.Ids { 379 | if _, ok := knownDescriptors[descr]; ok { 380 | // do not delete stuff still known to us 381 | continue 382 | } 383 | 384 | cf := cache.Feeds[id] 385 | if cf.LastCheck.IsZero() || util.Days(time.Since(cf.LastCheck)) > maxCacheDays { 386 | delete(cache.Feeds, id) 387 | delete(cache.Ids, descr) 388 | } 389 | } 390 | } 391 | 392 | func (cache *v1Cache) load(reader io.Reader) error { 393 | decoder := gob.NewDecoder(reader) 394 | return decoder.Decode(cache) 395 | } 396 | 397 | func (cache *v1Cache) store(writer io.Writer) error { 398 | encoder := gob.NewEncoder(writer) 399 | return encoder.Encode(cache) 400 | } 401 | -------------------------------------------------------------------------------- /internal/feed/cache/v2.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "compress/gzip" 5 | "fmt" 6 | "io" 7 | 8 | "github.com/Necoro/feed2imap-go/internal/feed" 9 | ) 10 | 11 | const v2Version Version = 2 12 | 13 | // v2Cache is identical to v1Cache, but uses gzip compression for storage 14 | type v2Cache v1Cache 15 | 16 | func newV2Cache() *v2Cache { 17 | return (*v2Cache)(newV1Cache()) 18 | } 19 | 20 | func (cache *v2Cache) asV1() *v1Cache { 21 | return (*v1Cache)(cache) 22 | } 23 | 24 | func (cache *v2Cache) cachedFeed(feed *feed.Feed) CachedFeed { 25 | return cache.asV1().cachedFeed(feed) 26 | } 27 | 28 | func (cache *v2Cache) transformTo(v Version) (Impl, error) { 29 | return nil, fmt.Errorf("Transformation not supported") 30 | } 31 | 32 | func (cache *v2Cache) cleanup(knownDescriptors map[feed.Descriptor]struct{}) { 33 | cache.asV1().cleanup(knownDescriptors) 34 | } 35 | 36 | func (cache *v2Cache) Version() Version { 37 | return v2Version 38 | } 39 | 40 | func (cache *v2Cache) Info() string { 41 | return cache.asV1().Info() 42 | } 43 | 44 | func (cache *v2Cache) SpecificInfo(i any) string { 45 | return cache.asV1().SpecificInfo(i) 46 | } 47 | 48 | func (cache *v2Cache) load(reader io.Reader) error { 49 | gzipReader, err := gzip.NewReader(reader) 50 | if err != nil { 51 | return err 52 | } 53 | defer gzipReader.Close() 54 | 55 | return cache.asV1().load(gzipReader) 56 | } 57 | 58 | func (cache *v2Cache) store(writer io.Writer) error { 59 | gzipWriter := gzip.NewWriter(writer) 60 | defer gzipWriter.Close() 61 | 62 | if err := cache.asV1().store(gzipWriter); err != nil { 63 | return err 64 | } 65 | 66 | return gzipWriter.Flush() 67 | } 68 | -------------------------------------------------------------------------------- /internal/feed/feed.go: -------------------------------------------------------------------------------- 1 | package feed 2 | 3 | import ( 4 | "fmt" 5 | "net/url" 6 | "strings" 7 | "time" 8 | 9 | "github.com/Necoro/gofeed" 10 | 11 | "github.com/Necoro/feed2imap-go/internal/feed/filter" 12 | "github.com/Necoro/feed2imap-go/internal/http" 13 | "github.com/Necoro/feed2imap-go/pkg/config" 14 | "github.com/Necoro/feed2imap-go/pkg/log" 15 | ) 16 | 17 | type Feed struct { 18 | *config.Feed 19 | feed *gofeed.Feed 20 | filter *filter.Filter 21 | items []Item 22 | Global config.GlobalOptions 23 | extID FeedID 24 | } 25 | 26 | type FeedID interface { 27 | String() string 28 | } 29 | 30 | type FilterFunc func(items []Item, ignHash, alwaysNew bool) []Item 31 | 32 | type Descriptor struct { 33 | Name string 34 | Url string 35 | } 36 | 37 | func (feed *Feed) Context() http.Context { 38 | return http.Context{ 39 | Timeout: feed.Global.Timeout, 40 | DisableTLS: feed.NoTLS, 41 | } 42 | } 43 | 44 | func (feed *Feed) Descriptor() Descriptor { 45 | var url string 46 | if feed.Url != "" { 47 | url = feed.Url 48 | } else { 49 | url = "exec://" + strings.Join(feed.Exec, "/") 50 | } 51 | return Descriptor{ 52 | Name: feed.Name, 53 | Url: url, 54 | } 55 | } 56 | 57 | func (feed *Feed) NeedsUpdate(updateTime time.Time) bool { 58 | if feed.MinFreq == 0 { // shortcut 59 | return true 60 | } 61 | if !updateTime.IsZero() && int(time.Since(updateTime).Hours()) < feed.MinFreq { 62 | log.Printf("Feed '%s' does not need updating, skipping.", feed.Name) 63 | return false 64 | } 65 | return true 66 | } 67 | 68 | func (feed *Feed) FetchSuccessful() bool { 69 | return feed.feed != nil 70 | } 71 | 72 | func Create(parsedFeed *config.Feed, global config.GlobalOptions) (*Feed, error) { 73 | var itemFilter *filter.Filter 74 | var err error 75 | if parsedFeed.ItemFilter != "" { 76 | if itemFilter, err = filter.New(parsedFeed.ItemFilter); err != nil { 77 | return nil, fmt.Errorf("Feed %s: Parsing item-filter: %w", parsedFeed.Name, err) 78 | } 79 | } 80 | return &Feed{Feed: parsedFeed, Global: global, filter: itemFilter}, nil 81 | } 82 | 83 | func (feed *Feed) filterItems() []Item { 84 | if feed.filter == nil { 85 | return feed.items 86 | } 87 | 88 | items := make([]Item, 0, len(feed.items)) 89 | 90 | for _, item := range feed.items { 91 | res, err := feed.filter.Run(item.Item) 92 | if err != nil { 93 | log.Errorf("Feed %s: Item %s: Error applying item filter: %s", feed.Name, printItem(item.Item), err) 94 | res = true // include 95 | } 96 | 97 | if res { 98 | if log.IsDebug() { 99 | log.Debugf("Filter '%s' matches for item %s", feed.ItemFilter, printItem(item.Item)) 100 | } 101 | items = append(items, item) 102 | } else if log.IsDebug() { // printItem is not for free 103 | log.Debugf("Filter '%s' does not match for item %s", feed.ItemFilter, printItem(item.Item)) 104 | } 105 | } 106 | return items 107 | } 108 | 109 | func (feed *Feed) Filter(filter FilterFunc) { 110 | if len(feed.items) > 0 { 111 | origLen := len(feed.items) 112 | 113 | log.Debugf("Filtering %s. Starting with %d items", feed.Name, origLen) 114 | 115 | items := feed.filterItems() 116 | newLen := len(items) 117 | if newLen < origLen { 118 | log.Printf("Item filter on %s: Reduced from %d to %d items.", feed.Name, origLen, newLen) 119 | origLen = newLen 120 | } 121 | 122 | feed.items = filter(items, feed.IgnHash, feed.AlwaysNew) 123 | 124 | newLen = len(feed.items) 125 | if newLen < origLen { 126 | log.Printf("Filtered %s. Reduced from %d to %d items.", feed.Name, origLen, newLen) 127 | } else { 128 | log.Printf("Filtered %s, no reduction.", feed.Name) 129 | } 130 | 131 | } else { 132 | log.Debugf("No items for %s. No filtering.", feed.Name) 133 | } 134 | } 135 | 136 | func (feed *Feed) SetExtID(extID FeedID) { 137 | feed.extID = extID 138 | } 139 | 140 | func (feed *Feed) id() string { 141 | if feed.extID == nil { 142 | return feed.Name 143 | } 144 | return feed.extID.String() 145 | } 146 | 147 | func (feed *Feed) url() *url.URL { 148 | var feedUrl *url.URL 149 | 150 | tryUrl := func(content, what string) bool { 151 | var err error 152 | if content != "" { 153 | feedUrl, err = url.Parse(content) 154 | if err != nil { 155 | log.Errorf("%s '%s' of feed '%s' is not a valid URL.", what, content, feed.Name) 156 | } else { 157 | return true 158 | } 159 | } 160 | return false 161 | } 162 | 163 | if !(tryUrl(feed.feed.FeedLink, "Self-Link") || 164 | tryUrl(feed.Url, "URL") || 165 | tryUrl(feed.feed.Link, "Link")) { 166 | panic(fmt.Sprintf("Could not find a valid URL for for feed '%s'. How have we ended up here?", feed.Name)) 167 | } 168 | 169 | return feedUrl 170 | } 171 | -------------------------------------------------------------------------------- /internal/feed/filter/filter.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import ( 4 | "github.com/Necoro/gofeed" 5 | "github.com/expr-lang/expr" 6 | "github.com/expr-lang/expr/vm" 7 | ) 8 | 9 | type Filter struct { 10 | prog *vm.Program 11 | } 12 | 13 | func (f *Filter) Run(item *gofeed.Item) (bool, error) { 14 | if res, err := expr.Run(f.prog, item); err != nil { 15 | return false, err 16 | } else { 17 | return res.(bool), nil 18 | } 19 | } 20 | 21 | func New(s string) (*Filter, error) { 22 | prog, err := expr.Compile(s, expr.AsBool(), expr.Env(gofeed.Item{})) 23 | if err != nil { 24 | return nil, err 25 | } 26 | return &Filter{prog}, nil 27 | } 28 | -------------------------------------------------------------------------------- /internal/feed/item.go: -------------------------------------------------------------------------------- 1 | package feed 2 | 3 | import ( 4 | "encoding/base64" 5 | "encoding/json" 6 | "fmt" 7 | "slices" 8 | "strings" 9 | "time" 10 | 11 | "github.com/Necoro/gofeed" 12 | "github.com/google/uuid" 13 | 14 | "github.com/Necoro/feed2imap-go/pkg/config" 15 | ) 16 | 17 | type feedImage struct { 18 | image []byte 19 | mime string 20 | name string 21 | } 22 | 23 | type ItemID uuid.UUID 24 | 25 | func newItemID() ItemID { 26 | return ItemID(uuid.New()) 27 | } 28 | 29 | type Item struct { 30 | *gofeed.Item // access fields implicitly 31 | Feed *gofeed.Feed // named explicitly to not shadow common fields with Item 32 | feed *Feed 33 | Body string 34 | TextBody string 35 | UpdateOnly bool 36 | ID ItemID 37 | reasons []string 38 | images []feedImage 39 | } 40 | 41 | func (item *Item) DateParsed() *time.Time { 42 | if item.UpdatedParsed == nil || item.UpdatedParsed.IsZero() { 43 | return item.PublishedParsed 44 | } 45 | return item.UpdatedParsed 46 | } 47 | 48 | func (item *Item) Date() string { 49 | if item.Updated == "" { 50 | return item.Published 51 | } 52 | return item.Updated 53 | } 54 | 55 | // Creator returns the name of the creating authors (comma separated). 56 | func (item *Item) Creator() string { 57 | names := make([]string, len(item.Authors)) 58 | for i, p := range item.Authors { 59 | names[i] = p.Name 60 | } 61 | return strings.Join(names, ", ") 62 | } 63 | 64 | func (item *Item) FeedLink() string { 65 | if item.Feed.FeedLink != "" { 66 | // the one in the feed itself 67 | return item.Feed.FeedLink 68 | } 69 | // the one in the config 70 | return item.feed.Url 71 | } 72 | 73 | func (item *Item) AddReason(reason string) { 74 | if !slices.Contains(item.reasons, reason) { 75 | item.reasons = append(item.reasons, reason) 76 | } 77 | } 78 | 79 | func (item *Item) addImage(img []byte, mime string, name string) int { 80 | i := feedImage{img, mime, name} 81 | item.images = append(item.images, i) 82 | return len(item.images) 83 | } 84 | 85 | func (item *Item) clearImages() { 86 | clear(item.images) 87 | item.images = []feedImage{} 88 | } 89 | 90 | func (item *Item) defaultEmail() string { 91 | return item.feed.Global.DefaultEmail 92 | } 93 | 94 | func (item *Item) Id() string { 95 | idStr := base64.RawURLEncoding.EncodeToString(item.ID[:]) 96 | return item.feed.id() + "#" + idStr 97 | } 98 | 99 | func (item *Item) messageId() string { 100 | return fmt.Sprintf("", item.Id(), config.Hostname()) 101 | } 102 | 103 | func printItem(item *gofeed.Item) string { 104 | // analogous to gofeed.Feed.String 105 | json, _ := json.MarshalIndent(item, "", " ") 106 | return string(json) 107 | } 108 | -------------------------------------------------------------------------------- /internal/feed/mail.go: -------------------------------------------------------------------------------- 1 | package feed 2 | 3 | import ( 4 | "bytes" 5 | "encoding/base64" 6 | "fmt" 7 | "io" 8 | "mime" 9 | "net/url" 10 | "path" 11 | "strings" 12 | "time" 13 | 14 | "github.com/Necoro/gofeed" 15 | "github.com/Necoro/html2text" 16 | "github.com/PuerkitoBio/goquery" 17 | "github.com/emersion/go-message" 18 | "github.com/emersion/go-message/mail" 19 | "github.com/gabriel-vasile/mimetype" 20 | "github.com/go-shiori/go-readability" 21 | "golang.org/x/net/html" 22 | "golang.org/x/net/html/charset" 23 | 24 | "github.com/Necoro/feed2imap-go/internal/feed/template" 25 | "github.com/Necoro/feed2imap-go/internal/http" 26 | "github.com/Necoro/feed2imap-go/internal/msg" 27 | "github.com/Necoro/feed2imap-go/pkg/config" 28 | "github.com/Necoro/feed2imap-go/pkg/log" 29 | "github.com/Necoro/feed2imap-go/pkg/rfc822" 30 | "github.com/Necoro/feed2imap-go/pkg/version" 31 | ) 32 | 33 | func address(name, address string) []*mail.Address { 34 | return []*mail.Address{{Name: name, Address: address}} 35 | } 36 | 37 | func author(authors []*gofeed.Person) *gofeed.Person { 38 | if len(authors) > 0 { 39 | return authors[0] 40 | } 41 | return nil 42 | } 43 | 44 | func (item *Item) fromAddress() []*mail.Address { 45 | itemAuthor := author(item.Authors) 46 | feedAuthor := author(item.Feed.Authors) 47 | switch { 48 | case itemAuthor != nil && itemAuthor.Email != "": 49 | return address(itemAuthor.Name, itemAuthor.Email) 50 | case itemAuthor != nil && itemAuthor.Name != "": 51 | return address(itemAuthor.Name, item.defaultEmail()) 52 | case feedAuthor != nil && feedAuthor.Email != "": 53 | return address(feedAuthor.Name, feedAuthor.Email) 54 | case feedAuthor != nil && feedAuthor.Name != "": 55 | return address(feedAuthor.Name, item.defaultEmail()) 56 | default: 57 | return address(item.feed.Name, item.defaultEmail()) 58 | } 59 | } 60 | 61 | func (item *Item) toAddress() []*mail.Address { 62 | return address(item.feed.Name, item.defaultEmail()) 63 | } 64 | 65 | func (item *Item) buildHeader() message.Header { 66 | var h mail.Header 67 | h.SetContentType("multipart/alternative", nil) 68 | h.SetAddressList("From", item.fromAddress()) 69 | h.SetAddressList("To", item.toAddress()) 70 | h.Set("Message-Id", item.messageId()) 71 | h.Set(msg.VersionHeader, version.Version()) 72 | h.Set(msg.ReasonHeader, strings.Join(item.reasons, ",")) 73 | h.Set(msg.IdHeader, item.Id()) 74 | h.Set(msg.CreateHeader, time.Now().Format(time.RFC1123Z)) 75 | if item.GUID != "" { 76 | h.Set(msg.GuidHeader, item.GUID) 77 | } 78 | 79 | { // date 80 | date := item.DateParsed() 81 | if date == nil { 82 | now := time.Now() 83 | date = &now 84 | } 85 | h.SetDate(*date) 86 | } 87 | { // subject 88 | subject := item.Title 89 | if subject == "" { 90 | subject = item.Date() 91 | } 92 | if subject == "" { 93 | subject = item.Link 94 | } 95 | h.SetSubject(subject) 96 | } 97 | 98 | return h.Header 99 | } 100 | 101 | func (item *Item) writeContentPart(w *message.Writer, typ string, tpl template.Template) error { 102 | var ih message.Header 103 | ih.SetContentType("text/"+typ, map[string]string{"charset": "utf-8"}) 104 | ih.SetContentDisposition("inline", nil) 105 | ih.Set("Content-Transfer-Encoding", "8bit") 106 | 107 | partW, err := w.CreatePart(ih) 108 | if err != nil { 109 | return err 110 | } 111 | defer partW.Close() 112 | 113 | if err = tpl.Execute(rfc822.Writer(w), item); err != nil { 114 | return fmt.Errorf("writing %s part: %w", typ, err) 115 | } 116 | 117 | return nil 118 | } 119 | 120 | func (item *Item) writeTextPart(w *message.Writer) error { 121 | return item.writeContentPart(w, "plain", template.Text) 122 | } 123 | 124 | func (item *Item) writeHtmlPart(w *message.Writer) error { 125 | return item.writeContentPart(w, "html", template.Html) 126 | } 127 | 128 | func (img *feedImage) buildNameMap(key string) map[string]string { 129 | if img.name == "" { 130 | return nil 131 | } 132 | return map[string]string{key: img.name} 133 | } 134 | 135 | func (img *feedImage) writeImagePart(w *message.Writer, cid string) error { 136 | var ih message.Header 137 | // set filename for both Type and Disposition 138 | // according to standard, it belongs to the latter -- but some clients expect the former 139 | ih.SetContentType(img.mime, img.buildNameMap("name")) 140 | ih.SetContentDisposition("inline", img.buildNameMap("filename")) 141 | ih.Set("Content-Transfer-Encoding", "base64") 142 | ih.SetText("Content-ID", fmt.Sprintf("<%s>", cid)) 143 | 144 | imgW, err := w.CreatePart(ih) 145 | if err != nil { 146 | return err 147 | } 148 | defer imgW.Close() 149 | 150 | if _, err = imgW.Write(img.image); err != nil { 151 | return err 152 | } 153 | 154 | return nil 155 | } 156 | 157 | func (item *Item) writeToBuffer(b *bytes.Buffer) error { 158 | h := item.buildHeader() 159 | item.buildBody() 160 | 161 | writer, err := message.CreateWriter(b, h) 162 | if err != nil { 163 | return err 164 | } 165 | defer writer.Close() 166 | 167 | if item.feed.Global.WithPartText() { 168 | if err = item.writeTextPart(writer); err != nil { 169 | return err 170 | } 171 | } 172 | 173 | if item.feed.Global.WithPartHtml() { 174 | var relWriter *message.Writer 175 | if len(item.images) > 0 { 176 | var rh message.Header 177 | rh.SetContentType("multipart/related", map[string]string{"type": "text/html"}) 178 | if relWriter, err = writer.CreatePart(rh); err != nil { 179 | return err 180 | } 181 | defer relWriter.Close() 182 | } else { 183 | relWriter = writer 184 | } 185 | 186 | if err = item.writeHtmlPart(relWriter); err != nil { 187 | return err 188 | } 189 | 190 | for idx, img := range item.images { 191 | cid := cidNr(idx + 1) 192 | if err = img.writeImagePart(relWriter, cid); err != nil { 193 | return err 194 | } 195 | } 196 | } 197 | 198 | item.clearImages() // safe memory 199 | return nil 200 | } 201 | 202 | func (item *Item) message() (msg.Message, error) { 203 | var b bytes.Buffer 204 | 205 | if err := item.writeToBuffer(&b); err != nil { 206 | return msg.Message{}, err 207 | } 208 | 209 | msg := msg.Message{ 210 | Content: b.String(), 211 | IsUpdate: item.UpdateOnly, 212 | ID: item.Id(), 213 | } 214 | 215 | return msg, nil 216 | } 217 | 218 | func (feed *Feed) Messages() (msg.Messages, error) { 219 | var ( 220 | err error 221 | mails = make([]msg.Message, len(feed.items)) 222 | ) 223 | for idx := range feed.items { 224 | if mails[idx], err = feed.items[idx].message(); err != nil { 225 | return nil, fmt.Errorf("creating mails for %s: %w", feed.Name, err) 226 | } 227 | } 228 | return mails, nil 229 | } 230 | 231 | func getImage(src string, ctx http.Context) ([]byte, string, error) { 232 | resp, cancel, err := http.Get(src, ctx) 233 | if err != nil { 234 | return nil, "", fmt.Errorf("fetching from '%s': %w", src, err) 235 | } 236 | defer cancel() 237 | 238 | img, err := io.ReadAll(resp.Body) 239 | if err != nil { 240 | return nil, "", fmt.Errorf("reading from '%s': %w", src, err) 241 | } 242 | 243 | var mimeStr string 244 | ext := path.Ext(src) 245 | if ext == "" { 246 | mimeStr = mimetype.Detect(img).String() 247 | } else { 248 | mimeStr = mime.TypeByExtension(ext) 249 | } 250 | return img, mimeStr, nil 251 | } 252 | 253 | func getFullArticle(src string, ctx http.Context) (string, error) { 254 | log.Debugf("Fetching article from '%s'", src) 255 | resp, cancel, err := http.Get(src, ctx) 256 | if err != nil { 257 | return "", fmt.Errorf("fetching from '%s': %w", src, err) 258 | } 259 | defer cancel() 260 | 261 | reader, err := charset.NewReader(resp.Body, resp.Header.Get("Content-Type")) 262 | if err != nil { 263 | return "", fmt.Errorf("detecting charset from '%s': %w", src, err) 264 | } 265 | 266 | doc, err := html.Parse(reader) 267 | if err != nil { 268 | return "", fmt.Errorf("parsing body from '%s': %w", src, err) 269 | } 270 | 271 | article, err := readability.FromDocument(doc, resp.Request.URL) 272 | if err != nil { 273 | return "", fmt.Errorf("parsing body from '%s': %w", src, err) 274 | } 275 | 276 | return article.Content, nil 277 | } 278 | 279 | func cidNr(idx int) string { 280 | return fmt.Sprintf("cid_%d", idx) 281 | } 282 | 283 | func (item *Item) getBody(bodyCfg config.Body) (string, error) { 284 | switch bodyCfg { 285 | case "default": 286 | if item.Content != "" { 287 | return item.Content, nil 288 | } 289 | return item.Description, nil 290 | case "description": 291 | return item.Description, nil 292 | case "content": 293 | return item.Content, nil 294 | case "both": 295 | return item.Description + item.Content, nil 296 | case "fetch": 297 | return getFullArticle(item.Link, item.feed.Context()) 298 | default: 299 | return "", fmt.Errorf("Unknown value for Body: %v", bodyCfg) 300 | } 301 | } 302 | 303 | func (item *Item) resolveUrl(otherUrlStr string) string { 304 | feed := item.feed 305 | feedUrl := feed.url() 306 | 307 | if feedUrl == nil { 308 | // no url, just return the original 309 | return otherUrlStr 310 | } 311 | 312 | otherUrl, err := url.Parse(otherUrlStr) 313 | if err != nil { 314 | log.Errorf("Feed %s: Item %s: Error parsing URL '%s' embedded in item: %s", 315 | feed.Name, item.Link, otherUrlStr, err) 316 | return "" 317 | } 318 | 319 | return feedUrl.ResolveReference(otherUrl).String() 320 | } 321 | 322 | func (item *Item) downloadImage(src string) string { 323 | feed := item.feed 324 | 325 | imgUrl := item.resolveUrl(src) 326 | 327 | img, mime, err := getImage(imgUrl, feed.Context()) 328 | if err != nil { 329 | log.Errorf("Feed %s: Item %s: Error fetching image: %s", 330 | feed.Name, item.Link, err) 331 | return "" 332 | } 333 | if img == nil { 334 | return "" 335 | } 336 | 337 | if feed.EmbedImages { 338 | return "data:" + mime + ";base64," + base64.StdEncoding.EncodeToString(img) 339 | } else { 340 | name := path.Base(src) 341 | if name == "/" || name == "." || name == " " { 342 | name = "" 343 | } 344 | 345 | idx := item.addImage(img, mime, name) 346 | return "cid:" + cidNr(idx) 347 | } 348 | } 349 | 350 | func (item *Item) buildBody() { 351 | var err error 352 | feed := item.feed 353 | 354 | if item.Body, err = item.getBody(feed.Body); err != nil { 355 | log.Errorf("Feed %s: Item %s: Error while fetching body: %s", feed.Name, item.Link, err) 356 | return 357 | } 358 | 359 | bodyNode, err := html.Parse(strings.NewReader(item.Body)) 360 | if err != nil { 361 | log.Errorf("Feed %s: Item %s: Error while parsing html: %s", feed.Name, item.Link, err) 362 | item.TextBody = item.Body 363 | return 364 | } 365 | 366 | doc := goquery.NewDocumentFromNode(bodyNode) 367 | doneAnything := false 368 | 369 | updateBody := func() { 370 | if doneAnything { 371 | html, err := goquery.OuterHtml(doc.Selection) 372 | if err != nil { 373 | item.clearImages() 374 | log.Errorf("Feed %s: Item %s: Error during rendering HTML: %s", 375 | feed.Name, item.Link, err) 376 | } else { 377 | item.Body = html 378 | } 379 | } 380 | } 381 | 382 | // make relative links absolute 383 | doc.Find("a").Each(func(i int, selection *goquery.Selection) { 384 | const attr = "href" 385 | 386 | src, ok := selection.Attr(attr) 387 | if !ok { 388 | return 389 | } 390 | 391 | if src != "" && src[0] == '/' { 392 | absUrl := item.resolveUrl(src) 393 | selection.SetAttr(attr, absUrl) 394 | doneAnything = true 395 | } 396 | }) 397 | 398 | if feed.Global.WithPartText() { 399 | if item.TextBody, err = html2text.FromHTMLNode(bodyNode, html2text.Options{CitationStyleLinks: true}); err != nil { 400 | log.Errorf("Feed %s: Item %s: Error while converting html to text: %s", feed.Name, item.Link, err) 401 | } 402 | } 403 | 404 | if !feed.Global.WithPartHtml() || err != nil { 405 | return 406 | } 407 | 408 | if !feed.InclImages { 409 | updateBody() 410 | return 411 | } 412 | 413 | // download images 414 | doc.Find("img").Each(func(i int, selection *goquery.Selection) { 415 | const attr = "src" 416 | 417 | src, ok := selection.Attr(attr) 418 | if !ok { 419 | return 420 | } 421 | 422 | if !strings.HasPrefix(src, "data:") { 423 | if imgStr := item.downloadImage(src); imgStr != "" { 424 | selection.SetAttr(attr, imgStr) 425 | } 426 | } 427 | 428 | // srcset overrides src and would reload all the images 429 | // we do not want to include all images in the srcset either, so just strip it 430 | selection.RemoveAttr("srcset") 431 | 432 | doneAnything = true 433 | }) 434 | 435 | updateBody() 436 | } 437 | -------------------------------------------------------------------------------- /internal/feed/parse.go: -------------------------------------------------------------------------------- 1 | package feed 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "os/exec" 7 | 8 | "github.com/Necoro/gofeed" 9 | 10 | "github.com/Necoro/feed2imap-go/internal/http" 11 | ) 12 | 13 | func (feed *Feed) Parse() error { 14 | fp := gofeed.NewParser() 15 | 16 | var reader io.Reader 17 | var cleanup func() error 18 | 19 | if feed.Url != "" { 20 | // we do not use the http support in gofeed, so that we can control the behavior of http requests 21 | // and ensure it to be the same in all places 22 | resp, cancel, err := http.Get(feed.Url, feed.Context()) 23 | if err != nil { 24 | return fmt.Errorf("while fetching %s from %s: %w", feed.Name, feed.Url, err) 25 | } 26 | defer cancel() // includes resp.Body.Close 27 | 28 | reader = resp.Body 29 | cleanup = func() error { return nil } 30 | } else { // exec 31 | // we use the same context as for HTTP 32 | ctx, cancel := feed.Context().StdContext() 33 | cmd := exec.CommandContext(ctx, feed.Exec[0], feed.Exec[1:]...) 34 | defer func() { 35 | cancel() 36 | // cmd.Wait might have already been called -- but call it again to be sure 37 | _ = cmd.Wait() 38 | }() 39 | 40 | stdout, err := cmd.StdoutPipe() 41 | if err != nil { 42 | return fmt.Errorf("preparing exec for feed '%s': %w", feed.Name, err) 43 | } 44 | 45 | if err = cmd.Start(); err != nil { 46 | return fmt.Errorf("starting exec for feed '%s: %w", feed.Name, err) 47 | } 48 | 49 | reader = stdout 50 | cleanup = cmd.Wait 51 | } 52 | 53 | parsedFeed, err := fp.Parse(reader) 54 | if err != nil { 55 | return fmt.Errorf("parsing feed '%s': %w", feed.Name, err) 56 | } 57 | 58 | feed.feed = parsedFeed 59 | feed.items = make([]Item, len(parsedFeed.Items)) 60 | for idx, feedItem := range parsedFeed.Items { 61 | feed.items[idx] = Item{Feed: parsedFeed, feed: feed, Item: feedItem, ID: newItemID()} 62 | } 63 | return cleanup() 64 | } 65 | -------------------------------------------------------------------------------- /internal/feed/template/funcs.go: -------------------------------------------------------------------------------- 1 | package template 2 | 3 | import ( 4 | "fmt" 5 | html "html/template" 6 | "strconv" 7 | "strings" 8 | text "text/template" 9 | 10 | "github.com/Necoro/feed2imap-go/pkg/log" 11 | ) 12 | 13 | // dict creates a map out of the passed in key/value pairs. 14 | func dict(v ...any) map[string]any { 15 | dict := make(map[string]any) 16 | lenv := len(v) 17 | for i := 0; i < lenv; i += 2 { 18 | key := v[i].(string) 19 | if i+1 >= lenv { 20 | dict[key] = "" 21 | continue 22 | } 23 | dict[key] = v[i+1] 24 | } 25 | return dict 26 | } 27 | 28 | // join takes a separator and a list of strings and puts the former in between each pair of the latter. 29 | func join(sep string, parts []string) string { 30 | return strings.Join(parts, sep) 31 | } 32 | 33 | // lastUrlPart returns the last part of a URL string 34 | func lastUrlPart(url string) string { 35 | split := strings.Split(url, "/") 36 | return split[len(split)-1] 37 | } 38 | 39 | // byteCount receives an integer as a string, that is interpreted as a size in bytes. 40 | // This size is then equipped with the corresponding unit: 41 | // 1023 --> 1023 B; 1024 --> 1.0 KB; ... 42 | func byteCount(str string) string { 43 | var b uint64 44 | if str != "" { 45 | var err error 46 | if b, err = strconv.ParseUint(str, 10, 64); err != nil { 47 | log.Printf("Cannot convert '%s' to byte count: %s", str, err) 48 | } 49 | } 50 | 51 | const unit = 1024 52 | if b < unit { 53 | return fmt.Sprintf("%d B", b) 54 | } 55 | div, exp := uint64(unit), 0 56 | for n := b / unit; n >= unit; n /= unit { 57 | div *= unit 58 | exp++ 59 | } 60 | return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "KMGTPE"[exp]) 61 | } 62 | 63 | func _html(s string) html.HTML { 64 | return html.HTML(s) 65 | } 66 | 67 | var funcMap = text.FuncMap{ 68 | "dict": dict, 69 | "join": join, 70 | "lastUrlPart": lastUrlPart, 71 | "byteCount": byteCount, 72 | "html": _html, 73 | } 74 | -------------------------------------------------------------------------------- /internal/feed/template/funcs_test.go: -------------------------------------------------------------------------------- 1 | package template 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/google/go-cmp/cmp" 7 | ) 8 | 9 | func TestByteCount(t *testing.T) { 10 | tests := map[string]struct { 11 | inp string 12 | out string 13 | }{ 14 | "Empty": {"", "0 B"}, 15 | "Byte": {"123", "123 B"}, 16 | "KByte": {"2048", "2.0 KB"}, 17 | "KByte slight": {"2049", "2.0 KB"}, 18 | "KByte round": {"2560", "2.5 KB"}, 19 | "MByte": {"2097152", "2.0 MB"}, 20 | } 21 | 22 | for name, tt := range tests { 23 | t.Run(name, func(tst *testing.T) { 24 | out := byteCount(tt.inp) 25 | 26 | if diff := cmp.Diff(tt.out, out); diff != "" { 27 | tst.Error(diff) 28 | } 29 | }) 30 | } 31 | } 32 | 33 | func TestDict(t *testing.T) { 34 | type i []any 35 | type o map[string]any 36 | 37 | tests := map[string]struct { 38 | inp i 39 | out o 40 | }{ 41 | "Empty": {i{}, o{}}, 42 | "One": {i{"1"}, o{"1": ""}}, 43 | "Two": {i{"1", 1}, o{"1": 1}}, 44 | "Three": {i{"1", "2", "3"}, o{"1": "2", "3": ""}}, 45 | "Four": {i{"1", 2, "3", '4'}, o{"1": 2, "3": '4'}}, 46 | } 47 | 48 | for name, tt := range tests { 49 | t.Run(name, func(tst *testing.T) { 50 | out := dict(tt.inp...) 51 | 52 | if diff := cmp.Diff(tt.out, o(out)); diff != "" { 53 | tst.Error(diff) 54 | } 55 | }) 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /internal/feed/template/html.tpl: -------------------------------------------------------------------------------- 1 | {{- /*gotype:github.com/Necoro/feed2imap-go/internal/feed.Item*/ -}} 2 | {{define "bottomLine"}} 3 | {{if .content}} 4 | 5 | 6 | {{.descr}}   7 | 8 | 9 | {{.content}} 10 | 11 | 12 | {{end}} 13 | {{end}} 14 | 15 | 16 | 17 | 22 | 23 | 24 | 25 | 30 | 31 |
Feed 18 | {{with .Feed.Link}}{{end}} 19 | {{or .Feed.Title .Feed.Link "Unnamed feed"}} 20 | {{if .Feed.Link}}{{end}} 21 |
Item 26 | {{with .Item.Link}}{{end}} 27 | {{or .Item.Title .Item.Link}} 28 | {{if .Item.Link}}{{end}} 29 |
32 | {{with .Body}} 33 | {{html .}} 34 | {{end}} 35 | {{with .Item.Enclosures}} 36 | 37 | 38 | 39 | 40 | {{range .}} 41 | 42 | 46 | 47 | {{end}} 48 |
Files:
43 |     44 | {{.URL | lastUrlPart}} ({{with .Length}}{{. | byteCount}}, {{end}}{{.Type}}) 45 |
49 | {{end}} 50 |
51 | 52 | {{template "bottomLine" (dict "descr" "Date:" "content" .Date)}} 53 | {{template "bottomLine" (dict "descr" "Author:" "content" .Creator)}} 54 | {{template "bottomLine" (dict "descr" "Filed under:" "content" (join ", " .Categories))}} 55 | {{with .FeedLink}} 56 | {{template "bottomLine" (dict "descr" "Feed-Link:" "content" (print "" . "" | html))}} 57 | {{end}} 58 |
-------------------------------------------------------------------------------- /internal/feed/template/template.go: -------------------------------------------------------------------------------- 1 | package template 2 | 3 | import ( 4 | _ "embed" 5 | "errors" 6 | "fmt" 7 | html "html/template" 8 | "io" 9 | "io/fs" 10 | "os" 11 | text "text/template" 12 | 13 | "github.com/Necoro/feed2imap-go/pkg/log" 14 | ) 15 | 16 | type template interface { 17 | Execute(wr io.Writer, data any) error 18 | Name() string 19 | } 20 | 21 | type Template struct { 22 | template 23 | useHtml bool 24 | dflt string 25 | } 26 | 27 | //go:embed html.tpl 28 | var defaultHtmlTpl string 29 | 30 | //go:embed text.tpl 31 | var defaultTextTpl string 32 | 33 | var Html = Template{ 34 | useHtml: true, 35 | dflt: defaultHtmlTpl, 36 | template: html.New("Html").Funcs(funcMap), 37 | } 38 | 39 | var Text = Template{ 40 | useHtml: false, 41 | dflt: defaultTextTpl, 42 | template: text.New("Text").Funcs(funcMap), 43 | } 44 | 45 | func (tpl *Template) loadDefault() { 46 | if err := tpl.load(tpl.dflt); err != nil { 47 | panic(err) 48 | } 49 | } 50 | 51 | func (tpl *Template) load(content string) (err error) { 52 | if tpl.useHtml { 53 | _, err = tpl.template.(*html.Template).Parse(content) 54 | } else { 55 | _, err = tpl.template.(*text.Template).Parse(content) 56 | } 57 | return 58 | } 59 | 60 | func (tpl *Template) LoadFile(file string) error { 61 | content, err := os.ReadFile(file) 62 | if err != nil { 63 | if errors.Is(err, fs.ErrNotExist) { 64 | log.Errorf("Template file '%s' does not exist, keeping default.", file) 65 | return nil 66 | } else { 67 | return fmt.Errorf("reading template file '%s': %w", file, err) 68 | } 69 | } 70 | 71 | return tpl.load(string(content)) 72 | } 73 | 74 | func init() { 75 | Html.loadDefault() 76 | Text.loadDefault() 77 | } 78 | -------------------------------------------------------------------------------- /internal/feed/template/template_test.go: -------------------------------------------------------------------------------- 1 | package template 2 | 3 | import "testing" 4 | 5 | func TestTemplateDefaults(t *testing.T) { 6 | // Dummy test to ensure init() works, i.e. the default templates are loaded 7 | } 8 | -------------------------------------------------------------------------------- /internal/feed/template/text.tpl: -------------------------------------------------------------------------------- 1 | {{- /*gotype:github.com/Necoro/feed2imap-go/internal/feed.Item*/ -}} 2 | {{- with .Item.Link -}} 3 | <{{.}}> 4 | 5 | {{ end -}} 6 | {{- with .TextBody -}} 7 | {{.}} 8 | {{ end -}} 9 | {{- with .Item.Enclosures}} 10 | Files: 11 | {{- range . }} 12 | {{ .URL}} ({{with .Length}}{{. | byteCount}}, {{end}}{{.Type}}) 13 | {{- end -}} 14 | {{- end}} 15 | -- 16 | Feed: {{ with .Feed.Title -}}{{.}}{{- end }} 17 | {{ with .Feed.Link -}} 18 | <{{.}}> 19 | {{end -}} 20 | Item: {{ with .Item.Title -}} 21 | {{.}} 22 | {{- end }} 23 | {{ with .Item.Link -}} 24 | <{{.}}> 25 | {{end -}} 26 | {{ with .Date -}} 27 | Date: {{.}} 28 | {{ end -}} 29 | {{ with .Creator -}} 30 | Author: {{.}} 31 | {{ end -}} 32 | {{ with (join ", " .Categories) -}} 33 | Filed under: {{.}} 34 | {{ end -}} 35 | {{ with .FeedLink -}} 36 | Feed-Link: {{.}} 37 | {{ end -}} -------------------------------------------------------------------------------- /internal/http/client.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | ctxt "context" 5 | "crypto/tls" 6 | "fmt" 7 | "net/http" 8 | "time" 9 | ) 10 | 11 | // share HTTP clients 12 | var ( 13 | stdClient *http.Client 14 | unsafeClient *http.Client 15 | ) 16 | 17 | // Error represents an HTTP error returned by a server. 18 | type Error struct { 19 | StatusCode int 20 | Status string 21 | } 22 | 23 | type Context struct { 24 | Timeout int 25 | DisableTLS bool 26 | } 27 | 28 | func (err Error) Error() string { 29 | return fmt.Sprintf("http error: %s", err.Status) 30 | } 31 | 32 | func init() { 33 | // std 34 | stdClient = &http.Client{Transport: http.DefaultTransport} 35 | 36 | // unsafe 37 | tlsConfig := &tls.Config{InsecureSkipVerify: true} 38 | transport := http.DefaultTransport.(*http.Transport).Clone() 39 | transport.TLSClientConfig = tlsConfig 40 | unsafeClient = &http.Client{Transport: transport} 41 | } 42 | 43 | func (ctx Context) StdContext() (ctxt.Context, ctxt.CancelFunc) { 44 | return ctxt.WithTimeout(ctxt.Background(), time.Duration(ctx.Timeout)*time.Second) 45 | } 46 | 47 | func client(disableTLS bool) *http.Client { 48 | if disableTLS { 49 | return unsafeClient 50 | } 51 | return stdClient 52 | } 53 | 54 | var noop ctxt.CancelFunc = func() {} 55 | 56 | func Get(url string, ctx Context) (resp *http.Response, cancel ctxt.CancelFunc, err error) { 57 | prematureExit := true 58 | stdCtx, ctxCancel := ctx.StdContext() 59 | 60 | cancel = func() { 61 | if resp != nil { 62 | _ = resp.Body.Close() 63 | } 64 | ctxCancel() 65 | } 66 | 67 | defer func() { 68 | if prematureExit { 69 | cancel() 70 | } 71 | }() 72 | 73 | req, err := http.NewRequestWithContext(stdCtx, "GET", url, nil) 74 | if err != nil { 75 | return nil, noop, err 76 | } 77 | req.Header.Set("User-Agent", "Feed2Imap-Go/1.0") 78 | 79 | resp, err = client(ctx.DisableTLS).Do(req) 80 | if err != nil { 81 | return nil, noop, err 82 | } 83 | 84 | if resp.StatusCode < 200 || resp.StatusCode >= 300 { 85 | return nil, noop, Error{ 86 | StatusCode: resp.StatusCode, 87 | Status: resp.Status, 88 | } 89 | } 90 | 91 | prematureExit = false 92 | return resp, cancel, nil 93 | } 94 | -------------------------------------------------------------------------------- /internal/imap/client.go: -------------------------------------------------------------------------------- 1 | package imap 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | "sync" 7 | "time" 8 | 9 | uidplus "github.com/emersion/go-imap-uidplus" 10 | imapClient "github.com/emersion/go-imap/client" 11 | 12 | "github.com/Necoro/feed2imap-go/pkg/config" 13 | "github.com/Necoro/feed2imap-go/pkg/log" 14 | ) 15 | 16 | type connConf struct { 17 | host string 18 | delimiter string 19 | toplevel Folder 20 | } 21 | 22 | type Client struct { 23 | connConf 24 | mailboxes *mailboxes 25 | commander *commander 26 | connections []*connection 27 | connChannel chan *connection 28 | connLock sync.Mutex 29 | disconnected bool 30 | } 31 | 32 | var dialer imapClient.Dialer 33 | 34 | func init() { 35 | dialer = &net.Dialer{Timeout: 30 * time.Second} 36 | } 37 | 38 | func newImapClient(url config.Url) (c *imapClient.Client, err error) { 39 | if url.ForceTLS() { 40 | if c, err = imapClient.DialWithDialerTLS(dialer, url.HostPort(), nil); err != nil { 41 | return nil, fmt.Errorf("connecting (TLS) to %s: %w", url.Host, err) 42 | } 43 | log.Print("Connected to ", url.HostPort(), " (TLS)") 44 | } else { 45 | if c, err = imapClient.DialWithDialer(dialer, url.HostPort()); err != nil { 46 | return nil, fmt.Errorf("connecting to %s: %w", url.Host, err) 47 | } 48 | } 49 | 50 | return 51 | } 52 | 53 | func (cl *Client) connect(url config.Url) (*connection, error) { 54 | c, err := newImapClient(url) 55 | if err != nil { 56 | return nil, err 57 | } 58 | 59 | cl.connLock.Lock() 60 | defer cl.connLock.Unlock() 61 | 62 | if cl.disconnected { 63 | return nil, nil 64 | } 65 | 66 | conn := cl.createConnection(c) 67 | 68 | if !url.ForceTLS() { 69 | if err = conn.startTls(); err != nil { 70 | return nil, err 71 | } 72 | } 73 | 74 | if err = c.Login(url.User, url.Password); err != nil { 75 | return nil, fmt.Errorf("login to %s: %w", url.Host, err) 76 | } 77 | 78 | cl.connChannel <- conn 79 | 80 | return conn, nil 81 | } 82 | 83 | func (cl *Client) Disconnect() { 84 | if cl != nil { 85 | cl.connLock.Lock() 86 | 87 | cl.stopCommander() 88 | close(cl.connChannel) 89 | 90 | connected := false 91 | for _, conn := range cl.connections { 92 | connected = conn.disconnect() || connected 93 | } 94 | 95 | if connected { 96 | log.Print("Disconnected from ", cl.host) 97 | } 98 | 99 | cl.disconnected = true 100 | 101 | cl.connLock.Unlock() 102 | } 103 | } 104 | 105 | func (cl *Client) createConnection(c *imapClient.Client) *connection { 106 | 107 | client := &client{c, uidplus.NewClient(c)} 108 | 109 | conn := &connection{ 110 | connConf: &cl.connConf, 111 | mailboxes: cl.mailboxes, 112 | c: client, 113 | } 114 | 115 | cl.connections = append(cl.connections, conn) 116 | return conn 117 | } 118 | 119 | func newClient() *Client { 120 | return &Client{ 121 | mailboxes: NewMailboxes(), 122 | connChannel: make(chan *connection, 0), 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /internal/imap/cmds.go: -------------------------------------------------------------------------------- 1 | package imap 2 | 3 | type ensureCommando struct { 4 | folder Folder 5 | } 6 | 7 | func (cmd ensureCommando) execute(conn *connection) error { 8 | return conn.ensureFolder(cmd.folder) 9 | } 10 | 11 | func (cl *Client) EnsureFolder(folder Folder) error { 12 | return cl.commander.execute(ensureCommando{folder}) 13 | } 14 | 15 | type addCommando struct { 16 | folder Folder 17 | messages []string 18 | } 19 | 20 | func (cmd addCommando) execute(conn *connection) error { 21 | return conn.putMessages(cmd.folder, cmd.messages) 22 | } 23 | 24 | func (cl *Client) PutMessages(folder Folder, messages []string) error { 25 | return cl.commander.execute(addCommando{folder, messages}) 26 | } 27 | 28 | type replaceCommando struct { 29 | folder Folder 30 | header string 31 | value string 32 | newContent string 33 | force bool 34 | } 35 | 36 | func (cmd replaceCommando) execute(conn *connection) error { 37 | return conn.replace(cmd.folder, cmd.header, cmd.value, cmd.newContent, cmd.force) 38 | } 39 | 40 | func (cl *Client) Replace(folder Folder, header, value, newContent string, force bool) error { 41 | return cl.commander.execute(replaceCommando{folder, header, value, newContent, force}) 42 | } 43 | -------------------------------------------------------------------------------- /internal/imap/commando.go: -------------------------------------------------------------------------------- 1 | package imap 2 | 3 | const maxPipeDepth = 10 4 | 5 | type commander struct { 6 | client *Client 7 | pipe chan<- execution 8 | done chan<- struct{} 9 | } 10 | 11 | type command interface { 12 | execute(*connection) error 13 | } 14 | 15 | type execution struct { 16 | cmd command 17 | done chan<- error 18 | } 19 | 20 | func (commander *commander) execute(command command) error { 21 | done := make(chan error) 22 | commander.pipe <- execution{command, done} 23 | return <-done 24 | } 25 | 26 | func executioner(conn *connection, pipe <-chan execution, done <-chan struct{}) { 27 | for { 28 | select { 29 | case <-done: 30 | return 31 | case execution := <-pipe: 32 | select { // break as soon as done is there 33 | case <-done: 34 | return 35 | default: 36 | } 37 | err := execution.cmd.execute(conn) 38 | execution.done <- err 39 | } 40 | } 41 | } 42 | 43 | func (cl *Client) startCommander() { 44 | if cl.commander != nil { 45 | return 46 | } 47 | 48 | pipe := make(chan execution, maxPipeDepth) 49 | done := make(chan struct{}) 50 | 51 | cl.commander = &commander{cl, pipe, done} 52 | 53 | go func() { 54 | for conn := range cl.connChannel { 55 | go executioner(conn, pipe, done) 56 | } 57 | }() 58 | } 59 | 60 | func (cl *Client) stopCommander() { 61 | if cl.commander == nil { 62 | return 63 | } 64 | 65 | close(cl.commander.done) 66 | 67 | cl.commander = nil 68 | } 69 | -------------------------------------------------------------------------------- /internal/imap/connection.go: -------------------------------------------------------------------------------- 1 | package imap 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "slices" 7 | "strings" 8 | "time" 9 | 10 | "github.com/emersion/go-imap" 11 | uidplus "github.com/emersion/go-imap-uidplus" 12 | imapClient "github.com/emersion/go-imap/client" 13 | 14 | "github.com/Necoro/feed2imap-go/pkg/log" 15 | ) 16 | 17 | type client struct { 18 | *imapClient.Client 19 | *uidplus.UidPlusClient 20 | } 21 | 22 | type connection struct { 23 | *connConf 24 | mailboxes *mailboxes 25 | c *client 26 | } 27 | 28 | func (conn *connection) startTls() error { 29 | hasStartTls, err := conn.c.SupportStartTLS() 30 | if err != nil { 31 | return fmt.Errorf("checking for starttls for %s: %w", conn.host, err) 32 | } 33 | 34 | if hasStartTls { 35 | if err = conn.c.StartTLS(nil); err != nil { 36 | return fmt.Errorf("enabling starttls for %s: %w", conn.host, err) 37 | } 38 | 39 | log.Print("Connected to ", conn.host, " (STARTTLS)") 40 | } else { 41 | log.Print("Connected to ", conn.host, " (Plain)") 42 | } 43 | 44 | return nil 45 | } 46 | 47 | func (conn *connection) disconnect() bool { 48 | if conn != nil { 49 | connected := (conn.c.State() & imap.ConnectedState) != 0 50 | _ = conn.c.Logout() 51 | return connected 52 | } 53 | return false 54 | } 55 | 56 | func (conn *connection) createFolder(folder string) error { 57 | err := conn.c.Create(folder) 58 | if err != nil { 59 | return fmt.Errorf("creating folder '%s': %w", folder, err) 60 | } 61 | 62 | err = conn.c.Subscribe(folder) 63 | if err != nil { 64 | return fmt.Errorf("subscribing to folder '%s': %w", folder, err) 65 | } 66 | 67 | log.Printf("Created folder '%s'", folder) 68 | 69 | return nil 70 | } 71 | 72 | func (conn *connection) list(folder string) (*imap.MailboxInfo, int, error) { 73 | mailboxes := make(chan *imap.MailboxInfo, 10) 74 | done := make(chan error, 1) 75 | go func() { 76 | done <- conn.c.List("", folder, mailboxes) 77 | }() 78 | 79 | found := 0 80 | var mbox *imap.MailboxInfo 81 | for m := range mailboxes { 82 | if found == 0 { 83 | mbox = m 84 | } 85 | found++ 86 | } 87 | 88 | if err := <-done; err != nil { 89 | return nil, 0, fmt.Errorf("while listing '%s': %w", folder, err) 90 | } 91 | 92 | return mbox, found, nil 93 | } 94 | 95 | func (conn *connection) fetchDelimiter() (string, error) { 96 | mbox, _, err := conn.list("") 97 | if err != nil { 98 | return "", err 99 | } 100 | 101 | return mbox.Delimiter, nil 102 | } 103 | 104 | func (conn *connection) ensureFolder(folder Folder) error { 105 | if conn.mailboxes.contains(folder) { 106 | return nil 107 | } 108 | 109 | if conn.mailboxes.locking(folder) { 110 | // someone else tried to create the MB -- try again, now that he's done 111 | return conn.ensureFolder(folder) 112 | } 113 | 114 | defer conn.mailboxes.unlocking(folder) 115 | 116 | log.Printf("Checking for folder '%s'", folder) 117 | 118 | mbox, found, err := conn.list(folder.str) 119 | if err != nil { 120 | return err 121 | } 122 | 123 | if mbox != nil && mbox.Delimiter != folder.delimiter { 124 | panic("Delimiters do not match") 125 | } 126 | 127 | switch { 128 | case found == 0 || (found == 1 && slices.Contains(mbox.Attributes, imap.NoSelectAttr)): 129 | return conn.createFolder(folder.str) 130 | case found == 1: 131 | conn.mailboxes.add(mbox) 132 | return nil 133 | default: 134 | return fmt.Errorf("Found multiple folders matching '%s'.", folder) 135 | } 136 | } 137 | 138 | func (conn *connection) delete(uids []uint32) error { 139 | storeItem := imap.FormatFlagsOp(imap.AddFlags, true) 140 | deleteFlag := []any{imap.DeletedFlag} 141 | 142 | seqSet := new(imap.SeqSet) 143 | seqSet.AddNum(uids...) 144 | 145 | if err := conn.c.UidStore(seqSet, storeItem, deleteFlag, nil); err != nil { 146 | return fmt.Errorf("marking as deleted: %w", err) 147 | } 148 | 149 | if ok, _ := conn.c.SupportUidPlus(); ok { 150 | if err := conn.c.UidExpunge(seqSet, nil); err != nil { 151 | return fmt.Errorf("expunging (uid): %w", err) 152 | } 153 | } else { 154 | if err := conn.c.Expunge(nil); err != nil { 155 | return fmt.Errorf("expunging: %w", err) 156 | } 157 | } 158 | 159 | return nil 160 | } 161 | 162 | func (conn *connection) fetchFlags(uid uint32) ([]string, error) { 163 | fetchItem := []imap.FetchItem{imap.FetchFlags} 164 | 165 | seqSet := new(imap.SeqSet) 166 | seqSet.AddNum(uid) 167 | 168 | messages := make(chan *imap.Message, 1) 169 | done := make(chan error) 170 | go func() { 171 | done <- conn.c.UidFetch(seqSet, fetchItem, messages) 172 | }() 173 | 174 | var flags []string 175 | for m := range messages { 176 | // unilateral flags messages may be sent by the server, which then clutter our messages 177 | // --> filter for our selected UID 178 | if m.Uid == uid { 179 | flags = m.Flags 180 | } 181 | } 182 | err := <-done 183 | 184 | if err == nil && flags == nil { 185 | err = errors.New("no flags returned") 186 | } 187 | 188 | if err != nil { 189 | return nil, fmt.Errorf("fetching flags for UID %d: %w", uid, err) 190 | } 191 | return flags, nil 192 | } 193 | 194 | func (conn *connection) replace(folder Folder, header, value, newContent string, force bool) error { 195 | var err error 196 | var msgIds []uint32 197 | 198 | if err = conn.selectFolder(folder); err != nil { 199 | return err 200 | } 201 | 202 | if msgIds, err = conn.searchHeader(header, value); err != nil { 203 | return err 204 | } 205 | 206 | if len(msgIds) == 0 { 207 | if force { 208 | return conn.append(folder, nil, newContent) 209 | } 210 | return nil // nothing to do 211 | } 212 | 213 | var flags []string 214 | if flags, err = conn.fetchFlags(msgIds[0]); err != nil { 215 | return err 216 | } 217 | 218 | // filter \Seen --> updating should be noted :) 219 | filteredFlags := make([]string, 0, len(flags)) 220 | for _, f := range flags { 221 | if f != imap.SeenFlag && f != imap.RecentFlag { 222 | filteredFlags = append(filteredFlags, f) 223 | } 224 | } 225 | 226 | if err = conn.delete(msgIds); err != nil { 227 | return err 228 | } 229 | 230 | if err = conn.append(folder, filteredFlags, newContent); err != nil { 231 | return err 232 | } 233 | 234 | return nil 235 | } 236 | 237 | func (conn *connection) searchHeader(header, value string) ([]uint32, error) { 238 | criteria := imap.NewSearchCriteria() 239 | criteria.Header.Set(header, value) 240 | criteria.WithoutFlags = []string{imap.DeletedFlag} 241 | ids, err := conn.search(criteria) 242 | if err != nil { 243 | return nil, fmt.Errorf("searching for header %q=%q: %w", header, value, err) 244 | } 245 | return ids, nil 246 | } 247 | 248 | func (conn *connection) search(criteria *imap.SearchCriteria) ([]uint32, error) { 249 | return conn.c.UidSearch(criteria) 250 | } 251 | 252 | func (conn *connection) selectFolder(folder Folder) error { 253 | if _, err := conn.c.Select(folder.str, false); err != nil { 254 | return fmt.Errorf("selecting folder %s: %w", folder, err) 255 | } 256 | 257 | return nil 258 | } 259 | 260 | func (conn *connection) append(folder Folder, flags []string, msg string) error { 261 | reader := strings.NewReader(msg) 262 | if err := conn.c.Client.Append(folder.str, flags, time.Now(), reader); err != nil { 263 | return fmt.Errorf("uploading message to %s: %w", folder, err) 264 | } 265 | 266 | return nil 267 | } 268 | 269 | func (conn *connection) putMessages(folder Folder, messages []string) error { 270 | if len(messages) == 0 { 271 | return nil 272 | } 273 | 274 | for _, msg := range messages { 275 | if err := conn.append(folder, nil, msg); err != nil { 276 | return err 277 | } 278 | } 279 | 280 | return nil 281 | } 282 | -------------------------------------------------------------------------------- /internal/imap/folder.go: -------------------------------------------------------------------------------- 1 | package imap 2 | 3 | import "strings" 4 | 5 | type Folder struct { 6 | str string 7 | delimiter string 8 | } 9 | 10 | func (f Folder) IsBlank() bool { 11 | return f.str == "" 12 | } 13 | 14 | func (f Folder) String() string { 15 | return f.str 16 | } 17 | 18 | func (f Folder) Append(other Folder) Folder { 19 | if f.delimiter != other.delimiter { 20 | panic("Delimiters do not match") 21 | } 22 | 23 | if other.str == "" { 24 | return f 25 | } 26 | 27 | var prefix string 28 | if f.str == "" { 29 | prefix = "" 30 | } else { 31 | prefix = f.str + f.delimiter 32 | } 33 | 34 | return Folder{ 35 | str: prefix + other.str, 36 | delimiter: f.delimiter, 37 | } 38 | } 39 | 40 | func buildFolderName(path []string, delimiter string) (name string) { 41 | name = strings.Join(path, delimiter) 42 | if delimiter != "" { 43 | name = strings.Trim(name, delimiter[0:1]) 44 | } 45 | return 46 | } 47 | 48 | func (cl *Client) folderName(path []string) Folder { 49 | return Folder{ 50 | buildFolderName(path, cl.delimiter), 51 | cl.delimiter, 52 | } 53 | } 54 | 55 | func (cl *Client) NewFolder(path []string) Folder { 56 | return cl.toplevel.Append(cl.folderName(path)) 57 | } 58 | -------------------------------------------------------------------------------- /internal/imap/imap.go: -------------------------------------------------------------------------------- 1 | package imap 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/Necoro/feed2imap-go/pkg/config" 7 | "github.com/Necoro/feed2imap-go/pkg/log" 8 | ) 9 | 10 | func Connect(url config.Url, numConnections int) (*Client, error) { 11 | var err error 12 | 13 | client := newClient() 14 | client.host = url.Host 15 | defer func() { 16 | if err != nil { 17 | client.Disconnect() 18 | } 19 | }() 20 | client.startCommander() 21 | 22 | var conn *connection // the main connection 23 | if conn, err = client.connect(url); err != nil { 24 | return nil, err 25 | } 26 | 27 | delim, err := conn.fetchDelimiter() 28 | if err != nil { 29 | return nil, fmt.Errorf("fetching delimiter: %w", err) 30 | } 31 | client.delimiter = delim 32 | 33 | client.toplevel = client.folderName(url.RootPath()) 34 | 35 | log.Printf("Determined '%s' as toplevel, with '%s' as delimiter", client.toplevel, client.delimiter) 36 | 37 | if !client.toplevel.IsBlank() { 38 | if err = conn.ensureFolder(client.toplevel); err != nil { 39 | return nil, err 40 | } 41 | } 42 | 43 | // the other connections 44 | for i := 1; i < numConnections; i++ { 45 | go func(id int) { 46 | if _, err := client.connect(url); err != nil { // explicitly new var 'err', b/c these are now harmless 47 | log.Warnf("connecting #%d: %s", id, err) 48 | } 49 | }(i) 50 | } 51 | 52 | return client, nil 53 | } 54 | -------------------------------------------------------------------------------- /internal/imap/mailboxes.go: -------------------------------------------------------------------------------- 1 | package imap 2 | 3 | import ( 4 | "sync" 5 | 6 | "github.com/emersion/go-imap" 7 | ) 8 | 9 | type mailboxes struct { 10 | mb map[string]*imap.MailboxInfo 11 | mu sync.RWMutex 12 | changeLocks map[string]chan struct{} 13 | } 14 | 15 | func (mbs *mailboxes) unlocking(elem Folder) { 16 | mbs.mu.Lock() 17 | defer mbs.mu.Unlock() 18 | 19 | ch, ok := mbs.changeLocks[elem.str] 20 | if !ok { 21 | panic("Unlocking where nothing is locked") 22 | } 23 | close(ch) 24 | delete(mbs.changeLocks, elem.str) 25 | } 26 | 27 | func (mbs *mailboxes) locking(elem Folder) bool { 28 | mbs.mu.Lock() 29 | 30 | // check again, if the folder has been created in the meantime 31 | _, ok := mbs.mb[elem.str] 32 | if ok { 33 | mbs.mu.Unlock() 34 | return true 35 | } 36 | 37 | ch, ok := mbs.changeLocks[elem.str] 38 | if !ok { 39 | ch = make(chan struct{}) 40 | mbs.changeLocks[elem.str] = ch 41 | mbs.mu.Unlock() 42 | // we created the lock, we are in charge and done here 43 | return false 44 | } else { 45 | // someone else is working, we wait till he's done 46 | mbs.mu.Unlock() // we are not doing anything... 47 | <-ch 48 | return true 49 | } 50 | } 51 | 52 | func (mbs *mailboxes) contains(elem Folder) bool { 53 | mbs.mu.RLock() 54 | defer mbs.mu.RUnlock() 55 | 56 | _, ok := mbs.mb[elem.str] 57 | return ok 58 | } 59 | 60 | func (mbs *mailboxes) add(elem *imap.MailboxInfo) { 61 | mbs.mu.Lock() 62 | defer mbs.mu.Unlock() 63 | 64 | mbs.mb[elem.Name] = elem 65 | } 66 | 67 | func NewMailboxes() *mailboxes { 68 | return &mailboxes{ 69 | mb: map[string]*imap.MailboxInfo{}, 70 | changeLocks: map[string]chan struct{}{}, 71 | mu: sync.RWMutex{}, 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /internal/msg/msg.go: -------------------------------------------------------------------------------- 1 | package msg 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/Necoro/feed2imap-go/internal/imap" 7 | "github.com/Necoro/feed2imap-go/pkg/log" 8 | ) 9 | 10 | // headers 11 | const ( 12 | VersionHeader = "X-Feed2Imap-Version" 13 | ReasonHeader = "X-Feed2Imap-Reason" 14 | IdHeader = "X-Feed2Imap-Item" 15 | GuidHeader = "X-Feed2Imap-Guid" 16 | CreateHeader = "X-Feed2Imap-Create-Date" 17 | ) 18 | 19 | type Messages []Message 20 | 21 | type Message struct { 22 | Content string 23 | IsUpdate bool 24 | ID string 25 | } 26 | 27 | func (m Messages) Upload(client *imap.Client, folder imap.Folder, reupload bool) error { 28 | toStore := make([]string, 0, len(m)) 29 | 30 | updateMsgs := make(chan Message, 5) 31 | ok := make(chan bool) 32 | go func() { /* update goroutine */ 33 | errHappened := false 34 | for msg := range updateMsgs { 35 | if err := client.Replace(folder, IdHeader, msg.ID, msg.Content, reupload); err != nil { 36 | log.Errorf("Error while updating mail with id '%s' in folder '%s'. Skipping.: %s", 37 | msg.ID, folder, err) 38 | errHappened = true 39 | } 40 | } 41 | 42 | ok <- errHappened 43 | }() 44 | 45 | for _, msg := range m { 46 | if !msg.IsUpdate { 47 | toStore = append(toStore, msg.Content) 48 | } else { 49 | updateMsgs <- msg 50 | } 51 | } 52 | 53 | close(updateMsgs) 54 | 55 | putErr := client.PutMessages(folder, toStore) 56 | updOk := <-ok 57 | 58 | if putErr != nil { 59 | return putErr 60 | } 61 | if updOk { 62 | return fmt.Errorf("Errors during updating mails.") 63 | } 64 | 65 | return nil 66 | } 67 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "os" 7 | 8 | "github.com/Necoro/feed2imap-go/internal/feed/cache" 9 | "github.com/Necoro/feed2imap-go/internal/feed/template" 10 | "github.com/Necoro/feed2imap-go/internal/imap" 11 | "github.com/Necoro/feed2imap-go/pkg/config" 12 | "github.com/Necoro/feed2imap-go/pkg/log" 13 | "github.com/Necoro/feed2imap-go/pkg/version" 14 | ) 15 | 16 | // flags 17 | var ( 18 | cfgFile string = "config.yml" 19 | cacheFile string 20 | printVersion bool = false 21 | dryRun bool = false 22 | buildCache bool = false 23 | verbose bool = false 24 | debug bool = false 25 | ) 26 | 27 | func init() { 28 | flag.StringVar(&cfgFile, "f", cfgFile, "configuration file") 29 | flag.StringVar(&cacheFile, "c", "", "override cache file location") 30 | flag.BoolVar(&printVersion, "version", printVersion, "print version and exit") 31 | flag.BoolVar(&dryRun, "dry-run", dryRun, "do everything short of uploading and writing the cache") 32 | flag.BoolVar(&buildCache, "build-cache", buildCache, "only (re)build the cache; useful after migration or when the cache is lost or corrupted") 33 | flag.BoolVar(&verbose, "v", verbose, "enable verbose output") 34 | flag.BoolVar(&debug, "d", debug, "enable debug output") 35 | } 36 | 37 | func processFeed(cf cache.CachedFeed, client *imap.Client, dryRun bool) { 38 | feed := cf.Feed() 39 | msgs, err := feed.Messages() 40 | if err != nil { 41 | log.Errorf("Processing items of feed %s: %s", feed.Name, err) 42 | return 43 | } 44 | 45 | if dryRun || len(msgs) == 0 { 46 | cf.Commit() 47 | return 48 | } 49 | 50 | folder := client.NewFolder(feed.Target) 51 | if err = client.EnsureFolder(folder); err != nil { 52 | log.Errorf("Creating folder of feed %s: %s", feed.Name, err) 53 | return 54 | } 55 | 56 | if err = msgs.Upload(client, folder, feed.Reupload); err != nil { 57 | log.Errorf("Uploading messages of feed %s: %s", feed.Name, err) 58 | return 59 | } 60 | 61 | log.Printf("Uploaded %d messages to '%s' @ %s", len(msgs), feed.Name, folder) 62 | 63 | cf.Commit() 64 | } 65 | 66 | func loadTemplate(path string, tpl template.Template) error { 67 | if path == "" { 68 | return nil 69 | } 70 | 71 | log.Printf("Loading custom %s template from %s", tpl.Name(), path) 72 | if err := tpl.LoadFile(path); err != nil { 73 | return fmt.Errorf("loading %s template from %s: %w", tpl.Name(), path, err) 74 | } 75 | return nil 76 | } 77 | 78 | func run() error { 79 | flag.Parse() 80 | if printVersion { 81 | println("Feed2Imap-Go, " + version.FullVersion()) 82 | return nil 83 | } 84 | 85 | if debug { 86 | log.SetDebug() 87 | } else if verbose { 88 | log.SetVerbose() 89 | } 90 | 91 | log.Printf("Starting up (%s)...", version.FullVersion()) 92 | 93 | cfg, err := config.Load(cfgFile) 94 | if err != nil { 95 | return err 96 | } 97 | 98 | state, err := cache.NewState(cfg) 99 | if err != nil { 100 | return err 101 | } 102 | 103 | cacheLocation := cacheFile 104 | if cacheLocation == "" { 105 | cacheLocation = cfg.Cache 106 | } 107 | log.Debugf("Using '%s' as cache location", cacheLocation) 108 | 109 | err = state.LoadCache(cacheLocation, buildCache) 110 | if err != nil { 111 | return err 112 | } 113 | defer state.UnlockCache() 114 | 115 | state.RemoveUndue() 116 | 117 | if state.NumFeeds() == 0 { 118 | log.Print("Nothing to do, exiting.") 119 | // nothing to do 120 | return nil 121 | } 122 | 123 | if !buildCache { 124 | if err = loadTemplate(cfg.HtmlTemplate, template.Html); err != nil { 125 | return err 126 | } 127 | if err = loadTemplate(cfg.TextTemplate, template.Text); err != nil { 128 | return err 129 | } 130 | } 131 | 132 | imapErr := make(chan error, 1) 133 | var c *imap.Client 134 | if !dryRun && !buildCache { 135 | go func() { 136 | var err error 137 | c, err = imap.Connect(cfg.Target, cfg.MaxConns) 138 | imapErr <- err 139 | }() 140 | 141 | defer func() { 142 | // capture c and not evaluate it, before connect has run 143 | c.Disconnect() 144 | }() 145 | } 146 | 147 | if success := state.Fetch(); success == 0 { 148 | return fmt.Errorf("No successful feed fetch.") 149 | } 150 | 151 | state.Filter() 152 | 153 | if buildCache { 154 | state.Foreach(cache.CachedFeed.Commit) 155 | } else { 156 | if !dryRun { 157 | if err = <-imapErr; err != nil { 158 | return err 159 | } 160 | } 161 | state.ForeachGo(func(f cache.CachedFeed) { 162 | processFeed(f, c, dryRun) 163 | }) 164 | } 165 | 166 | if !dryRun { 167 | if err = state.StoreCache(cacheLocation); err != nil { 168 | return err 169 | } 170 | } 171 | 172 | return nil 173 | } 174 | 175 | func main() { 176 | if err := run(); err != nil { 177 | log.Error(err) 178 | os.Exit(1) 179 | } 180 | } 181 | -------------------------------------------------------------------------------- /pkg/config/body.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "fmt" 5 | "slices" 6 | 7 | "gopkg.in/yaml.v3" 8 | ) 9 | 10 | type Body string 11 | 12 | var validBody = []string{"default", "both", "content", "description", "fetch"} 13 | 14 | func (b *Body) UnmarshalYAML(node *yaml.Node) error { 15 | var val string 16 | if err := node.Decode(&val); err != nil { 17 | return err 18 | } 19 | 20 | if val == "" { 21 | val = "default" 22 | } 23 | 24 | if !slices.Contains(validBody, val) { 25 | return TypeError("line %d: Invalid value for 'body': %q", node.Line, val) 26 | } 27 | 28 | *b = Body(val) 29 | return nil 30 | } 31 | 32 | func TypeError(format string, v ...any) *yaml.TypeError { 33 | return &yaml.TypeError{Errors: []string{fmt.Sprintf(format, v...)}} 34 | } 35 | -------------------------------------------------------------------------------- /pkg/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "os/user" 7 | "runtime" 8 | "slices" 9 | "strings" 10 | 11 | "github.com/Necoro/feed2imap-go/pkg/log" 12 | ) 13 | 14 | // Map is a convenience type for the non-mapped configuration options 15 | // Mostly used for legacy options 16 | type Map map[string]any 17 | 18 | // GlobalOptions are not feed specific 19 | type GlobalOptions struct { 20 | Cache string `yaml:"cache"` 21 | Timeout int `yaml:"timeout"` 22 | DefaultEmail string `yaml:"default-email"` 23 | Target Url `yaml:"target"` 24 | Parts []string `yaml:"parts"` 25 | MaxFailures int `yaml:"max-failures"` 26 | MaxConns int `yaml:"max-imap-connections"` 27 | AutoTarget bool `yaml:"auto-target"` 28 | HtmlTemplate string `yaml:"html-template"` 29 | TextTemplate string `yaml:"text-template"` 30 | } 31 | 32 | var DefaultGlobalOptions = GlobalOptions{ 33 | Cache: "feed.cache", 34 | Timeout: 30, 35 | MaxFailures: 10, 36 | MaxConns: 5, 37 | DefaultEmail: username() + "@" + Hostname(), 38 | Target: Url{}, 39 | Parts: []string{"text", "html"}, 40 | AutoTarget: true, 41 | HtmlTemplate: "", 42 | TextTemplate: "", 43 | } 44 | 45 | // Options are feed specific 46 | // NB: Always specify a yaml name, as it is later used in processing 47 | type Options struct { 48 | MinFreq int `yaml:"min-frequency"` 49 | InclImages bool `yaml:"include-images"` 50 | EmbedImages bool `yaml:"embed-images"` 51 | Disable bool `yaml:"disable"` 52 | IgnHash bool `yaml:"ignore-hash"` 53 | AlwaysNew bool `yaml:"always-new"` 54 | Reupload bool `yaml:"reupload-if-updated"` 55 | NoTLS bool `yaml:"tls-no-verify"` 56 | ItemFilter string `yaml:"item-filter"` 57 | Body Body `yaml:"body"` 58 | } 59 | 60 | var DefaultFeedOptions = Options{ 61 | Body: "default", 62 | MinFreq: 0, 63 | InclImages: true, 64 | EmbedImages: false, 65 | IgnHash: false, 66 | AlwaysNew: false, 67 | Disable: false, 68 | NoTLS: false, 69 | ItemFilter: "", 70 | } 71 | 72 | // Config holds the global configuration options and the configured feeds 73 | type Config struct { 74 | GlobalOptions `yaml:",inline"` 75 | FeedOptions Options `yaml:"options"` 76 | Feeds Feeds `yaml:"-"` 77 | } 78 | 79 | // WithDefault returns a configuration initialized with default values. 80 | func WithDefault() *Config { 81 | return &Config{ 82 | GlobalOptions: DefaultGlobalOptions, 83 | FeedOptions: DefaultFeedOptions, 84 | Feeds: Feeds{}, 85 | } 86 | } 87 | 88 | // Validate checks the configuration against common mistakes 89 | func (cfg *Config) Validate() error { 90 | if cfg.Target.Empty() { 91 | return fmt.Errorf("No target set!") 92 | } 93 | 94 | for _, feed := range cfg.Feeds { 95 | if feed.Url != "" && len(feed.Exec) > 0 { 96 | return fmt.Errorf("Feed %s: Both 'Url' and 'Exec' set, unsure what to do.", feed.Name) 97 | } 98 | } 99 | 100 | if cfg.Target.EmptyRoot() { 101 | for _, feed := range cfg.Feeds { 102 | if len(feed.Target) == 0 { 103 | return fmt.Errorf("Feed %s: No storage location (target) defined.", feed.Name) 104 | } 105 | } 106 | } 107 | 108 | if cfg.MaxConns < 1 { 109 | return fmt.Errorf("max-imap-connections is '%d', but must be at least 1.", cfg.MaxConns) 110 | } 111 | 112 | return nil 113 | } 114 | 115 | // WithPartText marks whether 'text' part should be included in mails 116 | func (opt GlobalOptions) WithPartText() bool { 117 | return slices.Contains(opt.Parts, "text") 118 | } 119 | 120 | // WithPartHtml marks whether 'html' part should be included in mails 121 | func (opt GlobalOptions) WithPartHtml() bool { 122 | return slices.Contains(opt.Parts, "html") 123 | } 124 | 125 | // Load configuration from file and validate it 126 | func Load(path string) (*Config, error) { 127 | log.Printf("Reading configuration file '%s'", path) 128 | 129 | f, err := os.Open(path) 130 | if err != nil { 131 | return nil, fmt.Errorf("while opening '%s': %w", path, err) 132 | } 133 | 134 | defer f.Close() 135 | 136 | stat, err := f.Stat() 137 | if err != nil { 138 | return nil, fmt.Errorf("while getting stats of '%s': %w", path, err) 139 | } 140 | 141 | if stat.Mode().Perm()&0004 != 0 { 142 | log.Warnf("Config file '%s' can be read by anyone. As this contains your IMAP credentials, you are advised to remove global read access.", path) 143 | } 144 | 145 | cfg := WithDefault() 146 | if err = cfg.parse(f); err != nil { 147 | return nil, fmt.Errorf("while parsing: %w", err) 148 | } 149 | 150 | if err = cfg.Validate(); err != nil { 151 | return nil, fmt.Errorf("Configuration invalid: %w", err) 152 | } 153 | 154 | return cfg, nil 155 | } 156 | 157 | // Hostname returns the current hostname, or 'localhost' if it cannot be determined 158 | func Hostname() (hostname string) { 159 | hostname, err := os.Hostname() 160 | if err != nil { 161 | hostname = "localhost" 162 | } 163 | return 164 | } 165 | 166 | func username() string { 167 | u, err := user.Current() 168 | switch { 169 | case err != nil: 170 | return "user" 171 | case runtime.GOOS == "windows": 172 | // the domain is attached -- remove it again 173 | split := strings.Split(u.Username, "\\") 174 | return split[len(split)-1] 175 | default: 176 | return u.Username 177 | } 178 | } 179 | -------------------------------------------------------------------------------- /pkg/config/deprecated.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/Necoro/feed2imap-go/pkg/log" 7 | ) 8 | 9 | type deprecated struct { 10 | msg string 11 | handle func(any, *GlobalOptions, *Options) 12 | } 13 | 14 | var unsupported = deprecated{ 15 | "It won't be supported and is ignored!", 16 | nil, 17 | } 18 | 19 | var deprecatedOpts = map[string]deprecated{ 20 | "dumpdir": unsupported, 21 | "debug-updated": {"Use '-d' as option instead.", nil}, 22 | "execurl": {"Use 'exec' instead.", nil}, 23 | "filter": {"Use 'item-filter' instead.", nil}, 24 | "disable-ssl-verification": {"Interpreted as 'tls-no-verify'.", func(i any, global *GlobalOptions, opts *Options) { 25 | if val, ok := i.(bool); ok { 26 | if val && !opts.NoTLS { 27 | // do not overwrite the set NoTLS flag! 28 | opts.NoTLS = val 29 | } 30 | } else { 31 | log.Errorf("disable-ssl-verification: value '%v' cannot be interpreted as a boolean. Ignoring!", i) 32 | } 33 | }}, 34 | } 35 | 36 | func handleDeprecated(option string, value any, feed string, global *GlobalOptions, opts *Options) bool { 37 | dep, ok := deprecatedOpts[option] 38 | if !ok { 39 | return false 40 | } 41 | 42 | var prefix string 43 | if feed != "" { 44 | prefix = fmt.Sprintf("Feed '%s': ", feed) 45 | } else { 46 | prefix = "Global " 47 | } 48 | 49 | log.Warnf("%sOption '%s' is deprecated: %s", prefix, option, dep.msg) 50 | 51 | if dep.handle != nil { 52 | dep.handle(value, global, opts) 53 | } 54 | 55 | return true 56 | } 57 | -------------------------------------------------------------------------------- /pkg/config/feed.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | // One stored feed 4 | type Feed struct { 5 | Name string 6 | Target []string 7 | Url string 8 | Exec []string 9 | Options 10 | } 11 | 12 | // Convenience type for all feeds 13 | type Feeds map[string]*Feed 14 | -------------------------------------------------------------------------------- /pkg/config/url.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | "net/url" 7 | "strings" 8 | 9 | "gopkg.in/yaml.v3" 10 | ) 11 | 12 | type Url struct { 13 | Scheme string `yaml:"scheme"` 14 | User string `yaml:"user"` 15 | Password string `yaml:"password"` 16 | Host string `yaml:"host"` 17 | Port string `yaml:"port"` 18 | Root string `yaml:"root"` 19 | } 20 | 21 | func (u *Url) Empty() bool { 22 | return u.Host == "" 23 | } 24 | 25 | func (u *Url) EmptyRoot() bool { 26 | return u.Root == "" || u.Root == "/" 27 | } 28 | 29 | func (u *Url) UnmarshalYAML(value *yaml.Node) (err error) { 30 | if value.ShortTag() == strTag { 31 | var val string 32 | var rawUrl *url.URL 33 | 34 | if err = value.Decode(&val); err != nil { 35 | return err 36 | } 37 | if rawUrl, err = url.Parse(val); err != nil { 38 | return err 39 | } 40 | 41 | u.Scheme = rawUrl.Scheme 42 | u.User = rawUrl.User.Username() 43 | u.Password, _ = rawUrl.User.Password() 44 | u.Host = rawUrl.Hostname() 45 | u.Port = rawUrl.Port() 46 | u.Root = rawUrl.Path 47 | } else { 48 | type _url Url // avoid recursion 49 | wrapped := (*_url)(u) 50 | if err = value.Decode(wrapped); err != nil { 51 | return err 52 | } 53 | } 54 | 55 | u.sanitize() 56 | 57 | if errors := u.validate(); len(errors) > 0 { 58 | errs := make([]string, len(errors)+1) 59 | copy(errs[1:], errors) 60 | errs[0] = fmt.Sprintf("line %d: Invalid target:", value.Line) 61 | return &yaml.TypeError{Errors: errs} 62 | } 63 | 64 | return nil 65 | } 66 | 67 | func (u Url) String() string { 68 | var sb strings.Builder 69 | 70 | sb.WriteString(u.Scheme) 71 | sb.WriteString("://") 72 | 73 | if u.User != "" { 74 | sb.WriteString(u.User) 75 | if u.Password != "" { 76 | sb.WriteString(":******") 77 | } 78 | sb.WriteRune('@') 79 | } 80 | 81 | sb.WriteString(u.HostPort()) 82 | 83 | if u.Root != "" { 84 | if u.Root[0] != '/' { 85 | sb.WriteRune('/') 86 | } 87 | sb.WriteString(u.Root) 88 | } 89 | 90 | return sb.String() 91 | } 92 | 93 | func (u *Url) HostPort() string { 94 | if u.Port != "" { 95 | return net.JoinHostPort(u.Host, u.Port) 96 | } 97 | return u.Host 98 | } 99 | 100 | const ( 101 | imapsPort = "993" 102 | imapPort = "143" 103 | imapsSchema = "imaps" 104 | imapsSchemaFull = imapsSchema + "://" 105 | imapSchema = "imap" 106 | imapSchemaFull = imapSchema + "://" 107 | maildirSchemaFull = "maildir://" 108 | ) 109 | 110 | func isRecognizedUrl(s string) bool { 111 | return isImapUrl(s) || isMaildirUrl(s) 112 | 113 | } 114 | 115 | func isImapUrl(s string) bool { 116 | return strings.HasPrefix(s, imapsSchemaFull) || strings.HasPrefix(s, imapSchemaFull) 117 | } 118 | 119 | func isMaildirUrl(s string) bool { 120 | return strings.HasPrefix(s, maildirSchemaFull) 121 | } 122 | 123 | func (u *Url) ForceTLS() bool { 124 | return u.Scheme == imapsSchema || u.Port == imapsPort 125 | } 126 | 127 | func (u *Url) setDefaultScheme() { 128 | if u.Scheme == "" { 129 | if u.Port == imapsPort { 130 | u.Scheme = imapsSchema 131 | } else { 132 | u.Scheme = imapSchema 133 | } 134 | } 135 | } 136 | 137 | func (u *Url) setDefaultPort() { 138 | if u.Port == "" { 139 | if u.Scheme == imapsSchema { 140 | u.Port = imapsPort 141 | } else { 142 | u.Port = imapPort 143 | } 144 | } 145 | } 146 | 147 | func (u *Url) sanitize() { 148 | u.setDefaultScheme() 149 | u.setDefaultPort() 150 | } 151 | 152 | func (u *Url) validate() (errors []string) { 153 | if u.Scheme != imapSchema && u.Scheme != imapsSchema { 154 | errors = append(errors, fmt.Sprintf("Unknown scheme %q", u.Scheme)) 155 | } 156 | 157 | if u.Host == "" { 158 | errors = append(errors, "Host not set") 159 | } 160 | 161 | return 162 | } 163 | 164 | func (u Url) BaseUrl() Url { 165 | // 'u' is not a pointer and thus a copy, modification is fine 166 | u.Root = "" 167 | return u 168 | } 169 | 170 | func (u *Url) CommonBaseUrl(other Url) bool { 171 | other.Root = "" 172 | return other == u.BaseUrl() 173 | } 174 | 175 | func (u *Url) RootPath() []string { 176 | path := u.Root 177 | if path != "" && path[0] == '/' { 178 | path = path[1:] 179 | } 180 | return strings.Split(path, "/") 181 | } 182 | -------------------------------------------------------------------------------- /pkg/config/url_test.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/google/go-cmp/cmp" 7 | "gopkg.in/yaml.v3" 8 | ) 9 | 10 | func TestUrl_Unmarshal(t *testing.T) { 11 | 12 | tests := []struct { 13 | name string 14 | inp string 15 | url Url 16 | wantErr bool 17 | str string 18 | }{ 19 | {name: "Empty", inp: `url: ""`, wantErr: true}, 20 | {name: "Simple String", inp: `url: "imap://user:pass@example.net:143/INBOX"`, url: Url{ 21 | Scheme: "imap", 22 | User: "user", 23 | Password: "pass", 24 | Host: "example.net", 25 | Port: "143", 26 | Root: "/INBOX", 27 | }, str: "imap://user:******@example.net:143/INBOX"}, 28 | {name: "Simple String with @", inp: `url: "imaps://user@example:pass@example.net:143/INBOX"`, url: Url{ 29 | Scheme: "imaps", 30 | User: "user@example", 31 | Password: "pass", 32 | Host: "example.net", 33 | Port: "143", 34 | Root: "/INBOX", 35 | }, str: "imaps://user@example:******@example.net:143/INBOX"}, 36 | {name: "Simple String with %40", inp: `url: "imap://user%40example:pass@example.net:4711/INBOX"`, url: Url{ 37 | Scheme: "imap", 38 | User: "user@example", 39 | Password: "pass", 40 | Host: "example.net", 41 | Port: "4711", 42 | Root: "/INBOX", 43 | }, str: "imap://user@example:******@example.net:4711/INBOX"}, 44 | {name: "Simple String without user", inp: `url: "imap://example.net:143/INBOX"`, url: Url{ 45 | Scheme: "imap", 46 | User: "", 47 | Password: "", 48 | Host: "example.net", 49 | Port: "143", 50 | Root: "/INBOX", 51 | }, str: "imap://example.net:143/INBOX"}, 52 | {name: "Err: Inv scheme", inp: `url: "smtp://user%40example:pass@example.net:4711/INBOX"`, wantErr: true}, 53 | {name: "Err: No Host", inp: `url: "imap://user%40example:pass/INBOX"`, wantErr: true}, 54 | {name: "Err: Scheme Only", inp: `url: "imap://"`, wantErr: true}, 55 | {name: "No Root", inp: `url: "imap://user:pass@example.net:143"`, url: Url{ 56 | Scheme: "imap", 57 | User: "user", 58 | Password: "pass", 59 | Host: "example.net", 60 | Port: "143", 61 | Root: "", 62 | }, str: "imap://user:******@example.net:143"}, 63 | {name: "No Root: Slash", inp: `url: "imap://user:pass@example.net:143/"`, url: Url{ 64 | Scheme: "imap", 65 | User: "user", 66 | Password: "pass", 67 | Host: "example.net", 68 | Port: "143", 69 | Root: "/", 70 | }, str: "imap://user:******@example.net:143/"}, 71 | {name: "Full", inp: `url: 72 | scheme: imap 73 | host: example.net 74 | user: user 75 | password: p4ss 76 | port: 143 77 | root: INBOX 78 | `, url: Url{ 79 | Scheme: "imap", 80 | User: "user", 81 | Password: "p4ss", 82 | Host: "example.net", 83 | Port: "143", 84 | Root: "INBOX", 85 | }, str: "imap://user:******@example.net:143/INBOX"}, 86 | {name: "Default Port", inp: `url: 87 | scheme: imap 88 | host: example.net 89 | user: user 90 | password: p4ss 91 | root: INBOX 92 | `, url: Url{ 93 | Scheme: "imap", 94 | User: "user", 95 | Password: "p4ss", 96 | Host: "example.net", 97 | Port: "143", 98 | Root: "INBOX", 99 | }, str: "imap://user:******@example.net:143/INBOX"}, 100 | {name: "Default Scheme", inp: `url: 101 | host: example.net 102 | user: user 103 | password: p4ss 104 | port: 993 105 | root: INBOX 106 | `, url: Url{ 107 | Scheme: "imaps", 108 | User: "user", 109 | Password: "p4ss", 110 | Host: "example.net", 111 | Port: "993", 112 | Root: "INBOX", 113 | }, str: "imaps://user:******@example.net:993/INBOX"}, 114 | } 115 | for _, tt := range tests { 116 | t.Run(tt.name, func(t *testing.T) { 117 | var u struct { 118 | Url Url `yaml:"url"` 119 | } 120 | err := yaml.Unmarshal([]byte(tt.inp), &u) 121 | if (err != nil) != tt.wantErr { 122 | t.Errorf("Unmarshal() error = %v, wantErr %v", err, tt.wantErr) 123 | return 124 | } 125 | 126 | if diff := cmp.Diff(u.Url, tt.url); err == nil && diff != "" { 127 | t.Error(diff) 128 | } 129 | 130 | if diff := cmp.Diff(u.Url.String(), tt.str); err == nil && diff != "" { 131 | t.Error(diff) 132 | } 133 | }) 134 | } 135 | } 136 | -------------------------------------------------------------------------------- /pkg/config/yaml.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "io" 7 | "strings" 8 | 9 | "github.com/mitchellh/mapstructure" 10 | "gopkg.in/yaml.v3" 11 | 12 | "github.com/Necoro/feed2imap-go/pkg/log" 13 | ) 14 | 15 | const ( 16 | strTag = "!!str" 17 | nullTag = "!!null" 18 | ) 19 | 20 | type config struct { 21 | *Config `yaml:",inline"` 22 | GlobalConfig Map `yaml:",inline"` 23 | Feeds []configGroupFeed 24 | } 25 | 26 | type group struct { 27 | Group string 28 | Feeds []configGroupFeed 29 | } 30 | 31 | type feed struct { 32 | Name string 33 | Url string 34 | Exec []string 35 | } 36 | 37 | type configGroupFeed struct { 38 | Target yaml.Node 39 | Feed feed `yaml:",inline"` 40 | Group group `yaml:",inline"` 41 | Options Map `yaml:",inline"` 42 | } 43 | 44 | func (grpFeed *configGroupFeed) isGroup() bool { 45 | return grpFeed.Group.Group != "" 46 | } 47 | 48 | func (grpFeed *configGroupFeed) isFeed() bool { 49 | return grpFeed.Feed.Name != "" || grpFeed.Feed.Url != "" || len(grpFeed.Feed.Exec) > 0 50 | } 51 | 52 | func (grpFeed *configGroupFeed) target(autoTarget bool) string { 53 | if !autoTarget || !grpFeed.Target.IsZero() { 54 | if grpFeed.Target.ShortTag() == nullTag { 55 | // null may be represented by ~ or NULL or ... 56 | // Value would hold this representation, which we do not want 57 | return "" 58 | } 59 | return grpFeed.Target.Value 60 | } 61 | 62 | if grpFeed.Feed.Name != "" { 63 | return grpFeed.Feed.Name 64 | } 65 | 66 | return grpFeed.Group.Group 67 | } 68 | 69 | func unmarshal(in io.Reader, cfg *Config) (config, error) { 70 | parsedCfg := config{Config: cfg} 71 | 72 | d := yaml.NewDecoder(in) 73 | d.KnownFields(true) 74 | if err := d.Decode(&parsedCfg); err != nil && err != io.EOF { 75 | return config{}, err 76 | } 77 | 78 | return parsedCfg, nil 79 | } 80 | 81 | func (cfg *Config) fixGlobalOptions(unparsed Map) { 82 | origMap := Map{} 83 | 84 | // copy map 85 | for k, v := range unparsed { 86 | origMap[k] = v 87 | } 88 | 89 | newOpts, _ := buildOptions(&cfg.FeedOptions, unparsed) 90 | 91 | for k, v := range origMap { 92 | if _, ok := unparsed[k]; !ok { 93 | log.Warnf("Global option '%s' should be inside the 'options' map. It currently overwrites the same key there.", k) 94 | } else if !handleDeprecated(k, v, "", &cfg.GlobalOptions, &newOpts) { 95 | log.Warnf("Unknown global option '%s'. Ignored!", k) 96 | } 97 | } 98 | 99 | cfg.FeedOptions = newOpts 100 | } 101 | 102 | func (cfg *Config) parse(in io.Reader) error { 103 | var ( 104 | err error 105 | parsedCfg config 106 | ) 107 | 108 | if parsedCfg, err = unmarshal(in, cfg); err != nil { 109 | var typeError *yaml.TypeError 110 | if errors.As(err, &typeError) { 111 | const sep = "\n\t" 112 | errMsgs := strings.Join(typeError.Errors, sep) 113 | return fmt.Errorf("config is invalid: %s%s", sep, errMsgs) 114 | } 115 | 116 | return fmt.Errorf("while unmarshalling: %w", err) 117 | } 118 | 119 | cfg.fixGlobalOptions(parsedCfg.GlobalConfig) 120 | 121 | if err := buildFeeds(parsedCfg.Feeds, []string{}, cfg.Feeds, &cfg.FeedOptions, cfg.AutoTarget, &cfg.Target); err != nil { 122 | return err 123 | } 124 | 125 | return nil 126 | } 127 | 128 | func appTarget(target []string, app string) []string { 129 | app = strings.TrimSpace(app) 130 | switch { 131 | case len(target) == 0 && app == "": 132 | return []string{} 133 | case len(target) == 0: 134 | return []string{app} 135 | case app == "": 136 | return target 137 | default: 138 | return append(target, app) 139 | } 140 | } 141 | 142 | func buildOptions(globalFeedOptions *Options, options Map) (Options, []string) { 143 | // copy global as default 144 | feedOptions := *globalFeedOptions 145 | 146 | if options == nil { 147 | // no options set for the feed: copy global options and be done 148 | return feedOptions, []string{} 149 | } 150 | 151 | var md mapstructure.Metadata 152 | mapstructureConfig := mapstructure.DecoderConfig{ 153 | TagName: "yaml", 154 | Metadata: &md, 155 | Result: &feedOptions, 156 | } 157 | 158 | var err error 159 | dec, err := mapstructure.NewDecoder(&mapstructureConfig) 160 | if err != nil { 161 | panic(err) 162 | } 163 | 164 | err = dec.Decode(options) 165 | if err != nil { 166 | panic(err) 167 | } 168 | 169 | return feedOptions, md.Unused 170 | } 171 | 172 | // Fetch the group structure and populate the `targetStr` fields in the feeds 173 | func buildFeeds(cfg []configGroupFeed, target []string, feeds Feeds, 174 | globalFeedOptions *Options, autoTarget bool, globalTarget *Url) (err error) { 175 | 176 | for _, f := range cfg { 177 | var fTarget []string 178 | 179 | rawTarget := f.target(autoTarget) 180 | if isRecognizedUrl(rawTarget) { 181 | // deprecated old-style URLs as target 182 | // as the full path is specified, `target` is ignored 183 | if fTarget, err = handleUrlTarget(rawTarget, &f.Target, globalTarget); err != nil { 184 | return err 185 | } 186 | } else { 187 | // new-style tree-like structure 188 | fTarget = appTarget(target, rawTarget) 189 | } 190 | 191 | switch { 192 | case f.isFeed() && f.isGroup(): 193 | return fmt.Errorf("Entry with targetStr %s is both a Feed and a group", fTarget) 194 | 195 | case f.isFeed(): 196 | name := f.Feed.Name 197 | if name == "" { 198 | return fmt.Errorf("Unnamed feed") 199 | } 200 | if _, ok := feeds[name]; ok { 201 | return fmt.Errorf("Duplicate Feed Name '%s'", name) 202 | } 203 | if len(f.Group.Feeds) > 0 { 204 | return fmt.Errorf("Feed '%s' tries to also be a group.", name) 205 | } 206 | if f.Feed.Url == "" && len(f.Feed.Exec) == 0 { 207 | return fmt.Errorf("Feed '%s' has not specified a URL or an Exec clause.", name) 208 | } 209 | 210 | opt, unknown := buildOptions(globalFeedOptions, f.Options) 211 | 212 | for _, optName := range unknown { 213 | if !handleDeprecated(optName, f.Options[optName], name, nil, &opt) { 214 | log.Warnf("Unknown option '%s' for feed '%s'. Ignored!", optName, name) 215 | } 216 | } 217 | 218 | feeds[name] = &Feed{ 219 | Name: name, 220 | Url: f.Feed.Url, 221 | Exec: f.Feed.Exec, 222 | Options: opt, 223 | Target: fTarget, 224 | } 225 | 226 | case f.isGroup(): 227 | if len(f.Group.Feeds) == 0 { 228 | log.Warnf("Group '%s' does not contain any feeds.", f.Group.Group) 229 | } 230 | 231 | opt, unknown := buildOptions(globalFeedOptions, f.Options) 232 | 233 | for _, optName := range unknown { 234 | log.Warnf("Unknown option '%s' for group '%s'. Ignored!", optName, f.Group.Group) 235 | } 236 | 237 | if err = buildFeeds(f.Group.Feeds, fTarget, feeds, &opt, autoTarget, globalTarget); err != nil { 238 | return err 239 | } 240 | } 241 | } 242 | 243 | return nil 244 | } 245 | 246 | func handleUrlTarget(targetStr string, targetNode *yaml.Node, globalTarget *Url) ([]string, error) { 247 | // this whole function is solely for compatibility with old feed2imap 248 | // there it was common to specify the whole URL for each feed 249 | if isMaildirUrl(targetStr) { 250 | // old feed2imap supported maildir, we don't 251 | return nil, fmt.Errorf("Line %d: Maildir is not supported.", targetNode.Line) 252 | } 253 | 254 | url := Url{} 255 | if err := url.UnmarshalYAML(targetNode); err != nil { 256 | return nil, err 257 | } 258 | 259 | if globalTarget.Empty() { 260 | // assign first feed as global url 261 | *globalTarget = url.BaseUrl() 262 | } else if !globalTarget.CommonBaseUrl(url) { 263 | // if we have a url, it must be the same prefix as the global url 264 | return nil, fmt.Errorf("Line %d: Given URL endpoint '%s' does not match previous endpoint '%s'.", 265 | targetNode.Line, 266 | url.BaseUrl(), 267 | globalTarget.BaseUrl()) 268 | } 269 | 270 | return url.RootPath(), nil 271 | } 272 | -------------------------------------------------------------------------------- /pkg/config/yaml_test.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "sort" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/google/go-cmp/cmp" 9 | "gopkg.in/yaml.v3" 10 | ) 11 | 12 | func t(s string) []string { 13 | if s == "" { 14 | return []string{} 15 | } 16 | return strings.Split(s, ".") 17 | } 18 | func n(s string) (n yaml.Node) { 19 | n.SetString(s) 20 | return 21 | } 22 | 23 | var null = yaml.Node{Tag: nullTag} 24 | 25 | func TestBuildOptions(tst *testing.T) { 26 | tests := []struct { 27 | name string 28 | inp Map 29 | opts Options 30 | out Options 31 | unknowns []string 32 | }{ 33 | {"Empty", nil, Options{}, Options{}, []string{}}, 34 | {"Simple copy", nil, Options{MinFreq: 75}, Options{MinFreq: 75}, []string{}}, 35 | {"Unknowns", Map{"foo": 1}, Options{}, Options{}, []string{"foo"}}, 36 | {"Override", Map{"include-images": true}, Options{InclImages: false}, Options{InclImages: true}, []string{}}, 37 | {"Non-Standard Type", Map{"body": "both"}, Options{}, Options{Body: "both"}, []string{}}, 38 | {"Mixed", Map{"min-frequency": 24}, Options{MinFreq: 6, InclImages: true}, Options{MinFreq: 24, InclImages: true}, []string{}}, 39 | {"All", 40 | Map{"max-frequency": 12, "include-images": true, "ignore-hash": true, "obsolete": 54}, 41 | Options{MinFreq: 6, InclImages: true, IgnHash: false}, 42 | Options{MinFreq: 6, InclImages: true, IgnHash: true}, 43 | []string{"max-frequency", "obsolete"}, 44 | }, 45 | } 46 | 47 | for _, tt := range tests { 48 | tst.Run(tt.name, func(tst *testing.T) { 49 | out, unk := buildOptions(&tt.opts, tt.inp) 50 | 51 | if diff := cmp.Diff(tt.out, out); diff != "" { 52 | tst.Error(diff) 53 | } 54 | 55 | sort.Strings(unk) 56 | sort.Strings(tt.unknowns) 57 | 58 | if diff := cmp.Diff(unk, tt.unknowns); diff != "" { 59 | tst.Error(diff) 60 | } 61 | }) 62 | } 63 | } 64 | 65 | func TestBuildFeeds(tst *testing.T) { 66 | tests := []struct { 67 | name string 68 | wantErr bool 69 | errMsg string 70 | target string 71 | feeds []configGroupFeed 72 | result Feeds 73 | noAutoTarget bool 74 | }{ 75 | {name: "Empty input", wantErr: false, target: "", feeds: nil, result: Feeds{}}, 76 | {name: "Empty Feed", wantErr: true, errMsg: "Unnamed feed", 77 | target: "", 78 | feeds: []configGroupFeed{ 79 | {Target: n("foo"), Feed: feed{Url: "google.de"}}, 80 | }, result: Feeds{}}, 81 | {name: "Empty Feed", wantErr: true, errMsg: "Unnamed feed", 82 | target: "", 83 | feeds: []configGroupFeed{ 84 | {Feed: feed{Url: "google.de"}}, 85 | }, result: Feeds{}}, 86 | {name: "Duplicate Feed Name", wantErr: true, errMsg: "Duplicate Feed Name 'Dup'", target: "", 87 | feeds: []configGroupFeed{ 88 | {Feed: feed{Name: "Dup", Url: "google.de"}}, 89 | {Feed: feed{Name: "Dup", Url: "bing.de"}}, 90 | }, result: Feeds{}}, 91 | {name: "No URL", wantErr: true, errMsg: "Feed 'muh' has not specified a URL or an Exec clause.", target: "", 92 | feeds: []configGroupFeed{ 93 | {Target: n("foo"), Feed: feed{Name: "muh"}}, 94 | }, 95 | result: Feeds{}, 96 | }, 97 | {name: "Simple", wantErr: false, target: "", 98 | feeds: []configGroupFeed{ 99 | {Target: n("foo"), Feed: feed{Name: "muh", Url: "google.de"}}, 100 | }, 101 | result: Feeds{"muh": &Feed{Name: "muh", Target: t("foo"), Url: "google.de"}}, 102 | }, 103 | {name: "Simple With Target", wantErr: false, target: "moep", 104 | feeds: []configGroupFeed{ 105 | {Target: n("foo"), Feed: feed{Name: "muh", Url: "google.de"}}, 106 | }, 107 | result: Feeds{"muh": &Feed{Name: "muh", Url: "google.de", Target: t("moep.foo")}}, 108 | }, 109 | {name: "Simple With Target and Whitespace", wantErr: false, target: "moep", 110 | feeds: []configGroupFeed{ 111 | {Target: n("\r\nfoo "), Feed: feed{Name: "muh", Url: "google.de"}}, 112 | }, 113 | result: Feeds{"muh": &Feed{Name: "muh", Url: "google.de", Target: t("moep.foo")}}, 114 | }, 115 | {name: "Simple With Target and NoAutoTarget", wantErr: false, target: "moep", noAutoTarget: true, 116 | feeds: []configGroupFeed{ 117 | {Target: n("foo"), Feed: feed{Name: "muh", Url: "google.de"}}, 118 | }, 119 | result: Feeds{"muh": &Feed{Name: "muh", Url: "google.de", Target: t("moep.foo")}}, 120 | }, 121 | {name: "Simple Without Target", wantErr: false, target: "moep", 122 | feeds: []configGroupFeed{ 123 | {Feed: feed{Name: "muh", Url: "google.de"}}, 124 | }, 125 | result: Feeds{"muh": &Feed{Name: "muh", Url: "google.de", Target: t("moep.muh")}}, 126 | }, 127 | {name: "Simple Without Target and NoAutoTarget", wantErr: false, target: "moep", noAutoTarget: true, 128 | feeds: []configGroupFeed{ 129 | {Feed: feed{Name: "muh", Url: "google.de"}}, 130 | }, 131 | result: Feeds{"muh": &Feed{Name: "muh", Url: "google.de", Target: t("moep")}}, 132 | }, 133 | {name: "Simple With Nil Target", wantErr: false, target: "moep", 134 | feeds: []configGroupFeed{ 135 | {Target: null, Feed: feed{Name: "muh", Url: "google.de"}}, 136 | }, 137 | result: Feeds{"muh": &Feed{Name: "muh", Url: "google.de", Target: t("moep")}}, 138 | }, 139 | {name: "Simple With Empty Target", wantErr: false, target: "moep", 140 | feeds: []configGroupFeed{ 141 | {Target: n(""), Feed: feed{Name: "muh", Url: "google.de"}}, 142 | }, 143 | result: Feeds{"muh": &Feed{Name: "muh", Url: "google.de", Target: t("moep")}}, 144 | }, 145 | {name: "Simple With Blank Target", wantErr: false, target: "moep", 146 | feeds: []configGroupFeed{ 147 | {Target: n(" "), Feed: feed{Name: "muh", Url: "google.de"}}, 148 | }, 149 | result: Feeds{"muh": &Feed{Name: "muh", Url: "google.de", Target: t("moep")}}, 150 | }, 151 | {name: "Multiple Feeds", wantErr: false, target: "moep", 152 | feeds: []configGroupFeed{ 153 | {Target: n("foo"), Feed: feed{Name: "muh", Url: "google.de"}}, 154 | {Feed: feed{Name: "bar", Url: "bing.de"}}, 155 | }, 156 | result: Feeds{ 157 | "muh": &Feed{Name: "muh", Url: "google.de", Target: t("moep.foo")}, 158 | "bar": &Feed{Name: "bar", Url: "bing.de", Target: t("moep.bar")}, 159 | }, 160 | }, 161 | {name: "URL Target", wantErr: false, target: "", 162 | feeds: []configGroupFeed{ 163 | {Target: n("imap://foo.bar:443/INBOX/Feed"), Feed: feed{Name: "muh", Url: "google.de"}}, 164 | }, 165 | result: Feeds{"muh": &Feed{Name: "muh", Url: "google.de", Target: t("INBOX.Feed")}}, 166 | }, 167 | {name: "Multiple URL Targets", wantErr: false, target: "", 168 | feeds: []configGroupFeed{ 169 | {Target: n("imap://foo.bar:443/INBOX/Feed"), Feed: feed{Name: "muh", Url: "google.de"}}, 170 | {Target: n("imap://foo.bar:443/INBOX/Feed2"), Feed: feed{Name: "bar", Url: "bing.de"}}, 171 | }, 172 | result: Feeds{ 173 | "muh": &Feed{Name: "muh", Url: "google.de", Target: t("INBOX.Feed")}, 174 | "bar": &Feed{Name: "bar", Url: "bing.de", Target: t("INBOX.Feed2")}, 175 | }, 176 | }, 177 | {name: "Mixed URL Targets", wantErr: true, errMsg: "Line 0: Given URL endpoint 'imap://other.bar:443' does not match previous endpoint 'imap://foo.bar:443'.", target: "", 178 | feeds: []configGroupFeed{ 179 | {Target: n("imap://foo.bar:443/INBOX/Feed"), Feed: feed{Name: "muh", Url: "google.de"}}, 180 | {Target: n("imap://other.bar:443/INBOX/Feed"), Feed: feed{Name: "bar", Url: "bing.de"}}, 181 | }, 182 | result: Feeds{}, 183 | }, 184 | {name: "Maildir URL Target", wantErr: true, errMsg: "Line 0: Maildir is not supported.", target: "", 185 | feeds: []configGroupFeed{ 186 | {Target: n("maildir:///home/foo/INBOX/Feed"), Feed: feed{Name: "muh"}}, 187 | }, 188 | result: Feeds{}, 189 | }, 190 | {name: "Empty Group", wantErr: false, target: "", 191 | feeds: []configGroupFeed{ 192 | {Group: group{Group: "G1"}}, 193 | }, 194 | result: Feeds{}, 195 | }, 196 | {name: "Group by accident", wantErr: true, errMsg: "Feed 'muh' tries to also be a group.", target: "", 197 | feeds: []configGroupFeed{ 198 | { 199 | Feed: feed{Name: "muh"}, 200 | Group: group{Feeds: []configGroupFeed{ 201 | {Feed: feed{Name: "F1", Url: "F1"}}, 202 | }}, 203 | }, 204 | }, 205 | result: Feeds{}, 206 | }, 207 | {name: "Simple Group", wantErr: false, target: "", 208 | feeds: []configGroupFeed{ 209 | {Group: group{Group: "G1", Feeds: []configGroupFeed{ 210 | {Target: n("bar"), Feed: feed{Name: "F1", Url: "F1"}}, 211 | {Target: n(""), Feed: feed{Name: "F2", Url: "F2"}}, 212 | {Feed: feed{Name: "F3", Url: "F3"}}, 213 | }}}, 214 | }, 215 | result: Feeds{ 216 | "F1": &Feed{Name: "F1", Url: "F1", Target: t("G1.bar")}, 217 | "F2": &Feed{Name: "F2", Url: "F2", Target: t("G1")}, 218 | "F3": &Feed{Name: "F3", Url: "F3", Target: t("G1.F3")}, 219 | }, 220 | }, 221 | {name: "Simple Group, NoAutoTarget", wantErr: false, target: "IN", noAutoTarget: true, 222 | feeds: []configGroupFeed{ 223 | {Group: group{Group: "G1", Feeds: []configGroupFeed{ 224 | {Target: n("bar"), Feed: feed{Name: "F1", Url: "F1"}}, 225 | {Target: n(""), Feed: feed{Name: "F2", Url: "F2"}}, 226 | {Feed: feed{Name: "F3", Url: "F3"}}, 227 | }}}, 228 | }, 229 | result: Feeds{ 230 | "F1": &Feed{Name: "F1", Url: "F1", Target: t("IN.bar")}, 231 | "F2": &Feed{Name: "F2", Url: "F2", Target: t("IN")}, 232 | "F3": &Feed{Name: "F3", Url: "F3", Target: t("IN")}, 233 | }, 234 | }, 235 | {name: "Nested Groups", wantErr: false, target: "", 236 | feeds: []configGroupFeed{ 237 | {Group: group{Group: "G1", Feeds: []configGroupFeed{ 238 | {Feed: feed{Name: "F0", Url: "F0"}}, 239 | {Target: n("bar"), Group: group{Group: "G2", 240 | Feeds: []configGroupFeed{{Feed: feed{Name: "F1", Url: "F1"}}}}}, 241 | {Target: n(""), Group: group{Group: "G3", 242 | Feeds: []configGroupFeed{{Target: n("baz"), Feed: feed{Name: "F2", Url: "F2"}}}}}, 243 | {Group: group{Group: "G4", 244 | Feeds: []configGroupFeed{{Feed: feed{Name: "F3", Url: "F3"}}}}}, 245 | }}}, 246 | }, 247 | result: Feeds{ 248 | "F0": &Feed{Name: "F0", Url: "F0", Target: t("G1.F0")}, 249 | "F1": &Feed{Name: "F1", Url: "F1", Target: t("G1.bar.F1")}, 250 | "F2": &Feed{Name: "F2", Url: "F2", Target: t("G1.baz")}, 251 | "F3": &Feed{Name: "F3", Url: "F3", Target: t("G1.G4.F3")}, 252 | }, 253 | }, 254 | } 255 | for _, tt := range tests { 256 | tst.Run(tt.name, func(tst *testing.T) { 257 | var feeds = Feeds{} 258 | var opts = Options{} 259 | var globalTarget = Url{} 260 | err := buildFeeds(tt.feeds, t(tt.target), feeds, &opts, !tt.noAutoTarget, &globalTarget) 261 | if tt.wantErr { 262 | if err == nil { 263 | tst.Error("Excepted error, but was successfull.") 264 | } else if diff := cmp.Diff(tt.errMsg, err.Error()); diff != "" { 265 | tst.Error(diff) 266 | } 267 | } else { 268 | if err != nil { 269 | tst.Errorf("Unexpected error %v", err) 270 | } else if diff := cmp.Diff(tt.result, feeds); diff != "" { 271 | tst.Error(diff) 272 | } 273 | } 274 | }) 275 | } 276 | } 277 | 278 | func defaultConfig(feeds []configGroupFeed, global Map) config { 279 | defCfg := WithDefault() 280 | return config{ 281 | Config: defCfg, 282 | Feeds: feeds, 283 | GlobalConfig: global, 284 | } 285 | } 286 | 287 | func TestUnmarshal(tst *testing.T) { 288 | tests := []struct { 289 | name string 290 | inp string 291 | wantErr bool 292 | errMsg string 293 | config config 294 | }{ 295 | {name: "Empty", 296 | inp: "", wantErr: false, config: defaultConfig(nil, nil)}, 297 | {name: "Trash", inp: "Something", wantErr: true, errMsg: "yaml: unmarshal errors:\n line 1: cannot unmarshal !!str `Something` into config.config"}, 298 | {name: "Simple config", 299 | inp: "something: 1\nsomething_else: 2", wantErr: false, config: defaultConfig(nil, Map{"something": 1, "something_else": 2})}, 300 | {name: "Known config", 301 | inp: "whatever: 2\ndefault-email: foo@foobar.de\ntimeout: 60\nsomething: 1", wantErr: false, config: func() config { 302 | c := defaultConfig(nil, Map{"something": 1, "whatever": 2}) 303 | c.Timeout = 60 304 | c.DefaultEmail = "foo@foobar.de" 305 | return c 306 | }()}, 307 | {name: "Known config with feed-options", 308 | inp: "whatever: 2\ntimeout: 60\noptions:\n min-frequency: 6", wantErr: false, config: func() config { 309 | c := defaultConfig(nil, Map{"whatever": 2}) 310 | c.Timeout = 60 311 | c.FeedOptions.MinFreq = 6 312 | return c 313 | }()}, 314 | {name: "Known config with invalid feed-options", 315 | inp: "options:\n max-frequency: 6", 316 | wantErr: true, errMsg: "yaml: unmarshal errors:\n line 2: field max-frequency not found in type config.Options", 317 | config: config{}}, 318 | {name: "Config with feed", 319 | inp: ` 320 | something: 1 321 | feeds: 322 | - name: Foo 323 | url: whatever 324 | target: bar 325 | include-images: true 326 | unknown-option: foo 327 | `, 328 | wantErr: false, 329 | config: defaultConfig([]configGroupFeed{{ 330 | Target: n("bar"), 331 | Feed: feed{ 332 | Name: "Foo", 333 | Url: "whatever", 334 | }, 335 | Options: Map{"include-images": true, "unknown-option": "foo"}, 336 | }}, Map{"something": 1})}, 337 | 338 | {name: "Feed with Exec", 339 | inp: ` 340 | feeds: 341 | - name: Foo 342 | exec: [whatever, -i, http://foo.bar] 343 | target: bar 344 | include-images: true 345 | unknown-option: foo 346 | `, 347 | wantErr: false, 348 | config: defaultConfig([]configGroupFeed{{ 349 | Target: n("bar"), 350 | Feed: feed{ 351 | Name: "Foo", 352 | Exec: []string{"whatever", "-i", "http://foo.bar"}, 353 | }, 354 | Options: Map{"include-images": true, "unknown-option": "foo"}, 355 | }}, nil)}, 356 | 357 | {name: "Feeds", 358 | inp: ` 359 | feeds: 360 | - name: Foo 361 | url: whatever 362 | min-frequency: 2 363 | - name: Shrubbery 364 | url: google.de 365 | target: bla 366 | include-images: false 367 | `, 368 | wantErr: false, 369 | config: defaultConfig([]configGroupFeed{ 370 | { 371 | Feed: feed{ 372 | Name: "Foo", 373 | Url: "whatever", 374 | }, 375 | Options: Map{"min-frequency": 2}, 376 | }, 377 | { 378 | Target: n("bla"), 379 | Feed: feed{ 380 | Name: "Shrubbery", 381 | Url: "google.de", 382 | }, 383 | Options: Map{"include-images": false}, 384 | }, 385 | }, nil), 386 | }, 387 | {name: "Empty Group", 388 | inp: ` 389 | feeds: 390 | - group: Foo 391 | target: bla 392 | `, 393 | wantErr: false, 394 | config: defaultConfig([]configGroupFeed{{Target: n("bla"), Group: group{"Foo", nil}}}, nil), 395 | }, 396 | {name: "Feeds and Groups", 397 | inp: ` 398 | feeds: 399 | - name: Foo 400 | url: whatever 401 | - group: G1 402 | target: target 403 | feeds: 404 | - group: G2 405 | target: "" 406 | feeds: 407 | - name: F1 408 | url: google.de 409 | - name: F2 410 | - name: F3 411 | target: 412 | - group: G3 413 | `, 414 | wantErr: false, 415 | config: defaultConfig([]configGroupFeed{ 416 | {Feed: feed{ 417 | Name: "Foo", 418 | Url: "whatever", 419 | }}, 420 | {Target: n("target"), Group: group{ 421 | Group: "G1", 422 | Feeds: []configGroupFeed{ 423 | {Target: n(""), Group: group{ 424 | Group: "G2", 425 | Feeds: []configGroupFeed{ 426 | {Feed: feed{Name: "F1", Url: "google.de"}}, 427 | }}, 428 | }, 429 | {Feed: feed{Name: "F2"}}, 430 | {Feed: feed{Name: "F3"}, Target: null}, 431 | {Group: group{Group: "G3"}}, 432 | }}, 433 | }, 434 | }, nil), 435 | }, 436 | } 437 | 438 | eqNode := cmp.Comparer(func(l, r yaml.Node) bool { 439 | return l.Tag == r.Tag && l.Value == r.Value 440 | }) 441 | 442 | for _, tt := range tests { 443 | tst.Run(tt.name, func(tst *testing.T) { 444 | in := strings.NewReader(tt.inp) 445 | 446 | got, err := unmarshal(in, WithDefault()) 447 | 448 | if tt.wantErr { 449 | if err == nil { 450 | tst.Error("Excepted error, but was successfull.") 451 | } else if diff := cmp.Diff(tt.errMsg, err.Error()); diff != "" { 452 | tst.Error(diff) 453 | } 454 | } else { 455 | if err != nil { 456 | tst.Errorf("Unexpected error %v", err) 457 | } else if diff := cmp.Diff(tt.config, got, eqNode); diff != "" { 458 | tst.Error(diff) 459 | } 460 | } 461 | }) 462 | } 463 | } 464 | 465 | func TestCompleteFeed(tst *testing.T) { 466 | inp := ` 467 | feeds: 468 | - name: Foo 469 | url: whatever 470 | - group: G1 471 | target: target 472 | feeds: 473 | - group: G2 474 | target: "" 475 | feeds: 476 | - name: F1 477 | url: google.de 478 | - name: F2 479 | url: F2 480 | - name: F3 481 | url: F3 482 | target: 483 | - name: F4 484 | exec: ['/bin/foo'] 485 | target: "G4" 486 | - name: F5 487 | url: F5 488 | target: ~ 489 | - name: F6 490 | url: F6 491 | target: "" 492 | - group: G3 493 | - group: G4 494 | feeds: 495 | - name: F7 496 | url: F7 497 | ` 498 | res := Feeds{ 499 | "Foo": &Feed{Name: "Foo", Target: t("Foo"), Url: "whatever"}, 500 | "F1": &Feed{Name: "F1", Target: t("target.F1"), Url: "google.de"}, 501 | "F2": &Feed{Name: "F2", Target: t("target.F2"), Url: "F2"}, 502 | "F3": &Feed{Name: "F3", Target: t("target"), Url: "F3"}, 503 | "F4": &Feed{Name: "F4", Target: t("target.G4"), Exec: []string{"/bin/foo"}}, 504 | "F5": &Feed{Name: "F5", Target: t("target"), Url: "F5"}, 505 | "F6": &Feed{Name: "F6", Target: t("target"), Url: "F6"}, 506 | "F7": &Feed{Name: "F7", Target: t("target.G4.F7"), Url: "F7"}, 507 | } 508 | 509 | c := WithDefault() 510 | c.FeedOptions = Options{} 511 | 512 | if err := c.parse(strings.NewReader(inp)); err != nil { 513 | tst.Error(err) 514 | } else { 515 | if diff := cmp.Diff(res, c.Feeds); diff != "" { 516 | tst.Error(diff) 517 | } 518 | } 519 | } 520 | 521 | func TestURLFeedWithoutGlobalTarget(tst *testing.T) { 522 | inp := ` 523 | feeds: 524 | - name: Foo 525 | url: Foo 526 | target: imap://foo.bar:443/INBOX/Feed 527 | ` 528 | res := Feeds{ 529 | "Foo": &Feed{Name: "Foo", Url: "Foo", Target: t("INBOX.Feed")}, 530 | } 531 | 532 | c := WithDefault() 533 | c.FeedOptions = Options{} 534 | 535 | if err := c.parse(strings.NewReader(inp)); err != nil { 536 | tst.Error(err) 537 | } else { 538 | if diff := cmp.Diff(res, c.Feeds); diff != "" { 539 | tst.Error(diff) 540 | } 541 | if diff := cmp.Diff("imap://foo.bar:443", c.Target.String()); diff != "" { 542 | tst.Error(diff) 543 | } 544 | } 545 | } 546 | 547 | func TestURLFeedWithGlobalTarget(tst *testing.T) { 548 | inp := ` 549 | target: imaps://foo.bar/INBOX/Feeds 550 | feeds: 551 | - name: Foo 552 | url: Foo 553 | target: imaps://foo.bar:993/Some/Other/Path 554 | ` 555 | res := Feeds{ 556 | "Foo": &Feed{Name: "Foo", Url: "Foo", Target: t("Some.Other.Path")}, 557 | } 558 | 559 | c := WithDefault() 560 | c.FeedOptions = Options{} 561 | 562 | if err := c.parse(strings.NewReader(inp)); err != nil { 563 | tst.Error(err) 564 | } else { 565 | if diff := cmp.Diff(res, c.Feeds); diff != "" { 566 | tst.Error(diff) 567 | } 568 | if diff := cmp.Diff("imaps://foo.bar:993/INBOX/Feeds", c.Target.String()); diff != "" { 569 | tst.Error(diff) 570 | } 571 | } 572 | } 573 | 574 | func TestURLFeedWithDifferentGlobalTarget(tst *testing.T) { 575 | inp := ` 576 | target: imaps://foo.bar/INBOX/Feeds 577 | feeds: 578 | - name: Foo 579 | url: Foo 580 | target: imaps://other.bar/INBOX/Feeds 581 | ` 582 | errorText := "Line 6: Given URL endpoint 'imaps://other.bar:993' does not match previous endpoint 'imaps://foo.bar:993'." 583 | c := WithDefault() 584 | c.FeedOptions = Options{} 585 | 586 | err := c.parse(strings.NewReader(inp)) 587 | if err == nil { 588 | tst.Error("Expected error.") 589 | } else { 590 | if diff := cmp.Diff(errorText, err.Error()); diff != "" { 591 | tst.Error(diff) 592 | } 593 | } 594 | } 595 | -------------------------------------------------------------------------------- /pkg/log/log.go: -------------------------------------------------------------------------------- 1 | package log 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "os" 7 | ) 8 | 9 | var debugLogger = log.New(os.Stdout, "DEBUG: ", log.LstdFlags|log.Lmsgprefix) 10 | var verboseLogger = log.New(os.Stdout, " INFO: ", log.LstdFlags|log.Lmsgprefix) 11 | var errorLogger = log.New(os.Stderr, "ERROR: ", log.LstdFlags|log.Lmsgprefix) 12 | var warnLogger = log.New(os.Stdout, " WARN: ", log.LstdFlags|log.Lmsgprefix) 13 | 14 | type logLevel byte 15 | 16 | const ( 17 | debug logLevel = iota 18 | info 19 | warn 20 | ) 21 | 22 | var level logLevel = warn 23 | 24 | func SetVerbose() { 25 | level = info 26 | } 27 | 28 | func SetDebug() { 29 | level = debug 30 | } 31 | 32 | func IsDebug() bool { 33 | return level == debug 34 | } 35 | 36 | func Debug(v ...any) { 37 | if level <= debug { 38 | _ = debugLogger.Output(2, fmt.Sprint(v...)) 39 | } 40 | } 41 | 42 | func Debugf(format string, v ...any) { 43 | if level <= debug { 44 | _ = debugLogger.Output(2, fmt.Sprintf(format, v...)) 45 | } 46 | } 47 | 48 | func Print(v ...any) { 49 | if level <= info { 50 | _ = verboseLogger.Output(2, fmt.Sprint(v...)) 51 | } 52 | } 53 | 54 | func Printf(format string, v ...any) { 55 | if level <= info { 56 | _ = verboseLogger.Output(2, fmt.Sprintf(format, v...)) 57 | } 58 | } 59 | 60 | func Error(v ...any) { 61 | _ = errorLogger.Output(2, fmt.Sprint(v...)) 62 | } 63 | 64 | //noinspection GoUnusedExportedFunction 65 | func Errorf(format string, a ...any) { 66 | _ = errorLogger.Output(2, fmt.Sprintf(format, a...)) 67 | } 68 | 69 | //noinspection GoUnusedExportedFunction 70 | func Warn(v ...any) { 71 | _ = warnLogger.Output(2, fmt.Sprint(v...)) 72 | } 73 | 74 | //noinspection GoUnusedExportedFunction 75 | func Warnf(format string, a ...any) { 76 | _ = warnLogger.Output(2, fmt.Sprintf(format, a...)) 77 | } 78 | -------------------------------------------------------------------------------- /pkg/rfc822/writer.go: -------------------------------------------------------------------------------- 1 | // Package rfc822 provides a writer that ensures the intrinsics of RFC 822. 2 | // 3 | // Rationale 4 | // 5 | // Cyrus IMAP really cares about the hard specifics of RFC 822, namely not allowing single \r and \n. 6 | // 7 | // See also: https://www.cyrusimap.org/imap/reference/faqs/interop-barenewlines.html 8 | // and: https://github.com/Necoro/feed2imap-go/issues/46 9 | // 10 | // NB: This package currently only cares about the newlines. 11 | package rfc822 12 | 13 | import "io" 14 | 15 | type rfc822Writer struct { 16 | w io.Writer 17 | } 18 | 19 | var lf = []byte{'\n'} 20 | var cr = []byte{'\r'} 21 | 22 | func (f rfc822Writer) Write(p []byte) (n int, err error) { 23 | crFound := false 24 | start := 0 25 | 26 | write := func(str []byte, count bool) { 27 | var j int 28 | j, err = f.w.Write(str) 29 | if count { 30 | n += j 31 | } 32 | } 33 | 34 | for idx, b := range p { 35 | if crFound && b != '\n' { 36 | // insert '\n' 37 | if write(p[start:idx], true); err != nil { 38 | return 39 | } 40 | if write(lf, false); err != nil { 41 | return 42 | } 43 | 44 | start = idx 45 | } else if !crFound && b == '\n' { 46 | // insert '\r' 47 | if write(p[start:idx], true); err != nil { 48 | return 49 | } 50 | if write(cr, false); err != nil { 51 | return 52 | } 53 | 54 | start = idx 55 | } 56 | crFound = b == '\r' 57 | } 58 | 59 | // write the remainder 60 | if write(p[start:], true); err != nil { 61 | return 62 | } 63 | 64 | if crFound { // dangling \r 65 | write(lf, false) 66 | } 67 | 68 | return 69 | } 70 | 71 | // Writer creates a new RFC 822 conform writer. 72 | func Writer(w io.Writer) io.Writer { 73 | return rfc822Writer{w} 74 | } 75 | -------------------------------------------------------------------------------- /pkg/rfc822/writer_test.go: -------------------------------------------------------------------------------- 1 | package rfc822 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | "testing" 7 | ) 8 | 9 | func TestRfc822Writer_Write(t *testing.T) { 10 | tests := []struct { 11 | before string 12 | after string 13 | }{ 14 | {"", ""}, 15 | {"foo", "foo"}, 16 | {"foo\r", "foo\r\n"}, 17 | {"foo\n", "foo\r\n"}, 18 | {"foo\r\n", "foo\r\n"}, 19 | {"\r", "\r\n"}, 20 | {"\n", "\r\n"}, 21 | {"\r\n", "\r\n"}, 22 | {"foo\rbar", "foo\r\nbar"}, 23 | {"foo\nbar", "foo\r\nbar"}, 24 | {"foo\r\nbar", "foo\r\nbar"}, 25 | {"\r\r", "\r\n\r\n"}, 26 | {"\n\n", "\r\n\r\n"}, 27 | {"\r\r\n", "\r\n\r\n"}, 28 | {"\n\r", "\r\n\r\n"}, 29 | {"\rbar", "\r\nbar"}, 30 | {"\nbar", "\r\nbar"}, 31 | {"\r\nbar", "\r\nbar"}, 32 | } 33 | for _, tt := range tests { 34 | t.Run(tt.before, func(t *testing.T) { 35 | b := bytes.Buffer{} 36 | w := Writer(&b) 37 | n, err := io.WriteString(w, tt.before) 38 | if err != nil { 39 | t.Errorf("Error: %v", err) 40 | return 41 | } 42 | if n != len(tt.before) { 43 | t.Errorf("Unexpected number of bytes written: %d, expected: %d", n, len(tt.before)) 44 | } 45 | res := b.String() 46 | if tt.after != res { 47 | t.Errorf("Expected: %q, got: %q", tt.after, res) 48 | } 49 | }) 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /pkg/util/util.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import "time" 4 | 5 | // TimeFormat formats the given time, where an empty time is formatted as "not set". 6 | func TimeFormat(t time.Time) string { 7 | if t.IsZero() { 8 | return "not set" 9 | } 10 | 11 | return t.Format(time.ANSIC) 12 | } 13 | 14 | // Days returns the number of days in a duration. Fraction of days are discarded. 15 | func Days(d time.Duration) int { 16 | return int(d.Hours() / 24) 17 | } 18 | -------------------------------------------------------------------------------- /pkg/version/version.go: -------------------------------------------------------------------------------- 1 | package version 2 | 3 | // this is set by the linker during build 4 | var ( 5 | version = "1.7.2-post" 6 | commit = "" 7 | ) 8 | 9 | // Version returns the current feed2imap-go version 10 | func Version() string { 11 | return version 12 | } 13 | 14 | // FullVersion returns the version including the commit hash 15 | func FullVersion() string { 16 | return "Version " + version + " Commit: " + commit 17 | } 18 | -------------------------------------------------------------------------------- /tools/README: -------------------------------------------------------------------------------- 1 | Tools for feed2imap-go 2 | ---------------------- 3 | 4 | Currently included: 5 | * `print-cache`: Print details about feeds part of the cache and the cached contents. 6 | 7 | Note to packagers: `print-cache` is mostly a debugging tool, inclusion in packages is optional. 8 | -------------------------------------------------------------------------------- /tools/print-cache/print-cache.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "log" 7 | 8 | "github.com/Necoro/feed2imap-go/internal/feed/cache" 9 | ) 10 | 11 | // flags 12 | var ( 13 | cacheFile string = "feed.cache" 14 | feedId string = "" 15 | ) 16 | 17 | func init() { 18 | flag.StringVar(&cacheFile, "c", cacheFile, "cache file") 19 | flag.StringVar(&feedId, "i", feedId, "id of the feed") 20 | } 21 | 22 | func main() { 23 | flag.Parse() 24 | 25 | cache, err := cache.Load(cacheFile, false) 26 | if err != nil { 27 | log.Fatal(err) 28 | } 29 | 30 | defer cache.Unlock() 31 | 32 | fmt.Printf("Cache version %d\n", cache.Version()) 33 | if feedId != "" { 34 | fmt.Print(cache.SpecificInfo(feedId)) 35 | } else { 36 | fmt.Print(cache.Info()) 37 | } 38 | } 39 | --------------------------------------------------------------------------------