├── .github ├── FUNDING.yml ├── build ├── run-tests.sh └── workflows │ ├── docker-publish.yml │ ├── pull_request.yml │ ├── push.yml │ └── release.yml ├── .gitignore ├── Dockerfile ├── LICENSE ├── README.md ├── add_cmd.go ├── add_cmd_test.go ├── config_cmd.go ├── config_cmd_test.go ├── configfile ├── FUZZING.md ├── configfile.go ├── configfile_test.go └── fuzz_test.go ├── cron_cmd.go ├── cron_cmd_test.go ├── daemon_cmd.go ├── daemon_cmd_test.go ├── del_cmd.go ├── del_cmd_test.go ├── docker-compose.yml ├── example_templates ├── README.md └── youtube.txt ├── export_cmd.go ├── export_cmd_test.go ├── go.mod ├── go.sum ├── httpfetch ├── httpfetch.go └── httpfetch_test.go ├── import_cmd.go ├── import_cmd_test.go ├── list_cmd.go ├── list_cmd_test.go ├── list_default_template_cmd.go ├── list_default_template_cmd_test.go ├── main.go ├── main_test.go ├── processor ├── emailer │ └── emailer.go ├── processor.go └── processor_test.go ├── seen_cmd.go ├── state └── state.go ├── template ├── template.go ├── template.txt └── template_test.go ├── unsee_cmd.go ├── usage_test.go ├── version_cmd.go ├── version_cmd_test.go └── withstate └── feeditem.go /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | github: skx 3 | custom: https://steve.fi/donate/ 4 | -------------------------------------------------------------------------------- /.github/build: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # The basename of our binary 4 | BASE="rss2email" 5 | 6 | # I don't even .. 7 | go env -w GOFLAGS="-buildvcs=false" 8 | 9 | # 10 | # We build on only a single platform/arch. 11 | # 12 | BUILD_PLATFORMS="linux darwin freebsd" 13 | BUILD_ARCHS="amd64 386" 14 | 15 | # For each platform 16 | for OS in ${BUILD_PLATFORMS[@]}; do 17 | 18 | # For each arch 19 | for ARCH in ${BUILD_ARCHS[@]}; do 20 | 21 | # Setup a suffix for the binary 22 | SUFFIX="${OS}" 23 | 24 | # i386 is better than 386 25 | if [ "$ARCH" = "386" ]; then 26 | SUFFIX="${SUFFIX}-i386" 27 | else 28 | SUFFIX="${SUFFIX}-${ARCH}" 29 | fi 30 | 31 | # Windows binaries should end in .EXE 32 | if [ "$OS" = "windows" ]; then 33 | SUFFIX="${SUFFIX}.exe" 34 | fi 35 | 36 | echo "Building for ${OS} [${ARCH}] -> ${BASE}-${SUFFIX}" 37 | 38 | # Run the build 39 | export GOARCH=${ARCH} 40 | export GOOS=${OS} 41 | export CGO_ENABLED=0 42 | 43 | go build -ldflags "-X main.version=$(git describe --tags)" -o "${BASE}-${SUFFIX}" 44 | 45 | done 46 | done 47 | -------------------------------------------------------------------------------- /.github/run-tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # I don't even .. 4 | go env -w GOFLAGS="-buildvcs=false" 5 | 6 | # Run golang tests 7 | go test ./... 8 | -------------------------------------------------------------------------------- /.github/workflows/docker-publish.yml: -------------------------------------------------------------------------------- 1 | name: Docker 2 | 3 | # This workflow uses actions that are not certified by GitHub. 4 | # They are provided by a third-party and are governed by 5 | # separate terms of service, privacy policy, and support 6 | # documentation. 7 | 8 | on: 9 | push: 10 | branches: [ "master" ] 11 | # Publish semver tags as releases. 12 | tags: [ 'v*.*.*' ] 13 | pull_request: 14 | branches: [ "master" ] 15 | 16 | env: 17 | # Use docker.io for Docker Hub if empty 18 | REGISTRY: ghcr.io 19 | # github.repository as / 20 | IMAGE_NAME: ${{ github.repository }} 21 | 22 | 23 | jobs: 24 | build: 25 | 26 | runs-on: ubuntu-latest 27 | permissions: 28 | contents: read 29 | packages: write 30 | # This is used to complete the identity challenge 31 | # with sigstore/fulcio when running outside of PRs. 32 | id-token: write 33 | 34 | steps: 35 | - name: Checkout repository 36 | uses: actions/checkout@v3 37 | with: 38 | fetch-depth: 0 39 | 40 | - name: Get our version 41 | id: get-build-version 42 | run: | 43 | echo "MYVERSION="`git describe --tags`"" >> $GITHUB_ENV 44 | echo "Our version is $(git describe --tags)" 45 | 46 | 47 | # Install the cosign tool except on PR 48 | # https://github.com/sigstore/cosign-installer 49 | - name: Install cosign 50 | if: github.event_name != 'pull_request' 51 | uses: sigstore/cosign-installer@v3.7.0 52 | with: 53 | cosign-release: 'v2.4.1' # optional 54 | 55 | # Set up BuildKit Docker container builder to be able to build 56 | # multi-platform images and export cache 57 | # https://github.com/docker/setup-buildx-action 58 | - name: Set up Docker Buildx 59 | uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0 60 | 61 | # Login against a Docker registry except on PR 62 | # https://github.com/docker/login-action 63 | - name: Log into registry ${{ env.REGISTRY }} 64 | if: github.event_name != 'pull_request' 65 | uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 66 | with: 67 | registry: ${{ env.REGISTRY }} 68 | username: ${{ github.actor }} 69 | password: ${{ secrets.GITHUB_TOKEN }} 70 | 71 | # Extract metadata (tags, labels) for Docker 72 | # https://github.com/docker/metadata-action 73 | - name: Extract Docker metadata 74 | id: meta 75 | uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0 76 | with: 77 | images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} 78 | 79 | # Build and push Docker image with Buildx (don't push on PR) 80 | # https://github.com/docker/build-push-action 81 | - name: Build and push Docker image 82 | id: build-and-push 83 | uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0 84 | with: 85 | context: . 86 | platforms: linux/amd64,linux/arm64 87 | push: ${{ github.event_name != 'pull_request' }} 88 | tags: ${{ steps.meta.outputs.tags }} 89 | labels: ${{ steps.meta.outputs.labels }} 90 | cache-from: type=gha 91 | cache-to: type=gha,mode=max 92 | build-args: | 93 | VERSION=${{ env.MYVERSION }} 94 | 95 | # Sign the resulting Docker image digest except on PRs. 96 | # This will only write to the public Rekor transparency log when the Docker 97 | # repository is public to avoid leaking data. If you would like to publish 98 | # transparency data even for private images, pass --force to cosign below. 99 | # https://github.com/sigstore/cosign 100 | - name: Sign the published Docker image 101 | if: ${{ github.event_name != 'pull_request' }} 102 | env: 103 | # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable 104 | TAGS: ${{ steps.meta.outputs.tags }} 105 | DIGEST: ${{ steps.build-and-push.outputs.digest }} 106 | # This step uses the identity token to provision an ephemeral certificate 107 | # against the sigstore community Fulcio instance. 108 | run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST} 109 | -------------------------------------------------------------------------------- /.github/workflows/pull_request.yml: -------------------------------------------------------------------------------- 1 | on: pull_request 2 | name: Pull Request 3 | jobs: 4 | golangci: 5 | name: lint 6 | runs-on: ubuntu-latest 7 | steps: 8 | - uses: actions/setup-go@v2 9 | - uses: actions/checkout@v2 10 | - name: golangci-lint 11 | uses: golangci/golangci-lint-action@v2 12 | test: 13 | name: Test 14 | runs-on: ubuntu-latest 15 | steps: 16 | - uses: actions/checkout@master 17 | - name: Test 18 | uses: skx/github-action-tester@master 19 | -------------------------------------------------------------------------------- /.github/workflows/push.yml: -------------------------------------------------------------------------------- 1 | on: 2 | push: 3 | branches: 4 | - master 5 | name: Push Event 6 | jobs: 7 | test: 8 | name: Test 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@master 12 | - name: Test 13 | uses: skx/github-action-tester@master 14 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | on: release 2 | name: Handle Release 3 | jobs: 4 | upload: 5 | name: Upload 6 | runs-on: ubuntu-latest 7 | steps: 8 | - name: Checkout the repository 9 | uses: actions/checkout@master 10 | - name: Generate the artifacts 11 | uses: skx/github-action-build@master 12 | - name: Upload the artifacts 13 | uses: skx/github-action-publish-binaries@master 14 | env: 15 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 16 | with: 17 | args: rss2email-* 18 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | rss2email 2 | rss2email- 3 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # 2 | # Trivial Dockerfile for rss2email. 3 | # 4 | # Build it like so: 5 | # 6 | # docker build -t rss2email:latest . 7 | # 8 | # I tag/push to a github repository like so: 9 | # 10 | # docker tag rss2email:latest docker.pkg.github.com/skx/docker/rss2email:7 11 | # docker push docker.pkg.github.com/skx/docker/rss2email:7 12 | # 13 | # Running it will be something like this: 14 | # 15 | # docker run -d \ 16 | # --env SMTP_HOST=smtp.gmail.com \ 17 | # --env SMTP_USERNAME=steve@example.com \ 18 | # --env SMTP_PASSWORD=secret \ 19 | # rss2email:latest daemon -verbose steve@example.com 20 | # 21 | 22 | # STEP1 - Build-image 23 | ########################################################################### 24 | FROM golang:alpine AS builder 25 | 26 | ARG VERSION 27 | 28 | LABEL org.opencontainers.image.source=https://github.com/skx/rss2email/ 29 | 30 | # Create a working-directory 31 | WORKDIR $GOPATH/src/github.com/skx/rss2email/ 32 | 33 | # Copy the source to it 34 | COPY . . 35 | 36 | # Build the binary - ensuring we pass the build-argument 37 | RUN go build -ldflags "-X main.version=$VERSION" -o /go/bin/rss2email 38 | 39 | # STEP2 - Deploy-image 40 | ########################################################################### 41 | FROM alpine 42 | 43 | # Copy the binary. 44 | COPY --from=builder /go/bin/rss2email /usr/local/bin/ 45 | 46 | # Set entrypoint 47 | ENTRYPOINT [ "/usr/local/bin/rss2email" ] 48 | 49 | # Set default command 50 | CMD help 51 | 52 | # Create a group and user 53 | RUN addgroup app && adduser -D -G app -h /app app 54 | 55 | # Tell docker that all future commands should run as the app user 56 | USER app 57 | 58 | # Set working directory 59 | WORKDIR /app 60 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 2, June 1991 3 | 4 | Copyright (C) 1989, 1991 Free Software Foundation, Inc., 5 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 6 | Everyone is permitted to copy and distribute verbatim copies 7 | of this license document, but changing it is not allowed. 8 | 9 | Preamble 10 | 11 | The licenses for most software are designed to take away your 12 | freedom to share and change it. By contrast, the GNU General Public 13 | License is intended to guarantee your freedom to share and change free 14 | software--to make sure the software is free for all its users. This 15 | General Public License applies to most of the Free Software 16 | Foundation's software and to any other program whose authors commit to 17 | using it. (Some other Free Software Foundation software is covered by 18 | the GNU Lesser General Public License instead.) You can apply it to 19 | your programs, too. 20 | 21 | When we speak of free software, we are referring to freedom, not 22 | price. Our General Public Licenses are designed to make sure that you 23 | have the freedom to distribute copies of free software (and charge for 24 | this service if you wish), that you receive source code or can get it 25 | if you want it, that you can change the software or use pieces of it 26 | in new free programs; and that you know you can do these things. 27 | 28 | To protect your rights, we need to make restrictions that forbid 29 | anyone to deny you these rights or to ask you to surrender the rights. 30 | These restrictions translate to certain responsibilities for you if you 31 | distribute copies of the software, or if you modify it. 32 | 33 | For example, if you distribute copies of such a program, whether 34 | gratis or for a fee, you must give the recipients all the rights that 35 | you have. You must make sure that they, too, receive or can get the 36 | source code. And you must show them these terms so they know their 37 | rights. 38 | 39 | We protect your rights with two steps: (1) copyright the software, and 40 | (2) offer you this license which gives you legal permission to copy, 41 | distribute and/or modify the software. 42 | 43 | Also, for each author's protection and ours, we want to make certain 44 | that everyone understands that there is no warranty for this free 45 | software. If the software is modified by someone else and passed on, we 46 | want its recipients to know that what they have is not the original, so 47 | that any problems introduced by others will not reflect on the original 48 | authors' reputations. 49 | 50 | Finally, any free program is threatened constantly by software 51 | patents. We wish to avoid the danger that redistributors of a free 52 | program will individually obtain patent licenses, in effect making the 53 | program proprietary. To prevent this, we have made it clear that any 54 | patent must be licensed for everyone's free use or not licensed at all. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | GNU GENERAL PUBLIC LICENSE 60 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 61 | 62 | 0. This License applies to any program or other work which contains 63 | a notice placed by the copyright holder saying it may be distributed 64 | under the terms of this General Public License. The "Program", below, 65 | refers to any such program or work, and a "work based on the Program" 66 | means either the Program or any derivative work under copyright law: 67 | that is to say, a work containing the Program or a portion of it, 68 | either verbatim or with modifications and/or translated into another 69 | language. (Hereinafter, translation is included without limitation in 70 | the term "modification".) Each licensee is addressed as "you". 71 | 72 | Activities other than copying, distribution and modification are not 73 | covered by this License; they are outside its scope. The act of 74 | running the Program is not restricted, and the output from the Program 75 | is covered only if its contents constitute a work based on the 76 | Program (independent of having been made by running the Program). 77 | Whether that is true depends on what the Program does. 78 | 79 | 1. You may copy and distribute verbatim copies of the Program's 80 | source code as you receive it, in any medium, provided that you 81 | conspicuously and appropriately publish on each copy an appropriate 82 | copyright notice and disclaimer of warranty; keep intact all the 83 | notices that refer to this License and to the absence of any warranty; 84 | and give any other recipients of the Program a copy of this License 85 | along with the Program. 86 | 87 | You may charge a fee for the physical act of transferring a copy, and 88 | you may at your option offer warranty protection in exchange for a fee. 89 | 90 | 2. You may modify your copy or copies of the Program or any portion 91 | of it, thus forming a work based on the Program, and copy and 92 | distribute such modifications or work under the terms of Section 1 93 | above, provided that you also meet all of these conditions: 94 | 95 | a) You must cause the modified files to carry prominent notices 96 | stating that you changed the files and the date of any change. 97 | 98 | b) You must cause any work that you distribute or publish, that in 99 | whole or in part contains or is derived from the Program or any 100 | part thereof, to be licensed as a whole at no charge to all third 101 | parties under the terms of this License. 102 | 103 | c) If the modified program normally reads commands interactively 104 | when run, you must cause it, when started running for such 105 | interactive use in the most ordinary way, to print or display an 106 | announcement including an appropriate copyright notice and a 107 | notice that there is no warranty (or else, saying that you provide 108 | a warranty) and that users may redistribute the program under 109 | these conditions, and telling the user how to view a copy of this 110 | License. (Exception: if the Program itself is interactive but 111 | does not normally print such an announcement, your work based on 112 | the Program is not required to print an announcement.) 113 | 114 | These requirements apply to the modified work as a whole. If 115 | identifiable sections of that work are not derived from the Program, 116 | and can be reasonably considered independent and separate works in 117 | themselves, then this License, and its terms, do not apply to those 118 | sections when you distribute them as separate works. But when you 119 | distribute the same sections as part of a whole which is a work based 120 | on the Program, the distribution of the whole must be on the terms of 121 | this License, whose permissions for other licensees extend to the 122 | entire whole, and thus to each and every part regardless of who wrote it. 123 | 124 | Thus, it is not the intent of this section to claim rights or contest 125 | your rights to work written entirely by you; rather, the intent is to 126 | exercise the right to control the distribution of derivative or 127 | collective works based on the Program. 128 | 129 | In addition, mere aggregation of another work not based on the Program 130 | with the Program (or with a work based on the Program) on a volume of 131 | a storage or distribution medium does not bring the other work under 132 | the scope of this License. 133 | 134 | 3. You may copy and distribute the Program (or a work based on it, 135 | under Section 2) in object code or executable form under the terms of 136 | Sections 1 and 2 above provided that you also do one of the following: 137 | 138 | a) Accompany it with the complete corresponding machine-readable 139 | source code, which must be distributed under the terms of Sections 140 | 1 and 2 above on a medium customarily used for software interchange; or, 141 | 142 | b) Accompany it with a written offer, valid for at least three 143 | years, to give any third party, for a charge no more than your 144 | cost of physically performing source distribution, a complete 145 | machine-readable copy of the corresponding source code, to be 146 | distributed under the terms of Sections 1 and 2 above on a medium 147 | customarily used for software interchange; or, 148 | 149 | c) Accompany it with the information you received as to the offer 150 | to distribute corresponding source code. (This alternative is 151 | allowed only for noncommercial distribution and only if you 152 | received the program in object code or executable form with such 153 | an offer, in accord with Subsection b above.) 154 | 155 | The source code for a work means the preferred form of the work for 156 | making modifications to it. For an executable work, complete source 157 | code means all the source code for all modules it contains, plus any 158 | associated interface definition files, plus the scripts used to 159 | control compilation and installation of the executable. However, as a 160 | special exception, the source code distributed need not include 161 | anything that is normally distributed (in either source or binary 162 | form) with the major components (compiler, kernel, and so on) of the 163 | operating system on which the executable runs, unless that component 164 | itself accompanies the executable. 165 | 166 | If distribution of executable or object code is made by offering 167 | access to copy from a designated place, then offering equivalent 168 | access to copy the source code from the same place counts as 169 | distribution of the source code, even though third parties are not 170 | compelled to copy the source along with the object code. 171 | 172 | 4. You may not copy, modify, sublicense, or distribute the Program 173 | except as expressly provided under this License. Any attempt 174 | otherwise to copy, modify, sublicense or distribute the Program is 175 | void, and will automatically terminate your rights under this License. 176 | However, parties who have received copies, or rights, from you under 177 | this License will not have their licenses terminated so long as such 178 | parties remain in full compliance. 179 | 180 | 5. You are not required to accept this License, since you have not 181 | signed it. However, nothing else grants you permission to modify or 182 | distribute the Program or its derivative works. These actions are 183 | prohibited by law if you do not accept this License. Therefore, by 184 | modifying or distributing the Program (or any work based on the 185 | Program), you indicate your acceptance of this License to do so, and 186 | all its terms and conditions for copying, distributing or modifying 187 | the Program or works based on it. 188 | 189 | 6. Each time you redistribute the Program (or any work based on the 190 | Program), the recipient automatically receives a license from the 191 | original licensor to copy, distribute or modify the Program subject to 192 | these terms and conditions. You may not impose any further 193 | restrictions on the recipients' exercise of the rights granted herein. 194 | You are not responsible for enforcing compliance by third parties to 195 | this License. 196 | 197 | 7. If, as a consequence of a court judgment or allegation of patent 198 | infringement or for any other reason (not limited to patent issues), 199 | conditions are imposed on you (whether by court order, agreement or 200 | otherwise) that contradict the conditions of this License, they do not 201 | excuse you from the conditions of this License. If you cannot 202 | distribute so as to satisfy simultaneously your obligations under this 203 | License and any other pertinent obligations, then as a consequence you 204 | may not distribute the Program at all. For example, if a patent 205 | license would not permit royalty-free redistribution of the Program by 206 | all those who receive copies directly or indirectly through you, then 207 | the only way you could satisfy both it and this License would be to 208 | refrain entirely from distribution of the Program. 209 | 210 | If any portion of this section is held invalid or unenforceable under 211 | any particular circumstance, the balance of the section is intended to 212 | apply and the section as a whole is intended to apply in other 213 | circumstances. 214 | 215 | It is not the purpose of this section to induce you to infringe any 216 | patents or other property right claims or to contest validity of any 217 | such claims; this section has the sole purpose of protecting the 218 | integrity of the free software distribution system, which is 219 | implemented by public license practices. Many people have made 220 | generous contributions to the wide range of software distributed 221 | through that system in reliance on consistent application of that 222 | system; it is up to the author/donor to decide if he or she is willing 223 | to distribute software through any other system and a licensee cannot 224 | impose that choice. 225 | 226 | This section is intended to make thoroughly clear what is believed to 227 | be a consequence of the rest of this License. 228 | 229 | 8. If the distribution and/or use of the Program is restricted in 230 | certain countries either by patents or by copyrighted interfaces, the 231 | original copyright holder who places the Program under this License 232 | may add an explicit geographical distribution limitation excluding 233 | those countries, so that distribution is permitted only in or among 234 | countries not thus excluded. In such case, this License incorporates 235 | the limitation as if written in the body of this License. 236 | 237 | 9. The Free Software Foundation may publish revised and/or new versions 238 | of the General Public License from time to time. Such new versions will 239 | be similar in spirit to the present version, but may differ in detail to 240 | address new problems or concerns. 241 | 242 | Each version is given a distinguishing version number. If the Program 243 | specifies a version number of this License which applies to it and "any 244 | later version", you have the option of following the terms and conditions 245 | either of that version or of any later version published by the Free 246 | Software Foundation. If the Program does not specify a version number of 247 | this License, you may choose any version ever published by the Free Software 248 | Foundation. 249 | 250 | 10. If you wish to incorporate parts of the Program into other free 251 | programs whose distribution conditions are different, write to the author 252 | to ask for permission. For software which is copyrighted by the Free 253 | Software Foundation, write to the Free Software Foundation; we sometimes 254 | make exceptions for this. Our decision will be guided by the two goals 255 | of preserving the free status of all derivatives of our free software and 256 | of promoting the sharing and reuse of software generally. 257 | 258 | NO WARRANTY 259 | 260 | 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY 261 | FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN 262 | OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES 263 | PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED 264 | OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 265 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS 266 | TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE 267 | PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, 268 | REPAIR OR CORRECTION. 269 | 270 | 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 271 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR 272 | REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, 273 | INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING 274 | OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED 275 | TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY 276 | YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER 277 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE 278 | POSSIBILITY OF SUCH DAMAGES. 279 | 280 | END OF TERMS AND CONDITIONS 281 | 282 | How to Apply These Terms to Your New Programs 283 | 284 | If you develop a new program, and you want it to be of the greatest 285 | possible use to the public, the best way to achieve this is to make it 286 | free software which everyone can redistribute and change under these terms. 287 | 288 | To do so, attach the following notices to the program. It is safest 289 | to attach them to the start of each source file to most effectively 290 | convey the exclusion of warranty; and each file should have at least 291 | the "copyright" line and a pointer to where the full notice is found. 292 | 293 | 294 | Copyright (C) 295 | 296 | This program is free software; you can redistribute it and/or modify 297 | it under the terms of the GNU General Public License as published by 298 | the Free Software Foundation; either version 2 of the License, or 299 | (at your option) any later version. 300 | 301 | This program is distributed in the hope that it will be useful, 302 | but WITHOUT ANY WARRANTY; without even the implied warranty of 303 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 304 | GNU General Public License for more details. 305 | 306 | You should have received a copy of the GNU General Public License along 307 | with this program; if not, write to the Free Software Foundation, Inc., 308 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 309 | 310 | Also add information on how to contact you by electronic and paper mail. 311 | 312 | If the program is interactive, make it output a short notice like this 313 | when it starts in an interactive mode: 314 | 315 | Gnomovision version 69, Copyright (C) year name of author 316 | Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 317 | This is free software, and you are welcome to redistribute it 318 | under certain conditions; type `show c' for details. 319 | 320 | The hypothetical commands `show w' and `show c' should show the appropriate 321 | parts of the General Public License. Of course, the commands you use may 322 | be called something other than `show w' and `show c'; they could even be 323 | mouse-clicks or menu items--whatever suits your program. 324 | 325 | You should also get your employer (if you work as a programmer) or your 326 | school, if any, to sign a "copyright disclaimer" for the program, if 327 | necessary. Here is a sample; alter the names: 328 | 329 | Yoyodyne, Inc., hereby disclaims all copyright interest in the program 330 | `Gnomovision' (which makes passes at compilers) written by James Hacker. 331 | 332 | , 1 April 1989 333 | Ty Coon, President of Vice 334 | 335 | This General Public License does not permit incorporating your program into 336 | proprietary programs. If your program is a subroutine library, you may 337 | consider it more useful to permit linking proprietary applications with the 338 | library. If this is what you want to do, use the GNU Lesser General 339 | Public License instead of this License. 340 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Go Report Card](https://goreportcard.com/badge/github.com/skx/rss2email)](https://goreportcard.com/report/github.com/skx/rss2email) 2 | [![license](https://img.shields.io/github/license/skx/rss2email.svg)](https://github.com/skx/rss2email/blob/master/LICENSE) 3 | [![Release](https://img.shields.io/github/release/skx/rss2email.svg)](https://github.com/skx/rss2email/releases/latest) 4 | 5 | Table of Contents 6 | ================= 7 | 8 | * [RSS2Email](#rss2email) 9 | * [Installation](#installation) 10 | * [bash completion](#bash-completion) 11 | * [Feed Configuration](#feed-configuration) 12 | * [Usage](#usage) 13 | * [Daemon Mode](#daemon-mode) 14 | * [Initial Run](#initial-run) 15 | * [Assumptions](#assumptions) 16 | * [Email Customization](#email-customization) 17 | * [Changing default From address](#changing-default-from-address) 18 | * [Implementation Overview](#implementation-overview) 19 | * [Logging Notes](#logging-notes) 20 | * [Github Setup](#github-setup) 21 | 22 | 23 | 24 | 25 | # RSS2Email 26 | 27 | This project began life as a naive port of the python-based [r2e](https://github.com/wking/rss2email) utility to golang. 28 | 29 | Over time we've now gained a few more features: 30 | 31 | * The ability to customize the email-templates which are generated and sent. 32 | * See [email customization](#email-customization) for details. 33 | * The ability to send email via STMP, or via `/usr/sbin/sendmail`. 34 | * See [SMTP-setup](#smtp-setup) for details. 35 | * The ability to include/exclude feed items from the emails. 36 | * For example receive emails only of feed items that contain the pattern "playstation". 37 | * A well-behaved HTTP-polling behaviour, using the appropriate cache-related HTTP-headers. 38 | 39 | 40 | 41 | # Installation 42 | 43 | If you have golang installed you can fetch, build, and install the latest binary by running: 44 | 45 | ```sh 46 | go install github.com/skx/rss2email@latest 47 | ``` 48 | 49 | If you prefer you can also fetch our latest binary release from [our release page](https://github.com/skx/rss2email/releases). 50 | 51 | To install from source simply clone the repository and build in the usual manner: 52 | 53 | ```sh 54 | git clone https://github.com/skx/rss2email 55 | cd rss2email 56 | go install . 57 | ``` 58 | 59 | Finally you can find automatically generated docker images, these are built on a nightly basis, and when releases are made: 60 | 61 | * https://github.com/skx/rss2email/pkgs/container/rss2email 62 | 63 | 64 | 65 | **Version NOTES**: 66 | 67 | * You'll need go version **1.21** or higher to build. 68 | * We use `go embed` to embed the (default) email-template within the binary, this was introduced with golang **v1.17**. 69 | * We use the [slog logging package](https://go.dev/blog/slog) introduced with golang **v1.21**. 70 | * We use the fuzzing support which was introduced with golang **v1.18** to test our configuration-file loading/parsing. 71 | * See [configfile/FUZZING.md](configfile/FUZZING.md) for details of using that. 72 | 73 | 74 | 75 | ## bash completion 76 | 77 | The binary has integrated support for TAB-completion, for bash. To enable this update your [dotfiles](https://github.com/skx/dotfiles/) to include the following: 78 | 79 | ``` 80 | source <(rss2email bash-completion) 81 | ``` 82 | 83 | 84 | 85 | 86 | # Feed Configuration 87 | 88 | Once you have installed the application you'll need to configure the feeds to monitor, this could be done by editing the configuration file: 89 | 90 | * `~/.rss2email/feeds.txt` 91 | 92 | There are several built-in sub-commands for manipulating the feed-list, for example you can add a new feed to monitor via the `add` sub-command: 93 | 94 | $ rss2email add https://example.com/blog.rss 95 | 96 | OPML files can be imported via the `import` sub-command: 97 | 98 | $ rss2email import feeds.opml 99 | 100 | The list of feeds can be displayed via the `list` subcommand (note that adding the `-verbose` flag will fetch each of the feeds and that will be slow): 101 | 102 | $ rss2email list [-verbose] 103 | 104 | Finally you can remove an entry from the feed-list via the `delete` sub-command: 105 | 106 | $ rss2email delete https://example.com/foo.rss 107 | 108 | The configuration file in its simplest form is nothing more than a list of URLs, one per line. However there is also support for adding per-feed options: 109 | 110 | https://foo.example.com/ 111 | - key:value 112 | https://foo.example.com/ 113 | - key2:value2 114 | 115 | This is documented and explained in the integrated help: 116 | 117 | $ rss2email help config 118 | 119 | Adding per-feed items allows excluding feed-entries by regular expression, for example this does what you'd expect: 120 | 121 | https://www.filfre.net/feed/rss/ 122 | - exclude-title: The Analog Antiquarian 123 | 124 | 125 | 126 | 127 | # Usage 128 | 129 | Once you've populated your feed list, via a series of `rss2email add ..` commands, or by editing the configuration file directly, you are now ready to actually launch the application. 130 | 131 | To run the application, announcing all new feed-items by email to `user@host.com` you'd run this: 132 | 133 | $ rss2email cron user@host.com 134 | 135 | Once the feed-list has been fetched, and items processed, the application will terminate. It is expected that you'll add an entry to your `crontab` file to ensure this runs regularly. For example you might wish to run the check & email process once every 15 minutes, so you could add this: 136 | 137 | # Announce feed-changes via email four times an hour 138 | */15 * * * * $HOME/go/bin/rss2email cron recipient@example.com 139 | 140 | When new items appear in the feeds they will then be sent to you via email. 141 | Each email will be multi-part, containing both `text/plain` and `text/html` 142 | versions of the new post(s). There is a default template which should contain 143 | the things you care about: 144 | 145 | * A link to the item posted. 146 | * The subject/title of the new feed item. 147 | * The HTML and Text content of the new feed item. 148 | 149 | If you wish you may customize the template which is used to generate the notification email, see [email-customization](#email-customization) for details. It is also possible to run in a [daemon mode](#daemon-mode) which will leave the process running forever, rather than terminating after walking the feeds once. 150 | 151 | The state of feed-entries is recorded beneath `~/.rss2email/state.db`, which is a [boltdb database](https://pkg.go.dev/go.etcd.io/bbolt). 152 | 153 | 154 | 155 | 156 | # Daemon Mode 157 | 158 | Typically you'd invoke `rss2email` with the `cron` sub-command as we documented above. This works in the naive way you'd expect: 159 | 160 | * Read the contents of each URL in the feed-list. 161 | * For each feed-item which is new generate and send an email. 162 | * Terminate. 163 | 164 | The `daemon` process does a similar thing, however it does __not__ terminate. Instead the process becomes: 165 | 166 | * Read the contents of each URL in the feed-list. 167 | * For each feed-item which is new generate and send an email. 168 | * Sleep for 5 minutes. 169 | * Begin the process once more. 170 | 171 | With this behaviour every feed will be fetched and processed every five minutes, which is almost certainly too frequently. To change this we have a notion of "frequency" - a feed will never be fetched more frequently than the given frequency value. 172 | 173 | * Set the `SLEEP` environmental variable if you wish to change globally. 174 | * e.g. "`export SLEEP=15`" will cause our main loop to fetch the feeds only once every fifteen minutes. 175 | * Set the per-feed `frequency` option to a different value. 176 | * That would mean all feeds would get fetched every fifteen minutes. 177 | * Except for the specific one that has a different value. 178 | 179 | > NOTE: Frequency values of less than five minutes will be ignored. 180 | 181 | In short the process runs forever, in the foreground. This is expected to be driven by `docker` or a systemd-service. Creating the appropriate configuration is left as an exercise, but you might examine the following two files for inspiration: 182 | 183 | * [Dockerfile](Dockerfile) 184 | * [docker-compose.yml](docker-compose.yml) 185 | 186 | 187 | 188 | 189 | # Initial Run 190 | 191 | When you add a new feed all the items contained within that feed will initially be unseen/new, and this means you'll receive a flood of emails if you were to run: 192 | 193 | $ rss2email add https://blog.steve.fi/index.rss 194 | $ rss2email cron user@domain.com 195 | 196 | To avoid this you can use the `-send=false` flag, which will merely 197 | record each item as having been seen, rather than sending you emails: 198 | 199 | $ rss2email add https://blog.steve.fi/index.rss 200 | $ rss2email cron -send=false user@domain.com 201 | 202 | 203 | 204 | 205 | # Assumptions 206 | 207 | Because this application is so minimal there are a number of assumptions baked in: 208 | 209 | * We assume that `/usr/sbin/sendmail` exists and will send email successfully. 210 | * You can cause emails to be sent via SMTP, see [SMTP-setup](#smtp-setup) for details. 211 | * We assume the recipient and sender email addresses can be the same. 212 | * i.e. If you mail output to `bob@example.com` that will be used as the sender address. 213 | * You can change the default sender via the [email-customization](#email-customization) process described next if you prefer though. 214 | 215 | 216 | 217 | 218 | # SMTP Setup 219 | 220 | By default the outgoing emails we generate are piped to `/usr/sbin/sendmail` to be delivered. If that is unavailable, or unsuitable, you can instead configure things such that SMTP is used directly. 221 | 222 | To configure SMTP you need to setup the following environmental-variables (environmental variables were selected as they're natural to use within Docker and systemd-service files). 223 | 224 | 225 | | Name | Example Value | 226 | |-------------------|-------------------| 227 | | **SMTP_HOST** | `smtp.gmail.com` | 228 | | **SMTP_PORT** | `587` | 229 | | **SMTP_USERNAME** | `bob@example.com` | 230 | | **SMTP_PASSWORD** | `secret!value` | 231 | 232 | If those values are present then SMTP will be used, otherwise the email will be sent via the local MTA. 233 | 234 | 235 | 236 | 237 | # Email Customization 238 | 239 | By default the emails are sent using a template file which is embedded in the application. You can override the template by creating the file `~/.rss2email/email.tmpl`, if that is present then it will be used instead of the default. 240 | 241 | You can view the default template via the following command: 242 | 243 | $ rss2email list-default-template 244 | 245 | You can copy the default-template to the right location by running the following, before proceeding to edit it as you wish: 246 | 247 | $ rss2email list-default-template > ~/.rss2email/email.tmpl 248 | 249 | The default template contains a brief header documenting the available fields, and functions, which you can use. As the template uses the standard Golang [text/template](https://golang.org/pkg/text/template/) facilities you can be pretty creative with it! 250 | 251 | If you're a developer who wishes to submit changes to the embedded version you should carry out the following two-step process to make your change. 252 | 253 | * Edit `template/template.txt`, which is the source of the template. 254 | * Rebuild the application to update the embedded copy. 255 | 256 | **NOTE**: If you read the earlier section on configuration you'll see that it is possible to add per-feed configuration values to the config file. One of the supported options is to setup a feed-specific template-file. 257 | 258 | 259 | 260 | ## Changing default From address 261 | 262 | As noted earlier when sending the notification emails the recipient address is used as the sender-address too. There are no flags for changing the From: address used to send the emails, however using the section above you can [use a customized email-template](#email-customization), and simply update the template to read something like this: 263 | 264 | ``` 265 | From: my.sender@example.com 266 | To: {{.To}} 267 | Subject: [rss2email] {{.Subject}} 268 | X-RSS-Link: {{.Link}} 269 | X-RSS-Feed: {{.Feed}} 270 | ``` 271 | 272 | * i.e. Change the `{{.From}}` to your preferred sender-address. 273 | 274 | 275 | 276 | 277 | # Implementation Overview 278 | 279 | The two main commands are `cron` and `daemon` and they work in roughly the same way: 280 | 281 | * They instantiate [processor/processor.go](processor/processor.go) to run the logic 282 | * That walks over the list of feeds from [configfile/configfile.go](configfile/configfile.go). 283 | * For each feed [httpfetch/httpfetch.go](httpfetch/httpfetch.go) is used to fetch the contents. 284 | * The result is a collection of `*gofeed.Feed` items, one for each entry in the remote feed. 285 | * These are wrapped via [withstate/feeditem.go](withstate/feeditem.go) so we can test if they're new. 286 | * [processor/emailer/emailer.go](processor/emailer/emailer.go) is used to send the email if necessary. 287 | * Either by SMTP or by executing `/usr/sbin/sendmail` 288 | 289 | The other subcommands mostly just interact with the feed-list, via the use of [configfile/configfile.go](configfile/configfile.go) to add/delete/list the contents of the feed-list. 290 | 291 | 292 | 293 | 294 | # Logging Notes 295 | 296 | The application is configured to use a common logger, which will output all messages to STDERR. The codebase will log messages at three levels: 297 | 298 | * DEBUG 299 | * This will be used when new features are added, and contain implementation-related notices. 300 | * The messages here will be helpful for debugging, or extending the application. 301 | * WARN 302 | * This level is shown by default. 303 | * This level is used for messages which are not fatal errors, but which a user might wish to be aware of. 304 | * For example failure to fetch a remote feed, or a count of retried HTTP-fetches. 305 | * ERROR 306 | * This level is shown by default. 307 | * This level is used for fatal-errors. 308 | * This should only be used for messages which are immediately followed by an application exit. 309 | 310 | There are several environmental variables which can be used to modify the logging output: 311 | 312 | * `LOG_LEVEL` 313 | * This may be set to one of several values: 314 | * `ERROR`: Show only errors. 315 | * `WARN`: Show errors, and warnings. 316 | * `DEBUG`: Show errors, warnings, and internal debugging messages (very verbose). 317 | * `LOG_FILE_PATH` 318 | * The name of the file to duplicate logging message to, defaults to being `rss2email.log`. 319 | * `LOG_FILE_DISABLE` 320 | * Set this to any non-empty value to disable the logfile entirely. 321 | * `LOG_JSON` 322 | * If this is set to a non-empty string the logging messages will be output in JSON format. 323 | * This is useful if you're collecting (container) messages in datadog, loki, sumologic, or something similar. 324 | 325 | Bot the `cron` and `daemon` sub-commands will switch to showing DEBUG messages if you supply the `-verbose` flag to them, which avoids the need for setting environmental variables. 326 | 327 | 328 | 329 | 330 | # Github Setup 331 | 332 | This repository is configured to run tests upon every commit, and when 333 | pull-requests are created/updated. The testing is carried out via 334 | [.github/run-tests.sh](.github/run-tests.sh) which is used by the 335 | [github-action-tester](https://github.com/skx/github-action-tester) action. 336 | 337 | Releases are automated in a similar fashion via [.github/build](.github/build), 338 | and the [github-action-publish-binaries](https://github.com/skx/github-action-publish-binaries) action. 339 | 340 | Steve 341 | -- 342 | -------------------------------------------------------------------------------- /add_cmd.go: -------------------------------------------------------------------------------- 1 | // 2 | // Add a new feed to the users' list of configured feeds. 3 | // 4 | 5 | package main 6 | 7 | import ( 8 | "flag" 9 | "log/slog" 10 | 11 | "github.com/skx/rss2email/configfile" 12 | ) 13 | 14 | // Structure for our options and state. 15 | type addCmd struct { 16 | 17 | // Configuration file, used for testing 18 | config *configfile.ConfigFile 19 | } 20 | 21 | // Arguments handles argument-flags we might have. 22 | // 23 | // In our case we use this as a hook to setup our configuration-file, 24 | // which allows testing. 25 | func (a *addCmd) Arguments(flags *flag.FlagSet) { 26 | a.config = configfile.New() 27 | } 28 | 29 | // Info is part of the subcommand-API 30 | func (a *addCmd) Info() (string, string) { 31 | return "add", `Add a new feed to our feed-list. 32 | 33 | Add one or more specified URLs to the configuration file. 34 | 35 | To see details of the configuration file, including the location, 36 | please run: 37 | 38 | $ rss2email help config 39 | 40 | Example: 41 | 42 | $ rss2email add https://blog.steve.fi/index.rss 43 | ` 44 | } 45 | 46 | // Execute is invoked if the user specifies `add` as the subcommand. 47 | func (a *addCmd) Execute(args []string) int { 48 | 49 | // Parse the existing file 50 | _, err := a.config.Parse() 51 | if err != nil { 52 | logger.Error("failed to parse configuration file", 53 | slog.String("configfile", a.config.Path()), 54 | slog.String("error", err.Error())) 55 | return 1 56 | } 57 | 58 | changed := false 59 | 60 | // For each argument add it to the list 61 | for _, entry := range args { 62 | 63 | // Add the entry 64 | a.config.Add(entry) 65 | 66 | changed = true 67 | 68 | } 69 | 70 | // Save the list. 71 | if changed { 72 | err = a.config.Save() 73 | if err != nil { 74 | logger.Error("failed to save the updated feed list", slog.String("error", err.Error())) 75 | return 1 76 | } 77 | } 78 | 79 | // All done, with no errors. 80 | return 0 81 | } 82 | -------------------------------------------------------------------------------- /add_cmd_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/skx/rss2email/configfile" 8 | ) 9 | 10 | func TestAdd(t *testing.T) { 11 | 12 | // Create an instance of the command, and setup a default 13 | // configuration file 14 | 15 | content := `# Comment here 16 | https://example.org/ 17 | https://example.net/ 18 | - foo: bar 19 | ` 20 | data := []byte(content) 21 | tmpfile, err := os.CreateTemp("", "example") 22 | if err != nil { 23 | t.Fatalf("Error creating temporary file") 24 | } 25 | 26 | if _, err = tmpfile.Write(data); err != nil { 27 | t.Fatalf("Error writing to config file") 28 | } 29 | if err = tmpfile.Close(); err != nil { 30 | t.Fatalf("Error creating temporary file") 31 | } 32 | 33 | add := addCmd{} 34 | add.Arguments(nil) 35 | config := configfile.NewWithPath(tmpfile.Name()) 36 | add.config = config 37 | 38 | // Add an entry 39 | add.Execute([]string{"https://blog.steve.fi/index.rss"}) 40 | 41 | // Open the file and confirm it has the content we expect 42 | x := configfile.NewWithPath(tmpfile.Name()) 43 | entries, err := x.Parse() 44 | if err != nil { 45 | t.Fatalf("Error parsing written file") 46 | } 47 | 48 | found := false 49 | for _, entry := range entries { 50 | if entry.URL == "https://blog.steve.fi/index.rss" { 51 | found = true 52 | } 53 | } 54 | 55 | if !found { 56 | t.Fatalf("Adding the entry failed") 57 | } 58 | 59 | os.Remove(tmpfile.Name()) 60 | } 61 | -------------------------------------------------------------------------------- /config_cmd.go: -------------------------------------------------------------------------------- 1 | // 2 | // Show help about our configuration file. 3 | // 4 | 5 | package main 6 | 7 | import ( 8 | "flag" 9 | "fmt" 10 | 11 | "github.com/skx/rss2email/configfile" 12 | ) 13 | 14 | // Structure for our options and state. 15 | type configCmd struct { 16 | 17 | // Configuration file, used for testing 18 | config *configfile.ConfigFile 19 | } 20 | 21 | // Arguments handles argument-flags we might have. 22 | // 23 | // In our case we use this as a hook to setup our configuration-file, 24 | // which allows testing. 25 | func (c *configCmd) Arguments(flags *flag.FlagSet) { 26 | c.config = configfile.New() 27 | } 28 | 29 | // Info is part of the subcommand-API 30 | func (c *configCmd) Info() (string, string) { 31 | 32 | // Get some details of the (new) configuration file. 33 | if c.config == nil { 34 | c.config = configfile.New() 35 | } 36 | path := c.config.Path() 37 | 38 | name := "config" 39 | doc := `Provide documentation for our configuration file. 40 | 41 | About 42 | ----- 43 | 44 | RSS2Email is a simple command-line utility which will fetch items from 45 | remote Atom and RSS feeds and generate emails. 46 | 47 | In order to operate it needs a list of remote Atom/RSS feeds to 48 | process, which are stored in a configuration file. 49 | 50 | 51 | Configuration File Location 52 | --------------------------- 53 | 54 | As of release 3.x of rss2email the configuration file will be loaded from 55 | the following location: 56 | 57 | ` + path 58 | 59 | doc += ` 60 | 61 | Configuration File Format 62 | ------------------------- 63 | 64 | The format of the configuration file is plain-text, and at its simplest 65 | it consists of nothing more than a series of URLs, one per line, like so: 66 | 67 | https://blog.steve.fi/index.rss 68 | http://floooh.github.io/feed.xml 69 | 70 | Entries can be commented out via the '#' character, temporarily: 71 | 72 | https://blog.steve.fi/index.rss 73 | # http://floooh.github.io/feed.xml 74 | 75 | In addition to containing a list of feed-locations the configuration file 76 | allows per-feed configuration options to be set. The general form of this 77 | support looks like this: 78 | 79 | https://foo.example.com/ 80 | - key:value 81 | - key:value2 82 | https://foo.example.com/ 83 | - key2:value2 84 | 85 | Here you see that lines prefixed with " -" will be used to specify a key 86 | and value separated with a ":" character. Configuration-options apply to 87 | the URL above their appearance. 88 | 89 | The first example demonstrates that configuration-keys may be repeated multiple 90 | times, if you desire. 91 | 92 | As configuration-items refer to feeds it is a fatal error for such a thing 93 | to appear before a URL. 94 | 95 | Per-Feed Configuration Options 96 | ------------------------------ 97 | 98 | Key | Purpose 99 | --------------+-------------------------------------------------------------- 100 | delay | The amount of time to sleep before retrying a failed HTTP-fetch 101 | | in seconds - "retry" configures the number of attempts to be made. 102 | exclude | Exclude any item which matches the given regular-expression. 103 | exclude-title | Exclude any item with a title matching the given regular-expression. 104 | exclude-older | Exclude any items whose publication date is older than the 105 | | specified number of days. 106 | frequency | How frequently to poll this feed, in minutes. 107 | include | Include only items which match the given regular-expression. 108 | include-title | Include only items with a title matching the given regular-expression. 109 | insecure | Ignore TLS failures when fetching feeds over https. 110 | | Disable the checks by setting this value to "true", or "yes". 111 | notify | Comma-delimited list of emails to send notifications to (if set, 112 | | replaces the emails specified in the cron/daemon command-line). 113 | retry | The maximum number of times to retry a failing HTTP-fetch. 114 | sleep | Sleep the specified number of seconds, before making the request. 115 | tag | Setup a tag for this feed, which can be accessed in the template. 116 | template | The path to a feed-specific email template to use. 117 | user-agent | Configure a specific User-Agent when making HTTP requests. 118 | 119 | 120 | Polling Frequency 121 | ----------------- 122 | 123 | Assuming you have a feed you only want to check once an hour, with the default 124 | behaviour our daemon command would poll the feed too often - as it processes 125 | its lists of feeds every five minutes by default. 126 | 127 | However if you changed the global sleep-delay to 60 then other feeds which 128 | you might want to track more closely would get delayed notifications. 129 | 130 | The "frequency" argument tracks the time between the last fetch of a feed, and 131 | allows you to say "Fetch this feed only if it was fetched >XX minutes ago". 132 | 133 | Note that frequencies of less than 5 minutes will be ignored, as that is how 134 | the sleep between executions takes. 135 | 136 | 137 | Regular Expression Tips 138 | ----------------------- 139 | 140 | Regular expressions are case-sensitive by default, to make them ignore any 141 | differences in case prefix them with "(?i)". 142 | 143 | For example the following entry will ignore any feed-items containing the 144 | word "cake" in their titles regardless of whether it is written as "cake", 145 | "Cake", or some other combination of upper and lower-cased letters: 146 | 147 | https://example.com/feed/path/here 148 | - exclude-title: (?i)cake 149 | 150 | ` 151 | return name, doc 152 | } 153 | 154 | // Execute is invoked if the user specifies `add` as the subcommand. 155 | func (c *configCmd) Execute(args []string) int { 156 | 157 | _, help := c.Info() 158 | fmt.Fprintf(out, "%s", help) 159 | 160 | // All done, with no errors. 161 | return 0 162 | } 163 | -------------------------------------------------------------------------------- /config_cmd_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "flag" 6 | "os" 7 | "strings" 8 | "testing" 9 | 10 | "github.com/skx/rss2email/configfile" 11 | ) 12 | 13 | func TestConfig(t *testing.T) { 14 | 15 | bak := out 16 | out = &bytes.Buffer{} 17 | defer func() { out = bak }() 18 | 19 | s := configCmd{} 20 | 21 | // 22 | // Call the handler. 23 | // 24 | s.Execute([]string{}) 25 | 26 | // 27 | // Look for some lines in the output 28 | // 29 | expected := []string{ 30 | "release 3.x", 31 | "RSS2Email is a simple", 32 | } 33 | 34 | // The text written to stdout 35 | output := out.(*bytes.Buffer).String() 36 | 37 | for _, txt := range expected { 38 | if !strings.Contains(output, txt) { 39 | t.Fatalf("Failed to find expected output") 40 | } 41 | } 42 | } 43 | 44 | // TestMissingConfig ensures we see a warning if the configuration 45 | // file is not present. 46 | func TestMissingConfig(t *testing.T) { 47 | 48 | // Create a temporary file, so we get a name of something 49 | // that doesn't exist 50 | tmpfile, err := os.CreateTemp("", "example") 51 | if err != nil { 52 | t.Fatalf("failed to create temporary file") 53 | } 54 | os.Remove(tmpfile.Name()) 55 | 56 | // 57 | // Setup a configuration-file, which doesn't exist. 58 | // 59 | s := configCmd{} 60 | flags := flag.NewFlagSet("test", flag.ContinueOnError) 61 | s.Arguments(flags) 62 | config := configfile.NewWithPath(tmpfile.Name()) 63 | s.config = config 64 | 65 | // Get the documentation 66 | _, doc := s.Info() 67 | 68 | // 69 | // Look for some lines in the output 70 | // 71 | expected := []string{ 72 | tmpfile.Name(), 73 | 74 | // known configuration options 75 | "include ", 76 | "include-title", 77 | "exclude ", 78 | "exclude-title", 79 | "exclude-older", 80 | } 81 | 82 | for _, txt := range expected { 83 | if !strings.Contains(doc, txt) { 84 | t.Fatalf("Failed to find expected output: %s", txt) 85 | } 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /configfile/FUZZING.md: -------------------------------------------------------------------------------- 1 | # Fuzz-Testing 2 | 3 | The 1.18 release of the golang compiler/toolset has integrated support for 4 | fuzz-testing. 5 | 6 | Fuzz-testing is basically magical and involves generating new inputs "randomly" 7 | and running test-cases with those inputs. 8 | 9 | 10 | ## Running 11 | 12 | If you're running 1.18beta1 or higher you can run the fuzz-testing against 13 | our configuration-file parser like so: 14 | 15 | $ go test -fuzztime=300s -parallel=1 -fuzz=FuzzParser -v 16 | .. 17 | --- PASS: TestFuzz (0.00s) 18 | === FUZZ FuzzParser 19 | fuzz: elapsed: 0s, gathering baseline coverage: 0/131 completed 20 | fuzz: elapsed: 0s, gathering baseline coverage: 131/131 completed, now fuzzing with 1 workers 21 | fuzz: elapsed: 3s, execs: 21154 (7050/sec), new interesting: 0 (total: 126) 22 | fuzz: elapsed: 6s, execs: 42006 (6951/sec), new interesting: 0 (total: 126) 23 | fuzz: elapsed: 9s, execs: 62001 (6663/sec), new interesting: 0 (total: 126) 24 | ... 25 | ... 26 | fuzz: elapsed: 4m57s, execs: 1143698 (0/sec), new interesting: 7 (total: 156) 27 | fuzz: elapsed: 5m0s, execs: 1143698 (0/sec), new interesting: 7 (total: 156) 28 | fuzz: elapsed: 5m1s, execs: 1143698 (0/sec), new interesting: 7 (total: 156) 29 | --- PASS: FuzzParser (301.07s) 30 | PASS 31 | ok github.com/skx/rss2email/configfile 301.135s 32 | 33 | 34 | You'll note that I've added `-parallel=1` to the test, because otherwise my desktop system becomes unresponsive while the testing is going on. 35 | -------------------------------------------------------------------------------- /configfile/configfile.go: -------------------------------------------------------------------------------- 1 | // Package configfile contains the logic to read a list of source 2 | // URLs, along with any (optional) configuration-directives. 3 | // 4 | // A configuration file looks like this: 5 | // 6 | // https://example.com/ 7 | // - foo:bar 8 | // 9 | // https://example.org/ 10 | // https://example.net/ 11 | // # comment 12 | // 13 | // It is assumed lines contain URLs, but anything prefixed with a "-" 14 | // is taken to be a parameter using a colon-deliminator. 15 | package configfile 16 | 17 | import ( 18 | "bufio" 19 | "fmt" 20 | "os" 21 | "path/filepath" 22 | "regexp" 23 | "strings" 24 | 25 | "github.com/skx/rss2email/state" 26 | ) 27 | 28 | // Option contain options which are used on a per-feed basis. 29 | // 30 | // We could use a map, but that would mean that each named option could 31 | // only be used once - and we want to allow multiple "exclude" values 32 | // for example. 33 | type Option struct { 34 | // Name holds the name of the configuration option. 35 | Name string 36 | 37 | // Value contains the specified value of the configuration option. 38 | Value string 39 | } 40 | 41 | // Feed is an entry which is read from our configuration-file. 42 | // 43 | // A feed consists of an URL pointing to an Atom/RSS feed, as well as 44 | // an optional set of parameters which are specific to that feed. 45 | type Feed struct { 46 | // URL is the URL of an Atom/RSS feed. 47 | URL string 48 | 49 | // Options contains a collection of any optional parameters 50 | // which have been read after an URL 51 | Options []Option 52 | } 53 | 54 | // ConfigFile contains our state. 55 | type ConfigFile struct { 56 | 57 | // Path contains the path to our config file 58 | path string 59 | 60 | // The entries we found. 61 | entries []Feed 62 | 63 | // Key:value regular expression 64 | re *regexp.Regexp 65 | } 66 | 67 | // New creates a new configuration-file reader. 68 | func New() *ConfigFile { 69 | return &ConfigFile{ 70 | re: regexp.MustCompile(`^([^:]+):(.*)$`), 71 | } 72 | } 73 | 74 | // NewWithPath creates a configuration-file reader, using the given file as 75 | // a source. This is primarily used for testing. 76 | func NewWithPath(file string) *ConfigFile { 77 | 78 | // Create new object - to avoid having to repeat our regexp 79 | // initialization. 80 | x := New() 81 | 82 | // Setup the path, and return the updated object. 83 | x.path = file 84 | return x 85 | } 86 | 87 | // Path returns the path to the configuration-file. 88 | func (c *ConfigFile) Path() string { 89 | 90 | // If we've not calculated the path then do so now. 91 | if c.path == "" { 92 | c.path = filepath.Join(state.Directory(), "feeds.txt") 93 | } 94 | 95 | return c.path 96 | } 97 | 98 | // Parse returns the entries from the config-file 99 | func (c *ConfigFile) Parse() ([]Feed, error) { 100 | 101 | // Remove all existing entries 102 | c.entries = []Feed{} 103 | 104 | // Open the file 105 | file, err := os.Open(c.Path()) 106 | if err != nil { 107 | return c.entries, err 108 | } 109 | defer file.Close() 110 | 111 | // Temporary entry 112 | var tmp Feed 113 | tmp.Options = []Option{} 114 | 115 | // Create a scanner to process the file. 116 | scanner := bufio.NewScanner(file) 117 | 118 | // Scan line by line 119 | for scanner.Scan() { 120 | 121 | // Get the line, and strip leading/trailing space 122 | line := scanner.Text() 123 | line = strings.TrimSpace(line) 124 | 125 | // skip comments 126 | if strings.HasPrefix(line, "#") { 127 | continue 128 | } 129 | 130 | // optional params have "-" prefix 131 | if strings.HasPrefix(line, "-") { 132 | 133 | // options go AFTER the URL to which they refer 134 | if tmp.URL == "" { 135 | return c.entries, fmt.Errorf("error: option outside a URL: %s", scanner.Text()) 136 | } 137 | 138 | // Remove the prefix and split by ":" 139 | line = strings.TrimPrefix(line, "-") 140 | 141 | // Look for "foo:bar" 142 | fields := c.re.FindStringSubmatch(line) 143 | 144 | // If we got key/val then save them away 145 | if len(fields) == 3 { 146 | key := strings.TrimSpace(fields[1]) 147 | val := strings.TrimSpace(fields[2]) 148 | tmp.Options = append(tmp.Options, Option{Name: key, Value: val}) 149 | } else { 150 | // If we have an URL show it, to help identify the section which is broken 151 | if tmp.URL != "" { 152 | return c.entries, fmt.Errorf("options should be of the form 'key:value', bogus entry found '%s', beneath feed %s", line, tmp.URL) 153 | } 154 | return c.entries, fmt.Errorf("options should be of the form 'key:value', bogus entry found '%s'", line) 155 | 156 | } 157 | } else { 158 | 159 | // If we already have a URL stored then append 160 | // it and reset our temporary structure 161 | if tmp.URL != "" { 162 | // store it, and reset our map 163 | c.entries = append(c.entries, tmp) 164 | tmp.Options = []Option{} 165 | } 166 | 167 | // set the url 168 | tmp.URL = line 169 | } 170 | } 171 | 172 | // Ensure we don't forget about the last item in the file. 173 | if tmp.URL != "" { 174 | c.entries = append(c.entries, tmp) 175 | } 176 | 177 | // Look for scanner-errors 178 | if err := scanner.Err(); err != nil { 179 | return c.entries, err 180 | } 181 | 182 | return c.entries, nil 183 | } 184 | 185 | // Add appends the given URIs to the config-file 186 | // 187 | // You must call `Save` if you wish this removal to be persisted. 188 | func (c *ConfigFile) Add(uris ...string) { 189 | 190 | for _, uri := range uris { 191 | 192 | // Look to see if it is already-present. 193 | found := false 194 | for _, ent := range c.entries { 195 | if ent.URL == uri { 196 | found = true 197 | } 198 | } 199 | 200 | // Not found? Then we can add it. 201 | if !found { 202 | f := Feed{URL: uri} 203 | c.entries = append(c.entries, f) 204 | } 205 | } 206 | } 207 | 208 | // Delete removes an entry from our list of feeds. 209 | // 210 | // You must call `Save` if you wish this removal to be persisted. 211 | func (c *ConfigFile) Delete(url string) { 212 | 213 | var keep []Feed 214 | 215 | for _, ent := range c.entries { 216 | if ent.URL != url { 217 | keep = append(keep, ent) 218 | } 219 | } 220 | 221 | c.entries = keep 222 | } 223 | 224 | // Save persists our list of feeds/options to disk. 225 | func (c *ConfigFile) Save() error { 226 | 227 | // Open the file 228 | file, err := os.Create(c.Path()) 229 | if err != nil { 230 | return err 231 | } 232 | 233 | // For each entry do the necessary 234 | for _, entry := range c.entries { 235 | 236 | fmt.Fprintf(file, "%s\n", entry.URL) 237 | 238 | for _, opt := range entry.Options { 239 | fmt.Fprintf(file, " - %s:%s\n", opt.Name, opt.Value) 240 | } 241 | 242 | } 243 | 244 | err = file.Close() 245 | return err 246 | } 247 | -------------------------------------------------------------------------------- /configfile/configfile_test.go: -------------------------------------------------------------------------------- 1 | package configfile 2 | 3 | import ( 4 | "os" 5 | "strings" 6 | "testing" 7 | ) 8 | 9 | // Test the default path works 10 | func TestDefaultPath(t *testing.T) { 11 | 12 | c := New() 13 | p := c.Path() 14 | if !strings.Contains(p, "feeds.txt") { 15 | t.Fatalf("expected path to be populated") 16 | } 17 | } 18 | 19 | // Test a file exists 20 | func TestExists(t *testing.T) { 21 | 22 | // Create a temporary file 23 | tmpfile, err := os.CreateTemp("", "example") 24 | if err != nil { 25 | t.Fatalf("error creating temporary file") 26 | } 27 | 28 | // Create the config-reader and pass it the 29 | // name of our temporary file 30 | conf := New() 31 | conf.path = tmpfile.Name() 32 | 33 | // Same again with the different constructor. 34 | conf2 := NewWithPath(tmpfile.Name()) 35 | 36 | // Remove the temporary file 37 | os.Remove(conf.path) 38 | 39 | // Parsing should return an error, when the file doesn't exist 40 | _, err = conf.Parse() 41 | if err == nil { 42 | t.Fatalf("Expected an error parsing a missing file, got none!") 43 | } 44 | 45 | // Parsing should return an error, when the file doesn't exist 46 | _, err = conf2.Parse() 47 | if err == nil { 48 | t.Fatalf("Expected an error parsing a missing file, got none!") 49 | } 50 | } 51 | 52 | // TestBasicFile tests parsing a basic file. 53 | func TestBasicFile(t *testing.T) { 54 | 55 | c := ParserHelper(t, `https://example.com/ 56 | https://example.net/ 57 | 58 | 59 | https://example.org`) 60 | 61 | out, err := c.Parse() 62 | if err != nil { 63 | t.Fatalf("Error parsing file: %v", err) 64 | } 65 | 66 | if len(out) != 3 { 67 | t.Fatalf("parsed wrong number of entries, got %d\n%v", len(out), out) 68 | } 69 | 70 | // All options should have no params 71 | for _, entry := range out { 72 | if len(entry.Options) != 0 { 73 | t.Fatalf("Found entry with unexpected parameters:%s\n", entry.URL) 74 | } 75 | } 76 | 77 | os.Remove(c.path) 78 | } 79 | 80 | // TestEmptyFile tests parsing an empty file. 81 | func TestEmptyFile(t *testing.T) { 82 | 83 | c := ParserHelper(t, ``) 84 | 85 | out, err := c.Parse() 86 | if err != nil { 87 | t.Fatalf("Error parsing file: %v", err) 88 | } 89 | 90 | if len(out) != 0 { 91 | t.Fatalf("parsed wrong number of entries, got %d\n%v", len(out), out) 92 | } 93 | 94 | os.Remove(c.path) 95 | } 96 | 97 | // TestEmptyFileComment tests parsing a file empty of everything but comments 98 | func TestEmptyFileComment(t *testing.T) { 99 | 100 | c := ParserHelper(t, `# Comment1 101 | #Comment2`) 102 | 103 | out, err := c.Parse() 104 | if err != nil { 105 | t.Fatalf("Error parsing file: %v", err) 106 | } 107 | 108 | if len(out) != 0 { 109 | t.Fatalf("parsed wrong number of entries, got %d\n%v", len(out), out) 110 | } 111 | 112 | os.Remove(c.path) 113 | } 114 | 115 | // TestOptions tests parsing a file with one URL with options 116 | func TestOptions(t *testing.T) { 117 | 118 | c := ParserHelper(t, ` 119 | http://example.com/ 120 | - foo:bar 121 | - retry: 7 122 | #Comment2`) 123 | 124 | out, err := c.Parse() 125 | if err != nil { 126 | t.Fatalf("Error parsing file: %v", err) 127 | } 128 | 129 | // One entry 130 | if len(out) != 1 { 131 | t.Fatalf("parsed wrong number of entries, got %d\n%v", len(out), out) 132 | } 133 | 134 | // We should have two options 135 | if len(out[0].Options) != 2 { 136 | t.Fatalf("Found wrong number of options, got %d", len(out[0].Options)) 137 | } 138 | 139 | for _, opt := range out[0].Options { 140 | if opt.Name != "foo" && 141 | opt.Name != "retry" { 142 | t.Fatalf("found bogus option %v", opt) 143 | } 144 | } 145 | 146 | os.Remove(c.path) 147 | } 148 | 149 | // TestEmptyOption tests that a key with no value raises an error 150 | func TestEmptyOption(t *testing.T) { 151 | 152 | c := ParserHelper(t, ` 153 | http://example.com/ 154 | - insecure 155 | `) 156 | 157 | _, err := c.Parse() 158 | if err == nil { 159 | t.Fatalf("Expected an error with an empty option, but got none") 160 | } 161 | os.Remove(c.path) 162 | } 163 | 164 | // TestComplexOption tests parsing an option containing ":" 165 | func TestComplexOption(t *testing.T) { 166 | 167 | c := ParserHelper(t, ` 168 | http://example.com/ 169 | - include:(?i)(foo:|bar:) 170 | `) 171 | 172 | out, err := c.Parse() 173 | if err != nil { 174 | t.Fatalf("Error parsing file: %v", err) 175 | } 176 | 177 | // One entry 178 | if len(out) != 1 { 179 | t.Fatalf("parsed wrong number of entries, got %d\n%v", len(out), out) 180 | } 181 | 182 | // We should have one option 183 | if len(out[0].Options) != 1 { 184 | t.Fatalf("Found wrong number of options, got %d", len(out[0].Options)) 185 | } 186 | 187 | if out[0].Options[0].Name != "include" { 188 | t.Fatalf("unexpected option name") 189 | } 190 | if out[0].Options[0].Value != "(?i)(foo:|bar:)" { 191 | t.Fatalf("unexpected option value") 192 | } 193 | 194 | os.Remove(c.path) 195 | } 196 | 197 | // TestBrokenOptions looks for options outside an URL 198 | func TestBrokenOptions(t *testing.T) { 199 | 200 | c := ParserHelper(t, `# https://example.com/index.rss 201 | - foo: bar`) 202 | 203 | _, err := c.Parse() 204 | if err == nil { 205 | t.Fatalf("Expected an error, got none!") 206 | } 207 | if !strings.Contains(err.Error(), "outside") { 208 | t.Fatalf("Got an error, but not the correct one:%s", err.Error()) 209 | } 210 | 211 | os.Remove(c.path) 212 | 213 | } 214 | 215 | // TestAdd tests adding an entry works 216 | func TestAdd(t *testing.T) { 217 | 218 | c := ParserHelper(t, ``) 219 | 220 | entries, err := c.Parse() 221 | if err != nil { 222 | t.Fatalf("unexpected error") 223 | } 224 | if len(entries) != 0 { 225 | t.Fatalf("expected no entries, but got some") 226 | } 227 | 228 | // add multiple times 229 | c.Add("https://example.com/") 230 | c.Add("https://example.com/") 231 | 232 | // Save 233 | err = c.Save() 234 | if err != nil { 235 | t.Fatalf("error saving") 236 | } 237 | 238 | // parse now we've saved 239 | entries, err = c.Parse() 240 | if err != nil { 241 | t.Fatalf("unexpected error") 242 | } 243 | if len(entries) != 1 { 244 | t.Fatalf("expected one entry, got %d", len(entries)) 245 | } 246 | 247 | os.Remove(c.path) 248 | } 249 | 250 | // TestAddProperties tests adding to a file with properties doesn't fail 251 | func TestAddProperties(t *testing.T) { 252 | 253 | c := ParserHelper(t, ` 254 | http://example.com/ 255 | - foo:bar 256 | - retry: 7 257 | #Comment2`) 258 | 259 | var out []Feed 260 | var err error 261 | 262 | _, err = c.Parse() 263 | if err != nil { 264 | t.Fatalf("Error parsing file: %v", err) 265 | } 266 | 267 | // Add another entry 268 | c.Add("https://blog.steve.fi/index.rss") 269 | 270 | // Now save and reload 271 | err = c.Save() 272 | if err != nil { 273 | t.Fatalf("Error saving file") 274 | } 275 | 276 | // Reparse 277 | out, err = c.Parse() 278 | if err != nil { 279 | t.Fatalf("Error parsing file: %v", err) 280 | } 281 | 282 | // Two entries now 283 | if len(out) != 2 { 284 | t.Fatalf("parsed wrong number of entries, got %d\n%v", len(out), out) 285 | } 286 | 287 | // We should have two options 288 | if len(out[0].Options) != 2 { 289 | t.Fatalf("Found wrong number of options, got %d", len(out[0].Options)) 290 | } 291 | 292 | for _, opt := range out[0].Options { 293 | if opt.Name != "foo" && 294 | opt.Name != "retry" { 295 | t.Fatalf("found bogus option %v", opt) 296 | } 297 | } 298 | 299 | os.Remove(c.path) 300 | } 301 | 302 | // TestDelete tests removing an entry. 303 | func TestDelete(t *testing.T) { 304 | 305 | c := ParserHelper(t, ` 306 | http://example.com/ 307 | - foo:bar 308 | - retry: 7 309 | #Comment2 310 | https://bob.com/index.rss`) 311 | 312 | var out []Feed 313 | var err error 314 | 315 | _, err = c.Parse() 316 | if err != nil { 317 | t.Fatalf("Error parsing file: %v", err) 318 | } 319 | 320 | // Add another entry 321 | c.Delete("https://bob.com/index.rss") 322 | 323 | // Now save and reload 324 | err = c.Save() 325 | if err != nil { 326 | t.Fatalf("Error saving file") 327 | } 328 | 329 | // Reparse 330 | out, err = c.Parse() 331 | if err != nil { 332 | t.Fatalf("Error parsing file: %v", err) 333 | } 334 | 335 | // One entries now 336 | if len(out) != 1 { 337 | t.Fatalf("parsed wrong number of entries, got %d\n%v", len(out), out) 338 | } 339 | 340 | // We should have two options 341 | if len(out[0].Options) != 2 { 342 | t.Fatalf("Found wrong number of options, got %d", len(out[0].Options)) 343 | } 344 | 345 | for _, opt := range out[0].Options { 346 | if opt.Name != "foo" && 347 | opt.Name != "retry" { 348 | t.Fatalf("found bogus option %v", opt) 349 | } 350 | } 351 | 352 | os.Remove(c.path) 353 | } 354 | 355 | // TestSaveBogusFile ensures that saving to a bogus file results in an error 356 | func TestSaveBogusFile(t *testing.T) { 357 | 358 | // Create an empty config 359 | c := ParserHelper(t, ``) 360 | 361 | // Remove the path, and setup something bogus 362 | os.Remove(c.path) 363 | c.path = "/dev/null/fsdf/C:/3ljs3" 364 | 365 | err := c.Save() 366 | if err == nil { 367 | t.Fatalf("Saving to a bogus file worked, and it shouldn't!") 368 | } 369 | 370 | } 371 | 372 | // ParserHelper writes the specified text to a configuration-file 373 | // and configures that path to be used for a ConfigFile object 374 | func ParserHelper(t *testing.T, content string) *ConfigFile { 375 | 376 | data := []byte(content) 377 | tmpfile, err := os.CreateTemp("", "example") 378 | if err != nil { 379 | t.Fatalf("Error creating temporary file") 380 | } 381 | 382 | if _, err := tmpfile.Write(data); err != nil { 383 | t.Fatalf("Error writing to config file") 384 | } 385 | if err := tmpfile.Close(); err != nil { 386 | t.Fatalf("Error creating temporary file") 387 | } 388 | 389 | // Create a new config-reader 390 | c := New() 391 | c.path = tmpfile.Name() 392 | 393 | return c 394 | } 395 | -------------------------------------------------------------------------------- /configfile/fuzz_test.go: -------------------------------------------------------------------------------- 1 | //go:build go1.18 2 | // +build go1.18 3 | 4 | package configfile 5 | 6 | import ( 7 | "os" 8 | "strings" 9 | "testing" 10 | ) 11 | 12 | func FuzzParser(f *testing.F) { 13 | f.Add([]byte("")) 14 | f.Add([]byte("https://example.com")) 15 | f.Add([]byte("https://example.com\r")) 16 | f.Add([]byte("https://example.com\r\n")) 17 | f.Add([]byte(` 18 | https://example.com 19 | - foo:bar 20 | - bar:baz 21 | https://example.com 22 | - foo:bar 23 | - bar:baz`)) 24 | 25 | f.Fuzz(func(t *testing.T, input []byte) { 26 | // Create a temporary file 27 | tmpfile, _ := os.CreateTemp("", "example") 28 | 29 | // Cleanup when we're done 30 | defer os.Remove(tmpfile.Name()) 31 | 32 | // Write it out 33 | _, err := tmpfile.Write(input) 34 | if err != nil { 35 | t.Fatalf("failed to write temporary file %s", err) 36 | } 37 | 38 | tmpfile.Close() 39 | 40 | // Create a new config-reader 41 | c := NewWithPath(tmpfile.Name()) 42 | 43 | // Parse, looking for errors 44 | _, err = c.Parse() 45 | if err != nil { 46 | 47 | // This is a known error, we expect to get 48 | if !strings.Contains(err.Error(), "option outside a URL") { 49 | t.Errorf("Input gave bad error: %s %s\n", input, err) 50 | } 51 | 52 | } 53 | }) 54 | } 55 | -------------------------------------------------------------------------------- /cron_cmd.go: -------------------------------------------------------------------------------- 1 | // 2 | // This is the cron-subcommand. 3 | // 4 | 5 | package main 6 | 7 | import ( 8 | "flag" 9 | "fmt" 10 | "log/slog" 11 | "os" 12 | "strings" 13 | 14 | "github.com/skx/rss2email/processor" 15 | ) 16 | 17 | // Structure for our options and state. 18 | type cronCmd struct { 19 | // Should we be verbose in operation? 20 | verbose bool 21 | 22 | // Should we send emails? 23 | send bool 24 | } 25 | 26 | // Info is part of the subcommand-API. 27 | func (c *cronCmd) Info() (string, string) { 28 | return "cron", `Send emails for each new entry in our feed lists. 29 | 30 | This sub-command polls all configured feeds, sending an email for 31 | new item in those feeds. 32 | 33 | The list of feeds is read from '~/.rss2email/feeds'. 34 | 35 | We record details of all the feed-items which have been seen beneath 36 | '~/.rss2email/seen/', and these entries will be expired automatically 37 | when the corresponding entries have fallen out of the source feed. 38 | 39 | Example: 40 | 41 | $ rss2email cron user1@example.com user2@example.com 42 | 43 | 44 | Email Sending: 45 | 46 | By default we pipe outgoing messages through '/usr/sbin/sendmail' for delivery, 47 | however it is possible to use SMTP for sending emails directly. If you 48 | wish to use SMTP you need to configure the following environmental variables: 49 | 50 | SMTP_HOST (e.g. "smtp.gmail.com") 51 | SMTP_PORT (e.g. "587") 52 | SMTP_USERNAME (e.g. "user@domain.com") 53 | SMTP_PASSWORD (e.g. "secret!word#here") 54 | 55 | 56 | Email Template: 57 | 58 | An embedded template is used to generate the emails which are sent, you 59 | may create a local override for this, for more details see : 60 | 61 | $ rss2email help list-default-template 62 | ` 63 | } 64 | 65 | // Arguments handles our flag-setup. 66 | func (c *cronCmd) Arguments(f *flag.FlagSet) { 67 | f.BoolVar(&c.verbose, "verbose", false, "Should we be extra verbose?") 68 | f.BoolVar(&c.send, "send", true, "Should we send emails, or just pretend to?") 69 | } 70 | 71 | // Entry-point 72 | func (c *cronCmd) Execute(args []string) int { 73 | 74 | // verbose will change the log-level of our logger 75 | if c.verbose { 76 | loggerLevel.Set(slog.LevelDebug) 77 | } 78 | 79 | // No argument? That's a bug 80 | if len(args) == 0 { 81 | fmt.Printf("Usage: rss2email cron email1@example.com .. emailN@example.com\n") 82 | return 1 83 | } 84 | 85 | // The list of addresses to notify, unless overridden by a per-feed 86 | // configuration option. 87 | recipients := []string{} 88 | 89 | // Save each argument away, checking it is fully-qualified. 90 | for _, email := range args { 91 | if strings.Contains(email, "@") { 92 | recipients = append(recipients, email) 93 | } else { 94 | fmt.Printf("Usage: rss2email cron [flags] email1 .. emailN\n") 95 | return 1 96 | } 97 | } 98 | 99 | // Create the helper 100 | p, err := processor.New() 101 | if err != nil { 102 | logger.Error("failed to create feed processor", 103 | slog.String("error", err.Error())) 104 | return 1 105 | } 106 | 107 | // Close the database handle, once processed. 108 | defer p.Close() 109 | 110 | // Setup the state 111 | p.SetSendEmail(c.send) 112 | p.SetLogger(logger) 113 | 114 | errors := p.ProcessFeeds(recipients) 115 | 116 | // If we found errors then show them. 117 | if len(errors) != 0 { 118 | for _, err := range errors { 119 | fmt.Fprintln(os.Stderr, err.Error()) 120 | } 121 | 122 | return 1 123 | } 124 | 125 | // All good. 126 | return 0 127 | } 128 | -------------------------------------------------------------------------------- /cron_cmd_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestCronNoArguments(t *testing.T) { 8 | 9 | c := cronCmd{} 10 | 11 | out := c.Execute([]string{}) 12 | if out != 1 { 13 | t.Fatalf("Expected error when called with no arguments") 14 | } 15 | } 16 | 17 | func TestCronNotEmails(t *testing.T) { 18 | 19 | d := cronCmd{} 20 | 21 | out := d.Execute([]string{"foo@example.com", "bar"}) 22 | if out != 1 { 23 | t.Fatalf("Expected error when called with non-email addresses") 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /daemon_cmd.go: -------------------------------------------------------------------------------- 1 | // 2 | // This is the daemon-subcommand. 3 | // 4 | 5 | package main 6 | 7 | import ( 8 | "flag" 9 | "fmt" 10 | "log/slog" 11 | "os" 12 | "strings" 13 | "time" 14 | 15 | "github.com/skx/rss2email/processor" 16 | ) 17 | 18 | // Structure for our options and state. 19 | type daemonCmd struct { 20 | 21 | // Should we be verbose in operation? 22 | verbose bool 23 | } 24 | 25 | // Info is part of the subcommand-API. 26 | func (d *daemonCmd) Info() (string, string) { 27 | return "daemon", `Send emails for each new entry in our feed lists. 28 | 29 | This sub-command polls all configured feeds, sending an email for 30 | each item which is new. Once the list of feeds has been processed 31 | the command will pause for 15 minutes, before beginning again. 32 | 33 | To see details of the configuration file, including the location, please 34 | run: 35 | 36 | $ rss2email help config 37 | 38 | In terms of implementation this command follows everything documented 39 | in the 'cron' sub-command. The only difference is this one never 40 | terminates - even if email-generation fails. 41 | 42 | 43 | Example: 44 | 45 | $ rss2email daemon user1@example.com user2@example.com 46 | ` 47 | } 48 | 49 | // Arguments handles our flag-setup. 50 | func (d *daemonCmd) Arguments(f *flag.FlagSet) { 51 | f.BoolVar(&d.verbose, "verbose", false, "Should we be extra verbose?") 52 | } 53 | 54 | // Entry-point 55 | func (d *daemonCmd) Execute(args []string) int { 56 | 57 | // If running verbosely change our log-level 58 | if d.verbose { 59 | loggerLevel.Set(slog.LevelDebug) 60 | } 61 | 62 | // No argument? That's a bug 63 | if len(args) == 0 { 64 | fmt.Printf("Usage: rss2email daemon email1@example.com .. emailN@example.com\n") 65 | return 1 66 | } 67 | 68 | // The list of addresses to notify, unless overridden by a per-feed 69 | // configuration option. 70 | recipients := []string{} 71 | 72 | // Save each argument away, checking it is fully-qualified. 73 | for _, email := range args { 74 | if strings.Contains(email, "@") { 75 | recipients = append(recipients, email) 76 | } else { 77 | fmt.Printf("Usage: rss2email daemon [flags] email1 .. emailN\n") 78 | return 1 79 | } 80 | } 81 | 82 | for { 83 | 84 | // Create the helper 85 | p, err := processor.New() 86 | 87 | if err != nil { 88 | logger.Error("failed to create feed processor", 89 | slog.String("error", err.Error())) 90 | return 1 91 | } 92 | 93 | // Ensure we send our version 94 | p.SetVersion(version) 95 | 96 | // Setup the state - note we ALWAYS send emails in this mode. 97 | p.SetSendEmail(true) 98 | p.SetLogger(logger) 99 | 100 | // Process all the feeds 101 | errors := p.ProcessFeeds(recipients) 102 | 103 | // If we found errors then show them. 104 | if len(errors) != 0 { 105 | for _, err := range errors { 106 | fmt.Fprintln(os.Stderr, err.Error()) 107 | } 108 | } 109 | 110 | // Close the database handle, once processed. 111 | p.Close() 112 | 113 | // Default time to sleep - in minutes 114 | n := 5 115 | 116 | logger.Warn("sleeping before polling feeds again", 117 | slog.Int("delay.minutes", n)) 118 | 119 | time.Sleep(time.Duration(n) * time.Minute) 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /daemon_cmd_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestDaemonNoArguments(t *testing.T) { 8 | 9 | d := daemonCmd{} 10 | 11 | out := d.Execute([]string{}) 12 | if out != 1 { 13 | t.Fatalf("Expected error when called with no arguments") 14 | } 15 | } 16 | 17 | func TestDaemonNotEmails(t *testing.T) { 18 | 19 | d := daemonCmd{} 20 | 21 | out := d.Execute([]string{"foo@example.com", "bart"}) 22 | if out != 1 { 23 | t.Fatalf("Expected error when called with non-email addresses") 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /del_cmd.go: -------------------------------------------------------------------------------- 1 | // 2 | // Delete a feed from our feed-list. 3 | // 4 | 5 | package main 6 | 7 | import ( 8 | "flag" 9 | "log/slog" 10 | 11 | "github.com/skx/rss2email/configfile" 12 | "github.com/skx/subcommands" 13 | ) 14 | 15 | // Structure for our options and state. 16 | type delCmd struct { 17 | 18 | // We embed the NoFlags option, because we accept no command-line flags. 19 | subcommands.NoFlags 20 | 21 | // Configuration file, used for testing 22 | config *configfile.ConfigFile 23 | } 24 | 25 | // Arguments handles argument-flags we might have. 26 | // 27 | // In our case we use this as a hook to setup our configuration-file, 28 | // which allows testing. 29 | func (d *delCmd) Arguments(flags *flag.FlagSet) { 30 | d.config = configfile.New() 31 | } 32 | 33 | // Info is part of the subcommand-API 34 | func (d *delCmd) Info() (string, string) { 35 | return "delete", `Remove a feed from our feed-list. 36 | 37 | Remove one or more specified URLs from the configuration file. 38 | 39 | To see details of the configuration file, including the location, 40 | please run: 41 | 42 | $ rss2email help config 43 | 44 | Example: 45 | 46 | $ rss2email delete https://blog.steve.fi/index.rss 47 | ` 48 | } 49 | 50 | // Entry-point. 51 | func (d *delCmd) Execute(args []string) int { 52 | 53 | // Parse the existing file 54 | _, err := d.config.Parse() 55 | if err != nil { 56 | logger.Error("failed to parse configuration file", 57 | slog.String("configfile", d.config.Path()), 58 | slog.String("error", err.Error())) 59 | return 1 60 | } 61 | 62 | changed := false 63 | 64 | // For each argument remove it from the list, if present. 65 | for _, entry := range args { 66 | d.config.Delete(entry) 67 | changed = true 68 | } 69 | 70 | // Save the list. 71 | if changed { 72 | err = d.config.Save() 73 | if err != nil { 74 | logger.Error("failed to save the updated feed list", slog.String("error", err.Error())) 75 | return 1 76 | } 77 | } 78 | 79 | // All done, with no errors. 80 | return 0 81 | 82 | } 83 | -------------------------------------------------------------------------------- /del_cmd_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/skx/rss2email/configfile" 8 | ) 9 | 10 | func TestDel(t *testing.T) { 11 | 12 | // Create an instance of the command, and setup a default 13 | // configuration file 14 | content := `# Comment here 15 | https://example.org/ 16 | https://example.net/ 17 | - foo: bar 18 | ` 19 | data := []byte(content) 20 | tmpfile, err := os.CreateTemp("", "example") 21 | if err != nil { 22 | t.Fatalf("Error creating temporary file") 23 | } 24 | 25 | if _, err = tmpfile.Write(data); err != nil { 26 | t.Fatalf("Error writing to config file") 27 | } 28 | if err = tmpfile.Close(); err != nil { 29 | t.Fatalf("Error creating temporary file") 30 | } 31 | 32 | del := delCmd{} 33 | del.Arguments(nil) // only for coverage 34 | 35 | config := configfile.NewWithPath(tmpfile.Name()) 36 | del.config = config 37 | 38 | // Delete an entry 39 | del.Execute([]string{"https://example.net/"}) 40 | 41 | // Open the file and confirm only one entry. 42 | x := configfile.NewWithPath(tmpfile.Name()) 43 | entries, err := x.Parse() 44 | if err != nil { 45 | t.Fatalf("Error parsing written file") 46 | } 47 | 48 | if len(entries) != 1 { 49 | t.Fatalf("Expected only one entry") 50 | } 51 | if entries[0].URL != "https://example.org/" { 52 | t.Fatalf("Wrong item deleted") 53 | } 54 | if len(entries[0].Options) != 0 { 55 | t.Fatalf("We have orphaned parameters") 56 | } 57 | 58 | os.Remove(tmpfile.Name()) 59 | } 60 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.8" 2 | services: 3 | rss2email: 4 | command: daemon -verbose steve@steve.fi 5 | entrypoint: rss2email 6 | working_dir: /app 7 | user: app 8 | environment: 9 | - TZ=UTC 10 | - SMTP_USERNAME=steve@example.com 11 | - SMTP_PASSWORD=blah.blah.blah! 12 | - SMTP_HOST=smtp.gmail.com 13 | - SMTP_PORT=587 14 | restart: unless-stopped 15 | image: ghcr.io/skx/rss2email:master 16 | volumes: 17 | - rss2email-data:/app/.rss2email 18 | 19 | volumes: 20 | rss2email-data: -------------------------------------------------------------------------------- /example_templates/README.md: -------------------------------------------------------------------------------- 1 | # Template Examples Folder 2 | 3 | This folder contains various template examples for use with the `rss2email` tool. Each template is designed to format and display RSS feed items in a specific way. Below, we provide instructions on how to download a YouTube template and a summary of the placeholders used in the template. 4 | 5 | ## YouTube Template 6 | 7 | ### Downloading the Template 8 | 9 | To use the YouTube template, follow these steps to download it into a local file named `email.tmpl`: 10 | 11 | ```bash 12 | curl -o email.tmpl https://raw.githubusercontent.com/skx/rss2email/master/example_templates/youtube.txt 13 | ``` 14 | 15 | ### Template Summary 16 | 17 | The YouTube template is designed to handle YouTube RSS feeds that use the Atom format. The description field in YouTube feeds is not parsed by `gofeed`, and it is stored in the extensions. The template discovers the field by looking for the description name. 18 | 19 | #### Placeholders: 20 | 21 | 1. `{{.RSSItem.Author.Name}}`: Represents the author name. 22 | 2. `{{.RSSItem.Published}}`: Represents the published date. 23 | 3. `{{.RSSItem.Extensions}}`: Represents the rest of the fields that were not parsed. They are saved using the Extensions type[^1]. 24 | 25 | The description has been retrieved by digging into the `{{.RSSItem.Extensions}}` attribute. 26 | 27 | For more details about the Extensions type and default mappings in `gofeed`, refer to the following documentation: 28 | - [Extensions Type Documentation](https://pkg.go.dev/github.com/mmcdole/gofeed@v1.2.1/extensions#Extensions)[^1] 29 | - [gofeed Default Mappings Documentation](https://pkg.go.dev/github.com/mmcdole/gofeed@v1.2.1#readme-default-mappings)[^2] 30 | 31 | [^1]: [Extensions Type Documentation](https://pkg.go.dev/github.com/mmcdole/gofeed@v1.2.1/extensions#Extensions) 32 | [^2]: [gofeed Default Mappings Documentation](https://pkg.go.dev/github.com/mmcdole/gofeed@v1.2.1#readme-default-mappings) 33 | 34 | ### Feed Format 35 | 36 | These feeds can be obtained with the following URL format: 37 | 38 | ```plaintext 39 | https://www.youtube.com/feeds/videos.xml?channel_id=${CHANNEL_ID} 40 | ``` 41 | 42 | Replace `CHANNEL_ID` with the actual channel ID, which can be retrieved from the channel URL (for instance: `https://www.youtube.com/channel/${CHANNEL_ID}`). 43 | -------------------------------------------------------------------------------- /example_templates/youtube.txt: -------------------------------------------------------------------------------- 1 | {{/* This is the default template which is used by default to generate emails for youtube video feeds. 2 | 3 | These feeds can be obtained with the following URL format: 4 | https://www.youtube.com/feeds/videos.xml?channel_id=${CHANNEL_ID} 5 | CHANNEL_ID can be retrieved in the channel URL ( for instance: https://www.youtube.com/channel/${CHANNEL_ID} ) . 6 | 7 | As you might imagine it is a Golang text/template file. 8 | 9 | Several fields and functions are available: 10 | 11 | {{.FeedTitle}} - The human-readable title of the source feed. 12 | {{.Feed}} - The URL of the feed from which the item came. 13 | {{.From}} - The email address which sends the email. 14 | {{.Link}} - The link to the new entry. 15 | {{.Subject}} - The subject of the new entry. 16 | {{.To}} - The recipient of the email. 17 | 18 | There is also access to the {{.RSSFeed}} and {{.RSSItem}} available, in 19 | case you need access to other fields which are not exported expliclty. 20 | Using that approach you can access {{.RSSItem.GUID}}, for example. 21 | 22 | The following functions are also available: 23 | 24 | {{env "USER"}} -> Return the given environmental variable 25 | {{quoteprintable .Link}} -> Quote the specified field. 26 | {{encodeHeader .Subject}} -> Quote the specified field to be used in mail header. 27 | {{split "STRING:HERE" ":"}} -> Split a string into an array by deliminator 28 | 29 | This comment will be stripped from the generated email. 30 | The RSSItem represents the single item related to this e-mail template. 31 | The following items represent: 32 | {{.RSSItem.Author.Name}} - The author name 33 | {{.RSSItem.Published}} - The published date 34 | {{.RSSItem.Extensions}} - Represents the rest of the fields that were not parsed. They are saved using the Extensions type[1]. The fields managed by this rss2email are detailed here [2]. 35 | 36 | Youtube uses the Atom format, therefore the description is not parsed by gofeed and is stored in the extensions. The template discovers the field by looking for the description name. 37 | [1] https://pkg.go.dev/github.com/mmcdole/gofeed@v1.2.1/extensions#Extensions 38 | [2] https://pkg.go.dev/github.com/mmcdole/gofeed@v1.2.1#readme-default-mappings 39 | */ -}} 40 | Content-Type: multipart/mixed; boundary=21ee3da964c7bf70def62adb9ee1a061747003c026e363e47231258c48f1 41 | From: {{.From}} 42 | To: {{.To}} 43 | Subject: [rss2mail] [video] [{{encodeHeader .RSSItem.Author.Name}}] {{if .Tag}}{{encodeHeader .Tag}} {{end}}{{encodeHeader .Subject}} 44 | X-RSS-Link: {{.Link}} 45 | X-RSS-Feed: {{.Feed}} 46 | {{- if .Tag}} 47 | X-RSS-Tags: {{.Tag}} 48 | {{- end}} 49 | X-RSS-GUID: {{.RSSItem.GUID}} 50 | Content-Base: {{.Link}} 51 | Mime-Version: 1.0 52 | 53 | --21ee3da964c7bf70def62adb9ee1a061747003c026e363e47231258c48f1 54 | Content-Type: multipart/related; boundary=76a1282373c08a65dd49db1dea2c55111fda9a715c89720a844fabb7d497 55 | 56 | --76a1282373c08a65dd49db1dea2c55111fda9a715c89720a844fabb7d497 57 | Content-Type: multipart/alternative; boundary=4186c39e13b2140c88094b3933206336f2bb3948db7ecf064c7a7d7473f2 58 | 59 | --4186c39e13b2140c88094b3933206336f2bb3948db7ecf064c7a7d7473f2 60 | Content-Type: text/plain; charset=UTF-8 61 | Content-Transfer-Encoding: quoted-printable 62 | 63 | Subject: {{.Subject}} 64 | Author: {{.RSSItem.Author.Name}} 65 | Date published: {{.RSSItem.Published}} 66 | Link: {{quoteprintable .Link}} 67 | 68 | --4186c39e13b2140c88094b3933206336f2bb3948db7ecf064c7a7d7473f2 69 | Content-Type: text/html; charset=UTF-8 70 | Content-Transfer-Encoding: quoted-printable 71 | 72 | Subject: {{.Subject}}
73 | Author: {{.RSSItem.Author.Name}}
74 | Date published: {{.RSSItem.Published}}
75 | {{quoteprintable .Link}} 76 |

77 | Description:
78 | {{- range $key, $value := .RSSItem.Extensions }} 79 | {{- range $type, $extensions := $value }} 80 | {{- range $index, $extension := $extensions }} 81 | {{- if eq $extension.Name "description" }} 82 | {{quoteprintable $extension.Value }} 83 | {{- else }} 84 | {{- range $attrKey, $attrValue := $extension.Attrs }} 85 | {{- if eq $attrKey "description" }} 86 | {{quoteprintable $attrValue }} 87 | {{- end }} 88 | {{- end }} 89 | {{- range $childKey, $childExtensions := $extension.Children }} 90 | {{- range $childIndex, $childExtension := $childExtensions }} 91 | {{- if eq $childExtension.Name "description" }} 92 | {{quoteprintable $childExtension.Value }} 93 | {{- end }} 94 | {{- end }} 95 | {{- end }} 96 | {{- end }} 97 | {{- end }} 98 | {{- end }} 99 | {{- end }} 100 |

101 | --4186c39e13b2140c88094b3933206336f2bb3948db7ecf064c7a7d7473f2-- 102 | 103 | --76a1282373c08a65dd49db1dea2c55111fda9a715c89720a844fabb7d497-- 104 | --21ee3da964c7bf70def62adb9ee1a061747003c026e363e47231258c48f1-- 105 | -------------------------------------------------------------------------------- /export_cmd.go: -------------------------------------------------------------------------------- 1 | // 2 | // Export our feeds in a standard format 3 | // 4 | 5 | package main 6 | 7 | import ( 8 | "flag" 9 | "log/slog" 10 | "text/template" 11 | 12 | "github.com/skx/rss2email/configfile" 13 | "github.com/skx/subcommands" 14 | ) 15 | 16 | // Structure for our options and state. 17 | type exportCmd struct { 18 | 19 | // We embed the NoFlags option, because we accept no command-line flags. 20 | subcommands.NoFlags 21 | 22 | // Configuration file, used for testing 23 | config *configfile.ConfigFile 24 | } 25 | 26 | // Info is part of the subcommand-API 27 | func (e *exportCmd) Info() (string, string) { 28 | return "export", `Export the feed list as an OPML file. 29 | 30 | This command exports the list of configured feeds as an OPML file. 31 | 32 | To see details of the configuration file, including the location, 33 | please run: 34 | 35 | $ rss2email help config 36 | 37 | Example: 38 | 39 | $ rss2email export 40 | ` 41 | } 42 | 43 | // Arguments handles argument-flags we might have. 44 | // 45 | // In our case we use this as a hook to setup our configuration-file, 46 | // which allows testing. 47 | func (e *exportCmd) Arguments(flags *flag.FlagSet) { 48 | e.config = configfile.New() 49 | } 50 | 51 | // Execute is invoked if the user specifies `add` as the subcommand. 52 | func (e *exportCmd) Execute(args []string) int { 53 | 54 | // Individual feed URL 55 | type Feed struct { 56 | URL string 57 | } 58 | 59 | // Template Data 60 | type TemplateData struct { 61 | Entries []Feed 62 | } 63 | data := TemplateData{} 64 | 65 | // Now do the parsing 66 | entries, err := e.config.Parse() 67 | if err != nil { 68 | logger.Error("failed to parse configuration file", 69 | slog.String("configfile", e.config.Path()), 70 | slog.String("error", err.Error())) 71 | return 1 72 | } 73 | 74 | // Populate our template variables 75 | for _, entry := range entries { 76 | data.Entries = append(data.Entries, Feed{URL: entry.URL}) 77 | } 78 | 79 | // Template 80 | tmpl := ` 81 | 82 | 83 | Feed Export 84 | 85 | 86 | {{range .Entries}} 87 | {{end}} 88 | 89 | ` 90 | // Compile the template and write to STDOUT 91 | t := template.Must(template.New("tmpl").Parse(tmpl)) 92 | err = t.Execute(out, data) 93 | if err != nil { 94 | logger.Error("error rendering template", slog.String("error", err.Error())) 95 | return 1 96 | } 97 | 98 | return 0 99 | } 100 | -------------------------------------------------------------------------------- /export_cmd_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "os" 6 | "strings" 7 | "testing" 8 | 9 | "github.com/skx/rss2email/configfile" 10 | ) 11 | 12 | func TestExport(t *testing.T) { 13 | 14 | // Replace the STDIO handle 15 | bak := out 16 | out = &bytes.Buffer{} 17 | defer func() { out = bak }() 18 | 19 | // Create a simple configuration file 20 | content := `# Comment here 21 | https://example.org/ 22 | https://example.net/ 23 | - foo: bar 24 | ` 25 | data := []byte(content) 26 | tmpfile, err := os.CreateTemp("", "example") 27 | if err != nil { 28 | t.Fatalf("Error creating temporary file") 29 | } 30 | 31 | if _, err := tmpfile.Write(data); err != nil { 32 | t.Fatalf("Error writing to config file") 33 | } 34 | if err := tmpfile.Close(); err != nil { 35 | t.Fatalf("Error creating temporary file") 36 | } 37 | 38 | // Create an instance of the command, and setup the config file 39 | ex := exportCmd{} 40 | ex.Arguments(nil) 41 | config := configfile.NewWithPath(tmpfile.Name()) 42 | ex.config = config 43 | 44 | // Run the export 45 | ex.Execute([]string{}) 46 | 47 | // 48 | // Look for some lines in the output 49 | // 50 | expected := []string{ 51 | "Feed Export", 52 | "xmlUrl=\"https://example.org/\"", 53 | "", 54 | } 55 | 56 | // The text written to stdout 57 | output := out.(*bytes.Buffer).String() 58 | 59 | for _, txt := range expected { 60 | if !strings.Contains(output, txt) { 61 | t.Fatalf("Failed to find expected output") 62 | } 63 | } 64 | 65 | // Cleanup 66 | os.Remove(tmpfile.Name()) 67 | } 68 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/skx/rss2email 2 | 3 | go 1.21 4 | 5 | toolchain go1.22.2 6 | 7 | require ( 8 | github.com/PuerkitoBio/goquery v1.9.2 9 | github.com/k3a/html2text v1.2.1 10 | github.com/mmcdole/gofeed v1.3.0 11 | github.com/skx/subcommands v0.9.2 12 | go.etcd.io/bbolt v1.3.10 13 | ) 14 | 15 | require ( 16 | github.com/andybalholm/cascadia v1.3.2 // indirect 17 | github.com/json-iterator/go v1.1.12 // indirect 18 | github.com/mmcdole/goxpp v1.1.1 // indirect 19 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 20 | github.com/modern-go/reflect2 v1.0.2 // indirect 21 | golang.org/x/net v0.33.0 // indirect 22 | golang.org/x/sys v0.28.0 // indirect 23 | golang.org/x/text v0.21.0 // indirect 24 | ) 25 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4yPeE= 2 | github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk= 3 | github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= 4 | github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= 5 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 6 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 7 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 8 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 9 | github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= 10 | github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= 11 | github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= 12 | github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= 13 | github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= 14 | github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= 15 | github.com/k3a/html2text v1.2.1 h1:nvnKgBvBR/myqrwfLuiqecUtaK1lB9hGziIJKatNFVY= 16 | github.com/k3a/html2text v1.2.1/go.mod h1:ieEXykM67iT8lTvEWBh6fhpH4B23kB9OMKPdIBmgUqA= 17 | github.com/mmcdole/gofeed v1.3.0 h1:5yn+HeqlcvjMeAI4gu6T+crm7d0anY85+M+v6fIFNG4= 18 | github.com/mmcdole/gofeed v1.3.0/go.mod h1:9TGv2LcJhdXePDzxiuMnukhV2/zb6VtnZt1mS+SjkLE= 19 | github.com/mmcdole/goxpp v1.1.1 h1:RGIX+D6iQRIunGHrKqnA2+700XMCnNv0bAOOv5MUhx8= 20 | github.com/mmcdole/goxpp v1.1.1/go.mod h1:v+25+lT2ViuQ7mVxcncQ8ch1URund48oH+jhjiwEgS8= 21 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 22 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 23 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 24 | github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= 25 | github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= 26 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 27 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 28 | github.com/skx/subcommands v0.9.2 h1:wG035k1U7Fn6A0hwOMg1ly7085cl62gnzLY1j78GISo= 29 | github.com/skx/subcommands v0.9.2/go.mod h1:HpOZHVUXT5Rc/Q7UCiyj7h5u6BleDfFjt+vxy2igonA= 30 | github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= 31 | github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= 32 | github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= 33 | github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= 34 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 35 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 36 | github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= 37 | github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= 38 | github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= 39 | go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= 40 | go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= 41 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 42 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 43 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= 44 | golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= 45 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 46 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 47 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 48 | golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= 49 | golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= 50 | golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= 51 | golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= 52 | golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= 53 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 54 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 55 | golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 56 | golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= 57 | golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= 58 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 59 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 60 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 61 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 62 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 63 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 64 | golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 65 | golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= 66 | golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 67 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 68 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 69 | golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= 70 | golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= 71 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 72 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 73 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 74 | golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= 75 | golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= 76 | golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= 77 | golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= 78 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 79 | golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 80 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 81 | golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= 82 | golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= 83 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 84 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 85 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 86 | -------------------------------------------------------------------------------- /httpfetch/httpfetch.go: -------------------------------------------------------------------------------- 1 | // Package httpfetch makes a remote HTTP call to retrieve an URL, 2 | // and parse that into a series of feed-items 3 | // 4 | // It is abstracted into its own class to allow testing. 5 | package httpfetch 6 | 7 | import ( 8 | "crypto/tls" 9 | "encoding/json" 10 | "errors" 11 | "fmt" 12 | "io" 13 | "log/slog" 14 | "net/http" 15 | "os" 16 | "path/filepath" 17 | "strconv" 18 | "strings" 19 | "time" 20 | 21 | "github.com/mmcdole/gofeed" 22 | "github.com/skx/rss2email/configfile" 23 | statePath "github.com/skx/rss2email/state" 24 | ) 25 | 26 | var ( 27 | // cache contains the values we can use to be cache-friendly. 28 | cache map[string]CacheHelper 29 | 30 | // ErrUnchanged is returned by our HTTP-fetcher if the content was previously 31 | // fetched and has not changed since then. 32 | ErrUnchanged = errors.New("UNCHANGED") 33 | ) 34 | 35 | // CacheHelper is a struct used to store modification-data relating to the 36 | // URL we're fetching. 37 | // 38 | // We use this to make conditional HTTP-requests, rather than fetching the 39 | // feed from scratch each time. 40 | type CacheHelper struct { 41 | 42 | // Etag contains the Etag the server sent, if any. 43 | Etag string 44 | 45 | // LastModified contains the Last-Modified header the server sent, if any. 46 | LastModified string 47 | 48 | // Updated contains the timestamp of when the feed was last fetched (successfully). 49 | Updated time.Time 50 | } 51 | 52 | // init is called once at startup, and creates the cache-map we use to avoid 53 | // making too many HTTP-probes against remote URLs (i.e. feeds) 54 | func init() { 55 | cache = make(map[string]CacheHelper) 56 | } 57 | 58 | // HTTPFetch is our state-storing structure 59 | type HTTPFetch struct { 60 | 61 | // The URL we should fetch 62 | url string 63 | 64 | // Contents of the remote URL, used for testing 65 | content string 66 | 67 | // How many times we should attempt to retry a failed 68 | // fetch before giving up. 69 | maxRetries int 70 | 71 | // insecure will cause invalid SSL certificate options to be ignored 72 | insecure bool 73 | 74 | // Between retries we should delay to avoid overwhelming 75 | // the remote server. This specifies how many times we should 76 | // do that. 77 | retryDelay time.Duration 78 | 79 | // frequency controls the poll frequency. If this is set to 1hr then 80 | // we don't fetch the feed until 1 hour after the last fetch, even if 81 | // we were executed as a daemon with a SLEEP setting of 5 (minutes). 82 | frequency time.Duration 83 | 84 | // The User-Agent header to send when making our HTTP fetch 85 | userAgent string 86 | 87 | // logger contains the logging handle to use, if any 88 | logger *slog.Logger 89 | } 90 | 91 | // New creates a new object which will fetch our content. 92 | func New(entry configfile.Feed, log *slog.Logger, version string) *HTTPFetch { 93 | 94 | // Create object with defaults 95 | state := &HTTPFetch{url: entry.URL, 96 | maxRetries: 3, 97 | retryDelay: 5 * time.Second, 98 | userAgent: fmt.Sprintf("rss2email %s (https://github.com/skx/rss2email)", version), 99 | } 100 | 101 | // Path to the cache file, which we read from-disk if we can. 102 | fileName := filepath.Join(statePath.Directory(), "httpcache.json") 103 | data, err := os.ReadFile(fileName) 104 | 105 | // If we got an error, and it wasn't a not-found log it 106 | if err != nil && !os.IsNotExist(err) { 107 | log.Debug("failed to read cache-values", 108 | slog.String("path", fileName), 109 | slog.String("error", err.Error())) 110 | } else { 111 | // We can't even log it usefully. 112 | err = json.Unmarshal(data, &cache) 113 | if err != nil { 114 | log.Debug("failed to unmarshall cache-values", 115 | slog.String("path", fileName), 116 | slog.String("error", err.Error())) 117 | } 118 | } 119 | 120 | // Get the user's sleep period - if overridden this will become the 121 | // default frequency for each feed item. 122 | sleep := os.Getenv("SLEEP") 123 | if sleep == "" { 124 | state.frequency = 15 * time.Minute 125 | } else { 126 | v, err := strconv.Atoi(sleep) 127 | if err == nil { 128 | state.frequency = time.Duration(v) * time.Minute 129 | } 130 | } 131 | 132 | // Are any of our options overridden? 133 | for _, opt := range entry.Options { 134 | 135 | // Max-retry count. 136 | if opt.Name == "retry" { 137 | 138 | num, err := strconv.Atoi(opt.Value) 139 | if err == nil { 140 | state.maxRetries = num 141 | } 142 | } 143 | 144 | // Disable fatal TLS errors. Horrid 145 | if opt.Name == "insecure" { 146 | 147 | // downcase the value 148 | val := strings.ToLower(opt.Value) 149 | 150 | // if it is enabled then set the flag 151 | if val == "yes" || val == "true" { 152 | state.insecure = true 153 | } 154 | } 155 | 156 | // Sleep-delay between failed fetch-attempts. 157 | if opt.Name == "delay" { 158 | 159 | num, err := strconv.Atoi(opt.Value) 160 | if err == nil { 161 | state.retryDelay = time.Duration(num) * time.Second 162 | } 163 | } 164 | 165 | // User-Agent 166 | if opt.Name == "user-agent" { 167 | state.userAgent = opt.Value 168 | } 169 | 170 | // Polling frequency 171 | if opt.Name == "frequency" { 172 | num, err := strconv.Atoi(opt.Value) 173 | if err == nil { 174 | state.frequency = time.Duration(num) * time.Minute 175 | } 176 | } 177 | } 178 | 179 | // Create a local logger with some dedicated information 180 | state.logger = log.With( 181 | slog.Group("httpfetch", 182 | slog.String("link", entry.URL), 183 | slog.String("user-agent", state.userAgent), 184 | slog.Bool("insecure", state.insecure), 185 | slog.Int("retry-max", state.maxRetries), 186 | slog.Duration("retry-delay", state.retryDelay), 187 | slog.Duration("frequency", state.frequency))) 188 | 189 | return state 190 | } 191 | 192 | // Fetch performs the HTTP-fetch, and returns the feed-contents. 193 | // 194 | // If our internal `content` field is non-empty it will be used in preference 195 | // to making a remote request, which is useful for testing. 196 | func (h *HTTPFetch) Fetch() (*gofeed.Feed, error) { 197 | 198 | var feed *gofeed.Feed 199 | var err error 200 | 201 | // Download contents, if not already present. 202 | for i := 0; h.content == "" && i < h.maxRetries; i++ { 203 | 204 | // Log the fetch attempt 205 | h.logger.Debug("fetching URL", 206 | slog.Int("attempt", i+1)) 207 | 208 | // fetch the contents 209 | err = h.fetch() 210 | 211 | // no error? that means we're good and we've retrieved 212 | // the content. 213 | if err == nil { 214 | break 215 | } 216 | 217 | // The remote content hasn't changed? 218 | if err == ErrUnchanged { 219 | return nil, ErrUnchanged 220 | } 221 | 222 | // if we got here we have to retry, but we should 223 | // show the error too. 224 | h.logger.Debug("fetching URL failed", 225 | slog.String("error", err.Error())) 226 | 227 | time.Sleep(h.retryDelay) 228 | 229 | } 230 | 231 | // Failed, after all the retries? 232 | if err != nil { 233 | return feed, err 234 | } 235 | 236 | // Parse it 237 | fp := gofeed.NewParser() 238 | feed, err2 := fp.ParseString(h.content) 239 | if err2 != nil { 240 | 241 | h.logger.Warn("failed to parse content", 242 | slog.String("error", err2.Error())) 243 | 244 | return nil, fmt.Errorf("error parsing %s contents: %s", h.url, err2.Error()) 245 | } 246 | 247 | return feed, nil 248 | } 249 | 250 | // fetch fetches the text from the remote URL. 251 | func (h *HTTPFetch) fetch() error { 252 | 253 | // Do we have a cache-entry? 254 | prevCache, okCache := cache[h.url] 255 | if okCache { 256 | h.logger.Debug("we have cached headers saved from a previous request", 257 | slog.String("etag", prevCache.Etag), 258 | slog.String("last-modified", prevCache.LastModified)) 259 | } 260 | 261 | // Create a HTTP-client 262 | client := &http.Client{} 263 | 264 | // Setup a transport which disables TLS-checks 265 | tr := &http.Transport{ 266 | TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, 267 | } 268 | 269 | // If we're ignoring the TLS then use a non-validating transport. 270 | if h.insecure { 271 | client.Transport = tr 272 | } 273 | 274 | // We only support making HTTP GET requests. 275 | req, err := http.NewRequest("GET", h.url, nil) 276 | if err != nil { 277 | return err 278 | } 279 | 280 | // If we've previously fetched this URL set the appropriate 281 | // cache-related headers in our new request. 282 | if okCache { 283 | 284 | // If there is a frequency for this feed AND the time has not yet 285 | // been reached then we terminate early. 286 | if time.Since(prevCache.Updated) < h.frequency { 287 | h.logger.Debug("avoiding this fetch, the feed was retrieved already within the frequency limit", 288 | slog.Time("last", prevCache.Updated), 289 | slog.Duration("duration", h.frequency)) 290 | return ErrUnchanged 291 | } 292 | 293 | // Otherwise set the cache-related headers. 294 | 295 | if prevCache.Etag != "" { 296 | h.logger.Debug("setting HTTP-header on outgoing request", 297 | slog.String("url", h.url), 298 | slog.String("If-None-Match", prevCache.Etag)) 299 | 300 | req.Header.Set("If-None-Match", prevCache.Etag) 301 | } 302 | if prevCache.LastModified != "" { 303 | h.logger.Debug("setting HTTP-header on outgoing request", 304 | slog.String("url", h.url), 305 | slog.String("If-Modified-Since", prevCache.LastModified)) 306 | req.Header.Set("If-Modified-Since", prevCache.LastModified) 307 | } 308 | 309 | } 310 | 311 | // Populate the HTTP User-Agent header - some sites (e.g. reddit) fail without this. 312 | req.Header.Set("User-Agent", h.userAgent) 313 | 314 | // Make the actual HTTP request. 315 | resp, err := client.Do(req) 316 | if err != nil { 317 | return err 318 | } 319 | defer resp.Body.Close() 320 | 321 | // Read the response headers and save any cache-like things 322 | // we can use to avoid excessive load in the future. 323 | x := CacheHelper{ 324 | Etag: resp.Header.Get("ETag"), 325 | LastModified: resp.Header.Get("Last-Modified"), 326 | Updated: time.Now(), 327 | } 328 | cache[h.url] = x 329 | 330 | // Save cache. 331 | encoded, errEncoding := json.Marshal(cache) 332 | if errEncoding == nil { 333 | fileName := filepath.Join(statePath.Directory(), "httpcache.json") 334 | errWrite := os.WriteFile(fileName, encoded, 0644) 335 | if errWrite != nil { 336 | h.logger.Debug("failed to write cache to json", 337 | slog.String("path", fileName), 338 | slog.String("error", errWrite.Error())) 339 | } 340 | } 341 | 342 | // 343 | // Did the remote page not change? 344 | // 345 | status := resp.StatusCode 346 | if status >= 300 && status < 400 { 347 | h.logger.Debug("response from request was unchanged", 348 | slog.String("status", resp.Status), 349 | slog.Int("code", resp.StatusCode)) 350 | return ErrUnchanged 351 | } 352 | 353 | // Otherwise we save the result away and 354 | // return any error/not as a result of reading 355 | // the body. 356 | data, err2 := io.ReadAll(resp.Body) 357 | h.content = string(data) 358 | 359 | h.logger.Debug("response from request", 360 | slog.String("url", h.url), 361 | slog.String("status", resp.Status), 362 | slog.Int("code", resp.StatusCode), 363 | slog.Int("size", len(h.content))) 364 | 365 | return err2 366 | } 367 | -------------------------------------------------------------------------------- /httpfetch/httpfetch_test.go: -------------------------------------------------------------------------------- 1 | package httpfetch 2 | 3 | import ( 4 | "fmt" 5 | "log/slog" 6 | "net/http" 7 | "net/http/httptest" 8 | "os" 9 | "strings" 10 | "testing" 11 | "time" 12 | 13 | "github.com/skx/rss2email/configfile" 14 | "github.com/skx/rss2email/withstate" 15 | ) 16 | 17 | var ( 18 | // logger contains a shared logging handle, the code we're testing assumes it exists. 19 | logger *slog.Logger 20 | ) 21 | 22 | // init runs at test-time. 23 | func init() { 24 | 25 | // setup logging-level 26 | lvl := &slog.LevelVar{} 27 | lvl.Set(slog.LevelWarn) 28 | 29 | // create a handler 30 | opts := &slog.HandlerOptions{Level: lvl} 31 | handler := slog.NewTextHandler(os.Stderr, opts) 32 | 33 | // ensure the global-variable is set. 34 | logger = slog.New(handler) 35 | } 36 | 37 | // TestNonFeed confirms we can cope with a remote URL which is not a feed. 38 | func TestNonFeed(t *testing.T) { 39 | 40 | // Not a feed. 41 | x := New(configfile.Feed{URL: "http://example.com/"}, logger, "v1.2.3") 42 | x.content = "this is not an XML file, so not a feed" 43 | 44 | // Parse it, which should fail. 45 | _, err := x.Fetch() 46 | if err == nil { 47 | t.Fatalf("We expected error, but got none!") 48 | } 49 | 50 | // And confirm it fails in the correct way. 51 | if !strings.Contains(err.Error(), "Failed to detect feed type") { 52 | t.Fatalf("got an error, but not what we expected; %s", err.Error()) 53 | } 54 | 55 | if !strings.Contains(x.userAgent, "v1.2.3") { 56 | t.Fatalf("our default agent doesn't contain our version string: '%s'", x.userAgent) 57 | } 58 | } 59 | 60 | // TestOneEntry confirms a feed contains a single entry 61 | func TestOneEntry(t *testing.T) { 62 | 63 | // The contents of our feed. 64 | x := New(configfile.Feed{URL: "https://blog.steve.fi/index.rss"}, logger, "unversioned") 65 | x.content = ` 66 | 73 | 74 | Steve Kemp's Blog 75 | https://blog.steve.fi/ 76 | Debian and Free Software 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | Brexit has come 86 | https://blog.steve.fi/brexit_has_come.html 87 | https://blog.steve.fi/brexit_has_come.html 88 | Hello, World 89 | 2020-05-22T09:00:00Z 90 | 91 | 92 | ` 93 | 94 | // Parse it which should not fail. 95 | out, err := x.Fetch() 96 | if err != nil { 97 | t.Fatalf("We didn't expect an error, but found %s", err.Error()) 98 | } 99 | 100 | // Confirm there is a single entry. 101 | if len(out.Items) != 1 { 102 | t.Fatalf("Expected one entry, but got %d", len(out.Items)) 103 | } 104 | } 105 | 106 | // TestRewrite ensures that a broken file is rewriting 107 | func TestRewrite(t *testing.T) { 108 | 109 | // The contents of our feed. 110 | x := New(configfile.Feed{URL: "https://blog.steve.fi/index.rss"}, logger, "unversioned") 111 | x.content = ` 112 | 119 | 120 | Steve Kemp's Blog 121 | https://blog.steve.fi/ 122 | Debian and Free Software 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | Brexit has come 132 | https://blog.steve.fi/brexit_has_come.html 133 | https://blog.steve.fi/brexit_has_come.html 134 | <a href="/foo">Foo</a> 135 | 2020-05-22T09:00:00Z 136 | 137 | 138 | ` 139 | 140 | // Parse it which should not fail. 141 | out, err := x.Fetch() 142 | if err != nil { 143 | t.Fatalf("We didn't expect an error, but found %s", err.Error()) 144 | } 145 | 146 | // Confirm there is a single entry. 147 | if len(out.Items) != 1 { 148 | t.Fatalf("Expected one entry, but got %d", len(out.Items)) 149 | } 150 | 151 | // Get the parsed-content 152 | item := withstate.FeedItem{Item: out.Items[0]} 153 | content, err := item.HTMLContent() 154 | if err != nil { 155 | t.Fatalf("unexpected error on item content: %v", err) 156 | } 157 | 158 | // Confirm that contains : href="https://blog.steve.fi/foo", 159 | // not: href="/foo" 160 | if strings.Contains(content, "\"/foo") { 161 | t.Fatalf("Failed to expand URLS: %s", content) 162 | } 163 | } 164 | 165 | func TestDelay(t *testing.T) { 166 | 167 | // Valid number 168 | n := New(configfile.Feed{URL: "https://blog.steve.fi/index.rss", 169 | Options: []configfile.Option{ 170 | {Name: "delay", Value: "15"}, 171 | }}, logger, "unversioned") 172 | 173 | if n.retryDelay != 15*time.Second { 174 | t.Errorf("failed to parse delay value") 175 | } 176 | 177 | // Invalid number - should have the default value 178 | i := New(configfile.Feed{URL: "https://blog.steve.fi/index.rss", 179 | Options: []configfile.Option{ 180 | {Name: "delay", Value: "steve"}, 181 | }}, logger, "unversioned") 182 | 183 | if i.retryDelay != 5*time.Second { 184 | t.Errorf("bogus value changed our delay-value") 185 | } 186 | } 187 | 188 | func TestRetry(t *testing.T) { 189 | 190 | // Valid number 191 | n := New(configfile.Feed{URL: "https://blog.steve.fi/index.rss", 192 | Options: []configfile.Option{ 193 | {Name: "retry", Value: "33"}, 194 | {Name: "moi", Value: "3"}, 195 | }}, logger, "unversioned") 196 | 197 | if n.maxRetries != 33 { 198 | t.Errorf("failed to parse retry value") 199 | } 200 | 201 | // Invalid number 202 | i := New(configfile.Feed{URL: "https://blog.steve.fi/index.rss", 203 | Options: []configfile.Option{ 204 | {Name: "retry", Value: "steve"}, 205 | }}, logger, "unversioned") 206 | 207 | if i.maxRetries != 3 { 208 | t.Errorf("bogus value changed our default") 209 | } 210 | } 211 | 212 | // Make a HTTP-request against a local entry 213 | func TestHTTPFetch(t *testing.T) { 214 | 215 | // Setup a stub server 216 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 217 | fmt.Fprintln(w, "Hello, client") 218 | })) 219 | defer ts.Close() 220 | 221 | // Create a config-entry which points to the fake HTTP-server 222 | conf := configfile.Feed{URL: ts.URL} 223 | 224 | // Create a fetcher 225 | obj := New(conf, logger, "unversioned") 226 | 227 | // Now make the HTTP-fetch 228 | _, err := obj.Fetch() 229 | 230 | if err == nil { 231 | t.Fatalf("expected an error from the fetch") 232 | } 233 | if !strings.Contains(err.Error(), "Failed to detect feed type") { 234 | t.Fatalf("got an error, but the wrong kind") 235 | } 236 | } 237 | 238 | // Make a HTTP-request against a local entry 239 | func TestHTTPFetchValid(t *testing.T) { 240 | 241 | feed := ` 242 | 249 | 250 | Steve Kemp's Blog 251 | https://blog.steve.fi/ 252 | Debian and Free Software 253 | 254 | 255 | 256 | 257 | 258 | 259 | 260 | 261 | Brexit has come 262 | https://blog.steve.fi/brexit_has_come.html 263 | https://blog.steve.fi/brexit_has_come.html 264 | Hello, World 265 | 2020-05-22T09:00:00Z 266 | 267 | 268 | ` 269 | // User-agent setup 270 | agent := "foo:bar:baz" 271 | 272 | // Setup a stub server 273 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 274 | fmt.Fprintln(w, feed) 275 | })) 276 | defer ts.Close() 277 | 278 | // Create a config-entry which points to the fake HTTP-server 279 | conf := configfile.Feed{URL: ts.URL, 280 | Options: []configfile.Option{ 281 | {Name: "user-agent", Value: agent}, 282 | }, 283 | } 284 | 285 | // Create a fetcher 286 | obj := New(conf, logger, "unversioned") 287 | 288 | if obj.userAgent != agent { 289 | t.Fatalf("failed to setup user-agent") 290 | } 291 | 292 | // Now make the HTTP-fetch 293 | res, err := obj.Fetch() 294 | 295 | if err != nil { 296 | t.Fatalf("unexpected error fetching feed") 297 | } 298 | 299 | if len(res.Items) != 1 { 300 | t.Fatalf("wrong feed count") 301 | } 302 | } 303 | -------------------------------------------------------------------------------- /import_cmd.go: -------------------------------------------------------------------------------- 1 | // 2 | // Import an OPML feedlist. 3 | // 4 | 5 | package main 6 | 7 | import ( 8 | "encoding/xml" 9 | "flag" 10 | "log/slog" 11 | "os" 12 | 13 | "github.com/skx/rss2email/configfile" 14 | "github.com/skx/subcommands" 15 | ) 16 | 17 | type opml struct { 18 | XMLName xml.Name `xml:"opml"` 19 | Version string `xml:"version,attr"` 20 | OpmlTitle string `xml:"head>title"` 21 | Outlines []outline `xml:"body>outline"` 22 | } 23 | 24 | type outline struct { 25 | Text string `xml:"text,attr"` 26 | Title string `xml:"title,attr"` 27 | Type string `xml:"type,attr"` 28 | XMLURL string `xml:"xmlUrl,attr"` 29 | HTMLURL string `xml:"htmlUrl,attr"` 30 | Favicon string `xml:"rssfr-favicon,attr"` 31 | } 32 | 33 | // Structure for our options and state. 34 | type importCmd struct { 35 | 36 | // We embed the NoFlags option, because we accept no command-line flags. 37 | subcommands.NoFlags 38 | 39 | // Configuration file, used for testing 40 | config *configfile.ConfigFile 41 | } 42 | 43 | // Info is part of the subcommand-API 44 | func (i *importCmd) Info() (string, string) { 45 | return "import", `Import a list of feeds via an OPML file. 46 | 47 | This command imports a series of feeds from the specified OPML 48 | file into the configuration file this application uses. 49 | 50 | To see details of the configuration file, including the location, 51 | please run: 52 | 53 | $ rss2email help config 54 | 55 | Example: 56 | 57 | $ rss2email import file1.opml file2.opml .. fileN.opml 58 | ` 59 | } 60 | 61 | // Arguments handles argument-flags we might have. 62 | // 63 | // In our case we use this as a hook to setup our configuration-file, 64 | // which allows testing. 65 | func (i *importCmd) Arguments(flags *flag.FlagSet) { 66 | i.config = configfile.New() 67 | } 68 | 69 | // Execute is invoked if the user specifies `import` as the subcommand. 70 | func (i *importCmd) Execute(args []string) int { 71 | 72 | _, err := i.config.Parse() 73 | if err != nil { 74 | logger.Error("failed to parse configuration file", 75 | slog.String("configfile", i.config.Path()), 76 | 77 | slog.String("error", err.Error())) 78 | return 1 79 | } 80 | 81 | // For each file on the command-line 82 | for _, file := range args { 83 | 84 | // Read content 85 | var data []byte 86 | data, err = os.ReadFile(file) 87 | if err != nil { 88 | logger.Warn("failed to read file", slog.String("file", file), slog.String("error", err.Error())) 89 | continue 90 | } 91 | 92 | // Parse 93 | o := opml{} 94 | err = xml.Unmarshal(data, &o) 95 | if err != nil { 96 | logger.Warn("failed to parse XML file", slog.String("file", file), slog.String("error", err.Error())) 97 | continue 98 | } 99 | 100 | for _, outline := range o.Outlines { 101 | 102 | if outline.XMLURL != "" { 103 | logger.Debug("Adding entry from file", slog.String("file", file), slog.String("url", outline.XMLURL)) 104 | i.config.Add(outline.XMLURL) 105 | } 106 | } 107 | 108 | } 109 | 110 | // Did we make a change? Then add them. 111 | err = i.config.Save() 112 | if err != nil { 113 | logger.Error("failed to save the updated feed list", slog.String("error", err.Error())) 114 | return 1 115 | } 116 | 117 | // All done. 118 | return 0 119 | } 120 | -------------------------------------------------------------------------------- /import_cmd_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/skx/rss2email/configfile" 8 | ) 9 | 10 | func TestImport(t *testing.T) { 11 | 12 | // Create a simple configuration file 13 | content := `# Comment here 14 | https://example.org/ 15 | https://example.net/ 16 | - foo: bar 17 | ` 18 | data := []byte(content) 19 | tmpfile, err := os.CreateTemp("", "example") 20 | if err != nil { 21 | t.Fatalf("Error creating temporary file") 22 | } 23 | 24 | if _, err = tmpfile.Write(data); err != nil { 25 | t.Fatalf("Error writing to config file") 26 | } 27 | if err = tmpfile.Close(); err != nil { 28 | t.Fatalf("Error creating temporary file") 29 | } 30 | 31 | // Create an OPML file to use as input 32 | opml, err := os.CreateTemp("", "opml") 33 | if err != nil { 34 | t.Fatalf("Error creating temporary file for OMPL input") 35 | } 36 | d1 := []byte(` 37 | 38 | 39 | 40 | Feed Value 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | `) 53 | err = os.WriteFile(opml.Name(), d1, 0644) 54 | if err != nil { 55 | t.Fatalf("failed to write OPML file") 56 | } 57 | 58 | // Create an instance of the command, and setup the config file 59 | im := importCmd{} 60 | im.Arguments(nil) 61 | config := configfile.NewWithPath(tmpfile.Name()) 62 | im.config = config 63 | 64 | // Run the import 65 | im.Execute([]string{opml.Name()}) 66 | 67 | // Look for the new entries in the feed. 68 | entries, err2 := config.Parse() 69 | if err2 != nil { 70 | t.Errorf("error parsing the (updated) config file") 71 | } 72 | if len(entries) != 9 { 73 | t.Fatalf("found %d entries", len(entries)) 74 | } 75 | 76 | // Cleanup 77 | os.Remove(tmpfile.Name()) 78 | os.Remove(opml.Name()) 79 | } 80 | -------------------------------------------------------------------------------- /list_cmd.go: -------------------------------------------------------------------------------- 1 | // 2 | // List our configured-feeds. 3 | // 4 | 5 | package main 6 | 7 | import ( 8 | "flag" 9 | "fmt" 10 | "log/slog" 11 | "time" 12 | 13 | "github.com/skx/rss2email/configfile" 14 | "github.com/skx/rss2email/httpfetch" 15 | ) 16 | 17 | var ( 18 | maxInt = int(^uint(0) >> 1) 19 | ) 20 | 21 | // Structure for our options and state. 22 | type listCmd struct { 23 | 24 | // Configuration file, used for testing 25 | config *configfile.ConfigFile 26 | 27 | // verbose controls whether our feed-list contains information 28 | // about feed entries and their ages 29 | verbose bool 30 | } 31 | 32 | // Arguments handles argument-flags we might have. 33 | // 34 | // In our case we use this as a hook to setup our configuration-file, 35 | // which allows testing. 36 | func (l *listCmd) Arguments(flags *flag.FlagSet) { 37 | 38 | // Setup configuration file 39 | l.config = configfile.New() 40 | 41 | // Are we listing verbosely? 42 | flags.BoolVar(&l.verbose, "verbose", false, "Show extra information about each feed (slow)?") 43 | } 44 | 45 | // Info is part of the subcommand-API 46 | func (l *listCmd) Info() (string, string) { 47 | return "list", `Output the list of feeds which are being polled. 48 | 49 | This subcommand lists the feeds which are specified in the 50 | configuration file. 51 | 52 | To see details of the configuration file, including the location, 53 | please run: 54 | 55 | $ rss2email help config 56 | 57 | 58 | You can add '-verbose' to see details about the feed contents, but note 59 | that this will require downloading the contents of each feed and will 60 | thus be slow - a simpler way of showing history would be to run: 61 | 62 | $ rss2email seen 63 | 64 | Example: 65 | 66 | $ rss2email list 67 | ` 68 | } 69 | 70 | func (l *listCmd) showFeedDetails(entry configfile.Feed) { 71 | 72 | // Fetch the details 73 | helper := httpfetch.New(entry, logger, version) 74 | feed, err := helper.Fetch() 75 | if err != nil { 76 | fmt.Fprintf(out, "# %s\n%s\n", err.Error(), entry.URL) 77 | return 78 | } 79 | 80 | // Handle single vs. plural entries 81 | entriesString := "entries" 82 | if len(feed.Items) == 1 { 83 | entriesString = "entry" 84 | } 85 | 86 | // get the age-range of the feed-entries 87 | oldest := -1 88 | newest := maxInt 89 | for _, item := range feed.Items { 90 | if item.PublishedParsed == nil { 91 | break 92 | } 93 | 94 | age := int(time.Since(*item.PublishedParsed) / (24 * time.Hour)) 95 | if age > oldest { 96 | oldest = age 97 | } 98 | 99 | if age < newest { 100 | newest = age 101 | } 102 | } 103 | 104 | // Now show the details, which is a bit messy. 105 | fmt.Fprintf(out, "# %d %s, aged %d-%d days\n", len(feed.Items), entriesString, newest, oldest) 106 | fmt.Fprintf(out, "%s\n", entry.URL) 107 | } 108 | 109 | // Entry-point. 110 | func (l *listCmd) Execute(args []string) int { 111 | 112 | // Now do the parsing 113 | entries, err := l.config.Parse() 114 | if err != nil { 115 | logger.Error("failed to parse configuration file", 116 | slog.String("configfile", l.config.Path()), 117 | slog.String("error", err.Error())) 118 | return 1 119 | } 120 | 121 | // Show the feeds 122 | for _, entry := range entries { 123 | 124 | if l.verbose { 125 | l.showFeedDetails(entry) 126 | } else { 127 | fmt.Fprintf(out, "%s\n", entry.URL) 128 | } 129 | } 130 | 131 | return 0 132 | } 133 | -------------------------------------------------------------------------------- /list_cmd_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "flag" 6 | "os" 7 | "strings" 8 | "testing" 9 | 10 | "github.com/skx/rss2email/configfile" 11 | ) 12 | 13 | // TestList confirms that listing the feed-list works as expected 14 | func TestList(t *testing.T) { 15 | 16 | bak := out 17 | out = &bytes.Buffer{} 18 | defer func() { out = bak }() 19 | 20 | // Create an instance of the command, and setup a default 21 | // configuration file 22 | 23 | content := `# Comment here 24 | https://example.org/ 25 | https://example.net/index.rss 26 | - foo: bar 27 | ` 28 | data := []byte(content) 29 | tmpfile, err := os.CreateTemp("", "example") 30 | if err != nil { 31 | t.Fatalf("Error creating temporary file") 32 | } 33 | 34 | if _, err := tmpfile.Write(data); err != nil { 35 | t.Fatalf("Error writing to config file") 36 | } 37 | if err := tmpfile.Close(); err != nil { 38 | t.Fatalf("Error creating temporary file") 39 | } 40 | 41 | list := listCmd{} 42 | flags := flag.NewFlagSet("test", flag.ContinueOnError) 43 | list.Arguments(flags) 44 | config := configfile.NewWithPath(tmpfile.Name()) 45 | list.config = config 46 | 47 | ret := list.Execute([]string{}) 48 | if ret != 0 { 49 | t.Fatalf("unexpected error running list") 50 | } 51 | 52 | output := out.(*bytes.Buffer).String() 53 | 54 | // We should have two URLs 55 | if !strings.Contains(output, "https://example.org/") { 56 | t.Errorf("List didn't contain expected output") 57 | } 58 | if !strings.Contains(output, "https://example.net/index.rss") { 59 | t.Errorf("List didn't contain expected output") 60 | } 61 | 62 | // We should not have comments, or parameters 63 | if strings.Contains(output, "foo") { 64 | t.Errorf("We found a parameter we didn't expect") 65 | } 66 | if strings.Contains(output, "#") { 67 | t.Errorf("We found a comment we didn't expect") 68 | } 69 | 70 | os.Remove(tmpfile.Name()) 71 | } 72 | -------------------------------------------------------------------------------- /list_default_template_cmd.go: -------------------------------------------------------------------------------- 1 | // 2 | // List our default email-template 3 | // 4 | 5 | package main 6 | 7 | import ( 8 | "fmt" 9 | 10 | "github.com/skx/rss2email/template" 11 | "github.com/skx/subcommands" 12 | ) 13 | 14 | // listDefaultTemplateCmd holds our state. 15 | type listDefaultTemplateCmd struct { 16 | 17 | // We embed the NoFlags option, because we accept no command-line flags. 18 | subcommands.NoFlags 19 | } 20 | 21 | // Info is part of the subcommand-API 22 | func (l *listDefaultTemplateCmd) Info() (string, string) { 23 | return "list-default-template", `Display the default email-template. 24 | 25 | An embedded template is used to format the emails which are sent by this 26 | application, when new feed items are discovered. If you wish to change 27 | the way the emails are formed, or formatted, you can replace this template 28 | with a local copy. 29 | 30 | To replace the template which is used simple create a new file located at 31 | '~/.rss2email/email.tmpl', with your content. 32 | 33 | This sub-command can be used to give you a starting point for your edits: 34 | 35 | $ rss2email list-default-template > ~/.rss2email/email.tmpl 36 | 37 | 38 | Example: 39 | 40 | $ rss2email list-default-template 41 | ` 42 | } 43 | 44 | // 45 | // Entry-point. 46 | // 47 | func (l *listDefaultTemplateCmd) Execute(args []string) int { 48 | 49 | // Load the default template from the embedded resource. 50 | content := template.EmailTemplate() 51 | fmt.Fprintf(out, "%s\n", string(content)) 52 | return 0 53 | } 54 | -------------------------------------------------------------------------------- /list_default_template_cmd_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "flag" 6 | "strings" 7 | "testing" 8 | ) 9 | 10 | func TestDefaultTemplate(t *testing.T) { 11 | 12 | bak := out 13 | out = &bytes.Buffer{} 14 | defer func() { out = bak }() 15 | 16 | s := listDefaultTemplateCmd{} 17 | 18 | // 19 | // Call the Arguments function for coverage. 20 | // 21 | flags := flag.NewFlagSet("test", flag.ContinueOnError) 22 | s.Arguments(flags) 23 | 24 | // 25 | // Call the handler. 26 | // 27 | s.Execute([]string{}) 28 | 29 | // 30 | // Look for some lines in the output 31 | // 32 | expected := []string{ 33 | "X-RSS-Link: {{.Link}}", 34 | "Content-Type: multipart/alternative;", 35 | "the default template which is used by default to generate emails", 36 | } 37 | 38 | // The text written to stdout 39 | output := out.(*bytes.Buffer).String() 40 | 41 | for _, txt := range expected { 42 | if !strings.Contains(output, txt) { 43 | t.Fatalf("Failed to find expected output") 44 | } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | // 2 | // Entry-point for our application. 3 | // 4 | 5 | package main 6 | 7 | import ( 8 | "fmt" 9 | "io" 10 | "log/slog" 11 | "os" 12 | "strings" 13 | 14 | "github.com/skx/subcommands" 15 | ) 16 | 17 | var ( 18 | // logger contains a shared logging handle, used by our sub-commands. 19 | logger *slog.Logger 20 | 21 | // loggerLevel allows changing the log-level at runtime 22 | loggerLevel *slog.LevelVar 23 | ) 24 | 25 | // Recovery is good 26 | func recoverPanic() { 27 | if r := recover(); r != nil { 28 | logger.Error("recovered from a panic", slog.String("error", fmt.Sprintf("%s", r))) 29 | } 30 | } 31 | 32 | // Register the subcommands, and run the one the user chose. 33 | func main() { 34 | 35 | // 36 | // Setup our default logging level, which will show 37 | // both warnings and errors. 38 | // 39 | loggerLevel = &slog.LevelVar{} 40 | loggerLevel.Set(slog.LevelWarn) 41 | 42 | // 43 | // If the user wants a different level they can choose it. 44 | // 45 | level := os.Getenv("LOG_LEVEL") 46 | 47 | // 48 | // Legacy/Compatibility 49 | // 50 | if os.Getenv("LOG_ALL") != "" { 51 | level = "DEBUG" 52 | } 53 | 54 | // Simplify things by only caring about upper-case 55 | level = strings.ToUpper(level) 56 | 57 | switch level { 58 | case "DEBUG": 59 | loggerLevel.Set(slog.LevelDebug) 60 | case "WARN": 61 | loggerLevel.Set(slog.LevelWarn) 62 | case "ERROR": 63 | loggerLevel.Set(slog.LevelError) 64 | case "": 65 | // NOP 66 | default: 67 | fmt.Printf("Unknown logging-level '%s'\n", level) 68 | return 69 | } 70 | 71 | // Those handler options 72 | opts := &slog.HandlerOptions{ 73 | Level: loggerLevel, 74 | AddSource: true, 75 | ReplaceAttr: func(groups []string, a slog.Attr) slog.Attr { 76 | if a.Key == slog.SourceKey { 77 | s := a.Value.Any().(*slog.Source) 78 | 79 | // Assume we have a source-path containing "rss2email" 80 | // if we do strip everything before that out. 81 | start := strings.Index(s.File, "rss2email") 82 | if start > 0 { 83 | s.File = s.File[start:] 84 | } 85 | 86 | // Assume we have a function containing "rss2email" 87 | // if we do strip everything before that out. 88 | start = strings.Index(s.Function, "rss2email") 89 | if start > 0 { 90 | s.Function = s.Function[start:] 91 | } 92 | 93 | } 94 | return a 95 | }, 96 | } 97 | 98 | // 99 | // Create a default writer, which the logger will use. 100 | // This will mostly go to STDERR, however it might also 101 | // be duplicated to a file. 102 | // 103 | multi := io.MultiWriter(os.Stderr) 104 | 105 | // 106 | // Default logfile path can be changed by LOG_FILE 107 | // environmental variable. 108 | // 109 | logPath := "rss2email.log" 110 | if os.Getenv("LOG_FILE_PATH") != "" { 111 | logPath = os.Getenv("LOG_FILE_PATH") 112 | } 113 | 114 | // 115 | // Create a logfile, if we can. 116 | // 117 | file, err := os.OpenFile(logPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) 118 | 119 | // 120 | // No error? Then update our writer to use it. 121 | // 122 | if err == nil { 123 | defer file.Close() 124 | 125 | // 126 | // Unless we've been disabled then update our 127 | // writer. 128 | // 129 | if os.Getenv("LOG_FILE_DISABLE") != "" { 130 | multi = io.MultiWriter(file, os.Stderr) 131 | } 132 | } 133 | 134 | // 135 | // Default to showing to STDERR [+file] in text. 136 | // 137 | var handler slog.Handler 138 | handler = slog.NewTextHandler(multi, opts) 139 | 140 | // 141 | // But allow JSON formatting too. 142 | // 143 | if os.Getenv("LOG_JSON") != "" { 144 | handler = slog.NewJSONHandler(multi, opts) 145 | } 146 | 147 | // 148 | // Create our logging handler, using the level we've just setup 149 | // 150 | logger = slog.New(handler) 151 | 152 | // 153 | // Catch errors 154 | // 155 | defer recoverPanic() 156 | 157 | // 158 | // Register each of our subcommands. 159 | // 160 | subcommands.Register(&addCmd{}) 161 | subcommands.Register(&cronCmd{}) 162 | subcommands.Register(&configCmd{}) 163 | subcommands.Register(&daemonCmd{}) 164 | subcommands.Register(&delCmd{}) 165 | subcommands.Register(&exportCmd{}) 166 | subcommands.Register(&importCmd{}) 167 | subcommands.Register(&listCmd{}) 168 | subcommands.Register(&listDefaultTemplateCmd{}) 169 | subcommands.Register(&seenCmd{}) 170 | subcommands.Register(&unseeCmd{}) 171 | subcommands.Register(&versionCmd{}) 172 | 173 | // 174 | // Execute the one the user chose. 175 | // 176 | os.Exit(subcommands.Execute()) 177 | } 178 | -------------------------------------------------------------------------------- /main_test.go: -------------------------------------------------------------------------------- 1 | // main_test.go - just setup a logger so the test-cases have 2 | // one available. 3 | 4 | package main 5 | 6 | import ( 7 | "log/slog" 8 | "os" 9 | ) 10 | 11 | // init runs at test-time. 12 | func init() { 13 | 14 | // setup logging-level 15 | lvl := &slog.LevelVar{} 16 | lvl.Set(slog.LevelWarn) 17 | 18 | // create a handler 19 | opts := &slog.HandlerOptions{Level: lvl} 20 | handler := slog.NewTextHandler(os.Stderr, opts) 21 | 22 | // ensure the global-variable is set. 23 | logger = slog.New(handler) 24 | } 25 | -------------------------------------------------------------------------------- /processor/emailer/emailer.go: -------------------------------------------------------------------------------- 1 | // Package emailer is responsible for sending out a feed 2 | // item via email. 3 | // 4 | // There are two ways emails are sent: 5 | // 6 | // 1. Via spawning /usr/sbin/sendmail. 7 | // 8 | // 2. Via SMTP. 9 | // 10 | // The choice is made based upon the presence of environmental 11 | // variables. 12 | package emailer 13 | 14 | import ( 15 | "bytes" 16 | "errors" 17 | "fmt" 18 | "html" 19 | "io" 20 | "log/slog" 21 | "mime/quotedprintable" 22 | "net/smtp" 23 | "os" 24 | "os/exec" 25 | "path/filepath" 26 | "strconv" 27 | "strings" 28 | "text/template" 29 | 30 | "github.com/mmcdole/gofeed" 31 | "github.com/skx/rss2email/configfile" 32 | "github.com/skx/rss2email/state" 33 | emailtemplate "github.com/skx/rss2email/template" 34 | "github.com/skx/rss2email/withstate" 35 | ) 36 | 37 | // Emailer stores our state 38 | type Emailer struct { 39 | 40 | // Feed is the source feed from which this item came 41 | feed *gofeed.Feed 42 | 43 | // Item is the feed item itself 44 | item withstate.FeedItem 45 | 46 | // Config options for the feed. 47 | opts []configfile.Option 48 | 49 | // logger contains a dedicated logging object 50 | logger *slog.Logger 51 | } 52 | 53 | // New creates a new Emailer object. 54 | // 55 | // The arguments are the source feed, the feed item which is being notified, 56 | // and any associated configuration values from the source feed. 57 | func New(feed *gofeed.Feed, item withstate.FeedItem, opts []configfile.Option, log *slog.Logger) *Emailer { 58 | 59 | // Default options 60 | obj := &Emailer{feed: feed, item: item, opts: opts} 61 | 62 | // Create a new logger 63 | obj.logger = log.With( 64 | slog.Group("email", 65 | slog.String("link", item.Link), 66 | slog.String("title", item.Title))) 67 | 68 | return obj 69 | } 70 | 71 | // env returns the contents of an environmental variable. 72 | // 73 | // This function exists to be used by our email-template. 74 | func env(s string) string { 75 | return (os.Getenv(s)) 76 | } 77 | 78 | // split converts a string to an array. 79 | // 80 | // This function exists to be used by our email-template. 81 | func split(in string, delim string) []string { 82 | return strings.Split(in, delim) 83 | } 84 | 85 | // loadTemplate loads the template used for sending the email notification. 86 | func (e *Emailer) loadTemplate() (*template.Template, error) { 87 | 88 | // Load the default template from the embedded resource. 89 | content := emailtemplate.EmailTemplate() 90 | 91 | // The directory within which we maintain state 92 | stateDir := state.Directory() 93 | 94 | // The path to the overridden template 95 | override := filepath.Join(stateDir, "email.tmpl") 96 | 97 | // If a per feed template was set, get it here. 98 | for _, opt := range e.opts { 99 | if opt.Name == "template" { 100 | override = filepath.Join(stateDir, opt.Value) 101 | } 102 | } 103 | 104 | // If the file exists, use it. 105 | _, err := os.Stat(override) 106 | if !os.IsNotExist(err) { 107 | content, err = os.ReadFile(override) 108 | if err != nil { 109 | 110 | e.logger.Debug("could not load template override file", 111 | slog.String("file", override), 112 | slog.String("error", err.Error())) 113 | 114 | return nil, fmt.Errorf("failed to read %s: %s", override, err.Error()) 115 | } 116 | } 117 | 118 | // 119 | // Function map allows exporting functions to the template 120 | // 121 | funcMap := template.FuncMap{ 122 | "env": env, 123 | "quoteprintable": toQuotedPrintable, 124 | "split": split, 125 | "encodeHeader": encodeHeader, 126 | } 127 | 128 | tmpl := template.Must(template.New("email.tmpl").Funcs(funcMap).Parse(string(content))) 129 | 130 | return tmpl, nil 131 | } 132 | 133 | // toQuotedPrintable will convert the given input-string to a 134 | // quoted-printable format. This is required for our MIME-part 135 | // body. 136 | // 137 | // NOTE: We use this function both directly, and from within our template. 138 | func toQuotedPrintable(s string) (string, error) { 139 | var ac bytes.Buffer 140 | w := quotedprintable.NewWriter(&ac) 141 | _, err := w.Write([]byte(s)) 142 | if err != nil { 143 | return "", err 144 | } 145 | err = w.Close() 146 | if err != nil { 147 | return "", err 148 | } 149 | return ac.String(), nil 150 | } 151 | 152 | // Encode email header entries to comply with the 7bit ASCII restriction 153 | // of RFC 5322 according to RFC 2047. 154 | // 155 | // We use quotedprintable encoding only if necessary. 156 | func encodeHeader(s string) string { 157 | se, err := toQuotedPrintable(s) 158 | if (err != nil) || (len(se) == len(s)) { 159 | return s 160 | } 161 | se = strings.Replace(strings.Replace(se, "?", "=3F", -1), " ", "=20", -1) 162 | se = strings.Replace(se, "=\r\n", "", -1) // remove soft line breaks 163 | return "=?utf-8?Q?" + se + "?=" 164 | } 165 | 166 | // Sendmail is a simple function that emails the given address. 167 | // 168 | // We send a MIME message with both a plain-text and a HTML-version of the 169 | // message. This should be nicer for users. 170 | func (e *Emailer) Sendmail(addresses []string, textstr string, htmlstr string) error { 171 | 172 | var err error 173 | 174 | // 175 | // Ensure we have a recipient. 176 | // 177 | if len(addresses) < 1 { 178 | 179 | e.logger.Error("missing recipient address") 180 | 181 | e := errors.New("empty recipient address, did you not setup a recipient?") 182 | return e 183 | } 184 | 185 | // 186 | // Process each address 187 | // 188 | for _, addr := range addresses { 189 | 190 | // 191 | // Here is a temporary structure we'll use to popular our email 192 | // template. 193 | // 194 | type TemplateParms struct { 195 | Feed string 196 | FeedTitle string 197 | From string 198 | HTML string 199 | Link string 200 | Subject string 201 | Tag string 202 | Text string 203 | To string 204 | 205 | // In case people need access to fields 206 | // we've not wrapped/exported explicitly 207 | RSSFeed *gofeed.Feed 208 | RSSItem withstate.FeedItem 209 | } 210 | 211 | // 212 | // Populate it appropriately. 213 | // 214 | var x TemplateParms 215 | x.Feed = e.feed.Link 216 | x.FeedTitle = e.feed.Title 217 | x.From = addr 218 | x.Link = e.item.Link 219 | x.Subject = e.item.Title 220 | x.To = addr 221 | x.RSSFeed = e.feed 222 | x.RSSItem = e.item 223 | x.Tag = e.item.Tag 224 | 225 | // The real meat of the mail is the text & HTML 226 | // parts. They need to be encoded, unconditionally. 227 | x.Text, err = toQuotedPrintable(textstr) 228 | if err != nil { 229 | return err 230 | } 231 | x.HTML, err = toQuotedPrintable(html.UnescapeString(htmlstr)) 232 | if err != nil { 233 | return err 234 | } 235 | 236 | // 237 | // Load the template we're going to render. 238 | // 239 | var t *template.Template 240 | t, err = e.loadTemplate() 241 | if err != nil { 242 | return err 243 | } 244 | 245 | // 246 | // Render the template into the buffer. 247 | // 248 | buf := &bytes.Buffer{} 249 | err = t.Execute(buf, x) 250 | if err != nil { 251 | return err 252 | } 253 | 254 | // 255 | // Are we sending via SMTP? 256 | // 257 | if e.isSMTP() { 258 | 259 | e.logger.Debug("preparing to send email", 260 | slog.String("recipient", addr), 261 | slog.String("method", "smtp")) 262 | 263 | err := e.sendSMTP(addr, buf.Bytes()) 264 | if err != nil { 265 | 266 | e.logger.Warn("error sending email", 267 | slog.String("recipient", addr), 268 | slog.String("method", "smtp"), 269 | slog.String("error", err.Error())) 270 | 271 | return err 272 | } 273 | 274 | e.logger.Debug("email sent", 275 | slog.String("recipient", addr), 276 | slog.String("method", "smtp")) 277 | 278 | } else { 279 | 280 | e.logger.Debug("preparing to send email", 281 | slog.String("recipient", addr), 282 | slog.String("method", "sendmail")) 283 | 284 | err := e.sendSendmail(addr, buf.Bytes()) 285 | if err != nil { 286 | e.logger.Warn("error sending email", 287 | slog.String("recipient", addr), 288 | slog.String("method", "sendmail"), 289 | slog.String("error", err.Error())) 290 | return err 291 | } 292 | 293 | e.logger.Debug("email sent", 294 | slog.String("recipient", addr), 295 | slog.String("method", "sendmail")) 296 | 297 | } 298 | } 299 | 300 | e.logger.Debug("emails sent", 301 | slog.Int("recipients", len(addresses))) 302 | 303 | return nil 304 | } 305 | 306 | // isSMTP determines whether we should use SMTP to send the email. 307 | // 308 | // We just check to see that the obvious mandatory parameters are set in the 309 | // environment. If they're wrong we'll get an error at delivery time, as 310 | // expected. 311 | func (e *Emailer) isSMTP() bool { 312 | 313 | // Mandatory environmental variables 314 | vars := []string{"SMTP_HOST", "SMTP_USERNAME", "SMTP_PASSWORD"} 315 | 316 | for _, name := range vars { 317 | if os.Getenv(name) == "" { 318 | return false 319 | } 320 | } 321 | 322 | return true 323 | } 324 | 325 | // sendSMTP sends the content of the email to the destination address 326 | // via SMTP. 327 | func (e *Emailer) sendSMTP(to string, content []byte) error { 328 | 329 | // basics 330 | host := os.Getenv("SMTP_HOST") 331 | port := os.Getenv("SMTP_PORT") 332 | 333 | p := 587 334 | if port != "" { 335 | n, err := strconv.Atoi(port) 336 | if err != nil { 337 | 338 | e.logger.Warn("error converting SMTP_PORT to integer", 339 | slog.String("port", port), 340 | slog.String("error", err.Error())) 341 | 342 | return err 343 | } 344 | p = n 345 | } 346 | 347 | // auth 348 | user := os.Getenv("SMTP_USERNAME") 349 | pass := os.Getenv("SMTP_PASSWORD") 350 | 351 | // Authenticate 352 | auth := smtp.PlainAuth("", user, pass, host) 353 | 354 | // Get the mailserver 355 | addr := fmt.Sprintf("%s:%d", host, p) 356 | 357 | // Send the mail 358 | err := smtp.SendMail(addr, auth, to, []string{to}, content) 359 | 360 | return err 361 | } 362 | 363 | // sendSendmail sends the content of the email to the destination address 364 | // via /usr/sbin/sendmail 365 | func (e *Emailer) sendSendmail(addr string, content []byte) error { 366 | 367 | // Get the command to run. 368 | sendmail := exec.Command("/usr/sbin/sendmail", "-i", "-f", addr, addr) 369 | stdin, err := sendmail.StdinPipe() 370 | if err != nil { 371 | 372 | e.logger.Warn("error creating STDIN pipe to sendmail", 373 | slog.String("recipient", addr), 374 | slog.String("error", err.Error())) 375 | 376 | return err 377 | } 378 | 379 | // 380 | // Get the output pipe. 381 | // 382 | stdout, err := sendmail.StdoutPipe() 383 | if err != nil { 384 | 385 | e.logger.Warn("error creating STDOUT pipe to sendmail", 386 | slog.String("recipient", addr), 387 | slog.String("error", err.Error())) 388 | 389 | return err 390 | } 391 | 392 | // 393 | // Run the command, and pipe in the rendered template-result 394 | // 395 | err = sendmail.Start() 396 | if err != nil { 397 | 398 | e.logger.Warn("error starting sendmail", 399 | slog.String("recipient", addr), 400 | slog.String("error", err.Error())) 401 | 402 | return err 403 | } 404 | _, err = stdin.Write(content) 405 | if err != nil { 406 | 407 | e.logger.Warn("error writing to sendmail pipe", 408 | slog.String("recipient", addr), 409 | slog.String("error", err.Error())) 410 | 411 | return err 412 | } 413 | stdin.Close() 414 | 415 | // 416 | // Read the output of Sendmail. 417 | // 418 | _, err = io.ReadAll(stdout) 419 | if err != nil { 420 | 421 | e.logger.Warn("error reading from sendmail pipe", 422 | slog.String("recipient", addr), 423 | slog.String("error", err.Error())) 424 | 425 | return err 426 | } 427 | 428 | // 429 | // Wait for the command to complete. 430 | // 431 | err = sendmail.Wait() 432 | if err != nil { 433 | 434 | e.logger.Warn("error awaiting sendmail completion", 435 | slog.String("recipient", addr), 436 | slog.String("error", err.Error())) 437 | 438 | return err 439 | } 440 | 441 | return err 442 | } 443 | -------------------------------------------------------------------------------- /processor/processor.go: -------------------------------------------------------------------------------- 1 | // Package processor contains the code which will actually poll 2 | // the list of URLs the user is watching, and send emails for those 3 | // entries which are new. 4 | // 5 | // Items which are excluded are treated the same as normal items, 6 | // in the sense they are processed once and then marked as having 7 | // been seen - the only difference is no email is actually generated 8 | // for them. 9 | package processor 10 | 11 | import ( 12 | "fmt" 13 | "log/slog" 14 | "net/url" 15 | "os" 16 | "path/filepath" 17 | "regexp" 18 | "strconv" 19 | "strings" 20 | "time" 21 | 22 | "github.com/k3a/html2text" 23 | "github.com/skx/rss2email/configfile" 24 | "github.com/skx/rss2email/httpfetch" 25 | "github.com/skx/rss2email/processor/emailer" 26 | "github.com/skx/rss2email/state" 27 | "github.com/skx/rss2email/withstate" 28 | "go.etcd.io/bbolt" 29 | ) 30 | 31 | // Processor stores our state 32 | type Processor struct { 33 | 34 | // send controls whether we send emails, or just pretend to. 35 | send bool 36 | 37 | // database holds a handle to the BoltDB database we use to 38 | // store feed-entry state within. 39 | dbHandle *bbolt.DB 40 | 41 | // logger stores the logging dbHandle. 42 | logger *slog.Logger 43 | 44 | // version stores the version of our application. 45 | version string 46 | } 47 | 48 | // New creates a new Processor object. 49 | // 50 | // This might return an error if we fail to open the database we use 51 | // for maintaining state. 52 | func New() (*Processor, error) { 53 | 54 | // Ensure we have a state-directory. 55 | dir := state.Directory() 56 | errM := os.MkdirAll(dir, 0666) 57 | if errM != nil { 58 | return nil, errM 59 | } 60 | 61 | // Now create the database, if missing, or open it if it exists. 62 | db, err := bbolt.Open(filepath.Join(dir, "state.db"), 0666, nil) 63 | if err != nil { 64 | return nil, err 65 | } 66 | 67 | return &Processor{send: true, dbHandle: db}, nil 68 | } 69 | 70 | // Close should be called to cleanup our internal database-handle. 71 | func (p *Processor) Close() { 72 | p.dbHandle.Close() 73 | } 74 | 75 | // ProcessFeeds is the main workhorse here, we process each feed and send 76 | // emails appropriately. 77 | func (p *Processor) ProcessFeeds(recipients []string) []error { 78 | 79 | // 80 | // If we receive errors we'll store them here, 81 | // so we can keep processing subsequent URIs. 82 | // 83 | var errors []error 84 | 85 | // Get the configuration-file 86 | conf := configfile.New() 87 | 88 | // Now do the parsing 89 | entries, err := conf.Parse() 90 | if err != nil { 91 | p.logger.Error("failed to parse configuration file", 92 | slog.String("configfile", conf.Path()), 93 | slog.String("error", err.Error())) 94 | return errors 95 | } 96 | 97 | // Keep track of the previous hostname from which we fetched a feed 98 | prev := "" 99 | 100 | // Keep track of each feed we've processed 101 | feeds := []string{} 102 | 103 | // We're about to process the feeds. 104 | p.logger.Debug("about to process feeds", 105 | slog.Int("feed_count", len(entries))) 106 | 107 | // For each feed contained in the configuration file 108 | for _, entry := range entries { 109 | 110 | p.logger.Debug("starting to process feed", 111 | slog.String("feed", entry.URL)) 112 | 113 | // Create a bucket to hold the entry-state here, 114 | // if we've not done so previously. 115 | // 116 | // We do this because we store the "seen" vs "unseen" 117 | // state in BoltDB database. 118 | // 119 | // BoltDB has a concept of "Buckets", which contain 120 | // key=value entries. 121 | // 122 | // Since we process feeds it seems logical to create 123 | // a bucket for each Feed URL, and then store the 124 | // URLs we've seen with a random value. 125 | // 126 | err = p.dbHandle.Update(func(tx *bbolt.Tx) error { 127 | _, err2 := tx.CreateBucketIfNotExists([]byte(entry.URL)) 128 | if err2 != nil { 129 | return fmt.Errorf("create bucket failed: %s", err) 130 | } 131 | return nil 132 | }) 133 | 134 | // If we have a DB-error then we return, this shouldn't happen. 135 | if err != nil { 136 | 137 | p.logger.Error("error creating bucket", 138 | slog.String("feed", entry.URL), 139 | slog.String("error", err.Error())) 140 | 141 | errors = append(errors, fmt.Errorf("error creating bucket for %s: %s", entry.URL, err)) 142 | return (errors) 143 | } 144 | 145 | // Record the URL of the feed in our list, 146 | // which is used for reaping obsolete feeds 147 | feeds = append(feeds, entry.URL) 148 | 149 | // Should we sleep before getting this feed? 150 | sleep := 0 151 | 152 | // We default to notifying the global recipient-list. 153 | // 154 | // But there might be a per-feed set of recipients which 155 | // we'll prefer if available. 156 | feedRecipients := recipients 157 | 158 | // parse the hostname form the URL 159 | // 160 | // We do this because some remote sites, such as Reddit, 161 | // will apply rate-limiting if we make too many consecutive 162 | // requests in a short period of time. 163 | host := "" 164 | u, err2 := url.Parse(entry.URL) 165 | if err2 == nil { 166 | host = u.Host 167 | } 168 | 169 | // Are we fetching from the same host as the previous feed? 170 | // If so then we'll add a delay to try to avoid annoying that 171 | // host. 172 | if host == prev { 173 | 174 | p.logger.Debug("fetching from same host as previous feed adding delay", 175 | slog.Int("sleep", 5), 176 | slog.String("host", prev), 177 | slog.String("feed", entry.URL)) 178 | 179 | sleep = 5 180 | } 181 | 182 | // Now look at each per-feed option, if any are set. 183 | for _, opt := range entry.Options { 184 | 185 | // Is it a set of recipients? 186 | if opt.Name == "notify" { 187 | 188 | // Save the values 189 | feedRecipients = strings.Split(opt.Value, ",") 190 | 191 | // But trim leading/trailing space 192 | for i := range feedRecipients { 193 | feedRecipients[i] = strings.TrimSpace(feedRecipients[i]) 194 | } 195 | } 196 | 197 | // Sleep setting? 198 | if opt.Name == "sleep" { 199 | 200 | // Convert the value, and if there was 201 | // no error save it away. 202 | num, nErr := strconv.Atoi(opt.Value) 203 | if nErr != nil { 204 | 205 | p.logger.Warn("failed to parse sleep value as number", 206 | slog.String("sleep", opt.Value), 207 | slog.String("error", nErr.Error())) 208 | 209 | // be conservative 210 | sleep = 10 211 | } else { 212 | sleep = num 213 | } 214 | } 215 | } 216 | 217 | // If we're supposed to sleep, do so 218 | if sleep != 0 { 219 | 220 | p.logger.Warn("sleeping", 221 | slog.Int("sleep", sleep)) 222 | 223 | time.Sleep(time.Duration(sleep) * time.Second) 224 | } 225 | 226 | // Process this specific entry. 227 | err = p.processFeed(entry, feedRecipients) 228 | if err != nil { 229 | errors = append(errors, fmt.Errorf("error processing %s - %s", entry.URL, err)) 230 | } 231 | 232 | // Now update with our current host. 233 | prev = host 234 | } 235 | 236 | // Reap feeds which are obsolete. 237 | err = p.pruneUnknownFeeds(feeds) 238 | if err != nil { 239 | 240 | p.logger.Warn("failed to prune unknown feeds", 241 | slog.String("error", err.Error())) 242 | 243 | errors = append(errors, err) 244 | } 245 | 246 | // We're about to process the feeds. 247 | p.logger.Debug("all feeds processed", 248 | slog.Int("feed_count", len(entries))) 249 | 250 | // All feeds were processed, return any errors we found along the way 251 | return errors 252 | } 253 | 254 | // processFeed takes a configuration entry as input, fetches the appropriate 255 | // remote contents, and then processes each feed item found within it. 256 | // 257 | // Feed items which are new/unread will generate an email, unless they are 258 | // specifically excluded by the per-feed options. 259 | func (p *Processor) processFeed(entry configfile.Feed, recipients []string) error { 260 | 261 | // Create a local logger with some dedicated information 262 | logger := p.logger.With( 263 | slog.Group("feed", 264 | slog.String("link", entry.URL))) 265 | 266 | // Is there a tag set for this feed? 267 | tag := "" 268 | 269 | // Look at each per-feed option to determine that 270 | for _, opt := range entry.Options { 271 | if strings.ToLower(opt.Name) == "tag" { 272 | tag = opt.Value 273 | } 274 | } 275 | 276 | // Fetch the feed for the input URL 277 | helper := httpfetch.New(entry, logger, p.version) 278 | feed, err := helper.Fetch() 279 | if err != nil { 280 | 281 | if err == httpfetch.ErrUnchanged { 282 | logger.Warn("remote feed unchanged, skipping") 283 | return nil 284 | } 285 | 286 | logger.Warn("failed to fetch feed", 287 | slog.String("error", err.Error())) 288 | return err 289 | } 290 | 291 | // Show how many entries we've found in the feed. 292 | logger.Debug("feed retrieved", slog.Int("entries", len(feed.Items))) 293 | 294 | // Count how many seen/unseen items there were. 295 | seen := 0 296 | unseen := 0 297 | 298 | // Keep track of all the items in the feed. 299 | items := []string{} 300 | 301 | // 302 | // Issue #111 reported an example feed which 303 | // contained duplicate URLs 304 | // 305 | // We can look over the links in the feed, before 306 | // we do anything else, and look to see if we have 307 | // duplicates 308 | // 309 | // Do we have dupes? 310 | // 311 | dupes := false 312 | 313 | // 314 | // Temporary map 315 | // 316 | seenDupes := make(map[string]int) 317 | for _, str := range feed.Items { 318 | if seenDupes[str.Link] > 0 { 319 | 320 | // only log the messages once. 321 | if !dupes { 322 | logger.Warn("feed contains duplicate links") 323 | } 324 | dupes = true 325 | } 326 | 327 | seenDupes[str.Link]++ 328 | } 329 | 330 | // For each entry in the feed .. 331 | for _, xp := range feed.Items { 332 | 333 | // If the feed contains duplicate entries 334 | // then we try to uniquify them. 335 | if dupes { 336 | xp.Link += "#" 337 | xp.Link += xp.GUID 338 | } 339 | 340 | // Wrap the feed-item in a class of our own, 341 | // so that we can get access to the content easily. 342 | // 343 | // Specifically here we turn relative URLs into absolute 344 | // ones, using the feed link as the base. 345 | // 346 | // We have some legacy code for determining "new" vs "seen", 347 | // but that will go away in the future. 348 | item := withstate.FeedItem{Item: xp} 349 | 350 | // Set the tag for the item, if present. 351 | if tag != "" { 352 | item.Tag = tag 353 | } 354 | 355 | // Keep track of the fact that we saw this feed-item. 356 | // 357 | // This is used for pruning the BoltDB state file. 358 | items = append(items, item.Link) 359 | 360 | // Assume this feed-entry is new, and we've not seen it 361 | // in the past. 362 | isNew := true 363 | 364 | // Is this link already in the BoltDB? 365 | // 366 | // If so it's not new. 367 | if p.seenItem(entry.URL, item.Link) { 368 | isNew = false 369 | } 370 | 371 | // If this entry is new then we must notify, unless 372 | // the entry is excluded for some reason. 373 | if isNew { 374 | 375 | // Bump the count 376 | unseen++ 377 | 378 | // Show that we got something 379 | logger.Debug("new entry found in feed", 380 | slog.String("title", item.Title), 381 | slog.String("link", item.Link)) 382 | 383 | // If we're supposed to send email then do that. 384 | if p.send { 385 | 386 | // Get the content of the feed-item. 387 | // 388 | // This has to be done ahead of sending email, 389 | // as we can use this to skip entries via 390 | // regular expression on the title/body contents. 391 | content := "" 392 | content, err = item.HTMLContent() 393 | if err != nil { 394 | content = item.RawContent() 395 | } 396 | 397 | // Should we skip this entry? 398 | // 399 | // Skipping here means that we don't send an email, 400 | // however we do mark it as read - so it will only 401 | // be processed once. 402 | 403 | // check for regular expressions 404 | skip := p.shouldSkip(logger, entry, item.Title, content) 405 | 406 | // check for age (exclude-older) 407 | skip = skip || p.shouldSkipOlder(logger, entry, item.Published) 408 | 409 | if !skip { 410 | // Convert the content to text. 411 | text := html2text.HTML2Text(content) 412 | 413 | // Send the mail 414 | helper := emailer.New(feed, item, entry.Options, logger) 415 | err = helper.Sendmail(recipients, text, content) 416 | if err != nil { 417 | 418 | logger.Warn("failed to send email", 419 | slog.String("recipients", strings.Join(recipients, ",")), 420 | slog.String("error", err.Error())) 421 | 422 | return err 423 | } 424 | } 425 | } 426 | } else { 427 | 428 | // Bump the count 429 | seen++ 430 | 431 | } 432 | 433 | // Mark the item as having been seen, after the email 434 | // was (probably) sent. 435 | // 436 | // This does run the risk that sending mail fails, 437 | // due to error, and that keeps happening forever... 438 | err = p.recordItem(entry.URL, item.Link) 439 | if err != nil { 440 | logger.Warn("failed to mark item as processed", 441 | slog.String("error", err.Error())) 442 | return err 443 | } 444 | } 445 | 446 | logger.Debug("feed processed", 447 | slog.Int("seen_count", seen), 448 | slog.Int("unseen_count", unseen)) 449 | 450 | // Now prune the items in this feed. 451 | err = p.pruneFeed(entry.URL, items) 452 | if err != nil { 453 | 454 | logger.Warn("failed to prune bolddb", 455 | slog.String("error", err.Error())) 456 | 457 | return fmt.Errorf("error pruning boltdb for %s: %s", entry.URL, err) 458 | } 459 | 460 | return nil 461 | } 462 | 463 | // seenItem returns true if we've seen this item. 464 | // 465 | // It does this by checking the BoltDB in which we record state. 466 | func (p *Processor) seenItem(feed string, entry string) bool { 467 | val := "" 468 | 469 | err := p.dbHandle.View(func(tx *bbolt.Tx) error { 470 | 471 | // Select the feed-bucket 472 | b := tx.Bucket([]byte(feed)) 473 | 474 | // Get the entry with key of the feed URL. 475 | v := b.Get([]byte(entry)) 476 | if v != nil { 477 | val = string(v) 478 | } 479 | return nil 480 | }) 481 | if err != nil { 482 | p.logger.Warn("error checking state of item", 483 | slog.String("feed", feed), 484 | slog.String("item", entry), 485 | slog.String("error", err.Error())) 486 | } 487 | 488 | return val != "" 489 | } 490 | 491 | // recordItem marks an URL as having been seen. 492 | // 493 | // It does this by updating the BoltDB in which we record state. 494 | func (p *Processor) recordItem(feed string, entry string) error { 495 | 496 | err := p.dbHandle.Update(func(tx *bbolt.Tx) error { 497 | 498 | // Select the feed-bucket 499 | b := tx.Bucket([]byte(feed)) 500 | 501 | // Set a value "seen" to the key of the feed item link 502 | err := b.Put([]byte(entry), []byte("seen")) 503 | return err 504 | }) 505 | 506 | if err != nil { 507 | p.logger.Warn("error recording state of item", 508 | slog.String("feed", feed), 509 | slog.String("item", entry), 510 | slog.String("error", err.Error())) 511 | } 512 | 513 | return err 514 | } 515 | 516 | // pruneFeed will remove unknown items from our state database. 517 | // 518 | // Here we are given the URL of the feed, and a set of feed-item links, 519 | // we remove items which are no longer in the remote feed. 520 | // 521 | // See also `pruneUnknownFeeds` for removing feeds which are no longer 522 | // fetched at all. 523 | func (p *Processor) pruneFeed(feed string, items []string) error { 524 | 525 | // A list of items to remove 526 | toRemove := []string{} 527 | 528 | // Create a map of the items we've already seen 529 | seen := make(map[string]bool) 530 | for _, str := range items { 531 | seen[str] = true 532 | } 533 | 534 | // Select the bucket, which we know will exist, 535 | // and see if we should remove any of the keys 536 | // that are present. 537 | // 538 | // (i.e. Remove the ones that are not in the map above) 539 | err := p.dbHandle.View(func(tx *bbolt.Tx) error { 540 | 541 | // Select the bucket, which we know must exist 542 | b := tx.Bucket([]byte(feed)) 543 | 544 | // Get a cursor to the key=value entries in the bucket 545 | c := b.Cursor() 546 | 547 | // Iterate over the key/value pairs. 548 | for k, _ := c.First(); k != nil; k, _ = c.Next() { 549 | 550 | // Convert the key to a string 551 | key := string(k) 552 | 553 | // Is this in our list of seen entries? 554 | _, ok := seen[key] 555 | if !ok { 556 | // If not remove the key/value pair 557 | toRemove = append(toRemove, key) 558 | } 559 | } 560 | return nil 561 | }) 562 | 563 | if err != nil { 564 | 565 | p.logger.Warn("error getting all bucket keys", 566 | slog.String("error", err.Error())) 567 | return err 568 | } 569 | 570 | // Remove each entry that we were supposed to remove. 571 | for _, ent := range toRemove { 572 | 573 | err := p.dbHandle.Update(func(tx *bbolt.Tx) error { 574 | 575 | // Select the bucket 576 | b := tx.Bucket([]byte(feed)) 577 | 578 | // Delete the key=value pair. 579 | return b.Delete([]byte(ent)) 580 | }) 581 | if err != nil { 582 | 583 | p.logger.Warn("error deleting key from bucket", 584 | slog.String("entry", ent), 585 | slog.String("error", err.Error())) 586 | 587 | return fmt.Errorf("failed to remove %s - %s", ent, err) 588 | } 589 | } 590 | 591 | return nil 592 | } 593 | 594 | // pruneUnknownFeeds removes feeds from our database which are no longer 595 | // contained within our configuration file. 596 | // 597 | // To recap BoltDB has a notion of buckets, which are used to store key=value 598 | // pairs. We create a bucket for every feed which is present in our 599 | // configuration value, then use the URL of feed-items as the keys. 600 | // 601 | // Here we remove buckets which are obsolete. 602 | func (p *Processor) pruneUnknownFeeds(feeds []string) error { 603 | 604 | // Create a map for lookup 605 | seen := make(map[string]bool) 606 | for _, str := range feeds { 607 | seen[str] = true 608 | } 609 | 610 | // Now walk the database and see which buckets should be removed. 611 | toRemove := []string{} 612 | 613 | err := p.dbHandle.View(func(tx *bbolt.Tx) error { 614 | 615 | return tx.ForEach(func(bucketName []byte, _ *bbolt.Bucket) error { 616 | 617 | // Does this name exist in our map? 618 | _, ok := seen[string(bucketName)] 619 | 620 | // If not then it should be removed. 621 | if !ok { 622 | toRemove = append(toRemove, string(bucketName)) 623 | } 624 | return nil 625 | }) 626 | }) 627 | 628 | if err != nil { 629 | p.logger.Warn("error finding orphaned buckets", 630 | slog.String("error", err.Error())) 631 | 632 | return err 633 | } 634 | // For each bucket we need to remove, remove it 635 | for _, bucket := range toRemove { 636 | 637 | err := p.dbHandle.Update(func(tx *bbolt.Tx) error { 638 | 639 | // Select the bucket, which we know must exist 640 | b := tx.Bucket([]byte(bucket)) 641 | 642 | // Get a cursor to the key=value entries in the bucket 643 | c := b.Cursor() 644 | 645 | // Iterate over the key/value pairs and delete them. 646 | for k, _ := c.First(); k != nil; k, _ = c.Next() { 647 | 648 | err := b.Delete(k) 649 | if err != nil { 650 | 651 | p.logger.Warn("error removing key from bucket", 652 | slog.String("bucket", bucket), 653 | slog.String("key", string(k)), 654 | slog.String("error", err.Error())) 655 | 656 | return (fmt.Errorf("failed to delete bucket key %s:%s - %s", bucket, k, err)) 657 | } 658 | } 659 | 660 | // Now delete the bucket itself 661 | err := tx.DeleteBucket([]byte(bucket)) 662 | if err != nil { 663 | p.logger.Warn("error removing bucket", 664 | slog.String("bucket", bucket), 665 | slog.String("error", err.Error())) 666 | return fmt.Errorf("failed to remove bucket %s: %s", bucket, err) 667 | } 668 | 669 | return nil 670 | }) 671 | if err != nil { 672 | return fmt.Errorf("error removing bucket %s: %s", bucket, err) 673 | } 674 | } 675 | 676 | return nil 677 | } 678 | 679 | // shouldSkip returns true if this entry should be skipped/ignored. 680 | // 681 | // Our configuration file allows a series of per-feed configuration items, 682 | // and those allow skipping the entry by regular expression matches on 683 | // the item title or body. 684 | // 685 | // Similarly there is an `include` setting which will ensure we only 686 | // email items matching a particular regular expression. 687 | // 688 | // Note that if an entry should be skipped it is still marked as 689 | // having been read, but no email is sent. 690 | func (p *Processor) shouldSkip(logger *slog.Logger, config configfile.Feed, title string, content string) bool { 691 | 692 | // Walk over the options to see if there are any exclude* options 693 | // specified. 694 | for _, opt := range config.Options { 695 | 696 | // Exclude by title? 697 | if opt.Name == "exclude-title" { 698 | match, _ := regexp.MatchString(opt.Value, title) 699 | if match { 700 | logger.Debug("excluding entry due to exclude-title", 701 | slog.String("exclude-title", opt.Value), 702 | slog.String("item-title", title)) 703 | // True: skip/ignore this entry 704 | return true 705 | } 706 | } 707 | 708 | // Exclude by body/content? 709 | if opt.Name == "exclude" { 710 | 711 | match, _ := regexp.MatchString(opt.Value, content) 712 | if match { 713 | logger.Debug("excluding entry due to exclude", 714 | slog.String("exclude", opt.Value), 715 | slog.String("item-title", title)) 716 | 717 | // True: skip/ignore this entry 718 | return true 719 | } 720 | } 721 | } 722 | 723 | // If we have an include-setting then we must skip the entry unless 724 | // it matches. 725 | // 726 | // There might be more than one include setting and a match against 727 | // any will suffice. 728 | // 729 | include := false 730 | 731 | it := "" 732 | i := "" 733 | 734 | for _, opt := range config.Options { 735 | if opt.Name == "include-title" { 736 | 737 | // Save 738 | it = opt.Value 739 | 740 | // We found (at least one) include option 741 | include = true 742 | 743 | // OK we've found a `include` setting, 744 | // so we MUST skip unless there is a match 745 | match, _ := regexp.MatchString(opt.Value, title) 746 | if match { 747 | logger.Debug("including entry due to 'include-title'", 748 | slog.String("include-title", opt.Value), 749 | slog.String("item-title", title)) 750 | 751 | // False: Do not skip/ignore this entry 752 | return false 753 | } 754 | } 755 | if opt.Name == "include" { 756 | 757 | // Save 758 | i = opt.Value 759 | 760 | // We found (at least one) include option 761 | include = true 762 | 763 | // OK we've found a `include` setting, 764 | // so we MUST skip unless there is a match 765 | match, _ := regexp.MatchString(opt.Value, content) 766 | if match { 767 | logger.Debug("including entry due to 'include'", 768 | slog.String("include", opt.Value), 769 | slog.String("item-title", title)) 770 | 771 | // False: Do not skip/ignore this entry 772 | return false 773 | } 774 | } 775 | } 776 | 777 | // If we had at least one "include" setting and we reach here 778 | // the we had no match. 779 | // 780 | // i.e. The entry did not include a string we regarded as mandatory. 781 | if include { 782 | logger.Debug("excluding entry due to 'include', or 'include-title'", 783 | slog.String("include", i), 784 | slog.String("include-title", it), 785 | slog.String("item-title", title)) 786 | 787 | // True: skip/ignore this entry 788 | return true 789 | } 790 | 791 | // False: Do not skip/ignore this entry 792 | return false 793 | } 794 | 795 | // shouldSkipOlder returns true if this entry should be skipped due to age. 796 | // 797 | // Age is configured with "exclude-older" in days. 798 | func (p *Processor) shouldSkipOlder(logger *slog.Logger, config configfile.Feed, published string) bool { 799 | 800 | // Walk over the options to see if there are any exclude-age options 801 | // specified. 802 | for _, opt := range config.Options { 803 | 804 | if opt.Name == "exclude-older" { 805 | pubTime, err := time.Parse(time.RFC1123, published) 806 | if err != nil { 807 | logger.Warn("failed to parse 'item.published' as date", 808 | slog.String("date", published), 809 | slog.String("error", err.Error())) 810 | return false 811 | } 812 | f, err := strconv.ParseFloat(opt.Value, 32) 813 | if err != nil { 814 | logger.Warn("failed to parse 'exclude-older' as float", 815 | slog.String("exclude-older", opt.Value), 816 | slog.String("error", err.Error())) 817 | 818 | return false 819 | } 820 | 821 | delta := time.Second * time.Duration(f*24*60*60) 822 | if pubTime.Add(delta).Before(time.Now()) { 823 | logger.Debug("excluding entry due to exclude-older setting", 824 | slog.String("exclude-older", opt.Value), 825 | slog.Float64("days", time.Since(pubTime).Hours()/24)) 826 | return true 827 | } 828 | } 829 | } 830 | 831 | // False: Do not skip/ignore this entry 832 | return false 833 | } 834 | 835 | // SetSendEmail updates the state of this object, when the send-flag 836 | // is false zero emails are generated. 837 | func (p *Processor) SetSendEmail(state bool) { 838 | p.send = state 839 | } 840 | 841 | // SetLogger ensures we have a logging-handle 842 | func (p *Processor) SetLogger(logger *slog.Logger) { 843 | p.logger = logger 844 | } 845 | 846 | // SetVersion ensures we can pass the version of our client to our HTTP-fetcher, 847 | // which means that the version will end up in our (default) user-agent. 848 | func (p *Processor) SetVersion(version string) { 849 | p.version = version 850 | } 851 | -------------------------------------------------------------------------------- /processor/processor_test.go: -------------------------------------------------------------------------------- 1 | package processor 2 | 3 | import ( 4 | "log/slog" 5 | "os" 6 | "testing" 7 | "time" 8 | 9 | "github.com/skx/rss2email/configfile" 10 | ) 11 | 12 | var ( 13 | // logger contains a shared logging handle, the code we're testing assumes it exists. 14 | logger *slog.Logger 15 | ) 16 | 17 | // init runs at test-time. 18 | func init() { 19 | 20 | // setup logging-level 21 | lvl := &slog.LevelVar{} 22 | lvl.Set(slog.LevelWarn) 23 | 24 | // create a handler 25 | opts := &slog.HandlerOptions{Level: lvl} 26 | handler := slog.NewTextHandler(os.Stderr, opts) 27 | 28 | // ensure the global-variable is set. 29 | logger = slog.New(handler) 30 | } 31 | 32 | func TestSendEmail(t *testing.T) { 33 | 34 | p, err := New() 35 | 36 | if err != nil { 37 | t.Fatalf("error creating processor %s", err.Error()) 38 | } 39 | defer p.Close() 40 | 41 | if !p.send { 42 | t.Fatalf("unexpected default to sending mail") 43 | } 44 | 45 | p.SetSendEmail(false) 46 | 47 | if p.send { 48 | t.Fatalf("unexpected send-setting") 49 | } 50 | 51 | } 52 | 53 | func TestVerbose(t *testing.T) { 54 | 55 | p, err := New() 56 | 57 | if err != nil { 58 | t.Fatalf("error creating processor %s", err.Error()) 59 | } 60 | 61 | defer p.Close() 62 | } 63 | 64 | // TestSkipExclude ensures that we can exclude items by regexp 65 | func TestSkipExclude(t *testing.T) { 66 | 67 | feed := configfile.Feed{ 68 | URL: "blah", 69 | Options: []configfile.Option{ 70 | {Name: "exclude", Value: "foo"}, 71 | {Name: "exclude-title", Value: "test"}, 72 | }, 73 | } 74 | 75 | // Create the new processor 76 | x, err := New() 77 | 78 | if err != nil { 79 | t.Fatalf("error creating processor %s", err.Error()) 80 | } 81 | defer x.Close() 82 | 83 | if !x.shouldSkip(logger, feed, "Title here", "

foo, bar baz

") { 84 | t.Fatalf("failed to skip entry by regexp") 85 | } 86 | 87 | if !x.shouldSkip(logger, feed, "test", "

This matches the title

") { 88 | t.Fatalf("failed to skip entry by title") 89 | } 90 | 91 | // With no options we're not going to skip 92 | feed = configfile.Feed{ 93 | URL: "blah", 94 | Options: []configfile.Option{}, 95 | } 96 | 97 | if x.shouldSkip(logger, feed, "Title here", "

foo, bar baz

") { 98 | t.Fatalf("skipped something with no options!") 99 | } 100 | 101 | } 102 | 103 | // TestSkipInclude ensures that we can exclude items by regexp 104 | func TestSkipInclude(t *testing.T) { 105 | 106 | feed := configfile.Feed{ 107 | URL: "blah", 108 | Options: []configfile.Option{ 109 | {Name: "include", Value: "good"}, 110 | }, 111 | } 112 | 113 | // Create the new processor 114 | x, err := New() 115 | 116 | if err != nil { 117 | t.Fatalf("error creating processor %s", err.Error()) 118 | } 119 | defer x.Close() 120 | 121 | if x.shouldSkip(logger, feed, "Title here", "

This is good

") { 122 | t.Fatalf("this should be included because it contains good") 123 | } 124 | 125 | if !x.shouldSkip(logger, feed, "Title here", "

This should be excluded.

") { 126 | t.Fatalf("This should be excluded; doesn't contain 'good'") 127 | } 128 | 129 | // If we don't try to make a mandatory include setting 130 | // nothing should be skipped 131 | feed = configfile.Feed{ 132 | URL: "blah", 133 | Options: []configfile.Option{}, 134 | } 135 | 136 | if x.shouldSkip(logger, feed, "Title here", "

This is good

") { 137 | t.Fatalf("nothing specified, shouldn't be skipped") 138 | } 139 | } 140 | 141 | // TestSkipIncludeTitle ensures that we can exclude items by regexp 142 | func TestSkipIncludeTitle(t *testing.T) { 143 | 144 | feed := configfile.Feed{ 145 | URL: "blah", 146 | Options: []configfile.Option{ 147 | {Name: "include", Value: "good"}, 148 | {Name: "include-title", Value: "(?i)cake"}, 149 | }, 150 | } 151 | 152 | // Create the new processor 153 | x, err := New() 154 | if err != nil { 155 | t.Fatalf("error creating processor %s", err.Error()) 156 | } 157 | 158 | if x.shouldSkip(logger, feed, "Title here", "

This is good

") { 159 | t.Fatalf("this should be included because it contains good") 160 | } 161 | if x.shouldSkip(logger, feed, "I like Cake!", "

Food is good.

") { 162 | t.Fatalf("this should be included because of the title") 163 | } 164 | 165 | // 166 | // Second test, only include titles 167 | // 168 | feed = configfile.Feed{ 169 | URL: "blah", 170 | Options: []configfile.Option{ 171 | {Name: "include-title", Value: "(?i)cake"}, 172 | {Name: "include-title", Value: "(?i)pie"}, 173 | }, 174 | } 175 | 176 | // 177 | // Some titles which are OK 178 | // 179 | valid := []string{"I like cake", "I like pie", "piecemeal", "cupcake", "pancake"} 180 | bogus := []string{"I do not like food", "I don't like cooked goods", "cheese is dead milk", "books are fun", "tv is good"} 181 | 182 | // Create the new processor 183 | x.Close() 184 | x, err = New() 185 | if err != nil { 186 | t.Fatalf("error creating processor %s", err.Error()) 187 | } 188 | defer x.Close() 189 | 190 | // include 191 | for _, entry := range valid { 192 | if x.shouldSkip(logger, feed, entry, "content") { 193 | t.Fatalf("this should be included due to include-title") 194 | } 195 | } 196 | 197 | // exclude 198 | for _, entry := range bogus { 199 | if !x.shouldSkip(logger, feed, entry, "content") { 200 | t.Fatalf("this shouldn't be included!") 201 | } 202 | } 203 | } 204 | 205 | // TestSkipOlder ensures that we can exclude items by age 206 | func TestSkipOlder(t *testing.T) { 207 | 208 | feed := configfile.Feed{ 209 | URL: "blah", 210 | Options: []configfile.Option{ 211 | {Name: "exclude-older", Value: "1"}, 212 | }, 213 | } 214 | 215 | // Create the new processor 216 | x, err := New() 217 | 218 | if err != nil { 219 | t.Fatalf("error creating processor %s", err.Error()) 220 | } 221 | defer x.Close() 222 | 223 | if x.shouldSkipOlder(logger, feed, "X") { 224 | t.Fatalf("failed to skip non correct published-date") 225 | } 226 | 227 | if !x.shouldSkipOlder(logger, feed, "Fri, 02 Dec 2022 16:43:04 +0000") { 228 | t.Fatalf("failed to skip old entry by age") 229 | } 230 | 231 | if !x.shouldSkipOlder(logger, feed, time.Now().Add(-time.Hour*24*2).Format(time.RFC1123)) { 232 | t.Fatalf("failed to skip newer entry by age") 233 | } 234 | 235 | if x.shouldSkipOlder(logger, feed, time.Now().Add(-time.Hour*12).Format(time.RFC1123)) { 236 | t.Fatalf("skipped new entry by age") 237 | } 238 | 239 | // With no options we're not going to skip 240 | feed = configfile.Feed{ 241 | URL: "blah", 242 | Options: []configfile.Option{}, 243 | } 244 | 245 | if x.shouldSkipOlder(logger, feed, time.Now().Add(-time.Hour*24*128).String()) { 246 | t.Fatalf("skipped age with no options!") 247 | } 248 | } 249 | -------------------------------------------------------------------------------- /seen_cmd.go: -------------------------------------------------------------------------------- 1 | // 2 | // Show feeds and their contents 3 | // 4 | 5 | package main 6 | 7 | import ( 8 | "fmt" 9 | "log/slog" 10 | "os" 11 | "path/filepath" 12 | 13 | "github.com/skx/rss2email/state" 14 | "github.com/skx/subcommands" 15 | "go.etcd.io/bbolt" 16 | ) 17 | 18 | // Structure for our options and state. 19 | type seenCmd struct { 20 | 21 | // We embed the NoFlags option, because we accept no command-line flags. 22 | subcommands.NoFlags 23 | } 24 | 25 | // Info is part of the subcommand-API. 26 | func (s *seenCmd) Info() (string, string) { 27 | return "seen", `Show all the feed-items we've seen. 28 | 29 | This sub-command will report upon all the feeds to which 30 | you're subscribed, and show the link to each feed-entry 31 | to which you've been notified in the past. 32 | 33 | (i.e. This walks the internal database which is used to 34 | store state, and outputs the list of recorded items which 35 | are no longer regarded as new/unseen.) 36 | ` 37 | } 38 | 39 | // Entry-point. 40 | func (s *seenCmd) Execute(args []string) int { 41 | 42 | // Ensure we have a state-directory. 43 | dir := state.Directory() 44 | errM := os.MkdirAll(dir, 0666) 45 | if errM != nil { 46 | logger.Error("failed to create directory", slog.String("directory", dir), slog.String("error", errM.Error())) 47 | return 1 48 | } 49 | 50 | // Now create the database, if missing, or open it if it exists. 51 | dbPath := filepath.Join(dir, "state.db") 52 | db, err := bbolt.Open(dbPath, 0666, nil) 53 | if err != nil { 54 | logger.Error("failed to open database", slog.String("database", dbPath), slog.String("error", err.Error())) 55 | return 1 56 | } 57 | 58 | // Ensure we close when we're done 59 | defer db.Close() 60 | 61 | // Keep track of buckets here 62 | var bucketNames [][]byte 63 | 64 | err = db.View(func(tx *bbolt.Tx) error { 65 | err = tx.ForEach(func(bucketName []byte, _ *bbolt.Bucket) error { 66 | bucketNames = append(bucketNames, bucketName) 67 | return nil 68 | }) 69 | return err 70 | }) 71 | if err != nil { 72 | logger.Error("failed to find bucket names", slog.String("database", dbPath), slog.String("error", err.Error())) 73 | return 1 74 | } 75 | 76 | // Now we have a list of buckets, we'll show the contents 77 | for _, buck := range bucketNames { 78 | 79 | fmt.Printf("%s\n", buck) 80 | 81 | err = db.View(func(tx *bbolt.Tx) error { 82 | 83 | // Select the bucket, which we know must exist 84 | b := tx.Bucket([]byte(buck)) 85 | 86 | // Get a cursor to the key=value entries in the bucket 87 | c := b.Cursor() 88 | 89 | // Iterate over the key/value pairs. 90 | for k, _ := c.First(); k != nil; k, _ = c.Next() { 91 | 92 | // Convert the key to a string 93 | key := string(k) 94 | 95 | fmt.Printf("\t%s\n", key) 96 | } 97 | 98 | return nil 99 | }) 100 | 101 | if err != nil { 102 | logger.Error("failed iterating over bucket", slog.String("database", dbPath), slog.String("bucket", string(buck)), slog.String("error", err.Error())) 103 | return 1 104 | } 105 | } 106 | 107 | return 0 108 | } 109 | -------------------------------------------------------------------------------- /state/state.go: -------------------------------------------------------------------------------- 1 | // Package state exists to provide a simple way of referring 2 | // to the directory beneath which we store state: 3 | // 4 | // 1. The location of the configuration-file. 5 | // 6 | // 2. The location of the BoltDB database. 7 | package state 8 | 9 | import ( 10 | "os" 11 | "os/user" 12 | "path/filepath" 13 | ) 14 | 15 | // Directory returns the path to a directory which can be used 16 | // for storing state. 17 | // 18 | // NOTE: This directory might not necessarily exist, we're just 19 | // returning the prefix directory that should/would be used for 20 | // persistent files. 21 | func Directory() string { 22 | 23 | // Default to using $HOME 24 | home := os.Getenv("HOME") 25 | 26 | if home == "" { 27 | // Get the current user, and use their home if possible. 28 | usr, err := user.Current() 29 | if err == nil { 30 | home = usr.HomeDir 31 | } 32 | } 33 | 34 | // Return the path 35 | return filepath.Join(home, ".rss2email") 36 | } 37 | -------------------------------------------------------------------------------- /template/template.go: -------------------------------------------------------------------------------- 1 | // Package template just holds our email-template. 2 | // 3 | // This is abstracted because we want to refer to it from our 4 | // processor-package, which is not in package-main, and also 5 | // the template-listing command. 6 | package template 7 | 8 | import ( 9 | _ "embed" // embedded-resource magic 10 | ) 11 | 12 | //go:embed template.txt 13 | var message string 14 | 15 | // EmailTemplate returns the embedded email template. 16 | func EmailTemplate() []byte { 17 | return []byte(message) 18 | } 19 | -------------------------------------------------------------------------------- /template/template.txt: -------------------------------------------------------------------------------- 1 | {{/* This is the default template which is used by default to generate emails. 2 | 3 | As you might imagine it is a Golang text/template file. 4 | 5 | Several fields and functions are available: 6 | 7 | {{.FeedTitle}} - The human-readable title of the source feed. 8 | {{.Feed}} - The URL of the feed from which the item came. 9 | {{.From}} - The email address which sends the email. 10 | {{.Link}} - The link to the new entry. 11 | {{.Subject}} - The subject of the new entry. 12 | {{.To}} - The recipient of the email. 13 | 14 | There is also access to the {{.RSSFeed}} and {{.RSSItem}} available, in 15 | case you need access to other fields which are not exported expliclty. 16 | Using that approach you can access {{.RSSItem.GUID}}, for example. 17 | 18 | The following functions are also available: 19 | 20 | {{env "USER"}} -> Return the given environmental variable 21 | {{quoteprintable .Link}} -> Quote the specified field. 22 | {{encodeHeader .Subject}} -> Quote the specified field to be used in mail header. 23 | {{split "STRING:HERE" ":"}} -> Split a string into an array by deliminator 24 | 25 | This comment will be stripped from the generated email. 26 | 27 | */ -}} 28 | Content-Type: multipart/mixed; boundary=21ee3da964c7bf70def62adb9ee1a061747003c026e363e47231258c48f1 29 | From: {{.From}} 30 | To: {{.To}} 31 | Subject: [rss2email] {{if .Tag}}{{encodeHeader .Tag}} {{end}}{{encodeHeader .Subject}} 32 | X-RSS-Link: {{.Link}} 33 | X-RSS-Feed: {{.Feed}} 34 | {{- if .Tag}} 35 | X-RSS-Tags: {{.Tag}} 36 | {{- end}} 37 | X-RSS-GUID: {{.RSSItem.GUID}} 38 | Content-Base: {{.Link}} 39 | Mime-Version: 1.0 40 | 41 | --21ee3da964c7bf70def62adb9ee1a061747003c026e363e47231258c48f1 42 | Content-Type: multipart/related; boundary=76a1282373c08a65dd49db1dea2c55111fda9a715c89720a844fabb7d497 43 | 44 | --76a1282373c08a65dd49db1dea2c55111fda9a715c89720a844fabb7d497 45 | Content-Type: multipart/alternative; boundary=4186c39e13b2140c88094b3933206336f2bb3948db7ecf064c7a7d7473f2 46 | 47 | --4186c39e13b2140c88094b3933206336f2bb3948db7ecf064c7a7d7473f2 48 | Content-Type: text/plain; charset=UTF-8 49 | Content-Transfer-Encoding: quoted-printable 50 | 51 | {{quoteprintable .Link}} 52 | 53 | {{.Text}} 54 | 55 | {{quoteprintable .Link}} 56 | --4186c39e13b2140c88094b3933206336f2bb3948db7ecf064c7a7d7473f2 57 | Content-Type: text/html; charset=UTF-8 58 | Content-Transfer-Encoding: quoted-printable 59 | 60 |

{{quoteprintable .Subject}}

61 | {{.HTML}} 62 |

{{quoteprintable .Subject}}

63 | --4186c39e13b2140c88094b3933206336f2bb3948db7ecf064c7a7d7473f2-- 64 | 65 | --76a1282373c08a65dd49db1dea2c55111fda9a715c89720a844fabb7d497-- 66 | --21ee3da964c7bf70def62adb9ee1a061747003c026e363e47231258c48f1-- 67 | -------------------------------------------------------------------------------- /template/template_test.go: -------------------------------------------------------------------------------- 1 | package template 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestTemplate(t *testing.T) { 8 | 9 | // content and expected length 10 | content := EmailTemplate() 11 | length := 2645 12 | 13 | if len(content) != length { 14 | t.Fatalf("unexpected template size %d != %d", length, len(content)) 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /unsee_cmd.go: -------------------------------------------------------------------------------- 1 | // 2 | // "Unsee" a feed item 3 | // 4 | 5 | package main 6 | 7 | import ( 8 | "flag" 9 | "fmt" 10 | "log/slog" 11 | "os" 12 | "path/filepath" 13 | "regexp" 14 | 15 | "github.com/skx/rss2email/state" 16 | "go.etcd.io/bbolt" 17 | ) 18 | 19 | // Structure for our options and state. 20 | type unseeCmd struct { 21 | 22 | // Are our arguments regular expressions? 23 | regexp bool 24 | } 25 | 26 | // Info is part of the subcommand-API. 27 | func (u *unseeCmd) Info() (string, string) { 28 | return "unsee", `Regard a feed item as new, and unseen. 29 | 30 | This sub-command will allow you to mark an item as 31 | unseen, or new, meaning the next time the cron or daemon 32 | commands run they'll trigger an email notification. 33 | 34 | You can see the URLs which we regard as having already seen 35 | via the 'seen' sub-command. 36 | ` 37 | } 38 | 39 | // Arguments handles our flag-setup. 40 | func (u *unseeCmd) Arguments(f *flag.FlagSet) { 41 | f.BoolVar(&u.regexp, "regexp", false, "Are our arguments regular expressions, instead of literal URLs?") 42 | } 43 | 44 | // Entry-point. 45 | func (u *unseeCmd) Execute(args []string) int { 46 | 47 | if len(args) < 1 { 48 | fmt.Printf("Please specify the URLs to unsee (i.e. remove from the state database).\n") 49 | return 1 50 | } 51 | 52 | // Ensure we have a state-directory. 53 | dir := state.Directory() 54 | errM := os.MkdirAll(dir, 0666) 55 | if errM != nil { 56 | logger.Error("failed to create directory", slog.String("directory", dir), slog.String("error", errM.Error())) 57 | return 1 58 | } 59 | 60 | // Now create the database, if missing, or open it if it exists. 61 | dbPath := filepath.Join(dir, "state.db") 62 | db, err := bbolt.Open(dbPath, 0666, nil) 63 | if err != nil { 64 | logger.Error("failed to open database", slog.String("database", dbPath), slog.String("error", err.Error())) 65 | return 1 66 | } 67 | 68 | // Ensure we close when we're done 69 | defer db.Close() 70 | 71 | // Keep track of buckets here 72 | var bucketNames []string 73 | 74 | // Record each bucket 75 | err = db.View(func(tx *bbolt.Tx) error { 76 | return tx.ForEach(func(bucketName []byte, _ *bbolt.Bucket) error { 77 | bucketNames = append(bucketNames, string(bucketName)) 78 | return nil 79 | }) 80 | }) 81 | 82 | if err != nil { 83 | logger.Error("failed to find bucket names", slog.String("database", dbPath), slog.String("error", err.Error())) 84 | return 1 85 | } 86 | 87 | // Process each bucket to find the item to remove. 88 | for _, buck := range bucketNames { 89 | 90 | err = db.Update(func(tx *bbolt.Tx) error { 91 | 92 | // Items to remove 93 | remove := []string{} 94 | 95 | // Select the bucket, which we know must exist 96 | b := tx.Bucket([]byte(buck)) 97 | 98 | // Get a cursor to the key=value entries in the bucket 99 | c := b.Cursor() 100 | 101 | // Iterate over the key/value pairs. 102 | for k, _ := c.First(); k != nil; k, _ = c.Next() { 103 | 104 | // Convert the key to a string 105 | key := string(k) 106 | 107 | // Is this something to remove? 108 | for _, arg := range args { 109 | 110 | // If so append it. 111 | if u.regexp { 112 | match, _ := regexp.MatchString(arg, key) 113 | if match { 114 | remove = append(remove, key) 115 | } 116 | } else { 117 | 118 | // Literal string-match 119 | if arg == key { 120 | remove = append(remove, key) 121 | 122 | logger.Debug("removed item from history", slog.String("item", key), slog.String("database", dbPath), slog.String("bucket", buck)) 123 | } 124 | } 125 | } 126 | } 127 | 128 | // Now remove 129 | for _, key := range remove { 130 | err = b.Delete([]byte(key)) 131 | if err != nil { 132 | logger.Debug("failed to remove item from history", slog.String("item", key), slog.String("database", dbPath), slog.String("bucket", buck), slog.String("error", err.Error())) 133 | 134 | } 135 | } 136 | return nil 137 | }) 138 | 139 | if err != nil { 140 | logger.Error("failed iterating over bucket", slog.String("database", dbPath), slog.String("bucket", buck), slog.String("error", err.Error())) 141 | return 1 142 | } 143 | } 144 | 145 | return 0 146 | } 147 | -------------------------------------------------------------------------------- /usage_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "github.com/skx/rss2email/configfile" 6 | "os" 7 | "testing" 8 | ) 9 | 10 | // TestUsage just calls the usage-function for each of our handlers, 11 | // and passes some bogus flags to the arguments handler. 12 | func TestUsage(t *testing.T) { 13 | 14 | add := addCmd{} 15 | add.Info() 16 | add.Arguments(flag.NewFlagSet("test", flag.ContinueOnError)) 17 | 18 | cron := cronCmd{} 19 | cron.Info() 20 | cron.Arguments(flag.NewFlagSet("test", flag.ContinueOnError)) 21 | 22 | config := configCmd{} 23 | config.Info() 24 | config.Arguments(flag.NewFlagSet("test", flag.ContinueOnError)) 25 | 26 | daemon := daemonCmd{} 27 | daemon.Info() 28 | daemon.Arguments(flag.NewFlagSet("test", flag.ContinueOnError)) 29 | 30 | del := delCmd{} 31 | del.Info() 32 | del.Arguments(flag.NewFlagSet("test", flag.ContinueOnError)) 33 | 34 | export := exportCmd{} 35 | export.Info() 36 | export.Arguments(flag.NewFlagSet("test", flag.ContinueOnError)) 37 | 38 | imprt := importCmd{} 39 | imprt.Info() 40 | imprt.Arguments(flag.NewFlagSet("test", flag.ContinueOnError)) 41 | 42 | list := listCmd{} 43 | list.Info() 44 | list.Arguments(flag.NewFlagSet("test", flag.ContinueOnError)) 45 | 46 | ldt := listDefaultTemplateCmd{} 47 | ldt.Info() 48 | ldt.Arguments(flag.NewFlagSet("test", flag.ContinueOnError)) 49 | 50 | seen := seenCmd{} 51 | seen.Info() 52 | seen.Arguments(flag.NewFlagSet("test", flag.ContinueOnError)) 53 | 54 | unse := unseeCmd{} 55 | unse.Info() 56 | unse.Arguments(flag.NewFlagSet("test", flag.ContinueOnError)) 57 | 58 | vers := versionCmd{} 59 | vers.Info() 60 | vers.Arguments(flag.NewFlagSet("test", flag.ContinueOnError)) 61 | } 62 | 63 | // TestBrokenConfig is used to test that the commands which assume 64 | // a broken configuration file do so. 65 | func TestBrokenConfig(t *testing.T) { 66 | 67 | data := []byte(`# This is bogus, options must follow URLs 68 | - foo:bar`) 69 | tmpfile, err := os.CreateTemp("", "example") 70 | if err != nil { 71 | t.Fatalf("Error creating temporary file") 72 | } 73 | 74 | if _, err := tmpfile.Write(data); err != nil { 75 | t.Fatalf("Error writing to config file") 76 | } 77 | if err := tmpfile.Close(); err != nil { 78 | t.Fatalf("Error creating temporary file") 79 | } 80 | 81 | // Test the various commands. 82 | a := addCmd{} 83 | a.config = configfile.NewWithPath(tmpfile.Name()) 84 | res := a.Execute([]string{}) 85 | if res != 1 { 86 | t.Fatalf("expected error with config file") 87 | } 88 | 89 | d := delCmd{} 90 | d.config = configfile.NewWithPath(tmpfile.Name()) 91 | res = d.Execute([]string{}) 92 | if res != 1 { 93 | t.Fatalf("expected error with config file") 94 | } 95 | 96 | e := exportCmd{} 97 | e.config = configfile.NewWithPath(tmpfile.Name()) 98 | res = e.Execute([]string{}) 99 | if res != 1 { 100 | t.Fatalf("expected error with config file") 101 | } 102 | 103 | i := importCmd{} 104 | i.config = configfile.NewWithPath(tmpfile.Name()) 105 | res = i.Execute([]string{}) 106 | if res != 1 { 107 | t.Fatalf("expected error with config file") 108 | } 109 | 110 | l := listCmd{} 111 | l.config = configfile.NewWithPath(tmpfile.Name()) 112 | res = l.Execute([]string{}) 113 | if res != 1 { 114 | t.Fatalf("expected error with config file") 115 | } 116 | 117 | // TODO : error-match 118 | 119 | os.Remove(tmpfile.Name()) 120 | } 121 | -------------------------------------------------------------------------------- /version_cmd.go: -------------------------------------------------------------------------------- 1 | // 2 | // Show our version. 3 | // 4 | 5 | package main 6 | 7 | import ( 8 | "flag" 9 | "fmt" 10 | "io" 11 | "os" 12 | "runtime" 13 | ) 14 | 15 | // 16 | // modified during testing 17 | // 18 | var out io.Writer = os.Stdout 19 | 20 | var ( 21 | version = "unreleased" 22 | ) 23 | 24 | // Structure for our options and state. 25 | type versionCmd struct { 26 | // verbose controls whether our version information includes 27 | // the go-version. 28 | verbose bool 29 | } 30 | 31 | // Info is part of the subcommand-API. 32 | func (v *versionCmd) Info() (string, string) { 33 | return "version", `Report upon our version, and exit.` 34 | } 35 | 36 | // Arguments handles our flag-setup. 37 | func (v *versionCmd) Arguments(f *flag.FlagSet) { 38 | f.BoolVar(&v.verbose, "verbose", false, "Show go version the binary was generated with.") 39 | } 40 | 41 | // 42 | // Show the version - using the "out"-writer. 43 | // 44 | func showVersion(verbose bool) { 45 | fmt.Fprintf(out, "%s\n", version) 46 | if verbose { 47 | fmt.Fprintf(out, "Built with %s\n", runtime.Version()) 48 | } 49 | } 50 | 51 | // 52 | // Entry-point. 53 | // 54 | func (v *versionCmd) Execute(args []string) int { 55 | 56 | showVersion(v.verbose) 57 | 58 | return 0 59 | } 60 | -------------------------------------------------------------------------------- /version_cmd_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "flag" 6 | "runtime" 7 | "testing" 8 | ) 9 | 10 | func TestVersion(t *testing.T) { 11 | bak := out 12 | out = &bytes.Buffer{} 13 | defer func() { out = bak }() 14 | 15 | // 16 | // Expected 17 | // 18 | expected := "unreleased\n" 19 | 20 | s := versionCmd{} 21 | 22 | // 23 | // Call the Arguments function for coverage. 24 | // 25 | flags := flag.NewFlagSet("test", flag.ContinueOnError) 26 | s.Arguments(flags) 27 | 28 | // 29 | // Call the handler. 30 | // 31 | s.Execute([]string{}) 32 | 33 | if out.(*bytes.Buffer).String() != expected { 34 | t.Errorf("Expected '%s' received '%s'", expected, out) 35 | } 36 | } 37 | 38 | func TestVersionVerbose(t *testing.T) { 39 | bak := out 40 | out = &bytes.Buffer{} 41 | defer func() { out = bak }() 42 | 43 | // 44 | // Expected 45 | // 46 | expected := "unreleased\nBuilt with " + runtime.Version() + "\n" 47 | 48 | s := versionCmd{verbose: true} 49 | 50 | // 51 | // Call the handler. 52 | // 53 | s.Execute([]string{}) 54 | 55 | if out.(*bytes.Buffer).String() != expected { 56 | t.Errorf("Expected '%s' received '%s'", expected, out) 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /withstate/feeditem.go: -------------------------------------------------------------------------------- 1 | // Package withstate provides a simple wrapper of the gofeed.Item, which 2 | // allows simple tracking of the seen vs. unseen (new vs. old) state of 3 | // an RSS feeds' entry. 4 | // 5 | // State for a feed-item is stored upon the local filesystem. 6 | package withstate 7 | 8 | import ( 9 | "fmt" 10 | "net/url" 11 | "strings" 12 | 13 | "github.com/PuerkitoBio/goquery" 14 | "github.com/mmcdole/gofeed" 15 | ) 16 | 17 | 18 | // FeedItem is a structure wrapping a gofeed.Item, to allow us to record 19 | // state. 20 | type FeedItem struct { 21 | 22 | // Wrapped structure 23 | *gofeed.Item 24 | 25 | // Tag is a field that can be set for this feed item, 26 | // inside our configuration file. 27 | Tag string 28 | } 29 | 30 | // RawContent provides content or fallback to description 31 | func (item *FeedItem) RawContent() string { 32 | // The body should be stored in the 33 | // "Content" field. 34 | content := item.Item.Content 35 | 36 | // If the Content field is empty then 37 | // use the Description instead, if it 38 | // is non-empty itself. 39 | if (content == "") && item.Item.Description != "" { 40 | content = item.Item.Description 41 | } 42 | 43 | return content 44 | } 45 | 46 | // HTMLContent provides processed HTML 47 | func (item *FeedItem) HTMLContent() (string, error) { 48 | rawContent := item.RawContent() 49 | 50 | doc, err := goquery.NewDocumentFromReader(strings.NewReader(rawContent)) 51 | if err != nil { 52 | return rawContent, err 53 | } 54 | doc.Find("a, img").Each(func(i int, e *goquery.Selection) { 55 | var attr string 56 | switch e.Get(0).Data { 57 | case "a": 58 | attr = "href" 59 | case "img": 60 | attr = "src" 61 | e.RemoveAttr("loading") 62 | e.RemoveAttr("srcset") 63 | } 64 | 65 | ref, _ := e.Attr(attr) 66 | switch { 67 | case ref == "": 68 | return 69 | case strings.HasPrefix(ref, "data:"): 70 | return 71 | case strings.HasPrefix(ref, "http://"): 72 | return 73 | case strings.HasPrefix(ref, "https://"): 74 | return 75 | default: 76 | e.SetAttr(attr, item.patchReference(ref)) 77 | } 78 | }) 79 | doc.Find("iframe").Each(func(i int, iframe *goquery.Selection) { 80 | src, _ := iframe.Attr("src") 81 | if src == "" { 82 | iframe.Remove() 83 | } else { 84 | iframe.ReplaceWithHtml(fmt.Sprintf(`%s`, src, src)) 85 | } 86 | }) 87 | doc.Find("script").Each(func(i int, script *goquery.Selection) { 88 | script.Remove() 89 | }) 90 | 91 | return doc.Html() 92 | } 93 | 94 | func (item *FeedItem) patchReference(ref string) string { 95 | resURL, err := url.Parse(ref) 96 | if err != nil { 97 | return ref 98 | } 99 | 100 | itemURL, err := url.Parse(item.Item.Link) 101 | if err != nil { 102 | return ref 103 | } 104 | 105 | if resURL.Host == "" { 106 | resURL.Host = itemURL.Host 107 | } 108 | if resURL.Scheme == "" { 109 | resURL.Scheme = itemURL.Scheme 110 | } 111 | 112 | return resURL.String() 113 | } 114 | --------------------------------------------------------------------------------