├── .dive-ci.yml ├── .dockerignore ├── .env.sample ├── .github ├── FUNDING.yml ├── dependabot.yml └── workflows │ ├── codeql.yml │ ├── dive-check.yml │ ├── docker-publish.yml │ ├── fly.yml │ ├── goreleaser-snapshot.yml │ ├── goreleaser.yml │ └── main.yml ├── .gitignore ├── .goreleaser.yaml ├── Dockerfile ├── Dockerfile.fly ├── Dockerfile.railwayapp ├── LICENSE ├── Makefile ├── README.md ├── SECURITY.md ├── cmd └── rsslay │ └── main.go ├── etc └── litefs.yml ├── fly.toml ├── go.mod ├── go.sum ├── internal └── handlers │ └── handlers.go ├── pkg ├── converter │ └── rules.go ├── custom_cache │ └── cache.go ├── events │ ├── events.go │ └── events_test.go ├── feed │ ├── feed.go │ ├── feed_test.go │ ├── translator.go │ └── translator_test.go ├── helpers │ ├── helpers.go │ └── helpers_test.go ├── metrics │ └── registries.go └── replayer │ └── replayer.go ├── screenshot.png ├── scripts ├── check_nitter_column.sql ├── create_nitter_column.sql ├── schema.sql └── scripts.go └── web ├── assets ├── images │ ├── favicon.ico │ └── logo.png └── js │ ├── copyclipboard.js │ └── nostr.js └── templates ├── created.html.tmpl ├── index.html.tmpl ├── search.html.tmpl └── templates.go /.dive-ci.yml: -------------------------------------------------------------------------------- 1 | rules: 2 | # If the efficiency is measured below X%, mark as failed. 3 | # Expressed as a ratio between 0-1. 4 | lowestEfficiency: 0.98 5 | 6 | # If the amount of wasted space is at least X or larger than X, mark as failed. 7 | # Expressed in B, KB, MB, and GB. 8 | highestWastedBytes: 5MB 9 | 10 | # If the amount of wasted space makes up for X% or more of the image, mark as failed. 11 | # Note: the base image layer is NOT included in the total image size. 12 | # Expressed as a ratio between 0-1; fails if the threshold is met or crossed. 13 | highestUserWastedPercent: 0.10 -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | # flyctl launch added from .gitignore 2 | **/relayer-rss-bridge 3 | **/db 4 | **/.idea 5 | 6 | # flyctl launch added from .idea/.gitignore 7 | # Default ignored files 8 | .idea/shelf 9 | .idea/workspace.xml 10 | # Editor-based HTTP Client requests 11 | .idea/httpRequests 12 | # Datasource local storage ignored files 13 | .idea/dataSources 14 | .idea/dataSources.local.xml 15 | fly.toml 16 | .vs 17 | *.sqlite 18 | coverage* 19 | rsslay 20 | -------------------------------------------------------------------------------- /.env.sample: -------------------------------------------------------------------------------- 1 | CGO_ENABLED=1 2 | GOARCH=amd64 3 | GOOS=linux 4 | PORT=8080 5 | DB_DIR="/db/rsslay.sqlite" 6 | DEFAULT_PROFILE_PICTURE_URL="https://i.imgur.com/MaceU96.png" 7 | SECRET="CHANGE_ME" 8 | VERSION=0.5.4 9 | REPLAY_TO_RELAYS=false 10 | RELAYS_TO_PUBLISH_TO="" 11 | NITTER_INSTANCES="" 12 | DEFAULT_WAIT_TIME_BETWEEN_BATCHES=60000 13 | DEFAULT_WAIT_TIME_FOR_RELAY_RESPONSE=3000 14 | MAX_EVENTS_TO_REPLAY=10 15 | ENABLE_AUTO_NIP05_REGISTRATION=false 16 | MAIN_DOMAIN_NAME="" 17 | OWNER_PUBLIC_KEY="" 18 | MAX_SUBROUTINES=20 19 | INFO_RELAY_NAME="rsslay" 20 | INFO_CONTACT="" 21 | MAX_CONTENT_LENGTH=250 22 | LOG_LEVEL=WARN 23 | DELETE_FAILING_FEEDS=false 24 | REDIS_CONNECTION_STRING="" -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | github: piraces 3 | ko_fi: piraces 4 | liberapay: piraces -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: gomod 4 | directory: "/" 5 | schedule: 6 | interval: daily 7 | time: "00:00" 8 | open-pull-requests-limit: 10 9 | 10 | - package-ecosystem: github-actions 11 | directory: "/.github/workflows" 12 | schedule: 13 | interval: daily 14 | time: "04:00" 15 | open-pull-requests-limit: 10 -------------------------------------------------------------------------------- /.github/workflows/codeql.yml: -------------------------------------------------------------------------------- 1 | name: "CodeQL" 2 | 3 | on: 4 | push: 5 | branches: [ "main" ] 6 | pull_request: 7 | branches: [ "main" ] 8 | schedule: 9 | - cron: '18 21 * * 1' 10 | 11 | jobs: 12 | analyze: 13 | name: Analyze 14 | runs-on: ubuntu-latest 15 | permissions: 16 | actions: read 17 | contents: read 18 | security-events: write 19 | 20 | strategy: 21 | fail-fast: false 22 | matrix: 23 | language: [ 'go' ] 24 | 25 | steps: 26 | - name: Checkout repository 27 | uses: actions/checkout@v4 28 | 29 | # Initializes the CodeQL tools for scanning. 30 | - name: Initialize CodeQL 31 | uses: github/codeql-action/init@v2 32 | with: 33 | languages: ${{ matrix.language }} 34 | # If you wish to specify custom queries, you can do so here or in a config file. 35 | # By default, queries listed here will override any specified in a config file. 36 | # Prefix the list here with "+" to use these queries and those in the config file. 37 | 38 | # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs 39 | # queries: security-extended,security-and-quality 40 | 41 | 42 | # Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java). 43 | # If this step fails, then you should remove it and run the build manually (see below) 44 | - name: Autobuild 45 | uses: github/codeql-action/autobuild@v2 46 | 47 | # ℹ️ Command-line programs to run using the OS shell. 48 | # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun 49 | 50 | # If the Autobuild fails above, remove it and uncomment the following three lines. 51 | # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. 52 | 53 | # - run: | 54 | # echo "Run, Build Application using script" 55 | # ./location_of_script_within_repo/buildscript.sh 56 | 57 | - name: Perform CodeQL Analysis 58 | uses: github/codeql-action/analyze@v2 59 | with: 60 | category: "/language:${{matrix.language}}" 61 | -------------------------------------------------------------------------------- /.github/workflows/dive-check.yml: -------------------------------------------------------------------------------- 1 | name: CI Dive Check 2 | on: 3 | push: 4 | branches: [ main ] 5 | pull_request: 6 | branches: [ main ] 7 | workflow_dispatch: 8 | 9 | jobs: 10 | check: 11 | runs-on: ubuntu-latest 12 | 13 | steps: 14 | - uses: actions/checkout@v4 15 | 16 | - name: Build image 17 | run: docker build . -t rsslay:temp 18 | - name: Dive 19 | run: docker run -e CI=true -e DOCKER_API_VERSION=1.37 --rm -v /var/run/docker.sock:/var/run/docker.sock --mount type=bind,source=/home/runner/work/rsslay/rsslay/.dive-ci.yml,target=/.dive-ci wagoodman/dive:latest rsslay:temp --ci-config /.dive-ci -------------------------------------------------------------------------------- /.github/workflows/docker-publish.yml: -------------------------------------------------------------------------------- 1 | name: Publish Docker image 2 | 3 | on: 4 | push: 5 | tags: 6 | - '*' 7 | workflow_dispatch: 8 | 9 | jobs: 10 | push_to_registries: 11 | name: Push Docker image to multiple registries 12 | runs-on: ubuntu-latest 13 | permissions: 14 | packages: write 15 | contents: read 16 | steps: 17 | - name: Check out the repo 18 | uses: actions/checkout@v4 19 | 20 | - name: Set up QEMU 21 | uses: docker/setup-qemu-action@v3 22 | 23 | - name: Set up Docker Buildx 24 | uses: docker/setup-buildx-action@v3 25 | 26 | - name: Log in to Docker Hub 27 | uses: docker/login-action@v3 28 | with: 29 | username: ${{ secrets.DOCKER_USERNAME }} 30 | password: ${{ secrets.DOCKER_PASSWORD }} 31 | 32 | - name: Log in to the Container registry 33 | uses: docker/login-action@v3 34 | with: 35 | registry: ghcr.io 36 | username: ${{ github.actor }} 37 | password: ${{ secrets.GITHUB_TOKEN }} 38 | 39 | - name: Extract metadata (tags, labels) for Docker 40 | id: meta 41 | uses: docker/metadata-action@v5 42 | with: 43 | images: | 44 | piraces/rsslay 45 | ghcr.io/piraces/rsslay 46 | 47 | - name: Build and push Docker images 48 | uses: docker/build-push-action@v5 49 | with: 50 | context: . 51 | push: true 52 | platforms: linux/amd64,linux/arm64/v8 53 | tags: ${{ steps.meta.outputs.tags }} 54 | labels: ${{ steps.meta.outputs.labels }} 55 | -------------------------------------------------------------------------------- /.github/workflows/fly.yml: -------------------------------------------------------------------------------- 1 | name: Fly Deploy 2 | on: 3 | push: 4 | branches: 5 | - main 6 | workflow_dispatch: 7 | jobs: 8 | deploy: 9 | name: Deploy app 10 | runs-on: ubuntu-latest 11 | env: 12 | FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }} 13 | steps: 14 | - uses: actions/checkout@v4 15 | - uses: superfly/flyctl-actions/setup-flyctl@master 16 | - run: flyctl deploy --remote-only --dockerfile Dockerfile.fly 17 | -------------------------------------------------------------------------------- /.github/workflows/goreleaser-snapshot.yml: -------------------------------------------------------------------------------- 1 | name: GoReleaser release (Snapshot) 2 | 3 | on: 4 | workflow_dispatch: 5 | 6 | permissions: 7 | contents: write 8 | packages: write 9 | issues: write 10 | 11 | jobs: 12 | goreleaser: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - uses: actions/checkout@v4 16 | with: 17 | fetch-depth: 0 18 | - run: git fetch --force --tags 19 | - uses: actions/setup-go@v5 20 | with: 21 | go-version: '>=1.21.0' 22 | cache: true 23 | - uses: goreleaser/goreleaser-action@v5 24 | with: 25 | distribution: goreleaser 26 | version: latest 27 | args: release --snapshot 28 | env: 29 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} -------------------------------------------------------------------------------- /.github/workflows/goreleaser.yml: -------------------------------------------------------------------------------- 1 | name: GoReleaser release 2 | 3 | on: 4 | push: 5 | tags: 6 | - '*' 7 | workflow_dispatch: 8 | 9 | permissions: 10 | contents: write 11 | packages: write 12 | issues: write 13 | 14 | jobs: 15 | goreleaser: 16 | runs-on: ubuntu-latest 17 | steps: 18 | - uses: actions/checkout@v4 19 | with: 20 | fetch-depth: 0 21 | - run: git fetch --force --tags 22 | - uses: actions/setup-go@v5 23 | with: 24 | go-version: '>=1.21.0' 25 | cache: true 26 | - uses: goreleaser/goreleaser-action@v5 27 | with: 28 | distribution: goreleaser 29 | version: latest 30 | args: release 31 | env: 32 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: CI Build & Test with coverage 2 | 3 | on: 4 | push: 5 | branches: [ "main" ] 6 | pull_request: 7 | branches: [ "main" ] 8 | 9 | jobs: 10 | 11 | build: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v4 15 | 16 | - name: Set up Go 17 | uses: actions/setup-go@v5 18 | with: 19 | go-version: 1.21.0 20 | 21 | - name: Build 22 | run: go build -v ./... 23 | 24 | - name: Test & Coverage 25 | run: go test -v ./... -race -covermode=atomic -coverprofile=coverage.out 26 | 27 | - name: Upload coverage reports to Codecov 28 | run: | 29 | curl -Os https://uploader.codecov.io/latest/linux/codecov 30 | chmod +x codecov 31 | ./codecov -t ${CODECOV_TOKEN} -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | relayer-rss-bridge 2 | db 3 | .idea 4 | .vs 5 | dist 6 | *.sqlite 7 | coverage* 8 | rsslay 9 | -------------------------------------------------------------------------------- /.goreleaser.yaml: -------------------------------------------------------------------------------- 1 | builds: 2 | - id: rsslay-linux 3 | main: ./cmd/rsslay 4 | ldflags: 5 | - -s -w -linkmode external -extldflags '-static' -X 'github.com/piraces/rsslay/pkg/version.BuildVersion={{.Version}}' 6 | env: 7 | - CGO_ENABLED=1 8 | goos: 9 | - linux 10 | ignore: 11 | - goos: linux 12 | goarch: 386 13 | - goos: linux 14 | goarch: arm64 15 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.21-alpine as build 2 | 3 | WORKDIR /app 4 | 5 | COPY go.mod ./ 6 | COPY go.sum ./ 7 | RUN go mod download 8 | 9 | COPY cmd/ ./cmd/ 10 | COPY internal/ ./internal/ 11 | COPY pkg/ ./pkg/ 12 | COPY web/ ./web/ 13 | COPY scripts/ ./scripts/ 14 | 15 | RUN apk add --no-cache build-base 16 | 17 | RUN CGO_ENABLED=1 go build -ldflags="-s -w -linkmode external -extldflags '-static'" -o /rsslay cmd/rsslay/main.go 18 | 19 | FROM alpine:latest 20 | 21 | LABEL org.opencontainers.image.title="rsslay" 22 | LABEL org.opencontainers.image.source=https://github.com/piraces/rsslay 23 | LABEL org.opencontainers.image.description="rsslay turns RSS or Atom feeds into Nostr profiles" 24 | LABEL org.opencontainers.image.authors="Raúl Piracés" 25 | LABEL org.opencontainers.image.licenses=MIT 26 | 27 | ENV PORT="8080" 28 | ENV DB_DIR="/db/rsslay.sqlite" 29 | ENV DEFAULT_PROFILE_PICTURE_URL="https://i.imgur.com/MaceU96.png" 30 | ENV SECRET="CHANGE_ME" 31 | ENV VERSION=0.5.4 32 | ENV REPLAY_TO_RELAYS=false 33 | ENV RELAYS_TO_PUBLISH_TO="" 34 | ENV NITTER_INSTANCES="" 35 | ENV DEFAULT_WAIT_TIME_BETWEEN_BATCHES=60000 36 | ENV DEFAULT_WAIT_TIME_FOR_RELAY_RESPONSE=3000 37 | ENV MAX_EVENTS_TO_REPLAY=10 38 | ENV ENABLE_AUTO_NIP05_REGISTRATION=false 39 | ENV MAIN_DOMAIN_NAME="" 40 | ENV OWNER_PUBLIC_KEY="" 41 | ENV MAX_SUBROUTINES=20 42 | ENV INFO_RELAY_NAME="rsslay" 43 | ENV INFO_CONTACT="" 44 | ENV MAX_CONTENT_LENGTH=250 45 | ENV LOG_LEVEL="WARN" 46 | ENV DELETE_FAILING_FEEDS=false 47 | ENV REDIS_CONNECTION_STRING="" 48 | 49 | COPY --from=build /rsslay . 50 | COPY --from=build /app/web/assets/ ./web/assets/ 51 | 52 | CMD [ "/rsslay" ] -------------------------------------------------------------------------------- /Dockerfile.fly: -------------------------------------------------------------------------------- 1 | # syntax=docker/dockerfile:1 2 | FROM flyio/litefs:0.4 AS litefs 3 | FROM golang:1.21-alpine as build 4 | 5 | WORKDIR /app 6 | 7 | COPY go.mod ./ 8 | COPY go.sum ./ 9 | RUN go mod download 10 | 11 | COPY cmd/ ./cmd/ 12 | COPY internal/ ./internal/ 13 | COPY pkg/ ./pkg/ 14 | COPY web/ ./web/ 15 | COPY scripts/ ./scripts/ 16 | 17 | RUN apk add --no-cache build-base 18 | 19 | RUN CGO_ENABLED=1 go build -ldflags="-s -w -linkmode external -extldflags '-static'" -tags osusergo,netgo -o /rsslay cmd/rsslay/main.go 20 | 21 | FROM alpine:latest 22 | 23 | LABEL org.opencontainers.image.title="rsslay" 24 | LABEL org.opencontainers.image.source=https://github.com/piraces/rsslay 25 | LABEL org.opencontainers.image.description="rsslay turns RSS or Atom feeds into Nostr profiles" 26 | LABEL org.opencontainers.image.authors="Raúl Piracés" 27 | LABEL org.opencontainers.image.licenses=MIT 28 | 29 | ENV PORT="8080" 30 | ENV DB_DIR="/db/rsslay.sqlite" 31 | ENV DEFAULT_PROFILE_PICTURE_URL="https://i.imgur.com/MaceU96.png" 32 | ENV SECRET="CHANGE_ME" 33 | ENV VERSION=0.5.4 34 | ENV REPLAY_TO_RELAYS=false 35 | ENV RELAYS_TO_PUBLISH_TO="" 36 | ENV NITTER_INSTANCES="" 37 | ENV DEFAULT_WAIT_TIME_BETWEEN_BATCHES=60000 38 | ENV DEFAULT_WAIT_TIME_FOR_RELAY_RESPONSE=3000 39 | ENV MAX_EVENTS_TO_REPLAY=10 40 | ENV ENABLE_AUTO_NIP05_REGISTRATION=false 41 | ENV MAIN_DOMAIN_NAME="" 42 | ENV OWNER_PUBLIC_KEY="" 43 | ENV MAX_SUBROUTINES=20 44 | ENV INFO_RELAY_NAME="rsslay" 45 | ENV INFO_CONTACT="" 46 | ENV MAX_CONTENT_LENGTH=250 47 | ENV LOG_LEVEL="WARN" 48 | ENV DELETE_FAILING_FEEDS=false 49 | ENV REDIS_CONNECTION_STRING="" 50 | 51 | COPY --from=litefs /usr/local/bin/litefs /usr/local/bin/litefs 52 | COPY --from=build /rsslay /usr/local/bin/rsslay 53 | COPY --from=build /app/web/assets/ ./web/assets/ 54 | 55 | ADD etc/litefs.yml /etc/litefs.yml 56 | 57 | RUN apk add bash fuse3 sqlite ca-certificates curl 58 | 59 | ENTRYPOINT litefs mount -- rsslay -dsn /litefs/rsslay -------------------------------------------------------------------------------- /Dockerfile.railwayapp: -------------------------------------------------------------------------------- 1 | FROM golang:1.21-alpine as build 2 | 3 | ARG PORT 4 | ARG DB_DIR 5 | ARG DEFAULT_PROFILE_PICTURE_URL 6 | ARG SECRET 7 | ARG REPLAY_TO_RELAYS 8 | ARG RELAYS_TO_PUBLISH_TO 9 | ARG NITTER_INSTANCES 10 | ARG DEFAULT_WAIT_TIME_BETWEEN_BATCHES 11 | ARG DEFAULT_WAIT_TIME_FOR_RELAY_RESPONSE 12 | ARG MAX_EVENTS_TO_REPLAY 13 | ARG ENABLE_AUTO_NIP05_REGISTRATION 14 | ARG MAIN_DOMAIN_NAME 15 | ARG OWNER_PUBLIC_KEY 16 | ARG MAX_SUBROUTINES 17 | ARG INFO_RELAY_NAME 18 | ARG INFO_CONTACT 19 | ARG MAX_CONTENT_LENGTH 20 | ARG LOG_LEVEL 21 | ARG DELETE_FAILING_FEEDS 22 | 23 | WORKDIR /app 24 | 25 | COPY go.mod ./ 26 | COPY go.sum ./ 27 | RUN go mod download 28 | 29 | COPY cmd/ ./cmd/ 30 | COPY internal/ ./internal/ 31 | COPY pkg/ ./pkg/ 32 | COPY web/ ./web/ 33 | COPY scripts/ ./scripts/ 34 | 35 | RUN apk add --no-cache build-base 36 | 37 | RUN CGO_ENABLED=1 go build -ldflags="-s -w -linkmode external -extldflags '-static'" -o /rsslay cmd/rsslay/main.go 38 | 39 | FROM alpine:latest 40 | 41 | ARG PORT 42 | ARG DB_DIR 43 | ARG DEFAULT_PROFILE_PICTURE_URL 44 | ARG SECRET 45 | ARG REPLAY_TO_RELAYS 46 | ARG RELAYS_TO_PUBLISH_TO 47 | ARG NITTER_INSTANCES 48 | ARG DEFAULT_WAIT_TIME_BETWEEN_BATCHES 49 | ARG DEFAULT_WAIT_TIME_FOR_RELAY_RESPONSE 50 | ARG MAX_EVENTS_TO_REPLAY 51 | ARG ENABLE_AUTO_NIP05_REGISTRATION 52 | ARG MAIN_DOMAIN_NAME 53 | ARG OWNER_PUBLIC_KEY 54 | ARG MAX_SUBROUTINES 55 | ARG INFO_RELAY_NAME 56 | ARG INFO_CONTACT 57 | ARG MAX_CONTENT_LENGTH 58 | ARG LOG_LEVEL 59 | ARG DELETE_FAILING_FEEDS 60 | ARG REDIS_CONNECTION_STRING 61 | 62 | LABEL org.opencontainers.image.title="rsslay" 63 | LABEL org.opencontainers.image.source=https://github.com/piraces/rsslay 64 | LABEL org.opencontainers.image.description="rsslay turns RSS or Atom feeds into Nostr profiles" 65 | LABEL org.opencontainers.image.authors="Raúl Piracés" 66 | LABEL org.opencontainers.image.licenses=MIT 67 | 68 | ENV PORT=$PORT 69 | ENV DB_DIR=$DB_DIR 70 | ENV DEFAULT_PROFILE_PICTURE_URL=$DEFAULT_PROFILE_PICTURE_URL 71 | ENV SECRET=$SECRET 72 | ENV VERSION=0.5.4 73 | ENV REPLAY_TO_RELAYS=false 74 | ENV RELAYS_TO_PUBLISH_TO="" 75 | ENV NITTER_INSTANCES="" 76 | ENV DEFAULT_WAIT_TIME_BETWEEN_BATCHES=60000 77 | ENV DEFAULT_WAIT_TIME_FOR_RELAY_RESPONSE=3000 78 | ENV MAX_EVENTS_TO_REPLAY=10 79 | ENV ENABLE_AUTO_NIP05_REGISTRATION="false" 80 | ENV MAIN_DOMAIN_NAME="" 81 | ENV OWNER_PUBLIC_KEY="" 82 | ENV MAX_SUBROUTINES=20 83 | ENV INFO_RELAY_NAME="rsslay" 84 | ENV INFO_CONTACT="" 85 | ENV MAX_CONTENT_LENGTH=250 86 | ENV LOG_LEVEL="WARN" 87 | ENV DELETE_FAILING_FEEDS=false 88 | ENV REDIS_CONNECTION_STRING=$REDIS_CONNECTION_STRING 89 | 90 | COPY --from=build /rsslay . 91 | COPY --from=build /app/web/assets/ ./web/assets/ 92 | 93 | CMD [ "/rsslay" ] -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | This is free and unencumbered software released into the public domain. 2 | 3 | Anyone is free to copy, modify, publish, use, compile, sell, or 4 | distribute this software, either in source code form or as a compiled 5 | binary, for any purpose, commercial or non-commercial, and by any 6 | means. 7 | 8 | In jurisdictions that recognize copyright laws, the author or authors 9 | of this software dedicate any and all copyright interest in the 10 | software to the public domain. We make this dedication for the benefit 11 | of the public at large and to the detriment of our heirs and 12 | successors. We intend this dedication to be an overt act of 13 | relinquishment in perpetuity of all present and future rights to this 14 | software under copyright law. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 19 | IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 | OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 | ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | OTHER DEALINGS IN THE SOFTWARE. 23 | 24 | For more information, please refer to -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | relayer-rss-bridge: $(shell find . -name "*.go") 2 | CC=$$(which musl-gcc) go build -ldflags="-s -w -linkmode external -extldflags '-static'" -o ./relayer-rss-bridge cmd/rsslay/main.go 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # rsslay 2 | 3 | [![CI Build & Test](https://github.com/piraces/rsslay/actions/workflows/main.yml/badge.svg)](https://github.com/piraces/rsslay/actions/workflows/main.yml) 4 | [![Fly Deploy](https://github.com/piraces/rsslay/actions/workflows/fly.yml/badge.svg)](https://github.com/piraces/rsslay/actions/workflows/fly.yml) 5 | [![CI Dive Check](https://github.com/piraces/rsslay/actions/workflows/dive-check.yml/badge.svg)](https://github.com/piraces/rsslay/actions/workflows/dive-check.yml) 6 | [![Publish Docker image](https://github.com/piraces/rsslay/actions/workflows/docker-publish.yml/badge.svg)](https://github.com/piraces/rsslay/actions/workflows/docker-publish.yml) 7 | [![codecov](https://codecov.io/gh/piraces/rsslay/branch/main/graph/badge.svg?token=tNKcOjlxLo)](https://codecov.io/gh/piraces/rsslay) 8 | 9 | ![Docker Hub](https://img.shields.io/docker/pulls/piraces/rsslay?logo=docker) 10 | [![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fpiraces%2Frsslay.svg?type=shield)](https://app.fossa.com/projects/git%2Bgithub.com%2Fpiraces%2Frsslay?ref=badge_shield) 11 | 12 | [![Go Report Card](https://goreportcard.com/badge/github.com/piraces/rsslay)](https://goreportcard.com/report/github.com/piraces/rsslay) 13 | [![Go Reference](https://pkg.go.dev/badge/github.com/piraces/rsslay.svg)](https://pkg.go.dev/github.com/piraces/rsslay) 14 | 15 | **Nostr relay that creates virtual nostr profiles for each RSS feed submitted** based on [relayer](https://github.com/fiatjaf/relayer/) by [fiatjaf](https://fiatjaf.com). 16 | 17 | **Donate for development ⚡:** [https://getalby.com/p/piraces](https://getalby.com/p/piraces) 18 | 19 | **Working relay: `wss://rsslay.nostr.moe`. Frontend available in [rsslay.nostr.moe](https://rsslay.nostr.moe).** 20 | 21 | 22 | [![Deploy on Railway](https://railway.app/button.svg)](https://railway.app/template/UDf6vC?referralCode=Zbo_gO) 23 | 24 | ![Screenshot of main page](screenshot.png) 25 | 26 | ## API 27 | 28 | `rsslay` exposes an API to work with it programmatically, so you can automate feed creation and retrieval. 29 | Checkout the [wiki entry](https://github.com/piraces/rsslay/wiki/API) for further info. 30 | 31 | ## Mirroring events ("replaying") 32 | 33 | _**Note:** since v0.5.3 its recommended to set `REPLAY_TO_RELAYS` to false. There is no need to perform replays to other relays, the main rsslay should be able to handle the events._ 34 | 35 | Actually `rsslay` makes usage of a method named `AttemptReplayEvents` which is made to send the events to other relays of confidence to attempt to make the events and the profile more reachable (they are just mirror relays)... 36 | 37 | Currently used relays: none. 38 | 39 | ## Feeds from Twitter via Nitter instances 40 | 41 | The [Nitter](https://github.com/zedeus/nitter) project is well integrated into `rsslay` and it performs special handling of this kind of feeds. 42 | Currently `rsslay` is doing the following with Nitter feeds: 43 | - Upgrading to `https` all instances that may be misconfigured. 44 | - Format some responses and retweets in a more "user-friendly" way. 45 | - As there are many instances available out there, if one is unreachable at the moment of parsing, a pool of instances is used (configurable): 46 | - [birdsite.xanny.family](https://birdsite.xanny.family/) 47 | - [notabird.site](https://notabird.site/) 48 | - [nitter.moomoo.me](https://nitter.moomoo.me/) 49 | - [nitter.fly.dev](https://nitter.fly.dev/) 50 | 51 | ## Running the project 52 | 53 | Running `rsslay` its easy, checkout [the wiki entry for it](https://github.com/piraces/rsslay/wiki/Running-the-project). 54 | 55 | ## Deploying your instance 56 | 57 | If you want to run your own instance, you are covered! 58 | Several options (including "one-click" ones) are available. 59 | Checkout [the wiki](https://github.com/piraces/rsslay/wiki/Deploy-your-own-instance). 60 | 61 | ## Caching 62 | 63 | Since version v0.5.1, rsslay uses cache by default (in-memory with [BigCache](https://github.com/allegro/bigcache) by default or with [Redis](https://redis.io/) if configured) enabled by default to improve performance. 64 | In the case of the main instance `rsslay.nostr.moe`, Redis is used in HA mode to improve performance for multiple requests for the same feed. 65 | 66 | **Nevertheless, there is a caveat using this approach that is that cached feeds do not refresh for 30 minutes (but I personally think it is worth for the performance gain).** 67 | 68 | ## Metrics 69 | 70 | Since version v0.5.1, rsslay uses [Prometheus](https://prometheus.io/) instrumenting with metrics exposed on `/metrics` path. 71 | So with this you can mount your own [Graphana](https://grafana.com/) dashboards and look into rsslay insights! 72 | 73 | # Related projects 74 | 75 | - **atomstr** by [@psic4t](https://github.com/psic4t): atomstr is a RSS/Atom gateway to Nostr. It fetches all sorts of RSS or Atom feeds, generates Nostr profiles for each and posts new entries to given Nostr relay(s). If you have one of these relays in your profile, you can find and subscribe to the feeds. 76 | - Source code: https://sr.ht/~psic4t/atomstr/ 77 | - Demo instance: https://atomstr.data.haus/ 78 | 79 | 80 | # Contributing 81 | 82 | Feel free to [open an issue](https://github.com/piraces/rsslay/issues/new), provide feedback in [discussions](https://github.com/piraces/rsslay/discussions), or fork the repo and open a PR with your contribution! 83 | 84 | **All kinds of contributions are welcome!** 85 | 86 | # Contact 87 | 88 | You can reach me on nostr [`npub1ftpy6thgy2354xypal6jd0m37wtsgsvcxljvzje5vskc9cg3a5usexrrtq`](https://snort.social/p/npub1ftpy6thgy2354xypal6jd0m37wtsgsvcxljvzje5vskc9cg3a5usexrrtq) 89 | 90 | Also on [the bird site](https://twitter.com/piraces_), and [Mastodon](https://hachyderm.io/@piraces). 91 | 92 | # License 93 | 94 | [Unlicense](https://unlicense.org). 95 | 96 | [![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fpiraces%2Frsslay.svg?type=large)](https://app.fossa.com/projects/git%2Bgithub.com%2Fpiraces%2Frsslay?ref=badge_large) 97 | 98 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | Security Policy for `rsslay`. 4 | Please refer to this document regarding security concerns, issues or vulnerabilities that can be disclosed. 5 | 6 | ## Supported Versions 7 | 8 | Only the latest minor version is currently being supported with security updates. 9 | 10 | As of now the following versions are covered with security updates: 11 | 12 | | Version | Supported | 13 | |---------| ------------------ | 14 | | 0.5.x | :white_check_mark: | 15 | | < 0.5.0 | :x: | 16 | 17 | ## Reporting a Vulnerability 18 | 19 | If you found a vulnerability, please report it to my mail raul[at]piraces.dev. 20 | You can sign the message using my PGP Key always updated at [KeyOxide](https://keys.openpgp.org/pks/lookup?op=get&options=mr&search=raul@piraces.dev). 21 | You can also [encrypt the message in the form that KeyOxide provides](https://keyoxide.org/hkp/raul%40piraces.dev). 22 | 23 | When contacting back, you can check my signature in [KeyOxide](https://keyoxide.org/hkp/raul%40piraces.dev) too. 24 | 25 | After receiving the corresponding report, the security issue should be addressed in two weeks or less. 26 | Either the vulnerability is accepted or declined it will be always welcomed, explained and addressed. 27 | 28 | If the vulnerability is accepted a public issue will be raised to track the vulnerability (depending the case, it may be after the vulnerability is solved with a patch). 29 | 30 | Thank you. 31 | -------------------------------------------------------------------------------- /cmd/rsslay/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "database/sql" 6 | _ "embed" 7 | "errors" 8 | "flag" 9 | "fmt" 10 | "github.com/eko/gocache/lib/v4/cache" 11 | "github.com/fiatjaf/relayer" 12 | _ "github.com/fiatjaf/relayer" 13 | "github.com/hashicorp/logutils" 14 | "github.com/hellofresh/health-go/v5" 15 | "github.com/kelseyhightower/envconfig" 16 | _ "github.com/mattn/go-sqlite3" 17 | "github.com/nbd-wtf/go-nostr" 18 | "github.com/nbd-wtf/go-nostr/nip11" 19 | "github.com/piraces/rsslay/internal/handlers" 20 | "github.com/piraces/rsslay/pkg/custom_cache" 21 | "github.com/piraces/rsslay/pkg/events" 22 | "github.com/piraces/rsslay/pkg/feed" 23 | "github.com/piraces/rsslay/pkg/metrics" 24 | "github.com/piraces/rsslay/pkg/replayer" 25 | "github.com/piraces/rsslay/scripts" 26 | "github.com/prometheus/client_golang/prometheus/promhttp" 27 | "golang.org/x/exp/slices" 28 | "log" 29 | "net/http" 30 | "os" 31 | "path" 32 | "sync" 33 | "time" 34 | ) 35 | 36 | // Command line flags. 37 | var ( 38 | dsn = flag.String("dsn", "", "datasource name") 39 | ) 40 | 41 | const assetsDir = "/assets/" 42 | 43 | type Relay struct { 44 | Secret string `envconfig:"SECRET" required:"true"` 45 | DatabaseDirectory string `envconfig:"DB_DIR" default:"db/rsslay.sqlite"` 46 | DefaultProfilePictureUrl string `envconfig:"DEFAULT_PROFILE_PICTURE_URL" default:"https://i.imgur.com/MaceU96.png"` 47 | Version string `envconfig:"VERSION" default:"unknown"` 48 | ReplayToRelays bool `envconfig:"REPLAY_TO_RELAYS" default:"false"` 49 | RelaysToPublish []string `envconfig:"RELAYS_TO_PUBLISH_TO" default:""` 50 | NitterInstances []string `envconfig:"NITTER_INSTANCES" default:""` 51 | DefaultWaitTimeBetweenBatches int64 `envconfig:"DEFAULT_WAIT_TIME_BETWEEN_BATCHES" default:"60000"` 52 | DefaultWaitTimeForRelayResponse int64 `envconfig:"DEFAULT_WAIT_TIME_FOR_RELAY_RESPONSE" default:"3000"` 53 | MaxEventsToReplay int `envconfig:"MAX_EVENTS_TO_REPLAY" default:"20"` 54 | EnableAutoNIP05Registration bool `envconfig:"ENABLE_AUTO_NIP05_REGISTRATION" default:"false"` 55 | MainDomainName string `envconfig:"MAIN_DOMAIN_NAME" default:""` 56 | OwnerPublicKey string `envconfig:"OWNER_PUBLIC_KEY" default:""` 57 | MaxSubroutines int `envconfig:"MAX_SUBROUTINES" default:"20"` 58 | RelayName string `envconfig:"INFO_RELAY_NAME" default:"rsslay"` 59 | Contact string `envconfig:"INFO_CONTACT" default:"~"` 60 | MaxContentLength int `envconfig:"MAX_CONTENT_LENGTH" default:"250"` 61 | DeleteFailingFeeds bool `envconfig:"DELETE_FAILING_FEEDS" default:"false"` 62 | RedisConnectionString string `envconfig:"REDIS_CONNECTION_STRING" default:""` 63 | 64 | updates chan nostr.Event 65 | lastEmitted sync.Map 66 | db *sql.DB 67 | healthCheck *health.Health 68 | mutex sync.Mutex 69 | routineQueueLength int 70 | cache *cache.Cache[string] 71 | } 72 | 73 | var relayInstance = &Relay{ 74 | updates: make(chan nostr.Event), 75 | } 76 | 77 | func CreateHealthCheck() { 78 | h, _ := health.New(health.WithComponent(health.Component{ 79 | Name: "rsslay", 80 | Version: os.Getenv("VERSION"), 81 | }), health.WithChecks(health.Config{ 82 | Name: "self", 83 | Timeout: time.Second * 5, 84 | SkipOnErr: false, 85 | Check: func(ctx context.Context) error { 86 | return nil 87 | }, 88 | }, 89 | )) 90 | relayInstance.healthCheck = h 91 | } 92 | 93 | func ConfigureLogging() { 94 | filter := &logutils.LevelFilter{ 95 | Levels: []logutils.LogLevel{"DEBUG", "INFO", "WARN", "ERROR", "FATAL"}, 96 | MinLevel: logutils.LogLevel(os.Getenv("LOG_LEVEL")), 97 | Writer: os.Stderr, 98 | } 99 | log.SetOutput(filter) 100 | } 101 | 102 | func ConfigureCache() { 103 | if relayInstance.RedisConnectionString != "" { 104 | custom_cache.RedisConnectionString = &relayInstance.RedisConnectionString 105 | } 106 | custom_cache.InitializeCache() 107 | } 108 | 109 | func (r *Relay) Name() string { 110 | return r.RelayName 111 | } 112 | 113 | func (r *Relay) OnInitialized(s *relayer.Server) { 114 | s.Router().Path("/").HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { 115 | handlers.HandleWebpage(writer, request, r.db, &r.MainDomainName) 116 | }) 117 | s.Router().Path("/create").HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { 118 | handlers.HandleCreateFeed(writer, request, r.db, &r.Secret, dsn) 119 | }) 120 | s.Router().Path("/search").HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { 121 | handlers.HandleSearch(writer, request, r.db) 122 | }) 123 | s.Router(). 124 | PathPrefix(assetsDir). 125 | Handler(http.StripPrefix(assetsDir, http.FileServer(http.Dir("./web/"+assetsDir)))) 126 | s.Router().Path("/healthz").HandlerFunc(relayInstance.healthCheck.HandlerFunc) 127 | s.Router().Path("/api/feed").HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { 128 | handlers.HandleApiFeed(writer, request, r.db, &r.Secret, dsn) 129 | }) 130 | s.Router().Path("/.well-known/nostr.json").HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { 131 | handlers.HandleNip05(writer, request, r.db, &r.OwnerPublicKey, &r.EnableAutoNIP05Registration) 132 | }) 133 | s.Router().Path("/metrics").Handler(promhttp.Handler()) 134 | } 135 | 136 | func (r *Relay) Init() error { 137 | flag.Parse() 138 | err := envconfig.Process("", r) 139 | if err != nil { 140 | return fmt.Errorf("couldn't process envconfig: %w", err) 141 | } else { 142 | log.Printf("[INFO] Running VERSION %s:\n - DSN=%s\n - DB_DIR=%s\n\n", r.Version, *dsn, r.DatabaseDirectory) 143 | } 144 | 145 | ConfigureCache() 146 | r.db = InitDatabase(r) 147 | 148 | go r.UpdateListeningFilters() 149 | 150 | return nil 151 | } 152 | 153 | func (r *Relay) UpdateListeningFilters() { 154 | for { 155 | time.Sleep(20 * time.Minute) 156 | metrics.ListeningFiltersOps.Inc() 157 | 158 | filters := relayer.GetListeningFilters() 159 | log.Printf("[DEBUG] Checking for updates; %d filters active", len(filters)) 160 | 161 | var parsedEvents []replayer.EventWithPrivateKey 162 | for _, filter := range filters { 163 | if filter.Kinds == nil || slices.Contains(filter.Kinds, nostr.KindTextNote) { 164 | for _, pubkey := range filter.Authors { 165 | parsedFeed, entity := events.GetParsedFeedForPubKey(pubkey, r.db, r.DeleteFailingFeeds, r.NitterInstances) 166 | if parsedFeed == nil { 167 | continue 168 | } 169 | 170 | for _, item := range parsedFeed.Items { 171 | defaultCreatedAt := time.Unix(time.Now().Unix(), 0) 172 | evt := feed.ItemToTextNote(pubkey, item, parsedFeed, defaultCreatedAt, entity.URL, relayInstance.MaxContentLength) 173 | last, ok := r.lastEmitted.Load(entity.URL) 174 | if last == nil { 175 | last = uint32(time.Now().Unix()) 176 | } 177 | if !ok || nostr.Timestamp(int64(last.(uint32))) < evt.CreatedAt { 178 | _ = evt.Sign(entity.PrivateKey) 179 | r.updates <- evt 180 | r.lastEmitted.Store(entity.URL, last.(uint32)) 181 | parsedEvents = append(parsedEvents, replayer.EventWithPrivateKey{Event: &evt, PrivateKey: entity.PrivateKey}) 182 | } 183 | } 184 | } 185 | } 186 | } 187 | r.AttemptReplayEvents(parsedEvents) 188 | } 189 | } 190 | 191 | func (r *Relay) AttemptReplayEvents(events []replayer.EventWithPrivateKey) { 192 | if relayInstance.ReplayToRelays && relayInstance.routineQueueLength < relayInstance.MaxSubroutines && len(events) > 0 { 193 | r.routineQueueLength++ 194 | metrics.ReplayRoutineQueueLength.Set(float64(r.routineQueueLength)) 195 | replayer.ReplayEventsToRelays(&replayer.ReplayParameters{ 196 | MaxEventsToReplay: relayInstance.MaxEventsToReplay, 197 | RelaysToPublish: relayInstance.RelaysToPublish, 198 | Mutex: &relayInstance.mutex, 199 | Queue: &relayInstance.routineQueueLength, 200 | WaitTime: relayInstance.DefaultWaitTimeBetweenBatches, 201 | WaitTimeForRelayResponse: relayInstance.DefaultWaitTimeForRelayResponse, 202 | Events: events, 203 | }) 204 | } 205 | } 206 | 207 | func (r *Relay) AcceptEvent(_ *nostr.Event) bool { 208 | metrics.InvalidEventsRequests.Inc() 209 | return false 210 | } 211 | 212 | func (r *Relay) Storage() relayer.Storage { 213 | return store{r.db} 214 | } 215 | 216 | type store struct { 217 | db *sql.DB 218 | } 219 | 220 | func (b store) Init() error { return nil } 221 | func (b store) SaveEvent(_ *nostr.Event) error { 222 | metrics.InvalidEventsRequests.Inc() 223 | return errors.New("blocked: we don't accept any events") 224 | } 225 | 226 | func (b store) DeleteEvent(_, _ string) error { 227 | metrics.InvalidEventsRequests.Inc() 228 | return errors.New("blocked: we can't delete any events") 229 | } 230 | 231 | func (b store) QueryEvents(filter *nostr.Filter) ([]nostr.Event, error) { 232 | var parsedEvents []nostr.Event 233 | var eventsToReplay []replayer.EventWithPrivateKey 234 | 235 | metrics.QueryEventsRequests.Inc() 236 | 237 | if filter.IDs != nil || len(filter.Tags) > 0 { 238 | return parsedEvents, nil 239 | } 240 | 241 | for _, pubkey := range filter.Authors { 242 | parsedFeed, entity := events.GetParsedFeedForPubKey(pubkey, relayInstance.db, relayInstance.DeleteFailingFeeds, relayInstance.NitterInstances) 243 | 244 | if parsedFeed == nil { 245 | continue 246 | } 247 | 248 | if filter.Kinds == nil || slices.Contains(filter.Kinds, nostr.KindSetMetadata) { 249 | evt := feed.EntryFeedToSetMetadata(pubkey, parsedFeed, entity.URL, relayInstance.EnableAutoNIP05Registration, relayInstance.DefaultProfilePictureUrl, relayInstance.MainDomainName) 250 | 251 | if filter.Since != nil && evt.CreatedAt < *filter.Since { 252 | continue 253 | } 254 | if filter.Until != nil && evt.CreatedAt > *filter.Until { 255 | continue 256 | } 257 | 258 | _ = evt.Sign(entity.PrivateKey) 259 | parsedEvents = append(parsedEvents, evt) 260 | if relayInstance.ReplayToRelays { 261 | eventsToReplay = append(eventsToReplay, replayer.EventWithPrivateKey{Event: &evt, PrivateKey: entity.PrivateKey}) 262 | } 263 | } 264 | 265 | if filter.Kinds == nil || slices.Contains(filter.Kinds, nostr.KindTextNote) { 266 | var last uint32 = 0 267 | for _, item := range parsedFeed.Items { 268 | defaultCreatedAt := time.Unix(time.Now().Unix(), 0) 269 | evt := feed.ItemToTextNote(pubkey, item, parsedFeed, defaultCreatedAt, entity.URL, relayInstance.MaxContentLength) 270 | 271 | // Feed need to have a date for each entry... 272 | if evt.CreatedAt == nostr.Timestamp(defaultCreatedAt.Unix()) { 273 | continue 274 | } 275 | 276 | if filter.Since != nil && evt.CreatedAt < *filter.Since { 277 | continue 278 | } 279 | if filter.Until != nil && evt.CreatedAt > *filter.Until { 280 | continue 281 | } 282 | 283 | _ = evt.Sign(entity.PrivateKey) 284 | 285 | if evt.CreatedAt > nostr.Timestamp(int64(last)) { 286 | last = uint32(evt.CreatedAt) 287 | } 288 | 289 | parsedEvents = append(parsedEvents, evt) 290 | if relayInstance.ReplayToRelays { 291 | eventsToReplay = append(eventsToReplay, replayer.EventWithPrivateKey{Event: &evt, PrivateKey: entity.PrivateKey}) 292 | } 293 | } 294 | 295 | relayInstance.lastEmitted.Store(entity.URL, last) 296 | } 297 | } 298 | 299 | relayInstance.AttemptReplayEvents(eventsToReplay) 300 | 301 | return parsedEvents, nil 302 | } 303 | 304 | func (r *Relay) InjectEvents() chan nostr.Event { 305 | return r.updates 306 | } 307 | 308 | func (r *Relay) GetNIP11InformationDocument() nip11.RelayInformationDocument { 309 | metrics.RelayInfoRequests.Inc() 310 | infoDocument := nip11.RelayInformationDocument{ 311 | Name: relayInstance.Name(), 312 | Description: "Relay that creates virtual nostr profiles for each RSS feed submitted, powered by the relayer framework", 313 | PubKey: relayInstance.OwnerPublicKey, 314 | Contact: relayInstance.Contact, 315 | SupportedNIPs: []int{5, 9, 11, 12, 15, 16, 19, 20}, 316 | Software: "git+https://github.com/piraces/rsslay.git", 317 | Version: relayInstance.Version, 318 | } 319 | 320 | if relayInstance.OwnerPublicKey == "" { 321 | infoDocument.PubKey = "~" 322 | } 323 | 324 | return infoDocument 325 | } 326 | 327 | func main() { 328 | CreateHealthCheck() 329 | ConfigureLogging() 330 | defer func(db *sql.DB) { 331 | err := db.Close() 332 | if err != nil { 333 | log.Fatalf("[FATAL] failed to close the database connection: %v", err) 334 | } 335 | }(relayInstance.db) 336 | 337 | if err := relayer.Start(relayInstance); err != nil { 338 | log.Fatalf("[FATAL] server terminated: %v", err) 339 | } 340 | } 341 | 342 | func InitDatabase(r *Relay) *sql.DB { 343 | finalConnection := dsn 344 | if *dsn == "" { 345 | log.Print("[INFO] dsn required is not present... defaulting to DB_DIR") 346 | finalConnection = &r.DatabaseDirectory 347 | } 348 | 349 | // Create empty dir if not exists 350 | dbPath := path.Dir(*finalConnection) 351 | err := os.MkdirAll(dbPath, 0660) 352 | if err != nil { 353 | log.Printf("[INFO] unable to initialize DB_DIR at: %s. Error: %v", dbPath, err) 354 | } 355 | 356 | // Connect to SQLite database. 357 | sqlDb, err := sql.Open("sqlite3", *finalConnection) 358 | if err != nil { 359 | log.Fatalf("[FATAL] open db: %v", err) 360 | } 361 | 362 | log.Printf("[INFO] database opened at %s", *finalConnection) 363 | 364 | // Run migrations 365 | if _, err := sqlDb.Exec(scripts.SchemaSQL); err != nil { 366 | log.Fatalf("[FATAL] cannot migrate schema: %v", err) 367 | } 368 | 369 | if _, err := sqlDb.Exec(scripts.CheckNitterColumnSQL); err != nil { 370 | _, err := sqlDb.Exec(scripts.CreateNitterColumnSQL) 371 | if err != nil { 372 | log.Fatalf("[FATAL] cannot migrate schema from previous versions: %v", err) 373 | } 374 | } 375 | 376 | return sqlDb 377 | } 378 | -------------------------------------------------------------------------------- /etc/litefs.yml: -------------------------------------------------------------------------------- 1 | # The fuse section describes settings for the FUSE file system. This file system 2 | # is used as a thin layer between the SQLite client in your application and the 3 | # storage on disk. It intercepts disk writes to determine transaction boundaries 4 | # so that those transactions can be saved and shipped to replicas. 5 | fuse: 6 | dir: "/litefs" 7 | 8 | # The data section describes settings for the internal LiteFS storage. We'll 9 | # mount a volume to the data directory so it can be persisted across restarts. 10 | # However, this data should not be accessed directly by the user application. 11 | data: 12 | dir: "/var/lib/litefs" 13 | 14 | # This flag ensure that LiteFS continues to run if there is an issue on starup. 15 | # It makes it easy to ssh in and debug any issues you might be having rather 16 | # than continually restarting on initialization failure. 17 | exit-on-error: true 18 | 19 | # The lease section specifies how the cluster will be managed. We're using the 20 | # "consul" lease type so that our application can dynamically change the primary. 21 | # 22 | # These environment variables will be available in your Fly.io application. 23 | # You must specify "experiement.enable_consul" for FLY_CONSUL_URL to be available. 24 | lease: 25 | type: "consul" 26 | candidate: ${FLY_REGION == PRIMARY_REGION} 27 | promote: true 28 | advertise-url: "http://${FLY_ALLOC_ID}.vm.${FLY_APP_NAME}.internal:20202" 29 | 30 | consul: 31 | url: "${FLY_CONSUL_URL}" 32 | key: "litefs/${FLY_APP_NAME}" 33 | -------------------------------------------------------------------------------- /fly.toml: -------------------------------------------------------------------------------- 1 | app = "rsslay" 2 | primary_region = "yyz" 3 | kill_signal = "SIGINT" 4 | kill_timeout = 5 5 | 6 | [experimental] 7 | enable_consul = true 8 | 9 | [metrics] 10 | port = 8080 11 | path = "/metrics" 12 | 13 | [env] 14 | DB_DIR = "/var/lib/litefs/db" 15 | DEFAULT_PROFILE_PICTURE_URL = "https://i.imgur.com/MaceU96.png" 16 | DEFAULT_WAIT_TIME_BETWEEN_BATCHES = "60000" 17 | DEFAULT_WAIT_TIME_FOR_RELAY_RESPONSE = "1000" 18 | DELETE_FAILING_FEEDS = "false" 19 | ENABLE_AUTO_NIP05_REGISTRATION = "true" 20 | INFO_CONTACT = "mailto:raul@piraces.dev" 21 | INFO_RELAY_NAME = "rsslay public instance" 22 | LOG_LEVEL = "INFO" 23 | MAIN_DOMAIN_NAME = "rsslay.nostr.moe" 24 | MAX_CONTENT_LENGTH = "5000" 25 | MAX_EVENTS_TO_REPLAY = "10" 26 | MAX_SUBROUTINES = "20" 27 | NITTER_INSTANCES = "birdsite.xanny.family,notabird.site,nitter.moomoo.me,nitter.fly.dev" 28 | OWNER_PUBLIC_KEY = "4ac24d2ee822a34a9881eff526bf71f39704419837e4c14b34642d82e111ed39" 29 | PORT = "8080" 30 | RELAYS_TO_PUBLISH_TO = "" 31 | REPLAY_TO_RELAYS = "false" 32 | VERSION = "0.5.4" 33 | 34 | [[mounts]] 35 | source = "rsslay_data_machines" 36 | destination = "/var/lib/litefs" 37 | processes = ["app"] 38 | 39 | [[services]] 40 | protocol = "tcp" 41 | internal_port = 8080 42 | 43 | [[services.ports]] 44 | port = 80 45 | handlers = ["http"] 46 | force_https = true 47 | 48 | [[services.ports]] 49 | port = 443 50 | handlers = ["tls", "http"] 51 | [services.concurrency] 52 | type = "connections" 53 | hard_limit = 250 54 | soft_limit = 100 55 | 56 | [[services.http_checks]] 57 | interval = "10s" 58 | timeout = "2s" 59 | grace_period = "5s" 60 | restart_limit = 0 61 | method = "get" 62 | path = "/healthz" 63 | protocol = "http" 64 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/piraces/rsslay 2 | 3 | go 1.21 4 | 5 | require ( 6 | github.com/DATA-DOG/go-sqlmock v1.5.0 7 | github.com/JohannesKaufmann/html-to-markdown v1.4.2 8 | github.com/PuerkitoBio/goquery v1.8.1 9 | github.com/allegro/bigcache v1.2.1 10 | github.com/eko/gocache/lib/v4 v4.1.5 11 | github.com/eko/gocache/store/bigcache/v4 v4.2.1 12 | github.com/eko/gocache/store/redis/v4 v4.2.1 13 | github.com/fiatjaf/relayer v1.7.3 14 | github.com/hashicorp/logutils v1.0.0 15 | github.com/hellofresh/health-go/v5 v5.5.1 16 | github.com/kelseyhightower/envconfig v1.4.0 17 | github.com/mattn/go-sqlite3 v1.14.18 18 | github.com/microcosm-cc/bluemonday v1.0.26 19 | github.com/mmcdole/gofeed v1.2.1 20 | github.com/nbd-wtf/go-nostr v0.24.2 21 | github.com/prometheus/client_golang v1.17.0 22 | github.com/redis/go-redis/v9 v9.3.0 23 | github.com/stretchr/testify v1.8.4 24 | golang.org/x/exp v0.0.0-20230809150735-7b3493d9a819 25 | ) 26 | 27 | require ( 28 | github.com/andybalholm/cascadia v1.3.2 // indirect 29 | github.com/aymerick/douceur v0.2.0 // indirect 30 | github.com/beorn7/perks v1.0.1 // indirect 31 | github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect 32 | github.com/btcsuite/btcd/btcutil v1.1.3 // indirect 33 | github.com/btcsuite/btcd/chaincfg/chainhash v1.0.2 // indirect 34 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 35 | github.com/davecgh/go-spew v1.1.1 // indirect 36 | github.com/decred/dcrd/crypto/blake256 v1.0.0 // indirect 37 | github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect 38 | github.com/dgraph-io/ristretto v0.1.1 // indirect 39 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect 40 | github.com/dustin/go-humanize v1.0.0 // indirect 41 | github.com/gobwas/httphead v0.1.0 // indirect 42 | github.com/gobwas/pool v0.2.1 // indirect 43 | github.com/gobwas/ws v1.2.0 // indirect 44 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect 45 | github.com/golang/mock v1.6.0 // indirect 46 | github.com/golang/protobuf v1.5.3 // indirect 47 | github.com/gorilla/css v1.0.0 // indirect 48 | github.com/gorilla/mux v1.8.0 // indirect 49 | github.com/gorilla/websocket v1.5.0 // indirect 50 | github.com/josharian/intern v1.0.0 // indirect 51 | github.com/json-iterator/go v1.1.12 // indirect 52 | github.com/mailru/easyjson v0.7.7 // indirect 53 | github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect 54 | github.com/mmcdole/goxpp v1.1.0 // indirect 55 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 56 | github.com/modern-go/reflect2 v1.0.2 // indirect 57 | github.com/pkg/errors v0.9.1 // indirect 58 | github.com/pmezard/go-difflib v1.0.0 // indirect 59 | github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect 60 | github.com/prometheus/common v0.44.0 // indirect 61 | github.com/prometheus/procfs v0.11.1 // indirect 62 | github.com/puzpuzpuz/xsync/v2 v2.5.0 // indirect 63 | github.com/rs/cors v1.7.0 // indirect 64 | github.com/tidwall/gjson v1.14.4 // indirect 65 | github.com/tidwall/match v1.1.1 // indirect 66 | github.com/tidwall/pretty v1.2.0 // indirect 67 | go.opentelemetry.io/otel v1.19.0 // indirect 68 | go.opentelemetry.io/otel/trace v1.19.0 // indirect 69 | golang.org/x/net v0.18.0 // indirect 70 | golang.org/x/sys v0.14.0 // indirect 71 | golang.org/x/text v0.14.0 // indirect 72 | google.golang.org/protobuf v1.31.0 // indirect 73 | gopkg.in/yaml.v3 v3.0.1 // indirect 74 | ) 75 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= 2 | github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= 3 | github.com/JohannesKaufmann/html-to-markdown v1.4.2 h1:Jt3i/2l98+yOb5uD0ovoIGwccF4DfNxBeUye4P5KP9g= 4 | github.com/JohannesKaufmann/html-to-markdown v1.4.2/go.mod h1:AwPLQeuGhVGKyWXJR8t46vR0iL1d3yGuembj8c1VcJU= 5 | github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= 6 | github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= 7 | github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= 8 | github.com/allegro/bigcache v1.2.1 h1:hg1sY1raCwic3Vnsvje6TT7/pnZba83LeFck5NrFKSc= 9 | github.com/allegro/bigcache v1.2.1/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= 10 | github.com/allegro/bigcache/v3 v3.1.0 h1:H2Vp8VOvxcrB91o86fUSVJFqeuz8kpyyB02eH3bSzwk= 11 | github.com/allegro/bigcache/v3 v3.1.0/go.mod h1:aPyh7jEvrog9zAwx5N7+JUQX5dZTSGpxF1LAR4dr35I= 12 | github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= 13 | github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= 14 | github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= 15 | github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= 16 | github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= 17 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 18 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 19 | github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= 20 | github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= 21 | github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= 22 | github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= 23 | github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= 24 | github.com/btcsuite/btcd v0.22.0-beta.0.20220111032746-97732e52810c/go.mod h1:tjmYdS6MLJ5/s0Fj4DbLgSbDHbEqLJrtnHecBFkdz5M= 25 | github.com/btcsuite/btcd v0.23.0/go.mod h1:0QJIIN1wwIXF/3G/m87gIwGniDMDQqjVn4SZgnFpsYY= 26 | github.com/btcsuite/btcd/btcec/v2 v2.1.0/go.mod h1:2VzYrv4Gm4apmbVVsSq5bqf1Ec8v56E48Vt0Y/umPgA= 27 | github.com/btcsuite/btcd/btcec/v2 v2.1.3/go.mod h1:ctjw4H1kknNJmRN4iP1R7bTQ+v3GJkZBd6mui8ZsAZE= 28 | github.com/btcsuite/btcd/btcec/v2 v2.3.2 h1:5n0X6hX0Zk+6omWcihdYvdAlGf2DfasC0GMf7DClJ3U= 29 | github.com/btcsuite/btcd/btcec/v2 v2.3.2/go.mod h1:zYzJ8etWJQIv1Ogk7OzpWjowwOdXY1W/17j2MW85J04= 30 | github.com/btcsuite/btcd/btcutil v1.0.0/go.mod h1:Uoxwv0pqYWhD//tfTiipkxNfdhG9UrLwaeswfjfdF0A= 31 | github.com/btcsuite/btcd/btcutil v1.1.0/go.mod h1:5OapHB7A2hBBWLm48mmw4MOHNJCcUBTwmWH/0Jn8VHE= 32 | github.com/btcsuite/btcd/btcutil v1.1.3 h1:xfbtw8lwpp0G6NwSHb+UE67ryTFHJAiNuipusjXSohQ= 33 | github.com/btcsuite/btcd/btcutil v1.1.3/go.mod h1:UR7dsSJzJUfMmFiiLlIrMq1lS9jh9EdCV7FStZSnpi0= 34 | github.com/btcsuite/btcd/chaincfg/chainhash v1.0.0/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= 35 | github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= 36 | github.com/btcsuite/btcd/chaincfg/chainhash v1.0.2 h1:KdUfX2zKommPRa+PD0sWZUyXe9w277ABlgELO7H04IM= 37 | github.com/btcsuite/btcd/chaincfg/chainhash v1.0.2/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= 38 | github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= 39 | github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= 40 | github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= 41 | github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= 42 | github.com/btcsuite/goleveldb v1.0.0/go.mod h1:QiK9vBlgftBg6rWQIj6wFzbPfRjiykIEhBH4obrXJ/I= 43 | github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= 44 | github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= 45 | github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= 46 | github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= 47 | github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 48 | github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= 49 | github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 50 | github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 51 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 52 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 53 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 54 | github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0= 55 | github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= 56 | github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= 57 | github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1owhMVTHFZIlnvd4= 58 | github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2UO1+lbSKsdiOoYi9Zzey7Fc= 59 | github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= 60 | github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= 61 | github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= 62 | github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= 63 | github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= 64 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= 65 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= 66 | github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= 67 | github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= 68 | github.com/eko/gocache/lib/v4 v4.1.5 h1:CeMQmdIzwBKKLRjk3FCDXzNFsQTyqJ01JLI7Ib0C9r8= 69 | github.com/eko/gocache/lib/v4 v4.1.5/go.mod h1:XaNfCwW8KYW1bRZ/KoHA1TugnnkMz0/gT51NDIu7LSY= 70 | github.com/eko/gocache/store/bigcache/v4 v4.2.1 h1:xf9R5HZqmrfT4+NzlJPQJQUWftfWW06FHbjz4IEjE08= 71 | github.com/eko/gocache/store/bigcache/v4 v4.2.1/go.mod h1:Q9+hxUE+XUVGSRGP1tqW8sPHcZ50PfyBVh9VKh0OjrA= 72 | github.com/eko/gocache/store/redis/v4 v4.2.1 h1:uPAgZIn7knH6a55tO4ETN9V93VD3Rcyx0ZIyozEqC0I= 73 | github.com/eko/gocache/store/redis/v4 v4.2.1/go.mod h1:JoLkNA5yeGNQUwINAM9529cDNQCo88WwiKlO9e/+39I= 74 | github.com/fiatjaf/relayer v1.7.3 h1:HbE67EFsabLO4bvbuB3uMRfRizsvDxseCIaeQHQstv8= 75 | github.com/fiatjaf/relayer v1.7.3/go.mod h1:cQGM8YSoU/7I79Mg9ULlLQWYm/U54/B/4k60fRXEY2o= 76 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 77 | github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= 78 | github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= 79 | github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= 80 | github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= 81 | github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= 82 | github.com/gobwas/ws v1.2.0 h1:u0p9s3xLYpZCA1z5JgCkMeB34CKCMMQbM+G8Ii7YD0I= 83 | github.com/gobwas/ws v1.2.0/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY= 84 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= 85 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 86 | github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= 87 | github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= 88 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 89 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= 90 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= 91 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= 92 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= 93 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= 94 | github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 95 | github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= 96 | github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= 97 | github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= 98 | github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 99 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 100 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 101 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 102 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 103 | github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= 104 | github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 105 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 106 | github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= 107 | github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= 108 | github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= 109 | github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= 110 | github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= 111 | github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= 112 | github.com/hashicorp/logutils v1.0.0 h1:dLEQVugN8vlakKOUE3ihGLTZJRB4j+M2cdTm/ORI65Y= 113 | github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= 114 | github.com/hellofresh/health-go/v5 v5.5.1 h1:QVGiq5Amv73HMlVh/KgrB3OqxZ+wSUpZKiOrIs5mNco= 115 | github.com/hellofresh/health-go/v5 v5.5.1/go.mod h1:GutnYy+rj2sZsyArSSt5VhfHP3FH8tiZ2tpKLOY+/0M= 116 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 117 | github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= 118 | github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= 119 | github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= 120 | github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= 121 | github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= 122 | github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= 123 | github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= 124 | github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= 125 | github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= 126 | github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= 127 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 128 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 129 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 130 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 131 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 132 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 133 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 134 | github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= 135 | github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= 136 | github.com/mattn/go-sqlite3 v1.14.18 h1:JL0eqdCOq6DJVNPSvArO/bIV9/P7fbGrV00LZHc+5aI= 137 | github.com/mattn/go-sqlite3 v1.14.18/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= 138 | github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= 139 | github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= 140 | github.com/microcosm-cc/bluemonday v1.0.26 h1:xbqSvqzQMeEHCqMi64VAs4d8uy6Mequs3rQ0k/Khz58= 141 | github.com/microcosm-cc/bluemonday v1.0.26/go.mod h1:JyzOCs9gkyQyjs+6h10UEVSe02CGwkhd72Xdqh78TWs= 142 | github.com/mmcdole/gofeed v1.2.1 h1:tPbFN+mfOLcM1kDF1x2c/N68ChbdBatkppdzf/vDe1s= 143 | github.com/mmcdole/gofeed v1.2.1/go.mod h1:2wVInNpgmC85q16QTTuwbuKxtKkHLCDDtf0dCmnrNr4= 144 | github.com/mmcdole/goxpp v1.1.0 h1:WwslZNF7KNAXTFuzRtn/OKZxFLJAAyOA9w82mDz2ZGI= 145 | github.com/mmcdole/goxpp v1.1.0/go.mod h1:v+25+lT2ViuQ7mVxcncQ8ch1URund48oH+jhjiwEgS8= 146 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 147 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 148 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 149 | github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= 150 | github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= 151 | github.com/nbd-wtf/go-nostr v0.24.2 h1:1PdFED7uHh3BlXfDVD96npBc0YAgj9hPT+l6NWog4kc= 152 | github.com/nbd-wtf/go-nostr v0.24.2/go.mod h1:eE8Qf8QszZbCd9arBQyotXqATNUElWsTEEx+LLORhyQ= 153 | github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= 154 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 155 | github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 156 | github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= 157 | github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= 158 | github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= 159 | github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= 160 | github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= 161 | github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= 162 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 163 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 164 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 165 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 166 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 167 | github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= 168 | github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= 169 | github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 h1:v7DLqVdK4VrYkVD5diGdl4sxJurKJEMnODWRJlxV9oM= 170 | github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= 171 | github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= 172 | github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= 173 | github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= 174 | github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= 175 | github.com/puzpuzpuz/xsync/v2 v2.5.0 h1:2k4qrO/orvmEXZ3hmtHqIy9XaQtPTwzMZk1+iErpE8c= 176 | github.com/puzpuzpuz/xsync/v2 v2.5.0/go.mod h1:gD2H2krq/w52MfPLE+Uy64TzJDVY7lP2znR9qmR35kU= 177 | github.com/redis/go-redis/v9 v9.3.0 h1:RiVDjmig62jIWp7Kk4XVLs0hzV6pI3PyTnnL0cnn0u0= 178 | github.com/redis/go-redis/v9 v9.3.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= 179 | github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= 180 | github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= 181 | github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= 182 | github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= 183 | github.com/sebdah/goldie/v2 v2.5.3 h1:9ES/mNN+HNUbNWpVAlrzuZ7jE+Nrczbj8uFRjM7624Y= 184 | github.com/sebdah/goldie/v2 v2.5.3/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI= 185 | github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= 186 | github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= 187 | github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= 188 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 189 | github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= 190 | github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= 191 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 192 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 193 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 194 | github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= 195 | github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= 196 | github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= 197 | github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= 198 | github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= 199 | github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= 200 | github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= 201 | github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= 202 | github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= 203 | github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= 204 | github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= 205 | github.com/yuin/goldmark v1.6.0 h1:boZcn2GTjpsynOsC0iJHnBWa4Bi0qzfJjthwauItG68= 206 | github.com/yuin/goldmark v1.6.0/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= 207 | go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs= 208 | go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY= 209 | go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1Dzxpg= 210 | go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo= 211 | golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 212 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 213 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 214 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 215 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 216 | golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= 217 | golang.org/x/exp v0.0.0-20230809150735-7b3493d9a819 h1:EDuYyU/MkFXllv9QF9819VlI9a4tzGuCbhG0ExK9o1U= 218 | golang.org/x/exp v0.0.0-20230809150735-7b3493d9a819/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= 219 | golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 220 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= 221 | golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= 222 | golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 223 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 224 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 225 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 226 | golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 227 | golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= 228 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 229 | golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= 230 | golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= 231 | golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= 232 | golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= 233 | golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= 234 | golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= 235 | golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= 236 | golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= 237 | golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= 238 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 239 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 240 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 241 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 242 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 243 | golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 244 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 245 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 246 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 247 | golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 248 | golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 249 | golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 250 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 251 | golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 252 | golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 253 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 254 | golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 255 | golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 256 | golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 257 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 258 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 259 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 260 | golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 261 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 262 | golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 263 | golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 264 | golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 265 | golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= 266 | golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 267 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 268 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 269 | golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= 270 | golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= 271 | golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= 272 | golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww= 273 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 274 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 275 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 276 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 277 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 278 | golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= 279 | golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= 280 | golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= 281 | golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= 282 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 283 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 284 | golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= 285 | golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= 286 | golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= 287 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 288 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 289 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 290 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 291 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= 292 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= 293 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= 294 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= 295 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= 296 | google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 297 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= 298 | google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= 299 | google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= 300 | google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= 301 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 302 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 303 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 304 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 305 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 306 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 307 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 308 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 309 | gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 310 | gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 311 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 312 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 313 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 314 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 315 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 316 | -------------------------------------------------------------------------------- /internal/handlers/handlers.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | "database/sql" 5 | "encoding/json" 6 | _ "github.com/mattn/go-sqlite3" 7 | "github.com/nbd-wtf/go-nostr" 8 | "github.com/nbd-wtf/go-nostr/nip05" 9 | "github.com/nbd-wtf/go-nostr/nip19" 10 | "github.com/piraces/rsslay/pkg/feed" 11 | "github.com/piraces/rsslay/pkg/helpers" 12 | "github.com/piraces/rsslay/pkg/metrics" 13 | "github.com/piraces/rsslay/web/templates" 14 | "github.com/prometheus/client_golang/prometheus" 15 | "html/template" 16 | "log" 17 | "net/http" 18 | "net/url" 19 | "os" 20 | "path/filepath" 21 | "strings" 22 | ) 23 | 24 | var t = template.Must(template.ParseFS(templates.Templates, "*.tmpl")) 25 | 26 | type Entry struct { 27 | PubKey string 28 | NPubKey string 29 | Url string 30 | Error bool 31 | ErrorMessage string 32 | ErrorCode int 33 | } 34 | 35 | type PageData struct { 36 | Count uint64 37 | FilteredCount uint64 38 | Entries []Entry 39 | MainDomainName string 40 | } 41 | 42 | func HandleWebpage(w http.ResponseWriter, r *http.Request, db *sql.DB, mainDomainName *string) { 43 | mustRedirect := handleOtherRegion(w, r) 44 | if mustRedirect { 45 | return 46 | } 47 | 48 | metrics.IndexRequests.Inc() 49 | var count uint64 50 | row := db.QueryRow(`SELECT count(*) FROM feeds`) 51 | err := row.Scan(&count) 52 | if err != nil { 53 | http.Error(w, err.Error(), http.StatusInternalServerError) 54 | return 55 | } 56 | 57 | var items []Entry 58 | rows, err := db.Query(`SELECT publickey, url FROM feeds ORDER BY RANDOM() LIMIT 50`) 59 | if err != nil { 60 | http.Error(w, err.Error(), http.StatusInternalServerError) 61 | return 62 | } 63 | 64 | for rows.Next() { 65 | var entry Entry 66 | if err := rows.Scan(&entry.PubKey, &entry.Url); err != nil { 67 | log.Printf("[ERROR] failed to scan row iterating feeds: %v", err) 68 | metrics.AppErrors.With(prometheus.Labels{"type": "SQL_SCAN"}).Inc() 69 | continue 70 | } 71 | 72 | entry.NPubKey, _ = nip19.EncodePublicKey(entry.PubKey) 73 | items = append(items, entry) 74 | } 75 | if err := rows.Close(); err != nil { 76 | http.Error(w, err.Error(), http.StatusInternalServerError) 77 | return 78 | } 79 | 80 | data := PageData{ 81 | Count: count, 82 | Entries: items, 83 | MainDomainName: *mainDomainName, 84 | } 85 | 86 | _ = t.ExecuteTemplate(w, "index.html.tmpl", data) 87 | } 88 | 89 | func HandleSearch(w http.ResponseWriter, r *http.Request, db *sql.DB) { 90 | mustRedirect := handleOtherRegion(w, r) 91 | if mustRedirect { 92 | return 93 | } 94 | 95 | metrics.SearchRequests.Inc() 96 | query := r.URL.Query().Get("query") 97 | if query == "" || len(query) <= 4 { 98 | http.Error(w, "Please enter more than 5 characters to search", 400) 99 | return 100 | } 101 | 102 | var count uint64 103 | row := db.QueryRow(`SELECT count(*) FROM feeds`) 104 | err := row.Scan(&count) 105 | if err != nil { 106 | http.Error(w, err.Error(), http.StatusInternalServerError) 107 | return 108 | } 109 | 110 | var items []Entry 111 | rows, err := db.Query(`SELECT publickey, url FROM feeds WHERE url like '%' || $1 || '%' LIMIT 50`, query) 112 | if err != nil { 113 | http.Error(w, err.Error(), http.StatusInternalServerError) 114 | return 115 | } 116 | 117 | for rows.Next() { 118 | var entry Entry 119 | if err := rows.Scan(&entry.PubKey, &entry.Url); err != nil { 120 | log.Printf("[ERROR] failed to scan row iterating feeds searching: %v", err) 121 | metrics.AppErrors.With(prometheus.Labels{"type": "SQL_SCAN"}).Inc() 122 | continue 123 | } 124 | 125 | entry.NPubKey, _ = nip19.EncodePublicKey(entry.PubKey) 126 | items = append(items, entry) 127 | } 128 | if err := rows.Close(); err != nil { 129 | http.Error(w, err.Error(), http.StatusInternalServerError) 130 | return 131 | } 132 | 133 | data := PageData{ 134 | Count: count, 135 | FilteredCount: uint64(len(items)), 136 | Entries: items, 137 | } 138 | 139 | _ = t.ExecuteTemplate(w, "search.html.tmpl", data) 140 | } 141 | 142 | func HandleCreateFeed(w http.ResponseWriter, r *http.Request, db *sql.DB, secret *string, dsn *string) { 143 | mustRedirect := handleRedirectToPrimaryNode(w, dsn) 144 | if mustRedirect { 145 | return 146 | } 147 | 148 | metrics.CreateRequests.Inc() 149 | entry := createFeedEntry(r, db, secret) 150 | _ = t.ExecuteTemplate(w, "created.html.tmpl", entry) 151 | } 152 | 153 | func HandleApiFeed(w http.ResponseWriter, r *http.Request, db *sql.DB, secret *string, dsn *string) { 154 | if r.Method == http.MethodGet || r.Method == http.MethodPost { 155 | handleCreateFeedEntry(w, r, db, secret, dsn) 156 | } else { 157 | http.Error(w, "Method not supported", http.StatusMethodNotAllowed) 158 | } 159 | } 160 | 161 | func HandleNip05(w http.ResponseWriter, r *http.Request, db *sql.DB, ownerPubKey *string, enableAutoRegistration *bool) { 162 | metrics.WellKnownRequests.Inc() 163 | name := r.URL.Query().Get("name") 164 | name, _ = url.QueryUnescape(name) 165 | w.Header().Set("Content-Type", "application/json") 166 | nip05WellKnownResponse := nip05.WellKnownResponse{ 167 | Names: map[string]string{ 168 | "_": *ownerPubKey, 169 | }, 170 | Relays: nil, 171 | } 172 | 173 | var response []byte 174 | if name != "" && name != "_" && *enableAutoRegistration { 175 | row := db.QueryRow("SELECT publickey FROM feeds WHERE url like '%' || $1 || '%'", name) 176 | 177 | var entity feed.Entity 178 | err := row.Scan(&entity.PublicKey) 179 | if err == nil { 180 | nip05WellKnownResponse = nip05.WellKnownResponse{ 181 | Names: map[string]string{ 182 | name: entity.PublicKey, 183 | }, 184 | Relays: nil, 185 | } 186 | } 187 | } 188 | 189 | response, _ = json.Marshal(nip05WellKnownResponse) 190 | _, _ = w.Write(response) 191 | } 192 | 193 | func handleCreateFeedEntry(w http.ResponseWriter, r *http.Request, db *sql.DB, secret *string, dsn *string) { 194 | mustRedirect := handleRedirectToPrimaryNode(w, dsn) 195 | if mustRedirect { 196 | return 197 | } 198 | 199 | metrics.CreateRequestsAPI.Inc() 200 | entry := createFeedEntry(r, db, secret) 201 | w.Header().Set("Content-Type", "application/json") 202 | 203 | if entry.ErrorCode >= 400 { 204 | w.WriteHeader(entry.ErrorCode) 205 | } else { 206 | w.WriteHeader(http.StatusOK) 207 | } 208 | 209 | response, _ := json.Marshal(entry) 210 | _, _ = w.Write(response) 211 | } 212 | 213 | func handleOtherRegion(w http.ResponseWriter, r *http.Request) bool { 214 | // If a different region is specified, redirect to that region. 215 | if region := r.URL.Query().Get("region"); region != "" && region != os.Getenv("FLY_REGION") { 216 | log.Printf("[DEBUG] redirecting from %q to %q", os.Getenv("FLY_REGION"), region) 217 | w.Header().Set("fly-replay", "region="+region) 218 | return true 219 | } 220 | return false 221 | } 222 | 223 | func handleRedirectToPrimaryNode(w http.ResponseWriter, dsn *string) bool { 224 | // If this node is not primary, look up and redirect to the current primary. 225 | primaryFilename := filepath.Join(filepath.Dir(*dsn), ".primary") 226 | primary, err := os.ReadFile(primaryFilename) 227 | if err != nil && !os.IsNotExist(err) { 228 | http.Error(w, err.Error(), http.StatusInternalServerError) 229 | return true 230 | } 231 | if string(primary) != "" { 232 | log.Printf("[DEBUG] redirecting to primary instance: %q", string(primary)) 233 | w.Header().Set("fly-replay", "instance="+string(primary)) 234 | return true 235 | } 236 | 237 | return false 238 | } 239 | 240 | func createFeedEntry(r *http.Request, db *sql.DB, secret *string) *Entry { 241 | urlParam := r.URL.Query().Get("url") 242 | entry := Entry{ 243 | Error: false, 244 | } 245 | 246 | if !helpers.IsValidHttpUrl(urlParam) { 247 | log.Printf("[DEBUG] tried to create feed from invalid feed url '%q' skipping...", urlParam) 248 | entry.ErrorCode = http.StatusBadRequest 249 | entry.Error = true 250 | entry.ErrorMessage = "Invalid URL provided (must be in absolute format and with https or https scheme)..." 251 | return &entry 252 | } 253 | 254 | feedUrl := feed.GetFeedURL(urlParam) 255 | if feedUrl == "" { 256 | entry.ErrorCode = http.StatusBadRequest 257 | entry.Error = true 258 | entry.ErrorMessage = "Could not find a feed URL in there..." 259 | return &entry 260 | } 261 | 262 | parsedFeed, err := feed.ParseFeed(feedUrl) 263 | if err != nil { 264 | entry.ErrorCode = http.StatusBadRequest 265 | entry.Error = true 266 | entry.ErrorMessage = "Bad feed: " + err.Error() 267 | return &entry 268 | } 269 | 270 | sk := feed.PrivateKeyFromFeed(feedUrl, *secret) 271 | publicKey, err := nostr.GetPublicKey(sk) 272 | if err != nil { 273 | entry.ErrorCode = http.StatusInternalServerError 274 | entry.Error = true 275 | entry.ErrorMessage = "bad private key: " + err.Error() 276 | return &entry 277 | } 278 | 279 | publicKey = strings.TrimSpace(publicKey) 280 | isNitterFeed := strings.Contains(parsedFeed.Description, "Twitter feed") 281 | defer insertFeed(err, feedUrl, publicKey, sk, isNitterFeed, db) 282 | 283 | entry.Url = feedUrl 284 | entry.PubKey = publicKey 285 | entry.NPubKey, _ = nip19.EncodePublicKey(publicKey) 286 | return &entry 287 | } 288 | 289 | func insertFeed(err error, feedUrl string, publicKey string, sk string, nitter bool, db *sql.DB) { 290 | row := db.QueryRow("SELECT privatekey, url FROM feeds WHERE publickey=$1", publicKey) 291 | 292 | var entity feed.Entity 293 | err = row.Scan(&entity.PrivateKey, &entity.URL) 294 | if err != nil && err == sql.ErrNoRows { 295 | log.Printf("[DEBUG] not found feed at url %q as publicKey %s", feedUrl, publicKey) 296 | if _, err := db.Exec(`INSERT INTO feeds (publickey, privatekey, url, nitter) VALUES (?, ?, ?, ?)`, publicKey, sk, feedUrl, nitter); err != nil { 297 | log.Printf("[ERROR] failure: %v", err) 298 | metrics.AppErrors.With(prometheus.Labels{"type": "SQL_WRITE"}).Inc() 299 | } else { 300 | log.Printf("[DEBUG] saved feed at url %q as publicKey %s", feedUrl, publicKey) 301 | } 302 | } else if err != nil { 303 | metrics.AppErrors.With(prometheus.Labels{"type": "SQL_SCAN"}).Inc() 304 | log.Fatalf("[ERROR] failed when trying to retrieve row with pubkey '%s': %v", publicKey, err) 305 | } else { 306 | log.Printf("[DEBUG] found feed at url %q as publicKey %s", feedUrl, publicKey) 307 | } 308 | } 309 | -------------------------------------------------------------------------------- /pkg/converter/rules.go: -------------------------------------------------------------------------------- 1 | package converter 2 | 3 | import ( 4 | "fmt" 5 | md "github.com/JohannesKaufmann/html-to-markdown" 6 | "github.com/PuerkitoBio/goquery" 7 | "strings" 8 | ) 9 | 10 | func GetConverterRules() []md.Rule { 11 | return []md.Rule{ 12 | { 13 | Filter: []string{"h1", "h2", "h3", "h4", "h5", "h6"}, 14 | Replacement: func(content string, selection *goquery.Selection, opt *md.Options) *string { 15 | content = strings.TrimSpace(content) 16 | return md.String(content) 17 | }, 18 | }, 19 | { 20 | Filter: []string{"img"}, 21 | AdvancedReplacement: func(content string, selec *goquery.Selection, opt *md.Options) (md.AdvancedResult, bool) { 22 | src := selec.AttrOr("src", "") 23 | src = strings.TrimSpace(src) 24 | if src == "" { 25 | return md.AdvancedResult{ 26 | Markdown: "", 27 | }, false 28 | } 29 | 30 | src = opt.GetAbsoluteURL(selec, src, "") 31 | 32 | text := fmt.Sprintf("\n%s\n", src) 33 | return md.AdvancedResult{ 34 | Markdown: text, 35 | }, false 36 | }, 37 | }, 38 | { 39 | Filter: []string{"a"}, 40 | AdvancedReplacement: func(content string, selec *goquery.Selection, opt *md.Options) (md.AdvancedResult, bool) { 41 | // if there is no href, no link is used. So just return the content inside the link 42 | href, ok := selec.Attr("href") 43 | if !ok || strings.TrimSpace(href) == "" || strings.TrimSpace(href) == "#" { 44 | return md.AdvancedResult{ 45 | Markdown: content, 46 | }, false 47 | } 48 | 49 | href = opt.GetAbsoluteURL(selec, href, "") 50 | 51 | // having multiline content inside a link is a bit tricky 52 | content = md.EscapeMultiLine(content) 53 | 54 | // if there is no link content (for example because it contains an svg) 55 | // the 'title' or 'aria-label' attribute is used instead. 56 | if strings.TrimSpace(content) == "" { 57 | content = selec.AttrOr("title", selec.AttrOr("aria-label", "")) 58 | } 59 | 60 | // a link without text won't de displayed anyway 61 | if content == "" { 62 | return md.AdvancedResult{ 63 | Markdown: "", 64 | }, false 65 | } 66 | 67 | replacement := fmt.Sprintf("%s (%s)", content, href) 68 | 69 | return md.AdvancedResult{ 70 | Markdown: replacement, 71 | }, false 72 | }, 73 | }, 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /pkg/custom_cache/cache.go: -------------------------------------------------------------------------------- 1 | package custom_cache 2 | 3 | import ( 4 | "context" 5 | "github.com/allegro/bigcache" 6 | "github.com/eko/gocache/lib/v4/cache" 7 | "github.com/eko/gocache/lib/v4/store" 8 | bigcache_store "github.com/eko/gocache/store/bigcache/v4" 9 | redis_store "github.com/eko/gocache/store/redis/v4" 10 | "github.com/redis/go-redis/v9" 11 | "log" 12 | "time" 13 | ) 14 | 15 | var MainCache *cache.Cache[[]byte] 16 | var MainCacheRedis *cache.Cache[string] 17 | var Initialized = false 18 | var RedisConnectionString *string 19 | 20 | func InitializeCache() { 21 | if RedisConnectionString != nil { 22 | initializeRedisCache() 23 | log.Printf("[INFO] Using Redis cache\n\n") 24 | Initialized = true 25 | return 26 | } 27 | initializeBigCache() 28 | log.Printf("[INFO] Using default memory cache\n\n") 29 | Initialized = true 30 | } 31 | 32 | func Get(key string) (string, error) { 33 | if !Initialized { 34 | InitializeCache() 35 | } 36 | if MainCacheRedis != nil { 37 | return getFromRedis(key) 38 | } 39 | return getFromBigCache(key) 40 | } 41 | 42 | func Set(key string, value string) error { 43 | if !Initialized { 44 | InitializeCache() 45 | } 46 | if MainCacheRedis != nil { 47 | return setToRedis(key, value) 48 | } 49 | 50 | return setToBigCache(key, value) 51 | } 52 | 53 | func initializeBigCache() { 54 | bigcacheClient, _ := bigcache.NewBigCache(bigcache.DefaultConfig(30 * time.Minute)) 55 | bigcacheStore := bigcache_store.NewBigcache(bigcacheClient) 56 | 57 | MainCache = cache.New[[]byte](bigcacheStore) 58 | } 59 | 60 | func initializeRedisCache() { 61 | opt, _ := redis.ParseURL(*RedisConnectionString) 62 | redisStore := redis_store.NewRedis(redis.NewClient(opt)) 63 | 64 | MainCacheRedis = cache.New[string](redisStore) 65 | } 66 | 67 | func getFromRedis(key string) (string, error) { 68 | value, err := MainCacheRedis.Get(context.Background(), key) 69 | switch err { 70 | case nil: 71 | log.Printf("[DEBUG] Get the key '%s' from the redis cache.", key) 72 | case redis.Nil: 73 | log.Printf("[DEBUG] Failed to find the key '%s' from the redis cache.", key) 74 | default: 75 | log.Printf("[DEBUG] Failed to get the value from the redis cache with key '%s': %v", key, err) 76 | } 77 | return value, err 78 | } 79 | 80 | func setToRedis(key string, value string) error { 81 | return MainCacheRedis.Set(context.Background(), key, value, store.WithExpiration(30*time.Minute)) 82 | } 83 | 84 | func getFromBigCache(key string) (string, error) { 85 | valueBytes, err := MainCache.Get(context.Background(), key) 86 | if err != nil { 87 | log.Printf("[DEBUG] Failed to find the key '%s' from the cache. Error: %v", key, err) 88 | return "", err 89 | } 90 | return string(valueBytes), nil 91 | } 92 | 93 | func setToBigCache(key string, value string) error { 94 | return MainCache.Set(context.Background(), key, []byte(value)) 95 | } 96 | -------------------------------------------------------------------------------- /pkg/events/events.go: -------------------------------------------------------------------------------- 1 | package events 2 | 3 | import ( 4 | "database/sql" 5 | "github.com/mmcdole/gofeed" 6 | "github.com/piraces/rsslay/pkg/feed" 7 | "github.com/piraces/rsslay/pkg/helpers" 8 | "github.com/piraces/rsslay/pkg/metrics" 9 | "github.com/prometheus/client_golang/prometheus" 10 | "log" 11 | "net/url" 12 | "strings" 13 | ) 14 | 15 | func GetParsedFeedForPubKey(pubKey string, db *sql.DB, deleteFailingFeeds bool, nitterInstances []string) (*gofeed.Feed, feed.Entity) { 16 | pubKey = strings.TrimSpace(pubKey) 17 | row := db.QueryRow("SELECT privatekey, url, nitter FROM feeds WHERE publickey=$1", pubKey) 18 | 19 | var entity feed.Entity 20 | err := row.Scan(&entity.PrivateKey, &entity.URL, &entity.Nitter) 21 | if err != nil && err == sql.ErrNoRows { 22 | return nil, entity 23 | } else if err != nil { 24 | log.Printf("[ERROR] failed when trying to retrieve row with pubkey '%s': %v", pubKey, err) 25 | metrics.AppErrors.With(prometheus.Labels{"type": "SQL_SCAN"}).Inc() 26 | return nil, entity 27 | } 28 | 29 | if !helpers.IsValidHttpUrl(entity.URL) { 30 | log.Printf("[INFO] retrieved invalid url from database %q", entity.URL) 31 | if deleteFailingFeeds { 32 | feed.DeleteInvalidFeed(entity.URL, db) 33 | } 34 | return nil, entity 35 | } 36 | 37 | parsedFeed, err := feed.ParseFeed(entity.URL) 38 | if err != nil && entity.Nitter { 39 | log.Printf("[DEBUG] failed to parse feed at url %q: %v. Now iterating through other Nitter instances", entity.URL, err) 40 | for i, instance := range nitterInstances { 41 | newUrl, _ := setHostname(entity.URL, instance) 42 | log.Printf("[DEBUG] attempt %d: use %q instead of %q", i, newUrl, entity.URL) 43 | parsedFeed, err = feed.ParseFeed(newUrl) 44 | if err == nil { 45 | log.Printf("[DEBUG] attempt %d: success with %q", i, newUrl) 46 | break 47 | } 48 | } 49 | } 50 | 51 | if err != nil { 52 | log.Printf("[DEBUG] failed to parse feed at url %q: %v", entity.URL, err) 53 | if deleteFailingFeeds { 54 | feed.DeleteInvalidFeed(entity.URL, db) 55 | } 56 | return nil, entity 57 | } 58 | 59 | if strings.Contains(parsedFeed.Description, "Twitter feed") && !entity.Nitter { 60 | updateDatabaseEntry(&entity, db) 61 | entity.Nitter = true 62 | } 63 | 64 | return parsedFeed, entity 65 | } 66 | 67 | func updateDatabaseEntry(entity *feed.Entity, db *sql.DB) { 68 | log.Printf("[DEBUG] attempting to set feed at url %q with publicKey %s as nitter instance", entity.URL, entity.PublicKey) 69 | if _, err := db.Exec(`UPDATE feeds SET nitter = ? WHERE publickey = ?`, 1, entity.PublicKey); err != nil { 70 | log.Printf("[ERROR] failure while updating record on db to set as nitter feed: %v", err) 71 | metrics.AppErrors.With(prometheus.Labels{"type": "SQL_WRITE"}).Inc() 72 | } else { 73 | log.Printf("[DEBUG] set feed at url %q with publicKey %s as nitter instance", entity.URL, entity.PublicKey) 74 | } 75 | } 76 | 77 | func setHostname(addr, hostname string) (string, error) { 78 | u, err := url.Parse(addr) 79 | if err != nil { 80 | return "", err 81 | } 82 | u.Host = hostname 83 | return u.String(), nil 84 | } 85 | -------------------------------------------------------------------------------- /pkg/events/events_test.go: -------------------------------------------------------------------------------- 1 | package events 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "github.com/DATA-DOG/go-sqlmock" 7 | "github.com/piraces/rsslay/pkg/feed" 8 | "github.com/stretchr/testify/assert" 9 | "testing" 10 | ) 11 | 12 | const samplePubKey = "73e247ee8c4ff09a50525bed7b0869c371864c0bf2b4d6a2639acaed07613958" 13 | const samplePrivateKey = "4d0888c07093941c9db16fcffb96fdf8af49a6839e865ea6110c7ab7cbd2d3d3" 14 | const sampleValidNitterFeedUrl = "https://nitter.moomoo.me/Twitter/rss" 15 | const sampleInvalidNitterFeedUrl = "https://example.com/Twitter/rss" 16 | const sampleValidUrl = "https://mastodon.social/" 17 | 18 | var nitterInstances = []string{"birdsite.xanny.family", "notabird.site", "nitter.moomoo.me", "nitter.fly.dev"} 19 | var sqlRows = []string{"privatekey", "url", "nitter"} 20 | 21 | func TestGetParsedFeedForNitterPubKey(t *testing.T) { 22 | t.Skip() 23 | db, mock, err := sqlmock.New() 24 | if err != nil { 25 | t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) 26 | } 27 | rows := sqlmock.NewRows(sqlRows) 28 | rows.AddRow(samplePrivateKey, sampleValidNitterFeedUrl, true) 29 | mock.ExpectQuery("SELECT privatekey, url, nitter FROM feeds").WillReturnRows(rows) 30 | mock.ExpectClose() 31 | 32 | parsedFeed, entity := GetParsedFeedForPubKey(samplePubKey, db, true, nitterInstances) 33 | assert.NotNil(t, parsedFeed) 34 | assert.Equal(t, feed.Entity{ 35 | PublicKey: "", 36 | PrivateKey: samplePrivateKey, 37 | URL: sampleValidNitterFeedUrl, 38 | Nitter: true, 39 | }, entity) 40 | _ = db.Close() 41 | } 42 | 43 | func TestGetParsedFeedForExistingOutdatedNitterPubKey(t *testing.T) { 44 | t.Skip() 45 | db, mock, err := sqlmock.New() 46 | if err != nil { 47 | t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) 48 | } 49 | rows := sqlmock.NewRows(sqlRows) 50 | rows.AddRow(samplePrivateKey, sampleValidNitterFeedUrl, false) 51 | mock.ExpectQuery("SELECT privatekey, url, nitter FROM feeds").WillReturnRows(rows) 52 | mock.ExpectExec("UPDATE feeds").WillReturnResult(sqlmock.NewResult(1, 1)) 53 | 54 | mock.ExpectClose() 55 | 56 | parsedFeed, entity := GetParsedFeedForPubKey(samplePubKey, db, true, nitterInstances) 57 | assert.NotNil(t, parsedFeed) 58 | assert.Equal(t, feed.Entity{ 59 | PublicKey: "", 60 | PrivateKey: samplePrivateKey, 61 | URL: sampleValidNitterFeedUrl, 62 | Nitter: true, 63 | }, entity) 64 | _ = db.Close() 65 | } 66 | 67 | func TestGetParsedFeedForErrorExistingOutdatedNitterPubKey(t *testing.T) { 68 | t.Skip() 69 | db, mock, err := sqlmock.New() 70 | if err != nil { 71 | t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) 72 | } 73 | rows := sqlmock.NewRows(sqlRows) 74 | rows.AddRow(samplePrivateKey, sampleValidNitterFeedUrl, false) 75 | mock.ExpectQuery("SELECT privatekey, url, nitter FROM feeds").WillReturnRows(rows) 76 | mock.ExpectExec("UPDATE feeds").WillReturnError(errors.New("error")) 77 | mock.ExpectClose() 78 | 79 | parsedFeed, entity := GetParsedFeedForPubKey(samplePubKey, db, true, nitterInstances) 80 | assert.NotNil(t, parsedFeed) 81 | assert.Equal(t, feed.Entity{ 82 | PublicKey: "", 83 | PrivateKey: samplePrivateKey, 84 | URL: sampleValidNitterFeedUrl, 85 | Nitter: true, 86 | }, entity) 87 | _ = db.Close() 88 | } 89 | 90 | func TestGetParsedFeedSQLErrorForNitterPubKey(t *testing.T) { 91 | db, mock, err := sqlmock.New() 92 | if err != nil { 93 | t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) 94 | } 95 | rows := sqlmock.NewRows(sqlRows) 96 | rows.AddRow(samplePrivateKey, sampleValidNitterFeedUrl, false) 97 | mock.ExpectQuery("SELECT privatekey, url, nitter FROM feeds").WillReturnError(errors.New("error")) 98 | mock.ExpectClose() 99 | 100 | parsedFeed, entity := GetParsedFeedForPubKey(samplePubKey, db, true, nitterInstances) 101 | assert.Nil(t, parsedFeed) 102 | assert.Empty(t, entity) 103 | _ = db.Close() 104 | } 105 | 106 | func TestGetParsedFeedInvalidFeedUrlForNitterPubKey(t *testing.T) { 107 | db, mock, err := sqlmock.New() 108 | if err != nil { 109 | t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) 110 | } 111 | rows := sqlmock.NewRows(sqlRows) 112 | rows.AddRow(samplePrivateKey, sampleInvalidNitterFeedUrl, false) 113 | mock.ExpectQuery("SELECT privatekey, url, nitter FROM feeds").WillReturnRows(rows) 114 | mock.ExpectClose() 115 | 116 | parsedFeed, entity := GetParsedFeedForPubKey(samplePubKey, db, true, nitterInstances) 117 | assert.Nil(t, parsedFeed) 118 | assert.Equal(t, feed.Entity{ 119 | PublicKey: "", 120 | PrivateKey: samplePrivateKey, 121 | URL: sampleInvalidNitterFeedUrl, 122 | Nitter: false, 123 | }, entity) 124 | _ = db.Close() 125 | } 126 | 127 | func TestGetParsedFeedInvalidFeedUrlForStandardPubKey(t *testing.T) { 128 | db, mock, err := sqlmock.New() 129 | if err != nil { 130 | t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) 131 | } 132 | rows := sqlmock.NewRows(sqlRows) 133 | rows.AddRow(samplePrivateKey, sampleValidUrl, false) 134 | mock.ExpectQuery("SELECT privatekey, url, nitter FROM feeds").WillReturnRows(rows) 135 | expectedDeleteQuery := fmt.Sprintf("DELETE FROM feeds WHERE url=%s", sampleValidUrl) 136 | mock.ExpectQuery(expectedDeleteQuery) 137 | mock.ExpectClose() 138 | 139 | parsedFeed, entity := GetParsedFeedForPubKey(samplePubKey, db, true, nitterInstances) 140 | assert.Nil(t, parsedFeed) 141 | assert.Equal(t, feed.Entity{ 142 | PublicKey: "", 143 | PrivateKey: samplePrivateKey, 144 | URL: sampleValidUrl, 145 | Nitter: false, 146 | }, entity) 147 | _ = db.Close() 148 | } 149 | 150 | func TestGetParsedFeedInvalidUrlForStandardPubKey(t *testing.T) { 151 | db, mock, err := sqlmock.New() 152 | if err != nil { 153 | t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) 154 | } 155 | rows := sqlmock.NewRows(sqlRows) 156 | rows.AddRow(samplePrivateKey, "not a url", false) 157 | mock.ExpectQuery("SELECT privatekey, url, nitter FROM feeds").WillReturnRows(rows) 158 | expectedDeleteQuery := fmt.Sprintf("DELETE FROM feeds WHERE url=%s", sampleValidUrl) 159 | mock.ExpectQuery(expectedDeleteQuery) 160 | mock.ExpectClose() 161 | 162 | parsedFeed, entity := GetParsedFeedForPubKey(samplePubKey, db, true, nitterInstances) 163 | assert.Nil(t, parsedFeed) 164 | assert.Equal(t, feed.Entity{ 165 | PublicKey: "", 166 | PrivateKey: samplePrivateKey, 167 | URL: "not a url", 168 | Nitter: false, 169 | }, entity) 170 | _ = db.Close() 171 | } 172 | -------------------------------------------------------------------------------- /pkg/feed/feed.go: -------------------------------------------------------------------------------- 1 | package feed 2 | 3 | import ( 4 | "crypto/hmac" 5 | "crypto/sha256" 6 | "database/sql" 7 | "encoding/hex" 8 | "encoding/json" 9 | "errors" 10 | "fmt" 11 | md "github.com/JohannesKaufmann/html-to-markdown" 12 | "github.com/PuerkitoBio/goquery" 13 | "github.com/microcosm-cc/bluemonday" 14 | "github.com/mmcdole/gofeed" 15 | "github.com/nbd-wtf/go-nostr" 16 | "github.com/piraces/rsslay/pkg/converter" 17 | "github.com/piraces/rsslay/pkg/custom_cache" 18 | "github.com/piraces/rsslay/pkg/helpers" 19 | "github.com/piraces/rsslay/pkg/metrics" 20 | "github.com/prometheus/client_golang/prometheus" 21 | "html" 22 | "log" 23 | "net/http" 24 | "net/url" 25 | "strings" 26 | "time" 27 | ) 28 | 29 | var ( 30 | fp = gofeed.NewParser() 31 | client = &http.Client{ 32 | CheckRedirect: func(req *http.Request, via []*http.Request) error { 33 | if len(via) >= 2 { 34 | return errors.New("stopped after 2 redirects") 35 | } 36 | return nil 37 | }, 38 | Timeout: 5 * time.Second, 39 | } 40 | ) 41 | 42 | type Entity struct { 43 | PublicKey string 44 | PrivateKey string 45 | URL string 46 | Nitter bool 47 | } 48 | 49 | var types = []string{ 50 | "rss+xml", 51 | "atom+xml", 52 | "feed+json", 53 | "text/xml", 54 | "application/xml", 55 | } 56 | 57 | func GetFeedURL(url string) string { 58 | resp, err := client.Get(url) 59 | if err != nil || resp.StatusCode >= 300 { 60 | return "" 61 | } 62 | 63 | ct := resp.Header.Get("Content-Type") 64 | for _, typ := range types { 65 | if strings.Contains(ct, typ) { 66 | return url 67 | } 68 | } 69 | 70 | if strings.Contains(ct, "text/html") { 71 | doc, err := goquery.NewDocumentFromReader(resp.Body) 72 | if err != nil { 73 | return "" 74 | } 75 | 76 | for _, typ := range types { 77 | href, _ := doc.Find(fmt.Sprintf("link[type*='%s']", typ)).Attr("href") 78 | if href == "" { 79 | continue 80 | } 81 | if !strings.HasPrefix(href, "http") && !strings.HasPrefix(href, "https") { 82 | href, _ = helpers.UrlJoin(url, href) 83 | } 84 | return href 85 | } 86 | } 87 | 88 | return "" 89 | } 90 | 91 | func ParseFeed(url string) (*gofeed.Feed, error) { 92 | feedString, err := custom_cache.Get(url) 93 | if err == nil { 94 | metrics.CacheHits.Inc() 95 | 96 | var feed gofeed.Feed 97 | err := json.Unmarshal([]byte(feedString), &feed) 98 | if err != nil { 99 | log.Printf("[ERROR] failure to parse cache stored feed: %v", err) 100 | metrics.AppErrors.With(prometheus.Labels{"type": "CACHE_PARSE"}).Inc() 101 | } else { 102 | return &feed, nil 103 | } 104 | } else { 105 | log.Printf("[DEBUG] entry not found in cache: %v", err) 106 | } 107 | 108 | metrics.CacheMiss.Inc() 109 | fp.RSSTranslator = NewCustomTranslator() 110 | feed, err := fp.ParseURL(url) 111 | if err != nil { 112 | return nil, err 113 | } 114 | 115 | // cleanup a little so we don't store too much junk 116 | for i := range feed.Items { 117 | feed.Items[i].Content = "" 118 | } 119 | 120 | marshal, err := json.Marshal(feed) 121 | if err == nil { 122 | err = custom_cache.Set(url, string(marshal)) 123 | } 124 | 125 | if err != nil { 126 | log.Printf("[ERROR] failure to store into cache feed: %v", err) 127 | metrics.AppErrors.With(prometheus.Labels{"type": "CACHE_SET"}).Inc() 128 | } 129 | 130 | return feed, nil 131 | } 132 | 133 | func EntryFeedToSetMetadata(pubkey string, feed *gofeed.Feed, originalUrl string, enableAutoRegistration bool, defaultProfilePictureUrl string, mainDomainName string) nostr.Event { 134 | // Handle Nitter special cases (http schema) 135 | if strings.Contains(feed.Description, "Twitter feed") { 136 | if strings.HasPrefix(originalUrl, "https://") { 137 | feed.Description = strings.ReplaceAll(feed.Description, "http://", "https://") 138 | feed.Title = strings.ReplaceAll(feed.Title, "http://", "https://") 139 | if feed.Image != nil { 140 | feed.Image.URL = strings.ReplaceAll(feed.Image.URL, "http://", "https://") 141 | } 142 | 143 | feed.Link = strings.ReplaceAll(feed.Link, "http://", "https://") 144 | } 145 | } 146 | 147 | var theDescription = feed.Description 148 | var theFeedTitle = feed.Title 149 | if strings.Contains(feed.Link, "reddit.com") { 150 | var subredditParsePart1 = strings.Split(feed.Link, "/r/") 151 | var subredditParsePart2 = strings.Split(subredditParsePart1[1], "/") 152 | theDescription = feed.Description + fmt.Sprintf(" #%s", subredditParsePart2[0]) 153 | 154 | theFeedTitle = "/r/" + subredditParsePart2[0] 155 | } 156 | metadata := map[string]string{ 157 | "name": theFeedTitle + " (RSS Feed)", 158 | "about": theDescription + "\n\n" + feed.Link, 159 | } 160 | 161 | if enableAutoRegistration { 162 | metadata["nip05"] = fmt.Sprintf("%s@%s", originalUrl, mainDomainName) 163 | } 164 | 165 | if feed.Image != nil { 166 | metadata["picture"] = feed.Image.URL 167 | } else if defaultProfilePictureUrl != "" { 168 | metadata["picture"] = defaultProfilePictureUrl 169 | } 170 | 171 | content, _ := json.Marshal(metadata) 172 | 173 | createdAt := time.Unix(time.Now().Unix(), 0) 174 | if feed.PublishedParsed != nil { 175 | createdAt = *feed.PublishedParsed 176 | } 177 | 178 | evt := nostr.Event{ 179 | PubKey: pubkey, 180 | CreatedAt: nostr.Timestamp(createdAt.Unix()), 181 | Kind: nostr.KindSetMetadata, 182 | Tags: nostr.Tags{[]string{"proxy", feed.FeedLink, "rss"}}, 183 | Content: string(content), 184 | } 185 | evt.ID = string(evt.Serialize()) 186 | 187 | return evt 188 | } 189 | 190 | func ItemToTextNote(pubkey string, item *gofeed.Item, feed *gofeed.Feed, defaultCreatedAt time.Time, originalUrl string, maxContentLength int) nostr.Event { 191 | content := "" 192 | if item.Title != "" { 193 | content = "**" + item.Title + "**" 194 | } 195 | 196 | mdConverter := md.NewConverter("", true, nil) 197 | mdConverter.AddRules(converter.GetConverterRules()...) 198 | 199 | description, err := mdConverter.ConvertString(item.Description) 200 | if err != nil { 201 | log.Printf("[WARN] failure to convert description to markdown (defaulting to plain text): %v", err) 202 | p := bluemonday.StripTagsPolicy() 203 | description = p.Sanitize(item.Description) 204 | } 205 | 206 | if !strings.EqualFold(item.Title, description) && !strings.Contains(feed.Link, "stacker.news") && !strings.Contains(feed.Link, "reddit.com") { 207 | content += "\n\n" + description 208 | } 209 | 210 | shouldUpgradeLinkSchema := false 211 | 212 | // Handle Nitter special cases (duplicates and http schema) 213 | if strings.Contains(feed.Description, "Twitter feed") { 214 | content = "" 215 | shouldUpgradeLinkSchema = true 216 | 217 | if strings.HasPrefix(originalUrl, "https://") { 218 | description = strings.ReplaceAll(description, "http://", "https://") 219 | } 220 | 221 | if strings.Contains(item.Title, "RT by @") { 222 | if len(item.DublinCoreExt.Creator) > 0 { 223 | content = "**" + "RT " + item.DublinCoreExt.Creator[0] + ":**\n\n" 224 | } 225 | } else if strings.Contains(item.Title, "R to @") { 226 | fields := strings.Fields(item.Title) 227 | if len(fields) >= 2 { 228 | replyingToHandle := fields[2] 229 | content = "**" + "Response to " + replyingToHandle + "**\n\n" 230 | } 231 | } 232 | content += description 233 | } 234 | 235 | if strings.Contains(feed.Link, "reddit.com") { 236 | var subredditParsePart1 = strings.Split(feed.Link, "/r/") 237 | var subredditParsePart2 = strings.Split(subredditParsePart1[1], "/") 238 | var theHashtag = fmt.Sprintf(" #%s", subredditParsePart2[0]) 239 | 240 | content = content + "\n\n" + theHashtag 241 | 242 | } 243 | 244 | content = html.UnescapeString(content) 245 | if len(content) > maxContentLength { 246 | content = content[0:(maxContentLength-1)] + "…" 247 | } 248 | 249 | if shouldUpgradeLinkSchema { 250 | item.Link = strings.ReplaceAll(item.Link, "http://", "https://") 251 | } 252 | 253 | // Handle comments 254 | if item.Custom != nil { 255 | if comments, ok := item.Custom["comments"]; ok { 256 | content += fmt.Sprintf("\n\nComments: %s", comments) 257 | } 258 | } 259 | 260 | content += "\n\n" + item.Link 261 | 262 | createdAt := defaultCreatedAt 263 | if item.UpdatedParsed != nil { 264 | createdAt = *item.UpdatedParsed 265 | } 266 | if item.PublishedParsed != nil { 267 | createdAt = *item.PublishedParsed 268 | } 269 | 270 | composedProxyLink := feed.FeedLink 271 | if item.GUID != "" { 272 | composedProxyLink += fmt.Sprintf("#%s", url.QueryEscape(item.GUID)) 273 | } 274 | 275 | evt := nostr.Event{ 276 | PubKey: pubkey, 277 | CreatedAt: nostr.Timestamp(createdAt.Unix()), 278 | Kind: nostr.KindTextNote, 279 | Tags: nostr.Tags{[]string{"proxy", composedProxyLink, "rss"}}, 280 | Content: strings.ToValidUTF8(content, ""), 281 | } 282 | evt.ID = string(evt.Serialize()) 283 | 284 | return evt 285 | } 286 | 287 | func PrivateKeyFromFeed(url string, secret string) string { 288 | m := hmac.New(sha256.New, []byte(secret)) 289 | m.Write([]byte(url)) 290 | r := m.Sum(nil) 291 | return hex.EncodeToString(r) 292 | } 293 | 294 | func DeleteInvalidFeed(url string, db *sql.DB) { 295 | if _, err := db.Exec(`DELETE FROM feeds WHERE url=?`, url); err != nil { 296 | log.Printf("[ERROR] failure to delete invalid feed: %v", err) 297 | metrics.AppErrors.With(prometheus.Labels{"type": "SQL_WRITE"}).Inc() 298 | } else { 299 | log.Printf("[DEBUG] deleted invalid feed with url %q", url) 300 | } 301 | } 302 | -------------------------------------------------------------------------------- /pkg/feed/feed_test.go: -------------------------------------------------------------------------------- 1 | package feed 2 | 3 | import ( 4 | "database/sql" 5 | "errors" 6 | "fmt" 7 | "github.com/DATA-DOG/go-sqlmock" 8 | "github.com/mmcdole/gofeed" 9 | ext "github.com/mmcdole/gofeed/extensions" 10 | "github.com/stretchr/testify/assert" 11 | "strings" 12 | "testing" 13 | "time" 14 | ) 15 | 16 | const samplePubKey = "1870bcd5f6081ef7ea4b17204ffa4e92de51670142be0c8140e0635b355ca85f" 17 | const sampleUrlForPublicKey = "https://nitter.moomoo.me/Bitcoin/rss" 18 | const samplePrivateKeyForPubKey = "27660ab89e69f59bb8d9f0bd60da4a8515cdd3e2ca4f91d72a242b086d6aaaa7" 19 | const testSecret = "test" 20 | 21 | const sampleInvalidUrl = "https:// nostr.example/" 22 | const sampleInvalidUrlContentType = "https://accounts.google.com/.well-known/openid-configuration" 23 | const sampleRedirectingUrl = "https://httpstat.us/301" 24 | const sampleValidDirectFeedUrl = "https://mastodon.social/@Gargron.rss" 25 | const sampleValidIndirectFeedUrl = "https://www.rssboard.org/" 26 | const sampleValidIndirectFeedUrlExpected = "http://feeds.rssboard.org/rssboard" 27 | const sampleValidWithoutFeedUrl = "https://go.dev/" 28 | const sampleValidWithRelativeFeedUrl = "https://golangweekly.com/" 29 | const sampleValidWithRelativeFeedUrlExpected = "https://golangweekly.com/rss" 30 | 31 | var actualTime = time.Unix(time.Now().Unix(), 0) 32 | var sampleNitterFeed = gofeed.Feed{ 33 | Title: "Coldplay / @coldplay", 34 | Description: "Twitter feed for: @coldplay. Generated by nitter.moomoo.me", 35 | Link: "http://nitter.moomoo.me/coldplay", 36 | FeedLink: "https://nitter.moomoo.me/coldplay/rss", 37 | Links: []string{"http://nitter.moomoo.me/coldplay"}, 38 | PublishedParsed: &actualTime, 39 | Language: "en-us", 40 | Image: &gofeed.Image{ 41 | URL: "http://nitter.moomoo.me/pic/pbs.twimg.com%2Fprofile_images%2F1417506973877211138%2FYIm7dOQH_400x400.jpg", 42 | Title: "Coldplay / @coldplay", 43 | }, 44 | } 45 | 46 | var sampleStackerNewsFeed = gofeed.Feed{ 47 | Title: "Stacker News", 48 | Description: "Like Hacker News, but we pay you Bitcoin.", 49 | Link: "https://stacker.news", 50 | FeedLink: "https://stacker.news/rss", 51 | Links: []string{"https://blog.cryptographyengineering.com/2014/11/zero-knowledge-proofs-illustrated-primer.html"}, 52 | PublishedParsed: &actualTime, 53 | Language: "en", 54 | } 55 | 56 | var sampleNitterFeedRTItem = gofeed.Item{ 57 | Title: "RT by @coldplay: TOMORROW", 58 | Description: "Sample description", 59 | Content: "Sample content", 60 | Link: "http://nitter.moomoo.me/coldplay/status/1622148481740685312#m", 61 | UpdatedParsed: &actualTime, 62 | PublishedParsed: &actualTime, 63 | GUID: "http://nitter.moomoo.me/coldplay/status/1622148481740685312#m", 64 | DublinCoreExt: &ext.DublinCoreExtension{ 65 | Creator: []string{"@nbcsnl"}, 66 | }, 67 | } 68 | 69 | var sampleNitterFeedResponseItem = gofeed.Item{ 70 | Title: "R to @coldplay: Sample", 71 | Description: "Sample description", 72 | Content: "Sample content", 73 | Link: "http://nitter.moomoo.me/elonmusk/status/1621544996167122944#m", 74 | UpdatedParsed: &actualTime, 75 | PublishedParsed: &actualTime, 76 | GUID: "http://nitter.moomoo.me/elonmusk/status/1621544996167122944#m", 77 | DublinCoreExt: &ext.DublinCoreExtension{ 78 | Creator: []string{"@elonmusk"}, 79 | }, 80 | } 81 | 82 | var sampleDefaultFeedItem = gofeed.Item{ 83 | Title: "Golang Weekly", 84 | Description: "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Phasellus nec condimentum orci. Vestibulum at nunc porta, placerat ex sit amet, consectetur augue. Donec cursus ipsum sed venenatis maximus. Nunc tincidunt dui nec congue lacinia. In mollis magna eu nisi viverra luctus. Ut ultrices eros gravida, lacinia nibh vitae, tristique massa. Sed eu scelerisque erat. Sed eget tortor et turpis feugiat interdum. Nulla sit amet nibh vel massa bibendum congue. Quisque sed tempor velit. Interdum et malesuada fames ac ante ipsum primis in faucibus. Curabitur suscipit mollis fringilla. Integer quis sodales tortor, at hendrerit lacus. Cras posuere maximus nisi. Mauris eget.", 85 | Content: "Sample content", 86 | Link: "https://golangweekly.com/issues/446", 87 | UpdatedParsed: &actualTime, 88 | PublishedParsed: &actualTime, 89 | GUID: "https://golangweekly.com/issues/446", 90 | } 91 | 92 | var sampleDefaultFeedItemWithComments = gofeed.Item{ 93 | Title: "Golang Weekly", 94 | Description: "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Phasellus nec condimentum orci. Vestibulum at nunc porta, placerat ex sit amet, consectetur augue. Donec cursus ipsum sed venenatis maximus. Nunc tincidunt dui nec congue lacinia. In mollis magna eu nisi viverra luctus. Ut ultrices eros gravida, lacinia nibh vitae, tristique massa. Sed eu scelerisque erat. Sed eget tortor et turpis feugiat interdum. Nulla sit amet nibh vel massa bibendum congue. Quisque sed tempor velit. Interdum et malesuada fames ac ante ipsum primis in faucibus. Curabitur suscipit mollis fringilla. Integer quis sodales tortor, at hendrerit lacus. Cras posuere maximus nisi. Mauris eget.", 95 | Content: "Sample content", 96 | Link: "https://golangweekly.com/issues/446", 97 | UpdatedParsed: &actualTime, 98 | PublishedParsed: &actualTime, 99 | GUID: "https://golangweekly.com/issues/446", 100 | Custom: map[string]string{ 101 | "comments": "https://golangweekly.com/issues/446", 102 | }, 103 | } 104 | 105 | var sampleDefaultFeedItemExpectedContent = fmt.Sprintf("**%s**\n\n%s", sampleDefaultFeedItem.Title, sampleDefaultFeedItem.Description) 106 | var sampleDefaultFeedItemExpectedContentSubstring = sampleDefaultFeedItemExpectedContent[0:249] 107 | 108 | var sampleStackerNewsFeedItem = gofeed.Item{ 109 | Title: "Zero Knowledge Proofs: An illustrated primer", 110 | Description: "Comments", 111 | Content: "Sample content", 112 | Link: "https://blog.cryptographyengineering.com/2014/11/zero-knowledge-proofs-illustrated-primer.html", 113 | UpdatedParsed: &actualTime, 114 | PublishedParsed: &actualTime, 115 | GUID: "https://stacker.news/items/131533", 116 | Custom: map[string]string{ 117 | "comments": "https://stacker.news/items/131533", 118 | }, 119 | } 120 | 121 | var sampleDefaultFeed = gofeed.Feed{ 122 | Title: "Golang Weekly", 123 | Description: "A weekly newsletter about the Go programming language", 124 | Link: "https://golangweekly.com/rss", 125 | FeedLink: "https://golangweekly.com/rss", 126 | Links: []string{"https://golangweekly.com/issues/446"}, 127 | PublishedParsed: &actualTime, 128 | Language: "en-us", 129 | Image: nil, 130 | } 131 | 132 | func TestGetFeedURLWithInvalidURLReturnsEmptyString(t *testing.T) { 133 | feed := GetFeedURL(sampleInvalidUrl) 134 | assert.Empty(t, feed) 135 | } 136 | 137 | func TestGetFeedURLWithInvalidContentTypeReturnsEmptyString(t *testing.T) { 138 | feed := GetFeedURL(sampleInvalidUrlContentType) 139 | assert.Empty(t, feed) 140 | } 141 | 142 | func TestGetFeedURLWithRedirectingURLReturnsEmptyString(t *testing.T) { 143 | feed := GetFeedURL(sampleRedirectingUrl) 144 | assert.Empty(t, feed) 145 | } 146 | 147 | func TestGetFeedURLWithValidUrlOfValidTypesReturnsSameUrl(t *testing.T) { 148 | feed := GetFeedURL(sampleValidDirectFeedUrl) 149 | assert.Equal(t, sampleValidDirectFeedUrl, feed) 150 | } 151 | 152 | func TestGetFeedURLWithValidUrlOfHtmlTypeWithFeedReturnsFoundFeed(t *testing.T) { 153 | feed := GetFeedURL(sampleValidIndirectFeedUrl) 154 | assert.Equal(t, sampleValidIndirectFeedUrlExpected, feed) 155 | } 156 | 157 | func TestGetFeedURLWithValidUrlOfHtmlTypeWithRelativeFeedReturnsFoundFeed(t *testing.T) { 158 | feed := GetFeedURL(sampleValidWithRelativeFeedUrl) 159 | assert.Equal(t, sampleValidWithRelativeFeedUrlExpected, feed) 160 | } 161 | 162 | func TestGetFeedURLWithValidUrlOfHtmlTypeWithoutFeedReturnsEmpty(t *testing.T) { 163 | feed := GetFeedURL(sampleValidWithoutFeedUrl) 164 | assert.Empty(t, feed) 165 | } 166 | 167 | func TestParseFeedWithValidUrlReturnsParsedFeed(t *testing.T) { 168 | feed, err := ParseFeed(sampleValidWithRelativeFeedUrlExpected) 169 | assert.NotNil(t, feed) 170 | assert.NoError(t, err) 171 | } 172 | 173 | func TestParseFeedWithValidUrlWithoutFeedReturnsError(t *testing.T) { 174 | feed, err := ParseFeed(sampleValidWithoutFeedUrl) 175 | assert.Nil(t, feed) 176 | assert.Error(t, err) 177 | } 178 | 179 | func TestParseFeedWithCachedUrlReturnsCachedParsedFeed(t *testing.T) { 180 | _, _ = ParseFeed(sampleValidWithRelativeFeedUrlExpected) 181 | feed, err := ParseFeed(sampleValidWithRelativeFeedUrlExpected) 182 | assert.NotNil(t, feed) 183 | assert.NoError(t, err) 184 | } 185 | 186 | func TestEntryFeedToSetMetadata(t *testing.T) { 187 | testCases := []struct { 188 | pubKey string 189 | feed *gofeed.Feed 190 | originalUrl string 191 | enableAutoRegistration bool 192 | defaultProfilePictureUrl string 193 | defaultMainDomain string 194 | }{ 195 | { 196 | pubKey: samplePubKey, 197 | feed: &sampleNitterFeed, 198 | originalUrl: sampleNitterFeed.FeedLink, 199 | enableAutoRegistration: true, 200 | defaultProfilePictureUrl: "https://image.example", 201 | defaultMainDomain: "rsslay.nostr.moe", 202 | }, 203 | { 204 | pubKey: samplePubKey, 205 | feed: &sampleDefaultFeed, 206 | originalUrl: sampleDefaultFeed.FeedLink, 207 | enableAutoRegistration: true, 208 | defaultProfilePictureUrl: "https://image.example", 209 | defaultMainDomain: "rsslay.nostr.moe", 210 | }, 211 | } 212 | for _, tc := range testCases { 213 | metadata := EntryFeedToSetMetadata(tc.pubKey, tc.feed, tc.originalUrl, tc.enableAutoRegistration, tc.defaultProfilePictureUrl, tc.defaultMainDomain) 214 | assert.NotEmpty(t, metadata) 215 | assert.Equal(t, samplePubKey, metadata.PubKey) 216 | assert.Equal(t, 0, metadata.Kind) 217 | assert.Empty(t, metadata.Sig) 218 | } 219 | } 220 | 221 | func TestPrivateKeyFromFeed(t *testing.T) { 222 | sk := PrivateKeyFromFeed(sampleUrlForPublicKey, testSecret) 223 | assert.Equal(t, samplePrivateKeyForPubKey, sk) 224 | } 225 | 226 | func TestItemToTextNote(t *testing.T) { 227 | testCases := []struct { 228 | pubKey string 229 | item *gofeed.Item 230 | feed *gofeed.Feed 231 | defaultCreatedAt time.Time 232 | originalUrl string 233 | expectedContent string 234 | maxContentLength int 235 | }{ 236 | { 237 | pubKey: samplePubKey, 238 | item: &sampleNitterFeedRTItem, 239 | feed: &sampleNitterFeed, 240 | defaultCreatedAt: actualTime, 241 | originalUrl: sampleNitterFeed.FeedLink, 242 | expectedContent: fmt.Sprintf("**RT %s:**\n\n%s\n\n%s", sampleNitterFeedRTItem.DublinCoreExt.Creator[0], sampleNitterFeedRTItem.Description, strings.ReplaceAll(sampleNitterFeedRTItem.Link, "http://", "https://")), 243 | maxContentLength: 250, 244 | }, 245 | { 246 | pubKey: samplePubKey, 247 | item: &sampleNitterFeedResponseItem, 248 | feed: &sampleNitterFeed, 249 | defaultCreatedAt: actualTime, 250 | originalUrl: sampleNitterFeed.FeedLink, 251 | expectedContent: fmt.Sprintf("**Response to %s:**\n\n%s\n\n%s", "@coldplay", sampleNitterFeedResponseItem.Description, strings.ReplaceAll(sampleNitterFeedResponseItem.Link, "http://", "https://")), 252 | maxContentLength: 250, 253 | }, 254 | { 255 | pubKey: samplePubKey, 256 | item: &sampleDefaultFeedItem, 257 | feed: &sampleDefaultFeed, 258 | defaultCreatedAt: actualTime, 259 | originalUrl: sampleDefaultFeed.FeedLink, 260 | expectedContent: sampleDefaultFeedItemExpectedContentSubstring + "…" + "\n\n" + sampleDefaultFeedItem.Link, 261 | maxContentLength: 250, 262 | }, 263 | { 264 | pubKey: samplePubKey, 265 | item: &sampleDefaultFeedItemWithComments, 266 | feed: &sampleDefaultFeed, 267 | defaultCreatedAt: actualTime, 268 | originalUrl: sampleDefaultFeed.FeedLink, 269 | expectedContent: sampleDefaultFeedItemExpectedContentSubstring + "…\n\nComments: " + sampleDefaultFeedItemWithComments.Custom["comments"] + "\n\n" + sampleDefaultFeedItem.Link, 270 | maxContentLength: 250, 271 | }, 272 | { 273 | pubKey: samplePubKey, 274 | item: &sampleDefaultFeedItemWithComments, 275 | feed: &sampleDefaultFeed, 276 | defaultCreatedAt: actualTime, 277 | originalUrl: sampleDefaultFeed.FeedLink, 278 | expectedContent: sampleDefaultFeedItemExpectedContent + "\n\nComments: " + sampleDefaultFeedItemWithComments.Custom["comments"] + "\n\n" + sampleDefaultFeedItem.Link, 279 | maxContentLength: 1500, 280 | }, 281 | { 282 | pubKey: samplePubKey, 283 | item: &sampleStackerNewsFeedItem, 284 | feed: &sampleStackerNewsFeed, 285 | defaultCreatedAt: actualTime, 286 | originalUrl: sampleStackerNewsFeed.FeedLink, 287 | expectedContent: fmt.Sprintf("**%s**\n\nComments: %s\n\n%s", sampleStackerNewsFeedItem.Title, sampleStackerNewsFeedItem.GUID, sampleStackerNewsFeedItem.Link), 288 | maxContentLength: 250, 289 | }, 290 | } 291 | for _, tc := range testCases { 292 | event := ItemToTextNote(tc.pubKey, tc.item, tc.feed, tc.defaultCreatedAt, tc.originalUrl, tc.maxContentLength) 293 | assert.NotEmpty(t, event) 294 | assert.Equal(t, tc.pubKey, event.PubKey) 295 | assert.Equal(t, tc.defaultCreatedAt, event.CreatedAt.Time()) 296 | assert.Equal(t, 1, event.Kind) 297 | assert.Equal(t, tc.expectedContent, event.Content) 298 | assert.Empty(t, event.Sig) 299 | assert.NotEmpty(t, event.Tags) 300 | } 301 | } 302 | 303 | func TestDeleteExistingInvalidFeed(t *testing.T) { 304 | db, mock, err := sqlmock.New() 305 | if err != nil { 306 | t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) 307 | } 308 | defer func(db *sql.DB) { 309 | err := db.Close() 310 | if err != nil { 311 | t.Fatalf("an error '%s' was not expected when closing a stub database connection", err) 312 | } 313 | }(db) 314 | 315 | mock.ExpectExec("DELETE FROM feeds").WillReturnResult(sqlmock.NewResult(0, 1)) 316 | mock.ExpectClose() 317 | DeleteInvalidFeed(sampleUrlForPublicKey, db) 318 | } 319 | 320 | func TestDeleteNonExistingInvalidFeed(t *testing.T) { 321 | db, mock, err := sqlmock.New() 322 | if err != nil { 323 | t.Fatalf("an error '%s' was not expected when opening a stub database connection", err) 324 | } 325 | defer func(db *sql.DB) { 326 | err := db.Close() 327 | if err != nil { 328 | t.Fatalf("an error '%s' was not expected when closing a stub database connection", err) 329 | } 330 | }(db) 331 | 332 | mock.ExpectExec("DELETE FROM feeds").WillReturnError(errors.New("")) 333 | mock.ExpectClose() 334 | DeleteInvalidFeed(sampleUrlForPublicKey, db) 335 | } 336 | -------------------------------------------------------------------------------- /pkg/feed/translator.go: -------------------------------------------------------------------------------- 1 | package feed 2 | 3 | import ( 4 | "fmt" 5 | "github.com/mmcdole/gofeed" 6 | "github.com/mmcdole/gofeed/rss" 7 | ) 8 | 9 | type CustomTranslator struct { 10 | defaultRSSTranslator *gofeed.DefaultRSSTranslator 11 | } 12 | 13 | func NewCustomTranslator() *CustomTranslator { 14 | t := &CustomTranslator{} 15 | 16 | t.defaultRSSTranslator = &gofeed.DefaultRSSTranslator{} 17 | return t 18 | } 19 | 20 | func (ct *CustomTranslator) Translate(feed interface{}) (*gofeed.Feed, error) { 21 | rssFeed, found := feed.(*rss.Feed) 22 | if !found { 23 | return nil, fmt.Errorf("feed did not match expected type of *rss.Feed") 24 | } 25 | 26 | f, err := ct.defaultRSSTranslator.Translate(rssFeed) 27 | if err != nil { 28 | return nil, err 29 | } 30 | 31 | for i, item := range rssFeed.Items { 32 | if item.Comments != "" { 33 | if f.Items[i].Custom == nil { 34 | f.Items[i].Custom = map[string]string{} 35 | } 36 | f.Items[i].Custom["comments"] = item.Comments 37 | } 38 | } 39 | 40 | return f, nil 41 | } 42 | -------------------------------------------------------------------------------- /pkg/feed/translator_test.go: -------------------------------------------------------------------------------- 1 | package feed 2 | 3 | import ( 4 | "github.com/mmcdole/gofeed" 5 | "github.com/stretchr/testify/assert" 6 | "testing" 7 | ) 8 | 9 | const feedWithComments = ` 10 | 11 | Stacker News 12 | https://stacker.news 13 | Like Hacker News, but we pay you Bitcoin. 14 | en 15 | Sat, 18 Feb 2023 12:35:17 GMT 16 | 17 | 18 | https://stacker.news/items/138518 19 | What is your favourite Linux distribution, and why? 20 | https://stacker.news/items/138518 21 | https://stacker.news/items/138518 22 | 23 | Comments ]]> 24 | 25 | Fri, 17 Feb 2023 18:29:20 GMT 26 | 27 | 28 | ` 29 | 30 | const feedWithoutComments = ` 31 | 32 | Stacker News 33 | https://stacker.news 34 | Like Hacker News, but we pay you Bitcoin. 35 | en 36 | Sat, 18 Feb 2023 12:35:17 GMT 37 | 38 | 39 | https://stacker.news/items/138518 40 | What is your favourite Linux distribution, and why? 41 | https://stacker.news/items/138518 42 | 43 | Comments ]]> 44 | 45 | Fri, 17 Feb 2023 18:29:20 GMT 46 | 47 | 48 | ` 49 | 50 | func TestCustomTranslator_TranslateWithComments(t *testing.T) { 51 | fp := gofeed.NewParser() 52 | fp.RSSTranslator = NewCustomTranslator() 53 | feed, _ := fp.ParseString(feedWithComments) 54 | item := feed.Items[0] 55 | assert.NotNil(t, item.Custom) 56 | assert.NotNil(t, item.Custom["comments"]) 57 | assert.Equal(t, "https://stacker.news/items/138518", item.Custom["comments"]) 58 | } 59 | 60 | func TestCustomTranslator_TranslateWithoutComments(t *testing.T) { 61 | fp := gofeed.NewParser() 62 | fp.RSSTranslator = NewCustomTranslator() 63 | feed, _ := fp.ParseString(feedWithoutComments) 64 | item := feed.Items[0] 65 | assert.Nil(t, item.Custom) 66 | } 67 | -------------------------------------------------------------------------------- /pkg/helpers/helpers.go: -------------------------------------------------------------------------------- 1 | package helpers 2 | 3 | import ( 4 | "golang.org/x/exp/slices" 5 | "net/url" 6 | "path" 7 | ) 8 | 9 | var validSchemas = []string{"https", "http"} 10 | 11 | func UrlJoin(baseUrl string, elem ...string) (result string, err error) { 12 | u, err := url.Parse(baseUrl) 13 | if err != nil { 14 | return 15 | } 16 | 17 | if len(elem) > 0 { 18 | elem = append([]string{u.Path}, elem...) 19 | u.Path = path.Join(elem...) 20 | } 21 | 22 | return u.String(), nil 23 | } 24 | 25 | func IsValidHttpUrl(rawUrl string) bool { 26 | parsedUrl, err := url.ParseRequestURI(rawUrl) 27 | if parsedUrl == nil { 28 | return false 29 | } 30 | if err != nil || !slices.Contains(validSchemas, parsedUrl.Scheme) { 31 | return false 32 | } 33 | return true 34 | } 35 | -------------------------------------------------------------------------------- /pkg/helpers/helpers_test.go: -------------------------------------------------------------------------------- 1 | package helpers 2 | 3 | import ( 4 | "fmt" 5 | "github.com/stretchr/testify/assert" 6 | "testing" 7 | ) 8 | 9 | const sampleInvalidUrl = "https:// nostr.example/" 10 | const sampleValidUrl = "https://nostr.example" 11 | 12 | func TestJoinWithInvalidUrlReturnsNil(t *testing.T) { 13 | join, err := UrlJoin(sampleInvalidUrl) 14 | assert.Equal(t, join, "") 15 | assert.ErrorContains(t, err, "invalid character") 16 | } 17 | 18 | func TestJoinWithValidUrlAndNoExtraElementsReturnsBaseUrl(t *testing.T) { 19 | join, err := UrlJoin(sampleValidUrl) 20 | assert.Equal(t, sampleValidUrl, join) 21 | assert.NoError(t, err) 22 | } 23 | 24 | func TestJoinWithValidUrlAndExtraElementsReturnsValidUrl(t *testing.T) { 25 | join, err := UrlJoin(sampleValidUrl, "rss") 26 | expectedJoinResult := fmt.Sprintf("%s/%s", sampleValidUrl, "rss") 27 | assert.Equal(t, expectedJoinResult, join) 28 | assert.NoError(t, err) 29 | } 30 | 31 | func TestIsValidUrl(t *testing.T) { 32 | testCases := []struct { 33 | rawUrl string 34 | expectedValid bool 35 | }{ 36 | { 37 | rawUrl: "hi/there?", 38 | expectedValid: false, 39 | }, 40 | { 41 | rawUrl: "http://golang.cafe/", 42 | expectedValid: true, 43 | }, 44 | { 45 | rawUrl: "http://golang.org/index.html?#page1", 46 | expectedValid: true, 47 | }, 48 | { 49 | rawUrl: "golang.org", 50 | expectedValid: false, 51 | }, 52 | { 53 | rawUrl: "https://golang.cafe/", 54 | expectedValid: true, 55 | }, 56 | { 57 | rawUrl: "wss://nostr.moe", 58 | expectedValid: false, 59 | }, 60 | { 61 | rawUrl: "ftp://nostr.moe", 62 | expectedValid: false, 63 | }, 64 | } 65 | for _, tc := range testCases { 66 | isValid := IsValidHttpUrl(tc.rawUrl) 67 | if tc.expectedValid { 68 | assert.True(t, isValid) 69 | } else { 70 | assert.False(t, isValid) 71 | } 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /pkg/metrics/registries.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "github.com/prometheus/client_golang/prometheus" 5 | "github.com/prometheus/client_golang/prometheus/promauto" 6 | ) 7 | 8 | var ( 9 | IndexRequests = promauto.NewCounter(prometheus.CounterOpts{ 10 | Name: "rsslay_processed_index_ops_total", 11 | Help: "The total number of processed index requests", 12 | }) 13 | SearchRequests = promauto.NewCounter(prometheus.CounterOpts{ 14 | Name: "rsslay_processed_search_ops_total", 15 | Help: "The total number of processed search requests", 16 | }) 17 | CreateRequests = promauto.NewCounter(prometheus.CounterOpts{ 18 | Name: "rsslay_processed_create_ops_total", 19 | Help: "The total number of processed create feed requests", 20 | }) 21 | CreateRequestsAPI = promauto.NewCounter(prometheus.CounterOpts{ 22 | Name: "rsslay_processed_create_api_ops_total", 23 | Help: "The total number of processed create feed requests via API", 24 | }) 25 | WellKnownRequests = promauto.NewCounter(prometheus.CounterOpts{ 26 | Name: "rsslay_processed_wellknown_ops_total", 27 | Help: "The total number of processed well-known requests", 28 | }) 29 | RelayInfoRequests = promauto.NewCounter(prometheus.CounterOpts{ 30 | Name: "rsslay_processed_relay_info_ops_total", 31 | Help: "The total number of processed relay info requests", 32 | }) 33 | QueryEventsRequests = promauto.NewCounter(prometheus.CounterOpts{ 34 | Name: "rsslay_processed_query_events_ops_total", 35 | Help: "The total number of processed query events requests", 36 | }) 37 | InvalidEventsRequests = promauto.NewCounter(prometheus.CounterOpts{ 38 | Name: "rsslay_processed_invalid_events_ops_total", 39 | Help: "The total number of processed invalid events requests", 40 | }) 41 | ListeningFiltersOps = promauto.NewCounter(prometheus.CounterOpts{ 42 | Name: "rsslay_processed_listening_filters_ops_total", 43 | Help: "The total number of updated listening filters", 44 | }) 45 | CacheHits = promauto.NewCounter(prometheus.CounterOpts{ 46 | Name: "rsslay_processed_cache_hits_ops_total", 47 | Help: "The total number of cache hits", 48 | }) 49 | CacheMiss = promauto.NewCounter(prometheus.CounterOpts{ 50 | Name: "rsslay_processed_cache_miss_ops_total", 51 | Help: "The total number of cache misses", 52 | }) 53 | AppErrors = prometheus.NewCounterVec(prometheus.CounterOpts{ 54 | Name: "rsslay_errors_total", 55 | Help: "Number of errors for the app.", 56 | }, []string{"type"}) 57 | ReplayRoutineQueueLength = promauto.NewGauge(prometheus.GaugeOpts{ 58 | Name: "rsslay_replay_routines_queue_length", 59 | Help: "Current number of subroutines to replay events to other relays", 60 | }) 61 | ReplayEvents = prometheus.NewCounterVec(prometheus.CounterOpts{ 62 | Name: "rsslay_replay_events_total", 63 | Help: "Number of correct replayed events by relay.", 64 | }, []string{"relay"}) 65 | ReplayErrorEvents = prometheus.NewCounterVec(prometheus.CounterOpts{ 66 | Name: "rsslay_replay_events_error_total", 67 | Help: "Number of error replayed events by relay.", 68 | }, []string{"relay"}) 69 | ) 70 | -------------------------------------------------------------------------------- /pkg/replayer/replayer.go: -------------------------------------------------------------------------------- 1 | package replayer 2 | 3 | import ( 4 | "context" 5 | "github.com/nbd-wtf/go-nostr" 6 | "github.com/piraces/rsslay/pkg/metrics" 7 | "github.com/prometheus/client_golang/prometheus" 8 | "log" 9 | "sort" 10 | "sync" 11 | "time" 12 | ) 13 | 14 | type ReplayParameters struct { 15 | MaxEventsToReplay int 16 | RelaysToPublish []string 17 | Mutex *sync.Mutex 18 | Queue *int 19 | WaitTime int64 20 | WaitTimeForRelayResponse int64 21 | Events []EventWithPrivateKey 22 | } 23 | 24 | type EventWithPrivateKey struct { 25 | Event *nostr.Event 26 | PrivateKey string 27 | } 28 | 29 | func ReplayEventsToRelays(parameters *ReplayParameters) { 30 | eventCount := len(parameters.Events) 31 | if eventCount == 0 { 32 | return 33 | } 34 | 35 | if eventCount > parameters.MaxEventsToReplay { 36 | sort.Slice(parameters.Events, func(i, j int) bool { 37 | return parameters.Events[i].Event.CreatedAt > parameters.Events[j].Event.CreatedAt 38 | }) 39 | parameters.Events = parameters.Events[:parameters.MaxEventsToReplay] 40 | } 41 | 42 | go func() { 43 | parameters.Mutex.Lock() 44 | // publish the event to predefined relays 45 | for _, url := range parameters.RelaysToPublish { 46 | statusSummary := 0 47 | for _, ev := range parameters.Events { 48 | relay := connectToRelay(url, ev.PrivateKey) 49 | if relay == nil { 50 | continue 51 | } 52 | _ = relay.Close() 53 | 54 | publishStatus := publishEvent(relay, *ev.Event, url) 55 | statusSummary = statusSummary | int(publishStatus) 56 | } 57 | if statusSummary < 0 { 58 | log.Printf("[WARN] Replayed %d events to %s with failed status summary %d\n", len(parameters.Events), url, statusSummary) 59 | } else { 60 | log.Printf("[DEBUG] Replayed %d events to %s with status summary %d\n", len(parameters.Events), url, statusSummary) 61 | } 62 | } 63 | time.Sleep(time.Duration(parameters.WaitTime) * time.Millisecond) 64 | *parameters.Queue-- 65 | metrics.ReplayRoutineQueueLength.Set(float64(*parameters.Queue)) 66 | parameters.Mutex.Unlock() 67 | }() 68 | } 69 | 70 | func publishEvent(relay *nostr.Relay, ev nostr.Event, url string) nostr.Status { 71 | publishStatus, err := relay.Publish(context.Background(), ev) 72 | switch publishStatus { 73 | case nostr.PublishStatusSent: 74 | metrics.ReplayEvents.With(prometheus.Labels{"relay": url}).Inc() 75 | break 76 | default: 77 | metrics.ReplayErrorEvents.With(prometheus.Labels{"relay": url}).Inc() 78 | break 79 | } 80 | _ = relay.Close() 81 | if err != nil { 82 | log.Printf("[INFO] Failed to replay event to %s with error: %v", url, err) 83 | } 84 | return publishStatus 85 | } 86 | 87 | func connectToRelay(url string, privateKey string) *nostr.Relay { 88 | relay, e := nostr.RelayConnect(context.Background(), url, nostr.WithAuthHandler(func(ctx context.Context, authEvent *nostr.Event) (ok bool) { 89 | err := authEvent.Sign(privateKey) 90 | if err != nil { 91 | log.Printf("[ERROR] Error while trying to authenticate with relay '%s': %v", url, err) 92 | return false 93 | } 94 | return true 95 | }), 96 | ) 97 | if e != nil { 98 | log.Printf("[ERROR] Error while trying to connect with relay '%s': %v", url, e) 99 | metrics.AppErrors.With(prometheus.Labels{"type": "REPLAY_CONNECT"}).Inc() 100 | return nil 101 | } 102 | 103 | return relay 104 | } 105 | -------------------------------------------------------------------------------- /screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/piraces/rsslay/6b7d062a1e6cd0e9ebdf90c88828d31c974b36b4/screenshot.png -------------------------------------------------------------------------------- /scripts/check_nitter_column.sql: -------------------------------------------------------------------------------- 1 | SELECT nitter from feeds -------------------------------------------------------------------------------- /scripts/create_nitter_column.sql: -------------------------------------------------------------------------------- 1 | ALTER TABLE feeds ADD COLUMN nitter INTEGER DEFAULT 0 -------------------------------------------------------------------------------- /scripts/schema.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE IF NOT EXISTS feeds ( 2 | publickey VARCHAR(64) PRIMARY KEY, 3 | privatekey VARCHAR(64) NOT NULL, 4 | url TEXT NOT NULL, 5 | nitter INTEGER DEFAULT 0 6 | ); 7 | 8 | -------------------------------------------------------------------------------- /scripts/scripts.go: -------------------------------------------------------------------------------- 1 | package scripts 2 | 3 | import _ "embed" 4 | 5 | //go:embed schema.sql 6 | var SchemaSQL string 7 | 8 | //go:embed check_nitter_column.sql 9 | var CheckNitterColumnSQL string 10 | 11 | //go:embed create_nitter_column.sql 12 | var CreateNitterColumnSQL string 13 | -------------------------------------------------------------------------------- /web/assets/images/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/piraces/rsslay/6b7d062a1e6cd0e9ebdf90c88828d31c974b36b4/web/assets/images/favicon.ico -------------------------------------------------------------------------------- /web/assets/images/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/piraces/rsslay/6b7d062a1e6cd0e9ebdf90c88828d31c974b36b4/web/assets/images/logo.png -------------------------------------------------------------------------------- /web/assets/js/copyclipboard.js: -------------------------------------------------------------------------------- 1 | async function copyToClipboard(name) { 2 | const input = document.getElementById(name); 3 | input.select(); 4 | let text = input.value; 5 | try { 6 | await navigator.clipboard.writeText(text); 7 | } catch (err) { 8 | console.error('Failed to copy: ', err); 9 | } 10 | } -------------------------------------------------------------------------------- /web/assets/js/nostr.js: -------------------------------------------------------------------------------- 1 | const rsslayPubKeyStorageKey = "rsslay.publicKey"; 2 | const rsslayRelaysStorageKey = "rsslay.relays"; 3 | 4 | const loginButton = document.getElementById('login'); 5 | const logoutButton = document.getElementById('logout'); 6 | const loginButtonText = document.getElementById('login-text'); 7 | 8 | let pubKey; 9 | let relays; 10 | let relaysUrls; 11 | let subs; 12 | let followListEvent; 13 | let pool; 14 | 15 | function tryAddToFollowList(pubKeyToFollow) { 16 | if (followListEvent) { 17 | const newFollowTag = ["p", pubKeyToFollow]; 18 | const tagsSet = new Set(followListEvent.tags); 19 | let found = false; 20 | followListEvent.tags.forEach((value) => { 21 | if (value[1] === pubKeyToFollow){ 22 | found = true; 23 | } 24 | }); 25 | if (found){ 26 | return followListEvent.tags; 27 | } 28 | tagsSet.add(newFollowTag); 29 | return [...tagsSet]; 30 | } else { 31 | swal({ 32 | title: "Connecting...", 33 | text: "Waiting for relays to retrieve your contact list. Please wait a few seconds and try again...", 34 | icon: "error", 35 | button: "Ok", 36 | }); 37 | } 38 | } 39 | 40 | function tryRemoveFromFollowList(publicKeyToUnfollow) { 41 | if (followListEvent) { 42 | const tagsSet = new Set(); 43 | followListEvent.tags.forEach((value) => { 44 | if (value[1] !== publicKeyToUnfollow){ 45 | tagsSet.add(value); 46 | } 47 | }); 48 | return [...tagsSet]; 49 | } else { 50 | swal({ 51 | title: "Connecting...", 52 | text: "Waiting for relays to retrieve your contact list. Please wait a few seconds and try again...", 53 | icon: "error", 54 | button: "Ok", 55 | }); 56 | } 57 | } 58 | 59 | async function tryUnfollow(pubKey) { 60 | if (pubKey) { 61 | const loggedIn = checkLogin(); 62 | if (!loggedIn){ 63 | await performLogin(); 64 | } 65 | let event = { 66 | kind: 3, 67 | created_at: Math.floor(Date.now() / 1000), 68 | tags: tryRemoveFromFollowList(pubKey), 69 | content: JSON.stringify(relays), 70 | } 71 | const signedEvent = await window.nostr.signEvent(event); 72 | 73 | let ok = window.NostrTools.validateEvent(signedEvent); 74 | let veryOk = window.NostrTools.verifySignature(signedEvent); 75 | if (ok && veryOk){ 76 | let alerted = false; 77 | let pubs = pool.publish(relaysUrls, signedEvent); 78 | pubs.forEach(pub => { 79 | pub.on('ok', () => { 80 | if (!alerted) { 81 | alerted = true; 82 | swal({ 83 | title: "Not following", 84 | text: "Your contact list has been updated, you're now no longer following the profile.", 85 | icon: "success", 86 | button: "Ok", 87 | }); 88 | const followButton = document.getElementById(pubKey); 89 | followButton.classList.remove("is-danger"); 90 | followButton.classList.add("is-link"); 91 | followButton.setAttribute('onclick', `tryFollow("${pubKey}")`); 92 | followButton.textContent = "Follow profile"; 93 | } 94 | }) 95 | pub.on('seen', () => { 96 | console.log(`we saw the event!`); 97 | }) 98 | pub.on('failed', reason => { 99 | console.log(`failed to publish: ${reason}`); 100 | }) 101 | }); 102 | } 103 | } 104 | } 105 | 106 | async function tryFollow(pubKey) { 107 | if (pubKey) { 108 | const loggedIn = checkLogin(); 109 | if (!loggedIn){ 110 | await performLogin(); 111 | } 112 | let event = { 113 | kind: 3, 114 | created_at: Math.floor(Date.now() / 1000), 115 | tags: tryAddToFollowList(pubKey), 116 | content: JSON.stringify(relays), 117 | } 118 | const signedEvent = await window.nostr.signEvent(event); 119 | 120 | let ok = window.NostrTools.validateEvent(signedEvent); 121 | let veryOk = window.NostrTools.verifySignature(signedEvent); 122 | if (ok && veryOk){ 123 | let alerted = false; 124 | let pubs = pool.publish(relaysUrls, signedEvent); 125 | pubs.forEach(pub => { 126 | pub.on('ok', () => { 127 | if (!alerted){ 128 | alerted = true; 129 | swal({ 130 | title: "Followed!", 131 | text: "Your contact list has been updated, you're now following the profile.", 132 | icon: "success", 133 | button: "Ok", 134 | }); 135 | const followButton = document.getElementById(pubKey); 136 | followButton.classList.remove("is-link"); 137 | followButton.classList.add("is-danger"); 138 | followButton.setAttribute('onclick', `tryUnfollow("${pubKey}")`); 139 | followButton.textContent = "Unfollow"; 140 | } 141 | }) 142 | pub.on('seen', () => { 143 | console.log(`we saw the event!`); 144 | }) 145 | pub.on('failed', reason => { 146 | console.log(`failed to publish: ${reason}`); 147 | }) 148 | }); 149 | } 150 | } 151 | } 152 | 153 | function parseFollowList(followListEvent) { 154 | const profilesPubKeys = new Set(); 155 | followListEvent.tags.forEach((tag) => { 156 | profilesPubKeys.add(tag[1]); 157 | }); 158 | profilesPubKeys.forEach((pubKey) => { 159 | const followButton = document.getElementById(pubKey); 160 | if (followButton) { 161 | followButton.classList.remove("is-link"); 162 | followButton.classList.add("is-danger"); 163 | followButton.setAttribute('onclick', `tryUnfollow("${pubKey}")`); 164 | followButton.textContent = "Unfollow"; 165 | } 166 | }); 167 | } 168 | 169 | function connectToRelays(){ 170 | pool = new window.NostrTools.SimplePool() 171 | 172 | subs = pool.sub([...relaysUrls], [{ 173 | authors: [pubKey], 174 | kind: 3, 175 | }]); 176 | 177 | subs.on('event', event => { 178 | if (event.kind === 3){ 179 | followListEvent = event; 180 | relays = JSON.parse(event.content); 181 | relaysUrls = Object.keys(relays); 182 | sessionStorage.setItem(rsslayRelaysStorageKey, JSON.stringify(relays)); 183 | parseFollowList(followListEvent); 184 | } 185 | }); 186 | } 187 | 188 | function checkLogin(){ 189 | if (typeof window.nostr !== 'undefined') { 190 | pubKey = sessionStorage.getItem(rsslayPubKeyStorageKey); 191 | relays = sessionStorage.getItem(rsslayRelaysStorageKey); 192 | if (pubKey && relays){ 193 | relays = JSON.parse(relays); 194 | relaysUrls = Object.keys(relays); 195 | afterLogin(); 196 | return true; 197 | } 198 | } 199 | return false; 200 | } 201 | 202 | async function performLogin() { 203 | if (typeof window.nostr !== 'undefined') { 204 | pubKey = sessionStorage.getItem(rsslayPubKeyStorageKey); 205 | relays = sessionStorage.getItem(rsslayRelaysStorageKey); 206 | if (!pubKey || !relays){ 207 | try { 208 | pubKey = await window.nostr.getPublicKey(); 209 | relays = await window.nostr.getRelays(); 210 | sessionStorage.setItem(rsslayPubKeyStorageKey, pubKey); 211 | sessionStorage.setItem(rsslayRelaysStorageKey, JSON.stringify(relays)); 212 | } catch (e) { 213 | swal({ 214 | title: "Oops...", 215 | text: "There was a problem to obtain public info from your profile... Try again later and make sure to grant correct permissions in your extension", 216 | icon: "error", 217 | button: "Ok", 218 | }); 219 | return; 220 | } 221 | } else { 222 | relays = JSON.parse(relays); 223 | } 224 | relaysUrls = Object.keys(relays); 225 | afterLogin(); 226 | } else { 227 | swal({ 228 | title: "Oops...", 229 | text: "There was a problem to obtain your public key... Try again later and make sure to grant correct permissions in your extension", 230 | icon: "error", 231 | button: "Ok", 232 | }); 233 | } 234 | } 235 | 236 | async function performLogout() { 237 | loginButton.disabled = false; 238 | loginButton.addEventListener('click', performLogin); 239 | loginButtonText.textContent = "Login"; 240 | logoutButton.disabled = true; 241 | logoutButton.removeEventListener('click', performLogout); 242 | sessionStorage.clear(); 243 | subs.unsub(); 244 | pool.close([...relaysUrls]); 245 | } 246 | 247 | function afterLogin() { 248 | loginButton.disabled = true; 249 | loginButton.removeEventListener('click', performLogin); 250 | loginButtonText.textContent = "Logged in!"; 251 | logoutButton.disabled = false; 252 | logoutButton.addEventListener('click', performLogout); 253 | connectToRelays(); 254 | } -------------------------------------------------------------------------------- /web/templates/created.html.tmpl: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | rsslay 10 | 11 | 12 | 13 | 54 | 55 |
56 |
57 |

rsslay

58 |

rsslay turns RSS or Atom feeds into Nostr profiles.

60 |
61 |
62 |
63 | {{if .Error}} 64 |
65 | {{.ErrorMessage}} 66 |
67 | {{else}} 68 |
69 | 70 |
71 |
72 | 73 |
74 |
75 |
76 |

77 | 78 |

79 |
80 | 85 |
86 |
87 |
88 |
89 | 90 |
91 |
92 | 93 |
94 |
95 |
96 |

97 | 98 |

99 |
100 | 105 |
106 |
107 |
108 |
109 | 110 |
111 |
112 | 113 |
114 |
115 |
116 |

117 | 118 |

119 |
120 | 125 |
126 |
127 |
128 |
129 |
130 | View in astral.ninja 131 | View in iris.to 132 | View in snort.social 133 | Open in default app 134 | Follow profile 135 |
136 |
137 | {{end}} 138 | 139 | 140 | 141 | 142 | Go home 143 | 144 |
145 | 154 | 155 | 156 | 157 | 158 | 177 | 178 | 179 | -------------------------------------------------------------------------------- /web/templates/index.html.tmpl: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | rsslay 10 | 11 | 12 | 13 | 54 | 55 |
56 |
57 |

rsslay

58 |

rsslay turns RSS or Atom feeds into Nostr profiles.

60 |
61 |
62 |
63 | 71 |

How to use

72 |
73 |
    74 |
  1. Get the blog URL or RSS or Atom feed URL and paste it below.
    2. 75 |
  2. Click the button to get its corresponding public key.
    3. 76 |
  3. Add the following relay to your Nostr client: wss://{{.MainDomainName}}
    4. 77 |
  4. Follow the feed's public key from your Nostr client.
    5. 78 |
79 |
80 |

RSS is powerful!

81 |
82 |

🍰 Everything is RSSible

83 | 110 |
111 |
112 |

Create a profile for a RSS feed:

113 |
114 |
115 |
116 | 118 |
119 |
120 | 126 |
127 |
128 |
129 |
130 |

Some of the existing feeds (50 random selected)

131 |
132 |
133 |

Search by URL (if you want to search by key you can use a normal 134 | client):

135 |
136 |
137 | 138 |
139 |
140 | 146 |
147 |
148 |
149 |
150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | {{range .Entries}} 159 | 160 | 162 | 164 | 166 | 175 | 176 | {{end}} 177 | 178 |
Public key (Hex)Public keyFeed URLView in clients
{{.PubKey}} 161 | {{.NPubKey}} 163 | {{.Url}} 165 | 167 |
168 | View in astral.ninja 169 | View in iris.to 170 | View in snort.social 171 | Open in default app 172 | Follow profile 173 |
174 |
179 |

Source Code

180 |

You can find it at github.com/piraces/rsslay

181 |

Upstream source code at github.com/fiatjaf/relayer 182 |

183 |
184 | 193 | 194 | 195 | 196 | 212 | 213 | 214 | 215 | -------------------------------------------------------------------------------- /web/templates/search.html.tmpl: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | rsslay 10 | 11 | 12 | 13 | 54 |
55 |
56 |

rsslay

57 |

rsslay turns RSS or Atom feeds into Nostr profiles.

59 |
60 |
61 |
62 | 76 |

How to use

77 |
78 |
    79 |
  1. Get the blog URL or RSS or Atom feed URL and paste it below.
    2. 80 |
  2. Click the button to get its corresponding public key.
    3. 81 |
  3. Add this relay to your Nostr client.
    4. 82 |
  4. Follow the feed's public key from your Nostr client.
    5. 83 |
84 |
85 |

RSS is powerful!

86 |
87 |

🍰 Everything is RSSible

88 | 112 |
113 |
114 |

Create a profile for a RSS feed:

115 |
116 |
117 |
118 | 120 |
121 |
122 | 128 |
129 |
130 |
131 |
132 |

Found feeds (showing a maximum of 50, refine your query if necessary)

133 |
134 |
135 |

Search by URL (if you want to search by key you can use a normal 136 | client):

137 |
138 |
139 | 140 |
141 |
142 | 148 |
149 |
150 |
151 |
152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | {{range .Entries}} 161 | 162 | 164 | 166 | 168 | 177 | 178 | {{end}} 179 | 180 |
Public key (Hex)Public keyFeed URLView in clients
{{.PubKey}} 163 | {{.NPubKey}} 165 | {{.Url}} 167 | 169 |
170 | View in astral.ninja 171 | View in iris.to 172 | View in snort.social 173 | Open in default app 174 | Follow profile 175 |
176 |
181 |

Source Code

182 |

You can find it at github.com/piraces/rsslay

183 |

Upstream source code at github.com/fiatjaf/relayer 184 |

185 |
186 | 195 | 196 | 197 | 198 | 214 | 215 | 216 | -------------------------------------------------------------------------------- /web/templates/templates.go: -------------------------------------------------------------------------------- 1 | package templates 2 | 3 | import "embed" 4 | 5 | //go:embed *.tmpl 6 | var Templates embed.FS 7 | --------------------------------------------------------------------------------