├── .docker ├── dgraph │ ├── .gitignore │ ├── docker-compose.multi.yml │ └── docker-compose.yml ├── elasticsearch │ ├── 5.6 │ │ ├── .dockerignore │ │ ├── Dockerfile │ │ ├── VERSION │ │ ├── config │ │ │ ├── elastic │ │ │ │ ├── elasticsearch.yml │ │ │ │ └── log4j2.properties │ │ │ └── logrotate │ │ ├── docker-healthcheck │ │ ├── elastic-entrypoint.sh │ │ └── hooks │ │ │ └── post_push │ └── 7.0.1 │ │ ├── Dockerfile │ │ ├── config │ │ ├── elasticsearch.yml │ │ ├── jvm.options │ │ └── log4j2.properties │ │ └── scripts │ │ └── docker-entrypoint.sh └── readeef │ ├── config │ └── readeef.toml │ └── scripts │ ├── build.sh │ └── entrypoint.sh ├── .dockerignore ├── .env ├── .gitignore ├── DOCKER.md ├── Dockerfile ├── Dockerfile.dev ├── LICENSE ├── Makefile ├── README.md ├── TODO ├── api ├── api.go ├── article.go ├── article_test.go ├── articlerepotype_string.go ├── articlestate_string.go ├── auth.go ├── auth_test.go ├── events.go ├── events_test.go ├── extract_generator_mock_test.go ├── features.go ├── features_test.go ├── feed.go ├── feed_manager_mock_test.go ├── feed_test.go ├── fever │ ├── favicon.go │ ├── feeds.go │ ├── groups.go │ ├── handler.go │ ├── ids.go │ ├── items.go │ ├── links.go │ ├── mark.go │ └── user.go ├── hubbub.go ├── hubbub_test.go ├── opml.go ├── opml_test.go ├── processor_article_mock_test.go ├── request.go ├── request_test.go ├── searcher_mock_test.go ├── tag.go ├── tag_test.go ├── token │ ├── bolt_storage.go │ └── token.go ├── token_storage_mock_test.go ├── ttrss │ ├── articles.go │ ├── auth.go │ ├── conversion.go │ ├── counters.go │ ├── feeds.go │ ├── generic.go │ ├── handler.go │ ├── session.go │ ├── settings.go │ └── tags.go ├── user.go ├── user_settings.go ├── user_settings_test.go └── user_test.go ├── cmd ├── readeef-client │ ├── config.go │ ├── main.go │ ├── open.go │ ├── platform_darwin.go │ ├── platform_linux.go │ └── platform_windows.go ├── readeef-static-locator │ └── main.go └── readeef │ ├── main.go │ ├── postgres.go │ ├── search-index.go │ ├── server.go │ ├── sqlite.go │ └── user-admin.go ├── config ├── config.go ├── default.go └── parts.go ├── content ├── article.go ├── article_test.go ├── err.go ├── extract.go ├── extract │ ├── extract.go │ ├── goose.go │ └── readability.go ├── extract_test.go ├── feed.go ├── feed_test.go ├── monitor │ ├── index.go │ ├── thumbnailer.go │ ├── unread.go │ └── user-filters.go ├── processor │ ├── absolutize_urls.go │ ├── cleanup.go │ ├── cleanup_test.go │ ├── common_test.go │ ├── insert_thumbnail_target.go │ ├── processor.go │ ├── proxy_http.go │ ├── relative_url.go │ ├── top_image_marker.go │ └── unescape.go ├── repo │ ├── article.go │ ├── article_test.go │ ├── common_test.go │ ├── eventable │ │ ├── article.go │ │ ├── bus.go │ │ ├── bus_test.go │ │ ├── feed.go │ │ └── service.go │ ├── extract.go │ ├── extract_test.go │ ├── feed.go │ ├── feed_test.go │ ├── logging │ │ ├── article.go │ │ ├── extract.go │ │ ├── feed.go │ │ ├── scores.go │ │ ├── service.go │ │ ├── subscription.go │ │ ├── tag.go │ │ ├── thumbnail.go │ │ └── user.go │ ├── mock_repo │ │ ├── article.go │ │ ├── extract.go │ │ ├── feed.go │ │ ├── scores.go │ │ ├── service.go │ │ ├── subscription.go │ │ ├── tag.go │ │ ├── thumbnail.go │ │ └── user.go │ ├── postgres_test.go │ ├── scores.go │ ├── scores_test.go │ ├── service.go │ ├── service_test.go │ ├── sql │ │ ├── article.go │ │ ├── db │ │ │ ├── base │ │ │ │ ├── article.go │ │ │ │ ├── extract.go │ │ │ │ ├── feed.go │ │ │ │ ├── helper.go │ │ │ │ ├── scores.go │ │ │ │ ├── subscription.go │ │ │ │ ├── tag.go │ │ │ │ ├── thumbnail.go │ │ │ │ └── user.go │ │ │ ├── db.go │ │ │ ├── helper.go │ │ │ ├── postgres │ │ │ │ ├── helper.go │ │ │ │ └── init.go │ │ │ └── sqlite3 │ │ │ │ ├── cgo.go │ │ │ │ ├── helper.go │ │ │ │ └── init.go │ │ ├── extract.go │ │ ├── feed.go │ │ ├── scores.go │ │ ├── service.go │ │ ├── subscription.go │ │ ├── tag.go │ │ ├── thumbnail.go │ │ └── user.go │ ├── sqlite3_test.go │ ├── subscription.go │ ├── subscription_test.go │ ├── tag.go │ ├── tag_test.go │ ├── thumbnail.go │ ├── thumbnail_test.go │ ├── user.go │ └── user_test.go ├── scores.go ├── scores_test.go ├── search │ ├── bleve.go │ ├── elastic.go │ └── search.go ├── subscription.go ├── subscription_test.go ├── tag.go ├── tag_test.go ├── thumbnail.go ├── thumbnail │ ├── description.go │ ├── extract.go │ └── thumbnail.go ├── thumbnail_test.go ├── user.go ├── user_test.go └── validation.go ├── docker-compose.yml ├── feed ├── favicon.go ├── favicon_test.go ├── scheduler.go ├── scheduler_test.go └── search.go ├── feed_manager.go ├── feed_manager_test.go ├── file.list ├── fs.go ├── fs_files.go ├── fs_files_nofs.go ├── go.mod ├── go.sum ├── hubbub.go ├── hubbub_test.go ├── internal └── legacy │ └── config.go ├── log ├── log.go ├── logrus.go └── std.go ├── parser ├── atom.go ├── atom_test.go ├── feed.go ├── opml.go ├── opml_test.go ├── parser.go ├── parser_test.go ├── pubsub.go ├── rss.go ├── rss1.go ├── rss1_test.go ├── rss2.go └── rss2_test.go ├── pool └── buffer.go ├── popularity ├── link.go ├── popularity.go ├── reddit.go └── twitter.go ├── rf-ng ├── .dockerignore ├── .editorconfig ├── .gitignore ├── Dockerfile ├── Makefile ├── README.md ├── angular.json ├── e2e │ ├── app.e2e-spec.ts │ ├── app.po.ts │ └── tsconfig.e2e.json ├── karma.conf.js ├── ngsw-config.json ├── package-lock.json ├── package.json ├── protractor.conf.js ├── src │ ├── app │ │ ├── app.module.ts │ │ ├── app.routing.ts │ │ ├── article-display │ │ │ ├── article-display.component.ts │ │ │ ├── article-display.css │ │ │ └── article-display.html │ │ ├── article-list │ │ │ ├── article-list.component.ts │ │ │ ├── article-list.css │ │ │ ├── article-list.html │ │ │ ├── list-item.component.ts │ │ │ ├── list-item.css │ │ │ └── list-item.html │ │ ├── components │ │ │ ├── app.css │ │ │ ├── app.html │ │ │ ├── app.spec.ts │ │ │ └── app.ts │ │ ├── guards │ │ │ └── auth.ts │ │ ├── login │ │ │ ├── login.component.ts │ │ │ ├── login.css │ │ │ └── login.html │ │ ├── main │ │ │ ├── main.component.ts │ │ │ ├── main.css │ │ │ ├── main.html │ │ │ └── routing-util.ts │ │ ├── services │ │ │ ├── api.ts │ │ │ ├── article.ts │ │ │ ├── auth.ts │ │ │ ├── events.ts │ │ │ ├── favicon.ts │ │ │ ├── features.ts │ │ │ ├── feed.ts │ │ │ ├── interaction.ts │ │ │ ├── preferences.ts │ │ │ ├── sharing.ts │ │ │ ├── tag.ts │ │ │ └── user.ts │ │ ├── settings │ │ │ ├── admin │ │ │ │ ├── admin.component.ts │ │ │ │ ├── admin.css │ │ │ │ ├── admin.html │ │ │ │ └── new-user.html │ │ │ ├── common.css │ │ │ ├── discovery │ │ │ │ ├── discovery.component.ts │ │ │ │ ├── discovery.css │ │ │ │ └── discovery.html │ │ │ ├── filters │ │ │ │ ├── filters.component.ts │ │ │ │ ├── filters.css │ │ │ │ ├── filters.html │ │ │ │ └── new-filter-dialog.html │ │ │ ├── general │ │ │ │ ├── general.component.ts │ │ │ │ ├── general.css │ │ │ │ ├── general.html │ │ │ │ └── password-form.html │ │ │ ├── management │ │ │ │ ├── error-dialog.html │ │ │ │ ├── management.component.ts │ │ │ │ ├── management.css │ │ │ │ └── management.html │ │ │ ├── settings.component.ts │ │ │ ├── settings.css │ │ │ ├── settings.html │ │ │ └── share-services │ │ │ │ ├── share-services.component.ts │ │ │ │ ├── share-services.css │ │ │ │ └── share-services.html │ │ ├── share-service │ │ │ └── share-service.component.ts │ │ ├── sidebar │ │ │ ├── side-bar-feed.html │ │ │ ├── side-bar-settings.html │ │ │ ├── side-bar.css │ │ │ ├── sidebar.feed.component.ts │ │ │ └── sidebar.settings.component.ts │ │ └── toolbar │ │ │ ├── toolbar-feed.html │ │ │ ├── toolbar-settings.html │ │ │ ├── toolbar.css │ │ │ ├── toolbar.feed.component.ts │ │ │ └── toolbar.settings.component.ts │ ├── assets │ │ ├── .gitkeep │ │ └── icons │ │ │ ├── readeef-114.png │ │ │ ├── readeef-144.png │ │ │ ├── readeef-72.png │ │ │ ├── readeef-small.png │ │ │ └── readeef.png │ ├── environments │ │ ├── environment.prod.ts │ │ └── environment.ts │ ├── gesture-config.ts │ ├── index.html │ ├── locale │ │ ├── messages.bg.xlf │ │ ├── messages.en.xlf │ │ └── messages.xlf │ ├── main.ts │ ├── manifest.webmanifest │ ├── polyfills.ts │ ├── styles.css │ ├── test.ts │ ├── tsconfig.app.json │ ├── tsconfig.spec.json │ └── typings.d.ts ├── tsconfig.app.json ├── tsconfig.json ├── tsconfig.spec.json ├── tslint.json ├── ui │ └── index.html ├── xliffmerge.json └── yarn.lock ├── systemd └── readeef.service ├── templates ├── goose-format-result.tmpl └── raw.tmpl ├── timeout_client.go └── web ├── proxy.go └── web.go /.docker/dgraph/.gitignore: -------------------------------------------------------------------------------- 1 | data0 2 | data1 3 | data2 4 | data3 5 | -------------------------------------------------------------------------------- /.docker/dgraph/docker-compose.multi.yml: -------------------------------------------------------------------------------- 1 | --- 2 | version: "3.7" 3 | services: 4 | 5 | zero: 6 | image: dgraph/dgraph:latest 7 | volumes: 8 | - ./data0/data-volume:/dgraph 9 | ports: 10 | - 5080:5080 11 | - 6080:6080 12 | command: dgraph zero --my=zero:5080 --replicas 3 13 | 14 | server_1: 15 | image: dgraph/dgraph:latest 16 | hostname: "server_1" 17 | volumes: 18 | - ./data1/data-volume:/dgraph 19 | ports: 20 | - 8080:8080 21 | - 9080:9080 22 | command: dgraph server --my=server_1:7080 --lru_mb=4096 --zero=zero:5080 23 | 24 | server_2: 25 | image: dgraph/dgraph:latest 26 | hostname: "server_2" 27 | volumes: 28 | - ./data2/data-volume:/dgraph 29 | ports: 30 | - 8081:8081 31 | - 9081:9081 32 | command: dgraph server --my=server_2:7081 --lru_mb=4096 --zero=zero:5080 -o 1 33 | 34 | server_3: 35 | image: dgraph/dgraph:latest 36 | hostname: "server_3" 37 | volumes: 38 | - ./data3/data-volume:/dgraph 39 | ports: 40 | - 8082:8082 41 | - 9082:9082 42 | command: dgraph server --my=server_3:7082 --lru_mb=4096 --zero=zero:5080 -o 2 43 | 44 | ratel: 45 | image: dgraph/dgraph:latest 46 | hostname: "ratel" 47 | ports: 48 | - 8000:8000 49 | command: dgraph-ratel 50 | -------------------------------------------------------------------------------- /.docker/dgraph/docker-compose.yml: -------------------------------------------------------------------------------- 1 | --- 2 | version: "3.7" 3 | services: 4 | 5 | zero: 6 | image: dgraph/dgraph:v${DGRAPH_VERSION} 7 | volumes: 8 | - type: volume 9 | source: dgraph 10 | target: /dgraph 11 | volume: 12 | nocopy: true 13 | ports: 14 | - 5080:5080 15 | - 6080:6080 16 | restart: on-failure 17 | command: dgraph zero --my=zero:5080 18 | 19 | server: 20 | image: dgraph/dgraph:v${DGRAPH_VERSION} 21 | volumes: 22 | - type: volume 23 | source: dgraph 24 | target: /dgraph 25 | volume: 26 | nocopy: true 27 | ports: 28 | - 8080:8080 29 | - 9080:9080 30 | restart: on-failure 31 | command: dgraph alpha --my=server:7080 --lru_mb=2048 --zero=zero:5080 32 | 33 | ratel: 34 | image: dgraph/dgraph:v1.1.0 35 | volumes: 36 | - type: volume 37 | source: dgraph 38 | target: /dgraph 39 | volume: 40 | nocopy: true 41 | ports: 42 | - 8000:8000 43 | command: dgraph-ratel 44 | 45 | volumes: 46 | dgraph: -------------------------------------------------------------------------------- /.docker/elasticsearch/5.6/.dockerignore: -------------------------------------------------------------------------------- 1 | # Ignore .git folder 2 | .git* 3 | .gitignore 4 | 5 | es-logo.png 6 | LICENSE 7 | README.md 8 | docker-compose.yml 9 | .DS_Store 10 | build 11 | release 12 | Makefile 13 | -------------------------------------------------------------------------------- /.docker/elasticsearch/5.6/VERSION: -------------------------------------------------------------------------------- 1 | 5.6 2 | -------------------------------------------------------------------------------- /.docker/elasticsearch/5.6/config/elastic/elasticsearch.yml: -------------------------------------------------------------------------------- 1 | network.host: 0.0.0.0 2 | 3 | # this value is required because we set "network.host" 4 | # be sure to modify it appropriately for a production cluster deployment 5 | discovery.zen.minimum_master_nodes: 1 6 | # bootstrap.memory_lock: true 7 | 8 | node.master: true 9 | node.ingest: true 10 | node.data: true 11 | -------------------------------------------------------------------------------- /.docker/elasticsearch/5.6/config/elastic/log4j2.properties: -------------------------------------------------------------------------------- 1 | status = error 2 | 3 | appender.console.type = Console 4 | appender.console.name = console 5 | appender.console.layout.type = PatternLayout 6 | appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n 7 | 8 | rootLogger.level = info 9 | rootLogger.appenderRef.console.ref = console 10 | -------------------------------------------------------------------------------- /.docker/elasticsearch/5.6/config/logrotate: -------------------------------------------------------------------------------- 1 | /var/log/elasticsearch/*.log { 2 | daily 3 | rotate 50 4 | size 50M 5 | copytruncate 6 | compress 7 | delaycompress 8 | missingok 9 | notifempty 10 | create 644 elasticsearch elasticsearch 11 | } 12 | -------------------------------------------------------------------------------- /.docker/elasticsearch/5.6/docker-healthcheck: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -eo pipefail 3 | 4 | host="$(hostname --ip-address || echo '127.0.0.1')" 5 | 6 | if health="$(curl -fsSL "http://$host:9200/_cat/health?h=status")"; then 7 | health="$(echo "$health" | sed -r 's/^[[:space:]]+|[[:space:]]+$//g')" # trim whitespace (otherwise we'll have "green ") 8 | if [ "$health" = 'green' ]; then 9 | exit 0 10 | fi 11 | echo >&2 "unexpected health status: $health" 12 | fi 13 | 14 | # If the probe returns 2 ("starting") when the container has already moved out of the "starting" state then it is treated as "unhealthy" instead. 15 | # https://github.com/docker/docker/blob/dcc65376bac8e73bb5930fce4cddc2350bb7baa2/docs/reference/builder.md#healthcheck 16 | exit 2 17 | -------------------------------------------------------------------------------- /.docker/elasticsearch/5.6/hooks/post_push: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | VERSION=$(cat Dockerfile | grep '^ENV VERSION' | cut -d" " -f3) 6 | TAGS=($VERSION 5) 7 | 8 | for TAG in "${TAGS[@]}"; do 9 | echo "===> Tagging $IMAGE_NAME as $DOCKER_REPO:$TAG" 10 | docker tag $IMAGE_NAME $DOCKER_REPO:$TAG 11 | echo "===> Pushing $DOCKER_REPO:$TAG" 12 | docker push $DOCKER_REPO:$TAG 13 | done 14 | -------------------------------------------------------------------------------- /.docker/elasticsearch/7.0.1/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM docker.elastic.co/elasticsearch/elasticsearch:7.0.1 2 | 3 | USER elasticsearch 4 | 5 | # Copy config files 6 | COPY ./config/jvm.options /usr/share/elasticsearch/config/ 7 | COPY ./config/log4j2.properties /usr/share/elasticsearch/config/ 8 | COPY ./config/elasticsearch.yml /usr/share/elasticsearch/config/ 9 | 10 | # Install Cerebro 11 | ARG CEREBRO_VERSION 12 | RUN cd /usr/share/elasticsearch/ \ 13 | && wget -O cerebro-${CEREBRO_VERSION}.tgz https://github.com/lmenezes/cerebro/releases/download/v${CEREBRO_VERSION}/cerebro-${CEREBRO_VERSION}.tgz \ 14 | && tar zxvf cerebro-${CEREBRO_VERSION}.tgz \ 15 | && rm cerebro-${CEREBRO_VERSION}.tgz \ 16 | && mkdir cerebro-${CEREBRO_VERSION}/logs \ 17 | && mv cerebro-${CEREBRO_VERSION} cerebro 18 | 19 | COPY ./scripts/docker-entrypoint.sh /usr/share/elasticsearch/start -------------------------------------------------------------------------------- /.docker/elasticsearch/7.0.1/config/elasticsearch.yml: -------------------------------------------------------------------------------- 1 | cluster.name: "docker-cluster" 2 | # cluster.initial_master_nodes 3 | network.host: 0.0.0.0 4 | 5 | node.name: elasticsearch 6 | cluster.initial_master_nodes: elasticsearch 7 | bootstrap.memory_lock: true 8 | #index.blocks.read_only_allow_delete: null 9 | # minimum_master_nodes need to be explicitly set when bound on a public IP 10 | # set to 1 to allow single node clusters 11 | # Details: https://github.com/elastic/elasticsearch/pull/17288 12 | discovery.zen.minimum_master_nodes: 1 13 | #discovery.seed_hosts 14 | #discovery.seed_providers 15 | xpack.license.self_generated.type: basic 16 | 17 | # ---------------------------------- Network ----------------------------------- 18 | 19 | # CORS Settings 20 | http.cors.enabled: true 21 | http.cors.allow-origin: "*" 22 | http.cors.allow-methods: OPTIONS, HEAD, GET, POST, PUT, DELETE 23 | http.cors.allow-headers: "X-Requested-With, Content-Type, Content-Length, X-User" 24 | 25 | # twint -u noneprivacy -es localhost:9200 -------------------------------------------------------------------------------- /.docker/elasticsearch/7.0.1/config/log4j2.properties: -------------------------------------------------------------------------------- 1 | status = error 2 | 3 | appender.console.type = Console 4 | appender.console.name = console 5 | appender.console.layout.type = PatternLayout 6 | appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n 7 | 8 | rootLogger.level = info 9 | rootLogger.appenderRef.console.ref = console -------------------------------------------------------------------------------- /.docker/elasticsearch/7.0.1/scripts/docker-entrypoint.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/bash 2 | 3 | # Start Cerebro 4 | /usr/share/elasticsearch/cerebro/bin/cerebro >> /usr/share/elasticsearch/cerebro/logs/cerebro.log 2>&1 & 5 | 6 | # Start ES 7 | /usr/share/elasticsearch/bin/elasticsearch -------------------------------------------------------------------------------- /.docker/readeef/config/readeef.toml: -------------------------------------------------------------------------------- 1 | [server] 2 | port = 8080 3 | 4 | [server.auto-cert] 5 | storage-path = "./storage/certs" 6 | 7 | [log] 8 | level = "info" # error, info, debug 9 | file = "-" # stderr, or a filename 10 | formatter = "text" # text, json 11 | access-file = "" # stdout or a filename 12 | 13 | [api] 14 | emulators = [] # ["tt-rss", "fever"] 15 | 16 | [api.limits] 17 | articles-per-query = 200 18 | 19 | [auth] 20 | session-storage-path = "./storage/session.db" 21 | token-storage-path = "./storage/token.db" 22 | 23 | [db] 24 | driver = "postgres" 25 | connect = "host=db user=readeef dbname=readeef password=readeef sslmode=disable" 26 | 27 | [feed-manager] 28 | update-interval = "30m" 29 | monitors = ["index", "thumbnailer"] 30 | 31 | [timeout] 32 | connect = "1s" 33 | read-write = "2s" 34 | 35 | [hubbub] 36 | from = "readeef" 37 | 38 | [popularity] 39 | delay = "5s" 40 | # providers = ["Reddit", "Twitter"] 41 | 42 | [feed-parser] 43 | processors = ["cleanup", "top-image-marker", "absolutize-urls"] 44 | proxy-http-url-template = "/proxy?url={{ . }}" 45 | 46 | [content] 47 | thumbnail-generator = "description" 48 | 49 | [content.extract] 50 | generator = "goose" # readability 51 | 52 | [content.search] 53 | provider = "bleve" 54 | batch-size = 100 55 | bleve-path = "./storage/search.bleve" 56 | elastic-url = "http://elasticsearch:9200" 57 | 58 | [content.article] 59 | processors = ["insert-thumbnail-target"] 60 | proxy-http-url-template = "/proxy?url={{ . }}" 61 | 62 | #[ui] 63 | # path = "/opt/readeef/ui" -------------------------------------------------------------------------------- /.docker/readeef/scripts/build.sh: -------------------------------------------------------------------------------- 1 | #/bin/sh 2 | 3 | set -x 4 | set -e 5 | 6 | # build binaries 7 | go mod tidy 8 | go get github.com/urandom/embed/cmd/embed 9 | go get github.com/urandom/readeef/cmd/readeef-static-locator 10 | 11 | # requirements for building ui 12 | cd rf-ng 13 | npm install --unsafe-perm -g node-gyp webpack-dev-server rimraf webpack typescript @angular/cli 14 | npm install 15 | cd .. 16 | 17 | make all 18 | ls -l ./rf-ng/ui -------------------------------------------------------------------------------- /.docker/readeef/scripts/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #/bin/sh 2 | 3 | set -x 4 | set -e 5 | 6 | case "$1" in 7 | 8 | 'server') 9 | exec readeef server $@ $ARGS 10 | ;; 11 | 12 | 'dev') 13 | apk add --no-cache nano bash 14 | exec /bin/bash $@ $ARGS 15 | ;; 16 | 17 | 'index') 18 | exec readeef search-index $@ $ARGS 19 | ;; 20 | 21 | 'readeef-static-locator') 22 | exec readeef-static-locator $@ $ARGS 23 | ;; 24 | 25 | *) 26 | exec readeef server $@ 27 | ;; 28 | esac -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | .git 2 | .git/ 3 | .git/* 4 | .git/** 5 | Dockerfile 6 | docker-compose.yml 7 | docker-sync.yml 8 | vendor 9 | vendor/ 10 | vendor/* 11 | vendor/** 12 | Makfile 13 | Dockerfile.dev 14 | /node_modules/ 15 | node_modules 16 | node_modules/ 17 | node_modules/* 18 | node_modules/** -------------------------------------------------------------------------------- /.env: -------------------------------------------------------------------------------- 1 | ## Readeef 2 | PROJECT_NAME=readeef 3 | TINI_VERSION=0.18.0 4 | READEEF_PORT=8080 5 | READEEF_NG_PORT=4200 6 | 7 | ## Postgresql 8 | POSTGRES_VERSION=10.1 9 | POSTGRES_PORT=5432 10 | POSTGRES_USER=readeef 11 | POSTGRES_PASSWORD=readeef 12 | POSTGRES_DB=readeef 13 | 14 | ## ELK 15 | ELASTIC_VERSION=7.0.1 16 | ELASTIC_PORT=9200 17 | KIBANA_VERSION=7.0.1 18 | KIBANA_PORT=5601 19 | CEREBRO_VERSION=0.8.5 20 | CEREBRO_PORT=9400 21 | 22 | ## DGRAPH 23 | DGRAPH_VERSION=1.1.0 -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | static/dist 2 | .bck 3 | vendor 4 | vendor/ 5 | vendor/* 6 | vendor/** 7 | node_modules 8 | node_modules/ 9 | node_modules/* 10 | node_modules/** 11 | storage 12 | storage/ 13 | storage/* 14 | storage/** 15 | # fs_files.go 16 | -------------------------------------------------------------------------------- /DOCKER.md: -------------------------------------------------------------------------------- 1 | readeef with docker 2 | =================== 3 | 4 | # Docker 5 | 6 | ## Requirements 7 | 8 | To use this docker compose file you should comply with this requirements: 9 | 10 | * Install [Docker Desktop](https://www.docker.com/products/docker-desktop) for Windows/Mac or [Docker Engine](https://docs.docker.com/install/linux/docker-ce/ubuntu/#install-docker-ce) for Linux 11 | * Install [docker-compose](https://docs.docker.com/compose/install/) (This is installed by default on Windows and Mac with Docker installation) 12 | 13 | ### Build the image 14 | ```bash 15 | make docker-build 16 | ``` 17 | 18 | ### Run the image 19 | ```bash 20 | make docker-run 21 | ``` 22 | 23 | ### Run with Docker-compose 24 | 25 | ```bash 26 | docker-compose up -d 27 | ``` 28 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:alpine3.10 AS builder 2 | MAINTAINER x0rzkov 3 | 4 | RUN apk add --no-cache make gcc g++ git ca-certificates musl-dev nodejs npm sqlite-dev sqlite 5 | 6 | COPY . /go/src/github.com/urandom/readeef 7 | WORKDIR /go/src/github.com/urandom/readeef 8 | 9 | RUN ./.docker/readeef/scripts/build.sh 10 | 11 | FROM alpine:3.10 AS runtime 12 | MAINTAINER x0rzkov 13 | 14 | ARG TINI_VERSION=${TINI_VERSION:-"v0.18.0"} 15 | 16 | # Install tini to /usr/local/sbin 17 | ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-muslc-amd64 /usr/local/sbin/tini 18 | 19 | # Install runtime dependencies & create runtime user 20 | RUN apk --no-cache --no-progress add ca-certificates git libssh2 openssl sqlite \ 21 | && chmod +x /usr/local/sbin/tini && mkdir -p /opt \ 22 | && adduser -D readeef -h /opt/readeef -s /bin/sh \ 23 | && su readeef -c 'cd /opt/readeef; mkdir -p bin config data ui' 24 | 25 | # Switch to user context 26 | USER readeef 27 | WORKDIR /opt/readeef 28 | 29 | # Copy readeef binaries to /opt/readeef/bin 30 | # COPY --from=builder /go/src/github.com/urandom/readeef/readeef/rf-ng/ui /opt/readeef/ui 31 | COPY --from=builder /go/src/github.com/urandom/readeef/readeef /opt/readeef/bin/readeef 32 | COPY --from=builder /go/bin/readeef-static-locator /opt/readeef/bin/readeef-static-locator 33 | COPY .docker/readeef/config/readeef.toml /opt/readeef/config/readeef.toml 34 | ENV PATH $PATH:/opt/readeef/bin 35 | 36 | # Container configuration 37 | EXPOSE 8080 38 | VOLUME ["/opt/readeef/data"] 39 | ENTRYPOINT ["tini", "-g", "--"] 40 | CMD ["/opt/readeef/bin/readeef", "server"] 41 | # Optional: create entrypoint file for multi-scenario start 42 | # ENTRYPOINT ["./.docker/readeef/scripts/entrypoint.sh"] 43 | # CMD ["dev"] 44 | -------------------------------------------------------------------------------- /Dockerfile.dev: -------------------------------------------------------------------------------- 1 | FROM golang:alpine3.10 2 | 3 | MAINTAINER x0rzkov 4 | 5 | RUN apk add --no-cache bash nano make gcc g++ git ca-certificates musl-dev nodejs npm sqlite-dev sqlite 6 | 7 | COPY . /go/src/github.com/urandom/readeef 8 | WORKDIR /go/src/github.com/urandom/readeef 9 | 10 | RUN ./.docker/readeef/scripts/build.sh 11 | 12 | CMD ["/bin/bash"] 13 | -------------------------------------------------------------------------------- /TODO: -------------------------------------------------------------------------------- 1 | * Create indexes for foreign key columns that are used in queries 2 | * Non-fatal API errors 3 | * TinyRSS API emulation 4 | -------------------------------------------------------------------------------- /api/articlerepotype_string.go: -------------------------------------------------------------------------------- 1 | // Code generated by "stringer -type articleRepoType api"; DO NOT EDIT. 2 | 3 | package api 4 | 5 | import "fmt" 6 | 7 | const _articleRepoType_name = "userRepoTypefavoriteRepoTypepopularRepoTypetagRepoTypefeedRepoType" 8 | 9 | var _articleRepoType_index = [...]uint8{0, 12, 28, 43, 54, 66} 10 | 11 | func (i articleRepoType) String() string { 12 | if i < 0 || i >= articleRepoType(len(_articleRepoType_index)-1) { 13 | return fmt.Sprintf("articleRepoType(%d)", i) 14 | } 15 | return _articleRepoType_name[_articleRepoType_index[i]:_articleRepoType_index[i+1]] 16 | } 17 | -------------------------------------------------------------------------------- /api/articlestate_string.go: -------------------------------------------------------------------------------- 1 | // Code generated by "stringer -type articleState"; DO NOT EDIT. 2 | 3 | package api 4 | 5 | import "fmt" 6 | 7 | const _articleState_name = "readfavorite" 8 | 9 | var _articleState_index = [...]uint8{0, 4, 12} 10 | 11 | func (i articleState) String() string { 12 | if i < 0 || i >= articleState(len(_articleState_index)-1) { 13 | return fmt.Sprintf("articleState(%d)", i) 14 | } 15 | return _articleState_name[_articleState_index[i]:_articleState_index[i+1]] 16 | } 17 | -------------------------------------------------------------------------------- /api/auth.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "net/http" 5 | 6 | jwt "github.com/dgrijalva/jwt-go" 7 | "github.com/urandom/handler/auth" 8 | "github.com/urandom/readeef/api/token" 9 | "github.com/urandom/readeef/content" 10 | "github.com/urandom/readeef/content/repo" 11 | "github.com/urandom/readeef/log" 12 | ) 13 | 14 | func tokenCreate(repo repo.User, secret []byte, log log.Log) http.Handler { 15 | return auth.TokenGenerator(nil, auth.AuthenticatorFunc(func(user, pass string) bool { 16 | u, err := repo.Get(content.Login(user)) 17 | if err != nil { 18 | log.Infof("Error fetching user %s: %+v", user, err) 19 | return false 20 | } 21 | 22 | ok, err := u.Authenticate(pass, secret) 23 | if err != nil { 24 | log.Infof("Error authenticating user %s: %+v", user, err) 25 | return false 26 | } 27 | return ok 28 | }), secret, auth.Logger(log)) 29 | } 30 | 31 | func tokenDelete(storage token.Storage, secret []byte, log log.Log) http.Handler { 32 | return auth.TokenBlacklister(nil, storage, secret, auth.Logger(log)) 33 | } 34 | 35 | func tokenValidator( 36 | repo repo.User, 37 | storage token.Storage, 38 | log log.Log, 39 | ) auth.TokenValidator { 40 | return auth.TokenValidatorFunc(func(token string, claims jwt.Claims) bool { 41 | exists, err := storage.Exists(token) 42 | 43 | if err != nil { 44 | log.Printf("Error using token storage: %+v\n", err) 45 | return false 46 | } 47 | 48 | if exists { 49 | return false 50 | } 51 | 52 | if c, ok := claims.(*jwt.StandardClaims); ok { 53 | _, err := repo.Get(content.Login(c.Subject)) 54 | 55 | if err != nil { 56 | if !content.IsNoContent(err) { 57 | log.Printf("Error getting user %s from repo: %+v\n", c.Subject, err) 58 | } 59 | 60 | return false 61 | } 62 | 63 | return true 64 | } 65 | 66 | return false 67 | }) 68 | } 69 | -------------------------------------------------------------------------------- /api/extract_generator_mock_test.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: github.com/urandom/readeef/content/extract (interfaces: Generator) 3 | 4 | // Package api is a generated GoMock package. 5 | package api 6 | 7 | import ( 8 | gomock "github.com/golang/mock/gomock" 9 | content "github.com/urandom/readeef/content" 10 | reflect "reflect" 11 | ) 12 | 13 | // MockGenerator is a mock of Generator interface 14 | type MockGenerator struct { 15 | ctrl *gomock.Controller 16 | recorder *MockGeneratorMockRecorder 17 | } 18 | 19 | // MockGeneratorMockRecorder is the mock recorder for MockGenerator 20 | type MockGeneratorMockRecorder struct { 21 | mock *MockGenerator 22 | } 23 | 24 | // NewMockGenerator creates a new mock instance 25 | func NewMockGenerator(ctrl *gomock.Controller) *MockGenerator { 26 | mock := &MockGenerator{ctrl: ctrl} 27 | mock.recorder = &MockGeneratorMockRecorder{mock} 28 | return mock 29 | } 30 | 31 | // EXPECT returns an object that allows the caller to indicate expected use 32 | func (m *MockGenerator) EXPECT() *MockGeneratorMockRecorder { 33 | return m.recorder 34 | } 35 | 36 | // Generate mocks base method 37 | func (m *MockGenerator) Generate(arg0 string) (content.Extract, error) { 38 | ret := m.ctrl.Call(m, "Generate", arg0) 39 | ret0, _ := ret[0].(content.Extract) 40 | ret1, _ := ret[1].(error) 41 | return ret0, ret1 42 | } 43 | 44 | // Generate indicates an expected call of Generate 45 | func (mr *MockGeneratorMockRecorder) Generate(arg0 interface{}) *gomock.Call { 46 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Generate", reflect.TypeOf((*MockGenerator)(nil).Generate), arg0) 47 | } 48 | -------------------------------------------------------------------------------- /api/features.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import "net/http" 4 | 5 | type features struct { 6 | Search bool `json:"search,omitempty"` 7 | Extractor bool `json:"extractor,omitempty"` 8 | ProxyHTTP bool `json:"proxyHTTP,omitempty"` 9 | Popularity bool `json:"popularity,omitempty"` 10 | } 11 | 12 | func featuresHandler(features features) http.HandlerFunc { 13 | return func(w http.ResponseWriter, r *http.Request) { 14 | args{"features": features}.WriteJSON(w) 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /api/features_test.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "encoding/json" 5 | "net/http/httptest" 6 | "reflect" 7 | "testing" 8 | ) 9 | 10 | func Test_featuresHandler(t *testing.T) { 11 | tests := []struct { 12 | name string 13 | features features 14 | wantErr bool 15 | }{ 16 | {"all", features{Search: true, Extractor: true, ProxyHTTP: true, Popularity: true}, false}, 17 | {"some", features{Extractor: true}, false}, 18 | } 19 | type data struct { 20 | Features features `json:"features"` 21 | } 22 | for _, tt := range tests { 23 | t.Run(tt.name, func(t *testing.T) { 24 | req := httptest.NewRequest("GET", "/features", nil) 25 | w := httptest.NewRecorder() 26 | 27 | featuresHandler(tt.features).ServeHTTP(w, req) 28 | got := data{} 29 | 30 | if err := json.Unmarshal(w.Body.Bytes(), &got); (err != nil) != tt.wantErr { 31 | t.Errorf("featuresHandler() error = %v, wantErr %v", err, tt.wantErr) 32 | return 33 | } 34 | 35 | if !reflect.DeepEqual(got.Features, tt.features) { 36 | t.Errorf("featuresHandler() = %v, want %v", got.Features, tt.features) 37 | } 38 | }) 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /api/fever/favicon.go: -------------------------------------------------------------------------------- 1 | package fever 2 | 3 | import ( 4 | "encoding/base64" 5 | "net/http" 6 | "sync" 7 | 8 | "github.com/pkg/errors" 9 | "github.com/urandom/readeef/content" 10 | "github.com/urandom/readeef/content/repo" 11 | rffeed "github.com/urandom/readeef/feed" 12 | "github.com/urandom/readeef/log" 13 | ) 14 | 15 | type favicon struct { 16 | ID content.FeedID `json:"id"` 17 | Data string `json:"data"` 18 | } 19 | 20 | var ( 21 | faviconCache = map[content.FeedID]string{} 22 | faviconCacheMu sync.RWMutex 23 | ) 24 | 25 | func favicons( 26 | r *http.Request, 27 | resp resp, 28 | user content.User, 29 | service repo.Service, 30 | log log.Log, 31 | ) error { 32 | log.Infoln("Fetching fever feeds favicons") 33 | 34 | var favicons []favicon 35 | 36 | feeds, err := service.FeedRepo().ForUser(user) 37 | if err != nil { 38 | return errors.WithMessage(err, "getting user feeds") 39 | } 40 | 41 | for _, f := range feeds { 42 | faviconCacheMu.RLock() 43 | data := faviconCache[f.ID] 44 | faviconCacheMu.RUnlock() 45 | 46 | if data != "" { 47 | favicons = append(favicons, favicon{f.ID, data}) 48 | continue 49 | } 50 | 51 | log.Debugf("Getting favicon for %q", f.SiteLink) 52 | b, ct, err := rffeed.Favicon(f.SiteLink) 53 | if err != nil { 54 | log.Printf("Error getting favicon for %q: %v", f.SiteLink, err) 55 | continue 56 | } 57 | 58 | data = ct + ";base64," + base64.StdEncoding.EncodeToString(b) 59 | faviconCacheMu.Lock() 60 | faviconCache[f.ID] = data 61 | faviconCacheMu.Unlock() 62 | 63 | favicons = append(favicons, favicon{f.ID, data}) 64 | } 65 | 66 | resp["favicons"] = favicons 67 | 68 | return nil 69 | } 70 | 71 | func init() { 72 | actions["favicons"] = favicons 73 | } 74 | -------------------------------------------------------------------------------- /api/fever/feeds.go: -------------------------------------------------------------------------------- 1 | package fever 2 | 3 | import ( 4 | "net/http" 5 | "time" 6 | 7 | "github.com/pkg/errors" 8 | "github.com/urandom/readeef/content" 9 | "github.com/urandom/readeef/content/repo" 10 | "github.com/urandom/readeef/log" 11 | ) 12 | 13 | type feed struct { 14 | Id content.FeedID `json:"id"` 15 | Title string `json:"title"` 16 | FaviconID content.FeedID `json:"favicon_id"` 17 | Url string `json:"url"` 18 | SiteUrl string `json:"site_url"` 19 | IsSpark int `json:"is_spark"` 20 | UpdateTime int64 `json:"last_updated_on_time"` 21 | } 22 | 23 | func feeds( 24 | r *http.Request, 25 | resp resp, 26 | user content.User, 27 | service repo.Service, 28 | log log.Log, 29 | ) error { 30 | log.Infoln("Fetching fever feeds") 31 | 32 | var feverFeeds []feed 33 | 34 | feeds, err := service.FeedRepo().ForUser(user) 35 | if err != nil { 36 | return errors.WithMessage(err, "getting user feeds") 37 | } 38 | 39 | now := time.Now().Unix() 40 | for _, f := range feeds { 41 | feed := feed{ 42 | Id: f.ID, FaviconID: f.ID, Title: f.Title, Url: f.Link, 43 | SiteUrl: f.SiteLink, UpdateTime: now, 44 | } 45 | 46 | feverFeeds = append(feverFeeds, feed) 47 | } 48 | 49 | resp["feeds"] = feverFeeds 50 | 51 | err = groups(r, resp, user, service, log) 52 | delete(resp, "groups") 53 | 54 | return err 55 | } 56 | 57 | func init() { 58 | actions["feeds"] = feeds 59 | } 60 | -------------------------------------------------------------------------------- /api/fever/groups.go: -------------------------------------------------------------------------------- 1 | package fever 2 | 3 | import ( 4 | "net/http" 5 | "strconv" 6 | "strings" 7 | 8 | "github.com/pkg/errors" 9 | "github.com/urandom/readeef/content" 10 | "github.com/urandom/readeef/content/repo" 11 | "github.com/urandom/readeef/log" 12 | ) 13 | 14 | type group struct { 15 | Id int64 `json:"id"` 16 | Title string `json:"title"` 17 | } 18 | 19 | type feedsGroup struct { 20 | GroupId int64 `json:"group_id"` 21 | FeedIds string `json:"feed_ids"` 22 | } 23 | 24 | func groups( 25 | r *http.Request, 26 | resp resp, 27 | user content.User, 28 | service repo.Service, 29 | log log.Log, 30 | ) error { 31 | log.Infoln("Fetching fever groups") 32 | 33 | tags, err := service.TagRepo().ForUser(user) 34 | if err != nil { 35 | return errors.WithMessage(err, "getting user tags") 36 | } 37 | 38 | g := make([]group, len(tags)) 39 | fg := make([]feedsGroup, len(tags)) 40 | 41 | feedRepo := service.FeedRepo() 42 | for i, tag := range tags { 43 | g[i] = group{Id: int64(tag.ID), Title: string(tag.Value)} 44 | 45 | feeds, err := feedRepo.ForTag(tag, user) 46 | if err != nil { 47 | return errors.WithMessage(err, "getting tag feeds") 48 | } 49 | 50 | ids := make([]string, len(feeds)) 51 | for j := range feeds { 52 | ids[j] = strconv.FormatInt(int64(feeds[j].ID), 10) 53 | } 54 | 55 | fg[i] = feedsGroup{GroupId: int64(tag.ID), FeedIds: strings.Join(ids, ",")} 56 | } 57 | 58 | resp["groups"], resp["feeds_groups"] = g, fg 59 | 60 | return nil 61 | } 62 | 63 | func init() { 64 | actions["groups"] = groups 65 | } 66 | -------------------------------------------------------------------------------- /api/fever/ids.go: -------------------------------------------------------------------------------- 1 | package fever 2 | 3 | import ( 4 | "net/http" 5 | "strconv" 6 | 7 | "github.com/pkg/errors" 8 | "github.com/urandom/readeef/content" 9 | "github.com/urandom/readeef/content/repo" 10 | "github.com/urandom/readeef/log" 11 | "github.com/urandom/readeef/pool" 12 | ) 13 | 14 | func unreadItemIDs( 15 | r *http.Request, 16 | resp resp, 17 | user content.User, 18 | service repo.Service, 19 | log log.Log, 20 | ) error { 21 | log.Infoln("Fetching unread fever item ids") 22 | 23 | ids, err := service.ArticleRepo().IDs(user, 24 | content.UnreadOnly, content.Filters(content.GetUserFilters(user))) 25 | if err != nil { 26 | return errors.WithMessage(err, "getting unread ids") 27 | } 28 | 29 | buf := pool.Buffer.Get() 30 | defer pool.Buffer.Put(buf) 31 | 32 | for i := range ids { 33 | if i != 0 { 34 | buf.WriteString(",") 35 | } 36 | 37 | buf.WriteString(strconv.FormatInt(int64(ids[i]), 10)) 38 | } 39 | 40 | resp["unread_item_ids"] = buf.String() 41 | 42 | return nil 43 | } 44 | 45 | func savedItemIDs( 46 | r *http.Request, 47 | resp resp, 48 | user content.User, 49 | service repo.Service, 50 | log log.Log, 51 | ) error { 52 | log.Infoln("Fetching saved fever item ids") 53 | 54 | ids, err := service.ArticleRepo().IDs(user, 55 | content.FavoriteOnly, content.Filters(content.GetUserFilters(user))) 56 | if err != nil { 57 | return errors.WithMessage(err, "getting unread ids") 58 | } 59 | 60 | buf := pool.Buffer.Get() 61 | defer pool.Buffer.Put(buf) 62 | 63 | for i := range ids { 64 | if i != 0 { 65 | buf.WriteString(",") 66 | } 67 | 68 | buf.WriteString(strconv.FormatInt(int64(ids[i]), 10)) 69 | } 70 | 71 | resp["saved_item_ids"] = buf.String() 72 | 73 | return nil 74 | } 75 | 76 | func init() { 77 | actions["unread_item_ids"] = unreadItemIDs 78 | actions["saved_item_ids"] = savedItemIDs 79 | } 80 | -------------------------------------------------------------------------------- /api/fever/user.go: -------------------------------------------------------------------------------- 1 | package fever 2 | 3 | import ( 4 | "encoding/hex" 5 | 6 | "github.com/pkg/errors" 7 | "github.com/urandom/readeef/content" 8 | "github.com/urandom/readeef/content/repo" 9 | "github.com/urandom/readeef/log" 10 | ) 11 | 12 | func readeefUser(repo repo.User, md5hex string, log log.Log) (content.User, error) { 13 | md5, err := hex.DecodeString(md5hex) 14 | 15 | if err != nil { 16 | return content.User{}, errors.Wrap(err, "decoding hex api_key") 17 | } 18 | 19 | user, err := repo.FindByMD5(md5) 20 | if err != nil { 21 | return content.User{}, errors.WithMessage(err, "getting user by md5") 22 | } 23 | return user, nil 24 | } 25 | -------------------------------------------------------------------------------- /api/processor_article_mock_test.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: github.com/urandom/readeef/content/processor (interfaces: Article) 3 | 4 | // Package api is a generated GoMock package. 5 | package api 6 | 7 | import ( 8 | reflect "reflect" 9 | 10 | gomock "github.com/golang/mock/gomock" 11 | content "github.com/urandom/readeef/content" 12 | ) 13 | 14 | // MockArticleProcessor is a mock of Article interface 15 | type MockArticleProcessor struct { 16 | ctrl *gomock.Controller 17 | recorder *MockArticleProcessorMockRecorder 18 | } 19 | 20 | // MockArticleProcessorMockRecorder is the mock recorder for MockArticleProcessor 21 | type MockArticleProcessorMockRecorder struct { 22 | mock *MockArticleProcessor 23 | } 24 | 25 | // NewMockArticleProcessor creates a new mock instance 26 | func NewMockArticleProcessor(ctrl *gomock.Controller) *MockArticleProcessor { 27 | mock := &MockArticleProcessor{ctrl: ctrl} 28 | mock.recorder = &MockArticleProcessorMockRecorder{mock} 29 | return mock 30 | } 31 | 32 | // EXPECT returns an object that allows the caller to indicate expected use 33 | func (m *MockArticleProcessor) EXPECT() *MockArticleProcessorMockRecorder { 34 | return m.recorder 35 | } 36 | 37 | // ProcessArticles mocks base method 38 | func (m *MockArticleProcessor) ProcessArticles(arg0 []content.Article) []content.Article { 39 | ret := m.ctrl.Call(m, "ProcessArticles", arg0) 40 | ret0, _ := ret[0].([]content.Article) 41 | return ret0 42 | } 43 | 44 | // ProcessArticles indicates an expected call of ProcessArticles 45 | func (mr *MockArticleProcessorMockRecorder) ProcessArticles(arg0 interface{}) *gomock.Call { 46 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ProcessArticles", reflect.TypeOf((*MockArticleProcessor)(nil).ProcessArticles), arg0) 47 | } 48 | -------------------------------------------------------------------------------- /api/request.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "net/http" 7 | 8 | jwt "github.com/dgrijalva/jwt-go" 9 | "github.com/urandom/handler/auth" 10 | "github.com/urandom/readeef/content" 11 | "github.com/urandom/readeef/content/repo" 12 | "github.com/urandom/readeef/log" 13 | ) 14 | 15 | type contextKey string 16 | 17 | var userKey = contextKey("user") 18 | 19 | func userContext(repo repo.User, next http.Handler, log log.Log) http.Handler { 20 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 21 | if c, ok := auth.Claims(r).(*jwt.StandardClaims); ok { 22 | user, err := repo.Get(content.Login(c.Subject)) 23 | 24 | if err != nil { 25 | if content.IsNoContent(err) { 26 | http.Error(w, "Not found", http.StatusNotFound) 27 | return 28 | } 29 | 30 | fatal(w, log, "Error loading user: %+v", err) 31 | return 32 | } 33 | 34 | ctx := context.WithValue(r.Context(), userKey, user) 35 | 36 | next.ServeHTTP(w, r.WithContext(ctx)) 37 | } else { 38 | http.Error(w, "Invalid claims", http.StatusBadRequest) 39 | } 40 | }) 41 | } 42 | 43 | func userValidator(next http.Handler) http.Handler { 44 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 45 | if _, stop := userFromRequest(w, r); stop { 46 | return 47 | } 48 | 49 | next.ServeHTTP(w, r) 50 | 51 | }) 52 | } 53 | 54 | func userFromRequest(w http.ResponseWriter, r *http.Request) (user content.User, stop bool) { 55 | var ok bool 56 | if user, ok = r.Context().Value(userKey).(content.User); ok { 57 | return user, false 58 | } 59 | 60 | http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) 61 | return content.User{}, true 62 | } 63 | 64 | type args map[string]interface{} 65 | 66 | func (a args) WriteJSON(w http.ResponseWriter) { 67 | b, err := json.Marshal(a) 68 | 69 | if err != nil { 70 | http.Error(w, err.Error(), http.StatusInternalServerError) 71 | } 72 | 73 | w.Write(b) 74 | } 75 | -------------------------------------------------------------------------------- /api/searcher_mock_test.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: ./api/article.go 3 | 4 | // Package api is a generated GoMock package. 5 | package api 6 | 7 | import ( 8 | gomock "github.com/golang/mock/gomock" 9 | content "github.com/urandom/readeef/content" 10 | reflect "reflect" 11 | ) 12 | 13 | // Mocksearcher is a mock of searcher interface 14 | type Mocksearcher struct { 15 | ctrl *gomock.Controller 16 | recorder *MocksearcherMockRecorder 17 | } 18 | 19 | // MocksearcherMockRecorder is the mock recorder for Mocksearcher 20 | type MocksearcherMockRecorder struct { 21 | mock *Mocksearcher 22 | } 23 | 24 | // NewMocksearcher creates a new mock instance 25 | func NewMocksearcher(ctrl *gomock.Controller) *Mocksearcher { 26 | mock := &Mocksearcher{ctrl: ctrl} 27 | mock.recorder = &MocksearcherMockRecorder{mock} 28 | return mock 29 | } 30 | 31 | // EXPECT returns an object that allows the caller to indicate expected use 32 | func (m *Mocksearcher) EXPECT() *MocksearcherMockRecorder { 33 | return m.recorder 34 | } 35 | 36 | // Search mocks base method 37 | func (m *Mocksearcher) Search(arg0 string, arg1 content.User, arg2 ...content.QueryOpt) ([]content.Article, error) { 38 | varargs := []interface{}{arg0, arg1} 39 | for _, a := range arg2 { 40 | varargs = append(varargs, a) 41 | } 42 | ret := m.ctrl.Call(m, "Search", varargs...) 43 | ret0, _ := ret[0].([]content.Article) 44 | ret1, _ := ret[1].(error) 45 | return ret0, ret1 46 | } 47 | 48 | // Search indicates an expected call of Search 49 | func (mr *MocksearcherMockRecorder) Search(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { 50 | varargs := append([]interface{}{arg0, arg1}, arg2...) 51 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Search", reflect.TypeOf((*Mocksearcher)(nil).Search), varargs...) 52 | } 53 | -------------------------------------------------------------------------------- /api/token/bolt_storage.go: -------------------------------------------------------------------------------- 1 | package token 2 | 3 | import ( 4 | "encoding/binary" 5 | "time" 6 | 7 | "github.com/boltdb/bolt" 8 | "github.com/pkg/errors" 9 | ) 10 | 11 | type BoltStorage struct { 12 | db *bolt.DB 13 | } 14 | 15 | var ( 16 | bucket = []byte("token-bucket") 17 | ) 18 | 19 | func NewBoltStorage(path string) (BoltStorage, error) { 20 | db, err := bolt.Open(path, 0660, nil) 21 | 22 | if err != nil { 23 | return BoltStorage{}, errors.Wrapf(err, "opening token bolt storage %s", path) 24 | } 25 | 26 | err = db.Update(func(tx *bolt.Tx) error { 27 | _, err := tx.CreateBucketIfNotExists(bucket) 28 | 29 | return err 30 | }) 31 | if err != nil { 32 | return BoltStorage{}, errors.Wrap(err, "creating token bolt bucket") 33 | } 34 | 35 | return BoltStorage{db}, nil 36 | 37 | } 38 | 39 | func (b BoltStorage) Store(token string, expiration time.Time) error { 40 | err := b.db.Update(func(tx *bolt.Tx) error { 41 | b := tx.Bucket(bucket) 42 | 43 | buf := make([]byte, 8) 44 | binary.LittleEndian.PutUint64(buf, uint64(expiration.Unix())) 45 | 46 | return b.Put([]byte(token), buf) 47 | }) 48 | 49 | if err != nil { 50 | err = errors.Wrap(err, "writing token to storage") 51 | } 52 | 53 | return err 54 | } 55 | 56 | func (b BoltStorage) Exists(token string) (bool, error) { 57 | exists := false 58 | 59 | err := b.db.View(func(tx *bolt.Tx) error { 60 | if v := tx.Bucket(bucket).Get([]byte(token)); v != nil { 61 | exists = true 62 | } 63 | 64 | return nil 65 | }) 66 | 67 | if err != nil { 68 | err = errors.Wrap(err, "looking up token") 69 | } 70 | 71 | return exists, err 72 | } 73 | 74 | func (b BoltStorage) RemoveExpired() error { 75 | now := time.Now() 76 | 77 | err := b.db.Update(func(tx *bolt.Tx) error { 78 | 79 | c := tx.Bucket(bucket).Cursor() 80 | 81 | for k, v := c.First(); k != nil; k, v = c.Next() { 82 | t := time.Unix(int64(binary.LittleEndian.Uint64(v)), 0) 83 | 84 | if now.Before(t) { 85 | continue 86 | } 87 | 88 | if err := c.Delete(); err != nil { 89 | return err 90 | } 91 | } 92 | 93 | return nil 94 | }) 95 | 96 | if err != nil { 97 | err = errors.Wrap(err, "cleaning expired tokens") 98 | } 99 | 100 | return err 101 | } 102 | -------------------------------------------------------------------------------- /api/token/token.go: -------------------------------------------------------------------------------- 1 | package token 2 | 3 | import "time" 4 | 5 | type Storage interface { 6 | Store(token string, expiration time.Time) error 7 | Exists(token string) (bool, error) 8 | RemoveExpired() error 9 | } 10 | -------------------------------------------------------------------------------- /api/ttrss/auth.go: -------------------------------------------------------------------------------- 1 | package ttrss 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "github.com/pkg/errors" 8 | "github.com/urandom/readeef/content" 9 | "github.com/urandom/readeef/content/repo" 10 | ) 11 | 12 | func registerAuthActions(sessionManager sessionManager, secret []byte) { 13 | actions["login"] = func(req request, u content.User, service repo.Service) (interface{}, error) { 14 | return login(req, u, sessionManager, secret) 15 | } 16 | actions["logout"] = func(req request, u content.User, service repo.Service) (interface{}, error) { 17 | return logout(req, u, sessionManager) 18 | } 19 | actions["isLoggedIn"] = func(req request, u content.User, service repo.Service) (interface{}, error) { 20 | return isLoggedIn(req, u, sessionManager) 21 | } 22 | } 23 | 24 | func login( 25 | req request, user content.User, sessionManager sessionManager, secret []byte, 26 | ) (interface{}, error) { 27 | if ok, err := user.Authenticate(req.Password, []byte(secret)); !ok { 28 | return nil, errors.WithStack(newErr(fmt.Sprintf( 29 | "authentication for TT-RSS user '%s'", user.Login, 30 | ), "LOGIN_ERROR")) 31 | } else if err != nil { 32 | return nil, errors.WithStack(newErr(fmt.Sprintf( 33 | "authentication for TT-RSS user '%s': %v", user.Login, err, 34 | ), "LOGIN_ERROR")) 35 | } 36 | 37 | sessId := sessionManager.update(session{login: user.Login, lastVisit: time.Now()}) 38 | 39 | return genericContent{ 40 | ApiLevel: API_LEVEL, 41 | SessionId: sessId, 42 | }, nil 43 | } 44 | 45 | func logout( 46 | req request, user content.User, sessionManager sessionManager, 47 | ) (interface{}, error) { 48 | sessionManager.remove(req.Sid) 49 | return genericContent{Status: "OK"}, nil 50 | } 51 | 52 | func isLoggedIn( 53 | req request, user content.User, sessionManager sessionManager, 54 | ) (interface{}, error) { 55 | s := sessionManager.get(req.Sid) 56 | return genericContent{Status: s.login != ""}, nil 57 | } 58 | -------------------------------------------------------------------------------- /api/ttrss/generic.go: -------------------------------------------------------------------------------- 1 | package ttrss 2 | 3 | import ( 4 | "github.com/pkg/errors" 5 | "github.com/urandom/readeef/content" 6 | "github.com/urandom/readeef/content/repo" 7 | ) 8 | 9 | type configContent struct { 10 | IconsDir string `json:"icons_dir"` 11 | IconsUrl string `json:"icons_url"` 12 | DaemonIsRunning bool `json:"daemon_is_running"` 13 | NumFeeds int `json:"num_feeds"` 14 | } 15 | 16 | type genericContent struct { 17 | Level int `json:"level,omitempty"` 18 | ApiLevel int `json:"api_level,omitempty"` 19 | Version string `json:"version,omitempty"` 20 | SessionId string `json:"session_id,omitempty"` 21 | Status interface{} `json:"status,omitempty"` 22 | Unread string `json:"unread,omitempty"` 23 | Updated int64 `json:"updated,omitempty"` 24 | Value interface{} `json:"value,omitempty"` 25 | Method string `json:"method,omitempty"` 26 | } 27 | 28 | func getApiLevel(req request, user content.User, service repo.Service) (interface{}, error) { 29 | return genericContent{Level: API_LEVEL}, nil 30 | } 31 | 32 | func getVersion(req request, user content.User, service repo.Service) (interface{}, error) { 33 | return genericContent{Version: API_VERSION}, nil 34 | } 35 | 36 | func getConfig(req request, user content.User, service repo.Service) (interface{}, error) { 37 | feeds, err := service.FeedRepo().ForUser(user) 38 | if err != nil { 39 | return nil, errors.WithMessage(err, "getting user feeds") 40 | } 41 | feedCount := len(feeds) 42 | 43 | return configContent{DaemonIsRunning: true, NumFeeds: feedCount}, nil 44 | } 45 | 46 | func unknown(req request, user content.User, service repo.Service) (interface{}, error) { 47 | return genericContent{Method: req.Op}, errors.WithStack(newErr("unknown method "+req.Op, "UNKNOWN_METHOD")) 48 | } 49 | 50 | func init() { 51 | actions["getApiLevel"] = getApiLevel 52 | actions["getVersion"] = getVersion 53 | actions["getConfig"] = getConfig 54 | } 55 | -------------------------------------------------------------------------------- /cmd/readeef-client/config.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | 7 | "github.com/BurntSushi/toml" 8 | "github.com/pkg/errors" 9 | ) 10 | 11 | type config struct { 12 | URL string 13 | } 14 | 15 | var ( 16 | configPath = filepath.Join(CONFIG_DIR, "config.toml") 17 | ) 18 | 19 | func readConfig() (config, error) { 20 | var c config 21 | 22 | if _, err := toml.DecodeFile(configPath, &c); err != nil { 23 | return c, errors.Wrap(err, "reading config file") 24 | } 25 | 26 | return c, nil 27 | } 28 | 29 | func writeConfig(c config) error { 30 | if err := os.MkdirAll(configPath, 0755); err != nil { 31 | return errors.Wrap(err, "creating config directory") 32 | } 33 | 34 | if err := os.Rename(configPath, configPath+".bkp"); err != nil && !os.IsNotExist(err) { 35 | return errors.Wrap(err, "backing up config file") 36 | } 37 | 38 | f, err := os.Create(configPath) 39 | if err != nil { 40 | return errors.Wrap(err, "creating config file") 41 | } 42 | defer f.Close() 43 | 44 | encoder := toml.NewEncoder(f) 45 | if err := encoder.Encode(c); err != nil { 46 | return errors.Wrap(err, "encoding config") 47 | } 48 | 49 | return nil 50 | } 51 | -------------------------------------------------------------------------------- /cmd/readeef-client/open.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os/exec" 6 | ) 7 | 8 | func open(u string) error { 9 | args := []string{} 10 | if len(OPEN) > 1 { 11 | args = append(args, OPEN...) 12 | } 13 | args = append(args, u) 14 | if err := exec.Command(OPEN[0], args...).Run(); err != nil { 15 | return fmt.Errorf("Error opening %s: %v", u, err) 16 | } 17 | 18 | return nil 19 | } 20 | -------------------------------------------------------------------------------- /cmd/readeef-client/platform_darwin.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | ) 7 | 8 | var ( 9 | OPEN = []string{"open"} 10 | CONFIG_DIR = filepath.Join(os.Getenv("HOME"), "Library/Preferences/readeef-client") 11 | ) 12 | -------------------------------------------------------------------------------- /cmd/readeef-client/platform_linux.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | ) 7 | 8 | var ( 9 | OPEN = []string{"xdg-open"} 10 | CONFIG_DIR = filepath.Join(os.Getenv("HOME"), ".config/readeef-client") 11 | ) 12 | -------------------------------------------------------------------------------- /cmd/readeef-client/platform_windows.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | ) 7 | 8 | var ( 9 | OPEN = []string{"cmd", "/c", "start"} 10 | CONFIG_DIR = filepath.Join(os.Getenv("%APPDATA%"), "readeef-client") 11 | ) 12 | -------------------------------------------------------------------------------- /cmd/readeef/postgres.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import _ "github.com/urandom/readeef/content/repo/sql/db/postgres" 4 | -------------------------------------------------------------------------------- /cmd/readeef/search-index.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | 6 | "github.com/pkg/errors" 7 | "github.com/urandom/readeef/config" 8 | "github.com/urandom/readeef/content/repo/sql" 9 | "github.com/urandom/readeef/content/search" 10 | ) 11 | 12 | var ( 13 | searchIndexVerbose bool 14 | ) 15 | 16 | func runSearchIndex(config config.Config, args []string) error { 17 | if searchIndexVerbose { 18 | config.Log.Level = "debug" 19 | } 20 | 21 | log := initLog(config.Log) 22 | service, err := sql.NewService(config.DB.Driver, config.DB.Connect, log) 23 | if err != nil { 24 | return errors.WithMessage(err, "creating content service") 25 | } 26 | 27 | searchProvider := initSearchProvider(config.Content, service, log) 28 | if searchProvider == nil { 29 | return errors.Errorf("unknown search provider %s", config.Content.Search.Provider) 30 | } 31 | 32 | log.Info("Starting feed indexing") 33 | 34 | if err := search.Reindex(searchProvider, service.ArticleRepo()); err != nil { 35 | return errors.WithMessage(err, "indexing all feeds") 36 | } 37 | 38 | return nil 39 | } 40 | 41 | func init() { 42 | flags := flag.NewFlagSet("search-index", flag.ExitOnError) 43 | flags.BoolVar(&searchIndexVerbose, "verbose", false, "verbose output") 44 | 45 | commands = append(commands, Command{ 46 | Name: "search-index", 47 | Desc: "re-index all feeds", 48 | Flags: flags, 49 | Run: runSearchIndex, 50 | }) 51 | } 52 | -------------------------------------------------------------------------------- /cmd/readeef/sqlite.go: -------------------------------------------------------------------------------- 1 | // +build cgo 2 | 3 | package main 4 | 5 | import _ "github.com/urandom/readeef/content/repo/sql/db/sqlite3" 6 | -------------------------------------------------------------------------------- /config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | 7 | "github.com/BurntSushi/toml" 8 | "github.com/pkg/errors" 9 | ) 10 | 11 | // Config is the readeef configuration 12 | type Config struct { 13 | Server Server `toml:"server"` 14 | Log Log `toml:"log"` 15 | API API `toml:"api"` 16 | Timeout Timeout `toml:"timeout"` 17 | DB DB `toml:"db"` 18 | Auth Auth `toml:"auth"` 19 | Hubbub Hubbub `toml:"hubbub"` 20 | Popularity Popularity `toml:"popularity"` 21 | FeedParser FeedParser `toml:"feed-parser"` 22 | FeedManager FeedManager `toml:"feed-manager"` 23 | Content Content `toml:"content"` 24 | UI UI `toml:"ui"` 25 | } 26 | 27 | // Read loads the config data from the given path 28 | func Read(path string) (Config, error) { 29 | c, err := defaultConfig() 30 | 31 | if err != nil { 32 | return Config{}, errors.WithMessage(err, "initializing default config") 33 | } 34 | 35 | c, err = readPath(c, path) 36 | if err != nil { 37 | return Config{}, err 38 | } 39 | 40 | for _, c := range []converter{&c.API, &c.Log, &c.Timeout, &c.FeedManager, &c.Popularity, &c.Content} { 41 | c.Convert() 42 | } 43 | 44 | return c, nil 45 | 46 | } 47 | 48 | func readPath(c Config, path string) (Config, error) { 49 | if path != "" { 50 | b, err := ioutil.ReadFile(path) 51 | if err != nil { 52 | if os.IsNotExist(err) { 53 | return c, nil 54 | } 55 | return Config{}, errors.Wrapf(err, "reading config from %s", path) 56 | } 57 | 58 | if err = toml.Unmarshal(b, &c); err != nil { 59 | return Config{}, errors.Wrapf(err, "unmarshaling toml config from %s", path) 60 | } 61 | } 62 | 63 | return c, nil 64 | } 65 | -------------------------------------------------------------------------------- /config/default.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "github.com/BurntSushi/toml" 5 | "github.com/pkg/errors" 6 | ) 7 | 8 | func defaultConfig() (Config, error) { 9 | var def Config 10 | 11 | err := toml.Unmarshal([]byte(DefaultCfg), &def) 12 | 13 | if err != nil { 14 | return Config{}, errors.Wrap(err, "parsing default config") 15 | } 16 | 17 | def.API.Version = apiversion 18 | return def, nil 19 | } 20 | 21 | // DefaultCfg shows the default configuration of the readeef server 22 | var DefaultCfg = ` 23 | [server] 24 | port = 8080 25 | [server.auto-cert] 26 | storage-path = "./storage/certs" 27 | [log] 28 | level = "info" # error, info, debug 29 | file = "-" # stderr, or a filename 30 | formatter = "text" # text, json 31 | access-file = "" # stdout or a filename 32 | [api] 33 | emulators = [] # ["tt-rss", "fever"] 34 | [api.limits] 35 | articles-per-query = 200 36 | [db] 37 | driver = "sqlite3" 38 | connect = "file:./storage/content.sqlite3?cache=shared&mode=rwc&_busy_timeout=50000000&_foreign_keys=1&_journal=wal" 39 | [auth] 40 | session-storage-path = "./storage/session.db" 41 | token-storage-path = "./storage/token.db" 42 | [feed-manager] 43 | update-interval = "30m" 44 | monitors = ["index", "thumbnailer"] 45 | [timeout] 46 | connect = "1s" 47 | read-write = "2s" 48 | [hubbub] 49 | from = "readeef" 50 | [popularity] 51 | delay = "5s" 52 | # providers = ["Reddit", "Twitter"] 53 | [feed-parser] 54 | processors = ["cleanup", "top-image-marker", "absolutize-urls"] 55 | proxy-http-url-template = "/proxy?url={{ . }}" 56 | [content.extract] 57 | generator = "goose" # readability 58 | [content.search] 59 | provider = "bleve" 60 | batch-size = 100 61 | bleve-path = "./storage/search.bleve" 62 | elastic-url = "http://localhost:9200" 63 | [content.article] 64 | processors = ["insert-thumbnail-target"] 65 | proxy-http-url-template = "/proxy?url={{ . }}" 66 | [content.thumbnail] 67 | store = true 68 | [ui] 69 | path = "./rf-ng/ui" 70 | ` 71 | -------------------------------------------------------------------------------- /content/article_test.go: -------------------------------------------------------------------------------- 1 | package content_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/urandom/readeef/content" 7 | ) 8 | 9 | func TestArticle_Validate(t *testing.T) { 10 | type fields struct { 11 | ID content.ArticleID 12 | FeedID content.FeedID 13 | Link string 14 | } 15 | tests := []struct { 16 | name string 17 | fields fields 18 | wantErr bool 19 | }{ 20 | {"valid", fields{ID: 1, FeedID: 1, Link: "http://sugr.org"}, false}, 21 | {"link not absolute", fields{ID: 1, FeedID: 1, Link: "sugr.org"}, true}, 22 | {"no link", fields{ID: 1, FeedID: 1}, true}, 23 | {"no feed id", fields{ID: 1, Link: "http://sugr.org"}, true}, 24 | {"no id", fields{FeedID: 1, Link: "http://sugr.org"}, true}, 25 | {"nothing", fields{}, true}, 26 | } 27 | for _, tt := range tests { 28 | t.Run(tt.name, func(t *testing.T) { 29 | a := content.Article{ 30 | ID: tt.fields.ID, 31 | FeedID: tt.fields.FeedID, 32 | Link: tt.fields.Link, 33 | } 34 | if err := a.Validate(); (err != nil) != tt.wantErr { 35 | t.Errorf("Article.Validate() error = %v, wantErr %v", err, tt.wantErr) 36 | } 37 | }) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /content/err.go: -------------------------------------------------------------------------------- 1 | package content 2 | 3 | import "github.com/pkg/errors" 4 | 5 | var ( 6 | ErrNoContent = errors.New("No content") 7 | ) 8 | 9 | func IsNoContent(err error) bool { 10 | cause := errors.Cause(err) 11 | 12 | return cause == ErrNoContent 13 | } 14 | -------------------------------------------------------------------------------- /content/extract.go: -------------------------------------------------------------------------------- 1 | package content 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | ) 7 | 8 | type Extract struct { 9 | ArticleID ArticleID `db:"article_id"` 10 | Title string 11 | Content string 12 | TopImage string `db:"top_image"` 13 | Language string 14 | } 15 | 16 | func (e Extract) Validate() error { 17 | if e.ArticleID == 0 { 18 | return NewValidationError(errors.New("Article extract has no article id")) 19 | } 20 | 21 | return nil 22 | } 23 | 24 | func (e Extract) String() string { 25 | return fmt.Sprintf("%d: %s", e.ArticleID, e.Title) 26 | } 27 | -------------------------------------------------------------------------------- /content/extract/extract.go: -------------------------------------------------------------------------------- 1 | package extract 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/pkg/errors" 8 | "github.com/urandom/readeef/content" 9 | "github.com/urandom/readeef/content/processor" 10 | "github.com/urandom/readeef/content/repo" 11 | ) 12 | 13 | type Generator interface { 14 | Generate(link string) (content.Extract, error) 15 | } 16 | 17 | // Get retrieves the extract from the repository. If it doesn't exist, a new 18 | // one is created and stored before returning. 19 | func Get( 20 | article content.Article, 21 | repo repo.Extract, 22 | generator Generator, 23 | processors []processor.Article, 24 | ) (content.Extract, error) { 25 | extract, err := repo.Get(article) 26 | 27 | if err != nil { 28 | if !content.IsNoContent(err) { 29 | return extract, errors.WithMessage(err, fmt.Sprintf("getting extract for %s from repo", article)) 30 | } 31 | 32 | if extract, err = generator.Generate(article.Link); err != nil { 33 | return extract, errors.WithMessage(err, fmt.Sprintf("generating extract from %s", article.Link)) 34 | } 35 | 36 | extract.ArticleID = article.ID 37 | 38 | if err = repo.Update(extract); err != nil { 39 | return content.Extract{}, errors.WithMessage(err, fmt.Sprintf("updating extract %s", extract)) 40 | } 41 | } 42 | 43 | if len(processors) > 0 { 44 | a := content.Article{Description: extract.Content} 45 | 46 | articles := []content.Article{a} 47 | 48 | if extract.TopImage != "" { 49 | articles = append(articles, content.Article{ 50 | Description: fmt.Sprintf(``, extract.TopImage), 51 | }) 52 | } 53 | 54 | articles = processor.Articles(processors).Process(articles) 55 | 56 | extract.Content = articles[0].Description 57 | 58 | if extract.TopImage != "" { 59 | content := articles[1].Description 60 | 61 | content = strings.Replace(content, ` 0 { 26 | if _, ok := tagIDs[original[i].TagID]; ok { 27 | // Add the feed id to the list of feeds 28 | var found bool 29 | for _, id := range original[i].FeedIDs { 30 | if id == data.Feed.ID { 31 | found = true 32 | break 33 | } 34 | } 35 | 36 | if !found { 37 | original[i].FeedIDs = append( 38 | original[i].FeedIDs, 39 | data.Feed.ID, 40 | ) 41 | changed = true 42 | } 43 | } else { 44 | // Remove the feed id, if it was in the list 45 | for j := range original[i].FeedIDs { 46 | if original[i].FeedIDs[j] == data.Feed.ID { 47 | original[i].FeedIDs = append( 48 | original[i].FeedIDs[:j], 49 | original[i].FeedIDs[j+1:]..., 50 | ) 51 | 52 | changed = true 53 | break 54 | } 55 | } 56 | } 57 | } 58 | 59 | if original[i].Valid() { 60 | filters = append(filters, original[i]) 61 | } 62 | } 63 | 64 | if len(filters) != len(original) || changed { 65 | data.User.ProfileData["filters"] = filters 66 | if err := userRepo.Update(data.User); err != nil { 67 | log.Printf("Error updating user %s: %+v", data.User, err) 68 | } 69 | } 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /content/processor/absolutize_urls.go: -------------------------------------------------------------------------------- 1 | package processor 2 | 3 | import ( 4 | "net/url" 5 | "path" 6 | "strings" 7 | 8 | "github.com/PuerkitoBio/goquery" 9 | "github.com/urandom/readeef/log" 10 | "github.com/urandom/readeef/parser" 11 | ) 12 | 13 | type AbsolutizeURLs struct { 14 | log log.Log 15 | } 16 | 17 | func NewAbsolutizeURLs(l log.Log) AbsolutizeURLs { 18 | return AbsolutizeURLs{log: l} 19 | } 20 | 21 | func (p AbsolutizeURLs) ProcessFeed(f parser.Feed) parser.Feed { 22 | p.log.Infof("Converting relative urls of feed '%s' to absolute\n", f.Title) 23 | 24 | for i := range f.Articles { 25 | if d, err := goquery.NewDocumentFromReader(strings.NewReader(f.Articles[i].Description)); err == nil { 26 | articleLink, err := url.Parse(f.Articles[i].Link) 27 | if err != nil { 28 | continue 29 | } 30 | 31 | if convertRelativeLinksToAbsolute(d, articleLink) { 32 | if content, err := d.Html(); err == nil { 33 | // net/http tries to provide valid html, adding html, head and body tags 34 | content = content[strings.Index(content, "")+6 : strings.LastIndex(content, "")] 35 | 36 | f.Articles[i].Description = content 37 | } 38 | } 39 | } 40 | } 41 | 42 | return f 43 | } 44 | 45 | func convertRelativeLinksToAbsolute(d *goquery.Document, articleLink *url.URL) bool { 46 | changed := false 47 | d.Find("[src]").Each(func(i int, s *goquery.Selection) { 48 | val, ok := s.Attr("src") 49 | if !ok { 50 | return 51 | } 52 | 53 | u, err := url.Parse(val) 54 | if err != nil { 55 | return 56 | } 57 | 58 | if !u.IsAbs() { 59 | u.Scheme = articleLink.Scheme 60 | 61 | if u.Host == "" { 62 | u.Host = articleLink.Host 63 | 64 | if len(u.Path) > 0 && u.Path[0] != '/' { 65 | u.Path = path.Join(path.Dir(articleLink.Path), u.Path) 66 | } 67 | } 68 | 69 | } 70 | 71 | s.SetAttr("src", u.String()) 72 | 73 | changed = true 74 | return 75 | }) 76 | 77 | return changed 78 | } 79 | -------------------------------------------------------------------------------- /content/processor/common_test.go: -------------------------------------------------------------------------------- 1 | package processor 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/urandom/readeef/config" 7 | "github.com/urandom/readeef/log" 8 | ) 9 | 10 | var ( 11 | logger log.Log 12 | ) 13 | 14 | func init() { 15 | cfg := config.Log{} 16 | cfg.Converted.Writer = os.Stderr 17 | cfg.Converted.Prefix = "[testing] " 18 | logger = log.WithStd(cfg) 19 | } 20 | -------------------------------------------------------------------------------- /content/processor/insert_thumbnail_target.go: -------------------------------------------------------------------------------- 1 | package processor 2 | 3 | import ( 4 | "fmt" 5 | "html/template" 6 | "strings" 7 | 8 | "github.com/PuerkitoBio/goquery" 9 | "github.com/urandom/readeef/content" 10 | "github.com/urandom/readeef/log" 11 | ) 12 | 13 | type InsertThumbnailTarget struct { 14 | log log.Log 15 | urlTemplate *template.Template 16 | } 17 | 18 | func NewInsertThumbnailTarget(l log.Log) InsertThumbnailTarget { 19 | return InsertThumbnailTarget{log: l} 20 | } 21 | 22 | func (p InsertThumbnailTarget) ProcessArticles(articles []content.Article) []content.Article { 23 | if len(articles) == 0 { 24 | return articles 25 | } 26 | 27 | p.log.Infof("Proxying urls of feed '%d'\n", articles[0].FeedID) 28 | 29 | for i := range articles { 30 | if articles[i].ThumbnailLink == "" { 31 | continue 32 | } 33 | 34 | if d, err := goquery.NewDocumentFromReader(strings.NewReader(articles[i].Description)); err == nil { 35 | if insertThumbnailTarget(d, articles[i].ThumbnailLink, p.log) { 36 | if content, err := d.Html(); err == nil { 37 | // net/http tries to provide valid html, adding html, head and body tags 38 | content = content[strings.Index(content, "")+6 : strings.LastIndex(content, "")] 39 | 40 | articles[i].Description = content 41 | } 42 | } 43 | } 44 | } 45 | 46 | return articles 47 | } 48 | 49 | func insertThumbnailTarget(d *goquery.Document, thumbnailLink string, log log.Log) bool { 50 | changed := false 51 | 52 | if d.Find(".top-image").Length() > 0 { 53 | return changed 54 | } 55 | 56 | thumbDoc, err := goquery.NewDocumentFromReader(strings.NewReader(fmt.Sprintf(``, thumbnailLink))) 57 | if err != nil { 58 | log.Infof("Error generating thumbnail image node: %v\n", err) 59 | return changed 60 | } 61 | 62 | d.Find("body").PrependSelection(thumbDoc.Find("img")) 63 | changed = true 64 | 65 | return changed 66 | } 67 | -------------------------------------------------------------------------------- /content/processor/processor.go: -------------------------------------------------------------------------------- 1 | package processor 2 | 3 | import ( 4 | "github.com/urandom/readeef/content" 5 | "github.com/urandom/readeef/parser" 6 | ) 7 | 8 | type Article interface { 9 | ProcessArticles([]content.Article) []content.Article 10 | } 11 | 12 | type Feed interface { 13 | ProcessFeed(parser.Feed) parser.Feed 14 | } 15 | 16 | type Articles []Article 17 | 18 | func (processors Articles) Process(articles []content.Article) []content.Article { 19 | for _, p := range processors { 20 | articles = p.ProcessArticles(articles) 21 | } 22 | 23 | return articles 24 | } 25 | -------------------------------------------------------------------------------- /content/processor/top_image_marker.go: -------------------------------------------------------------------------------- 1 | package processor 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/PuerkitoBio/goquery" 7 | "github.com/urandom/readeef/log" 8 | "github.com/urandom/readeef/parser" 9 | "golang.org/x/net/html" 10 | ) 11 | 12 | type TopImageMarker struct { 13 | log log.Log 14 | } 15 | 16 | func NewTopImageMarker(l log.Log) TopImageMarker { 17 | return TopImageMarker{log: l} 18 | } 19 | 20 | func (p TopImageMarker) ProcessFeed(f parser.Feed) parser.Feed { 21 | p.log.Infof("Locating suitable top images in articles of '%s'\n", f.Title) 22 | 23 | for i := range f.Articles { 24 | if d, err := goquery.NewDocumentFromReader(strings.NewReader(f.Articles[i].Description)); err == nil { 25 | if markTopImage(d) { 26 | if content, err := d.Html(); err == nil { 27 | // net/http tries to provide valid html, adding html, head and body tags 28 | content = content[strings.Index(content, "")+6 : strings.LastIndex(content, "")] 29 | f.Articles[i].Description = content 30 | } 31 | } 32 | } 33 | } 34 | 35 | return f 36 | } 37 | 38 | func markTopImage(d *goquery.Document) bool { 39 | changed := false 40 | 41 | totalTextCount := len(d.Text()) 42 | img := d.Find("img") 43 | 44 | if img.Length() == 0 { 45 | return changed 46 | } 47 | 48 | if totalTextCount == 0 { 49 | img.AddClass("top-image") 50 | changed = true 51 | } else { 52 | afterTextCount := len(img.NextAll().Text()) 53 | // Add any text-node siblings of the image 54 | for n := img.Get(0).NextSibling; n != nil; n = n.NextSibling { 55 | if n.Type == html.TextNode { 56 | afterTextCount += len(n.Data) 57 | } 58 | } 59 | 60 | img.Parents().Each(func(i int, s *goquery.Selection) { 61 | afterTextCount += len(s.NextAll().Text()) 62 | }) 63 | 64 | if totalTextCount-afterTextCount <= afterTextCount { 65 | img.AddClass("top-image") 66 | changed = true 67 | } 68 | } 69 | 70 | return changed 71 | } 72 | -------------------------------------------------------------------------------- /content/processor/unescape.go: -------------------------------------------------------------------------------- 1 | package processor 2 | 3 | import ( 4 | "html" 5 | 6 | "github.com/urandom/readeef/content" 7 | "github.com/urandom/readeef/log" 8 | "github.com/urandom/readeef/parser" 9 | ) 10 | 11 | type Unescape struct { 12 | log log.Log 13 | } 14 | 15 | func NewUnescape(l log.Log) Unescape { 16 | return Unescape{log: l} 17 | } 18 | 19 | func (p Unescape) ProcessArticles(articles []content.Article) []content.Article { 20 | if len(articles) == 0 { 21 | return articles 22 | } 23 | 24 | p.log.Infof("Unescaping articles of feed %d", articles[0].FeedID) 25 | 26 | for i := range articles { 27 | articles[i].Description = p.processDescription(articles[i].Description) 28 | } 29 | 30 | return articles 31 | } 32 | 33 | func (p Unescape) ProcessFeed(f parser.Feed) parser.Feed { 34 | for i := range f.Articles { 35 | f.Articles[i].Description = p.processDescription(f.Articles[i].Description) 36 | } 37 | 38 | return f 39 | } 40 | 41 | func (p Unescape) processDescription(description string) string { 42 | return html.UnescapeString(description) 43 | } 44 | -------------------------------------------------------------------------------- /content/repo/article.go: -------------------------------------------------------------------------------- 1 | package repo 2 | 3 | import "github.com/urandom/readeef/content" 4 | 5 | // Article allows fetching and manipulating content.Article objects 6 | type Article interface { 7 | ForUser(content.User, ...content.QueryOpt) ([]content.Article, error) 8 | 9 | All(...content.QueryOpt) ([]content.Article, error) 10 | 11 | Count(content.User, ...content.QueryOpt) (int64, error) 12 | IDs(content.User, ...content.QueryOpt) ([]content.ArticleID, error) 13 | 14 | Read(bool, content.User, ...content.QueryOpt) error 15 | Favor(bool, content.User, ...content.QueryOpt) error 16 | 17 | RemoveStaleUnreadRecords() error 18 | } 19 | -------------------------------------------------------------------------------- /content/repo/common_test.go: -------------------------------------------------------------------------------- 1 | package repo_test 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/urandom/readeef/config" 8 | "github.com/urandom/readeef/content/repo" 9 | "github.com/urandom/readeef/log" 10 | ) 11 | 12 | var ( 13 | skip = true 14 | 15 | cfg config.Log 16 | logger log.Log 17 | service repo.Service 18 | ) 19 | 20 | func skipTest(t *testing.T) { 21 | if skip { 22 | t.Skip("No database tag selected") 23 | } 24 | } 25 | 26 | func init() { 27 | cfg.Converted.Writer = os.Stderr 28 | cfg.Converted.Prefix = "[testing] " 29 | 30 | logger = log.WithStd(cfg) 31 | } 32 | -------------------------------------------------------------------------------- /content/repo/eventable/bus.go: -------------------------------------------------------------------------------- 1 | package eventable 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/urandom/readeef/content" 7 | ) 8 | 9 | type Event struct { 10 | Name string 11 | Data interface{} 12 | } 13 | 14 | type UserData interface { 15 | UserLogin() content.Login 16 | } 17 | 18 | type FeedData interface { 19 | FeedID() content.FeedID 20 | } 21 | 22 | type Stream chan Event 23 | 24 | type busCall func(*busPayload) 25 | 26 | type busPayload struct { 27 | listeners []Stream 28 | } 29 | 30 | type bus struct { 31 | ops chan busCall 32 | } 33 | 34 | func newBus(ctx context.Context) bus { 35 | b := bus{ 36 | ops: make(chan busCall), 37 | } 38 | 39 | go b.loop(ctx) 40 | 41 | return b 42 | } 43 | 44 | func (b bus) Dispatch(name string, data interface{}) { 45 | b.ops <- func(p *busPayload) { 46 | event := Event{name, data} 47 | for i := range p.listeners { 48 | p.listeners[i] <- event 49 | } 50 | } 51 | } 52 | 53 | func (b bus) Listener() Stream { 54 | ret := make(chan Event, 10) 55 | 56 | b.ops <- func(p *busPayload) { 57 | p.listeners = append(p.listeners, ret) 58 | } 59 | 60 | return ret 61 | } 62 | 63 | func (b bus) loop(ctx context.Context) { 64 | payload := busPayload{ 65 | []Stream{}, 66 | } 67 | 68 | for { 69 | select { 70 | case op := <-b.ops: 71 | op(&payload) 72 | case <-ctx.Done(): 73 | return 74 | } 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /content/repo/eventable/bus_test.go: -------------------------------------------------------------------------------- 1 | package eventable 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | "time" 7 | 8 | "github.com/urandom/readeef/content" 9 | ) 10 | 11 | type event1data struct { 12 | data int 13 | } 14 | 15 | func (e event1data) UserLogin() content.Login { 16 | return "user1" 17 | } 18 | 19 | type event2data struct { 20 | data string 21 | } 22 | 23 | func (e event2data) UserLogin() content.Login { 24 | return "user2" 25 | } 26 | 27 | func Test_bus_Dispatch(t *testing.T) { 28 | type args struct { 29 | name string 30 | data UserData 31 | } 32 | tests := []struct { 33 | name string 34 | events []args 35 | listeners int 36 | }{ 37 | {"single event", []args{args{"event1", event1data{42}}}, 1}, 38 | {"multiple event", []args{ 39 | args{"event1", event1data{42}}, 40 | args{"event2", event2data{"event2"}}, 41 | }, 1}, 42 | {"single event, multi listeners", []args{args{"event1", event1data{42}}}, 3}, 43 | {"multiple event, multi listeners", []args{ 44 | args{"event1", event1data{42}}, 45 | args{"event2", event2data{"event2"}}, 46 | }, 5}, 47 | } 48 | for _, tt := range tests { 49 | t.Run(tt.name, func(t *testing.T) { 50 | ctx, cancel := context.WithCancel(context.Background()) 51 | defer cancel() 52 | b := newBus(ctx) 53 | 54 | done := make(chan struct{}) 55 | 56 | for i := 0; i < tt.listeners; i++ { 57 | go func() { 58 | l := b.Listener() 59 | i := 0 60 | 61 | for { 62 | select { 63 | case e := <-l: 64 | if e.Name != tt.events[i].name || e.Data != tt.events[i].data { 65 | t.Errorf("Test_bus_Dispatch(), expected %#v, got %#v", e, tt.events[i]) 66 | return 67 | } 68 | i++ 69 | case <-done: 70 | if i != len(tt.events) { 71 | t.Errorf("Received events = %d do not match expected = %d", i, len(tt.events)) 72 | } 73 | return 74 | case <-ctx.Done(): 75 | return 76 | } 77 | } 78 | }() 79 | } 80 | 81 | time.Sleep(500 * time.Millisecond) 82 | 83 | for _, e := range tt.events { 84 | b.Dispatch(e.name, e.data) 85 | } 86 | 87 | time.Sleep(500 * time.Millisecond) 88 | 89 | close(done) 90 | 91 | time.Sleep(500 * time.Millisecond) 92 | }) 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /content/repo/eventable/service.go: -------------------------------------------------------------------------------- 1 | package eventable 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/urandom/readeef/content/repo" 7 | "github.com/urandom/readeef/log" 8 | ) 9 | 10 | type Service struct { 11 | repo.Service 12 | eventBus bus 13 | 14 | article articleRepo 15 | feed feedRepo 16 | } 17 | 18 | func NewService(ctx context.Context, s repo.Service, log log.Log) Service { 19 | bus := newBus(ctx) 20 | 21 | return Service{ 22 | s, bus, 23 | articleRepo{s.ArticleRepo(), bus, log}, 24 | feedRepo{s.FeedRepo(), bus, log}, 25 | } 26 | } 27 | 28 | func (s Service) Listener() Stream { 29 | return s.eventBus.Listener() 30 | } 31 | 32 | func (s Service) ArticleRepo() repo.Article { 33 | return s.article 34 | } 35 | 36 | func (s Service) FeedRepo() repo.Feed { 37 | return s.feed 38 | } 39 | -------------------------------------------------------------------------------- /content/repo/extract.go: -------------------------------------------------------------------------------- 1 | package repo 2 | 3 | import "github.com/urandom/readeef/content" 4 | 5 | // Extract allows fetching and manipulating content.Extract objects 6 | type Extract interface { 7 | Get(content.Article) (content.Extract, error) 8 | Update(content.Extract) error 9 | } 10 | -------------------------------------------------------------------------------- /content/repo/feed.go: -------------------------------------------------------------------------------- 1 | package repo 2 | 3 | import "github.com/urandom/readeef/content" 4 | 5 | // Feed allows fetching and manipulating content.Feed objects 6 | type Feed interface { 7 | Get(content.FeedID, content.User) (content.Feed, error) 8 | FindByLink(link string) (content.Feed, error) 9 | 10 | ForUser(content.User) ([]content.Feed, error) 11 | ForTag(content.Tag, content.User) ([]content.Feed, error) 12 | All() ([]content.Feed, error) 13 | 14 | IDs() ([]content.FeedID, error) 15 | Unsubscribed() ([]content.Feed, error) 16 | 17 | Update(*content.Feed) ([]content.Article, error) 18 | Delete(content.Feed) error 19 | 20 | Users(content.Feed) ([]content.User, error) 21 | AttachTo(content.Feed, content.User) error 22 | DetachFrom(content.Feed, content.User) error 23 | 24 | SetUserTags(content.Feed, content.User, []*content.Tag) error 25 | } 26 | -------------------------------------------------------------------------------- /content/repo/logging/article.go: -------------------------------------------------------------------------------- 1 | package logging 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/urandom/readeef/content" 7 | "github.com/urandom/readeef/content/repo" 8 | "github.com/urandom/readeef/log" 9 | ) 10 | 11 | type articleRepo struct { 12 | repo.Article 13 | 14 | log log.Log 15 | } 16 | 17 | func (r articleRepo) ForUser(user content.User, opts ...content.QueryOpt) ([]content.Article, error) { 18 | start := time.Now() 19 | 20 | articles, err := r.Article.ForUser(user, opts...) 21 | 22 | r.log.Infof("repo.Article.ForUser took %s", time.Now().Sub(start)) 23 | 24 | return articles, err 25 | } 26 | 27 | func (r articleRepo) All(opts ...content.QueryOpt) ([]content.Article, error) { 28 | start := time.Now() 29 | 30 | articles, err := r.Article.All(opts...) 31 | 32 | r.log.Infof("repo.Article.All took %s", time.Now().Sub(start)) 33 | 34 | return articles, err 35 | } 36 | 37 | func (r articleRepo) Count(user content.User, opts ...content.QueryOpt) (int64, error) { 38 | start := time.Now() 39 | 40 | count, err := r.Article.Count(user, opts...) 41 | 42 | r.log.Infof("repo.Article.Count took %s", time.Now().Sub(start)) 43 | 44 | return count, err 45 | } 46 | 47 | func (r articleRepo) IDs(user content.User, opts ...content.QueryOpt) ([]content.ArticleID, error) { 48 | start := time.Now() 49 | 50 | ids, err := r.Article.IDs(user, opts...) 51 | 52 | r.log.Infof("repo.Article.IDs took %s", time.Now().Sub(start)) 53 | 54 | return ids, err 55 | } 56 | 57 | func (r articleRepo) Read(state bool, user content.User, opts ...content.QueryOpt) error { 58 | start := time.Now() 59 | 60 | err := r.Article.Read(state, user, opts...) 61 | 62 | r.log.Infof("repo.Article.Read took %s", time.Now().Sub(start)) 63 | 64 | return err 65 | } 66 | 67 | func (r articleRepo) Favor(state bool, user content.User, opts ...content.QueryOpt) error { 68 | start := time.Now() 69 | 70 | err := r.Article.Favor(state, user, opts...) 71 | 72 | r.log.Infof("repo.Article.Favor took %s", time.Now().Sub(start)) 73 | 74 | return err 75 | } 76 | 77 | func (r articleRepo) RemoveStaleUnreadRecords() error { 78 | start := time.Now() 79 | 80 | err := r.Article.RemoveStaleUnreadRecords() 81 | 82 | r.log.Infof("repo.Article.RemoveStaleUnreadRecords took %s", time.Now().Sub(start)) 83 | 84 | return err 85 | } 86 | -------------------------------------------------------------------------------- /content/repo/logging/extract.go: -------------------------------------------------------------------------------- 1 | package logging 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/urandom/readeef/content" 7 | "github.com/urandom/readeef/content/repo" 8 | "github.com/urandom/readeef/log" 9 | ) 10 | 11 | type extractRepo struct { 12 | repo.Extract 13 | 14 | log log.Log 15 | } 16 | 17 | func (r extractRepo) Get(article content.Article) (content.Extract, error) { 18 | start := time.Now() 19 | 20 | extract, err := r.Extract.Get(article) 21 | 22 | r.log.Infof("repo.Extract.Get took %s", time.Now().Sub(start)) 23 | 24 | return extract, err 25 | } 26 | 27 | func (r extractRepo) Update(extract content.Extract) error { 28 | start := time.Now() 29 | 30 | err := r.Extract.Update(extract) 31 | 32 | r.log.Infof("repo.Extract.Update took %s", time.Now().Sub(start)) 33 | 34 | return err 35 | } 36 | -------------------------------------------------------------------------------- /content/repo/logging/scores.go: -------------------------------------------------------------------------------- 1 | package logging 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/urandom/readeef/content" 7 | "github.com/urandom/readeef/content/repo" 8 | "github.com/urandom/readeef/log" 9 | ) 10 | 11 | type scoresRepo struct { 12 | repo.Scores 13 | 14 | log log.Log 15 | } 16 | 17 | func (r scoresRepo) Get(article content.Article) (content.Scores, error) { 18 | start := time.Now() 19 | 20 | scores, err := r.Scores.Get(article) 21 | 22 | r.log.Infof("repo.Scores.Get took %s", time.Now().Sub(start)) 23 | 24 | return scores, err 25 | } 26 | 27 | func (r scoresRepo) Update(scores content.Scores) error { 28 | start := time.Now() 29 | 30 | err := r.Scores.Update(scores) 31 | 32 | r.log.Infof("repo.Scores.Update took %s", time.Now().Sub(start)) 33 | 34 | return err 35 | } 36 | -------------------------------------------------------------------------------- /content/repo/logging/service.go: -------------------------------------------------------------------------------- 1 | package logging 2 | 3 | import ( 4 | "github.com/urandom/readeef/content/repo" 5 | "github.com/urandom/readeef/log" 6 | ) 7 | 8 | type Service struct { 9 | repo.Service 10 | 11 | article articleRepo 12 | extract extractRepo 13 | feed feedRepo 14 | scores scoresRepo 15 | subscription subscriptionRepo 16 | tag tagRepo 17 | thumbnail thumbnailRepo 18 | user userRepo 19 | } 20 | 21 | func NewService(s repo.Service, log log.Log) Service { 22 | return Service{ 23 | s, 24 | articleRepo{s.ArticleRepo(), log}, 25 | extractRepo{s.ExtractRepo(), log}, 26 | feedRepo{s.FeedRepo(), log}, 27 | scoresRepo{s.ScoresRepo(), log}, 28 | subscriptionRepo{s.SubscriptionRepo(), log}, 29 | tagRepo{s.TagRepo(), log}, 30 | thumbnailRepo{s.ThumbnailRepo(), log}, 31 | userRepo{s.UserRepo(), log}, 32 | } 33 | } 34 | 35 | func (s Service) ArticleRepo() repo.Article { 36 | return s.article 37 | } 38 | 39 | func (s Service) ExtractRepo() repo.Extract { 40 | return s.extract 41 | } 42 | 43 | func (s Service) FeedRepo() repo.Feed { 44 | return s.feed 45 | } 46 | 47 | func (s Service) ScoresRepo() repo.Scores { 48 | return s.scores 49 | } 50 | 51 | func (s Service) SubscriptionRepo() repo.Subscription { 52 | return s.subscription 53 | } 54 | 55 | func (s Service) TagRepo() repo.Tag { 56 | return s.tag 57 | } 58 | 59 | func (s Service) ThumbnailRepo() repo.Thumbnail { 60 | return s.thumbnail 61 | } 62 | 63 | func (s Service) UserRepo() repo.User { 64 | return s.user 65 | } 66 | -------------------------------------------------------------------------------- /content/repo/logging/subscription.go: -------------------------------------------------------------------------------- 1 | package logging 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/urandom/readeef/content" 7 | "github.com/urandom/readeef/content/repo" 8 | "github.com/urandom/readeef/log" 9 | ) 10 | 11 | type subscriptionRepo struct { 12 | repo.Subscription 13 | 14 | log log.Log 15 | } 16 | 17 | func (r subscriptionRepo) Get(feed content.Feed) (content.Subscription, error) { 18 | start := time.Now() 19 | 20 | subscription, err := r.Subscription.Get(feed) 21 | 22 | r.log.Infof("repo.Subscription.Get took %s", time.Now().Sub(start)) 23 | 24 | return subscription, err 25 | } 26 | 27 | func (r subscriptionRepo) All() ([]content.Subscription, error) { 28 | start := time.Now() 29 | 30 | subscriptions, err := r.Subscription.All() 31 | 32 | r.log.Infof("repo.Subscription.All took %s", time.Now().Sub(start)) 33 | 34 | return subscriptions, err 35 | } 36 | 37 | func (r subscriptionRepo) Update(subscription content.Subscription) error { 38 | start := time.Now() 39 | 40 | err := r.Subscription.Update(subscription) 41 | 42 | r.log.Infof("repo.Subscription.Update took %s", time.Now().Sub(start)) 43 | 44 | return err 45 | } 46 | -------------------------------------------------------------------------------- /content/repo/logging/tag.go: -------------------------------------------------------------------------------- 1 | package logging 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/urandom/readeef/content" 7 | "github.com/urandom/readeef/content/repo" 8 | "github.com/urandom/readeef/log" 9 | ) 10 | 11 | type tagRepo struct { 12 | repo.Tag 13 | 14 | log log.Log 15 | } 16 | 17 | func (r tagRepo) Get(id content.TagID, user content.User) (content.Tag, error) { 18 | start := time.Now() 19 | 20 | tag, err := r.Tag.Get(id, user) 21 | 22 | r.log.Infof("repo.Tag.Get took %s", time.Now().Sub(start)) 23 | 24 | return tag, err 25 | } 26 | 27 | func (r tagRepo) ForUser(user content.User) ([]content.Tag, error) { 28 | start := time.Now() 29 | 30 | tags, err := r.Tag.ForUser(user) 31 | 32 | r.log.Infof("repo.Tag.ForUser took %s", time.Now().Sub(start)) 33 | 34 | return tags, err 35 | } 36 | 37 | func (r tagRepo) ForFeed(feed content.Feed, user content.User) ([]content.Tag, error) { 38 | start := time.Now() 39 | 40 | tags, err := r.Tag.ForFeed(feed, user) 41 | 42 | r.log.Infof("repo.Tag.ForFeed took %s", time.Now().Sub(start)) 43 | 44 | return tags, err 45 | } 46 | 47 | func (r tagRepo) FeedIDs(tag content.Tag, user content.User) ([]content.FeedID, error) { 48 | start := time.Now() 49 | 50 | ids, err := r.Tag.FeedIDs(tag, user) 51 | 52 | r.log.Infof("repo.Tag.FeedIDs took %s", time.Now().Sub(start)) 53 | 54 | return ids, err 55 | } 56 | -------------------------------------------------------------------------------- /content/repo/logging/thumbnail.go: -------------------------------------------------------------------------------- 1 | package logging 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/urandom/readeef/content" 7 | "github.com/urandom/readeef/content/repo" 8 | "github.com/urandom/readeef/log" 9 | ) 10 | 11 | type thumbnailRepo struct { 12 | repo.Thumbnail 13 | 14 | log log.Log 15 | } 16 | 17 | func (r thumbnailRepo) Get(article content.Article) (content.Thumbnail, error) { 18 | start := time.Now() 19 | 20 | thumbnail, err := r.Thumbnail.Get(article) 21 | 22 | r.log.Infof("repo.Thumbnail.Get took %s", time.Now().Sub(start)) 23 | 24 | return thumbnail, err 25 | } 26 | 27 | func (r thumbnailRepo) Update(thumbnail content.Thumbnail) error { 28 | start := time.Now() 29 | 30 | err := r.Thumbnail.Update(thumbnail) 31 | 32 | r.log.Infof("repo.Thumbnail.Update took %s", time.Now().Sub(start)) 33 | 34 | return err 35 | } 36 | -------------------------------------------------------------------------------- /content/repo/logging/user.go: -------------------------------------------------------------------------------- 1 | package logging 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/urandom/readeef/content" 7 | "github.com/urandom/readeef/content/repo" 8 | "github.com/urandom/readeef/log" 9 | ) 10 | 11 | type userRepo struct { 12 | repo.User 13 | 14 | log log.Log 15 | } 16 | 17 | func (r userRepo) Get(login content.Login) (content.User, error) { 18 | start := time.Now() 19 | 20 | user, err := r.User.Get(login) 21 | 22 | r.log.Infof("repo.User.Get took %s", time.Now().Sub(start)) 23 | 24 | return user, err 25 | } 26 | 27 | func (r userRepo) All() ([]content.User, error) { 28 | start := time.Now() 29 | 30 | users, err := r.User.All() 31 | 32 | r.log.Infof("repo.User.All took %s", time.Now().Sub(start)) 33 | 34 | return users, err 35 | } 36 | 37 | func (r userRepo) Update(user content.User) error { 38 | start := time.Now() 39 | 40 | err := r.User.Update(user) 41 | 42 | r.log.Infof("repo.User.Update took %s", time.Now().Sub(start)) 43 | 44 | return err 45 | } 46 | 47 | func (r userRepo) Delete(user content.User) error { 48 | start := time.Now() 49 | 50 | err := r.User.Delete(user) 51 | 52 | r.log.Infof("repo.User.Delete took %s", time.Now().Sub(start)) 53 | 54 | return err 55 | } 56 | 57 | func (r userRepo) FindByMD5(data []byte) (content.User, error) { 58 | start := time.Now() 59 | 60 | user, err := r.User.FindByMD5(data) 61 | 62 | r.log.Infof("repo.User.FindByMD5 took %s", time.Now().Sub(start)) 63 | 64 | return user, err 65 | } 66 | -------------------------------------------------------------------------------- /content/repo/mock_repo/extract.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: github.com/urandom/readeef/content/repo (interfaces: Extract) 3 | 4 | // Package mock_repo is a generated GoMock package. 5 | package mock_repo 6 | 7 | import ( 8 | gomock "github.com/golang/mock/gomock" 9 | content "github.com/urandom/readeef/content" 10 | reflect "reflect" 11 | ) 12 | 13 | // MockExtract is a mock of Extract interface 14 | type MockExtract struct { 15 | ctrl *gomock.Controller 16 | recorder *MockExtractMockRecorder 17 | } 18 | 19 | // MockExtractMockRecorder is the mock recorder for MockExtract 20 | type MockExtractMockRecorder struct { 21 | mock *MockExtract 22 | } 23 | 24 | // NewMockExtract creates a new mock instance 25 | func NewMockExtract(ctrl *gomock.Controller) *MockExtract { 26 | mock := &MockExtract{ctrl: ctrl} 27 | mock.recorder = &MockExtractMockRecorder{mock} 28 | return mock 29 | } 30 | 31 | // EXPECT returns an object that allows the caller to indicate expected use 32 | func (m *MockExtract) EXPECT() *MockExtractMockRecorder { 33 | return m.recorder 34 | } 35 | 36 | // Get mocks base method 37 | func (m *MockExtract) Get(arg0 content.Article) (content.Extract, error) { 38 | ret := m.ctrl.Call(m, "Get", arg0) 39 | ret0, _ := ret[0].(content.Extract) 40 | ret1, _ := ret[1].(error) 41 | return ret0, ret1 42 | } 43 | 44 | // Get indicates an expected call of Get 45 | func (mr *MockExtractMockRecorder) Get(arg0 interface{}) *gomock.Call { 46 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockExtract)(nil).Get), arg0) 47 | } 48 | 49 | // Update mocks base method 50 | func (m *MockExtract) Update(arg0 content.Extract) error { 51 | ret := m.ctrl.Call(m, "Update", arg0) 52 | ret0, _ := ret[0].(error) 53 | return ret0 54 | } 55 | 56 | // Update indicates an expected call of Update 57 | func (mr *MockExtractMockRecorder) Update(arg0 interface{}) *gomock.Call { 58 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockExtract)(nil).Update), arg0) 59 | } 60 | -------------------------------------------------------------------------------- /content/repo/mock_repo/scores.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: github.com/urandom/readeef/content/repo (interfaces: Scores) 3 | 4 | // Package mock_repo is a generated GoMock package. 5 | package mock_repo 6 | 7 | import ( 8 | gomock "github.com/golang/mock/gomock" 9 | content "github.com/urandom/readeef/content" 10 | reflect "reflect" 11 | ) 12 | 13 | // MockScores is a mock of Scores interface 14 | type MockScores struct { 15 | ctrl *gomock.Controller 16 | recorder *MockScoresMockRecorder 17 | } 18 | 19 | // MockScoresMockRecorder is the mock recorder for MockScores 20 | type MockScoresMockRecorder struct { 21 | mock *MockScores 22 | } 23 | 24 | // NewMockScores creates a new mock instance 25 | func NewMockScores(ctrl *gomock.Controller) *MockScores { 26 | mock := &MockScores{ctrl: ctrl} 27 | mock.recorder = &MockScoresMockRecorder{mock} 28 | return mock 29 | } 30 | 31 | // EXPECT returns an object that allows the caller to indicate expected use 32 | func (m *MockScores) EXPECT() *MockScoresMockRecorder { 33 | return m.recorder 34 | } 35 | 36 | // Get mocks base method 37 | func (m *MockScores) Get(arg0 content.Article) (content.Scores, error) { 38 | ret := m.ctrl.Call(m, "Get", arg0) 39 | ret0, _ := ret[0].(content.Scores) 40 | ret1, _ := ret[1].(error) 41 | return ret0, ret1 42 | } 43 | 44 | // Get indicates an expected call of Get 45 | func (mr *MockScoresMockRecorder) Get(arg0 interface{}) *gomock.Call { 46 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockScores)(nil).Get), arg0) 47 | } 48 | 49 | // Update mocks base method 50 | func (m *MockScores) Update(arg0 content.Scores) error { 51 | ret := m.ctrl.Call(m, "Update", arg0) 52 | ret0, _ := ret[0].(error) 53 | return ret0 54 | } 55 | 56 | // Update indicates an expected call of Update 57 | func (mr *MockScoresMockRecorder) Update(arg0 interface{}) *gomock.Call { 58 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockScores)(nil).Update), arg0) 59 | } 60 | -------------------------------------------------------------------------------- /content/repo/mock_repo/thumbnail.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: github.com/urandom/readeef/content/repo (interfaces: Thumbnail) 3 | 4 | // Package mock_repo is a generated GoMock package. 5 | package mock_repo 6 | 7 | import ( 8 | gomock "github.com/golang/mock/gomock" 9 | content "github.com/urandom/readeef/content" 10 | reflect "reflect" 11 | ) 12 | 13 | // MockThumbnail is a mock of Thumbnail interface 14 | type MockThumbnail struct { 15 | ctrl *gomock.Controller 16 | recorder *MockThumbnailMockRecorder 17 | } 18 | 19 | // MockThumbnailMockRecorder is the mock recorder for MockThumbnail 20 | type MockThumbnailMockRecorder struct { 21 | mock *MockThumbnail 22 | } 23 | 24 | // NewMockThumbnail creates a new mock instance 25 | func NewMockThumbnail(ctrl *gomock.Controller) *MockThumbnail { 26 | mock := &MockThumbnail{ctrl: ctrl} 27 | mock.recorder = &MockThumbnailMockRecorder{mock} 28 | return mock 29 | } 30 | 31 | // EXPECT returns an object that allows the caller to indicate expected use 32 | func (m *MockThumbnail) EXPECT() *MockThumbnailMockRecorder { 33 | return m.recorder 34 | } 35 | 36 | // Get mocks base method 37 | func (m *MockThumbnail) Get(arg0 content.Article) (content.Thumbnail, error) { 38 | ret := m.ctrl.Call(m, "Get", arg0) 39 | ret0, _ := ret[0].(content.Thumbnail) 40 | ret1, _ := ret[1].(error) 41 | return ret0, ret1 42 | } 43 | 44 | // Get indicates an expected call of Get 45 | func (mr *MockThumbnailMockRecorder) Get(arg0 interface{}) *gomock.Call { 46 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockThumbnail)(nil).Get), arg0) 47 | } 48 | 49 | // Update mocks base method 50 | func (m *MockThumbnail) Update(arg0 content.Thumbnail) error { 51 | ret := m.ctrl.Call(m, "Update", arg0) 52 | ret0, _ := ret[0].(error) 53 | return ret0 54 | } 55 | 56 | // Update indicates an expected call of Update 57 | func (mr *MockThumbnailMockRecorder) Update(arg0 interface{}) *gomock.Call { 58 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockThumbnail)(nil).Update), arg0) 59 | } 60 | -------------------------------------------------------------------------------- /content/repo/postgres_test.go: -------------------------------------------------------------------------------- 1 | // +build postgres 2 | 3 | package repo_test 4 | 5 | import ( 6 | "os" 7 | "testing" 8 | 9 | "github.com/urandom/readeef/content/repo/sql" 10 | "github.com/urandom/readeef/content/repo/sql/db" 11 | _ "github.com/urandom/readeef/content/repo/sql/db/postgres" 12 | ) 13 | 14 | func TestMain(m *testing.M) { 15 | db := db.New(logger) 16 | if err := db.Open("postgres", "host=/var/run/postgresql user=urandom dbname=readeef-test sslmode=disable"); err != nil { 17 | panic(err) 18 | } 19 | 20 | db.Exec("TRUNCATE articles CASCADE") 21 | db.Exec("TRUNCATE articles_scores CASCADE") 22 | db.Exec("TRUNCATE feed_images CASCADE") 23 | db.Exec("TRUNCATE feeds CASCADE") 24 | db.Exec("TRUNCATE hubbub_subscriptions CASCADE") 25 | db.Exec("TRUNCATE users CASCADE") 26 | db.Exec("TRUNCATE users_articles_states CASCADE") 27 | db.Exec("TRUNCATE users_feeds CASCADE") 28 | db.Exec("TRUNCATE users_feeds_tags CASCADE") 29 | 30 | db.Close() 31 | 32 | var err error 33 | service, err = sql.NewService("postgres", "host=/var/run/postgresql user=urandom dbname=readeef-test sslmode=disable", logger) 34 | if err != nil { 35 | panic(err) 36 | } 37 | 38 | skip = false 39 | ret := m.Run() 40 | 41 | os.Exit(ret) 42 | } 43 | -------------------------------------------------------------------------------- /content/repo/scores.go: -------------------------------------------------------------------------------- 1 | package repo 2 | 3 | import "github.com/urandom/readeef/content" 4 | 5 | // Scores allows fetching and manipulating content.Scores objects 6 | type Scores interface { 7 | Get(content.Article) (content.Scores, error) 8 | Update(content.Scores) error 9 | } 10 | -------------------------------------------------------------------------------- /content/repo/service.go: -------------------------------------------------------------------------------- 1 | package repo 2 | 3 | // Service provices access to the different content repositories. 4 | type Service interface { 5 | UserRepo() User 6 | TagRepo() Tag 7 | FeedRepo() Feed 8 | SubscriptionRepo() Subscription 9 | ArticleRepo() Article 10 | ExtractRepo() Extract 11 | ThumbnailRepo() Thumbnail 12 | ScoresRepo() Scores 13 | } 14 | -------------------------------------------------------------------------------- /content/repo/service_test.go: -------------------------------------------------------------------------------- 1 | package repo_test 2 | 3 | import "testing" 4 | 5 | func TestService(t *testing.T) { 6 | skipTest(t) 7 | 8 | if service.UserRepo() == nil { 9 | t.Fatal("service.UserRepo() = nil") 10 | } 11 | 12 | if service.TagRepo() == nil { 13 | t.Fatal("service.TagRepo() = nil") 14 | } 15 | 16 | if service.FeedRepo() == nil { 17 | t.Fatal("service.FeedRepo() = nil") 18 | } 19 | 20 | if service.SubscriptionRepo() == nil { 21 | t.Fatal("service.SubscriptionRepo() = nil") 22 | } 23 | 24 | if service.ArticleRepo() == nil { 25 | t.Fatal("service.ArticleRepo() = nil") 26 | } 27 | 28 | if service.ExtractRepo() == nil { 29 | t.Fatal("service.ExtractRepo() = nil") 30 | } 31 | 32 | if service.ThumbnailRepo() == nil { 33 | t.Fatal("service.ThumbnailRepo() = nil") 34 | } 35 | 36 | if service.ScoresRepo() == nil { 37 | t.Fatal("service.ScoresRepo() = nil") 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /content/repo/sql/db/base/extract.go: -------------------------------------------------------------------------------- 1 | package base 2 | 3 | func init() { 4 | sqlStmts.Extract.Get = getArticleExtract 5 | sqlStmts.Extract.Create = createArticleExtract 6 | sqlStmts.Extract.Update = updateArticleExtract 7 | } 8 | 9 | const ( 10 | getArticleExtract = ` 11 | SELECT ae.title, ae.content, ae.top_image, ae.language 12 | FROM articles_extracts ae 13 | WHERE ae.article_id = :article_id 14 | ` 15 | createArticleExtract = ` 16 | INSERT INTO articles_extracts(article_id, title, content, top_image, language) 17 | SELECT :article_id, :title, :content, :top_image, :language EXCEPT SELECT article_id, title, content, top_image, language FROM articles_extracts WHERE article_id = :article_id 18 | ` 19 | updateArticleExtract = ` 20 | UPDATE articles_extracts SET title = :title, content = :content, top_image = :top_image, language = :language WHERE article_id = :article_id` 21 | ) 22 | -------------------------------------------------------------------------------- /content/repo/sql/db/base/helper.go: -------------------------------------------------------------------------------- 1 | package base 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | "strings" 7 | 8 | "github.com/jmoiron/sqlx" 9 | "github.com/urandom/readeef/content/repo/sql/db" 10 | ) 11 | 12 | type Helper struct { 13 | sql db.SqlStmts 14 | } 15 | 16 | func NewHelper() *Helper { 17 | return &Helper{sql: sqlStmts} 18 | } 19 | 20 | func (h Helper) SQL() db.SqlStmts { 21 | return h.sql 22 | } 23 | 24 | func (h *Helper) Set(override db.SqlStmts) { 25 | oursPtr := reflect.ValueOf(&h.sql) 26 | ours := oursPtr.Elem() 27 | theirs := reflect.ValueOf(override) 28 | 29 | for i := 0; i < ours.NumField(); i++ { 30 | ourInner := ours.Field(i) 31 | theirInner := theirs.Field(i) 32 | 33 | if theirInner.IsValid() { 34 | for j := 0; j < theirInner.NumField(); j++ { 35 | ourField := ourInner.Field(j) 36 | theirField := theirInner.Field(j) 37 | 38 | if theirField.IsValid() && ourField.CanSet() && ourField.Kind() == reflect.String { 39 | s := theirField.String() 40 | if s != "" { 41 | ourField.SetString(s) 42 | } 43 | } 44 | } 45 | } 46 | } 47 | } 48 | 49 | func (h Helper) Upgrade(db *db.DB, old, new int) error { 50 | return nil 51 | } 52 | 53 | func (h Helper) CreateWithID(tx *sqlx.Tx, sql string, arg interface{}) (int64, error) { 54 | var id int64 55 | 56 | stmt, err := tx.PrepareNamed(sql) 57 | if err != nil { 58 | return 0, err 59 | } 60 | defer stmt.Close() 61 | 62 | res, err := stmt.Exec(arg) 63 | if err != nil { 64 | return 0, err 65 | } 66 | 67 | id, err = res.LastInsertId() 68 | if err != nil { 69 | return 0, err 70 | } 71 | 72 | return id, nil 73 | } 74 | 75 | func (h Helper) WhereMultipleORs(column, prefix string, length int, equal bool) string { 76 | orSlice := make([]string, length) 77 | for i := 0; i < length; i++ { 78 | orSlice[i] = fmt.Sprintf(":%s%d", prefix, i) 79 | } 80 | 81 | sign := "IN" 82 | if !equal { 83 | sign = "NOT IN" 84 | } 85 | 86 | return fmt.Sprintf("%s %s (%s)", column, sign, strings.Join(orSlice, ", ")) 87 | } 88 | 89 | func (h Helper) RetryableErr(err error) bool { 90 | return false 91 | } 92 | 93 | var ( 94 | sqlStmts = db.SqlStmts{} 95 | ) 96 | -------------------------------------------------------------------------------- /content/repo/sql/db/base/scores.go: -------------------------------------------------------------------------------- 1 | package base 2 | 3 | func init() { 4 | sqlStmts.Scores.Get = getArticleScores 5 | sqlStmts.Scores.Create = createArticleScores 6 | sqlStmts.Scores.Update = updateArticleScores 7 | } 8 | 9 | const ( 10 | getArticleScores = ` 11 | SELECT asco.score, asco.score1, asco.score2, asco.score3, asco.score4, asco.score5 12 | FROM articles_scores asco 13 | WHERE asco.article_id = :article_id 14 | ` 15 | createArticleScores = ` 16 | INSERT INTO articles_scores(article_id, score, score1, score2, score3, score4, score5) 17 | SELECT :article_id, :score, :score1, :score2, :score3, :score4, :score5 EXCEPT SELECT article_id, score, score1, score2, score3, score4, score5 FROM articles_scores WHERE article_id = :article_id` 18 | updateArticleScores = `UPDATE articles_scores SET score = :score, score1 = :score1, score2 = :score2, score3 = :score3, score4 = :score4, score5 = :score5 WHERE article_id = :article_id` 19 | ) 20 | -------------------------------------------------------------------------------- /content/repo/sql/db/base/subscription.go: -------------------------------------------------------------------------------- 1 | package base 2 | 3 | func init() { 4 | sqlStmts.Subscription.GetForFeed = getFeedHubbubSubscription 5 | sqlStmts.Subscription.All = getHubbubSubscriptions 6 | sqlStmts.Subscription.Create = createHubbubSubscription 7 | sqlStmts.Subscription.Update = updateHubbubSubscription 8 | } 9 | 10 | const ( 11 | getFeedHubbubSubscription = ` 12 | SELECT link, lease_duration, verification_time, subscription_failure 13 | FROM hubbub_subscriptions WHERE feed_id = :feed_id` 14 | getHubbubSubscriptions = ` 15 | SELECT link, feed_id, lease_duration, verification_time, subscription_failure 16 | FROM hubbub_subscriptions` 17 | 18 | createHubbubSubscription = ` 19 | INSERT INTO hubbub_subscriptions(feed_id, link, lease_duration, verification_time, subscription_failure) 20 | SELECT :feed_id, :link, :lease_duration, :verification_time, :subscription_failure EXCEPT 21 | SELECT feed_id, link, lease_duration, verification_time, subscription_failure 22 | FROM hubbub_subscriptions WHERE feed_id = :feed_id 23 | ` 24 | updateHubbubSubscription = ` 25 | UPDATE hubbub_subscriptions SET link = :link, lease_duration = :lease_duration, 26 | verification_time = :verification_time, subscription_failure = :subscription_failure WHERE feed_id = :feed_id 27 | ` 28 | ) 29 | -------------------------------------------------------------------------------- /content/repo/sql/db/base/tag.go: -------------------------------------------------------------------------------- 1 | package base 2 | 3 | func init() { 4 | sqlStmts.Tag.Get = getUserTag 5 | sqlStmts.Tag.GetByValue = getTagByValue 6 | sqlStmts.Tag.GetUserFeedIDs = getUserTagFeedIDs 7 | sqlStmts.Tag.AllForUser = getUserTags 8 | sqlStmts.Tag.AllForFeed = getUserFeedTags 9 | sqlStmts.Tag.Create = createTag 10 | sqlStmts.Tag.DeleteStale = deleteStaleTags 11 | } 12 | 13 | const ( 14 | getUserTag = ` 15 | SELECT t.value 16 | FROM tags t LEFT OUTER JOIN users_feeds_tags uft 17 | ON t.id = uft.tag_id 18 | WHERE id = :id AND uft.user_login = :user_login 19 | ` 20 | getTagByValue = `SELECT id FROM tags WHERE value = :value` 21 | getUserFeedTags = ` 22 | SELECT t.id, t.value 23 | FROM users_feeds_tags uft INNER JOIN tags t 24 | ON uft.tag_id = t.id 25 | WHERE uft.user_login = :user_login AND uft.feed_id = :feed_id` 26 | getUserTags = ` 27 | SELECT DISTINCT t.id, t.value 28 | FROM tags t LEFT OUTER JOIN users_feeds_tags uft 29 | ON t.id = uft.tag_id 30 | WHERE uft.user_login = :user_login 31 | ` 32 | getUserTagFeedIDs = ` 33 | SELECT uft.feed_id 34 | FROM users_feeds_tags uft 35 | WHERE uft.user_login = :user_login AND uft.tag_id = :id 36 | ` 37 | 38 | createTag = ` 39 | INSERT INTO tags (value) 40 | SELECT :value EXCEPT SELECT value FROM tags WHERE value = :value 41 | ` 42 | 43 | deleteStaleTags = `DELETE FROM tags WHERE id NOT IN (SELECT tag_id FROM users_feeds_tags)` 44 | ) 45 | -------------------------------------------------------------------------------- /content/repo/sql/db/base/thumbnail.go: -------------------------------------------------------------------------------- 1 | package base 2 | 3 | func init() { 4 | sqlStmts.Thumbnail.Get = getArticleThumbnail 5 | sqlStmts.Thumbnail.Create = createArticleThumbnail 6 | sqlStmts.Thumbnail.Update = updateArticleThumbnail 7 | } 8 | 9 | const ( 10 | getArticleThumbnail = ` 11 | SELECT at.thumbnail, at.link, at.processed 12 | FROM articles_thumbnails at 13 | WHERE at.article_id = :article_id 14 | ` 15 | createArticleThumbnail = ` 16 | INSERT INTO articles_thumbnails(article_id, thumbnail, link, processed) 17 | SELECT :article_id, :thumbnail, :link, :processed EXCEPT SELECT article_id, thumbnail, link, processed FROM articles_thumbnails WHERE article_id = :article_id 18 | ` 19 | updateArticleThumbnail = ` 20 | UPDATE articles_thumbnails SET thumbnail = :thumbnail, link = :link, processed = :processed WHERE article_id = :article_id` 21 | ) 22 | -------------------------------------------------------------------------------- /content/repo/sql/db/base/user.go: -------------------------------------------------------------------------------- 1 | package base 2 | 3 | func init() { 4 | sqlStmts.User.Get = getUser 5 | sqlStmts.User.GetByMD5API = getUserByMD5Api 6 | sqlStmts.User.All = getUsers 7 | sqlStmts.User.Create = createUser 8 | sqlStmts.User.Update = updateUser 9 | sqlStmts.User.Delete = deleteUser 10 | } 11 | 12 | const ( 13 | getUser = `SELECT first_name, last_name, email, admin, active, profile_data, hash_type, salt, hash, md5_api FROM users WHERE login = :login` 14 | getUserByMD5Api = `SELECT login, first_name, last_name, email, admin, active, profile_data, hash_type, salt, hash FROM users WHERE md5_api = :md5_api` 15 | getUsers = `SELECT login, first_name, last_name, email, admin, active, profile_data, hash_type, salt, hash, md5_api FROM users` 16 | 17 | createUser = ` 18 | INSERT INTO users(login, first_name, last_name, email, admin, active, profile_data, hash_type, salt, hash, md5_api) 19 | SELECT :login, :first_name, :last_name, :email, :admin, :active, :profile_data, :hash_type, :salt, :hash, :md5_api EXCEPT 20 | SELECT login, first_name, last_name, email, admin, active, profile_data, hash_type, salt, hash, md5_api FROM users WHERE login = :login` 21 | updateUser = ` 22 | UPDATE users SET first_name = :first_name, last_name = :last_name, email = :email, admin = :admin, active = :active, profile_data = :profile_data, hash_type = :hash_type, salt = :salt, hash = :hash, md5_api = :md5_api 23 | WHERE login = :login` 24 | deleteUser = `DELETE FROM users WHERE login = :login` 25 | ) 26 | -------------------------------------------------------------------------------- /content/repo/sql/db/sqlite3/cgo.go: -------------------------------------------------------------------------------- 1 | // +build cgo 2 | 3 | package sqlite3 4 | 5 | import _ "github.com/mattn/go-sqlite3" 6 | -------------------------------------------------------------------------------- /content/repo/sql/extract.go: -------------------------------------------------------------------------------- 1 | package sql 2 | 3 | import ( 4 | "database/sql" 5 | 6 | "github.com/jmoiron/sqlx" 7 | "github.com/pkg/errors" 8 | "github.com/urandom/readeef/content" 9 | "github.com/urandom/readeef/content/repo/sql/db" 10 | "github.com/urandom/readeef/log" 11 | ) 12 | 13 | type extractRepo struct { 14 | db *db.DB 15 | 16 | log log.Log 17 | } 18 | 19 | func (r extractRepo) Get(article content.Article) (content.Extract, error) { 20 | if err := article.Validate(); err != nil { 21 | return content.Extract{}, errors.WithMessage(err, "validating article") 22 | } 23 | 24 | r.log.Infof("Getting extract for article %s", article) 25 | 26 | extract := content.Extract{ArticleID: article.ID} 27 | if err := r.db.WithNamedStmt(r.db.SQL().Extract.Get, nil, func(stmt *sqlx.NamedStmt) error { 28 | return stmt.Get(&extract, extract) 29 | }); err != nil { 30 | if err == sql.ErrNoRows { 31 | err = content.ErrNoContent 32 | } 33 | 34 | return content.Extract{}, errors.Wrapf(err, "getting extract for article %s", article) 35 | } 36 | 37 | return extract, nil 38 | } 39 | 40 | func (r extractRepo) Update(extract content.Extract) error { 41 | if err := extract.Validate(); err != nil { 42 | return errors.WithMessage(err, "validating extract") 43 | } 44 | 45 | r.log.Infof("Updating extract %s", extract) 46 | 47 | return r.db.WithTx(func(tx *sqlx.Tx) error { 48 | s := r.db.SQL() 49 | return r.db.WithNamedStmt(s.Extract.Update, tx, func(stmt *sqlx.NamedStmt) error { 50 | res, err := stmt.Exec(extract) 51 | if err != nil { 52 | return errors.Wrap(err, "executing extract update stmt") 53 | } 54 | 55 | if num, err := res.RowsAffected(); err == nil && num > 0 { 56 | return nil 57 | } 58 | 59 | return r.db.WithNamedStmt(s.Extract.Create, tx, func(stmt *sqlx.NamedStmt) error { 60 | if _, err = stmt.Exec(extract); err != nil { 61 | return errors.Wrap(err, "executing extract create stmt") 62 | } 63 | return nil 64 | }) 65 | }) 66 | }) 67 | } 68 | -------------------------------------------------------------------------------- /content/repo/sql/scores.go: -------------------------------------------------------------------------------- 1 | package sql 2 | 3 | import ( 4 | "database/sql" 5 | 6 | "github.com/jmoiron/sqlx" 7 | "github.com/pkg/errors" 8 | "github.com/urandom/readeef/content" 9 | "github.com/urandom/readeef/content/repo/sql/db" 10 | "github.com/urandom/readeef/log" 11 | ) 12 | 13 | type scoresRepo struct { 14 | db *db.DB 15 | 16 | log log.Log 17 | } 18 | 19 | func (r scoresRepo) Get(article content.Article) (content.Scores, error) { 20 | if err := article.Validate(); err != nil { 21 | return content.Scores{}, errors.WithMessage(err, "validating article") 22 | } 23 | 24 | r.log.Debugf("Getting scores for article %s", article) 25 | 26 | scores := content.Scores{ArticleID: article.ID} 27 | if err := r.db.WithNamedStmt(r.db.SQL().Scores.Get, nil, func(stmt *sqlx.NamedStmt) error { 28 | return stmt.Get(&scores, scores) 29 | }); err != nil { 30 | if err == sql.ErrNoRows { 31 | err = content.ErrNoContent 32 | } 33 | 34 | return content.Scores{}, errors.Wrapf(err, "getting scores for article %s", article) 35 | } 36 | 37 | return scores, nil 38 | } 39 | 40 | func (r scoresRepo) Update(scores content.Scores) error { 41 | if err := scores.Validate(); err != nil { 42 | return errors.WithMessage(err, "validating scores") 43 | } 44 | 45 | r.log.Infof("Updating scores %s", scores) 46 | 47 | s := r.db.SQL() 48 | return r.db.WithTx(func(tx *sqlx.Tx) error { 49 | return r.db.WithNamedStmt(s.Scores.Update, tx, func(stmt *sqlx.NamedStmt) error { 50 | res, err := stmt.Exec(scores) 51 | if err != nil { 52 | return errors.Wrap(err, "executing scores update stmt") 53 | } 54 | 55 | if num, err := res.RowsAffected(); err == nil && num > 0 { 56 | return nil 57 | } 58 | 59 | return r.db.WithNamedStmt(s.Scores.Create, tx, func(stmt *sqlx.NamedStmt) error { 60 | if _, err = stmt.Exec(scores); err != nil { 61 | return errors.Wrap(err, "executing scores create stmt") 62 | } 63 | 64 | return nil 65 | }) 66 | }) 67 | }) 68 | } 69 | -------------------------------------------------------------------------------- /content/repo/sql/service.go: -------------------------------------------------------------------------------- 1 | package sql 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/pkg/errors" 7 | "github.com/urandom/readeef/content/repo" 8 | "github.com/urandom/readeef/content/repo/sql/db" 9 | "github.com/urandom/readeef/log" 10 | ) 11 | 12 | type Service struct { 13 | user repo.User 14 | tag repo.Tag 15 | feed repo.Feed 16 | subscription repo.Subscription 17 | article repo.Article 18 | extract repo.Extract 19 | scores repo.Scores 20 | thumbnail repo.Thumbnail 21 | } 22 | 23 | func NewService(driver, source string, log log.Log) (Service, error) { 24 | switch driver { 25 | case "sqlite3", "postgres": 26 | db := db.New(log) 27 | if err := db.Open(driver, source); err != nil { 28 | return Service{}, errors.Wrap(err, "connecting to database") 29 | } 30 | 31 | log.Infof("Initializing sql repo with driver %s", driver) 32 | return Service{ 33 | user: userRepo{db, log}, 34 | tag: tagRepo{db, log}, 35 | feed: feedRepo{db, log}, 36 | subscription: subscriptionRepo{db, log}, 37 | article: articleRepo{db, log}, 38 | extract: extractRepo{db, log}, 39 | scores: scoresRepo{db, log}, 40 | thumbnail: thumbnailRepo{db, log}, 41 | }, nil 42 | default: 43 | panic(fmt.Sprintf("Cannot provide a repo for driver '%s'\n", driver)) 44 | } 45 | } 46 | 47 | func (s Service) UserRepo() repo.User { 48 | return s.user 49 | } 50 | 51 | func (s Service) TagRepo() repo.Tag { 52 | return s.tag 53 | } 54 | 55 | func (s Service) FeedRepo() repo.Feed { 56 | return s.feed 57 | } 58 | 59 | func (s Service) SubscriptionRepo() repo.Subscription { 60 | return s.subscription 61 | } 62 | 63 | func (s Service) ArticleRepo() repo.Article { 64 | return s.article 65 | } 66 | 67 | func (s Service) ExtractRepo() repo.Extract { 68 | return s.extract 69 | } 70 | 71 | func (s Service) ScoresRepo() repo.Scores { 72 | return s.scores 73 | } 74 | 75 | func (s Service) ThumbnailRepo() repo.Thumbnail { 76 | return s.thumbnail 77 | } 78 | -------------------------------------------------------------------------------- /content/repo/sql/thumbnail.go: -------------------------------------------------------------------------------- 1 | package sql 2 | 3 | import ( 4 | "database/sql" 5 | 6 | "github.com/jmoiron/sqlx" 7 | "github.com/pkg/errors" 8 | "github.com/urandom/readeef/content" 9 | "github.com/urandom/readeef/content/repo/sql/db" 10 | "github.com/urandom/readeef/log" 11 | ) 12 | 13 | type thumbnailRepo struct { 14 | db *db.DB 15 | 16 | log log.Log 17 | } 18 | 19 | func (r thumbnailRepo) Get(article content.Article) (content.Thumbnail, error) { 20 | if err := article.Validate(); err != nil { 21 | return content.Thumbnail{}, errors.WithMessage(err, "validating article") 22 | } 23 | 24 | r.log.Infof("Getting thumbnail for article %s", article) 25 | 26 | thumbnail := content.Thumbnail{ArticleID: article.ID} 27 | if err := r.db.WithNamedStmt(r.db.SQL().Thumbnail.Get, nil, func(stmt *sqlx.NamedStmt) error { 28 | return stmt.Get(&thumbnail, thumbnail) 29 | }); err != nil { 30 | if err == sql.ErrNoRows { 31 | err = content.ErrNoContent 32 | } 33 | 34 | return content.Thumbnail{}, errors.Wrapf(err, "getting thumbnail for article %s", article) 35 | } 36 | 37 | return thumbnail, nil 38 | } 39 | 40 | func (r thumbnailRepo) Update(thumbnail content.Thumbnail) error { 41 | if err := thumbnail.Validate(); err != nil { 42 | return errors.WithMessage(err, "validating thumbnail") 43 | } 44 | 45 | r.log.Infof("Updating thumbnail %s", thumbnail) 46 | 47 | return r.db.WithTx(func(tx *sqlx.Tx) error { 48 | s := r.db.SQL() 49 | return r.db.WithNamedStmt(s.Thumbnail.Update, tx, func(stmt *sqlx.NamedStmt) error { 50 | res, err := stmt.Exec(thumbnail) 51 | if err != nil { 52 | return errors.Wrap(err, "executing thumbnail update stmt") 53 | } 54 | 55 | if num, err := res.RowsAffected(); err == nil && num > 0 { 56 | return nil 57 | } 58 | 59 | return r.db.WithNamedStmt(s.Thumbnail.Create, tx, func(stmt *sqlx.NamedStmt) error { 60 | if _, err := stmt.Exec(thumbnail); err != nil { 61 | return errors.Wrap(err, "executing thumbnail create stmt") 62 | } 63 | 64 | return nil 65 | }) 66 | }) 67 | }) 68 | } 69 | -------------------------------------------------------------------------------- /content/repo/sqlite3_test.go: -------------------------------------------------------------------------------- 1 | // +build sqlite3 2 | 3 | package repo_test 4 | 5 | import ( 6 | "os" 7 | "testing" 8 | 9 | "github.com/urandom/readeef/content/repo/sql" 10 | "github.com/urandom/readeef/content/repo/sql/db" 11 | _ "github.com/urandom/readeef/content/repo/sql/db/sqlite3" 12 | ) 13 | 14 | func TestMain(m *testing.M) { 15 | db := db.New(logger) 16 | if err := db.Open("sqlite3", "file:/tmp/readeef-test.sqlite3?cache=shared&_foreign_keys=1&_journal=wal"); err != nil { 17 | // if err := db.Open("sqlite3", "file::memory:?cache=shared"); err != nil { 18 | panic(err) 19 | } 20 | 21 | db.Exec("DELETE FROM articles") 22 | db.Exec("DELETE FROM articles_scores") 23 | db.Exec("DELETE FROM feed_images") 24 | db.Exec("DELETE FROM feeds") 25 | db.Exec("DELETE FROM hubbub_subscriptions") 26 | db.Exec("DELETE FROM users") 27 | db.Exec("DELETE FROM users_articles_states") 28 | db.Exec("DELETE FROM users_feeds") 29 | db.Exec("DELETE FROM users_feeds_tags") 30 | 31 | var err error 32 | service, err = sql.NewService("sqlite3", "file:/tmp/readeef-test.sqlite3?cache=shared&_foreign_keys=1&_journal=wal", logger) 33 | if err != nil { 34 | panic(err) 35 | } 36 | 37 | skip = false 38 | ret := m.Run() 39 | 40 | os.Exit(ret) 41 | } 42 | -------------------------------------------------------------------------------- /content/repo/subscription.go: -------------------------------------------------------------------------------- 1 | package repo 2 | 3 | import "github.com/urandom/readeef/content" 4 | 5 | // Subscription allows fetching and manipulating content.Subscription objects 6 | type Subscription interface { 7 | Get(content.Feed) (content.Subscription, error) 8 | All() ([]content.Subscription, error) 9 | 10 | Update(content.Subscription) error 11 | } 12 | -------------------------------------------------------------------------------- /content/repo/tag.go: -------------------------------------------------------------------------------- 1 | package repo 2 | 3 | import "github.com/urandom/readeef/content" 4 | 5 | // Tag allows fetching and manipulating content.Tag objects 6 | type Tag interface { 7 | Get(content.TagID, content.User) (content.Tag, error) 8 | 9 | ForUser(content.User) ([]content.Tag, error) 10 | ForFeed(content.Feed, content.User) ([]content.Tag, error) 11 | 12 | FeedIDs(content.Tag, content.User) ([]content.FeedID, error) 13 | } 14 | -------------------------------------------------------------------------------- /content/repo/thumbnail.go: -------------------------------------------------------------------------------- 1 | package repo 2 | 3 | import "github.com/urandom/readeef/content" 4 | 5 | // Thumbnail allows fetching and manipulating content.Thumbnail objects 6 | type Thumbnail interface { 7 | Get(content.Article) (content.Thumbnail, error) 8 | Update(content.Thumbnail) error 9 | } 10 | -------------------------------------------------------------------------------- /content/repo/user.go: -------------------------------------------------------------------------------- 1 | package repo 2 | 3 | import "github.com/urandom/readeef/content" 4 | 5 | // User allows manipulating content.User objects 6 | type User interface { 7 | Get(content.Login) (content.User, error) 8 | All() ([]content.User, error) 9 | Update(content.User) error 10 | Delete(content.User) error 11 | 12 | FindByMD5([]byte) (content.User, error) 13 | } 14 | -------------------------------------------------------------------------------- /content/scores.go: -------------------------------------------------------------------------------- 1 | package content 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/pkg/errors" 7 | ) 8 | 9 | // Scores contains the calculated popularity score of an article. 10 | type Scores struct { 11 | ArticleID ArticleID `db:"article_id"` 12 | Score int64 13 | Score1 int64 14 | Score2 int64 15 | Score3 int64 16 | Score4 int64 17 | Score5 int64 18 | } 19 | 20 | // Calculate returns the overall score of the article. 21 | func (s Scores) Calculate() int64 { 22 | return s.Score1 + int64(0.1*float64(s.Score2)) + int64(0.01*float64(s.Score3)) + int64(0.001*float64(s.Score4)) + int64(0.0001*float64(s.Score5)) 23 | } 24 | 25 | // Validate validates the score data. 26 | func (s Scores) Validate() error { 27 | if s.ArticleID == 0 { 28 | return NewValidationError(errors.New("Articls sxtract has no articls id")) 29 | } 30 | 31 | return nil 32 | } 33 | 34 | func (s Scores) String() string { 35 | return fmt.Sprintf("%d: %d", s.ArticleID, s.Score) 36 | } 37 | -------------------------------------------------------------------------------- /content/scores_test.go: -------------------------------------------------------------------------------- 1 | package content_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/urandom/readeef/content" 7 | ) 8 | 9 | func TestScores_Validate(t *testing.T) { 10 | tests := []struct { 11 | name string 12 | ArticleID content.ArticleID 13 | wantErr bool 14 | }{ 15 | {"valid", 1, false}, 16 | {"invalid", 0, true}, 17 | } 18 | for _, tt := range tests { 19 | t.Run(tt.name, func(t *testing.T) { 20 | s := content.Scores{ 21 | ArticleID: tt.ArticleID, 22 | } 23 | if err := s.Validate(); (err != nil) != tt.wantErr { 24 | t.Errorf("Scores.Validate() error = %v, wantErr %v", err, tt.wantErr) 25 | } 26 | }) 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /content/search/search.go: -------------------------------------------------------------------------------- 1 | package search 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/pkg/errors" 7 | "github.com/urandom/readeef/content" 8 | "github.com/urandom/readeef/content/repo" 9 | ) 10 | 11 | type indexOperation int 12 | 13 | const ( 14 | BatchAdd indexOperation = iota + 1 15 | BatchDelete 16 | ) 17 | 18 | type Provider interface { 19 | IsNewIndex() bool 20 | Search(string, content.User, ...content.QueryOpt) ([]content.Article, error) 21 | BatchIndex(articles []content.Article, op indexOperation) error 22 | RemoveFeed(content.FeedID) error 23 | } 24 | 25 | func Reindex(p Provider, repo repo.Article) error { 26 | limit := 2000 27 | offset := 0 28 | 29 | for { 30 | articles, err := repo.All(content.Paging(limit, offset)) 31 | if err != nil { 32 | return errors.WithMessage(err, fmt.Sprintf( 33 | "getting articles in window %d-%d", offset, offset+limit, 34 | )) 35 | } 36 | 37 | if err = p.BatchIndex(articles, BatchAdd); err != nil { 38 | return errors.WithMessage(err, "adding batch to index") 39 | } 40 | 41 | if len(articles) < limit { 42 | return nil 43 | } 44 | 45 | offset += limit 46 | 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /content/subscription.go: -------------------------------------------------------------------------------- 1 | package content 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "net/url" 7 | "time" 8 | ) 9 | 10 | type Subscription struct { 11 | FeedID FeedID `db:"feed_id"` 12 | Link string `db:"link"` 13 | LeaseDuration int64 `db:"lease_duration"` 14 | VerificationTime time.Time `db:"verification_time"` 15 | SubscriptionFailure bool `db:"subscription_failure"` 16 | } 17 | 18 | func (s Subscription) Validate() error { 19 | if s.FeedID == 0 { 20 | return NewValidationError(errors.New("Invalid feed id")) 21 | } 22 | 23 | if s.Link == "" { 24 | return NewValidationError(errors.New("No subscription link")) 25 | } 26 | 27 | if u, err := url.Parse(s.Link); err != nil || !u.IsAbs() { 28 | return NewValidationError(errors.New("Invalid subscription link")) 29 | } 30 | 31 | return nil 32 | } 33 | 34 | func (s Subscription) String() string { 35 | return fmt.Sprintf("%s: %d", s.Link, s.FeedID) 36 | } 37 | -------------------------------------------------------------------------------- /content/subscription_test.go: -------------------------------------------------------------------------------- 1 | package content_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/urandom/readeef/content" 7 | ) 8 | 9 | func TestSubscription_Validate(t *testing.T) { 10 | type fields struct { 11 | Link string 12 | FeedID content.FeedID 13 | } 14 | tests := []struct { 15 | name string 16 | fields fields 17 | wantErr bool 18 | }{ 19 | {"valid", fields{FeedID: 1, Link: "http://sugr.org"}, false}, 20 | {"link not absolute", fields{FeedID: 1, Link: "sugr.org"}, true}, 21 | {"no link", fields{FeedID: 1}, true}, 22 | {"no feed id", fields{Link: "http://sugr.org"}, true}, 23 | {"nothing", fields{}, true}, 24 | } 25 | for _, tt := range tests { 26 | t.Run(tt.name, func(t *testing.T) { 27 | s := content.Subscription{ 28 | Link: tt.fields.Link, 29 | FeedID: tt.fields.FeedID, 30 | } 31 | if err := s.Validate(); (err != nil) != tt.wantErr { 32 | t.Errorf("Subscription.Validate() error = %v, wantErr %v", err, tt.wantErr) 33 | } 34 | }) 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /content/tag.go: -------------------------------------------------------------------------------- 1 | package content 2 | 3 | import ( 4 | "database/sql/driver" 5 | "errors" 6 | "fmt" 7 | ) 8 | 9 | type TagID int64 10 | type TagValue string 11 | 12 | type Tag struct { 13 | ID TagID `json:"id"` 14 | Value TagValue `json:"value"` 15 | } 16 | 17 | func (t Tag) Validate() error { 18 | if t.Value == "" { 19 | return NewValidationError(errors.New("Tag has no value")) 20 | } 21 | 22 | return nil 23 | } 24 | 25 | func (t Tag) String() string { 26 | return string(t.Value) 27 | } 28 | 29 | func (id *TagID) Scan(src interface{}) error { 30 | asInt, ok := src.(int64) 31 | if !ok { 32 | return fmt.Errorf("Scan source '%#v' (%T) was not of type int64 (TagId)", src, src) 33 | } 34 | 35 | *id = TagID(asInt) 36 | 37 | return nil 38 | } 39 | 40 | func (id TagID) Value() (driver.Value, error) { 41 | return int64(id), nil 42 | } 43 | 44 | func (val *TagValue) Scan(src interface{}) error { 45 | switch t := src.(type) { 46 | case string: 47 | *val = TagValue(t) 48 | case []byte: 49 | *val = TagValue(t) 50 | default: 51 | return fmt.Errorf("Scan source '%#v' (%T) was not of type string (TagValue)", src, src) 52 | } 53 | 54 | return nil 55 | } 56 | 57 | func (val TagValue) Value() (driver.Value, error) { 58 | return string(val), nil 59 | } 60 | -------------------------------------------------------------------------------- /content/tag_test.go: -------------------------------------------------------------------------------- 1 | package content_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/urandom/readeef/content" 7 | ) 8 | 9 | func TestTag_Validate(t *testing.T) { 10 | tests := []struct { 11 | name string 12 | Value content.TagValue 13 | wantErr bool 14 | }{ 15 | {"valid", "tag", false}, 16 | {"invalid", "", true}, 17 | } 18 | for _, tt := range tests { 19 | t.Run(tt.name, func(t *testing.T) { 20 | tag := content.Tag{ 21 | Value: tt.Value, 22 | } 23 | if err := tag.Validate(); (err != nil) != tt.wantErr { 24 | t.Errorf("Tag.Validate() error = %v, wantErr %v", err, tt.wantErr) 25 | } 26 | }) 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /content/thumbnail.go: -------------------------------------------------------------------------------- 1 | package content 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | ) 7 | 8 | type Thumbnail struct { 9 | ArticleID ArticleID `db:"article_id"` 10 | Thumbnail string 11 | Link string 12 | Processed bool 13 | } 14 | 15 | func (t Thumbnail) Validate() error { 16 | if t.ArticleID == 0 { 17 | return NewValidationError(errors.New("Article thumbnail has no article id")) 18 | } 19 | 20 | return nil 21 | } 22 | 23 | func (t Thumbnail) String() string { 24 | return fmt.Sprintf("%d: %s", t.ArticleID, t.Link) 25 | } 26 | -------------------------------------------------------------------------------- /content/thumbnail/description.go: -------------------------------------------------------------------------------- 1 | package thumbnail 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/pkg/errors" 8 | "github.com/urandom/readeef/content" 9 | "github.com/urandom/readeef/content/repo" 10 | "github.com/urandom/readeef/log" 11 | ) 12 | 13 | type description struct { 14 | repo repo.Thumbnail 15 | store bool 16 | log log.Log 17 | } 18 | 19 | func FromDescription(repo repo.Thumbnail, store bool, log log.Log) Generator { 20 | return description{repo: repo, store: store, log: log} 21 | } 22 | 23 | func (t description) Generate(a content.Article) error { 24 | thumbnail := content.Thumbnail{ArticleID: a.ID, Processed: true} 25 | 26 | t.log.Debugf("Generating thumbnail for article %s from description", a) 27 | 28 | thumbnail.Thumbnail, thumbnail.Link = 29 | generateThumbnailFromDescription(strings.NewReader(a.Description)) 30 | 31 | if !t.store { 32 | thumbnail.Thumbnail = "" 33 | } 34 | 35 | if err := t.repo.Update(thumbnail); err != nil { 36 | return errors.WithMessage(err, fmt.Sprintf("saving thumbnail of %s", a)) 37 | } 38 | 39 | return nil 40 | } 41 | -------------------------------------------------------------------------------- /content/thumbnail_test.go: -------------------------------------------------------------------------------- 1 | package content_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/urandom/readeef/content" 7 | ) 8 | 9 | func TestThumbnail_Validate(t *testing.T) { 10 | tests := []struct { 11 | name string 12 | ArticleID content.ArticleID 13 | wantErr bool 14 | }{ 15 | {"valid", 1, false}, 16 | {"invalid", 0, true}, 17 | } 18 | for _, tt := range tests { 19 | t.Run(tt.name, func(t *testing.T) { 20 | thumb := content.Thumbnail{ 21 | ArticleID: tt.ArticleID, 22 | } 23 | if err := thumb.Validate(); (err != nil) != tt.wantErr { 24 | t.Errorf("Thumbnail.Validate() error = %v, wantErr %v", err, tt.wantErr) 25 | } 26 | }) 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /content/validation.go: -------------------------------------------------------------------------------- 1 | package content 2 | 3 | import "github.com/pkg/errors" 4 | 5 | type ValidationError struct { 6 | error 7 | } 8 | 9 | func NewValidationError(err error) error { 10 | return errors.WithStack(ValidationError{err}) 11 | } 12 | 13 | func IsValidationError(err error) bool { 14 | _, ok := errors.Cause(err).(ValidationError) 15 | return ok 16 | } 17 | -------------------------------------------------------------------------------- /feed/favicon.go: -------------------------------------------------------------------------------- 1 | package feed 2 | 3 | import ( 4 | "io/ioutil" 5 | "net/http" 6 | "net/url" 7 | 8 | "github.com/PuerkitoBio/goquery" 9 | "github.com/pkg/errors" 10 | ) 11 | 12 | func Favicon(site string) ([]byte, string, error) { 13 | doc, err := goquery.NewDocument(site) 14 | if err != nil { 15 | return nil, "", errors.Wrapf(err, "querying site: %q", site) 16 | } 17 | 18 | icons := doc.Find(`link[rel="shortcut icon"], link[rel="icon"], link[rel="icon shortcut"]`). 19 | Filter(`[href]`). 20 | Map(func(_ int, s *goquery.Selection) string { 21 | return s.AttrOr("href", "") 22 | }) 23 | 24 | var siteURL *url.URL 25 | for _, icon := range icons { 26 | if icon == "" { 27 | continue 28 | } 29 | 30 | iconURL, err := url.Parse(icon) 31 | if err != nil { 32 | continue 33 | } 34 | 35 | if !iconURL.IsAbs() { 36 | if siteURL == nil { 37 | siteURL, err = url.Parse(site) 38 | if err != nil { 39 | continue 40 | } 41 | } 42 | 43 | iconURL.Scheme = siteURL.Scheme 44 | if iconURL.Host == "" { 45 | iconURL.Host = siteURL.Host 46 | } 47 | } 48 | 49 | resp, err := http.Get(iconURL.String()) 50 | if err != nil { 51 | return nil, "", errors.Wrapf(err, "getting favicon %q", iconURL) 52 | } 53 | defer resp.Body.Close() 54 | 55 | b, err := ioutil.ReadAll(resp.Body) 56 | 57 | return b, resp.Header.Get("content-type"), err 58 | } 59 | 60 | return nil, "", nil 61 | } 62 | -------------------------------------------------------------------------------- /feed/favicon_test.go: -------------------------------------------------------------------------------- 1 | package feed 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestFavicon(t *testing.T) { 8 | tests := []struct { 9 | name string 10 | site string 11 | want bool 12 | wantCT string 13 | wantErr bool 14 | }{ 15 | {name: "no url", wantErr: true}, 16 | {name: "broken url", site: "https://example-nonexistent.com", wantErr: true}, 17 | {name: "wikipedia", site: "https://en.wikipedia.org", want: true, wantCT: "image/vnd.microsoft.icon"}, 18 | } 19 | for _, tt := range tests { 20 | t.Run(tt.name, func(t *testing.T) { 21 | got, ct, err := Favicon(tt.site) 22 | if (err != nil) != tt.wantErr { 23 | t.Errorf("Favicon() error = %v, wantErr %v", err, tt.wantErr) 24 | return 25 | } 26 | 27 | if (len(got) > 0) != tt.want { 28 | t.Errorf("Favicon() = %v, want %v", got, tt.want) 29 | } 30 | 31 | if ct != tt.wantCT { 32 | t.Errorf("Favicon() content-type = %q, want %q", ct, tt.wantCT) 33 | } 34 | }) 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /file.list: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/urandom/readeef/95c851e5ee6c3a12b0a8888985802698a56bf47b/file.list -------------------------------------------------------------------------------- /fs.go: -------------------------------------------------------------------------------- 1 | // +build !nofs 2 | 3 | package readeef 4 | 5 | //go:generate embed -output fs_files.go -package-name readeef -build-tags !nofs -fallback rf-ng/ui/... templates/raw.tmpl templates/goose-format-result.tmpl 6 | -------------------------------------------------------------------------------- /fs_files_nofs.go: -------------------------------------------------------------------------------- 1 | // +build nofs 2 | 3 | package readeef 4 | 5 | import "net/http" 6 | 7 | // NewFileSystem creates a new http.Dir filesystem 8 | func NewFileSystem() (http.FileSystem, error) { 9 | return http.Dir("."), nil 10 | } 11 | -------------------------------------------------------------------------------- /log/log.go: -------------------------------------------------------------------------------- 1 | package log 2 | 3 | // Log provides some common methods for outputting messages. 4 | // It may be used to exchange the default log.Logger error logger with another 5 | // provider. 6 | type Log interface { 7 | Print(v ...interface{}) 8 | Printf(format string, v ...interface{}) 9 | Println(v ...interface{}) 10 | 11 | Info(v ...interface{}) 12 | Infof(format string, v ...interface{}) 13 | Infoln(v ...interface{}) 14 | 15 | Debug(v ...interface{}) 16 | Debugf(format string, v ...interface{}) 17 | Debugln(v ...interface{}) 18 | } 19 | -------------------------------------------------------------------------------- /log/logrus.go: -------------------------------------------------------------------------------- 1 | package log 2 | 3 | import ( 4 | lg "github.com/Sirupsen/logrus" 5 | "github.com/urandom/readeef/config" 6 | ) 7 | 8 | type logrus struct { 9 | *lg.Logger 10 | } 11 | 12 | func WithLogrus(cfg config.Log) Log { 13 | logger := logrus{Logger: lg.New()} 14 | 15 | logger.Out = cfg.Converted.Writer 16 | 17 | switch cfg.Formatter { 18 | case "text", "": 19 | logger.Formatter = &lg.TextFormatter{DisableTimestamp: true} 20 | case "json": 21 | logger.Formatter = &lg.JSONFormatter{} 22 | } 23 | 24 | switch cfg.Level { 25 | case "info": 26 | logger.Level = lg.InfoLevel 27 | case "debug": 28 | logger.Level = lg.DebugLevel 29 | case "error", "": 30 | logger.Level = lg.ErrorLevel 31 | } 32 | 33 | return logger 34 | } 35 | 36 | func (l logrus) Print(args ...interface{}) { 37 | l.Logger.Error(args...) 38 | } 39 | 40 | func (l logrus) Printf(format string, args ...interface{}) { 41 | l.Logger.Errorf(format, args...) 42 | } 43 | 44 | func (l logrus) Errorln(args ...interface{}) { 45 | l.Logger.Errorln(args...) 46 | } 47 | -------------------------------------------------------------------------------- /log/std.go: -------------------------------------------------------------------------------- 1 | package log 2 | 3 | import ( 4 | "log" 5 | 6 | "github.com/urandom/readeef/config" 7 | ) 8 | 9 | type stdLogger struct { 10 | *log.Logger 11 | } 12 | 13 | // WithStd creates a logger that uses the stdlib log facilities. 14 | func WithStd(cfg config.Log) Log { 15 | return stdLogger{Logger: log.New(cfg.Converted.Writer, cfg.Converted.Prefix, 0)} 16 | } 17 | 18 | func (st stdLogger) Info(v ...interface{}) { 19 | st.Print(v...) 20 | } 21 | 22 | func (st stdLogger) Infof(format string, v ...interface{}) { 23 | st.Printf(format, v...) 24 | } 25 | 26 | func (st stdLogger) Infoln(v ...interface{}) { 27 | st.Println(v...) 28 | } 29 | 30 | func (st stdLogger) Debug(v ...interface{}) { 31 | st.Print(v...) 32 | } 33 | 34 | func (st stdLogger) Debugf(format string, v ...interface{}) { 35 | st.Printf(format, v...) 36 | } 37 | 38 | func (st stdLogger) Debugln(v ...interface{}) { 39 | st.Println(v...) 40 | } 41 | -------------------------------------------------------------------------------- /parser/atom.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "bytes" 5 | "encoding/xml" 6 | "io" 7 | "time" 8 | ) 9 | 10 | type atomFeed struct { 11 | XMLName xml.Name `xml:"feed"` 12 | Title string `xml:"title"` 13 | Description string `xml:"description"` 14 | Link atomLink `xml:"link"` 15 | Image rssImage `xml:"image"` 16 | Items []atomItem `xml:"entry"` 17 | } 18 | 19 | type atomItem struct { 20 | XMLName xml.Name `xml:"entry"` 21 | Id string `xml:"id"` 22 | Title string `xml:"title"` 23 | Description rssContent `xml:"summary"` 24 | Content rssContent `xml:"content"` 25 | Link atomLink `xml:"link"` 26 | Date string `xml:"updated"` 27 | PubDate string `xml:"published"` 28 | } 29 | 30 | type atomLink struct { 31 | Rel string `xml:"rel,attr,omitempty"` 32 | Href string `xml:"href,attr"` 33 | } 34 | 35 | func ParseAtom(b []byte) (Feed, error) { 36 | var f Feed 37 | var rss atomFeed 38 | 39 | decoder := xml.NewDecoder(bytes.NewReader(b)) 40 | decoder.DefaultSpace = "parserfeed" 41 | 42 | if err := decoder.Decode(&rss); err != nil { 43 | return f, err 44 | } 45 | 46 | f = Feed{ 47 | Title: rss.Title, 48 | Description: rss.Description, 49 | SiteLink: rss.Link.Href, 50 | Image: Image{ 51 | rss.Image.Title, rss.Image.Url, 52 | rss.Image.Width, rss.Image.Height}, 53 | } 54 | 55 | var lastValidDate time.Time 56 | for _, i := range rss.Items { 57 | article := Article{Title: i.Title, Link: i.Link.Href, Guid: i.Id} 58 | article.Description = getLargerContent(i.Content, i.Description) 59 | 60 | var err error 61 | if i.PubDate != "" { 62 | article.Date, err = parseDate(i.PubDate) 63 | } else if i.Date != "" { 64 | article.Date, err = parseDate(i.Date) 65 | } else { 66 | err = io.EOF 67 | } 68 | 69 | if err == nil { 70 | lastValidDate = article.Date.Add(time.Second) 71 | } else if lastValidDate.IsZero() { 72 | article.Date = unknownTime 73 | } else { 74 | article.Date = lastValidDate 75 | } 76 | 77 | f.Articles = append(f.Articles, article) 78 | } 79 | f.HubLink = getHubLink(b) 80 | 81 | return f, nil 82 | } 83 | -------------------------------------------------------------------------------- /parser/feed.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import "time" 4 | 5 | type Feed struct { 6 | Title string 7 | Description string 8 | SiteLink string 9 | HubLink string 10 | Image Image 11 | Articles []Article 12 | TTL time.Duration 13 | SkipHours map[int]bool 14 | SkipDays map[string]bool 15 | } 16 | 17 | type Article struct { 18 | Title string 19 | Description string 20 | Link string 21 | Guid string 22 | Date time.Time 23 | } 24 | 25 | type Image struct { 26 | Title string 27 | Url string 28 | Width int 29 | Height int 30 | } 31 | -------------------------------------------------------------------------------- /parser/parser.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "encoding/xml" 5 | "net/http" 6 | "strings" 7 | "time" 8 | ) 9 | 10 | const ( 11 | RFC1123NoSecond = "Mon, 02 Jan 2006 15:04 MST" 12 | HackerNewsTimeFormat = "Mon, _2 Jan 2006 15:04:05 -0700" 13 | ) 14 | 15 | var ( 16 | unknownTime = time.Unix(0, 0) 17 | ) 18 | 19 | func ParseFeed(source []byte, funcs ...func([]byte) (Feed, error)) (Feed, error) { 20 | var feed Feed 21 | var err error 22 | 23 | for _, f := range funcs { 24 | feed, err = f(source) 25 | if err != nil { 26 | if _, ok := err.(xml.UnmarshalError); !ok { 27 | return feed, err 28 | } 29 | } else { 30 | break 31 | } 32 | } 33 | 34 | return feed, err 35 | } 36 | 37 | func parseDate(date string) (time.Time, error) { 38 | formats := []string{ 39 | time.ANSIC, 40 | time.UnixDate, 41 | time.RubyDate, 42 | time.RFC822, 43 | time.RFC822Z, 44 | time.RFC850, 45 | time.RFC1123, 46 | time.RFC1123Z, 47 | RFC1123NoSecond, 48 | time.RFC3339, 49 | time.RFC3339Nano, 50 | http.TimeFormat, 51 | HackerNewsTimeFormat, 52 | } 53 | 54 | date = strings.TrimSpace(date) 55 | var err error 56 | var t time.Time 57 | for _, f := range formats { 58 | t, err = time.Parse(f, date) 59 | if err == nil { 60 | return t, nil 61 | } 62 | } 63 | 64 | return t, err 65 | } 66 | -------------------------------------------------------------------------------- /parser/parser_test.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import "testing" 4 | 5 | func TestParseFeed(t *testing.T) { 6 | _, err := ParseFeed([]byte(singleAtomXML), ParseRss2, ParseAtom, ParseRss1) 7 | 8 | if err != nil { 9 | t.Fatal(err) 10 | } 11 | 12 | _, err = ParseFeed([]byte(singleRss1XML), ParseRss2, ParseAtom, ParseRss1) 13 | 14 | if err != nil { 15 | t.Fatal(err) 16 | } 17 | 18 | _, err = ParseFeed([]byte(singleRss2XML), ParseRss2, ParseAtom, ParseRss1) 19 | 20 | if err != nil { 21 | t.Fatal(err) 22 | } 23 | 24 | _, err = ParseFeed([]byte(singleRss1XML), ParseRss2, ParseAtom) 25 | 26 | if err == nil { 27 | t.Fatalf("Expected an error\n") 28 | } 29 | 30 | _, err = ParseFeed([]byte(singleRss2XML), ParseRss1, ParseAtom) 31 | 32 | if err == nil { 33 | t.Fatalf("Expected an error\n") 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /parser/pubsub.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import "encoding/xml" 4 | 5 | type pubsubFeed struct { 6 | Link []pubsubLink `xml:"link"` 7 | Channel pubsubChannel `xml:"channel"` 8 | } 9 | 10 | type pubsubChannel struct { 11 | Link []pubsubLink `xml:"http://www.w3.org/2005/Atom link"` 12 | } 13 | 14 | type pubsubLink struct { 15 | Rel string `xml:"rel,attr,omitempty"` 16 | Href string `xml:"href,attr"` 17 | } 18 | 19 | func getHubLink(b []byte) string { 20 | var f pubsubFeed 21 | 22 | if err := xml.Unmarshal(b, &f); err == nil { 23 | links := append(f.Link, f.Channel.Link...) 24 | for _, link := range links { 25 | if link.Rel == "hub" { 26 | return link.Href 27 | } 28 | } 29 | } 30 | 31 | return "" 32 | } 33 | -------------------------------------------------------------------------------- /parser/rss.go: -------------------------------------------------------------------------------- 1 | package parser 2 | 3 | import ( 4 | "encoding/xml" 5 | "strings" 6 | ) 7 | 8 | type rssImage struct { 9 | XMLName xml.Name `xml:"image"` 10 | Title string `xml:"title"` 11 | Url string `xml:"url"` 12 | Width int `xml:"width"` 13 | Height int `xml:"height"` 14 | } 15 | 16 | // RssItem is the base content for both rss1 and rss2 feeds. The only reason 17 | // it's public is because of the refrect package 18 | type RssItem struct { 19 | XMLName xml.Name `xml:"item"` 20 | Id string `xml:"guid"` 21 | Title string `xml:"title"` 22 | Link string `xml:"link"` 23 | Description rssContent `xml:"description"` 24 | Content rssContent `xml:"http://purl.org/rss/1.0/modules/content/ encoded"` 25 | PubDate string `xml:"pubDate"` 26 | Date string `xml:"date"` 27 | TTL int `xml:"ttl"` 28 | SkipHours []int `xml:"skipHours>hour"` 29 | SkipDays []string `xml:"skipDays>day"` 30 | } 31 | 32 | type rssContent struct { 33 | XMLName xml.Name 34 | InnerXML string `xml:",innerxml"` 35 | Chardata string `xml:",chardata"` 36 | } 37 | 38 | func (c rssContent) Content() string { 39 | if strings.TrimSpace(c.Chardata) == "" { 40 | return c.InnerXML 41 | } else { 42 | return c.Chardata 43 | } 44 | } 45 | 46 | func getLargerContent(first, second rssContent) string { 47 | c1, c2 := first.Content(), second.Content() 48 | 49 | if len(c1) < len(c2) { 50 | return c2 51 | } else { 52 | return c1 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /pool/buffer.go: -------------------------------------------------------------------------------- 1 | package pool 2 | 3 | import ( 4 | "bytes" 5 | "sync" 6 | ) 7 | 8 | type bufp struct { 9 | sync.Pool 10 | } 11 | 12 | // Buffer is a utility variable that provides bytes.Buffer objects. 13 | var Buffer bufp 14 | 15 | // Get returns a bytes.Buffer pointer from the pool. 16 | func (b *bufp) Get() *bytes.Buffer { 17 | buffer := b.Pool.Get().(*bytes.Buffer) 18 | buffer.Reset() 19 | 20 | return buffer 21 | } 22 | 23 | func init() { 24 | Buffer = bufp{ 25 | Pool: sync.Pool{ 26 | New: func() interface{} { 27 | return new(bytes.Buffer) 28 | }, 29 | }, 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /popularity/link.go: -------------------------------------------------------------------------------- 1 | package popularity 2 | 3 | import ( 4 | "net/url" 5 | "regexp" 6 | "strings" 7 | ) 8 | 9 | var ( 10 | dqLinkMatcher = regexp.MustCompile(`.*?`) 11 | sqLinkMatcher = regexp.MustCompile(`.*?`) 12 | ) 13 | 14 | func realLink(link, text string) string { 15 | u, err := url.Parse(link) 16 | if err != nil { 17 | return "" 18 | } 19 | u.RawQuery = "" 20 | link = u.String() 21 | 22 | if link, ok := processReddit(link, text); ok { 23 | return link 24 | } 25 | 26 | return link 27 | } 28 | 29 | func processReddit(link, text string) (string, bool) { 30 | if strings.Contains(link, "://reddit.com/r") || strings.Contains(link, "://www.reddit.com/r") { 31 | res := dqLinkMatcher.FindAllStringSubmatch(text, -1) 32 | for _, l := range res { 33 | if checkRedditLink(l[1], link) { 34 | return l[1], true 35 | } 36 | } 37 | 38 | res = sqLinkMatcher.FindAllStringSubmatch(text, -1) 39 | for _, l := range res { 40 | if checkRedditLink(l[1], link) { 41 | return l[1], true 42 | } 43 | } 44 | } 45 | 46 | return "", false 47 | } 48 | 49 | func checkRedditLink(link, original string) bool { 50 | if strings.Contains(link, "reddit.com/user") { 51 | return false 52 | } 53 | 54 | if strings.Contains(link, "reddit.com/") && strings.Contains(link, "/comments/") { 55 | return false 56 | } 57 | 58 | if link != original { 59 | return true 60 | } 61 | 62 | return false 63 | } 64 | -------------------------------------------------------------------------------- /popularity/reddit.go: -------------------------------------------------------------------------------- 1 | package popularity 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/pkg/errors" 7 | "github.com/turnage/graw/reddit" 8 | "github.com/urandom/readeef/config" 9 | "github.com/urandom/readeef/log" 10 | ) 11 | 12 | type Reddit struct { 13 | bot reddit.Bot 14 | log log.Log 15 | } 16 | 17 | func FromReddit(config config.Popularity, log log.Log) (Reddit, error) { 18 | if config.Reddit.ID == "" || config.Reddit.Secret == "" { 19 | return Reddit{}, errors.New("invalid credentials") 20 | } 21 | 22 | bot, err := reddit.NewBot(reddit.BotConfig{ 23 | Agent: fmt.Sprintf("readeef:score_bot by /u/%s", config.Reddit.Username), 24 | App: reddit.App{ 25 | ID: config.Reddit.ID, 26 | Secret: config.Reddit.Secret, 27 | Username: config.Reddit.Username, 28 | Password: config.Reddit.Password, 29 | }, 30 | }) 31 | if err != nil { 32 | return Reddit{}, errors.Wrap(err, "initializing reddit api") 33 | } 34 | return Reddit{bot: bot, log: log}, nil 35 | } 36 | 37 | func (r Reddit) Score(link string) (int64, error) { 38 | var score int64 = -1 39 | 40 | harvest, err := r.bot.ListingWithParams("/api/info", map[string]string{ 41 | "url": link, 42 | }) 43 | 44 | if err != nil { 45 | return score, errors.Wrapf(err, "getting reddit info for link %s", link) 46 | } 47 | 48 | score = 0 49 | for _, post := range harvest.Posts { 50 | score += int64(post.Score + post.NumComments) 51 | } 52 | 53 | r.log.Debugf("Popularity: Reddit score and comments for url %s: %d", link, score) 54 | 55 | return score, nil 56 | } 57 | 58 | func (re Reddit) String() string { 59 | return "Reddit score provider" 60 | } 61 | -------------------------------------------------------------------------------- /popularity/twitter.go: -------------------------------------------------------------------------------- 1 | package popularity 2 | 3 | import ( 4 | "fmt" 5 | "net/url" 6 | 7 | "github.com/ChimeraCoder/anaconda" 8 | "github.com/urandom/readeef/config" 9 | "github.com/urandom/readeef/log" 10 | ) 11 | 12 | type Twitter struct { 13 | api *anaconda.TwitterApi 14 | log log.Log 15 | } 16 | 17 | func FromTwitter(config config.Popularity, log log.Log) Twitter { 18 | anaconda.SetConsumerKey(config.Twitter.ConsumerKey) 19 | anaconda.SetConsumerSecret(config.Twitter.ConsumerSecret) 20 | 21 | return Twitter{api: anaconda.NewTwitterApi(config.Twitter.AccessToken, config.Twitter.AccessTokenSecret), log: log} 22 | } 23 | 24 | func (t Twitter) Score(link string) (int64, error) { 25 | link = url.QueryEscape(link) 26 | 27 | var score int64 28 | 29 | values := make(url.Values) 30 | values.Set("count", "100") 31 | for { 32 | searchResults, err := t.api.GetSearch(link, values) 33 | if err != nil { 34 | return 0, fmt.Errorf("twitter scoring: %v", err) 35 | } 36 | 37 | score += int64(len(searchResults.Statuses)) 38 | 39 | if searchResults.Metadata.NextResults == "" { 40 | break 41 | } 42 | 43 | v, err := url.ParseQuery(searchResults.Metadata.NextResults[1:]) 44 | if err != nil { 45 | panic(err) 46 | } 47 | 48 | values.Set("max_id", v.Get("max_id")) 49 | } 50 | 51 | t.log.Debugf("Popularity: Tweets for url %s: %d", link, score) 52 | 53 | return score, nil 54 | } 55 | 56 | func (t Twitter) String() string { 57 | return "Twitter score provider" 58 | } 59 | -------------------------------------------------------------------------------- /rf-ng/.dockerignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | node_modules/ 3 | node_modules/* 4 | node_modules/** 5 | Makefile 6 | Dockerfile -------------------------------------------------------------------------------- /rf-ng/.editorconfig: -------------------------------------------------------------------------------- 1 | # Editor configuration, see http://editorconfig.org 2 | root = true 3 | 4 | [*] 5 | charset = utf-8 6 | indent_style = space 7 | indent_size = 2 8 | insert_final_newline = true 9 | trim_trailing_whitespace = true 10 | 11 | [*.md] 12 | max_line_length = off 13 | trim_trailing_whitespace = false 14 | -------------------------------------------------------------------------------- /rf-ng/.gitignore: -------------------------------------------------------------------------------- 1 | # See http://help.github.com/ignore-files/ for more about ignoring files. 2 | 3 | # compiled output 4 | /dist 5 | /tmp 6 | /out-tsc 7 | /ui/en 8 | /ui/bg 9 | 10 | # dependencies 11 | /node_modules 12 | 13 | # IDEs and editors 14 | /.idea 15 | .project 16 | .classpath 17 | .c9/ 18 | *.launch 19 | .settings/ 20 | *.sublime-workspace 21 | 22 | # IDE - VSCode 23 | .vscode/* 24 | !.vscode/settings.json 25 | !.vscode/tasks.json 26 | !.vscode/launch.json 27 | !.vscode/extensions.json 28 | 29 | # misc 30 | /.sass-cache 31 | /connect.lock 32 | /coverage 33 | /libpeerconnection.log 34 | npm-debug.log 35 | testem.log 36 | /typings 37 | yarn-error.log 38 | 39 | # e2e 40 | /e2e/*.js 41 | /e2e/*.map 42 | 43 | # System Files 44 | .DS_Store 45 | Thumbs.db 46 | -------------------------------------------------------------------------------- /rf-ng/Dockerfile: -------------------------------------------------------------------------------- 1 | # Use a Node.js image and assign it as our build 2 | FROM mhart/alpine-node:12 as build 3 | 4 | # Install system dependencies, mostly for libpng 5 | RUN apk --no-cache update \ 6 | && apk --no-cache add g++ make bash zlib-dev libpng-dev nano bash \ 7 | && rm -fr /var/cache/apk/* 8 | 9 | COPY . /opt/rf-ng 10 | 11 | WORKDIR /opt/rf-ng 12 | RUN npm install && \ 13 | npm run-script build 14 | 15 | VOLUME /opt/rf-ng/build 16 | 17 | CMD ["npm", "start"] -------------------------------------------------------------------------------- /rf-ng/Makefile: -------------------------------------------------------------------------------- 1 | docker-build: 2 | @docker build -t urandum/rf-ng:alpine-node12 . 3 | 4 | docker-run: 5 | @docker run -ti -p 4200:4200 urandum/rf-ng:alpine-node12 6 | -------------------------------------------------------------------------------- /rf-ng/README.md: -------------------------------------------------------------------------------- 1 | # RfNg 2 | 3 | This project was generated with [Angular CLI](https://github.com/angular/angular-cli) version 1.3.0. 4 | 5 | ## Development server 6 | 7 | Run `ng serve` for a dev server. Navigate to `http://localhost:4200/`. The app will automatically reload if you change any of the source files. 8 | 9 | ## Code scaffolding 10 | 11 | Run `ng generate component component-name` to generate a new component. You can also use `ng generate directive|pipe|service|class|guard|interface|enum|module`. 12 | 13 | ## Build 14 | 15 | Run `ng build` to build the project. The build artifacts will be stored in the `dist/` directory. Use the `-prod` flag for a production build. 16 | 17 | ## Running unit tests 18 | 19 | Run `ng test` to execute the unit tests via [Karma](https://karma-runner.github.io). 20 | 21 | ## Running end-to-end tests 22 | 23 | Run `ng e2e` to execute the end-to-end tests via [Protractor](http://www.protractortest.org/). 24 | Before running the tests make sure you are serving the app via `ng serve`. 25 | 26 | ## Further help 27 | 28 | To get more help on the Angular CLI use `ng help` or go check out the [Angular CLI README](https://github.com/angular/angular-cli/blob/master/README.md). 29 | -------------------------------------------------------------------------------- /rf-ng/e2e/app.e2e-spec.ts: -------------------------------------------------------------------------------- 1 | import { AppPage } from './app.po'; 2 | 3 | describe('rf-ng App', () => { 4 | let page: AppPage; 5 | 6 | beforeEach(() => { 7 | page = new AppPage(); 8 | }); 9 | 10 | it('should display welcome message', () => { 11 | page.navigateTo(); 12 | expect(page.getParagraphText()).toEqual('Welcome to app!'); 13 | }); 14 | }); 15 | -------------------------------------------------------------------------------- /rf-ng/e2e/app.po.ts: -------------------------------------------------------------------------------- 1 | import { browser, by, element } from 'protractor'; 2 | 3 | export class AppPage { 4 | navigateTo() { 5 | return browser.get('/'); 6 | } 7 | 8 | getParagraphText() { 9 | return element(by.css('app-root h1')).getText(); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /rf-ng/e2e/tsconfig.e2e.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../tsconfig.json", 3 | "compilerOptions": { 4 | "outDir": "../out-tsc/e2e", 5 | "baseUrl": "./", 6 | "module": "commonjs", 7 | "target": "es5", 8 | "types": [ 9 | "jasmine", 10 | "jasminewd2", 11 | "node" 12 | ] 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /rf-ng/karma.conf.js: -------------------------------------------------------------------------------- 1 | // Karma configuration file, see link for more information 2 | // https://karma-runner.github.io/1.0/config/configuration-file.html 3 | 4 | module.exports = function (config) { 5 | config.set({ 6 | basePath: '', 7 | frameworks: ['jasmine', '@angular-devkit/build-angular'], 8 | plugins: [ 9 | require('karma-jasmine'), 10 | require('karma-chrome-launcher'), 11 | require('karma-jasmine-html-reporter'), 12 | require('karma-coverage-istanbul-reporter'), 13 | require('@angular-devkit/build-angular/plugins/karma') 14 | ], 15 | client: { 16 | clearContext: false // leave Jasmine Spec Runner output visible in browser 17 | }, 18 | coverageIstanbulReporter: { 19 | dir: require('path').join(__dirname, './coverage/rf-ng'), 20 | reports: ['html', 'lcovonly', 'text-summary'], 21 | fixWebpackSourcePaths: true 22 | }, 23 | reporters: ['progress', 'kjhtml'], 24 | port: 9876, 25 | colors: true, 26 | logLevel: config.LOG_INFO, 27 | autoWatch: true, 28 | browsers: ['Chrome'], 29 | singleRun: false, 30 | restartOnFileChange: true 31 | }); 32 | }; 33 | -------------------------------------------------------------------------------- /rf-ng/ngsw-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "./node_modules/@angular/service-worker/config/schema.json", 3 | "index": "/index.html", 4 | "assetGroups": [ 5 | { 6 | "name": "app", 7 | "installMode": "prefetch", 8 | "resources": { 9 | "files": [ 10 | "/favicon.ico", 11 | "/index.html", 12 | "/manifest.webmanifest", 13 | "/*.css", 14 | "/*.js" 15 | ], 16 | "urls": [ 17 | "https://fonts.googleapis.com/css?family=Material+Icons", 18 | "https://fonts.gstatic.com/s/materialicons/**" 19 | ] 20 | } 21 | }, 22 | { 23 | "name": "assets", 24 | "installMode": "lazy", 25 | "updateMode": "prefetch", 26 | "resources": { 27 | "files": [ 28 | "/assets/**", 29 | "/*.(eot|svg|cur|jpg|png|webp|gif|otf|ttf|woff|woff2|ani)" 30 | ] 31 | } 32 | } 33 | ] 34 | } 35 | -------------------------------------------------------------------------------- /rf-ng/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "rf-ng", 3 | "version": "0.0.0", 4 | "scripts": { 5 | "ng": "ng", 6 | "start": "ng serve", 7 | "build": "ng build", 8 | "test": "ng test", 9 | "lint": "ng lint", 10 | "e2e": "ng e2e" 11 | }, 12 | "private": true, 13 | "dependencies": { 14 | "@angular/animations": "~11.0.5", 15 | "@angular/cdk": "~11.0.3", 16 | "@angular/common": "~11.0.5", 17 | "@angular/compiler": "~11.0.5", 18 | "@angular/core": "~11.0.5", 19 | "@angular/forms": "~11.0.5", 20 | "@angular/localize": "^11.0.5", 21 | "@angular/material": "^11.0.3", 22 | "@angular/platform-browser": "~11.0.5", 23 | "@angular/platform-browser-dynamic": "~11.0.5", 24 | "@angular/router": "~11.0.5", 25 | "@angular/service-worker": "~11.0.5", 26 | "@ng-bootstrap/ng-bootstrap": "^8.0.0", 27 | "bootstrap": "^4.4.1", 28 | "hammerjs": "^2.0.8", 29 | "jwt-decode": "^2.2.0", 30 | "moment": "^2.29.4", 31 | "ngx-virtual-scroller": "^3.0.3", 32 | "rxjs": "~6.6.3", 33 | "tslib": "^2.0.0", 34 | "zone.js": "~0.10.3" 35 | }, 36 | "devDependencies": { 37 | "@angular-devkit/build-angular": "~15.2.4", 38 | "@angular/cli": "~11.2.19", 39 | "@angular/compiler-cli": "~11.0.5", 40 | "@angular/language-service": "~11.0.5", 41 | "@types/node": "^12.11.1", 42 | "@types/jasmine": "~3.6.0", 43 | "@types/jasminewd2": "~2.0.3", 44 | "jasmine-core": "~3.6.0", 45 | "jasmine-spec-reporter": "~5.0.0", 46 | "karma": "~6.3.16", 47 | "karma-chrome-launcher": "~3.1.0", 48 | "karma-coverage-istanbul-reporter": "~3.0.2", 49 | "karma-jasmine": "~4.0.0", 50 | "karma-jasmine-html-reporter": "^1.5.0", 51 | "protractor": "~7.0.0", 52 | "ts-node": "~7.0.0", 53 | "tslint": "~6.1.0", 54 | "typescript": "~4.0.5" 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /rf-ng/protractor.conf.js: -------------------------------------------------------------------------------- 1 | // Protractor configuration file, see link for more information 2 | // https://github.com/angular/protractor/blob/master/lib/config.ts 3 | 4 | const { SpecReporter } = require('jasmine-spec-reporter'); 5 | 6 | exports.config = { 7 | allScriptsTimeout: 11000, 8 | specs: [ 9 | './e2e/**/*.e2e-spec.ts' 10 | ], 11 | capabilities: { 12 | 'browserName': 'chrome' 13 | }, 14 | directConnect: true, 15 | baseUrl: 'http://localhost:4200/', 16 | framework: 'jasmine', 17 | jasmineNodeOpts: { 18 | showColors: true, 19 | defaultTimeoutInterval: 30000, 20 | print: function() {} 21 | }, 22 | onPrepare() { 23 | require('ts-node').register({ 24 | project: 'e2e/tsconfig.e2e.json' 25 | }); 26 | jasmine.getEnv().addReporter(new SpecReporter({ spec: { displayStacktrace: true } })); 27 | } 28 | }; 29 | -------------------------------------------------------------------------------- /rf-ng/src/app/article-display/article-display.css: -------------------------------------------------------------------------------- 1 | :host { 2 | flex-grow: 1; 3 | display: flex; 4 | height: calc(100% - 64px); 5 | } 6 | 7 | ngb-carousel { 8 | user-select: text !important; 9 | overflow-y: auto; 10 | flex-grow: 1; 11 | } 12 | 13 | ::ng-deep .carousel-item { 14 | transition: -webkit-transform 0.2s ease-in-out; 15 | transition: transform 0.2s ease-in-out; 16 | transition: transform 0.2s ease-in-out, -webkit-transform 0.6s ease-in-out; 17 | } 18 | 19 | ::ng-deep span.carousel-control-next-icon { 20 | background-image: url("data:image/svg+xml;charset=utf8,%3Csvg xmlns='http://www.w3.org/2000/svg' fill='%23999' viewBox='0 0 8 8'%3E%3Cpath d='M1.5 0l-1.5 1.5 2.5 2.5-2.5 2.5 1.5 1.5 4-4-4-4z'/%3E%3C/svg%3E") !important; 21 | } 22 | 23 | ::ng-deep span.carousel-control-prev-icon { 24 | background-image: url("data:image/svg+xml;charset=utf8,%3Csvg xmlns='http://www.w3.org/2000/svg' fill='%23999' viewBox='0 0 8 8'%3E%3Cpath d='M4 0l-4 4 4 4 1.5-1.5-2.5-2.5 2.5-2.5-1.5-1.5z'/%3E%3C/svg%3E") !important; 25 | } 26 | 27 | ::ng-deep .carousel-control-prev, 28 | ::ng-deep .carousel-control-next { 29 | max-width: 100px; 30 | } 31 | 32 | ::ng-deep .carousel-indicators { 33 | display: none; 34 | } 35 | 36 | a { 37 | color: black; 38 | } 39 | 40 | .title { 41 | margin-top: 1rem; 42 | } 43 | 44 | .info { 45 | padding-bottom: 8px; 46 | } 47 | 48 | .feed { 49 | font-weight: 200; 50 | font-size: 10px; 51 | text-align: start; 52 | margin-left: 15px; 53 | margin-right: 15px; 54 | text-overflow: ellipsis; 55 | overflow: hidden; 56 | white-space: nowrap; 57 | } 58 | 59 | .index { 60 | font-size: 12px; 61 | text-align: end; 62 | } 63 | 64 | .time { 65 | font-size: 12px; 66 | text-align: end; 67 | } 68 | 69 | .description { 70 | width: 100%; 71 | } 72 | 73 | .description ::ng-deep img { 74 | max-width: 100%; 75 | height: auto; 76 | } 77 | 78 | .description ::ng-deep img.float { 79 | float: left; 80 | padding-right: 16px; 81 | padding-bottom: 16px; 82 | } 83 | 84 | .description ::ng-deep img.center { 85 | display: block; 86 | margin: 0 auto; 87 | } 88 | -------------------------------------------------------------------------------- /rf-ng/src/app/article-display/article-display.html: -------------------------------------------------------------------------------- 1 | ; 6 | 7 |
8 |

9 | 14 | 15 | 16 |

17 |
18 |
19 | {{ item.feed }} 20 |
21 |
22 | {{ index }} 23 |
24 |
25 | {{ item.time }} 26 |
27 |
28 |

29 | 30 |
31 |
32 | 33 | View 34 | 35 |
36 |
37 | 40 |
41 |
42 | 45 |
46 |
47 |
48 | 49 | 50 | -------------------------------------------------------------------------------- /rf-ng/src/app/article-list/article-list.css: -------------------------------------------------------------------------------- 1 | :host { 2 | flex-grow: 1; 3 | display: flex; 4 | height: calc(100% - 64px); 5 | } 6 | 7 | .empty-list { 8 | display: flex; 9 | justify-content: center; 10 | align-items: center; 11 | width: 100%; 12 | height: 100%; 13 | } 14 | 15 | virtual-scroller { 16 | overflow: hidden; 17 | overflow-y: auto; 18 | display: block; 19 | flex-grow: 1; 20 | background: #eee; 21 | } 22 | 23 | list-item { 24 | display: inline-block; 25 | overflow: hidden; 26 | vertical-align: top; 27 | border: 0; 28 | width: calc(25% - 16px); 29 | margin: 8px; 30 | } 31 | 32 | @media (max-width: 1350px) { 33 | list-item { 34 | width: calc(33% - 16px); 35 | } 36 | } 37 | 38 | @media (max-width: 900px) { 39 | list-item { 40 | width: calc(50% - 16px); 41 | } 42 | } 43 | 44 | @media (max-width: 600px) { 45 | list-item { 46 | width: calc(100% - 16px); 47 | } 48 | } 49 | 50 | .loader { 51 | height: 2em; 52 | width: 100%; 53 | display: block; 54 | line-height: 2em; 55 | text-align: center; 56 | position: relative; 57 | } 58 | .loader:before { 59 | content: ' '; 60 | position: absolute; 61 | top: 0; 62 | left: 0; 63 | width: 20%; 64 | height: 2px; 65 | background: red; 66 | animation: loader-animation 2s ease-out infinite; 67 | } 68 | @keyframes loader-animation { 69 | 0% { 70 | transform: translate(0%); 71 | } 72 | 100% { 73 | transform: translate(500%); 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /rf-ng/src/app/article-list/article-list.html: -------------------------------------------------------------------------------- 1 |

2 | 3 | No articles to display 4 | 5 |

6 |

7 | Loading... 8 |

9 | 10 | 11 | 12 | 13 | 14 |
Loading...
15 | -------------------------------------------------------------------------------- /rf-ng/src/app/article-list/list-item.component.ts: -------------------------------------------------------------------------------- 1 | import { Component, Input } from '@angular/core'; 2 | import { Router, ActivatedRoute } from "@angular/router"; 3 | import { ArticleService, Article } from "../services/article"; 4 | 5 | @Component({ 6 | selector: "list-item", 7 | templateUrl: "./list-item.html", 8 | styleUrls: ["./list-item.css"], 9 | }) 10 | export class ListItemComponent { 11 | @Input() 12 | item: Article 13 | 14 | constructor( 15 | private articleService: ArticleService, 16 | private router: Router, 17 | private route: ActivatedRoute, 18 | ) { } 19 | 20 | openArticle(article: Article) { 21 | this.router.navigate(['article', article.id], {relativeTo: this.route}); 22 | } 23 | 24 | favor(id: number, favor: boolean) { 25 | this.articleService.favor(id, favor).subscribe( 26 | success => { }, 27 | error => console.log(error) 28 | ) 29 | } 30 | } -------------------------------------------------------------------------------- /rf-ng/src/app/article-list/list-item.css: -------------------------------------------------------------------------------- 1 | :host { 2 | height: 500px; 3 | } 4 | 5 | mat-card { 6 | display: flex; 7 | flex-direction: column; 8 | height: calc(100% - 12px); 9 | cursor: pointer; 10 | } 11 | 12 | mat-card-header { 13 | overflow: hidden; 14 | } 15 | 16 | [mat-card-image] { 17 | height: 240px; 18 | flex: 0 0 auto; 19 | background-position: center center; 20 | background-repeat: no-repeat; 21 | background-size: contain; 22 | } 23 | 24 | mat-card-content { 25 | flex: 1 1 auto; 26 | overflow: hidden; 27 | text-overflow: ellipsis-lastline; 28 | font-size: 12px; 29 | line-height: 18px; 30 | max-height: 90px; 31 | display: -webkit-box; 32 | -webkit-line-clamp: 5; 33 | -webkit-box-orient: vertical; 34 | } 35 | 36 | mat-card-actions { 37 | flex: 0 0 auto; 38 | } 39 | 40 | .read { 41 | font-weight: 100; 42 | color: #666; 43 | } 44 | 45 | .read .title { 46 | font-weight: 300; 47 | } 48 | 49 | .title { 50 | font-weight: bold; 51 | font-size: 14px; 52 | overflow: hidden; 53 | text-overflow: ellipsis; 54 | line-height: 1; 55 | height: 3.85em; 56 | } 57 | 58 | .info { 59 | margin: 0; 60 | } 61 | 62 | .feed { 63 | font-weight: 200; 64 | font-size: 10px; 65 | text-align: start; 66 | text-overflow: ellipsis; 67 | overflow: hidden; 68 | white-space: nowrap; 69 | } 70 | 71 | .time { 72 | font-size: 11px; 73 | text-align: end; 74 | } 75 | 76 | button:hover, button:focus { 77 | outline: none; 78 | text-decoration: none; 79 | } 80 | -------------------------------------------------------------------------------- /rf-ng/src/app/article-list/list-item.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 9 | 10 | 11 | 12 |
13 |
14 | 15 |
16 | 17 | 18 |
19 |
20 | {{ item.feed }} 21 |
22 |
23 | {{ item.time }} 24 |
25 |
26 | 27 | 28 | -------------------------------------------------------------------------------- /rf-ng/src/app/components/app.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/urandom/readeef/95c851e5ee6c3a12b0a8888985802698a56bf47b/rf-ng/src/app/components/app.css -------------------------------------------------------------------------------- /rf-ng/src/app/components/app.html: -------------------------------------------------------------------------------- 1 | 2 |
3 | 4 |
-------------------------------------------------------------------------------- /rf-ng/src/app/components/app.spec.ts: -------------------------------------------------------------------------------- 1 | import { TestBed, waitForAsync } from '@angular/core/testing'; 2 | 3 | import { AppComponent } from './app'; 4 | 5 | describe('AppComponent', () => { 6 | beforeEach(waitForAsync(() => { 7 | TestBed.configureTestingModule({ 8 | declarations: [ 9 | AppComponent 10 | ], 11 | }).compileComponents(); 12 | })); 13 | 14 | it('should create the app', waitForAsync(() => { 15 | const fixture = TestBed.createComponent(AppComponent); 16 | const app = fixture.debugElement.componentInstance; 17 | expect(app).toBeTruthy(); 18 | })); 19 | 20 | it(`should have as title 'app'`, waitForAsync(() => { 21 | const fixture = TestBed.createComponent(AppComponent); 22 | const app = fixture.debugElement.componentInstance; 23 | expect(app.title).toEqual('app'); 24 | })); 25 | 26 | it('should render title in a h1 tag', waitForAsync(() => { 27 | const fixture = TestBed.createComponent(AppComponent); 28 | fixture.detectChanges(); 29 | const compiled = fixture.debugElement.nativeElement; 30 | expect(compiled.querySelector('h1').textContent).toContain('Welcome to app!'); 31 | })); 32 | }); 33 | -------------------------------------------------------------------------------- /rf-ng/src/app/components/app.ts: -------------------------------------------------------------------------------- 1 | import { Component } from '@angular/core'; 2 | 3 | @Component({ 4 | selector: 'app-root', 5 | templateUrl: './app.html', 6 | styleUrls: ['./app.css'] 7 | }) 8 | export class AppComponent { 9 | title = 'app'; 10 | } 11 | -------------------------------------------------------------------------------- /rf-ng/src/app/guards/auth.ts: -------------------------------------------------------------------------------- 1 | import { Injectable } from '@angular/core' 2 | import { Router, CanActivate, ActivatedRouteSnapshot, RouterStateSnapshot } from '@angular/router' 3 | import * as jwt from 'jwt-decode' 4 | 5 | @Injectable({providedIn: "root"}) 6 | export class AuthGuard implements CanActivate { 7 | constructor(private router: Router) {} 8 | 9 | canActivate(route: ActivatedRouteSnapshot, state: RouterStateSnapshot) : boolean { 10 | var token = localStorage.getItem("token") 11 | if (token) { 12 | var res = jwt(token) 13 | if ( 14 | (!res["exp"] || res["exp"] >= new Date().getTime() / 1000) && 15 | (!res["nbf"] || res["nbf"] < new Date().getTime() / 1000) 16 | ) { 17 | return true; 18 | } 19 | } 20 | 21 | this.router.navigate(['/login'], { queryParams: { returnUrl: state.url }}); 22 | return false; 23 | } 24 | } -------------------------------------------------------------------------------- /rf-ng/src/app/login/login.component.ts: -------------------------------------------------------------------------------- 1 | import { HttpErrorResponse } from "@angular/common/http"; 2 | import { Component, OnInit } from "@angular/core"; 3 | import { ActivatedRoute, Router } from "@angular/router"; 4 | import { TokenService } from "../services/auth"; 5 | 6 | @Component({ 7 | templateUrl: "./login.html", 8 | styleUrls: ["./login.css"], 9 | }) 10 | export class LoginComponent implements OnInit { 11 | user: string; 12 | password: string; 13 | loading = false; 14 | invalidLogin = false; 15 | 16 | returnURL: string; 17 | 18 | constructor( 19 | private router: Router, 20 | private route: ActivatedRoute, 21 | private tokenService: TokenService, 22 | ) { } 23 | 24 | ngOnInit(): void { 25 | this.tokenService.delete(); 26 | 27 | this.returnURL = this.route.snapshot.queryParams["returnURL"] || '/'; 28 | } 29 | 30 | login() { 31 | this.loading = true 32 | this.tokenService.create(this.user, this.password).subscribe( 33 | data => { 34 | this.invalidLogin = false; 35 | this.router.navigate([this.returnURL]); 36 | }, 37 | (error: HttpErrorResponse) => { 38 | if (error.status == 401) { 39 | this.invalidLogin = true; 40 | } 41 | this.loading = false; 42 | } 43 | ) 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /rf-ng/src/app/login/login.css: -------------------------------------------------------------------------------- 1 | .content { 2 | display: flex; 3 | height: 100vh; 4 | align-items: center; 5 | justify-content: center; 6 | } 7 | 8 | button { 9 | margin-bottom: 1rem; 10 | } -------------------------------------------------------------------------------- /rf-ng/src/app/login/login.html: -------------------------------------------------------------------------------- 1 |
2 |
3 | 4 | 5 | 6 | 7 | 9 | 10 |
11 | 12 | Invalid username or password 13 |
14 |
-------------------------------------------------------------------------------- /rf-ng/src/app/main/main.component.ts: -------------------------------------------------------------------------------- 1 | import { Component, OnInit, OnDestroy, ViewChild } from "@angular/core"; 2 | import { NavigationEnd, Router, RouterEvent } from '@angular/router'; 3 | import { Subscription } from "rxjs"; 4 | import { listRoute, articleRoute } from "./routing-util" 5 | import { TokenService } from "../services/auth"; 6 | import { filter, switchMap, map, combineLatest } from "rxjs/operators"; 7 | import { MatSidenav } from '@angular/material/sidenav'; 8 | 9 | @Component({ 10 | templateUrl: "./main.html", 11 | styleUrls: ["./main.css"], 12 | }) 13 | export class MainComponent implements OnInit, OnDestroy { 14 | showsArticle = false 15 | inSearch = false 16 | 17 | private subscriptions = new Array(); 18 | 19 | @ViewChild('sidenav', {static: true}) 20 | private sidenav: MatSidenav; 21 | 22 | constructor( 23 | private tokenService: TokenService, 24 | private router: Router, 25 | ) {} 26 | 27 | ngOnInit() { 28 | this.subscriptions.push(this.tokenService.tokenObservable().pipe( 29 | filter(token => token != ""), 30 | switchMap(_ => 31 | articleRoute(this.router).pipe( 32 | map(route => route != null), 33 | combineLatest( 34 | listRoute(this.router).pipe(map(route => 35 | route != null && route.data["primary"] == "search" 36 | )), 37 | (inArticles, inSearch) : [boolean, boolean] => 38 | [inArticles, inSearch] 39 | ), 40 | ), 41 | ), 42 | ).subscribe( 43 | data => { 44 | this.showsArticle = data[0]; 45 | this.inSearch = data[1]; 46 | }, 47 | error => console.log(error) 48 | )); 49 | 50 | this.subscriptions.push(this.router.events.pipe( 51 | filter(e => e instanceof NavigationEnd) 52 | ).subscribe(_ => { 53 | this.sidenav.close(); 54 | }, err => console.log(err))); 55 | } 56 | 57 | ngOnDestroy(): void { 58 | this.subscriptions.forEach(s => s.unsubscribe()); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /rf-ng/src/app/main/main.css: -------------------------------------------------------------------------------- 1 | html, body, mat-sidenav-container, .content { 2 | width: 100%; 3 | height: 100vh; 4 | display: flex; 5 | flex-direction: column; 6 | overflow: hidden; 7 | } 8 | 9 | mat-sidenav { 10 | width: 200px; 11 | } 12 | -------------------------------------------------------------------------------- /rf-ng/src/app/main/routing-util.ts: -------------------------------------------------------------------------------- 1 | import { Router, NavigationEnd, ActivatedRouteSnapshot } from '@angular/router'; 2 | import { Observable } from "rxjs"; 3 | import { filter, map, startWith, distinctUntilChanged, shareReplay } from 'rxjs/operators'; 4 | 5 | export function listRoute(router: Router) : Observable { 6 | return router.events.pipe( 7 | filter(event => event instanceof NavigationEnd), 8 | map(_ => { 9 | return getListRoute([router.routerState.snapshot.root]) 10 | }), 11 | startWith(getListRoute([router.routerState.snapshot.root])), 12 | distinctUntilChanged(), 13 | shareReplay(1), 14 | ); 15 | } 16 | 17 | export function getListRoute(routes: ActivatedRouteSnapshot[]): ActivatedRouteSnapshot { 18 | for (let route of routes) { 19 | if ("primary" in route.data) { 20 | return route; 21 | } 22 | 23 | let r = getListRoute(route.children); 24 | if (r != null) { 25 | return r; 26 | } 27 | } 28 | 29 | return null; 30 | } 31 | 32 | export function articleRoute(router: Router): Observable { 33 | return router.events.pipe( 34 | filter(event => event instanceof NavigationEnd), 35 | map(v => { 36 | return getArticleRoute([router.routerState.snapshot.root]) 37 | }), 38 | startWith(getArticleRoute([router.routerState.snapshot.root])), 39 | distinctUntilChanged(), 40 | shareReplay(1), 41 | ); 42 | } 43 | 44 | export function getArticleRoute(routes: ActivatedRouteSnapshot[]): ActivatedRouteSnapshot { 45 | for (let route of routes) { 46 | if ("articleID" in route.params) { 47 | return route; 48 | } 49 | 50 | let r = getArticleRoute(route.children); 51 | if (r != null) { 52 | return r; 53 | } 54 | } 55 | 56 | return null; 57 | } 58 | -------------------------------------------------------------------------------- /rf-ng/src/app/services/favicon.ts: -------------------------------------------------------------------------------- 1 | import { Injectable } from '@angular/core' 2 | 3 | @Injectable({providedIn: "root"}) 4 | export class FaviconService { 5 | private parser = document.createElement("a") 6 | 7 | iconURL(url: string) : string { 8 | this.parser.href = url; 9 | 10 | let domain = this.parser.hostname; 11 | 12 | return `//www.google.com/s2/favicons?domain=${domain}` 13 | } 14 | } -------------------------------------------------------------------------------- /rf-ng/src/app/services/features.ts: -------------------------------------------------------------------------------- 1 | import { Injectable } from '@angular/core' 2 | import { Router } from '@angular/router' 3 | import { Observable, ConnectableObservable } from "rxjs"; 4 | import { APIService } from "./api"; 5 | import { TokenService } from "./auth"; 6 | import { switchMap, publishReplay, map } from 'rxjs/operators'; 7 | 8 | 9 | 10 | interface featuresPayload { 11 | features: Features 12 | } 13 | 14 | export interface Features { 15 | i18n: boolean; 16 | search: boolean; 17 | extractor: boolean; 18 | proxyHTTP: boolean; 19 | popularity: boolean; 20 | } 21 | 22 | @Injectable({providedIn: "root"}) 23 | export class FeaturesService { 24 | private features: Observable; 25 | 26 | constructor( 27 | private api: APIService, 28 | private tokenService: TokenService, 29 | ) { 30 | var features = this.tokenService.tokenObservable().pipe( 31 | switchMap(token => this.api.get("features").pipe( 32 | map(p => p.features) 33 | )), 34 | publishReplay(1) 35 | ) as ConnectableObservable; 36 | features.connect(); 37 | 38 | this.features = features; 39 | } 40 | 41 | getFeatures() : Observable { 42 | return this.features; 43 | } 44 | } -------------------------------------------------------------------------------- /rf-ng/src/app/services/interaction.ts: -------------------------------------------------------------------------------- 1 | import { Injectable, Output, EventEmitter } from '@angular/core' 2 | import { Location } from '@angular/common'; 3 | import { Router } from '@angular/router'; 4 | 5 | @Injectable({providedIn: "root"}) 6 | export class InteractionService { 7 | @Output() toolbarTitleClickEvent = new EventEmitter(); 8 | 9 | constructor( 10 | private location: Location, 11 | private router: Router, 12 | ) {} 13 | 14 | toolbarTitleClick() { 15 | this.toolbarTitleClickEvent.emit(); 16 | } 17 | 18 | navigateUp(): boolean { 19 | let path = this.location.path() 20 | let idx = path.indexOf("/article/") 21 | if (idx != -1) { 22 | this.router.navigateByUrl(path.substring(0, idx), {state: {"articleID": path.substring(idx+9)}}); 23 | return true; 24 | } 25 | return false; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /rf-ng/src/app/services/tag.ts: -------------------------------------------------------------------------------- 1 | import { Injectable } from '@angular/core' 2 | import { Router } from '@angular/router' 3 | import { Observable } from "rxjs"; 4 | import { APIService } from "./api"; 5 | import { map } from 'rxjs/operators'; 6 | 7 | 8 | export class Tag { 9 | id: number; 10 | value: string; 11 | } 12 | 13 | interface TagsResponse { 14 | tags: Tag[]; 15 | } 16 | 17 | export class TagFeedIDs { 18 | tag: Tag 19 | ids: number[] 20 | } 21 | 22 | interface TagsFeedIDs { 23 | tagFeeds: TagFeedIDs[]; 24 | } 25 | 26 | interface FeedIDsResponse { 27 | feedIDs: number[]; 28 | } 29 | 30 | @Injectable({providedIn: "root"}) 31 | export class TagService { 32 | constructor(private api: APIService) { } 33 | 34 | getTags() : Observable { 35 | return this.api.get("tag").pipe( 36 | map(response => response.tags) 37 | ); 38 | } 39 | 40 | getFeedIDs(tag: {id: number}): Observable { 41 | return this.api.get(`tag/${tag.id}/feedIDs`).pipe( 42 | map(response => response.feedIDs) 43 | ); 44 | } 45 | 46 | getTagsFeedIDs(): Observable { 47 | return this.api.get("tag/feedIDs").pipe( 48 | map(response => response.tagFeeds) 49 | ); 50 | } 51 | } -------------------------------------------------------------------------------- /rf-ng/src/app/settings/admin/admin.css: -------------------------------------------------------------------------------- 1 | .user { 2 | display: flex; 3 | align-items: center; 4 | border-bottom: 1px solid #ccc; 5 | padding-bottom: 4px; 6 | margin-bottom: 4px; 7 | } 8 | 9 | .user mat-checkbox { 10 | flex: 1 0 0; 11 | } 12 | 13 | .footer { 14 | text-align: end; 15 | } -------------------------------------------------------------------------------- /rf-ng/src/app/settings/admin/admin.html: -------------------------------------------------------------------------------- 1 |
Manage Users
2 |

3 | Current user: {{ current.login }} 4 |

5 |
6 |
List of users:
7 |
8 | {{ user.login }} 9 | 12 |
13 |
14 |
15 | 16 |
-------------------------------------------------------------------------------- /rf-ng/src/app/settings/admin/new-user.html: -------------------------------------------------------------------------------- 1 |
Add a new user
2 |
3 | 4 | 5 | 6 | Empty user name 7 | 8 | 9 |
10 | 11 | 12 | 13 | Empty password 14 | 15 | 16 |
17 |
18 | 19 |
20 |
-------------------------------------------------------------------------------- /rf-ng/src/app/settings/common.css: -------------------------------------------------------------------------------- 1 | .title { 2 | padding-top: 16px; 3 | } -------------------------------------------------------------------------------- /rf-ng/src/app/settings/discovery/discovery.css: -------------------------------------------------------------------------------- 1 | mat-form-field { 2 | width: 100%; 3 | } 4 | 5 | input[type="file"] { 6 | width: 100%; 7 | border-bottom: 1px solid grey; 8 | padding-bottom: 4px; 9 | margin-bottom: 1em; 10 | } 11 | 12 | mat-progress-bar { 13 | margin-top: 1em; 14 | } 15 | 16 | .discovery-link { 17 | display: block; 18 | text-align: end; 19 | } 20 | -------------------------------------------------------------------------------- /rf-ng/src/app/settings/filters/filters.css: -------------------------------------------------------------------------------- 1 | .filter { 2 | display: flex; 3 | flex-wrap: wrap; 4 | align-items: center; 5 | margin-bottom: 20px; 6 | } 7 | 8 | .filter .info { 9 | flex: 1 0 0; 10 | } 11 | 12 | .filter span { 13 | padding-left: 4px; 14 | padding-right: 4px; 15 | } 16 | 17 | .filter .term, .filter .tag, .filter .feeds { 18 | padding-left: 8px; 19 | padding-right: 8px; 20 | font-weight: bold; 21 | } 22 | 23 | .filter .url-and-title { 24 | font-style: italic; 25 | } 26 | 27 | .delete { 28 | margin-left: 8px; 29 | } 30 | 31 | .error { 32 | margin-left: 8px; 33 | visibility: hidden; 34 | } 35 | 36 | .error.visible { 37 | visibility: visible; 38 | } 39 | 40 | .footer { 41 | text-align: end; 42 | padding-bottom: 8px; 43 | } 44 | 45 | form { 46 | display: flex; 47 | flex-direction: column; 48 | margin-bottom: 20px; 49 | } 50 | 51 | form mat-form-field { 52 | flex: 1 0 auto; 53 | } 54 | 55 | form mat-slide-toggle { 56 | align-self: center; 57 | flex: 1 0 auto; 58 | } 59 | 60 | form mat-checkbox { 61 | font-size: 14px; 62 | } 63 | 64 | form mat-checkbox ::ng-deep .mat-checkbox-layout { 65 | white-space: normal; 66 | } 67 | 68 | form p { 69 | border-bottom: 1px solid #ddd; 70 | margin-top: 12px; 71 | margin-bottom: 24px; 72 | font-size: 13px; 73 | } 74 | 75 | form mat-form-field { 76 | width: 100%; 77 | } -------------------------------------------------------------------------------- /rf-ng/src/app/settings/filters/filters.html: -------------------------------------------------------------------------------- 1 |
Manage Filters
2 |

3 | No filters have been defined 4 |

5 |
6 |
7 | 8 | Matches 9 | Does not match 10 | {{ filter.urlTerm }} 11 | by URL 12 | 13 | 14 | and 15 | 16 | 17 | Matches 18 | Does not match 19 | {{ filter.titleTerm }} 20 | by title 21 | 22 | excluding tag: 23 | excluding feeds: 24 | on tag: 25 | on feeds: 26 | {{ tagLabel(filter.tagID) }} 27 | {{ feedsLabel(filter.feedIDs) }} 28 |
29 | 32 |
33 |
34 | 35 |
-------------------------------------------------------------------------------- /rf-ng/src/app/settings/general/general.css: -------------------------------------------------------------------------------- 1 | mat-form-field { 2 | width: 100%; 3 | } 4 | 5 | .footer { 6 | text-align: end; 7 | } -------------------------------------------------------------------------------- /rf-ng/src/app/settings/general/general.html: -------------------------------------------------------------------------------- 1 |
Personalize your feed reader
2 | 3 | 4 | 5 |
6 | 7 | 8 | 9 |
10 | 11 | 12 | 13 | Please enter a valid email address 14 | 15 | 16 |
17 | 18 | 19 | English 20 | Български 21 | 22 | 23 |
24 |
25 | 26 |
27 | -------------------------------------------------------------------------------- /rf-ng/src/app/settings/general/password-form.html: -------------------------------------------------------------------------------- 1 |
Change your password
2 | 3 | 4 | 5 | Invalid password 6 | 7 | 8 |
9 | 10 | 11 | 12 | Passwords do not match 13 | 14 | 15 |
16 | 17 | 18 | 19 |
20 |
21 | 22 |
-------------------------------------------------------------------------------- /rf-ng/src/app/settings/management/error-dialog.html: -------------------------------------------------------------------------------- 1 |
    2 |
  • {{ error }}
    • 3 |
4 |
5 | 6 |
-------------------------------------------------------------------------------- /rf-ng/src/app/settings/management/management.css: -------------------------------------------------------------------------------- 1 | .feed { 2 | display: flex; 3 | flex-wrap: wrap; 4 | align-items: center; 5 | margin-bottom: 20px; 6 | } 7 | 8 | .favicon { 9 | width: 16px; 10 | height: 16px; 11 | margin-right: 5px; 12 | flex-shrink: 1; 13 | } 14 | .name { 15 | flex: 1 0 0; 16 | min-width: 250px; 17 | } 18 | .tags { 19 | flex: 1 0 0; 20 | min-width: 250px; 21 | width: auto; 22 | margin-left: 8px; 23 | } 24 | 25 | .delete { 26 | margin-left: 8px; 27 | } 28 | 29 | .error { 30 | margin-left: 8px; 31 | visibility: hidden; 32 | } 33 | 34 | .error.visible { 35 | visibility: visible; 36 | } 37 | 38 | .footer { 39 | text-align: end; 40 | padding-bottom: 8px; 41 | } -------------------------------------------------------------------------------- /rf-ng/src/app/settings/management/management.html: -------------------------------------------------------------------------------- 1 |
Manage Feeds
2 |

3 | No feeds have been added 4 |

5 |
6 | 7 | {{ feed[0].title }} 8 | 11 | 12 | 13 | 14 | 17 |
18 |
19 | 20 | 21 |
-------------------------------------------------------------------------------- /rf-ng/src/app/settings/settings.component.ts: -------------------------------------------------------------------------------- 1 | import { Component, OnInit, OnDestroy } from "@angular/core" 2 | 3 | @Component({ 4 | selector: "settings-container", 5 | templateUrl: "./settings.html", 6 | styleUrls: ["./settings.css"], 7 | }) 8 | export class SettingsComponent implements OnInit { 9 | ngOnInit(): void { 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /rf-ng/src/app/settings/settings.css: -------------------------------------------------------------------------------- 1 | :host { 2 | overflow: auto; 3 | } 4 | -------------------------------------------------------------------------------- /rf-ng/src/app/settings/settings.html: -------------------------------------------------------------------------------- 1 |
2 | 3 |
-------------------------------------------------------------------------------- /rf-ng/src/app/settings/share-services/share-services.component.ts: -------------------------------------------------------------------------------- 1 | import { Component, OnInit } from "@angular/core" ; 2 | import { MatSlideToggle } from "@angular/material/slide-toggle"; 3 | import { SharingService, ShareService } from "../../services/sharing" 4 | 5 | @Component({ 6 | selector: "settings-share-services", 7 | templateUrl: "./share-services.html", 8 | styleUrls: ["../common.css", "./share-services.css"] 9 | }) 10 | export class ShareServicesSettingsComponent implements OnInit { 11 | services: [ShareService, boolean][][] 12 | 13 | constructor( 14 | private sharingService: SharingService, 15 | ) {} 16 | 17 | ngOnInit(): void { 18 | this.services = this.sharingService.groupedList(); 19 | } 20 | 21 | toggleService(id: string, enabled: boolean) { 22 | this.sharingService.toggle(id, enabled); 23 | } 24 | } -------------------------------------------------------------------------------- /rf-ng/src/app/settings/share-services/share-services.css: -------------------------------------------------------------------------------- 1 | .cards-container { 2 | display: flex; 3 | flex-direction: row; 4 | flex-wrap: wrap; 5 | } 6 | 7 | mat-card { 8 | flex: 1 0 0; 9 | margin: 8px; 10 | min-width: 250px; 11 | } 12 | 13 | .service { 14 | margin-bottom: 16px; 15 | } -------------------------------------------------------------------------------- /rf-ng/src/app/settings/share-services/share-services.html: -------------------------------------------------------------------------------- 1 |
Share Services
2 |
3 | 4 | 5 | 6 |
7 | {{ group[0][0].category }} 8 |
9 | 10 | 11 | 12 |
13 | {{ service[0].description }} 14 |
15 | 16 | 17 |
-------------------------------------------------------------------------------- /rf-ng/src/app/share-service/share-service.component.ts: -------------------------------------------------------------------------------- 1 | import { Directive, OnInit, Input } from "@angular/core" 2 | import { SharingService, ShareService } from "../services/sharing" 3 | 4 | @Directive({ 5 | selector: "share-service", 6 | }) 7 | export class ShareServiceComponent implements OnInit { 8 | @Input() 9 | id: string 10 | 11 | @Input() 12 | description: string 13 | 14 | @Input() 15 | category: string 16 | 17 | @Input() 18 | link: string 19 | 20 | @Input() 21 | share: string 22 | 23 | constructor( 24 | private sharingService: SharingService 25 | ) {} 26 | 27 | ngOnInit(): void { 28 | this.sharingService.register({ 29 | id: this.id, 30 | description: this.description, 31 | category: this.category, 32 | link: this.link, 33 | template: this.share, 34 | }) 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /rf-ng/src/app/sidebar/side-bar-settings.html: -------------------------------------------------------------------------------- 1 | 2 | Settings 3 | 4 |
5 | General 6 | Add Feed 7 | Manage Feeds 8 | Filters 9 | Share Services 10 | Administration 11 |
12 | 13 | Feeds 14 | Logout 15 |
-------------------------------------------------------------------------------- /rf-ng/src/app/sidebar/side-bar.css: -------------------------------------------------------------------------------- 1 | .content > * { 2 | margin-bottom: 4px; 3 | } 4 | .content > a { 5 | display: flex; 6 | font-size: 16px; 7 | } 8 | 9 | .items > a { 10 | display: flex; 11 | font-size: 13px; 12 | } 13 | 14 | .category { 15 | display: flex; 16 | } 17 | 18 | .category > .expander { 19 | padding-left: 8px; 20 | padding-right: 8px; 21 | min-width: 24px; 22 | flex-shrink: 1; 23 | } 24 | 25 | .category > a { 26 | flex-grow: 1; 27 | text-align: start; 28 | } 29 | 30 | .category > .items { 31 | flex: 0 1 100%; 32 | } 33 | 34 | .content hr { 35 | margin: 0; 36 | } 37 | 38 | .favicon { 39 | padding-right: 4px; 40 | } -------------------------------------------------------------------------------- /rf-ng/src/app/sidebar/sidebar.settings.component.ts: -------------------------------------------------------------------------------- 1 | import { Component, OnInit, OnDestroy } from '@angular/core'; 2 | import { UserService } from "../services/user" 3 | import { Subscription } from "rxjs"; 4 | import { map } from 'rxjs/operators'; 5 | 6 | @Component({ 7 | selector: 'side-bar', 8 | templateUrl: './side-bar-settings.html', 9 | styleUrls: ['./side-bar.css'] 10 | }) 11 | export class SideBarSettingsComponent implements OnInit, OnDestroy { 12 | admin: boolean 13 | 14 | private subscriptions = new Array() 15 | 16 | constructor( 17 | private userService: UserService, 18 | ) { } 19 | 20 | ngOnInit(): void { 21 | this.subscriptions.push(this.userService.getCurrentUser().pipe(map( 22 | user => user.admin 23 | )).subscribe( 24 | admin => this.admin = admin, 25 | error => console.log(error), 26 | )) 27 | } 28 | 29 | ngOnDestroy(): void { 30 | this.subscriptions.forEach(s => s.unsubscribe()); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /rf-ng/src/app/toolbar/toolbar-settings.html: -------------------------------------------------------------------------------- 1 | Settings 2 | -------------------------------------------------------------------------------- /rf-ng/src/app/toolbar/toolbar.css: -------------------------------------------------------------------------------- 1 | :host { 2 | display: flex; 3 | flex: 1 1 auto; 4 | align-items: center; 5 | } 6 | 7 | .spacer { 8 | flex: 1 1 auto; 9 | } 10 | 11 | .title { 12 | cursor: default; 13 | user-select: none; 14 | } 15 | 16 | .search { 17 | min-height: 0; 18 | border: none; 19 | background: transparent; 20 | } 21 | 22 | ::ng-deep label { 23 | margin-bottom: 0; 24 | } 25 | -------------------------------------------------------------------------------- /rf-ng/src/app/toolbar/toolbar.settings.component.ts: -------------------------------------------------------------------------------- 1 | import { Component, OnInit } from '@angular/core'; 2 | 3 | @Component({ 4 | templateUrl: './toolbar-settings.html', 5 | styleUrls: ['./toolbar.css'] 6 | }) 7 | export class ToolbarSettingsComponent implements OnInit { 8 | constructor( 9 | ) { 10 | } 11 | 12 | ngOnInit(): void { 13 | } 14 | } -------------------------------------------------------------------------------- /rf-ng/src/assets/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/urandom/readeef/95c851e5ee6c3a12b0a8888985802698a56bf47b/rf-ng/src/assets/.gitkeep -------------------------------------------------------------------------------- /rf-ng/src/assets/icons/readeef-114.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/urandom/readeef/95c851e5ee6c3a12b0a8888985802698a56bf47b/rf-ng/src/assets/icons/readeef-114.png -------------------------------------------------------------------------------- /rf-ng/src/assets/icons/readeef-144.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/urandom/readeef/95c851e5ee6c3a12b0a8888985802698a56bf47b/rf-ng/src/assets/icons/readeef-144.png -------------------------------------------------------------------------------- /rf-ng/src/assets/icons/readeef-72.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/urandom/readeef/95c851e5ee6c3a12b0a8888985802698a56bf47b/rf-ng/src/assets/icons/readeef-72.png -------------------------------------------------------------------------------- /rf-ng/src/assets/icons/readeef-small.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/urandom/readeef/95c851e5ee6c3a12b0a8888985802698a56bf47b/rf-ng/src/assets/icons/readeef-small.png -------------------------------------------------------------------------------- /rf-ng/src/assets/icons/readeef.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/urandom/readeef/95c851e5ee6c3a12b0a8888985802698a56bf47b/rf-ng/src/assets/icons/readeef.png -------------------------------------------------------------------------------- /rf-ng/src/environments/environment.prod.ts: -------------------------------------------------------------------------------- 1 | export const environment = { 2 | production: true, 3 | apiEndpoint: "/api/v2/", 4 | routeTracing: false, 5 | }; 6 | -------------------------------------------------------------------------------- /rf-ng/src/environments/environment.ts: -------------------------------------------------------------------------------- 1 | // The file contents for the current environment will overwrite these during build. 2 | // The build system defaults to the dev environment which uses `environment.ts`, but if you do 3 | // `ng build --env=prod` then `environment.prod.ts` will be used instead. 4 | // The list of which env maps to which file can be found in `.angular-cli.json`. 5 | 6 | export const environment = { 7 | production: false, 8 | apiEndpoint: "/api/v2/", 9 | routeTracing: false, 10 | }; 11 | -------------------------------------------------------------------------------- /rf-ng/src/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | readeef: feed aggregator 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 26 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /rf-ng/src/main.ts: -------------------------------------------------------------------------------- 1 | import { enableProdMode } from '@angular/core'; 2 | import { platformBrowserDynamic } from '@angular/platform-browser-dynamic'; 3 | 4 | import { AppModule } from './app/app.module'; 5 | import { environment } from './environments/environment'; 6 | 7 | if (environment.production) { 8 | enableProdMode(); 9 | } 10 | 11 | platformBrowserDynamic().bootstrapModule(AppModule); 12 | -------------------------------------------------------------------------------- /rf-ng/src/manifest.webmanifest: -------------------------------------------------------------------------------- 1 | { 2 | "name": "rf-ng", 3 | "short_name": "rf-ng", 4 | "theme_color": "#1976d2", 5 | "background_color": "#fafafa", 6 | "display": "standalone", 7 | "scope": "./", 8 | "start_url": "./", 9 | "icons": [ 10 | { 11 | "src": "assets/icons/readeef-small.png", 12 | "sizes": "48x48", 13 | "type": "image/png", 14 | "purpose": "maskable any" 15 | }, 16 | { 17 | "src": "assets/icons/readeef-72.png", 18 | "sizes": "72x72", 19 | "type": "image/png", 20 | "purpose": "maskable any" 21 | }, 22 | { 23 | "src": "assets/icons/readeef-114.png", 24 | "sizes": "114x114", 25 | "type": "image/png", 26 | "purpose": "maskable any" 27 | }, 28 | { 29 | "src": "assets/icons/readeef-144.png", 30 | "sizes": "144x144", 31 | "type": "image/png", 32 | "purpose": "maskable any" 33 | }, 34 | { 35 | "src": "assets/icons/readeef.png", 36 | "sizes": "192x192", 37 | "type": "image/png", 38 | "purpose": "maskable any" 39 | } 40 | ] 41 | } 42 | -------------------------------------------------------------------------------- /rf-ng/src/styles.css: -------------------------------------------------------------------------------- 1 | /* You can add global styles to this file, and also import other style files */ 2 | 3 | @import "~@angular/material/prebuilt-themes/indigo-pink.css"; 4 | 5 | a:hover, a:focus, 6 | button:focus, button:focus { 7 | outline: none; 8 | text-decoration: none; 9 | } 10 | -------------------------------------------------------------------------------- /rf-ng/src/test.ts: -------------------------------------------------------------------------------- 1 | // This file is required by karma.conf.js and loads recursively all the .spec and framework files 2 | 3 | import 'zone.js/dist/long-stack-trace-zone'; 4 | import 'zone.js/dist/proxy.js'; 5 | import 'zone.js/dist/sync-test'; 6 | import 'zone.js/dist/jasmine-patch'; 7 | import 'zone.js/dist/async-test'; 8 | import 'zone.js/dist/fake-async-test'; 9 | import { getTestBed } from '@angular/core/testing'; 10 | import { 11 | BrowserDynamicTestingModule, 12 | platformBrowserDynamicTesting 13 | } from '@angular/platform-browser-dynamic/testing'; 14 | 15 | // Unfortunately there's no typing for the `__karma__` variable. Just declare it as any. 16 | declare const __karma__: any; 17 | declare const require: any; 18 | 19 | // Prevent Karma from running prematurely. 20 | __karma__.loaded = function () {}; 21 | 22 | // First, initialize the Angular testing environment. 23 | getTestBed().initTestEnvironment( 24 | BrowserDynamicTestingModule, 25 | platformBrowserDynamicTesting() 26 | ); 27 | // Then we find all the tests. 28 | const context = require.context('./', true, /\.spec\.ts$/); 29 | // And load the modules. 30 | context.keys().map(context); 31 | // Finally, start Karma to run the tests. 32 | __karma__.start(); 33 | -------------------------------------------------------------------------------- /rf-ng/src/tsconfig.app.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../tsconfig.json", 3 | "compilerOptions": { 4 | "outDir": "../out-tsc/app", 5 | "baseUrl": "./", 6 | "module": "es2015", 7 | "types": [] 8 | }, 9 | "exclude": [ 10 | "test.ts", 11 | "**/*.spec.ts" 12 | ] 13 | } 14 | -------------------------------------------------------------------------------- /rf-ng/src/tsconfig.spec.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "../tsconfig.json", 3 | "compilerOptions": { 4 | "outDir": "../out-tsc/spec", 5 | "baseUrl": "./", 6 | "module": "commonjs", 7 | "target": "es5", 8 | "types": [ 9 | "jasmine", 10 | "node" 11 | ] 12 | }, 13 | "files": [ 14 | "test.ts" 15 | ], 16 | "include": [ 17 | "**/*.spec.ts", 18 | "**/*.d.ts" 19 | ] 20 | } 21 | -------------------------------------------------------------------------------- /rf-ng/src/typings.d.ts: -------------------------------------------------------------------------------- 1 | /* SystemJS module definition */ 2 | declare var module: NodeModule; 3 | interface NodeModule { 4 | id: string; 5 | } 6 | -------------------------------------------------------------------------------- /rf-ng/tsconfig.app.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "compilerOptions": { 4 | "outDir": "./out-tsc/app", 5 | "types": [] 6 | }, 7 | "files": [ 8 | "src/main.ts", 9 | "src/polyfills.ts" 10 | ], 11 | "include": [ 12 | "src/**/*.d.ts" 13 | ], 14 | "exclude": [ 15 | "src/test.ts", 16 | "src/**/*.spec.ts" 17 | ] 18 | } 19 | -------------------------------------------------------------------------------- /rf-ng/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compileOnSave": false, 3 | "compilerOptions": { 4 | "baseUrl": "./", 5 | "outDir": "./dist/out-tsc", 6 | "sourceMap": true, 7 | "declaration": false, 8 | "downlevelIteration": true, 9 | "experimentalDecorators": true, 10 | "module": "es2020", 11 | "moduleResolution": "node", 12 | "importHelpers": true, 13 | "target": "es2015", 14 | "typeRoots": [ 15 | "node_modules/@types" 16 | ], 17 | "lib": [ 18 | "es2018", 19 | "dom" 20 | ] 21 | }, 22 | "angularCompilerOptions": { 23 | "fullTemplateTypeCheck": true, 24 | "strictInjectionParameters": true 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /rf-ng/tsconfig.spec.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "compilerOptions": { 4 | "outDir": "./out-tsc/spec", 5 | "types": [ 6 | "jasmine", 7 | "node" 8 | ] 9 | }, 10 | "files": [ 11 | "src/test.ts", 12 | "src/polyfills.ts" 13 | ], 14 | "include": [ 15 | "src/**/*.spec.ts", 16 | "src/**/*.d.ts" 17 | ] 18 | } 19 | -------------------------------------------------------------------------------- /rf-ng/ui/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | readeef: feed aggregator 6 | 9 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /rf-ng/xliffmerge.json: -------------------------------------------------------------------------------- 1 | { 2 | "xliffmergeOptions": { 3 | "srcDir": "src/locale", 4 | "genDir": "src/locale" 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /systemd/readeef.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=readeef Feed Reader 3 | After=network.target 4 | 5 | [Service] 6 | User=readeef 7 | Type=simple 8 | WorkingDirectory=/home/readeef/readeef 9 | ExecStart=/home/readeef/readeef/start.sh 10 | ExecStop=/usr/bin/killall readeef-server 11 | 12 | [Install] 13 | WantedBy=multi-user.target 14 | -------------------------------------------------------------------------------- /templates/goose-format-result.tmpl: -------------------------------------------------------------------------------- 1 | {% define "content" %} 2 |
3 |
4 | 5 |
6 | {% range .paragraphs %} 7 |

{% . %}

8 | {% end %} 9 |
10 | {% end %} 11 | -------------------------------------------------------------------------------- /templates/raw.tmpl: -------------------------------------------------------------------------------- 1 | {% template "content" . %} 2 | -------------------------------------------------------------------------------- /timeout_client.go: -------------------------------------------------------------------------------- 1 | package readeef 2 | 3 | import ( 4 | "net" 5 | "net/http" 6 | "time" 7 | ) 8 | 9 | func TimeoutDialer(ct time.Duration, rwt time.Duration) func(net, addr string) (c net.Conn, err error) { 10 | return func(netw, addr string) (net.Conn, error) { 11 | conn, err := net.DialTimeout(netw, addr, ct) 12 | if err != nil { 13 | return nil, err 14 | } 15 | conn.SetDeadline(time.Now().Add(rwt)) 16 | return conn, nil 17 | } 18 | } 19 | 20 | func NewTimeoutClient(connectTimeout time.Duration, readWriteTimeout time.Duration) *http.Client { 21 | return &http.Client{ 22 | Transport: &http.Transport{ 23 | Dial: TimeoutDialer(connectTimeout, readWriteTimeout), 24 | }, 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /web/proxy.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "context" 5 | "io/ioutil" 6 | "net/http" 7 | "net/url" 8 | "time" 9 | 10 | "github.com/alexedwards/scs" 11 | "github.com/pkg/errors" 12 | ) 13 | 14 | func ProxyHandler(sessionManager *scs.Manager) http.HandlerFunc { 15 | return func(w http.ResponseWriter, r *http.Request) { 16 | session := sessionManager.Load(r) 17 | if ok, err := session.GetBool(visitorKey); !ok || err != nil { 18 | w.WriteHeader(http.StatusForbidden) 19 | return 20 | } 21 | 22 | r.ParseForm() 23 | var err error 24 | 25 | switch { 26 | default: 27 | var u *url.URL 28 | 29 | u, err = url.Parse(r.Form.Get("url")) 30 | if err != nil { 31 | err = errors.Wrapf(err, "parsing url to proxy (%s)", r.Form.Get("url")) 32 | break 33 | } 34 | if u.Scheme == "" { 35 | u.Scheme = "http" 36 | } 37 | 38 | var req *http.Request 39 | 40 | req, err = http.NewRequest("GET", u.String(), nil) 41 | if err != nil { 42 | err = errors.Wrapf(err, "creating proxy request to %s", u) 43 | break 44 | } 45 | 46 | ctx, cancel := context.WithTimeout(req.Context(), 30*time.Second) 47 | defer cancel() 48 | 49 | var resp *http.Response 50 | 51 | resp, err = http.DefaultClient.Do(req.WithContext(ctx)) 52 | if err != nil { 53 | err = errors.Wrapf(err, "Error getting proxy response from %s", u) 54 | break 55 | } 56 | 57 | defer resp.Body.Close() 58 | 59 | for k, values := range resp.Header { 60 | for _, v := range values { 61 | w.Header().Add(k, v) 62 | } 63 | } 64 | 65 | var b []byte 66 | 67 | b, err = ioutil.ReadAll(resp.Body) 68 | if err != nil { 69 | err = errors.Wrapf(err, "reading proxy response from %s", u) 70 | break 71 | } 72 | 73 | _, err = w.Write(b) 74 | } 75 | 76 | if err != nil { 77 | http.Error(w, err.Error(), http.StatusNotAcceptable) 78 | return 79 | } 80 | } 81 | } 82 | --------------------------------------------------------------------------------