├── .gitignore ├── Dockerfile ├── LICENSE ├── README.md ├── config.example.ini ├── requirements.txt ├── service.py └── wallabag_kindle_consumer ├── __init__.py ├── config.py ├── consumer.py ├── interface.py ├── models.py ├── refresher.py ├── sender.py ├── static └── css │ └── custom.css ├── templates ├── base.html ├── index.html ├── macros.html └── relogin.html └── wallabag.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by .ignore support plugin (hsz.mobi) 2 | ### Python template 3 | # Byte-compiled / optimized / DLL files 4 | __pycache__/ 5 | *.py[cod] 6 | *$py.class 7 | 8 | # C extensions 9 | *.so 10 | 11 | # Distribution / packaging 12 | .Python 13 | build/ 14 | develop-eggs/ 15 | dist/ 16 | downloads/ 17 | eggs/ 18 | .eggs/ 19 | lib/ 20 | lib64/ 21 | parts/ 22 | sdist/ 23 | var/ 24 | wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | .hypothesis/ 50 | 51 | # Translations 52 | *.mo 53 | *.pot 54 | 55 | # Django stuff: 56 | *.log 57 | .static_storage/ 58 | .media/ 59 | local_settings.py 60 | 61 | # Flask stuff: 62 | instance/ 63 | .webassets-cache 64 | 65 | # Scrapy stuff: 66 | .scrapy 67 | 68 | # Sphinx documentation 69 | docs/_build/ 70 | 71 | # PyBuilder 72 | target/ 73 | 74 | # Jupyter Notebook 75 | .ipynb_checkpoints 76 | 77 | # pyenv 78 | .python-version 79 | 80 | # celery beat schedule file 81 | celerybeat-schedule 82 | 83 | # SageMath parsed files 84 | *.sage.py 85 | 86 | # Environments 87 | .env 88 | .venv 89 | env/ 90 | venv/ 91 | ENV/ 92 | env.bak/ 93 | venv.bak/ 94 | 95 | # Spyder project settings 96 | .spyderproject 97 | .spyproject 98 | 99 | # Rope project settings 100 | .ropeproject 101 | 102 | # mkdocs documentation 103 | /site 104 | 105 | # mypy 106 | .mypy_cache/ 107 | ### JetBrains template 108 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm 109 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 110 | 111 | # User-specific stuff: 112 | .idea/**/workspace.xml 113 | .idea/**/tasks.xml 114 | .idea/dictionaries 115 | 116 | # Sensitive or high-churn files: 117 | .idea/**/dataSources/ 118 | .idea/**/dataSources.ids 119 | .idea/**/dataSources.xml 120 | .idea/**/dataSources.local.xml 121 | .idea/**/sqlDataSources.xml 122 | .idea/**/dynamic.xml 123 | .idea/**/uiDesigner.xml 124 | 125 | # Gradle: 126 | .idea/**/gradle.xml 127 | .idea/**/libraries 128 | 129 | # CMake 130 | cmake-build-debug/ 131 | cmake-build-release/ 132 | 133 | # Mongo Explorer plugin: 134 | .idea/**/mongoSettings.xml 135 | 136 | ## File-based project format: 137 | *.iws 138 | 139 | ## Plugin-specific files: 140 | 141 | # IntelliJ 142 | out/ 143 | 144 | # mpeltonen/sbt-idea plugin 145 | .idea_modules/ 146 | 147 | # JIRA plugin 148 | atlassian-ide-plugin.xml 149 | 150 | # Cursive Clojure plugin 151 | .idea/replstate.xml 152 | 153 | # Crashlytics plugin (for Android Studio and IntelliJ) 154 | com_crashlytics_export_strings.xml 155 | crashlytics.properties 156 | crashlytics-build.properties 157 | fabric.properties 158 | ### Vim template 159 | # Swap 160 | [._]*.s[a-v][a-z] 161 | [._]*.sw[a-p] 162 | [._]s[a-v][a-z] 163 | [._]sw[a-p] 164 | 165 | # Session 166 | Session.vim 167 | 168 | # Temporary 169 | .netrwhist 170 | *~ 171 | # Auto-generated tag files 172 | tags 173 | 174 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.7-alpine 2 | MAINTAINER Jan Losinski 3 | 4 | ADD requirements.txt /tmp 5 | RUN apk add -U --virtual .bdep \ 6 | build-base \ 7 | gcc \ 8 | && \ 9 | pip install -r /tmp/requirements.txt && \ 10 | apk del .bdep 11 | 12 | ADD . /app 13 | VOLUME /data 14 | 15 | WORKDIR /app 16 | 17 | EXPOSE 8080 18 | 19 | CMD ./service.py --refresher --consumer --interface --env -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Jan Losinski 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Push tagged wallabag articles to kindle 2 | 3 | 4 | This is a service that polls a [Wallabag](https://wallabag.org/en) 5 | instance for articles with a certain tag (`kindle`, `kindle-mobi` or 6 | `kindle-pdf`). If it finds an article tagged with one of thse tags, it 7 | exports the article in the requested format and sends it via email to a 8 | configured kindle adress. 9 | 10 | The software consists of three parts: 11 | 12 | * **consumer:** This is the part that consumes the tags from wallabag 13 | and pushes them to the kindle addresses. 14 | * **refresher:** As wallabag gives no permanent api tokens we need to 15 | refresh them regularly. This part asks for a new token every time the 16 | old one is due to expire. With this method we don't need to save the 17 | users password. If we miss a refresh cycle (for example because the 18 | service was stopped) it will send the user an email to its notification 19 | address asking him to log in again to get a fresh token. 20 | * **interface:** This provides a basic web interface to register new 21 | users, refresh logins or delete users. 22 | 23 | All parts can run in one process or you can use seperate processes for 24 | each part. All they share is the configuration and the database. 25 | 26 | ## Configuration 27 | 28 | The configuration can be done either by a ini-format configuration file 29 | or via environment variables. The later is used for running the service 30 | via docker. All config option have ro be below a section called `DEFAULT`. 31 | You can just copy and modify the example config. 32 | 33 | The following tabe describes all options: 34 | 35 | | file-option | env-var | type | default | description | 36 | |-------------------|--------------------|----------|------------|----------| 37 | |`wallabag_host` | `WALLABAG_HOST` | required | | The http url of the wallabag host. | 38 | |`db_uri` | `DB_URI` | required | | The dbapi url for the database. | 39 | |`client_id` | `CLIENT_ID` | required | | The Wallabag client id. Read [here](https://doc.wallabag.org/en/developer/api/oauth.html) how to get it. | 40 | |`client_secret` | `CLIENT_SECRET` | required | | The Wallabag client secret. Read above. | 41 | |`domain` | `DOMAIN` | required | | the domain the interface for the service is accessible on. | 42 | |`smtp_from` | `SMTP_FROM` | required | | The from-address to use. | 43 | |`smtp_host` | `SMTP_HOST` | required | | The SMTP hostname or ip. | 44 | |`smtp_port` | `SMTP_PORT` | required | | The Port of the SMTP server. | 45 | |`smtp_user` | `SMTP_USER` | required | | The user for SMTP auth. | 46 | |`smtp_passwd` | `SMTP_PASSWD` | required | | The password for SMTP auth. | 47 | |`tag` | `TAG` | optional | `kindle` | The tag to consume. | 48 | |`refresh_grace` | `REFRESH_GRACE` | optional | `120` | The amount of seconds the token is refreshed before expiring. | 49 | |`consume_interval` | `CONSUME_INTERVAL` | optional | `30` | The time in seconds between two consume cycles. | 50 | |`interface_host` | `INTERFACE_HOST` | optional | `120.0.0.1`| The IP the user interface should bind to. | 51 | |`interface_port` | `INTERFACE_PORT` | optional | `8080` | The port the user interface should bind. | 52 | 53 | The config file is read either from the current working dicectory where it 54 | would be called `config.ini` or from a path hiven to the commandline option 55 | `--cfg`. To use configuration via environment use `--env`. 56 | 57 | 58 | ## Running 59 | 60 | Every component must be enabled via a commandline switch. To runn all three 61 | in one prcess use all three: 62 | ``` 63 | $ ./service.py --refresher --consumer --interface 64 | ``` 65 | 66 | For more information about commandline options use `--help`. 67 | 68 | 69 | ## Database 70 | 71 | The servie uses a database. The easiest option is to use sqlite. To save 72 | all data in a file called database.db in the current working directory set 73 | `sqlite:///database.db` in the config option `db_uri` or the env variable 74 | `DB_URI`. 75 | 76 | To initialize the database, run the server once with `--create_db`. 77 | 78 | 79 | ## Docker 80 | 81 | The provided dockerfile makes it easy to use the software as a docker 82 | container. The latest state of the master branch can be found as a 83 | [Automated build on the Docker hub](https://hub.docker.com/r/janlo/wallabag-kindle-consumer). 84 | 85 | You can easily configure the container via the environment variables from 86 | above. Just make sure, that the database is in a volume if you use sqlite. 87 | 88 | 89 | ## Usage 90 | 91 | As soon as you've set it all up you can reister your wallabag user in the 92 | Web-UI. The service does not store any login information other than the 93 | user name and the API token for wallabag. All actions that need a valid 94 | user like register/login/delete a user are authenticated directly against 95 | the wallabag server. 96 | 97 | ## License 98 | 99 | MIT 100 | -------------------------------------------------------------------------------- /config.example.ini: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | db_uri = sqlite:///database.db 3 | wallabag_host = https://wallabag.myserver.org 4 | client_id = 1_3o53gl30vhgk0c8ks4cocww08o84448osgo40wgw4gwkoo8skc 5 | client_secret = 636ocbqo978ckw0gsw4gcwwocg8044sco0w8w84cws48ggogs4 6 | domain = https://consumer.myserver.org 7 | smtp_from = wallabag@myserver.org 8 | smtp_host = myserver.org 9 | smtp_port = 587 10 | smtp_user = mail_user 11 | smtp_passwd = mail_pass -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | uvloop 2 | email_validator 3 | jinja2 4 | aiohttp_jinja2 5 | aiohttp 6 | sqlalchemy 7 | logbook 8 | psycopg2-binary 9 | -------------------------------------------------------------------------------- /service.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import argparse 4 | import asyncio 5 | import signal 6 | 7 | import logbook 8 | import uvloop 9 | import sys 10 | 11 | from logbook import Logger, StreamHandler 12 | 13 | from wallabag_kindle_consumer import models 14 | from wallabag_kindle_consumer.config import Config 15 | from wallabag_kindle_consumer.consumer import Consumer 16 | from wallabag_kindle_consumer.interface import App 17 | from wallabag_kindle_consumer.refresher import Refresher 18 | from wallabag_kindle_consumer.sender import Sender 19 | from wallabag_kindle_consumer.wallabag import Wallabag 20 | 21 | logger = Logger("kindle-consumer") 22 | 23 | 24 | def parse_args(): 25 | parser = argparse.ArgumentParser(description="Wallabag-Kindle-Consumer") 26 | parser.add_argument("--cfg", help="config file", required=False) 27 | parser.add_argument("--env", help="Read config from env", action="store_true") 28 | parser.add_argument("--refresher", help="Start token refresher", action="store_true") 29 | parser.add_argument("--interface", help="Start web interface", action="store_true") 30 | parser.add_argument("--consumer", help="Start article consumer", action="store_true") 31 | parser.add_argument("--create_db", help="Try to create the db", action="store_true") 32 | parser.add_argument("--debug", help="Enable debug logging", action="store_true") 33 | 34 | return parser.parse_args() 35 | 36 | 37 | if __name__ == "__main__": 38 | asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) 39 | 40 | loop = asyncio.get_event_loop() 41 | 42 | args = parse_args() 43 | 44 | level = logbook.INFO 45 | if args.debug: 46 | level = logbook.DEBUG 47 | 48 | StreamHandler(sys.stdout, level=level).push_application() 49 | 50 | config = Config.from_file("config.ini") 51 | 52 | if 'cfg' in args and args.cfg is not None: 53 | new = Config.from_file(args.cfg) 54 | if new is not None: 55 | config = new 56 | 57 | if 'env' in args and args.env: 58 | new = Config.from_env() 59 | if new is not None: 60 | config = new 61 | 62 | if args.create_db: 63 | models.create_db(config) 64 | logger.info("Database created.") 65 | 66 | on_stop = [] 67 | 68 | 69 | def _stop(): 70 | for cb in on_stop: 71 | cb() 72 | 73 | loop.stop() 74 | 75 | 76 | loop.add_signal_handler(signal.SIGTERM, _stop) 77 | loop.add_signal_handler(signal.SIGINT, _stop) 78 | 79 | wallabag = Wallabag(config) 80 | sender = Sender(loop, config.smtp_from, config.smtp_host, config.smtp_port, config.smtp_user, config.smtp_passwd) 81 | 82 | if args.refresher: 83 | logger.info("Create Refresher") 84 | refresher = Refresher(config, wallabag, sender) 85 | loop.create_task(refresher.refresh()) 86 | on_stop.append(lambda: refresher.stop()) 87 | 88 | if args.consumer: 89 | logger.info("Create Consumer") 90 | consumer = Consumer(wallabag, config, sender) 91 | loop.create_task(consumer.consume()) 92 | on_stop.append(lambda: consumer.stop()) 93 | 94 | if args.interface: 95 | logger.info("Create Interface") 96 | webapp = App(config, wallabag) 97 | loop.create_task(webapp.register_server()) 98 | on_stop.append(lambda: webapp.stop()) 99 | 100 | loop.run_forever() 101 | -------------------------------------------------------------------------------- /wallabag_kindle_consumer/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/janLo/wallabag-kindle-consumer/e69f5c182a1b7b27a36bae449f795fc77921ee33/wallabag_kindle_consumer/__init__.py -------------------------------------------------------------------------------- /wallabag_kindle_consumer/config.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | import configparser 3 | import os 4 | 5 | from logbook import Logger 6 | 7 | logger = Logger(__name__) 8 | 9 | 10 | @dataclasses.dataclass 11 | class Config: 12 | wallabag_host: str 13 | db_uri: str 14 | client_id: str 15 | client_secret: str 16 | domain: str 17 | smtp_from: str 18 | smtp_host: str 19 | smtp_port: int 20 | smtp_user: str 21 | smtp_passwd: str 22 | tag: str = "kindle" 23 | refresh_grace: int = 120 24 | consume_interval: int = 30 25 | interface_host: str = "127.0.0.1" 26 | interface_port: int = 8080 27 | 28 | @staticmethod 29 | def from_file(filename): 30 | logger.info("read config from file {file}", file=filename) 31 | 32 | if not os.path.exists(filename): 33 | logger.warn("Config file {filename} does not exist", filename=filename) 34 | return None 35 | 36 | parser = configparser.ConfigParser() 37 | parser.read(filename) 38 | 39 | if 'DEFAULT' not in parser: 40 | logger.warn("Config file {filename} does not contain a section DEFAULT", filename=filename) 41 | return None 42 | 43 | dflt = parser['DEFAULT'] 44 | 45 | tmp = {} 46 | missing = [] 47 | for field in dataclasses.fields(Config): 48 | if field.name in dflt: 49 | tmp[field.name] = field.type(dflt[field.name]) 50 | else: 51 | if field.default is dataclasses.MISSING: 52 | missing.append(field.name) 53 | 54 | if 0 != len(missing): 55 | logger.warn("Config file {filename} does not contain configs for: {lst}", filename=filename, 56 | lst=", ".join(missing)) 57 | return None 58 | 59 | return Config(**tmp) 60 | 61 | @staticmethod 62 | def from_env(): 63 | logger.info("Read config from environment") 64 | tmp = {} 65 | missing = [] 66 | for field in dataclasses.fields(Config): 67 | if field.name.upper() in os.environ: 68 | tmp[field.name] = field.type(os.environ[field.name.upper()]) 69 | else: 70 | if field.default is dataclasses.MISSING: 71 | missing.append(field.name.upper()) 72 | 73 | if 0 != len(missing): 74 | logger.warn("Environment config does not contain configs for: {lst}", lst=", ".join(missing)) 75 | return None 76 | 77 | return Config(**tmp) 78 | -------------------------------------------------------------------------------- /wallabag_kindle_consumer/consumer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import asyncio 3 | import datetime 4 | 5 | from logbook import Logger 6 | from sqlalchemy.orm import joinedload 7 | 8 | from wallabag_kindle_consumer.models import User, Job, context_session 9 | 10 | logger = Logger(__name__) 11 | 12 | 13 | class Consumer: 14 | def __init__(self, wallabag, cfg, sender): 15 | self.wallabag = wallabag 16 | self.sessionmaker = context_session(cfg) 17 | self.interval = cfg.consume_interval 18 | self.sender = sender 19 | self.running = True 20 | 21 | self._wait_fut = None # type: asyncio.Future 22 | 23 | async def fetch_jobs(self, user): 24 | logger.debug("Fetch entries for user {}", user.name) 25 | async for entry in self.wallabag.fetch_entries(user): 26 | logger.info("Schedule job to send entry {}", entry.id) 27 | job = Job(article=entry.id, title=entry.title, format=entry.tag.format) 28 | user.jobs.append(job) 29 | await self.wallabag.remove_tag(user, entry) 30 | 31 | async def process_job(self, job, session): 32 | logger.info("Process export for job {id} ({format})", id=job.article, format=job.format) 33 | data = await self.wallabag.export_article(job.user, job.article, job.format) 34 | await self.sender.send_mail(job, data) 35 | session.delete(job) 36 | 37 | async def _wait_since(self, since: datetime.datetime): 38 | now = datetime.datetime.utcnow() 39 | wait = max(0.0, self.interval - (now - since).total_seconds()) 40 | 41 | if not self.running: 42 | return 43 | 44 | self._wait_fut = asyncio.ensure_future(asyncio.sleep(wait)) 45 | 46 | try: 47 | await self._wait_fut 48 | except asyncio.CancelledError: 49 | pass 50 | finally: 51 | self._wait_fut = None 52 | 53 | async def consume(self): 54 | while self.running: 55 | start = datetime.datetime.utcnow() 56 | 57 | with self.sessionmaker as session: 58 | logger.debug("Start consume run") 59 | fetches = [self.fetch_jobs(user) for user in session.query(User).filter(User.active == True).all()] 60 | await asyncio.gather(*fetches) 61 | session.commit() 62 | 63 | jobs = [self.process_job(job, session) for job in session.query(Job).options(joinedload(Job.user))] 64 | await asyncio.gather(*jobs) 65 | session.commit() 66 | 67 | await self._wait_since(start) 68 | 69 | def stop(self): 70 | self.running = False 71 | self._wait_fut.cancel() 72 | -------------------------------------------------------------------------------- /wallabag_kindle_consumer/interface.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import os 3 | 4 | import aiohttp_jinja2 5 | import jinja2 6 | from aiohttp import web 7 | 8 | from logbook import Logger 9 | from email_validator import validate_email, EmailNotValidError 10 | 11 | from . import wallabag 12 | from . import models 13 | 14 | logger = Logger(__name__) 15 | 16 | 17 | class Validator: 18 | def __init__(self, loop, data): 19 | self.loop = loop 20 | self.data = data 21 | self.errors = {} 22 | self.username = None 23 | self.password = None 24 | self.kindle_email = None 25 | self.notify_email = None 26 | 27 | async def validate_credentials(self): 28 | errors = {} 29 | if "username" not in self.data or 0 == len(self.data['username']): 30 | errors['username'] = 'Username not given or empty' 31 | else: 32 | self.username = self.data['username'] 33 | 34 | if 'password' not in self.data or 0 == len(self.data['password']): 35 | errors['password'] = "Password not given or empty" 36 | else: 37 | self.password = self.data['password'] 38 | 39 | self.errors.update(errors) 40 | return 0 == len(errors) 41 | 42 | async def _validate_email(self, address): 43 | val = await self.loop.run_in_executor(None, validate_email, address) 44 | return val['email'] 45 | 46 | async def validate_emails(self): 47 | errors = {} 48 | if 'kindleEmail' not in self.data or 0 == len(self.data['kindleEmail']): 49 | errors['kindleEmail'] = "Kindle email address not given or empty" 50 | else: 51 | try: 52 | kindleEmail = await self._validate_email(self.data['kindleEmail']) 53 | if kindleEmail.endswith('@kindle.com') or kindleEmail.endswith('@free.kindle.com'): 54 | self.kindle_email = kindleEmail 55 | else: 56 | errors['kindleEmail'] = 'Given Kindle email does not end with @kindle.com or @free.kindle.com' 57 | except EmailNotValidError: 58 | errors['kindleEmail'] = "Kindle email is not a valid email address" 59 | 60 | if 'notifyEmail' not in self.data or 0 == len(self.data['notifyEmail']): 61 | errors['notifyEmail'] = "Notification email not given or empty" 62 | else: 63 | try: 64 | self.notify_email = await self._validate_email(self.data['notifyEmail']) 65 | except EmailNotValidError: 66 | errors['notifyEmail'] = "Notification email is not a valid email address" 67 | 68 | self.errors.update(errors) 69 | return 0 == len(errors) 70 | 71 | @property 72 | def success(self): 73 | return 0 == len(self.errors) 74 | 75 | 76 | class ViewBase(web.View): 77 | def __init__(self, *args, **kwargs): 78 | super().__init__(*args, **kwargs) 79 | self._errors = {} 80 | self._data = {} 81 | self._messages = [] 82 | 83 | @property 84 | def _cfg(self): 85 | return self.request.app['config'] 86 | 87 | @property 88 | def _wallabag(self): 89 | return self.request.app['wallabag'] 90 | 91 | def _template(self, vars): 92 | vars.update({'errors': self._errors, 'data': self._data, 'messages': self._messages, 93 | 'wallabag_host': self._cfg.wallabag_host, 94 | 'tags': [t.tag for t in wallabag.make_tags(self._cfg.tag)]}) 95 | return vars 96 | 97 | def _add_errors(self, errors): 98 | self._errors.update(errors) 99 | 100 | def _set_data(self, data): 101 | self._data = data 102 | 103 | def _add_message(self, msg): 104 | self._messages.append(msg) 105 | 106 | @property 107 | def _session(self): 108 | return self.request.app['session_maker'] 109 | 110 | 111 | class IndexView(ViewBase): 112 | @aiohttp_jinja2.template("index.html") 113 | async def get(self): 114 | return self._template({}) 115 | 116 | @aiohttp_jinja2.template("index.html") 117 | async def post(self): 118 | data = await self.request.post() 119 | self._set_data(data) 120 | 121 | validator = Validator(self.request.app.loop, data) 122 | 123 | await asyncio.gather(validator.validate_emails(), 124 | validator.validate_credentials()) 125 | self._add_errors(validator.errors) 126 | 127 | if validator.success: 128 | user = models.User(name=validator.username, kindle_mail=validator.kindle_email, 129 | email=validator.notify_email) 130 | 131 | with self._session as session: 132 | if session.query(models.User.name).filter(models.User.name == validator.username).count() != 0: 133 | self._add_errors({'user': "User is already registered"}) 134 | elif not await self._wallabag.get_token(user, validator.password): 135 | self._add_errors({'auth': 'Cannot authenticate at wallabag server to get a token'}) 136 | else: 137 | session.add(user) 138 | session.commit() 139 | self._add_message(f'User {validator.username} successfully registered') 140 | self._set_data({}) 141 | logger.info("User {user} registered", user=validator.username) 142 | 143 | return self._template({}) 144 | 145 | 146 | class ReLoginView(ViewBase): 147 | @aiohttp_jinja2.template("relogin.html") 148 | async def get(self): 149 | return self._template({'action': 'update', 'description': 'Refresh'}) 150 | 151 | @aiohttp_jinja2.template("relogin.html") 152 | async def post(self): 153 | data = await self.request.post() 154 | self._set_data(data) 155 | 156 | validator = Validator(self.request.app.loop, data) 157 | await validator.validate_credentials() 158 | self._add_errors(validator.errors) 159 | 160 | if validator.success: 161 | with self._session as session: 162 | user = session.query(models.User).filter(models.User.name == validator.username).first() 163 | if user is None: 164 | self._add_errors({'user': 'User not registered'}) 165 | else: 166 | if await self._wallabag.get_token(user, validator.password): 167 | user.active = True 168 | session.commit() 169 | self._add_message(f"User {validator.username} successfully updated.") 170 | logger.info("User {user} successfully updated.", user=user) 171 | else: 172 | self._add_errors({'auth': "Authentication against wallabag server failed"}) 173 | 174 | return self._template({'action': 'update', 'description': 'Refresh'}) 175 | 176 | 177 | class DeleteView(ViewBase): 178 | @aiohttp_jinja2.template("relogin.html") 179 | async def get(self): 180 | return self._template({'action': 'delete', 'description': 'Delete'}) 181 | 182 | @aiohttp_jinja2.template("relogin.html") 183 | async def post(self): 184 | data = await self.request.post() 185 | self._set_data(data) 186 | 187 | validator = Validator(self.request.app.loop, data) 188 | await validator.validate_credentials() 189 | self._add_errors(validator.errors) 190 | 191 | if validator.success: 192 | with self._session as session: 193 | user = session.query(models.User).filter(models.User.name == validator.username).first() 194 | if user is None: 195 | self._add_errors({'user': 'User not registered'}) 196 | else: 197 | if await self._wallabag.get_token(user, validator.password): 198 | session.delete(user) 199 | session.commit() 200 | self._add_message(f"User {validator.username} successfully deleted.") 201 | logger.info("User {user} successfully deleted.", user=user) 202 | else: 203 | self._add_errors({'auth': "Authentication against wallabag server failed"}) 204 | 205 | return self._template({'action': 'delete', 'description': 'Delete'}) 206 | 207 | 208 | class App: 209 | def __init__(self, config, wallabag): 210 | self.config = config 211 | self.wallabag = wallabag 212 | self.app = web.Application() 213 | self.site = None # type: web.TCPSite 214 | 215 | self.setup_app() 216 | self.setup_routes() 217 | 218 | def setup_app(self): 219 | self.app['config'] = self.config 220 | self.app['wallabag'] = self.wallabag 221 | self.app['session_maker'] = models.context_session(self.config) 222 | aiohttp_jinja2.setup( 223 | self.app, loader=jinja2.PackageLoader('wallabag_kindle_consumer', 'templates')) 224 | 225 | self.app['static_root_url'] = '/static' 226 | 227 | def setup_routes(self): 228 | self.app.router.add_static('/static/', 229 | path=os.path.join(os.path.dirname(__file__), 'static'), 230 | name='static') 231 | self.app.router.add_view("/", IndexView) 232 | self.app.router.add_view("/delete", DeleteView) 233 | self.app.router.add_view("/update", ReLoginView) 234 | 235 | def run(self): 236 | web.run_app(self.app, host=self.config.interface_host, port=self.config.interface_port) 237 | 238 | async def register_server(self): 239 | app_runner = web.AppRunner(self.app, access_log=logger) 240 | await app_runner.setup() 241 | self.site = web.TCPSite(app_runner, self.config.interface_host, self.config.interface_port) 242 | await self.site.start() 243 | 244 | def stop(self): 245 | if self.site is not None: 246 | asyncio.get_event_loop().create_task(self.site.stop()) 247 | -------------------------------------------------------------------------------- /wallabag_kindle_consumer/models.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import Integer, String, DateTime, Column, ForeignKey, Enum, Boolean 2 | from sqlalchemy import create_engine 3 | from sqlalchemy.ext.declarative import declarative_base 4 | from sqlalchemy.orm import sessionmaker, relationship 5 | 6 | Base = declarative_base() 7 | 8 | 9 | class User(Base): 10 | __tablename__ = "user" 11 | 12 | name = Column(String, primary_key=True) 13 | token = Column(String()) 14 | auth_token = Column(String) 15 | refresh_token = Column(String) 16 | token_valid = Column(DateTime) 17 | last_check = Column(DateTime) 18 | email = Column(String) 19 | kindle_mail = Column(String) 20 | active = Column(Boolean, default=True) 21 | 22 | jobs = relationship('Job', backref='user') 23 | 24 | 25 | class Job(Base): 26 | __tablename__ = "job" 27 | 28 | id = Column(Integer, primary_key=True, autoincrement=True) 29 | article = Column(Integer) 30 | title = Column(String) 31 | user_name = Column(String, ForeignKey("user.name")) 32 | format = Column(Enum('pdf', 'mobi', 'epub', name='format_enum')) 33 | 34 | 35 | class ContextSession: 36 | def __init__(self, session_maker): 37 | self.session_maker = session_maker 38 | 39 | def __enter__(self): 40 | self.session = self.session_maker() 41 | return self.session 42 | 43 | def __exit__(self, exc_type, exc_val, exc_tb): 44 | self.session.close() 45 | 46 | 47 | def context_session(config): 48 | return ContextSession(session_maker(config)) 49 | 50 | 51 | def session_maker(config): 52 | Session = sessionmaker(autocommit=False, 53 | autoflush=False, 54 | bind=create_engine(config.db_uri)) 55 | return Session 56 | 57 | 58 | def create_db(config): 59 | engine = create_engine(config.db_uri) 60 | Base.metadata.create_all(engine) 61 | 62 | 63 | def re_create_db(config): 64 | engine = create_engine(config.db_uri) 65 | Base.metadata.drop_all(engine) 66 | Base.metadata.create_all(engine) 67 | -------------------------------------------------------------------------------- /wallabag_kindle_consumer/refresher.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from datetime import datetime, timedelta 3 | 4 | from logbook import Logger 5 | from sqlalchemy import func 6 | 7 | from .models import User, context_session 8 | 9 | logger = Logger(__name__) 10 | 11 | 12 | class Refresher: 13 | def __init__(self, config, wallabag, sender): 14 | self.sessionmaker = context_session(config) 15 | self.wallabag = wallabag 16 | self.grace = config.refresh_grace 17 | self.sender = sender 18 | self.config = config 19 | 20 | self._running = True 21 | self._wait_fut = None # type: asyncio.Future 22 | 23 | def _wait_time(self, session): 24 | next = session.query(func.min(User.token_valid).label("min")).filter(User.active == True).first() 25 | if next is None or next.min is None: 26 | return 3 27 | delta = next.min - datetime.utcnow() 28 | if delta < timedelta(seconds=self.grace): 29 | return 0 30 | 31 | calculated = delta - timedelta(seconds=self.grace) 32 | return calculated.total_seconds() 33 | 34 | async def refresh(self): 35 | while self._running: 36 | with self.sessionmaker as session: 37 | self._wait_fut = asyncio.ensure_future(asyncio.sleep(self._wait_time(session))) 38 | try: 39 | await self._wait_fut 40 | except asyncio.CancelledError: 41 | continue 42 | finally: 43 | self._wait_fut = None 44 | 45 | ts = datetime.utcnow() + timedelta(seconds=self.grace) 46 | refreshes = [self._refresh_user(user) for user 47 | in session.query(User).filter(User.active == True).filter(User.token_valid < ts).all()] 48 | await asyncio.gather(*refreshes) 49 | 50 | session.commit() 51 | 52 | async def _refresh_user(self, user): 53 | logger.info("Refresh token for {}", user.name) 54 | if not await self.wallabag.refresh_token(user): 55 | await self.sender.send_warning(user, self.config) 56 | user.active = False 57 | 58 | def stop(self): 59 | self._running = False 60 | if self._wait_fut is not None: 61 | self._wait_fut.cancel() 62 | -------------------------------------------------------------------------------- /wallabag_kindle_consumer/sender.py: -------------------------------------------------------------------------------- 1 | import smtplib 2 | from email.encoders import encode_base64 3 | from email.mime.application import MIMEApplication 4 | from email.mime.multipart import MIMEMultipart 5 | from email.mime.text import MIMEText 6 | from email.utils import formatdate, make_msgid 7 | 8 | from logbook import Logger 9 | 10 | logger = Logger(__name__) 11 | 12 | 13 | class Sender: 14 | def __init__(self, loop, from_addr, smtp_server, smtp_port, smtp_user=None, smtp_passwd=None): 15 | self.from_addr = from_addr 16 | self.loop = loop 17 | self.host = smtp_server 18 | self.port = smtp_port 19 | self.user = smtp_user 20 | self.passwd = smtp_passwd 21 | 22 | def _send_mail(self, title, article, format, email, data): 23 | msg = MIMEMultipart() 24 | msg['Subject'] = "Send article {}".format(article) 25 | msg['From'] = self.from_addr 26 | msg['To'] = email 27 | msg['Date'] = formatdate(localtime=True) 28 | msg['Message-ID'] = make_msgid('wallabag-kindle') 29 | 30 | text = 'This email has been automatically sent.' 31 | msg.attach(MIMEText(text)) 32 | 33 | mobi = MIMEApplication(data) 34 | encode_base64(mobi) 35 | mobi.add_header('Content-Disposition', 'attachment', 36 | filename='{title}.{format}'.format(title=title, format=format)) 37 | 38 | msg.attach(mobi) 39 | 40 | smtp = smtplib.SMTP(host=self.host, port=self.port) 41 | smtp.starttls() 42 | if self.user is not None: 43 | smtp.login(self.user, self.passwd) 44 | smtp.sendmail(self.from_addr, [email], msg.as_string()) 45 | smtp.quit() 46 | logger.info("Mail with article {article} in format {format} with title {title} sent to {email}".format(article=article, 47 | title=title, 48 | format=format, 49 | email=email)) 50 | 51 | async def send_mail(self, job, data): 52 | return self.loop.run_in_executor(None, self._send_mail, job.title, job.article, job.format, 53 | job.user.kindle_mail, data) 54 | 55 | def _send_warning(self, email, config): 56 | msg = MIMEMultipart() 57 | msg['Subject'] = "Wallabag-Kindle-Consumer Notice" 58 | msg['From'] = self.from_addr 59 | msg['To'] = email 60 | msg['Date'] = formatdate(localtime=True) 61 | 62 | txt = MIMEText(("the Wallabag-Kindle-Consumer for your Wallabag " 63 | "account on {wallabag} was not able to refresh " 64 | "the access token. Please go to {url}/update and log " 65 | "in again to retrieve a new api token.").format(wallabag=config.wallabag_host, 66 | url=config.domain)) 67 | 68 | msg.attach(txt) 69 | 70 | smtp = smtplib.SMTP(host=self.host, port=self.port) 71 | smtp.starttls() 72 | if self.user is not None: 73 | smtp.login(self.user, self.passwd) 74 | smtp.sendmail(self.from_addr, [email], msg.as_string()) 75 | smtp.quit() 76 | logger.info("Notify mail sent to {user}", user=email) 77 | 78 | async def send_warning(self, user, config): 79 | return self.loop.run_in_executor(None, self._send_warning, user.email, config) 80 | -------------------------------------------------------------------------------- /wallabag_kindle_consumer/static/css/custom.css: -------------------------------------------------------------------------------- 1 | html { 2 | position: relative; 3 | min-height: 100%; 4 | } 5 | body { 6 | margin-bottom: 80px; /* Margin bottom by footer height */ 7 | } 8 | .footer { 9 | position: absolute; 10 | bottom: 0; 11 | width: 100%; 12 | height: 60px; /* Set the fixed height of the footer here */ 13 | line-height: 60px; /* Vertically center the text there */ 14 | background-color: #f5f5f5; 15 | } 16 | -------------------------------------------------------------------------------- /wallabag_kindle_consumer/templates/base.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 11 | 12 | 13 | Wallabag Kindle Consumer 14 | 15 | 16 |
17 | 18 |

Walabag Kindle Consumer

19 | 20 |

This transfers all articles tagged with {{ tags|join(", ") }} in {{ wallabag_host }} 22 | to the given kindle account.

23 | 24 | {% block body %}{% endblock %} 25 | 26 | 27 |
28 | 34 | 35 | 36 | 39 | 42 | 45 | 46 | 47 | -------------------------------------------------------------------------------- /wallabag_kindle_consumer/templates/index.html: -------------------------------------------------------------------------------- 1 | {% extends 'base.html' %} 2 | 3 | {% import 'macros.html' as forms %} 4 | 5 | {% block body %} 6 | 7 | 18 | 19 | {{ forms.messages(errors=errors, messages=messages) }} 20 | 21 |
22 |
Add an account
23 |
24 |
25 |
26 | {{ forms.input('username', "Username", description='Your wallabag username at ' + wallabag_host, data=data, errors=errors) }} 27 | {{ forms.input('password', "Password", type='password', description='Your wallabag password at ' + wallabag_host + ". The password will not be stored", data=data, errors=errors) }} 28 |
29 |
30 | {{ forms.input('kindleEmail', "Kindle Email", type='email', description='Your kindle email addess ' + wallabag_host, data=data, errors=errors) }} 31 | {{ forms.input('notifyEmail', "Alt. Email",type='email', description='An alternative email where we can send any problems. ' + wallabag_host, data=data, errors=errors) }} 32 |
33 |
34 |
35 | 36 |
37 |
38 |
39 |
40 |
41 | 42 | {% endblock %} -------------------------------------------------------------------------------- /wallabag_kindle_consumer/templates/macros.html: -------------------------------------------------------------------------------- 1 | {% macro input(id, label, type='text', description='', placeholder='', data={}, errors={}) %} 2 |
3 | 4 | 7 | {{ description }} 8 |
9 | {% endmacro %} 10 | 11 | {% macro messages(errors, messages=[]) %} 12 | {% for msg in errors.values() %} 13 | 16 | {% endfor %} 17 | {% for msg in messages %} 18 | 21 | {% endfor %} 22 | {% endmacro %} -------------------------------------------------------------------------------- /wallabag_kindle_consumer/templates/relogin.html: -------------------------------------------------------------------------------- 1 | {% extends 'base.html' %} 2 | 3 | {% import 'macros.html' as forms %} 4 | 5 | {% block body %} 6 | 17 | 18 | {{ forms.messages(errors=errors, messages=messages) }} 19 | 20 |
21 |
{{ description }} an account
22 |
23 |
24 |
25 | {{ forms.input('username', "Username", description='Your wallabag username at ' + wallabag_host, data=data, errors=errors) }} 26 | {{ forms.input('password', "Password", type='password', description='Your wallabag password at ' + wallabag_host + ". The password will not be stored", data=data, errors=errors) }} 27 |
28 |
29 |
30 | 31 |
32 |
33 |
34 |
35 |
36 | 37 | {% endblock %} -------------------------------------------------------------------------------- /wallabag_kindle_consumer/wallabag.py: -------------------------------------------------------------------------------- 1 | import aiohttp 2 | from datetime import datetime, timedelta 3 | from collections import namedtuple 4 | 5 | from logbook import Logger 6 | 7 | logger = Logger(__name__) 8 | 9 | 10 | class Article: 11 | def __init__(self, id, tags, title, tag, **kwargs): 12 | self.id = id 13 | self.tags = tags 14 | self.title = title 15 | self.tag = tag 16 | 17 | def tag_id(self): 18 | for t in self.tags: 19 | if t['label'] == self.tag.tag: 20 | return t['id'] 21 | 22 | return -1 23 | 24 | 25 | Tag = namedtuple('Tag', ['tag', 'format']) 26 | 27 | 28 | def make_tags(tag): 29 | return (Tag(tag='{tag}'.format(tag=tag), format='epub'), 30 | Tag(tag='{tag}-epub'.format(tag=tag), format='epub'), 31 | Tag(tag='{tag}-mobi'.format(tag=tag), format='mobi'), 32 | Tag(tag='{tag}-pdf'.format(tag=tag), format='pdf')) 33 | 34 | 35 | class Wallabag: 36 | def __init__(self, config): 37 | self.config = config 38 | self.tag = "kindle" 39 | self.tags = make_tags(self.tag) 40 | 41 | async def get_token(self, user, passwd): 42 | params = {'grant_type': 'password', 43 | 'client_id': self.config.client_id, 44 | 'client_secret': self.config.client_secret, 45 | 'username': user.name, 46 | 'password': passwd} 47 | 48 | async with aiohttp.ClientSession() as session: 49 | async with session.post(self._url('/oauth/v2/token'), json=params) as resp: 50 | if resp.status != 200: 51 | logger.warn("Cannot get token for user {user}", user=user.name) 52 | return False 53 | data = await resp.json() 54 | user.auth_token = data["access_token"] 55 | user.refresh_token = data["refresh_token"] 56 | user.token_valid = datetime.utcnow() + timedelta(seconds=data["expires_in"]) 57 | logger.info("Got new token for {}", user.name) 58 | 59 | return True 60 | 61 | async def refresh_token(self, user): 62 | params = {'grant_type': 'refresh_token', 63 | 'client_id': self.config.client_id, 64 | 'client_secret': self.config.client_secret, 65 | 'refresh_token': user.refresh_token, 66 | 'username': user.name} 67 | 68 | async with aiohttp.ClientSession() as session: 69 | async with session.post(self._url('/oauth/v2/token'), json=params) as resp: 70 | if resp.status != 200: 71 | logger.warn("Cannot refresh token for user {user}", user=user.name) 72 | return False 73 | data = await resp.json() 74 | user.auth_token = data["access_token"] 75 | user.refresh_token = data["refresh_token"] 76 | user.token_valid = datetime.utcnow() + timedelta(seconds=data["expires_in"]) 77 | 78 | return True 79 | 80 | def _api_params(self, user, params=None): 81 | if params is None: 82 | params = {} 83 | 84 | params['access_token'] = user.auth_token 85 | return params 86 | 87 | def _url(self, url): 88 | return self.config.wallabag_host + url 89 | 90 | async def fetch_entries(self, user): 91 | if user.auth_token is None: 92 | logger.warn("No auth token for {}".format(user.name)) 93 | return 94 | async with aiohttp.ClientSession() as session: 95 | for tag in self.tags: 96 | params = self._api_params(user, {"tags": tag.tag}) 97 | async with session.get(self._url('/api/entries.json'), params=params) as resp: 98 | if resp.status != 200: 99 | logger.warn("Could not get entries of tag {tag} for user {user}", tag=tag.tag, user=user.name) 100 | return 101 | 102 | data = await resp.json() 103 | if data['pages'] == 1: 104 | user.last_check = datetime.utcnow() 105 | 106 | articles = data['_embedded']['items'] 107 | for article in articles: 108 | yield Article(tag=tag, **article) 109 | 110 | async def remove_tag(self, user, article): 111 | params = self._api_params(user) 112 | url = self._url('/api/entries/{entry}/tags/{tag}.json'.format(entry=article.id, 113 | tag=article.tag_id())) 114 | 115 | async with aiohttp.ClientSession() as session: 116 | async with session.delete(url, params=params) as resp: 117 | if resp.status != 200: 118 | logger.warn("Cannot remove tag {tag} from entry {entry} of user {user}", user=user.name, 119 | entry=article.id, tag=article.tag.tag) 120 | return 121 | 122 | logger.info("Removed tag {tag} from article {article} of user {user}", user=user.name, 123 | article=article.id, tag=article.tag.tag) 124 | 125 | async def export_article(self, user, article_id, format): 126 | params = self._api_params(user) 127 | url = self._url("/api/entries/{entry}/export.{format}".format(entry=article_id, format=format)) 128 | 129 | async with aiohttp.ClientSession() as session: 130 | async with session.get(url, params=params) as resp: 131 | if resp.status != 200: 132 | logger.warn("Cannot export article {article} of user {user} in format {format}", user=user.name, 133 | article=article_id, format=format) 134 | return 135 | 136 | return await resp.read() 137 | --------------------------------------------------------------------------------