├── .gitignore ├── doc ├── 01.png ├── 02.png ├── 03.png ├── 04.jpg ├── 05.jpg ├── 06.jpg ├── 07.png ├── cover.png ├── grant.png ├── more.png ├── banner.png ├── masthead.png ├── settings.png ├── starit.png ├── bannerBoox.png ├── screenshot_2016_08_29T20_43_53_0200.png └── screenshot_2020_05_16T18_02_13+0200.png ├── push ├── imgs │ ├── 1.png │ ├── 2.png │ ├── 3.png │ └── trigger-file.png ├── push-crontab.sh ├── readme.md └── push-script.py ├── .github └── FUNDING.yml ├── SendToBoox_Template.sh ├── SendToKindle_Template.sh ├── README.md └── Pocket.recipe /.gitignore: -------------------------------------------------------------------------------- 1 | test* 2 | bucket/ 3 | *.mobi 4 | *.epub 5 | mobi/ 6 | secret.sh 7 | -------------------------------------------------------------------------------- /doc/01.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ramiro/Pocket-Plus-Calibre-Plugin/master/doc/01.png -------------------------------------------------------------------------------- /doc/02.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ramiro/Pocket-Plus-Calibre-Plugin/master/doc/02.png -------------------------------------------------------------------------------- /doc/03.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ramiro/Pocket-Plus-Calibre-Plugin/master/doc/03.png -------------------------------------------------------------------------------- /doc/04.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ramiro/Pocket-Plus-Calibre-Plugin/master/doc/04.jpg -------------------------------------------------------------------------------- /doc/05.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ramiro/Pocket-Plus-Calibre-Plugin/master/doc/05.jpg -------------------------------------------------------------------------------- /doc/06.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ramiro/Pocket-Plus-Calibre-Plugin/master/doc/06.jpg -------------------------------------------------------------------------------- /doc/07.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ramiro/Pocket-Plus-Calibre-Plugin/master/doc/07.png -------------------------------------------------------------------------------- /doc/cover.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ramiro/Pocket-Plus-Calibre-Plugin/master/doc/cover.png -------------------------------------------------------------------------------- /doc/grant.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ramiro/Pocket-Plus-Calibre-Plugin/master/doc/grant.png -------------------------------------------------------------------------------- /doc/more.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ramiro/Pocket-Plus-Calibre-Plugin/master/doc/more.png -------------------------------------------------------------------------------- /doc/banner.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ramiro/Pocket-Plus-Calibre-Plugin/master/doc/banner.png -------------------------------------------------------------------------------- /doc/masthead.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ramiro/Pocket-Plus-Calibre-Plugin/master/doc/masthead.png -------------------------------------------------------------------------------- /doc/settings.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ramiro/Pocket-Plus-Calibre-Plugin/master/doc/settings.png -------------------------------------------------------------------------------- /doc/starit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ramiro/Pocket-Plus-Calibre-Plugin/master/doc/starit.png -------------------------------------------------------------------------------- /push/imgs/1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ramiro/Pocket-Plus-Calibre-Plugin/master/push/imgs/1.png -------------------------------------------------------------------------------- /push/imgs/2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ramiro/Pocket-Plus-Calibre-Plugin/master/push/imgs/2.png -------------------------------------------------------------------------------- /push/imgs/3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ramiro/Pocket-Plus-Calibre-Plugin/master/push/imgs/3.png -------------------------------------------------------------------------------- /doc/bannerBoox.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ramiro/Pocket-Plus-Calibre-Plugin/master/doc/bannerBoox.png -------------------------------------------------------------------------------- /push/imgs/trigger-file.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ramiro/Pocket-Plus-Calibre-Plugin/master/push/imgs/trigger-file.png -------------------------------------------------------------------------------- /doc/screenshot_2016_08_29T20_43_53_0200.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ramiro/Pocket-Plus-Calibre-Plugin/master/doc/screenshot_2016_08_29T20_43_53_0200.png -------------------------------------------------------------------------------- /doc/screenshot_2020_05_16T18_02_13+0200.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ramiro/Pocket-Plus-Calibre-Plugin/master/doc/screenshot_2020_05_16T18_02_13+0200.png -------------------------------------------------------------------------------- /push/push-crontab.sh: -------------------------------------------------------------------------------- 1 | #!/bin/zsh 2 | 3 | source ~/.zshrc 4 | 5 | every=5 6 | N=$((60/$every)) 7 | 8 | while [ $N -ne 0 ] 9 | do 10 | python /Users/magnus/workspace/Pocket-Plus-Calibre-Plugin/push/push-script.py \ 11 | /Users/magnus/Library/Mobile\ Documents/iCloud~is~workflow~my~workflows/Documents \ 12 | /Users/magnus/workspace/Pocket-Plus-Calibre-Plugin/push/push.sh # &> ~/Desktop/log.txt 13 | sleep $every 14 | ((N--)) 15 | # echo $N 16 | done 17 | exit 18 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: mmagnus # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] 4 | patreon: mmagnus ## 5 | open_collective: # Replace with a single Open Collective username 6 | ko_fi: mmagnus 7 | tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel 8 | community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry 9 | liberapay: # Replace with a single Liberapay username 10 | issuehunt: # Replace with a single IssueHunt username 11 | otechie: # Replace with a single Otechie username 12 | custom: https://paypal.me/MarcinMagnus 13 | -------------------------------------------------------------------------------- /SendToBoox_Template.sh: -------------------------------------------------------------------------------- 1 | ebook-convert Pocket.recipe .pdf #& open Pocket.mobi 2 | 3 | # this will overwrite a file there ! add something to the name if you want to prevent that 4 | mv -v Pocket.pdf "/Users/magnus/Dropbox/boox/Pocket `date +'%b %d %Y %H%M'`.pdf" 5 | # --backup=numbered --suffix '.pdf'# go to a folder and clean up naming 6 | #cd /Users/magnus/Dropbox/boox 7 | #mmv -v "*.*~*~" "#1_#3.#2" 8 | #cd - 9 | - 10 | # credits 11 | # "*.*~*~" "#1_#3.#2" 12 | # Pocket Mar 08 2021.pdf.~1~ -> Pocket Mar 08 2021_1.pdf. : done 13 | # Pocket Mar 08 2021.pdf.~2~ -> Pocket Mar 08 2021_2.pdf. : done 14 | - 15 | # sync 16 | boox.py 17 | -------------------------------------------------------------------------------- /push/readme.md: -------------------------------------------------------------------------------- 1 | Push 2 | ------------------------------------------------------------------------------- 3 | 4 | ## Create a Shortcut to create a file in given folder 5 | 6 | Here I use Shortcut for iPhone, the trigger phrase is "Push". 7 | 8 |
9 | 10 | ## This Shortcut will make a file in iCloud/Shortcuts 11 | 12 | ![](imgs/trigger-file.png) 13 | 14 | ## This new file will be detected by this script and my calibre push script will be executed 15 | 16 | * * * * * /Users/magnus/workspace/Pocket-Plus-Calibre-Plugin/push/push-crontab.sh 17 | # this is just fancy wrapper to get cron run every 5 s 18 | -------------------------------------------------------------------------------- /SendToKindle_Template.sh: -------------------------------------------------------------------------------- 1 | function sent { 2 | # a secret.sh is a file with config for sending books to Amazon 3 | # polityka_username=ma.. 4 | # polityka_passwd=KT.. 5 | # mail_username=mag.. 6 | # mail_from=mag.. 7 | # mail_passwd=xa.. 8 | # kindle_mail=mag.. 9 | # mail_server=poc.. 10 | source secret.sh 11 | echo 'mail: sending...' 12 | Calibre-smtp -a Pocket.mobi -u $mail_username --password $mail_passwd $mail_from $kindle_mail BODY --encryption-method SSL -r $mail_server 13 | } 14 | 15 | #archive 16 | #dev 17 | # rm /Users/magnus/Dropbox/feed.xml # reset feed ;-) 18 | #export pocketx_archive=False # not dev does not work at all 19 | rm Pocket.mobi 20 | rm ~/Documents/My\ Kindle\ Content/Pocket.mobi 21 | ebook-convert Pocket.recipe .mobi # --debug-pipeline debug #; & open Pocket.epub 22 | if test -f "Pocket.mobi"; then 23 | #open Pocket.mobi # check of missing images or problems 24 | /usr/local/bin/ebook-viewer Pocket.mobi 25 | # sent 26 | fi 27 | -------------------------------------------------------------------------------- /push/push-script.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | python push-script.py /Users/magnus/Library/Mobile\ Documents/iCloud~is~workflow~my~workflows/Documents 5 | """ 6 | import argparse 7 | import glob 8 | import os 9 | import shutil 10 | 11 | def get_parser(): 12 | parser = argparse.ArgumentParser( 13 | description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) 14 | 15 | #parser.add_argument('-', "--", help="", default="") 16 | parser.add_argument("-d", "--dev", 17 | action="store_true", help="don't remove files") 18 | parser.add_argument("-v", "--verbose", 19 | action="store_true", help="be verbose") 20 | parser.add_argument("path", help="", default="") # nargs='+') 21 | parser.add_argument("script", help="", default="") # nargs='+') 22 | return parser 23 | 24 | 25 | if __name__ == '__main__': 26 | parser = get_parser() 27 | args = parser.parse_args() 28 | path = args.path + '/*' 29 | if len(glob.glob(path)): 30 | os.system("osascript -e 'display notification \"Push Pocket\" with title \"Pocket+\"'") 31 | os.system('source ' + args.script) 32 | if not args.dev: 33 | # fetch file list again to remove all files there 34 | for f in glob.glob(path): 35 | os.remove(f) 36 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 |
3 | 4 | 5 |

6 | Pocket+ recipe for Calibre 7 |

8 | 9 | Buy Me a Coffee at ko-fi.com 10 | 11 | 12 | [![tag](https://img.shields.io/github/release/mmagnus/Pocket-Plus-Calibre-Plugin.svg)](https://github.com/mmagnus/Pocket-Plus-Calibre-Plugin/releases) PayPal donate button 13 |

14 | This plugin allows users to get their Pocket-ed articles with Calibre and send them as an e-book to their prefered e-book reader. You can schedule this process and every day get the freshest e-book with your Pocket-ed articles! 15 | 16 | There is also experimental version of this plugin: https://github.com/mmagnus/PocketX-Calibre-Plugin 17 | 18 |
19 | 20 | Table of contents: 21 | 22 | * [Settings](#settings) 23 | * [Installation](#installation) 24 | * [Changelog](#changelog) 25 | * [Tips](#tips) 26 | * [Development](#development) 27 | * [Workflow](#workflow) 28 | * [Troubleshooting](#troubleshooting) 29 | 30 | [Pocket](https://getpocket.com/), previously known as Read It Later, is an application and service for managing a reading list of articles from the Internet. The application allows the user to save an article or web page to the cloud for later reading. The article is then sent to the user's Pocket list (synced to all of their devices) for offline reading. Pocket removes clutter from articles and allows the user to adjust text settings for easier reading [Source](https://en.wikipedia.org/wiki/Pocket_%28application%29). 31 | 32 | [Calibre](http://calibre-ebook.com/) is a free and open source e-book library management application developed by users of e-books for users of e-books. The programs also allows users to create own e-books and syncing with a variaty of e-book readers (e.g. Kindle, that's how I got the screenshots below) [Source](https://en.wikipedia.org/wiki/Calibre_%28software%29). Calibre has a plugin management system and .. 33 | 34 | Follow the discussion at 35 | 36 | If you have no idea where to start, take a look here this is an explantion how to use Calibre to send news to your Kindle. 37 | 38 | This is a fork of the original plugin and a merge of a fix by [@dlo9](https://github.com/dlo9). 39 | 40 | I modified the plugin to get an e-book including: 41 | 42 | * the Untagged (more or less as the original version of the plugin) 43 | * your content organized by Pocket tags! 44 | * or, alternatively, your articles organized by the first level domain of the URL. 45 | 46 | Now, you get **Untagged** and **The Sections created based on you Pocket tags**: 47 | 48 |
49 | 50 | Or sections based on **domains** of URsL (great contribution from [@alvaroreig](https://github.com/alvaroreig)): 51 | 52 |
53 | 54 | This is a fork of the original 2011 Calibre ReadItLater plugin. 55 | 56 | # Sections by domain 57 | 58 | If you want to use the sections by domain functionality, you have to 59 | 60 | * active the SECTIONS_BY_DOMAIN flag 61 | * uncomment the tld import. For more details, check https://github.com/mmagnus/Pocket-Plus-Calibre-Plugin/pull/31 62 | 63 | # Settings 64 | 65 | To change settings, click on: 66 | 67 | Fetch news -> Add custom news source -> Pocket (Edit this recipe) 68 | 69 | and edit the Python code. 70 | 71 | ![](doc/settings.png) 72 | 73 | **TAGS** (list of strings or empty list: []) if [] (empty list) then the plugin will connect Pocket and fetch articles based on the configuration of the plugin. 74 | Next, the plugin will get tags of these articles and group them into sections in the final ebook. 75 | If TAGS has elements, e.g., TAGS = ['tag1', 'tag2'] then only these tags will be fetched from Pocket. 76 | 77 | **TAGS_EXCEPTIONS** (list of strings or empty list: []) if [] (empty list) then the plugin will ignore it. 78 | If TAGS_EXCEPTIONS has elements, e.g., TAGS_EXCEPTIONS = ['tag3', 'tag4'] then the articles tagged with this tags will be ignored. 79 | That is, tag3 and tag4 won't appear as sections, and it's articles won't appear in the "Untagged" section. 80 | This variable is meant to be used with TAGS = [], as it doesn’t make any sense to specify a tag both in TAGS and in TAGS_EXCEPTIONS. 81 | 82 | **SECTIONS_BY_DOMAIN** If activated, the articles will be grouped by first level domain. This will override any 83 | tag configuration (that is: TAGS, TAGS_EXCEPTIONS, INCLUDE_UNTAGGED). This is because the recipe ignores duplicated 84 | articles, and therefore an article can't appear under a "real" (pocket) tag and under the fake tag with its domain. 85 | 86 | **SECTIONS_BY_DOMAIN_USING_TLD** you can install TLD and use it to get domains, but this requires installed library in 87 | a way that Calibre will see it (I had a huge problem to get this running @mmagnus), so there is a new 88 | way to get domain based on parsing URL, less sophisticated but more reliable (in my opinion @mmagnus) 89 | 90 | **INCLUDE_UNTAGGED** (True or False) if True then put all fetched and untagged articles in the last section 'Untagged'. 91 | If False then skip these articles and don't create the section 'Untagged'. Bear in mind that if TAGS is populated ( e.g. TAGS = ['tag1', 'tag2']), 92 | INCLUDE_UNTAGED = True and other tags exist in Pokcet (e.g. tag3,tag4) then the Untagged section will include untagged articles 93 | in Pocket AND articles tagged with tag3 and tag4. That behavior can be avoided using TAGS_EXCEPTION 94 | 95 | **ARCHIVE_DOWNLOADED** (True or False) do you want to archive articles after fetching 96 | 97 | **MAX_ARTICLES_PER_FEED** (number) how many articles do you want to fetch for FEED (FEED could be also 98 | considered as TAG, so for each TAG you this value will be applied. 99 | 100 | **SORT_METHOD** ('oldest' or 'newest') way how the articles are sorted 101 | 102 | **OLDEST_ARTICLE** (number) fetch articles added (modified) in Pocket for number of days, 7 will give you articles added/modified in Pocket for the last week 103 | 104 | **TO_PULL** ('all' or 'unread') What articles to pull? unread only or all? 105 | 106 | **TITLE_WITH_TAGS** (True or False) if True will the ebook filename will be like 107 | Pocket: INVEST P2P [Sun, 05 Jan 2020] for many tags this might be to long, if you make a single tag ebook this might be super fun! 108 | 109 | **ALLOW_DUPLICATES** (True or False) if True articles that have multiple tags matching those defined in TAGS are duplicated in each matched tag 110 | Eg.: TAGS = ['tag1','tag2'] then article1 that has both tags will appear in both sections tag1 and tag2. 111 | 112 | # Installation 113 | 114 | * Download files https://github.com/mmagnus/Pocket-Plus-Calibre-Plugin/archive/master.zip 115 | * Go to Calibre, under the "Fetch News" drop down select "Add or edit a custom news source" 116 | * Click "Load Recipe From File" and choose the Pocket.recipe file 117 | * Edit the settings in the windows that will pop up, for example, set up tags (See Settings), click Save, then Close 118 | * Click "Fetch News", the Custom you will find Pocket, change "Schedule for download" if you want 119 | * Click "Download Now" to download now if you want. 120 | * You will be asked to grant access to your Plugin, click on "here" (to be able to click you have to close "Schedule news download" by clicking "OK" or "Cancel" first) 121 | 122 | ![](doc/grant.png) 123 | (if the window disappears, it might be behind the main window of Calibre) 124 | 125 | * When the access is granted, simply click of "Fetch News" again to start! 126 | 127 | If you have any problem read more [at Pocket](https://help.getpocket.com/customer/portal/articles/361724-how-to-configure-calibre-with-pocket) 128 | 129 | Report any issues here: https://github.com/mmagnus/Pocket-Plus-Calibre-Plugin/issues 130 | 131 | # Changelog 132 | 133 | https://github.com/mmagnus/Pocket-Plus-Calibre-Plugin/releases 134 | 135 | * 210731 [v2.7.3] Replace tld with standard Python way (not as good as tld but works without extra package) if SECTIONS_BY_DOMAIN_USING_TLD = False 136 | * 201122 [v2.6.3] With fix from @AkashPatel95 #26 137 | * 200515 [v2.4.0] Auto tags! Automatically group articles into Sections based on Pocket's tags. 138 | * 200514 [v2.3.x] Redesigned tags system, attempt to fix `sort_id` problem, move OLDEST_ARTICLE to the top 139 | * 200104 Incorporate code from David Orchard (@dlo9, https://github.com/dlo9/calibre-recipes) to fix an issue with Pocket Authentication API 140 | * 170503 Download all images from every article by Stefan Wagner (@bompo) 141 | * 170503 Decide what to pull (all vs unread) 142 | * 170502 `Pocket + [Mon, 05 Dec 2016]` 143 | * 160817 Add links to articles 144 | * 160205 Modified version of the plugin to get (1) The latest (more or less as the original version of the plugin) (2) and your content organized by tags! 145 | 146 | # Tips 147 | 148 | 1. Don't forget that you can have multiple modified recipes and schedule each of them independently. 149 | 150 | ![](doc/more.png) 151 | 152 | 2. You can also take a look here if you want someone else to send your articles for you, so you don't have to have access to your computer. Both services were tested by me and they can be recommended! 153 | 154 | - https://p2k.co 155 | - https://www.crofflr.com/#/home 156 | 157 | # Development 158 | Links on development of recipes: 159 | 160 | * https://manual.calibre-ebook.com/news.html 161 | * https://manual.calibre-ebook.com/news_recipe.html 162 | * https://manual.calibre-ebook.com/creating_plugins.html#more-plugin-examples 163 | 164 | The default Calibre plugin is here https://github.com/kovidgoyal/calibre/blob/master/recipes/readitlater.recipe 165 | 166 | calibre-debug --paths --gui-debug ~/Desktop/calibre.txt 167 | 168 | # ToDo 169 | * Properly document how to automate this recipe with ebook-convert and calibre-smtp 170 | * Refactor the code that fetch the tags from pocket. 171 | * lazy load "from tld import fld" 172 | 173 | # Workflow 174 | (some potential workflow that @mmagnus is using at the moment) 175 | 176 | **MY CURRENT WORKFLOW** 177 | 178 | I often (each day or every second day) send new things [1] from my Pocket to my Kindle. When I research some particular topic I use tags to fetch only articles related to the given topic, e.g., “python testing”, and then I have a nice book only on that topic. For each of this type of books you can just copy paste this plugin and change the variables at the top of the file to have a few lines of action in your workflow. 179 | 180 | ```python 181 | # [1] 182 | TAGS = [] 183 | ARCHIVE_DOWNLOADED = True 184 | MAX_ARTICLES_PER_FEED = 100 185 | SORT_METHOD = 'newest' 186 | TO_PULL = 'unread' 187 | 188 | # [2] 189 | TAGS = ['python testing'] 190 | ARCHIVE_DOWNLOADED = True 191 | MAX_ARTICLES_PER_FEED = 100 192 | SORT_METHOD = 'newest' 193 | TO_PULL = 'all' 194 | ``` 195 | 196 | **ANOTHER WORKFLOW** 197 | 198 | The new AUTOTAGS feature has the ability to automatically generate the ebook without explicitly specifying the tags in the recipe. However, in my workflow there are certain articles that I save to Pocket but are not meant to be read in Kindle. They might have a lot of links, or are involved in software development and I want to read them as I code, or any other reason. In that context, the variable TAGS_EXCEPTIONS is very useful. I can use a tag like "nokindle" and guarantee that these articles won't be downloaded (and archived): 199 | 200 | ```python 201 | # [1] 202 | TAGS = [] 203 | TAGS_EXCEPTIONS = ['nokindle'] 204 | ARCHIVE_DOWNLOADED = True 205 | MAX_ARTICLES_PER_FEED = 100 206 | SORT_METHOD = 'newest' 207 | TO_PULL = 'unread' 208 | ``` 209 | 210 | **AUTOMATED [no Calibre]** 211 | 212 | One thing, this plugin needs that the Calibre is open and running. If you want someone else to send your articles for you, you can use these services. This is pretty cool because you don't have to have access to your computer. Both services were tested by me and I can recommended them! At the moment I prefer to be in 100% control of what and when is sent to my Kindle so I use only the plugin from this repo. 213 | 214 | https://p2k.co https://www.crofflr.com/#/home 215 | 216 | **PUSH TO KINDLE NOW AS ONE FILE** 217 | 218 | Sometimes I want to read something NOW or as a single e-book and I don't want to go through Calibre, then I use these plugins to push an article directly to Kindle. 219 | 220 | https://www.fivefilters.org/push-to-kindle/ (for Safari, Chrome, Firefox) https://www.amazon.com/gp/sendtokindle/chrome (for Chrome) 221 | 222 | # Troubleshooting 223 | ## FileNotFoundError: "custom_recipes/Pocket.js" 224 | 225 | I think this is a problem if you haven't granted access (happen to @mmagnus, when I run the bash script on a new machine). Go to Installation and grant access to your Pocket. 226 | 227 | Conversion options changed from defaults: 228 | test: None 229 | 1% Converting input to HTML... 230 | InputFormatPlugin: Recipe Input running 231 | {} 232 | Using custom recipe 233 | Using user agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36 234 | Traceback (most recent call last): 235 | File "runpy.py", line 194, in _run_module_as_main 236 | File "runpy.py", line 87, in _run_code 237 | File "site.py", line 39, in 238 | File "site.py", line 35, in main 239 | File "calibre/ebooks/conversion/cli.py", line 419, in main 240 | File "calibre/ebooks/conversion/plumber.py", line 1111, in run 241 | File "calibre/customize/conversion.py", line 244, in __call__ 242 | File "calibre/ebooks/conversion/plugins/recipe_input.py", line 137, in convert 243 | File "calibre/web/feeds/news.py", line 931, in __init__ 244 | File "", line 518, in get_browser 245 | File "", line 506, in ensure_authorization 246 | File "", line 133, in save 247 | FileNotFoundError: [Errno 2] No such file or directory: '/Users/magnus/Library/Preferences/calibre/custom_recipes/Pocket.js 248 | -------------------------------------------------------------------------------- /Pocket.recipe: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # vim:ft=python tabstop=8 expandtab shiftwidth=4 softtabstop=4 3 | from __future__ import print_function 4 | __version__ = '2.7.4' 5 | 6 | """ 7 | 2.7.4: Fix #46 KeyError: u'resolved_title' 8 | 2.7.3: Replace tld with standard Python way (not as good as tld but works without extra package) if SECTIONS_BY_DOMAIN_USING_TLD = False 9 | 2.7.2: Change default MAX_ARTICLES_PER_FEED to 30 10 | 2.7.1: A fix for import tld 11 | 2.7: Introduce ~/.pocket.py local config system 12 | 2.6.4: SECTIONS_BY_DOMAIN, an excellent implementation from @alvaroreig to make sections 13 | by domains (of URLs) 210322 14 | 2.6.3: With fix from @AkashPatel95 #26 201122 15 | 2.6.2: Fix. Add also removal of H1 to clean up all titles, to insert a new one 16 | 17 | **TAGS** (list of strings or empty list: []) if [] (empty list) then the plugin will connect Pocket and fetch articles based on the configuration of the plugin. 18 | Next, the plugin will get tags of these articles and group them into sections in the final ebook. 19 | If TAGS has elements, e.g., TAGS = ['tag1', 'tag2'] then only these tags will be fetched from Pocket. 20 | 21 | **TAGS_EXCEPTIONS** (list of strings or empty list: []) if [] (empty list) then the plugin will ignore it. 22 | If TAGS_EXCEPTIONS has elements, e.g., TAGS_EXCEPTIONS = ['tag3', 'tag4'] then the articles tagged with this tags will be ignored. 23 | That is, tag3 and tag4 won't appear as sections, and it's articles won't appear in the "Untagged" section. 24 | This variable is meant to be used with TAGS = [], as it doesn’t make any sense to specify a tag both in TAGS and in TAGS_EXCEPTIONS. 25 | 26 | **URL_KEYWORD_EXCEPTIONS** (list of keywords such as, if the URL of the article contains any keyword, then the plugin will ignore the article) 27 | 28 | **SECTIONS_BY_DOMAIN** If activated, the articles will be grouped by first level domain. This will override any 29 | tag configuration (that is: TAGS, TAGS_EXCEPTIONS, INCLUDE_UNTAGGED). This is because the recipe ignores duplicated 30 | articles, and therefore an article can't appear under a "real" (pocket) tag and under the fake tag with its domain. 31 | 32 | **SECTIONS_BY_DOMAIN_USING_TLD** you can install TLD and use it to get domains, but this requires installed library in 33 | a way that Calibre will see it (I had a huge problem to get this running @mmagnus), so there is a new 34 | way to get domain based on parsing URL, less sophisticated but more reliable (in my opinion @mmagnus) 35 | 36 | **INCLUDE_UNTAGGED** (True or False) if True then put all fetched and untagged articles in the last section 'Untagged'. 37 | If False then skip these articles and don't create the section 'Untagged'. Bear in mind that if TAGS is populated ( e.g. TAGS = ['tag1', 'tag2']), 38 | INCLUDE_UNTAGED = True and other tags exist in Pokcet (e.g. tag3,tag4) then the Untagged section will include untagged articles 39 | in Pocket AND articles tagged with tag3 and tag4. That behavior can be avoided using TAGS_EXCEPTION 40 | 41 | **ARCHIVE_DOWNLOADED** (True or False) do you want to archive articles after fetching 42 | 43 | **MAX_ARTICLES_PER_FEED** (number) how many articles do you want to fetch for FEED (FEED could be also 44 | considered as TAG, so for each TAG you this value will be applied. 45 | 46 | **SORT_METHOD** ('oldest' or 'newest') way how the articles are sorted 47 | 48 | **OLDEST_ARTICLE** (number) fetch articles added (modified) in Pocket for number of days, 7 will give you articles added/modified in Pocket for the last week 49 | 50 | **TO_PULL** ('all' or 'unread') What articles to pull? unread only or all? 51 | 52 | **TITLE_WITH_TAGS** (True or False) if True will the ebook filename will be like 53 | Pocket: INVEST P2P [Sun, 05 Jan 2020] for many tags this might be to long, if you make a single tag ebook this might be super fun! 54 | 55 | **ALLOW_DUPLICATES** (True or False) if True articles that have multiple tags matching those defined in TAGS are duplicated in each matched tag 56 | Eg.: TAGS = ['tag1','tag2'] then article1 that has both tags will appear in both sections tag1 and tag2. 57 | """ 58 | # CONFIGURATION ########################################################### 59 | TAGS = [] # [] or ['tag1', 'tag2'] 60 | TAGS_EXCEPTIONS = [] # [] or ['tag3', 'tag4'] 61 | URL_KEYWORD_EXCEPTIONS = [] # [] or ['keyword1', 'keyword2'] 62 | SECTIONS_BY_DOMAIN = False 63 | INCLUDE_UNTAGGED = True 64 | ARCHIVE_DOWNLOADED = True 65 | MAX_ARTICLES_PER_FEED = 30 66 | OLDEST_ARTICLE = 365 67 | SORT_METHOD = 'newest' 68 | SORT_WITHIN_TAG_BY_TITLE = False 69 | TO_PULL = 'unread' 70 | TITLE_WITH_TAGS = False 71 | ALLOW_DUPLICATES = True 72 | USE_GLOBAL_CONFIG = False 73 | # ADV CONFIGURATION ######################################################### 74 | SITE_PACKAGE_PATH = '' 75 | ############################################################################# 76 | # code for configuration set in your home folder 77 | if USE_GLOBAL_CONFIG: 78 | import os 79 | try: 80 | # variables as Python code into ~/.pocket.py to overwrite variables above, e.g.: 81 | # MAX_ARTICLES_PER_FEED = 30 82 | user_path = os.path.expanduser("~") 83 | exec(open(user_path + '/.pocket.py').read()) # python3 84 | except: # FileNotFoundError: noooot perfect! 85 | pass 86 | ############################################################################# 87 | from calibre.constants import config_dir 88 | from calibre.utils.config import JSONConfig 89 | from calibre.web.feeds.news import BasicNewsRecipe 90 | from collections import namedtuple 91 | from os import path 92 | from time import localtime, strftime, time 93 | ############################################################################# 94 | import sys 95 | SITE_PACKAGE_PATH = '' 96 | if SECTIONS_BY_DOMAIN: 97 | from urllib.parse import urlparse 98 | 99 | import errno 100 | import json 101 | import mechanize 102 | import operator 103 | 104 | try: 105 | from urllib.error import HTTPError 106 | except ImportError: 107 | from urllib2 import HTTPError 108 | 109 | __license__ = 'GPL v3' 110 | __copyright__ = '2019, David Orchard' 111 | 112 | 113 | 114 | class PocketConfig: 115 | __file_path = path.join(config_dir, 'custom_recipes', 'Pocket.json') 116 | 117 | class AuthState: 118 | FirstRun = 1 119 | Authorizing = 2 120 | Authorized = 3 121 | 122 | def __init__(self, state = AuthState.FirstRun, token = None, user = None): 123 | # Default values 124 | self.state = state 125 | self.token = token 126 | self.user = user 127 | 128 | @staticmethod 129 | def from_file(): 130 | config = PocketConfig() 131 | config.load() 132 | return config 133 | 134 | def load(self): 135 | try: 136 | with open(self.__file_path) as config: 137 | config = json.load(config) 138 | 139 | if isinstance(config, dict): 140 | for key in self.__dict__.keys(): 141 | if config[key]: 142 | setattr(self, key, config[key]) 143 | except IOError as e: 144 | # File not found 145 | if e.errno != errno.ENOENT: 146 | raise e 147 | 148 | def save(self): 149 | with open(self.__file_path, 'w') as config: 150 | json.dump(self.__dict__, config) 151 | 152 | 153 | class Pocket(BasicNewsRecipe): 154 | config = PocketConfig.from_file() 155 | 156 | __author__ = 'David Orchard' 157 | description = ''' 158 | 159 | Modified by Marcin Magnus. 160 | 161 | Fetches articles saved with Pocket and archives them.
162 | ''' + (''' 163 | Click here 164 | to disconnect Calibre from the Pocket account "{}". 165 | '''.format(config.user) if config.user else ''' 166 | Run 'Fetch News' with this source scheduled to initiate authentication with Pocket. 167 | ''') 168 | publisher = 'Pocket.com' 169 | category = 'info, custom, Pocket' 170 | 171 | # User-configurable settings ----------------------------------------------- 172 | tagsList = TAGS 173 | oldest_article = OLDEST_ARTICLE 174 | max_articles_per_feed = MAX_ARTICLES_PER_FEED 175 | archive_downloaded = ARCHIVE_DOWNLOADED 176 | include_untagged = INCLUDE_UNTAGGED 177 | series_name = 'Pocket' 178 | sort_method = SORT_METHOD 179 | to_pull = TO_PULL 180 | 181 | publication_type = 'magazine' 182 | title = "Pocket" 183 | # timefmt = '' # uncomment to remove date from the filenames, if commented then you will get something like `Pocket [Wed, 13 May 2020]` 184 | masthead_url = "https://github.com/mmagnus/Pocket-Plus-Calibre-Plugin/raw/master/doc/masthead.png" 185 | # will make square cover; this will replace text and cover of the default 186 | # cover_url = "https://github.com/mmagnus/Pocket-Plus-Calibre-Plugin/raw/master/doc/cover.png" 187 | # -------------------------------------------------------------------------- 188 | 189 | # Inherited developer settings 190 | auto_cleanup = True 191 | no_stylesheets = True 192 | use_embedded_content = False 193 | if ALLOW_DUPLICATES: 194 | ignore_duplicate_articles = {} 195 | else: 196 | ignore_duplicate_articles = {'url'} 197 | 198 | # Custom developer settings 199 | consumer_key = '87006-2ecad30a91903f54baf0ee05' 200 | redirect_uri = 'https://calibre-ebook.com/' 201 | base_url = 'https://app.getpocket.com' 202 | to_archive = [] 203 | 204 | simultaneous_downloads = 10 205 | 206 | extra_css = '.touchscreen_navbar {display: none;}' 207 | extra_css = '.calibre_navbar { visibility: hidden; }' 208 | # TITLE_WITH_TAGS 209 | tags_title = ' ' 210 | if tagsList: 211 | if tagsList[-1] != '' and TITLE_WITH_TAGS: # ugly hack 212 | tags_title = ':' + ' '.join(tagsList).upper() + ' ' 213 | 214 | def first_run(self): 215 | request = mechanize.Request("https://getpocket.com/v3/oauth/request", 216 | (u'{{' 217 | '"consumer_key":"{0}",' 218 | '"redirect_uri":"{1}"' 219 | '}}').format( 220 | self.consumer_key, 221 | self.redirect_uri 222 | ), 223 | headers = { 224 | 'Content-Type': 'application/json; charset=UTF8', 225 | 'X-Accept': 'application/json' 226 | } 227 | ) 228 | response = self.browser.open(request) 229 | response = json.load(response) 230 | self.config = PocketConfig( 231 | state = PocketConfig.AuthState.Authorizing, 232 | token = response['code'] 233 | ) 234 | 235 | def authorize(self): 236 | assert self.config.state == PocketConfig.AuthState.Authorizing, "Authorization process not yet begun" 237 | assert self.config.token, "No request token" 238 | request = mechanize.Request("https://getpocket.com/v3/oauth/authorize", 239 | (u'{{' 240 | '"consumer_key":"{0}",' 241 | '"code":"{1}"' 242 | '}}').format( 243 | self.consumer_key, 244 | self.config.token 245 | ), 246 | headers = { 247 | 'Content-Type': 'application/json; charset=UTF8', 248 | 'X-Accept': 'application/json' 249 | } 250 | ) 251 | try: 252 | response = self.browser.open(request) 253 | response = json.load(response) 254 | self.config = PocketConfig( 255 | state = PocketConfig.AuthState.Authorized, 256 | token = response["access_token"], 257 | user = response["username"], 258 | ) 259 | except HTTPError as e: 260 | if e.code == 403: 261 | # The code has already been used, or the user denied access 262 | self.reauthorize() 263 | raise e 264 | 265 | def parse_index(self): 266 | assert self.config.state == PocketConfig.AuthState.Authorized, "Not yet authorized" 267 | assert self.config.token, "No access token" 268 | 269 | articles = [] 270 | section_dict = {} #dictionary with the sections and its articles. the sections 271 | #cant be domains or tags depending on SECTIONS_BY_DOMAIN 272 | 273 | ############ GET ALL ITEMS ############# 274 | # get every item and iterate them. Build the section_dict 275 | # with tags or domains as keys, depending on SECTIONS_BY_DOMAIN 276 | request = mechanize.Request("https://getpocket.com/v3/get", 277 | (u'{{' 278 | '"consumer_key":"{0}",' 279 | '"access_token":"{1}",' 280 | '"count":"{2}",' 281 | '"since":"{3}",' 282 | '"state":"{5}",' 283 | '"detailType":"complete",' 284 | '"sort":"{4}"' '}}').format( 285 | self.consumer_key, 286 | self.config.token, 287 | self.max_articles_per_feed * 1000, # something "unlimited" 288 | int(time()) - 86400 * self.oldest_article, 289 | self.sort_method, 290 | self.to_pull, 291 | ), 292 | headers = { 293 | 'Content-Type': 'application/json; charset=UTF8', 294 | 'X-Accept': 'application/json' 295 | } 296 | ) 297 | 298 | try: 299 | response = self.browser.open(request) 300 | response = json.load(response) 301 | except HTTPError as e: 302 | if e.code == 401: 303 | # Calibre access has been removed 304 | self.reauthorize() 305 | raise e 306 | 307 | if not response['list']: 308 | self.abort_recipe_processing('No unread articles in the Pocket account "{}"'.format(self.config.user)) 309 | else: 310 | for item in dict(response['list']): 311 | if response['list'][item]['status'] == '2': 312 | del response['list'][item] 313 | # If the URL contains any URL_KEYWORD_EXCEPTIONS, ignore article 314 | elif any(pattern in response['list'][item]['given_url'] for pattern in URL_KEYWORD_EXCEPTIONS): 315 | print("Ignoring article due to keyword patterns:" + response['list'][item]['given_url']) 316 | del response['list'][item] 317 | elif SECTIONS_BY_DOMAIN: 318 | # the keys of section_dict will be domains 319 | # Extract domain from the URL 320 | domain = urlparse(response['list'][item]['resolved_url']).netloc.replace('www.', '') 321 | 322 | url = response['list'][item]['resolved_url'] 323 | print('>> url', url, file=sys.stderr) 324 | print('>>> domain', domain, file=sys.stderr) 325 | 326 | # Add the article under its domain 327 | if domain not in section_dict: 328 | section_dict[domain] = [item] 329 | else: 330 | section_dict[domain].append(item) 331 | else: 332 | # the keys of section_dict will be tags 333 | try: 334 | tag = list(response['list'][item]['tags'].keys()) 335 | except KeyError: 336 | if INCLUDE_UNTAGGED: 337 | tag = ['Untagged'] 338 | else: 339 | tag = None 340 | 341 | # tag could be None if article untagged and INCLUDE_UNTAGGED=False 342 | if tag and not any(tagcheck in TAGS_EXCEPTIONS for tagcheck in tag): 343 | if (len(TAGS) == 0): 344 | #autotags enabled, insert tag and item 345 | for tagcheck in tag: 346 | if tagcheck not in section_dict: 347 | section_dict[tagcheck] = [item] 348 | else: 349 | section_dict[tagcheck].append(item) 350 | else: 351 | # explicit tags, check that 352 | # the tag belongs to the explicit array TAGS 353 | # OR is is an untagged article and INCLUDE_UNTAGGED=True 354 | for tagcheck in tag: 355 | if tagcheck in TAGS or (tagcheck == 'Untagged' and INCLUDE_UNTAGGED): 356 | if tagcheck not in section_dict: 357 | section_dict[tagcheck] = [item] 358 | else: 359 | section_dict[tagcheck].append(item) 360 | 361 | ############ APPEND ARTS FOR EACH TAG/DOMAIN ############# 362 | # At this point the section_dict is completed, either with 363 | # domains or with tags 364 | 365 | for section in section_dict: 366 | arts = [] 367 | for item in section_dict.get(section): 368 | try: 369 | title = response['list'][item]['resolved_title'] 370 | except KeyError: 371 | title = 'error: title' 372 | try: 373 | url = response['list'][item]['resolved_url'] 374 | except KeyError: 375 | url = 'error: url' 376 | try: 377 | desc = response['list'][item]['excerpt'] 378 | except KeyError: 379 | desc = 'error: description' 380 | arts.append({ 381 | 'title': title, 382 | 'url': url, 383 | 'date': response['list'][item]['time_added'], 384 | 'description': desc,}) 385 | 386 | if ( 387 | self.archive_downloaded 388 | and response['list'][item]['item_id'] not in self.to_archive 389 | ): 390 | self.to_archive.append(response['list'][item]['item_id']) 391 | 392 | 393 | if not SECTIONS_BY_DOMAIN and SORT_WITHIN_TAG_BY_TITLE: 394 | arts = sorted(arts, key = lambda i: i['title']) 395 | 396 | if arts: 397 | articles.append((section, arts)) 398 | 399 | if not articles: 400 | self.abort_recipe_processing('No articles in the Pocket account %s to download' % (self.config.user)) #, ' '.join(self.tags))) \n[tags: %s] 401 | return articles 402 | 403 | 404 | def reauthorize(self): 405 | self.config = PocketConfig(); 406 | self.ensure_authorization() 407 | 408 | def ensure_authorization(self): 409 | if self.config.state is PocketConfig.AuthState.FirstRun: 410 | self.first_run() 411 | self.config.save() 412 | self.abort_recipe_processing(''' 413 | Calibre must be granted access to your Pocket account. Please click 414 | here 415 | to authenticate via a browser, and then re-fetch the news. 416 | '''.format(self.config.token, self.redirect_uri)) 417 | elif self.config.state is PocketConfig.AuthState.Authorizing: 418 | self.authorize() 419 | self.config.save() 420 | 421 | def get_browser(self, *args, **kwargs): 422 | self.browser = BasicNewsRecipe.get_browser(self) 423 | self.ensure_authorization() 424 | return self.browser 425 | 426 | def archive(self): 427 | assert self.config.state == PocketConfig.AuthState.Authorized, "Not yet authorized" 428 | assert self.config.token, "No access token" 429 | 430 | if not self.to_archive: 431 | return 432 | 433 | archived_time = int(time()) 434 | request = mechanize.Request("https://getpocket.com/v3/send", 435 | (u'{{' 436 | '"consumer_key":"{0}",' 437 | '"access_token":"{1}",' 438 | '"actions":{2}' 439 | '}}').format( 440 | self.consumer_key, 441 | self.config.token, 442 | json.dumps([{ 443 | 'action': 'archive', 444 | 'item_id': item_id, 445 | 'time': archived_time, 446 | } for item_id in self.to_archive]) 447 | ), 448 | headers = { 449 | 'Content-Type': 'application/json; charset=UTF8', 450 | 'X-Accept': 'application/json' 451 | } 452 | ) 453 | response = self.browser.open(request) 454 | 455 | def cleanup(self): 456 | # If we're in another state, then downloading didn't complete 457 | # (e.g. reauthorization needed) so there is no archiving to do 458 | if self.config.state == PocketConfig.AuthState.Authorized: 459 | self.archive() 460 | 461 | # TODO: This works with EPUB, but not mobi/azw3 462 | # BUG: https://bugs.launchpad.net/calibre/+bug/1838486 463 | def postprocess_book(self, oeb, opts, log): 464 | oeb.metadata.add('series', self.series_name) 465 | 466 | def postprocess_html(self, soup, first): 467 | title = soup.find('title').text # get title 468 | 469 | h1s = soup.findAll('h1') # get all h1 headers 470 | for h1 in h1s: 471 | if title in h1.text: 472 | h1 = h1.clear() # clean this tag, so the h1 will be there only 473 | 474 | h2s = soup.findAll('h2') # get all h2 headers 475 | for h2 in h2s: 476 | if title in h2.text: 477 | h2 = h2.clear() # clean this tag, so the h1 will be there only 478 | 479 | body = soup.find('body') 480 | new_tag = soup.new_tag('h1') 481 | new_tag.append(title) 482 | body.insert(0, new_tag) 483 | # print(soup.prettify(), file=sys.stderr) 484 | return soup 485 | 486 | def default_cover(self, cover_file): 487 | """ 488 | Create a generic cover for recipes that don't have a cover 489 | This override adds time to the cover 490 | """ 491 | try: 492 | from calibre.ebooks import calibre_cover 493 | title = self.title if isinstance(self.title, unicode) else \ 494 | self.title.decode('utf-8', 'replace') 495 | # print('>> title', title, file=sys.stderr) 496 | date = strftime(self.timefmt) 497 | time = strftime('%a %d %b %Y %-H:%M') 498 | img_data = calibre_cover(title, date, time) 499 | cover_file.write(img_data) 500 | cover_file.flush() 501 | except: 502 | self.log.exception('Failed to generate default cover') 503 | return False 504 | return True 505 | --------------------------------------------------------------------------------