├── .gitignore ├── conf_sample.py ├── phantom └── rasterize.js ├── README.md └── PinboardEvernoteSync.py /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | conf.py* 3 | -------------------------------------------------------------------------------- /conf_sample.py: -------------------------------------------------------------------------------- 1 | # Copy this file to conf.py and replace values 2 | # with the real data to enable unit tests. 3 | 4 | # Get your pinboard token here: https://pinboard.in/settings/password 5 | # ... but somehow it's not working! Please use your username/password instead. #FIXME 6 | # pinboard_token = 'bob:CDE72B0D1090404C2609' 7 | pinboard_username = 'bob' 8 | pinboard_pass = 'mypassword' 9 | # Get your evernote token here: https://www.evernote.com/api/DeveloperToken.action 10 | evernote_token = 'S=s1:U=6154a:E=144bc355fbc:C=13d648433bd:P=1cd:A=en-devtoken:V=2:H=2d23742aaed9c17b9b2adf7bac62d64b' 11 | # If you have a readability parser token (get it here: http://www.readability.com/account/api) 12 | # we will use it to parse a nice excerpt of the page into evernote whenever needed 13 | readability_token = '2c63eb0esds658f6203fc78c4aaaa6b9828a59' -------------------------------------------------------------------------------- /phantom/rasterize.js: -------------------------------------------------------------------------------- 1 | // from https://raw.github.com/ariya/phantomjs/master/examples/rasterize.js 2 | // LICENSED UNDER: https://github.com/ariya/phantomjs/blob/master/LICENSE.BSD 3 | 4 | var page = require('webpage').create(), 5 | system = require('system'), 6 | address, output, size; 7 | 8 | if (system.args.length < 3 || system.args.length > 5) { 9 | console.log('Usage: rasterize.js URL filename [paperwidth*paperheight|paperformat] [zoom]'); 10 | console.log(' paper (pdf output) examples: "5in*7.5in", "10cm*20cm", "A4", "Letter"'); 11 | phantom.exit(1); 12 | } else { 13 | address = system.args[1]; 14 | output = system.args[2]; 15 | page.viewportSize = { width: 800, height: 600 }; 16 | if (system.args.length > 3 && system.args[2].substr(-4) === ".pdf") { 17 | size = system.args[3].split('*'); 18 | page.paperSize = size.length === 2 ? { width: size[0], height: size[1], margin: '0px' } 19 | : { format: system.args[3], orientation: 'portrait', margin: '1cm' }; 20 | } 21 | if (system.args.length > 4) { 22 | page.zoomFactor = system.args[4]; 23 | } 24 | page.open(address, function (status) { 25 | if (status !== 'success') { 26 | console.log('Unable to load the address!'); 27 | phantom.exit(); 28 | } else { 29 | window.setTimeout(function () { 30 | page.render(output); 31 | phantom.exit(); 32 | }, 200); 33 | } 34 | }); 35 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Pinboard-Evernote Sync 2 | 3 | ## What is it? 4 | 5 | Sync Pinboard and Evernote bookmarks. 6 | 7 | ## How it works 8 | 9 | The script will: 10 | 11 | 1. retrieve all your bookmarks from pinboard 12 | 2. if there is no entry in Evernote (in any notebook) with the same URL, create one in the notebook "Bookmarks" 13 | * The notebook will be created if you don't have it yet 14 | 3. Add all your pinboard bookmarks into evernote 15 | * If you have lynx installed, the script will save a text dump of the page into Evernote 16 | * If you have phantom.JS installed, the script will also save a screenshot of the page into Evernote 17 | 4. Add all your evernote bookmarks into Pinboard 18 | 19 | ## How to use it 20 | 21 | 1. Get your pinboard API Token here: https://pinboard.in/settings/password 22 | 2. Get an Evernote developer token at: https://www.evernote.com/api/DeveloperToken.action 23 | 3. Create a conf.py file with your credentials (tokens or passwords). The file conf_sample.py will show you how 24 | 4. install https://github.com/evernote/evernote-sdk-python 25 | 5. install https://github.com/mgan59/python-pinboard 26 | 6. [Optional] install lynx, if you want a nice text dump of the pages in your bookmark note 27 | 7. [Optional] get a Readability parser key if you want to use it instead of lynx 28 | 8. [Optional] install phantomJS from http://phantomjs.org/ if you want to keep a screenshot of each bookmarked site in evernote 29 | 9. Run PinboardEvernoteSync.py whenever you feel like syncing your accounts 30 | 10. goto 9 31 | 32 | ## TODO 33 | 34 | * Sync tags 35 | * Add an option to choose the Evernote notebook. Currently only uses the "Bookmarks" default 36 | -------------------------------------------------------------------------------- /PinboardEvernoteSync.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """PinboardEvernoteSync.py 5 | 6 | Pinboard is great. Evernote too. 7 | 8 | Make love, not war, use both and keep them in sync. 9 | 10 | """ 11 | 12 | __version__ = "1.0" 13 | __license__ = "WTFPL" 14 | __copyright__ = "Copyright 2013-2015, Olivier Thereaux" 15 | __author__ = "Olivier Thereaux " 16 | 17 | import os 18 | import socket 19 | import subprocess 20 | import re 21 | import sys 22 | import pinboard 23 | import time 24 | from datetime import datetime, date 25 | import hashlib 26 | import binascii 27 | import cgi 28 | import urllib 29 | import urllib2 30 | import json 31 | import hashlib 32 | import binascii 33 | import evernote.edam.userstore.constants as UserStoreConstants 34 | import evernote.edam.limits.constants as EvernoteLimits 35 | from evernote.edam.notestore import NoteStore 36 | import evernote.edam.type.ttypes as Types 37 | import evernote.edam.error.ttypes as Errors 38 | from evernote.api.client import EvernoteClient 39 | 40 | 41 | # initialise clients with tokens 42 | pinboard_username = None 43 | pinboard_pass = None 44 | evernote_token = None 45 | readability_token = None 46 | from conf import * 47 | 48 | class Usage(Exception): 49 | def __init__(self, msg): 50 | self.msg = msg 51 | 52 | 53 | def canhaslynx(): 54 | # test whether lynx is installed 55 | # strongly inspired by http://stackoverflow.com/questions/377017/test-if-executable-exists-in-python 56 | lynx_exe = None 57 | for path in os.environ["PATH"].split(os.pathsep): 58 | path = path.strip('"') 59 | exe_file = os.path.join(path, "lynx") 60 | if os.path.isfile(exe_file) and os.access(exe_file, os.X_OK): 61 | lynx_exe = exe_file 62 | return lynx_exe 63 | 64 | def canhasphantom(): 65 | # test whether phantomJS is installed 66 | phantom_exe = None 67 | for path in os.environ["PATH"].split(os.pathsep): 68 | path = path.strip('"') 69 | exe_file = os.path.join(path, "phantomjs") 70 | if os.path.isfile(exe_file) and os.access(exe_file, os.X_OK): 71 | phantom_exe = exe_file 72 | return phantom_exe 73 | 74 | def clean_href(href): 75 | clean_href = href 76 | try: #python 3.3 and above 77 | from shlex import quote 78 | clean_href = quote(clean_href) 79 | except: 80 | _find_unsafe = re.compile(r'[^\w@%+=:,./-]').search 81 | if _find_unsafe(clean_href): 82 | clean_href = "'" + clean_href.replace("'", "'\"'\"'") + "'" 83 | return clean_href 84 | 85 | def savescreenshotfromphantom(phantom_exe, href): 86 | href = clean_href(href) 87 | phantom_args = [phantom_exe, "phantom/rasterize.js", href, "screenshot.png"] 88 | try: 89 | subprocess.check_output(phantom_args, stderr=open('exceptions.txt', 'w')) 90 | except: 91 | pass 92 | return None 93 | 94 | def getsummaryfromreadability(href, readability_token): 95 | readability_query = 'https://readability.com/api/content/v1/parser?url='+href+'&token='+readability_token 96 | readable_text = urllib.urlopen(readability_query).read() 97 | try: 98 | return '

'+cgi.escape(json.loads(readable_text)['excerpt']).encode("utf-8")+'

' 99 | except: 100 | return '' 101 | 102 | def getsummaryfromlynx(lynx_exe, href): 103 | href = clean_href(href) 104 | lynx_args = [lynx_exe,'-dump','-display_charset=utf-8','-assume_charset=utf-8','-nomargins','-hiddenlinks=ignore','-nonumbers',href] 105 | try: 106 | lynx_dump = cgi.escape(subprocess.check_output(lynx_args, stderr=open('/dev/null', 'w'))) 107 | return '
'+lynx_dump+'
' 108 | except: 109 | return '' 110 | 111 | def save2evernote(pinbookmark, note_store, bookmark_guid, lynx_exe, phantom_exe, readability_token): 112 | note = Types.Note() 113 | # ourNote.notebookGuid = parentNotebook.guid 114 | cleantitle = cgi.escape(pinbookmark["description"]) 115 | # According to https://dev.evernote.com/doc/reference/Types.html#Struct_Note 116 | # The subject of the note. Can't begin or end with a space. Doing some zealous replacement which seems to work 117 | cleantitle = cleantitle.replace(u"\xa0", u" ").encode("utf-8") 118 | cleantitle = re.sub(r"\s+", " ", cleantitle) 119 | cleantitle = re.sub(r"\t+", " ", cleantitle) 120 | cleantitle = re.sub(r"\s*$", "", cleantitle) 121 | cleantitle = re.sub(r"^\s*", "", cleantitle) 122 | # as per https://dev.evernote.com/doc/reference/Limits.html 123 | re.sub(r"[^\p{Cc}\p{Z}]([^\p{Cc}\p{Zl}\p{Zp}]{0,253}[^\p{Cc}\p{Z}])?$", "", cleantitle) 124 | if len(cleantitle) >= EvernoteLimits.EDAM_NOTE_TITLE_LEN_MAX: 125 | cleantitle = cleantitle[:250]+"..." 126 | print "Note title: %s" % cleantitle 127 | note.title = cleantitle 128 | attributes = Types.NoteAttributes(sourceURL = pinbookmark["href"]) 129 | note.attributes = attributes 130 | note.notebookGuid = bookmark_guid 131 | page_dump = '' 132 | if lynx_exe: 133 | # print "Can Has Lynx" 134 | page_dump = getsummaryfromlynx(lynx_exe, pinbookmark["href"]) 135 | if page_dump == "": #lynx failed, or no present. Try readability 136 | # print "Lynx no worky" 137 | if readability_token != None: 138 | # print "Using Readability instead" 139 | page_dump = getsummaryfromreadability(pinbookmark["href"], readability_token) 140 | resource_embed = "" 141 | if phantom_exe: 142 | savescreenshotfromphantom(phantom_exe, pinbookmark['href']) 143 | try: 144 | image = open('screenshot.png', 'rb').read() 145 | except: 146 | image = None 147 | if image: 148 | md5 = hashlib.md5() 149 | md5.update(image) 150 | hash = md5.digest() 151 | 152 | data = Types.Data() 153 | data.size = len(image) 154 | data.bodyHash = hash 155 | data.body = image 156 | 157 | resource = Types.Resource() 158 | resource.mime = 'image/png' 159 | resource.data = data 160 | note.resources = [resource] 161 | hash_hex = binascii.hexlify(hash) 162 | resource_embed = '' 163 | 164 | if resource_embed != "": 165 | os.remove('screenshot.png') 166 | resource_embed += """ 167 |
168 | """ 169 | note.content = ''' 170 | 171 | 172 | %s 173 |
174 | %s 175 | %s 176 | ''' % (cgi.escape(pinbookmark["extended"]).encode("utf-8"), resource_embed, page_dump) 177 | note.created = 1000*int(time.mktime(pinbookmark[u'time_parsed'])) 178 | note.updated = 1000*int(time.mktime(pinbookmark[u'time_parsed'])) 179 | return note_store.createNote(note) 180 | 181 | 182 | def main(): 183 | # Checking if we have Lynx / phantomJS for optional features 184 | #Is Lynx installed? 185 | print "Can we use lynx to extract text?" 186 | lynx_exe = canhaslynx() 187 | if lynx_exe: 188 | print "yes" 189 | else: 190 | print "no" 191 | #Is phantomJS installed? 192 | print "Can we use phantomJS to extract screenshots?" 193 | phantom_exe = canhasphantom() 194 | if phantom_exe: 195 | print "yes" 196 | else: 197 | print "no" 198 | 199 | print "connecting to Evernote…" 200 | client = EvernoteClient(token=evernote_token, sandbox=False) 201 | user_store = client.get_user_store() 202 | try: 203 | note_store = client.get_note_store() 204 | except Errors.EDAMSystemException, e: 205 | if e.errorCode == 19: # rate limit reached 206 | print "Rate limit reached" 207 | print "Retry your request in %d seconds" % e.rateLimitDuration 208 | time.sleep(e.rateLimitDuration+1) 209 | note_store = client.get_note_store() 210 | 211 | # look for notebook "Bookmarks". If there is none, create it 212 | notebooks = note_store.listNotebooks() 213 | bookmark_notebook_guid = None 214 | for notebook in notebooks: 215 | if notebook.name == "Bookmarks": 216 | bookmark_notebook_guid = notebook.guid 217 | if bookmark_notebook_guid == None: 218 | print "Creating Evernote Notebook…" 219 | new_notebook = Types.Notebook() 220 | new_notebook.name = "Bookmarks" 221 | bookmark_notebook_guid = note_store.createNotebook(new_notebook).guid 222 | 223 | # retrieve all ids and URIs in the Evernote notebook 224 | print "Retrieving all Evernote bookmarks…" 225 | filter = NoteStore.NoteFilter(notebookGuid = bookmark_notebook_guid) 226 | spec = NoteStore.NotesMetadataResultSpec() 227 | spec.includeTitle = True 228 | spec.includeAttributes = True 229 | spec.includeCreated = True 230 | note_offset = 0 231 | keep_looking = True 232 | all_evernote_bookmarks = list() 233 | while keep_looking: 234 | found_notes = note_store.findNotesMetadata(filter, note_offset, 250, spec) 235 | # oh FFS Evernote, can you make your API even less user-friendly? 236 | all_evernote_bookmarks += found_notes.notes 237 | if found_notes.totalNotes == len(all_evernote_bookmarks): 238 | keep_looking = False 239 | else: 240 | note_offset = len(all_evernote_bookmarks) 241 | time.sleep(1) 242 | print "Success. Retrieved %d Bookmarks." % len(all_evernote_bookmarks) 243 | all_evernote_bookmarks_map = dict() 244 | all_evernote_uris = list() 245 | for note in all_evernote_bookmarks: 246 | if note.attributes.sourceURL != None: 247 | all_evernote_bookmarks_map[note.attributes.sourceURL] = note 248 | all_evernote_uris.append(note.attributes.sourceURL) 249 | all_evernote_uris.reverse() 250 | 251 | # Now we do the same dance with Pinboard 252 | p = pinboard.open(token=pinboard_token) 253 | print "connecting to Pinboard…" 254 | p = pinboard.open(pinboard_username, pinboard_pass) 255 | print "Retrieving all Pinboard bookmarks…" 256 | all_pinboard_posts = p.posts() 257 | print "Success. Retrieved %d Bookmarks." % len(all_pinboard_posts) 258 | all_pinboard_uris = list() 259 | all_pinboard_posts_map = dict() 260 | for post in all_pinboard_posts: 261 | if post["href"] != None: 262 | all_pinboard_uris.append(post["href"]) 263 | all_pinboard_posts_map[post["href"]] = post 264 | all_pinboard_uris.reverse() 265 | 266 | 267 | # now we compare and prep the sync 268 | missing_from_evernote = list() 269 | missing_from_pinboard = list() 270 | for evernote_uri in all_evernote_uris: 271 | if evernote_uri not in all_pinboard_uris: 272 | missing_from_pinboard.append(evernote_uri) 273 | for pinboard_uri in all_pinboard_uris: 274 | if pinboard_uri not in all_evernote_uris: 275 | missing_from_evernote.append(pinboard_uri) 276 | 277 | if len(missing_from_pinboard): 278 | print "Processing %d bookmarks missing from Pinboard:" % len(missing_from_pinboard) 279 | post_counter = 1 280 | post_counter_total = len(missing_from_pinboard) 281 | for evernote_uri in missing_from_pinboard: 282 | note = all_evernote_bookmarks_map[evernote_uri] 283 | print post_counter, "/", post_counter_total, ": ", evernote_uri 284 | 285 | # remember? p is our pinboard API object 286 | p.add(url=evernote_uri, description=note.title, date = datetime.fromtimestamp(note.created/1000)) 287 | time.sleep(1) 288 | post_counter += 1 289 | print 290 | 291 | if len(missing_from_evernote): 292 | print "Processing %d bookmarks missing from Evernote…" % len(missing_from_evernote) 293 | post_counter = 1 294 | post_counter_total = len(missing_from_evernote) 295 | for pinboard_uri in missing_from_evernote: 296 | post = all_pinboard_posts_map[pinboard_uri] 297 | print post_counter, "/", post_counter_total, ": ", post['href'] 298 | 299 | note_filter = NoteStore.NoteFilter(words='sourceURL:"'+post["href"].encode("utf-8")+'"') 300 | try: 301 | existing_notes = note_store.findNotes(note_filter, 0, 1) 302 | except socket.error: 303 | print "Evernote server timeout, waiting to re-try" 304 | time.sleep(5) 305 | existing_notes = note_store.findNotes(note_filter, 0, 1) 306 | except Errors.EDAMSystemException, e: 307 | if e.errorCode == 19: # limit rate reached 308 | print "Rate limit reached" 309 | print "Retry your request in %d seconds" % e.rateLimitDuration 310 | time.sleep(e.rateLimitDuration+1) 311 | existing_notes = note_store.findNotes(note_filter, 0, 1) 312 | # if this bombs again, let it crash. for now. 313 | if len(existing_notes.notes) > 0: 314 | print "Skipping post: already in Evernote" 315 | pass 316 | else: 317 | try: 318 | created_note = save2evernote(post, note_store, bookmark_notebook_guid, lynx_exe=lynx_exe, phantom_exe=phantom_exe, readability_token=readability_token) 319 | print "Successfully created a new note with GUID: ", created_note.guid 320 | except Exception,e: 321 | print "Could not create a note. Error was: ", e 322 | pass 323 | post_counter += 1 324 | print 325 | 326 | 327 | if __name__ == "__main__": 328 | sys.exit(main()) 329 | --------------------------------------------------------------------------------