├── .gitignore ├── LICENSE.txt ├── README.md ├── app.py ├── db.py ├── enote ├── __init__.py ├── test.py └── util.py ├── environment.yaml ├── github ├── __init__.py ├── test.py └── util.py ├── images ├── demo-subplots.gif ├── evernote-token.png ├── github-token-created.png └── github-token.png ├── requirements.txt ├── settings.py ├── setup.py └── web ├── __init__.py ├── test.py └── util.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | .static_storage/ 56 | .media/ 57 | local_settings.py 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | 106 | # pycharm 107 | .idea 108 | 109 | # credentials 110 | secret.py -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Meng Lee @ b98705001@gmail.com 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Gist-Evernote 2 | 3 | A Python application that sync your Github Gists and save them to your [Evernote](https://evernote.com/) notebook as screenshots. With the app, you can search your work (Jupyter notebooks, Markdown files, Python scripts, etc) all in one place with pretty result. :sunglasses: :notebook: 4 | 5 | ## Demo 6 | A simple use case for search notebooks including `subplot`: 7 | 8 | ![demo-subplots](images/demo-subplots.gif) 9 | 10 | ## Background 11 | As a heavy Evernote user and a data scientist, I wondered won't it be wonderful to be able to search all my [Jupyter notebooks](http://jupyter.org/) directly inside Evernote. Thanks to the [OCR technology](https://help.evernote.com/hc/en-us/articles/208314518-How-Evernote-makes-text-inside-images-searchable) built by Evernote, this become possible. Without further ado, try it yourself :wink: 12 | 13 | ## Getting Started 14 | This project is written in Python and tested only on macOS currently. 15 | To start synchronizing your gist to Evernote, follow the instruction below to setup the application. 16 | 17 | ### Prerequisites 18 | 19 | #### Python 2.7 20 | 21 | ```bash 22 | python --version 23 | Python 2.7.14 :: Anaconda, Inc. 24 | ``` 25 | 26 | This is due to [Evernote API](https://github.com/evernote/evernote-sdk-python)'s limited support of 27 | [Python3](https://github.com/evernote/evernote-sdk-python3), but will try to migrate in near future. 28 | 29 | #### [Chrome browser](https://www.google.com.tw/chrome/browser/desktop/index.html) 30 | 31 | #### [Chrome driver](https://sites.google.com/a/chromium.org/chromedriver/downloads) 32 | Latest version is recommended. After downloading, put the driver under path like `/usr/local/bin`. 33 | 34 | Check whether the driver is in the path: 35 | ```bash 36 | ls /usr/local/bin | grep chromedriver 37 | chromedriver 38 | ``` 39 | 40 | Make sure the path where the driver locate is accessible via `$PATH` environment variable: 41 | 42 | ```bash 43 | echo $PATH 44 | ... :/usr/local/bin: ... 45 | ``` 46 | 47 | If the path is not included in `$PATH`, you can add the path to `$PATH` temporarily: 48 | ```bash 49 | export PATH=$PATH:/usr/local/bin 50 | ``` 51 | 52 | #### brew and dependencies 53 | 54 | ```commandline 55 | brew install portmidi pygobject pkg-config cairo gobject-introspection 56 | ``` 57 | 58 | #### path setting 59 | 60 | Make sure to include necessary path in `~/.bash_profile`: 61 | 62 | ```text 63 | export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:/usr/local/lib:/usr/local/opt/libffi/lib/pkgconfig 64 | ``` 65 | 66 | #### [Github Personal Access Token](https://github.com/settings/tokens) 67 | To grant the app to retrieve your gists, 68 | you have to give the app a access token. You can read more about the token [here](https://github.com/blog/1509-personal-api-tokens). 69 | 70 | Allow following permission when creating the token: 71 | 72 | ```commandline 73 | admin:org_hook, gist, notifications, read:gpg_key, read:public_key, read:repo_hook, repo, user 74 | ``` 75 | 76 | After creating the token, you should save the token for later usage: 77 | 78 | github-token 79 | 80 | 81 | #### [Evernote Developer Tokens](https://dev.evernote.com/doc/articles/dev_tokens.php) 82 | To grant the app to create/update notes to your Evernote notebooks, create a [Production Token](https://www.evernote.com/api/DeveloperToken.action) and save for later usage. 83 | 84 | evernote-token 85 | 86 | ### Installation 87 | First clone the repo and get into the root folder: 88 | 89 | ```commandline 90 | git clone https://github.com/leemengtaiwan/gist-evernote.git 91 | cd gist-evernote 92 | ``` 93 | 94 | To install all the dependencies, [Conda](https://conda.io/docs/) is recommended: 95 | 96 | ```commandline 97 | conda env create -n gist-evernote -f environment.yaml 98 | source activate gist-evernote 99 | ``` 100 | 101 | Or use `pip` instead: 102 | 103 | ```commandline 104 | pip install -r requirements.txt 105 | ``` 106 | 107 | ### Setup 108 | Tell the app about your Github and Evernote tokens acquired [previously](#Github Personal Access Token): 109 | 110 | ```commandline 111 | python setup.py 112 | ``` 113 | 114 | You will be asked to provided your tokens and the name of notebook you want your gists to be sync. Alternative, you can modify the [settings.py](settings.py) directly: 115 | 116 | ```python 117 | GITHUB_AUTH_TOKEN = "" 118 | EVERNOTE_PROD_TOKEN = "" 119 | EVERNOTE_SANDBOX_TOKEN = "" 120 | NOTEBOOK_TO_SYNC = "gist-evernote" 121 | 122 | ``` 123 | 124 | ### Start to sync your gists 125 | Finally, after input our credentials, now we can run the sync app: 126 | 127 | ```commandline 128 | python app.py 129 | ``` 130 | 131 | If anything go well, it will start to fetch your gists, 132 | take screenshots of them and save them as new notes in Evernote. Have a coffee before it finish :sunglasses::coffee: 133 | 134 | To reflect modification to synced gists to Evernote, just run the app again and it will try to do its best: 135 | 136 | ```commandline 137 | python app.py 138 | ``` 139 | 140 | ## Contributing 141 | 142 | There are still many things left to be improved. Any advice or pull request is highly appreciated. 143 | Notes that the Python Docstrings of this repo follow 144 | [Numpy Style](http://www.sphinx-doc.org/ja/stable/ext/example_numpy.html#example-numpy). 145 | 146 | 147 | ## Authors 148 | 149 | * **Meng Lee** - find me at `b98705001@gmail.com` 150 | 151 | ## License 152 | 153 | This project is licensed under the MIT License - 154 | see the [LICENSE.txt](LICENSE.txt) for details. 155 | 156 | ## Acknowledgements 157 | 158 | * [Github GraphQL API v4](https://developer.github.com/v4/) 159 | * [Evernote Python SDK](https://github.com/evernote/evernote-sdk-python) 160 | 161 | ## Future Work 162 | -------------------------------------------------------------------------------- /app.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | import os 3 | import time 4 | from multiprocessing import Pool, cpu_count 5 | from selenium import webdriver 6 | from selenium.webdriver.support import expected_conditions as EC 7 | from selenium.webdriver.common.by import By 8 | from selenium.webdriver.support.ui import WebDriverWait 9 | from selenium.common.exceptions import TimeoutException 10 | from enote.util import get_note, get_notebook, get_notebooks, \ 11 | create_resource, create_note, create_notebook, update_note 12 | from github.util import get_user_name, get_all_gists 13 | from web.util import fullpage_screenshot, get_gist_hash, create_chrome_driver 14 | from settings import NOTEBOOK_TO_SYNC 15 | from db import get_db 16 | 17 | DATE_FORMAT = "%Y-%m-%dT%H:%M:%SZ" 18 | GIST_BASE_URL = 'https://gist.github.com' 19 | 20 | notebook = None 21 | github_user = get_user_name() # get current login github user for fetching gist content 22 | db = get_db() # database to store synchronization info 23 | 24 | 25 | def app(): 26 | start = time.time() 27 | global notebook 28 | 29 | # find notebook to put new notes 30 | notebooks = get_notebooks() 31 | for n in notebooks: 32 | if n.name == NOTEBOOK_TO_SYNC: 33 | notebook = get_notebook(n.guid) 34 | 35 | # create notebook with the specified name if not found 36 | if not notebook: 37 | notebook = create_notebook(NOTEBOOK_TO_SYNC) 38 | print('Using notebook: %s' % notebook.name) 39 | 40 | # initialize, get all available gists 41 | if db.is_empty() or db.is_cold_start(): 42 | gists = get_all_gists() 43 | # sync only gists that were pushed after last synchronization 44 | else: 45 | last_sync_date = db.get_last_sync() 46 | print("Find gists that are updated after last sync (UTC): {}".format(last_sync_date)) 47 | gists = get_all_gists(after_date=last_sync_date) 48 | 49 | print("Total number of gists to be synchronized: %d" % len(gists)) 50 | 51 | # headless mode to reduce overhead and distraction 52 | driver = create_chrome_driver() if gists else None 53 | for gist in gists: 54 | _ = sync_gist(gist, driver=driver) 55 | if driver: 56 | driver.quit() 57 | 58 | # TODO multi-processes + mysql 59 | # setup multiple selenium drivers to speed up if multiple cpu available 60 | # num_processes = min(4, cpu_count() - 1) if cpu_count() > 1 else 1 61 | # print("Number of %d processes being created" % num_processes) 62 | # pool = Pool(num_processes) 63 | # 64 | # notes = pool.map(sync_gist, gists) 65 | # 66 | # pool.terminate() 67 | # pool.close() 68 | # pool.join() 69 | 70 | # sync all gists successfully, set to warm-start mode 71 | if db.is_cold_start(): 72 | db.toggle_cold_start() 73 | 74 | print("Synchronization took {:.0f} seconds.".format(time.time() - start)) 75 | 76 | 77 | def sync_gist(gist, driver): 78 | """Sync the Github gist to the corresponding Evernote note. 79 | 80 | Create a new Evernote note if there is no corresponding one with the gist. 81 | Overwrite existing note's content if gist has been changed. 82 | 83 | Parameters 84 | ---------- 85 | gist : dict 86 | A Gist acquired by Github GraphQL API with format like: 87 | { 88 | 'id': 'gist_id', 89 | 'name': 'gist_name', 90 | 'description': 'description', 91 | 'pushAt': '2018-01-15T00:48:23Z' 92 | 93 | } 94 | 95 | driver : selenium.webdriver 96 | The web driver used to access gist url 97 | 98 | Returns 99 | ------- 100 | note : evernote.edam.type.ttpyes.Note 101 | None if no new note created or updated 102 | 103 | """ 104 | note_exist = False 105 | gist_url = '/'.join((GIST_BASE_URL, gist['name'])) 106 | 107 | # check existing gist hash before fetch if available 108 | prev_hash = db.get_hash_by_id(gist['id']) 109 | note_guid = db.get_note_guid_by_id(gist['id']) 110 | if prev_hash and note_guid: 111 | note_exist = True 112 | cur_hash = get_gist_hash(github_user, gist['name']) 113 | if prev_hash == cur_hash: 114 | print('Gist {} remain the same, ignore.'.format(gist_url)) 115 | db.update_gist(gist, note_guid, cur_hash) 116 | return None 117 | 118 | 119 | 120 | driver.get(gist_url) 121 | # wait at most x seconds for Github rendering gist context 122 | delay_seconds = 10 123 | try: 124 | WebDriverWait(driver, delay_seconds).until(EC.presence_of_element_located((By.CLASS_NAME, 'is-render-ready'))) 125 | except TimeoutException: 126 | print("Take longer than {} seconds to load page.".format(delay_seconds)) 127 | 128 | # get first file name as default note title 129 | gist_title = driver.find_element(By.CLASS_NAME, 'gist-header-title>a').text 130 | 131 | # take screen shot for the gist and save it temporally 132 | image_path = 'images/{}.png'.format(gist['name']) 133 | fullpage_screenshot(driver, image_path) 134 | 135 | # build skeleton for note (including screenshot) 136 | resource, _ = create_resource(image_path) 137 | note_title = gist['description'] if gist['description'] else gist_title 138 | note_body = format_note_body(gist) 139 | 140 | # get hash of raw gist content and save gist info to database 141 | gist_hash = get_gist_hash(github_user, gist['name']) 142 | 143 | # create new note / update existing note 144 | if not note_exist: 145 | note = create_note(note_title, note_body, [resource], parent_notebook=notebook) 146 | db.save_gist(gist, note.guid, gist_hash) 147 | else: 148 | note = get_note(note_guid) 149 | update_note(note, note_title, note_body, note_guid, [resource]) 150 | db.update_gist(gist, note_guid, gist_hash) 151 | 152 | os.remove(image_path) 153 | print("Finish creating note for gist {}".format(gist_url)) 154 | return note 155 | 156 | 157 | def format_note_body(gist): 158 | """Create the note content that will be shown before attachments. 159 | 160 | Parameters 161 | ---------- 162 | gist : dict 163 | Dict that contains all information of the gist 164 | 165 | Returns 166 | ------- 167 | note_body : str 168 | 169 | """ 170 | blocks = [] 171 | 172 | desc = gist['description'] 173 | if desc: 174 | blocks.append(desc) 175 | 176 | gist_url = '/'.join((GIST_BASE_URL, gist['name'])) 177 | blocks.append('Gist on Github'.format(gist_url)) 178 | note_body = '
'.join(blocks) 179 | 180 | return note_body 181 | 182 | 183 | if __name__ == '__main__': 184 | app() 185 | -------------------------------------------------------------------------------- /db.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | import fire 4 | from datetime import datetime 5 | DB_FILE = 'db.json' 6 | ENV_FILE = 'env.json' 7 | DATE_FORMAT = "%Y-%m-%dT%H:%M:%SZ" 8 | 9 | 10 | class Database(object): 11 | """Storage class to keep track of gist sync information. 12 | 13 | This class is independent of the actual implementation 14 | of the database, thus make it easier to change in the future. 15 | """ 16 | 17 | def __init__(self): 18 | if not os.path.isfile(DB_FILE) or not os.path.isfile(ENV_FILE): 19 | self.info = {"num_gists": 0} 20 | self.env = { 21 | "cold_start": True, 22 | "sync_at": datetime.strftime(datetime(1990, 10, 22), DATE_FORMAT)} 23 | self.sync_info("save") 24 | self.sync_env("save") 25 | return 26 | 27 | # restore db and env from previous execution result 28 | self.sync_info('load') 29 | self.sync_env('load') 30 | 31 | def is_empty(self): 32 | """Indicate whether there is any gist in database. 33 | 34 | Returns 35 | ------- 36 | bool 37 | 38 | """ 39 | return self.info.get('num_gists', 0) == 0 40 | 41 | def is_cold_start(self): 42 | """Indicate whether it is needed to synchronize all gists. 43 | 44 | Returns 45 | ------- 46 | bool 47 | 48 | """ 49 | return self.env.get('cold_start', True) 50 | 51 | def get_last_sync(self): 52 | """Return the UTC datetime indicating the last synchronization 53 | 54 | Returns 55 | ------- 56 | last_sync_date : datetime.datetime 57 | """ 58 | return datetime.strptime(self.env['sync_at'], DATE_FORMAT) 59 | 60 | def toggle_cold_start(self): 61 | """Toggle value of cold_start""" 62 | self.env['cold_start'] = not self.env.get('cold_start', True) 63 | self.sync_env('save') 64 | 65 | def get_hash_by_id(self, gist_id): 66 | """Get hash value of the gist using `gist_id` as key 67 | 68 | Parameters 69 | ---------- 70 | gist_id : str 71 | Unique gist identifier called `id` available in Github API 72 | e.g. "MDQ6R2lzdGUzOTNkODgxMjIyODg1ZjU5ZWYwOWExNDExNzE1OWM4" 73 | 74 | Returns 75 | ------- 76 | hash : str 77 | "" if no gist can be found in database by `gist_id` 78 | """ 79 | return self.info.get(gist_id, {}).get('hash', '') 80 | 81 | def get_note_guid_by_id(self, gist_id): 82 | """Get guid of note related to the gist with `gist_id` 83 | 84 | Parameters 85 | ---------- 86 | gist_id : str 87 | Unique gist identifier called `id` available in Github API 88 | e.g. "MDQ6R2lzdGUzOTNkODgxMjIyODg1ZjU5ZWYwOWExNDExNzE1OWM4" 89 | 90 | Returns 91 | ------- 92 | guid : str 93 | 94 | """ 95 | return self.info.get(gist_id, {}).get('note_guid', '') 96 | 97 | 98 | def save_gist(self, gist, note_guid, hash): 99 | """Save information of a given gist into database. 100 | 101 | Parameters 102 | ---------- 103 | gist : dict 104 | A Gist acquired by Github GraphQL API with format like: 105 | { 106 | 'id': 'gist_id', 107 | 'name': 'gist_name', 108 | 'description': 'description', 109 | 'pushedAt': '2018-01-15T00:48:23Z', 110 | } 111 | note_guid : str 112 | hash : str 113 | """ 114 | gist['note_guid'] = note_guid 115 | gist['hash'] = hash 116 | 117 | self.info[gist['id']] = gist 118 | self.info['num_gists'] = self.info.get('num_gists', 0) + 1 119 | self.sync_info('save') 120 | self.update_sync_time(gist['pushedAt']) 121 | 122 | def update_gist(self, gist, note_guid, hash): 123 | """Update information of a given gist into database. 124 | 125 | Parameters 126 | ---------- 127 | gist : dict 128 | A Gist acquired by Github GraphQL API with format like: 129 | { 130 | 'id': 'gist_id', 131 | 'name': 'gist_name', 132 | 'description': 'description', 133 | 'pushedAt': '2018-01-15T00:48:23Z' 134 | } 135 | note_guid : str 136 | hash : str 137 | 138 | """ 139 | 140 | gist['note_guid'] = note_guid 141 | gist['hash'] = hash 142 | 143 | self.info[gist['id']] = gist 144 | self.sync_info('save') 145 | self.update_sync_time(gist['pushedAt']) 146 | 147 | def update_sync_time(self, sync_date): 148 | """Update last synchronization time 149 | 150 | Parameters 151 | ---------- 152 | sync_date : str 153 | String indicating valid datetime like '2018-01-15T00:48:23Z', 154 | which defined by global environment `DATE_FORMAT`. 155 | 156 | """ 157 | self.env['sync_at'] = sync_date 158 | self.sync_env('save') 159 | 160 | def sync_env(self, mode): 161 | """Synchronize runtime information between current Database obj and permanent storage 162 | 163 | Parameters 164 | ---------- 165 | mode : str 166 | Indicating to save or load environment. Valid values: ["save", "load"] 167 | 168 | Returns 169 | ------- 170 | bool 171 | 172 | """ 173 | if mode == 'save': 174 | with open(ENV_FILE, 'w') as fp: 175 | json.dump(self.env, fp, indent=2) 176 | elif mode == 'load': 177 | with open(ENV_FILE, "r") as fp: 178 | env = json.load(fp) 179 | self.env = env 180 | return True 181 | 182 | def sync_info(self, mode): 183 | """Synchronize gist info between current Database obj and permanent storage 184 | 185 | Parameters 186 | ---------- 187 | mode : str 188 | Indicating to save or load environment. Valid values: ["save", "load"] 189 | 190 | Returns 191 | ------- 192 | bool 193 | 194 | """ 195 | if mode == 'save': 196 | with open(DB_FILE, 'w') as fp: 197 | json.dump(self.info, fp, indent=2) 198 | elif mode == 'load': 199 | with open(DB_FILE, "r") as fp: 200 | info = json.load(fp) 201 | self.info = info 202 | return True 203 | 204 | 205 | def get_db(): 206 | """Get a database instance for storing gist information 207 | 208 | Returns 209 | ------- 210 | db : Database instance 211 | 212 | """ 213 | return Database() 214 | 215 | 216 | if __name__ == '__main__': 217 | fire.Fire() -------------------------------------------------------------------------------- /enote/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leemengtw/gist-evernote/90d8573870ded37dc82575ba25968d7a06efe219/enote/__init__.py -------------------------------------------------------------------------------- /enote/test.py: -------------------------------------------------------------------------------- 1 | import time 2 | import sys 3 | import unittest 4 | import util 5 | 6 | 7 | class Test(unittest.TestCase): 8 | """ Test Evernote API""" 9 | 10 | def setUp(self): 11 | pass 12 | 13 | def tearDown(self): 14 | pass 15 | 16 | def test_access(self): 17 | """Test simple access to Evernote API""" 18 | util.simple_access() 19 | 20 | def test_create_note(self): 21 | auth_token = util.get_auth_token() 22 | note_store = util.get_note_store() 23 | util.create_note(auth_token, note_store, 'Test Note', 'Hello world') 24 | 25 | def test_create_note_with_attachments(self): 26 | auth_token = util.get_auth_token() 27 | note_store = util.get_note_store() 28 | 29 | resources = [] 30 | resource = util.create_resource('images/test.png') 31 | resources.append(resource) 32 | 33 | util.create_note(auth_token, note_store, 34 | 'Test Note with attachments', 'Hello world', resources) 35 | 36 | 37 | if __name__ == "__main__": 38 | unittest.main(argv=[sys.argv[0]]) 39 | -------------------------------------------------------------------------------- /enote/util.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | from __future__ import print_function 3 | from __future__ import unicode_literals 4 | 5 | import fire 6 | import time 7 | import hashlib 8 | import tzlocal 9 | from datetime import datetime 10 | from evernote.api.client import EvernoteClient 11 | from evernote.edam.type import ttypes 12 | from evernote.edam.error import ttypes as Errors 13 | from evernote.edam.limits.constants import EDAM_NOTE_TITLE_LEN_MAX 14 | from secret import EVERNOTE_PROD_TOKEN, EVERNOTE_SANDBOX_TOKEN 15 | 16 | 17 | def get_evernote_auth_token(env="prod"): 18 | """Return either a valid production / dev Evernote developer token. 19 | 20 | Parameters 21 | ---------- 22 | env : str 23 | Indicate which environment's token to be returned. 24 | Valid options: ["prod", "dev"] 25 | 26 | Returns 27 | ------- 28 | token : str 29 | 30 | """ 31 | return EVERNOTE_PROD_TOKEN if env == 'prod' else EVERNOTE_SANDBOX_TOKEN 32 | 33 | 34 | def get_client(env="prod"): 35 | """Return a Evernote API client 36 | 37 | Parameters 38 | ---------- 39 | env : str 40 | Indicate which environment's token to be returned. 41 | Valid options: ["prod", "dev"] 42 | 43 | Returns 44 | ------- 45 | client : evernote.api.client.EvernoteClient 46 | 47 | """ 48 | sandbox = False if env == 'prod' else True 49 | return EvernoteClient(token=get_evernote_auth_token(env), sandbox=sandbox) 50 | 51 | 52 | def get_note_store(env="prod"): 53 | """Return a NoteStore used to used to manipulate notes, notebooks in a user account. 54 | 55 | Parameters 56 | ---------- 57 | env : str 58 | Indicate which environment's token to be returned. 59 | Valid options: ["prod", "dev"] 60 | 61 | Returns 62 | ------- 63 | NoteStore 64 | 65 | Notes 66 | ----- 67 | Evernote documentation: 68 | https://dev.evernote.com/doc/start/python.php 69 | 70 | """ 71 | return get_client(env).get_note_store() 72 | 73 | 74 | def get_note(guid=None, env='prod'): 75 | """Return a specific Note instance by guid. 76 | 77 | Parameters 78 | ---------- 79 | guid : str 80 | 81 | env : str 82 | Indicate which environment's token to be returned. 83 | Valid options: ["prod", "dev"] 84 | 85 | Returns 86 | ------- 87 | evernote.edam.type.ttypes.Note 88 | 89 | """ 90 | auth_token = get_evernote_auth_token(env) 91 | 92 | 93 | assert guid is not None, 'Guid is not available.' 94 | return get_note_store().getNote(auth_token, guid, False, False, False , False) 95 | 96 | 97 | def get_notebook(guid=None): 98 | """Return a specific Notebook instance by guid. 99 | 100 | Parameters 101 | ---------- 102 | guid : str 103 | 104 | Returns 105 | ------- 106 | evernote.edam.type.ttypes.Notebook 107 | 108 | """ 109 | assert guid is not None, 'Guid is not available.' 110 | return get_note_store().getNotebook(guid) 111 | 112 | 113 | def get_notebooks(env="prod"): 114 | """Get all available Notebook instances in a user account. 115 | 116 | Parameters 117 | ---------- 118 | env : str 119 | Indicate which environment's token to be returned. 120 | Valid options: ["prod", "dev"] 121 | 122 | Returns 123 | ------- 124 | Notebooks : list of evernote.edam.type.ttypes.Notebook 125 | 126 | """ 127 | client = get_client(env) 128 | 129 | # get information about current user 130 | userStore = client.get_user_store() 131 | user = userStore.getUser() 132 | print('Current user:', user.username) 133 | 134 | # get information about notes 135 | noteStore = client.get_note_store() 136 | notebooks = noteStore.listNotebooks() 137 | # for n in notebooks: 138 | # print(n.name, n.guid) 139 | return notebooks 140 | 141 | 142 | def create_notebook(name=None): 143 | """Create a new notebook with given `name`. 144 | 145 | Parameters 146 | ---------- 147 | name : str 148 | Indicating the name of the notebook to be created 149 | 150 | Returns 151 | ------- 152 | evernote.edam.type.ttypes.Notebook 153 | 154 | """ 155 | assert name is not None, 'Notebook name is not specified.' 156 | notebook = ttypes.Notebook() 157 | notebook.name = name 158 | return get_note_store().createNotebook(notebook) 159 | 160 | 161 | def create_resource(file_path, mime='image/png'): 162 | """Create a Resource instance for attaching to evernote Note instance 163 | 164 | Parameters 165 | ---------- 166 | file_path : str 167 | Indicate file path of the file 168 | 169 | mime : str, optional 170 | Valid MIME type indicating type of the file 171 | 172 | Returns 173 | ------- 174 | evernote.edam.type.ttypes.Resource 175 | The Resource must contain three parts: 176 | - MIME type 177 | - content: evernote.edam.type.ttypes.Data 178 | - hash 179 | 180 | Notes 181 | ----- 182 | Create string of MD5 sum: 183 | https://stackoverflow.com/questions/5297448/how-to-get-md5-sum-of-a-string-using-python 184 | 185 | """ 186 | 187 | file_data = None 188 | with open(file_path, 'rb') as f: 189 | byte_str = f.read() 190 | file_data = bytearray(byte_str) 191 | 192 | md5 = hashlib.md5() 193 | md5.update(file_data) 194 | hexhash = md5.hexdigest() 195 | data = ttypes.Data() 196 | 197 | # build Resource's necessary data 198 | data.size = len(file_data) 199 | data.bodyHash = hexhash 200 | data.body = file_data 201 | 202 | # build Resource Type 203 | resource = ttypes.Resource() 204 | resource.mime = mime 205 | resource.data = data 206 | return resource, hexhash 207 | 208 | 209 | def create_note(note_title, note_body, resources=[], parent_notebook=None, env="prod"): 210 | """Create new Note with the given attachments in user's notebook 211 | 212 | Parameters 213 | ---------- 214 | note_title : str 215 | Text to used as new note's title 216 | 217 | note_body : str 218 | Text to insert into note 219 | 220 | resources : list of evernote.edam.type.ttypes.Resource 221 | List of attachments to combined with the note 222 | 223 | parent_notebook : evernote.edam.type.ttypes.Notebook, optional 224 | Notebook instance to insert new note 225 | 226 | env : str 227 | Indicate which environment's token to be returned. 228 | Valid options: ["prod", "dev"] 229 | 230 | Returns 231 | ------- 232 | evernote.edam.type.ttypes.Note 233 | The newly created Note instance 234 | 235 | Notes 236 | ----- 237 | Evernote error documentation: 238 | http://dev.evernote.com/documentation/reference/Errors.html#Enum_EDAMErrorCode 239 | 240 | Evernote Struct: Note 241 | https://dev.evernote.com/doc/reference/Types.html#Struct_Note 242 | 243 | """ 244 | auth_token = get_evernote_auth_token(env) 245 | note_store = get_note_store(env) 246 | 247 | # create note object 248 | new_note = ttypes.Note() 249 | 250 | # get formatted title 251 | new_note.title = build_note_title(note_title) 252 | 253 | # build body of note 254 | new_note.resources = resources 255 | new_note.content = build_note_content(note_body, resources) 256 | 257 | # parent_notebook is optional. if omitted, default notebook is used 258 | if parent_notebook and hasattr(parent_notebook, 'guid'): 259 | new_note.notebookGuid = parent_notebook.guid 260 | 261 | # attempt to create note in Evernote account 262 | try: 263 | note = note_store.createNote(auth_token, new_note) 264 | except Errors.EDAMUserException, edue: 265 | print("EDAMUserException:", edue) 266 | return None 267 | except Errors.EDAMNotFoundException, ednfe: 268 | # Parent Notebook GUID doesn't correspond to an actual notebook 269 | print("EDAMNotFoundException: Invalid parent notebook GUID") 270 | return None 271 | 272 | # Return created note object 273 | return note 274 | 275 | 276 | def update_note(note, note_title, note_body, note_guid, resources): 277 | """Update existing note in Evernote identified by `note_guid`. 278 | 279 | Parameters 280 | ---------- 281 | note : evernote.edam.type.ttypes.Note 282 | Note instance to be updated 283 | 284 | note_title : str 285 | Text to used as new note's title 286 | 287 | note_body : str 288 | Text to insert into note 289 | 290 | note_guid : str 291 | 292 | resources : list of evernote.edam.type.ttypes.Resource 293 | List of attachments to combined with the note 294 | 295 | Returns 296 | ------- 297 | evernote.edam.type.ttypes.Note 298 | The updated Note instance 299 | 300 | """ 301 | 302 | auth_token = get_evernote_auth_token() 303 | note_store = get_note_store() 304 | 305 | note.guid = note_guid 306 | note.title = build_note_title(note_title) 307 | 308 | # build body of note with new resources 309 | note.resources = resources 310 | note.content = build_note_content(note_body, resources) 311 | 312 | # update `updated time` of the note in timezone-aware manner 313 | tz = tzlocal.get_localzone() 314 | dt = tz.localize(datetime.now()) 315 | note.updated = time.mktime(dt.timetuple()) * 1e3 316 | 317 | # attempt to update note in Evernote account 318 | try: 319 | note = note_store.updateNote(auth_token, note) 320 | except Errors.EDAMNotFoundException, ednfe: 321 | # Parent Notebook GUID doesn't correspond to an actual notebook 322 | print("EDAMNotFoundException: Invalid parent notebook GUID") 323 | return None 324 | 325 | # Return created note object 326 | return note 327 | 328 | 329 | def build_note_title(note_title): 330 | """Return a formatted title with right encoding. 331 | 332 | The func handle special formatting and encoding for title to avoid Evernote API Error. 333 | 334 | Parameters 335 | ---------- 336 | note_title : str / unicode 337 | 338 | Returns 339 | ------- 340 | formatted_note_title : str 341 | 342 | Notes 343 | ----- 344 | Evernote API documentation 345 | https://dev.evernote.com/doc/reference/NoteStore.html#Fn_NoteStore_updateNote 346 | 347 | """ 348 | formatted_note_title = note_title.strip().replace('\n', ' ') 349 | for title_charset in 'US-ASCII', 'ISO-8859-1', 'UTF-8': 350 | try: 351 | formatted_note_title = formatted_note_title.encode(title_charset) 352 | except UnicodeError: 353 | pass 354 | else: 355 | break 356 | 357 | # restrict the size of title in sure to be able to sync to evernote 358 | formatted_note_title = formatted_note_title[:EDAM_NOTE_TITLE_LEN_MAX] 359 | 360 | return formatted_note_title 361 | 362 | 363 | def build_note_content(note_body, resources): 364 | """Return notebook content with attachments written in Evernote Markup Language. 365 | 366 | Parameters 367 | ---------- 368 | note_body : str 369 | Custom content to insert before attachments 370 | 371 | resources : list of evernote.edam.type.ttypes.Resource 372 | List of attachments to combined with the note 373 | 374 | Returns 375 | ------- 376 | note_content : str 377 | 378 | Notes 379 | ----- 380 | Understanding the Evernote Markup Language: 381 | https://dev.evernote.com/doc/articles/enml.php 382 | 383 | 384 | """ 385 | note_content = '' 386 | note_content += '' 387 | note_content += "%s" % note_body 388 | 389 | if resources: 390 | # add Resource objects to note content 391 | note_content += "
" * 2 392 | 393 | for resource in resources: 394 | hexhash = resource.data.bodyHash 395 | note_content += "

" % (resource.mime, hexhash) 396 | note_content += "" 397 | 398 | # handling encoding 399 | for title_charset in 'US-ASCII', 'ISO-8859-1', 'UTF-8': 400 | try: 401 | note_content = note_content.encode(title_charset) 402 | except UnicodeError: 403 | pass 404 | else: 405 | break 406 | 407 | return note_content 408 | 409 | 410 | if __name__ == '__main__': 411 | fire.Fire() -------------------------------------------------------------------------------- /environment.yaml: -------------------------------------------------------------------------------- 1 | name: gist-evernote 2 | channels: 3 | - conda-forge 4 | - defaults 5 | dependencies: 6 | - fire=0.1.2=py_0 7 | - freetype=2.8.1=0 8 | - imageio=2.2.0=py27_0 9 | - jpeg=9b=2 10 | - libpng=1.6.34=0 11 | - libtiff=4.0.9=0 12 | - olefile=0.44=py27_0 13 | - pillow=5.0.0=py27_0 14 | - selenium=3.8.1=py27_0 15 | - setuptools=38.4.0=py27_0 16 | - six=1.11.0=py27_1 17 | - xz=5.2.3=0 18 | - asn1crypto=0.24.0=py27_0 19 | - ca-certificates=2017.08.26=ha1e5d58_0 20 | - certifi=2017.11.5=py27hfa9a1c4_0 21 | - cffi=1.11.4=py27h342bebf_0 22 | - chardet=3.0.4=py27h2842e91_1 23 | - cryptography=2.1.4=py27hdbc5e8f_0 24 | - enum34=1.1.6=py27hf475452_1 25 | - idna=2.6=py27hedea723_1 26 | - intel-openmp=2018.0.0=h8158457_8 27 | - ipaddress=1.0.19=py27_0 28 | - libcxx=4.0.1=h579ed51_0 29 | - libcxxabi=4.0.1=hebd6815_0 30 | - libedit=3.1=hb4e282d_0 31 | - libffi=3.2.1=h475c297_4 32 | - libgfortran=3.0.1=h93005f0_2 33 | - mkl=2018.0.1=hfbd8650_4 34 | - ncurses=6.0=hd04f020_2 35 | - numpy=1.14.0=py27h8a80b8c_0 36 | - openssl=1.0.2n=hdbc3d79_0 37 | - pip=9.0.1=py27h1567d89_4 38 | - pycparser=2.18=py27h0d28d88_1 39 | - pyopenssl=17.5.0=py27hfda213f_0 40 | - pysocks=1.6.7=py27h1cff6a6_1 41 | - python=2.7.14=hde5916a_29 42 | - readline=7.0=hc1231fa_4 43 | - requests=2.18.4=py27h9b2b37c_1 44 | - sqlite=3.20.1=h7e4c145_2 45 | - tk=8.6.7=h35a86e2_3 46 | - urllib3=1.22=py27hc3787e9_0 47 | - wheel=0.30.0=py27h677a027_1 48 | - zlib=1.2.11=hf3cbc9b_2 49 | - pip: 50 | - altgraph==0.15 51 | - better-exceptions==0.2.1 52 | - cycler==0.10.0 53 | - dis3==0.1.2 54 | - evernote==1.25.3 55 | - functools32==3.2.3.post2 56 | - future==0.16.0 57 | - httplib2==0.10.3 58 | - macholib==1.9 59 | - mercurial==4.3.3 60 | - nose==1.3.7 61 | - oauth2==1.9.0.post1 62 | - pefile==2017.11.5 63 | - pygobject 64 | - pyperclip==1.6.0 65 | - python-dateutil==2.6.0 66 | - pytz==2018.3 67 | - subprocess32==3.2.7 68 | - tzlocal==1.5.1 69 | prefix: /Users/leemeng/anaconda3/envs/gist-evernote 70 | 71 | -------------------------------------------------------------------------------- /github/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leemengtw/gist-evernote/90d8573870ded37dc82575ba25968d7a06efe219/github/__init__.py -------------------------------------------------------------------------------- /github/test.py: -------------------------------------------------------------------------------- 1 | import time 2 | import sys 3 | import unittest 4 | import util 5 | 6 | 7 | class Test(unittest.TestCase): 8 | """ Test Github GraphQL API""" 9 | 10 | def setUp(self): 11 | pass 12 | 13 | def tearDown(self): 14 | pass 15 | 16 | def test_access(self): 17 | """Test simple access to Github GraphQL API""" 18 | pass 19 | 20 | query = "{\"query\":\"query {\\n viewer {\\n login\\n }\\n}\"}" 21 | print(util.query_graphql(query)) 22 | 23 | 24 | 25 | if __name__ == "__main__": 26 | unittest.main(argv=[sys.argv[0]]) 27 | -------------------------------------------------------------------------------- /github/util.py: -------------------------------------------------------------------------------- 1 | import fire 2 | import requests 3 | from datetime import datetime 4 | from secret import GITHUB_AUTH_TOKEN 5 | 6 | GITHUB_GRAPHQL_URL = 'https://api.github.com/graphql' 7 | DATE_FORMAT = "%Y-%m-%dT%H:%M:%SZ" 8 | 9 | 10 | def get_user_name(token=GITHUB_AUTH_TOKEN): 11 | """Return current login user's name with given token 12 | 13 | Parameters 14 | ---------- 15 | token : str 16 | String representing Github Developer Access Token 17 | 18 | Returns 19 | ------- 20 | user_name : str 21 | """ 22 | payload = "{\"query\":\"query {\\n viewer {\\n login\\n }\\n}\"}" 23 | res = query_graphql(payload, token) 24 | return res['data']['viewer']['login'] 25 | 26 | 27 | def query_graphql(payload, token=GITHUB_AUTH_TOKEN, url=GITHUB_GRAPHQL_URL): 28 | """Helper to query Github GraphQL API 29 | 30 | Parameters 31 | ---------- 32 | payload : str 33 | Valid GraphQL query string. 34 | e.g., "{\"query\":\"query {\\n viewer {\\n login\\n }\\n}\"}" 35 | 36 | Returns 37 | ------- 38 | res : dict 39 | 40 | """ 41 | headers = { 42 | 'content-type': "application/json", 43 | 'authorization': "Bearer {}".format(token) 44 | } 45 | 46 | res = requests.request("POST", url, data=payload, headers=headers).json() 47 | assert res.get('data', False), 'No data available from Github: {}'.format(res) 48 | return res 49 | 50 | 51 | def get_gists(cursor=None, size=100): 52 | """Return all gists (public & secret) and end_cursor for pagination 53 | 54 | Parameters 55 | ---------- 56 | cursor : str 57 | String indicating endCursor in previous API request. 58 | e.g. "Y3Vyc29yOnYyOpK5MjAxOC0wMS0yM1QxMTo1NDo0MSswOTowMM4FGyp6" 59 | 60 | size : int 61 | Specify how many gists to fetch in a HTTP request to Github. 62 | Default set to Node limit specified by Github GraphQL resource limit. 63 | 64 | Returns 65 | ------- 66 | gists : list of dict 67 | List of gists with each gist is a dict of form: 68 | { 69 | "id": "id", 70 | "description": "some description", 71 | "name": "just a name", 72 | "pushedAt": "2018-01-15T08:32:57Z" 73 | } 74 | 75 | total : int 76 | Indicate how many gists available 77 | 78 | end_cursor : str 79 | A string representing the endCursor in gists.pageInfo 80 | 81 | has_next_page : bool 82 | Indicating whether there are gists remains 83 | 84 | Notes 85 | ----- 86 | Github GraphQL resource limit 87 | https://developer.github.com/v4/guides/resource-limitations/ 88 | 89 | """ 90 | first_payload = "{\"query\":\"query {viewer {gists(first:%d, privacy:ALL, orderBy: {field: UPDATED_AT, direction: DESC}) {totalCount edges { node { id description name pushedAt } cursor } pageInfo { endCursor hasNextPage } } } }\"}" 91 | payload_template = "{\"query\":\"query {viewer {gists(first:%d, privacy:ALL, orderBy: {field: UPDATED_AT, direction: DESC}, after:\\\"%s\\\") {totalCount edges { node { id description name pushedAt } cursor } pageInfo { endCursor hasNextPage } } } }\"}" 92 | 93 | if not cursor: 94 | payload = first_payload % size 95 | else: 96 | payload = payload_template % (size, cursor) 97 | 98 | res = query_graphql(payload) 99 | 100 | 101 | # parse nested response for easier usage 102 | gists = res['data']['viewer']['gists'] 103 | total = gists['totalCount'] 104 | page_info = gists['pageInfo'] 105 | end_cursor, has_next_page = page_info['endCursor'], page_info['hasNextPage'] 106 | gists = [e['node'] for e in gists['edges']] 107 | 108 | return gists, total, end_cursor, has_next_page 109 | 110 | 111 | def get_number_of_gists(): 112 | """Get total number of gists available in the user account 113 | 114 | Returns 115 | ------- 116 | num_gists : int 117 | """ 118 | payload = "{\"query\":\"query { viewer { gists(privacy:ALL) {totalCount}}}\"}" 119 | res = query_graphql(payload) 120 | return res['data']['viewer']['gists']['totalCount'] 121 | 122 | 123 | def get_all_gists(size=None, after_date=None, filter_on='pushedAt'): 124 | """Get number of `size` gists at once without pagination. 125 | 126 | A wrapper over `get_gists` func. Handle the pagination automatically. 127 | If size is not set by user, query Github for total number of gists. 128 | If a valid `after_date` is given, gists with field `filter_on` earlier than 129 | `after_date` will be dropped. 130 | 131 | Parameters 132 | ---------- 133 | size : int, optional 134 | Number of gists to fetch. Set to total number of gists if not set by user 135 | 136 | after_date : datetime.datetime 137 | UTC date to filter gists 138 | 139 | filter_on : str 140 | Date field corresponding to Github API for Gist 141 | 142 | Returns 143 | ------- 144 | gists : list of dict 145 | List of gists with each gist is a dict of form: 146 | { 147 | "id": "id", 148 | "description": "some description", 149 | "name": "just a name", 150 | "pushedAt": "2018-01-15T08:32:57Z" 151 | } 152 | 153 | See Also 154 | -------- 155 | get_gists : Return all gists (public & secret) and end_cursor for pagination 156 | 157 | 158 | """ 159 | if not size: 160 | size = get_number_of_gists() 161 | 162 | end_cursor = None 163 | gists = [] 164 | 165 | while True: 166 | cur_gists, total, end_cursor, has_next_page = get_gists(end_cursor) 167 | for gist in cur_gists: 168 | pushed_date = datetime.strptime(gist[filter_on], DATE_FORMAT) 169 | if after_date and pushed_date <= after_date: 170 | return gists 171 | 172 | if len(gists) >= size: 173 | return gists 174 | gists.append(gist) 175 | 176 | if not has_next_page: 177 | break 178 | 179 | return gists 180 | 181 | 182 | if __name__ == '__main__': 183 | fire.Fire() 184 | -------------------------------------------------------------------------------- /images/demo-subplots.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leemengtw/gist-evernote/90d8573870ded37dc82575ba25968d7a06efe219/images/demo-subplots.gif -------------------------------------------------------------------------------- /images/evernote-token.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leemengtw/gist-evernote/90d8573870ded37dc82575ba25968d7a06efe219/images/evernote-token.png -------------------------------------------------------------------------------- /images/github-token-created.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leemengtw/gist-evernote/90d8573870ded37dc82575ba25968d7a06efe219/images/github-token-created.png -------------------------------------------------------------------------------- /images/github-token.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leemengtw/gist-evernote/90d8573870ded37dc82575ba25968d7a06efe219/images/github-token.png -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | altgraph==0.15 2 | asn1crypto==0.24.0 3 | better-exceptions==0.2.1 4 | certifi==2017.11.5 5 | cffi==1.11.4 6 | chardet==3.0.4 7 | cryptography==2.1.4 8 | cycler==0.10.0 9 | dis3==0.1.2 10 | enum34==1.1.6 11 | evernote==1.25.3 12 | fire==0.1.2 13 | functools32==3.2.3.post2 14 | future==0.16.0 15 | httplib2==0.10.3 16 | idna==2.6 17 | imageio==2.2.0 18 | ipaddress==1.0.19 19 | macholib==1.9 20 | mercurial==4.3.3 21 | nose==1.3.7 22 | numpy==1.13.3 23 | oauth2==1.9.0.post1 24 | olefile==0.44 25 | pefile==2017.11.5 26 | Pillow==5.0.0 27 | pycparser==2.18 28 | pygobject==3.24.1 29 | pyOpenSSL==17.5.0 30 | pyperclip==1.6.0 31 | pyportmidi==0.0.7 32 | PySocks==1.6.7 33 | python-dateutil==2.6.0 34 | pytz==2018.3 35 | requests==2.18.4 36 | selenium==3.8.1 37 | six==1.11.0 38 | subprocess32==3.2.7 39 | TBB==0.1 40 | tzlocal==1.5.1 41 | urllib3==1.22 42 | -------------------------------------------------------------------------------- /settings.py: -------------------------------------------------------------------------------- 1 | GITHUB_AUTH_TOKEN = "" 2 | EVERNOTE_PROD_TOKEN = "" 3 | EVERNOTE_SANDBOX_TOKEN = "" 4 | NOTEBOOK_TO_SYNC = "gist-evernote" 5 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | GITHUB_SECRET_FILE = 'github/secret.py' 3 | EVERNOTE_SECRET_FILE = 'enote/secret.py' 4 | SETTING_FILE = 'settings.py' 5 | NOTEBOOK = 'gist-evernote' 6 | 7 | def initialize(): 8 | """Use settings.py to setup/update environment""" 9 | from settings import GITHUB_AUTH_TOKEN, \ 10 | EVERNOTE_PROD_TOKEN, EVERNOTE_SANDBOX_TOKEN 11 | 12 | setting_str = '' 13 | 14 | # setup github credential 15 | while not GITHUB_AUTH_TOKEN: 16 | GITHUB_AUTH_TOKEN = raw_input("Github Personal Access Token: ") 17 | 18 | with open(GITHUB_SECRET_FILE, 'w') as f: 19 | string = "GITHUB_AUTH_TOKEN = \"{}\"\n".format(GITHUB_AUTH_TOKEN) 20 | f.write(string) 21 | setting_str += string 22 | 23 | # setup evernote credential 24 | while not EVERNOTE_PROD_TOKEN: 25 | EVERNOTE_PROD_TOKEN = raw_input("Evernote Production Developer Token: ") 26 | 27 | with open(EVERNOTE_SECRET_FILE, 'w') as f: 28 | string = "EVERNOTE_PROD_TOKEN = \"{}\"\n".format(EVERNOTE_PROD_TOKEN) 29 | f.write(string) 30 | setting_str += string 31 | 32 | # optional for local debug 33 | string = "EVERNOTE_SANDBOX_TOKEN = \"{}\"\n".format(EVERNOTE_SANDBOX_TOKEN) 34 | f.write(string) 35 | setting_str += string 36 | 37 | # setup notebook 38 | notebook = raw_input("Evernote notebook name you want to put gists (default set to {}): ".format(NOTEBOOK)) 39 | notebook = notebook if notebook else NOTEBOOK 40 | setting_str += "NOTEBOOK_TO_SYNC = \"{}\"\n".format(notebook) 41 | 42 | # update setting 43 | with open(SETTING_FILE, "w") as f: 44 | f.write(setting_str) 45 | 46 | print("You're all set! run\n\n python app.py\n\nto start synchronization :) ") 47 | 48 | return True 49 | 50 | 51 | if __name__ == '__main__': 52 | initialize() 53 | -------------------------------------------------------------------------------- /web/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leemengtw/gist-evernote/90d8573870ded37dc82575ba25968d7a06efe219/web/__init__.py -------------------------------------------------------------------------------- /web/test.py: -------------------------------------------------------------------------------- 1 | """ 2 | This script uses a simplified version of the one here: 3 | https://snipt.net/restrada/python-selenium-workaround-for-full-page-screenshot-using-chromedriver-2x/ 4 | 5 | It contains the *crucial* correction added in the comments by Jason Coutu. 6 | """ 7 | 8 | import sys 9 | import time 10 | 11 | from selenium import webdriver 12 | import unittest 13 | 14 | import util 15 | 16 | class Test(unittest.TestCase): 17 | """ Demonstration: Get Chrome to generate fullscreen screenshot """ 18 | 19 | def setUp(self): 20 | self.driver = webdriver.Chrome() 21 | 22 | def tearDown(self): 23 | self.driver.quit() 24 | 25 | def test_fullpage_screenshot(self): 26 | ''' Generate document-height screenshot ''' 27 | url = "https://gist.github.com/leemengtaiwan/e393d881222885f59ef09a14117159c8" 28 | self.driver.get(url) 29 | time.sleep(5) 30 | util.fullpage_screenshot(self.driver, "test.png") 31 | 32 | 33 | if __name__ == "__main__": 34 | unittest.main(argv=[sys.argv[0]]) -------------------------------------------------------------------------------- /web/util.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | import time 4 | import fire 5 | import hashlib 6 | import requests 7 | from PIL import Image 8 | 9 | GIST_BASE_URL = 'https://gist.github.com' 10 | DRIVER_WIDTH, DRIVER_HEIGHT = 1200, 1373 11 | 12 | def generate_hexhash(content): 13 | """Generate string representation of MD5 sum of given data 14 | 15 | Parameters 16 | ---------- 17 | content : dict 18 | A python dictionary representing gist's content 19 | 20 | Returns 21 | ------- 22 | hexhash : str. e.g. "c0e14a771bac3b4944318b430efe2884" 23 | """ 24 | 25 | data = bytearray(str(content)) 26 | md5 = hashlib.md5() 27 | md5.update(data) 28 | hexhash = md5.hexdigest() 29 | return hexhash 30 | 31 | 32 | def get_gist_hash(github_user, gist_name): 33 | """Acquire the raw content of the given `gist_name` and return string repr of the MD5 sum. 34 | 35 | Parameters 36 | ---------- 37 | github_user : str 38 | String representing valid Github account. e.g. "leemengtaiwan" 39 | 40 | gist_name : str 41 | Valid gist identifier appear in url 42 | 43 | Returns 44 | ------- 45 | hash: str 46 | 47 | """ 48 | # TODO update example for gist_name 49 | 50 | gist_raw_url = '/'.join((GIST_BASE_URL, github_user, gist_name, 'raw')) 51 | res = requests.request( 52 | method='GET', 53 | url=gist_raw_url 54 | ) 55 | assert res.status_code == requests.codes.ok, "Problem occurred when requesting raw gist." 56 | try: 57 | data = res.json() 58 | except ValueError: 59 | data = res.content 60 | 61 | return generate_hexhash(data) 62 | 63 | 64 | def fullpage_screenshot(driver, file): 65 | """Take multiple screenshots of already-opened webpage and save the concatenated image 66 | 67 | Parameters 68 | ---------- 69 | driver : selenium.webdriver 70 | The current active web driver staying in the page to take screenshots 71 | file : str 72 | The file path to save concatenated image 73 | 74 | Returns 75 | ------- 76 | bool 77 | 78 | Notes 79 | ----- 80 | Generate Fullpage Screenshot in Chrome 81 | http://seleniumpythonqa.blogspot.jp/2015/08/generate-full-page-screenshot-in-chrome.html 82 | 83 | """ 84 | image_id = re.findall(r'/([0-9a-zA-Z]+).', file)[0] 85 | 86 | 87 | print("Starting chrome full page screenshot workaround ...") 88 | total_width = driver.execute_script("return document.body.offsetWidth") 89 | total_height = driver.execute_script("return document.body.parentNode.scrollHeight") 90 | viewport_width = driver.execute_script("return document.body.clientWidth") 91 | viewport_height = driver.execute_script("return window.innerHeight") 92 | print("Total: ({0}, {1}), Viewport: ({2},{3})".format(total_width, total_height,viewport_width,viewport_height)) 93 | rectangles = [] 94 | 95 | i = 0 96 | while i < total_height: 97 | ii = 0 98 | top_height = i + viewport_height 99 | 100 | if top_height > total_height: 101 | top_height = total_height 102 | 103 | while ii < total_width: 104 | top_width = ii + viewport_width 105 | 106 | if top_width > total_width: 107 | top_width = total_width 108 | 109 | print("Appending rectangle ({0},{1},{2},{3})".format(ii, i, top_width, top_height)) 110 | rectangles.append((ii, i, top_width,top_height)) 111 | 112 | ii = ii + viewport_width 113 | 114 | i = i + viewport_height 115 | 116 | previous = None 117 | part = 0 118 | screenshots = [] # list of dict with 'file_name' and 'offset' fields 119 | total_width, total_height = 0, 0 120 | 121 | for rectangle in rectangles: 122 | if not previous is None: 123 | driver.execute_script("window.scrollTo({0}, {1})".format(rectangle[0], rectangle[1])) 124 | print("Scrolled To ({0},{1})".format(rectangle[0], rectangle[1])) 125 | time.sleep(.2) 126 | 127 | file_name = "{0}_part_{1}.png".format(image_id, part) 128 | print("Capturing {0} ...".format(file_name)) 129 | 130 | driver.get_screenshot_as_file(file_name) 131 | screenshot = Image.open(file_name) 132 | img_width, img_height = screenshot.size 133 | 134 | 135 | screenshots.append({ 136 | 'file_name': file_name, 'offset': (0, img_height * part) 137 | }) 138 | total_width = img_width 139 | total_height += img_height 140 | 141 | del screenshot 142 | part = part + 1 143 | previous = rectangle 144 | 145 | # concatenate all partial images 146 | print(total_width, total_height) 147 | stitched_image = Image.new('RGB', (total_width, total_height)) 148 | 149 | for screenshot in screenshots: 150 | file_name, offset = screenshot['file_name'], screenshot['offset'] 151 | print("Adding to stitched image with offset ({0}, {1})".format(offset[0], offset[1])) 152 | image = Image.open(file_name) 153 | stitched_image.paste(image, offset) 154 | os.remove(file_name) 155 | 156 | stitched_image.save(file) 157 | print("Finishing chrome full page screenshot workaround...") 158 | return True 159 | 160 | 161 | def create_chrome_driver(mode="headless", width=DRIVER_WIDTH, height=DRIVER_HEIGHT): 162 | """Create a headless/visible Chrome driver. 163 | 164 | Parameters 165 | ---------- 166 | mode : str, optional 167 | "headless" for creating a headless Chrome driver. 168 | Otherwise the driver will be visible. 169 | 170 | width : int, optional 171 | Width of the web driver window 172 | 173 | height : int, optional 174 | Height of the web driver window 175 | 176 | Returns 177 | ------- 178 | driver 179 | """ 180 | from selenium import webdriver 181 | options = webdriver.ChromeOptions() 182 | if mode == 'headless': 183 | options.add_argument("headless") 184 | else: 185 | pass 186 | driver = webdriver.Chrome(chrome_options=options) 187 | driver.get('https://github.com/') 188 | driver.set_window_size(width, height) 189 | return driver 190 | 191 | 192 | if __name__ == '__main__': 193 | fire.Fire() --------------------------------------------------------------------------------