├── .gitignore
├── README.md
└── qt.py


/.gitignore:
--------------------------------------------------------------------------------
 1 | # Byte-compiled / optimized / DLL files
 2 | __pycache__/
 3 | *.py[cod]
 4 | *$py.class
 5 | 
 6 | # C extensions
 7 | *.so
 8 | 
 9 | # Distribution / packaging
10 | .Python
11 | env/
12 | build/
13 | develop-eggs/
14 | dist/
15 | downloads/
16 | eggs/
17 | .eggs/
18 | lib/
19 | lib64/
20 | parts/
21 | sdist/
22 | var/
23 | *.egg-info/
24 | .installed.cfg
25 | *.egg
26 | 
27 | # PyInstaller
28 | #  Usually these files are written by a python script from a template
29 | #  before PyInstaller builds the exe, so as to inject date/other infos into it.
30 | *.manifest
31 | *.spec
32 | 
33 | # Installer logs
34 | pip-log.txt
35 | pip-delete-this-directory.txt
36 | 
37 | # Unit test / coverage reports
38 | htmlcov/
39 | .tox/
40 | .coverage
41 | .coverage.*
42 | .cache
43 | nosetests.xml
44 | coverage.xml
45 | *,cover
46 | .hypothesis/
47 | 
48 | # Translations
49 | *.mo
50 | *.pot
51 | 
52 | # Django stuff:
53 | *.log
54 | 
55 | # Sphinx documentation
56 | docs/_build/
57 | 
58 | # PyBuilder
59 | target/
60 | 
61 | #Ipython Notebook
62 | .ipynb_checkpoints
63 | 
64 | # WingIDE project
65 | *.wpr
66 | *.wpu
67 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | quiver-tools aka qt
 2 | ===================
 3 | 
 4 | An utility to search or export Quiver notebooks to a markdown files.
 5 | 
 6 | [Quiver](http://happenapps.com/#quiver) is a great notebook app built for programmers. Code and text can be freely mixed. 
 7 | 
 8 | ## Features
 9 | 
10 | - Keep links between notes. 
11 | - Export and link resources
12 | - Automatic download javascript requirement:
13 | - Preserve sequence and flowchart diagrams
14 | - Create Notebooks and notes Index
15 | 
16 | 
17 | ## Installation
18 | 
19 | - [Download the script](https://raw.githubusercontent.com/mbelletti/quiver-tools/master/qt.py)
20 | - change `LIBRARY_PATH = "/changeme/Quiver.qvlibrary"` to your `Quiver.qvlibrary` or you can set with `-L` argument.
21 | - Put it on your path
22 | - `chmod u+x qt.py`
23 | 
24 | ## Usage
25 | 
26 | To export all your notes to `./tmp`
27 | 
28 | `./qt.py -s ./tmp`
29 | 
30 | 	~#./qt.py -h
31 | 
32 | 	usage: qt.py [-h] [-l] [-q QUERY] [-n NOTEBOOKS] [-e EXCLUDE_NOTEBOOKS] [-x EXPORT] [-v] [-i] [-L LIBRARY]
33 | 
34 | 	Quiver helper
35 | 
36 | 	options:
37 | 	-h, --help            show this help message and exit
38 | 	-l, --list            List notebooks
39 | 	-q QUERY, --query QUERY
40 | 							Search <query> in notes
41 | 	-n NOTEBOOKS, --notebooks NOTEBOOKS
42 | 							Restrict search in notebooks. Needs uuids space separated
43 | 	-e EXCLUDE_NOTEBOOKS, --exclude_notebooks EXCLUDE_NOTEBOOKS
44 | 							Exclude notebooks from search. Needs uuids space separated
45 | 	-x EXPORT, --export EXPORT
46 | 							Export to folder
47 | 	-v, --verbose         Verbose
48 | 	-i, --index           Create an index for each notebook
49 | 	-L LIBRARY, --library LIBRARY
50 | 							Quiver library path
51 | 
52 | 
53 | ## Feature Requests and Issues
54 | 
55 | Feel free to [add an issue or feature request](https://github.com/mbelletti/quiver-tools/issues). Pull requests are welcome as well! 
56 | 


--------------------------------------------------------------------------------
/qt.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python
  2 | # coding:utf-8
  3 | 
  4 | """
  5 |   Author: Massimiliano Belletti <max@belletti.net>
  6 |   Purpose: Helper for Quiver. Markdown export and Alfred search. 
  7 |   Created: 09/02/16
  8 | """
  9 | 
 10 | import sys
 11 | import os
 12 | import json
 13 | import argparse
 14 | import re
 15 | import unicodedata
 16 | import string
 17 | import imghdr
 18 | import pathlib
 19 | from typing import Any
 20 | import logging
 21 | 
 22 | 
 23 | log = logging.getLogger(__file__)  # create logger
 24 | # log_handler = logging.FileHandler(__APPNAME__ + ".log")
 25 | log_handler = logging.StreamHandler(sys.stdout)
 26 | log_formatter = logging.Formatter("%(asctime)s %(levelname)s %(message)s")
 27 | log_handler.setFormatter(log_formatter)
 28 | log.addHandler(log_handler)
 29 | log.setLevel(logging.ERROR)
 30 | 
 31 | LIBRARY_PATH = "/changeme/Quiver.qvlibrary"
 32 | 
 33 | 
 34 | def quiver(path):
 35 |     """
 36 |     Extract Notebooks and Notes from quiver json
 37 |     """
 38 |     book_ext = ".qvnotebook"
 39 | 
 40 |     def _get_notebooks() -> dict:
 41 |         """
 42 |         Notebooks generator
 43 |         """
 44 |         for n in sorted(os.listdir(path)):
 45 |             if n.endswith(book_ext):
 46 |                 d = json.loads(open(os.path.join(path, n, "meta.json")).read())
 47 |                 d["notes"] = _get_notes(d)
 48 |                 yield d
 49 | 
 50 |     def _get_notes(nb):
 51 |         """
 52 |         Notes generator
 53 |         """
 54 |         lpath = os.path.join(path, nb["uuid"] + book_ext)
 55 |         for n in sorted(os.listdir(lpath)):
 56 |             if ".json" in os.path.splitext(n):
 57 |                 pass
 58 |             else:
 59 |                 yield _get_note(nb, n)
 60 | 
 61 |     def _get_note(nb, notedir):
 62 |         "Note"
 63 |         lpath = os.path.join(path, nb["uuid"] + book_ext, notedir)
 64 |         n = json.loads(open(os.path.join(lpath, "meta.json")).read())
 65 |         n.update(
 66 |             dict(
 67 |                 nb=nb["name"],
 68 |                 nb_uuid=nb["uuid"],
 69 |             )
 70 |         )
 71 |         n.update(json.loads(open(os.path.join(lpath, "content.json")).read()))
 72 |         if "resources" in os.listdir(lpath):
 73 |             n.update({"resources": os.path.join(lpath, "resources")})
 74 |         return n
 75 | 
 76 |     return _get_notebooks()
 77 | 
 78 | 
 79 | def check_note(note, query) -> Any:
 80 |     """
 81 |     Filter note against query
 82 |     """
 83 |     if re.search(query, note["title"], flags=re.I):
 84 |         return note
 85 |     else:
 86 |         for c in note["cells"]:
 87 |             if re.search(query, c["data"], flags=re.I):
 88 |                 return note
 89 | 
 90 | 
 91 | def alfred_search(query, lib=LIBRARY_PATH, on_notebooks=None, exclude_notebooks=None):
 92 |     """
 93 |     Perform a search and return Alfred formatted items
 94 |     """
 95 | 
 96 |     min_chars = 2
 97 | 
 98 |     tag_tpl = "<%(name)s%(attrs)s>%(value)s</%(name)s>"
 99 |     tag = lambda name, value, attrs=[]: tag_tpl % dict(
100 |         name=name, value=value, attrs=(" " + " ".join(attrs) if attrs else "")
101 |     )
102 | 
103 |     attr = lambda name, value: '%s="%s"' % (name, value)
104 |     default_attrs = [attr("valid", "no")]
105 | 
106 |     def ae(title, sub, attrs=None):
107 |         """
108 |         Alfred element
109 | 
110 |         Represent a row in Alfred
111 | 
112 |         """
113 |         attrs = attrs or default_attrs
114 |         return tag(
115 |             "item", "".join([tag("title", title), tag("subtitle", sub)]), attrs=attrs
116 |         )
117 | 
118 |     def output(items):
119 |         # items is a list tags
120 |         xml_head = '<?xml version="1.0"?>'
121 |         out = tag("items", "".join([i for i in items]))
122 |         print(xml_head + out)
123 | 
124 |     note_ae = lambda note, attrs=default_attrs: ae(
125 |         note["title"], note["uuid"], attrs=attrs
126 |     )
127 |     notes_to_alfred = lambda notes: output(
128 |         [
129 |             note_ae(
130 |                 n,
131 |                 attrs=[
132 |                     attr("valid", "YES"),
133 |                     attr("arg", "quiver:///notes/" + n["uuid"]),
134 |                     attr("type", "file"),
135 |                 ],
136 |             )
137 |             for n in notes
138 |         ]
139 |     )
140 | 
141 |     if len(query) < min_chars:
142 |         items = [
143 |             ae(
144 |                 "Query too short",
145 |                 "The query needs to be at least " + str(min_chars) + " characters long",
146 |             )
147 |         ]
148 |         output(items)
149 |         return
150 | 
151 |     notebooks = quiver(lib)
152 | 
153 |     if exclude_notebooks:
154 |         notebooks = [nb for nb in notebooks if not nb["uuid"] in exclude_notebooks]
155 | 
156 |     if on_notebooks:
157 |         notebooks = [nb for nb in notebooks if nb["uuid"] in on_notebooks]
158 | 
159 |     notes = searchin_notebook(notebooks, query, exclude_notebooks)
160 |     notes_to_alfred(notes)
161 | 
162 | 
163 | def searchin_notebook(notebooks, query, exclude_notebooks=None):
164 |     for nb in notebooks:
165 |         if exclude_notebooks:
166 |             if not nb["uuid"] in exclude_notebooks:
167 |                 for n in searchin_notes(nb["notes"], query):
168 |                     yield n
169 |         else:
170 |             for n in searchin_notes(nb["notes"], query):
171 |                 yield n
172 | 
173 | 
174 | def searchin_notes(notes, query):
175 |     _ = []
176 |     for n in notes:
177 |         c = check_note(n, query)
178 |         if c:
179 |             _.append(n)
180 |     return _
181 | 
182 | 
183 | def md_export(notebooks, folder, index=True):
184 |     """Export quiver contents in markdown"""
185 | 
186 |     def create_index(nb_index, nb, sane, nf):
187 |         nb_index.append("[{}]({})\n".format(nb["name"], sane(nb["name"]) + "/index.md"))
188 |         index = []
189 |         for kk in nb["notes"]:
190 |             n = nb["notes"][kk]
191 |             index.append("[{}]({})\n".format(n["title"], sane(n["title"]) + ".md"))
192 |         with open(os.path.join(nf, "index.md"), mode="wb") as f:
193 |             f.write("[Notebooks](../index.md)\n\n".encode("utf8"))
194 |             f.write("# Index\n\n---\n".encode("utf8"))
195 |             h = None
196 |             for kk in sorted([i.lower() for i in index]):
197 |                 if h != kk[1].lower():
198 |                     h = kk[1].lower()
199 |                     f.write("## {}\n".format(h).encode("utf8"))
200 | 
201 |                 f.write("- {}\n".format(kk).encode("utf8"))
202 | 
203 |     validFilenameChars = "-_.(){}{}".format(string.ascii_letters, string.digits)
204 | 
205 |     def sane(filename):
206 |         cleanedFilename = (
207 |             unicodedata.normalize("NFKD", filename)
208 |             .encode("ASCII", "ignore")
209 |             .decode("ascii")
210 |             .replace(" ", "_")
211 |         )
212 |         return "".join(c for c in cleanedFilename if c in validFilenameChars)
213 | 
214 |     def check_fname(fname):
215 |         i = 0
216 |         name = fname
217 |         while os.path.exists(name):
218 |             i += 1
219 |             name = (
220 |                 os.path.splitext(fname)[0] + "_" + str(i) + os.path.splitext(fname)[1]
221 |             )
222 |         return name
223 | 
224 |     url_vendor = "/Applications/Quiver.app/Contents/Resources/dist/vendor"
225 | 
226 |     js_include = b"""\n<div>
227 |     <script src="http://code.jquery.com/jquery-1.4.2.min.js"></script>
228 |     <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
229 | 
230 |     <script src="../.resources/vendor/raphael-min.js"></script>
231 |     <script src="../.resources/vendor/underscore-min.js"></script>
232 |     <script src="../.resources/vendor/sequence-diagram-min.js"></script>
233 |     <script src="../.resources/vendor/flowchart.min.js"></script>
234 | </div>
235 |     """
236 | 
237 |     tpl_flow = b"""\n<div>
238 |     <script>
239 |         var diagram = flowchart.parse(document.getElementById('flowtext').innerText);
240 |         diagram.drawSVG('flow');
241 |     </script>
242 | </div>
243 |     """
244 |     tpl_seq = b"""\n<div>
245 |     <script>
246 |         $(".sequence").sequenceDiagram({theme: 'simple'});
247 |     </script>
248 | </div>
249 |     """
250 | 
251 |     def get_tree():
252 |         return {
253 |             nb["uuid"]: {
254 |                 "name": nb["name"],
255 |                 "notes": {n["uuid"]: n for n in nb["notes"]},
256 |             }
257 |             for nb in notebooks
258 |         }
259 | 
260 |     node_tree = get_tree()
261 | 
262 |     def get_note_filename(n):
263 |         try:
264 |             return sane(n["title"]) + ".md"
265 |         except Exception as e:
266 |             log.error(e)
267 |             raise e
268 | 
269 |     def search_in_tree(tree, k):
270 |         for x in tree:
271 |             if k in tree[x]["notes"]:
272 |                 return (
273 |                     "../"
274 |                     + sane(tree[x]["name"])
275 |                     + "/"
276 |                     + get_note_filename(tree[x]["notes"][k])
277 |                 )
278 |         return None
279 | 
280 |     def fix_image_link(stringa):
281 |         pattern = "(!\[IMAGE\]\(.*)(\ \=\d*x\d*)\)"
282 |         ms = stringa
283 |         m = re.findall(pattern, stringa)
284 |         while m:
285 |             ms = re.sub(pattern, "\g<1>)", ms)
286 |             m = re.findall(pattern, ms)
287 |             # print(ms)
288 |         return ms
289 | 
290 |     re_note_link = "quiver-note-url/([a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12})"
291 | 
292 |     folder = folder or "notes"
293 |     os.system('mkdir -p "%s"' % os.path.join(folder, ".resources"))
294 | 
295 |     os.system("cp -r %s %s" % (url_vendor, os.path.join(folder, ".resources")))
296 | 
297 |     nb_index = []
298 |     for k in node_tree:
299 |         nb = node_tree[k]
300 |         log.debug(nb["name"])
301 |         nf = os.path.join(folder, sane(nb["name"]))
302 |         os.system('mkdir -p "%s"' % nf)
303 |         if index:
304 |             create_index(nb_index, nb, sane, nf)
305 |         for kk in nb["notes"]:
306 |             n = nb["notes"][kk]
307 |             log.debug(n["title"])
308 |             resources = n.get("resources")
309 |             resources_path = pathlib.Path(nf) / "resources"
310 |             resources_renamed = {}
311 |             if resources:
312 |                 os.system('cp -r "%s" "%s"' % (n["resources"], nf))
313 |                 note_resources = resources_path.glob("*")
314 |                 for resource in note_resources:
315 |                     if not resource.suffix:
316 |                         ext = imghdr.what(resource)
317 |                         if ext:
318 |                             _ = resource.with_suffix("." + ext)
319 |                             resources_renamed[resource] = _
320 |                             resource.rename(_)
321 | 
322 |             j_included = False
323 |             fname = check_fname(os.path.join(nf, sane(n["title"]) + ".md"))
324 |             with open(fname, mode="wb") as f:
325 |                 if index:
326 |                     f.write("[Index](index.md)\n\n".encode("utf8"))
327 |                     f.write(f'# {n["title"]} \n\n'.encode("utf8"))
328 |                 if n["tags"]:
329 |                     f.write("tags: [".encode("utf8"))
330 |                     f.write(", ".join([f"`{t}`" for t in n["tags"]]).encode("utf8"))
331 |                     f.write("]".encode("utf8"))
332 |                     f.write("\n\n".encode("utf8"))
333 |                 for c in n["cells"]:
334 |                     # pattern = "\ \=.*x\d*"
335 | 
336 |                     s = c["data"].replace("quiver-image-url", "resources")
337 | 
338 |                     # s = re.sub(pattern, "", s)
339 |                     s = fix_image_link(s)
340 |                     for r in resources_renamed:
341 |                         s = s.replace(r.name, resources_renamed[r].name)
342 |                     s = s.replace("quiver-file-url", "resources")
343 |                     links = re.findall(re_note_link, s, re.I)
344 |                     for l in links:
345 |                         if not l in nb["notes"]:
346 |                             x = search_in_tree(node_tree, l)
347 |                             s = s.replace("quiver-note-url/" + l, x)
348 |                         else:
349 |                             s = s.replace(
350 |                                 "quiver-note-url/" + l,
351 |                                 get_note_filename(nb["notes"].get(l, "")),
352 |                             )
353 |                     s += "\n"
354 |                     if c["type"] == "code":
355 |                         s = "```\n" + s + "\n```"
356 |                         f.write(s.encode("utf8"))
357 |                     elif c["type"] == "diagram":
358 |                         if not j_included:
359 |                             f.write(js_include)
360 |                             j_included = True
361 |                         if c["diagramType"] == "sequence":
362 |                             s = '\n<div class="sequence">' + s + "</div>\n"
363 |                             f.write(s.encode("utf8"))
364 |                         elif c["diagramType"] == "flow":
365 |                             f.write(
366 |                                 str(
367 |                                     '\n<div id="flowtext">'
368 |                                     + s.replace("\n", "<br>")
369 |                                     + "</div>\n"
370 |                                 ).encode("utf8")
371 |                             )
372 |                             f.write('\n<div id="flow"></div>\n'.encode("utf8"))
373 |                             f.write(tpl_flow)
374 |                     else:
375 |                         f.write(s.encode("utf8"))
376 |                 if j_included:
377 |                     f.write(tpl_seq)
378 | 
379 |         if index:
380 |             with open(os.path.join(folder, "index.md"), mode="wb") as f:
381 |                 f.write("# Notebooks\n\n".format(n).encode("utf8"))
382 |                 for n in sorted(nb_index):
383 |                     f.write("- {}\n".format(n).encode("utf8"))
384 | 
385 | 
386 | def main():
387 |     parser = argparse.ArgumentParser(description="Quiver helper")
388 |     parser.add_argument("-l", "--list", help="List notebooks", action="store_true")
389 |     parser.add_argument(
390 |         "-q", "--query", help="Search <query> in notes", default=".*", type=str
391 |     )
392 |     parser.add_argument(
393 |         "-n",
394 |         "--notebooks",
395 |         help="""Restrict search in notebooks. Needs uuids space separated""",
396 |         type=str,
397 |     )
398 |     parser.add_argument(
399 |         "-e",
400 |         "--exclude_notebooks",
401 |         help="Exclude notebooks from search. Needs uuids space separated",
402 |         type=str,
403 |     )
404 |     parser.add_argument("-x", "--export", help="Export to folder", default="", type=str)
405 |     parser.add_argument(
406 |         "-v", "--verbose", help="Verbose", default=False, action="store_true"
407 |     )
408 |     parser.add_argument(
409 |         "-i", "--index", help="Create an index for each notebook", action="store_true"
410 |     )
411 |     parser.add_argument(
412 |         "-L", "--library", help="Quiver library path", default=LIBRARY_PATH, type=str
413 |     )
414 |     args = parser.parse_args()
415 | 
416 |     if not os.path.exists(args.library):
417 |         log.error("Quiver library not found. ")
418 |         log.error("%s doesn't exists." % args.library)
419 |         sys.exit(1)
420 | 
421 |     if args.verbose:
422 |         log.setLevel(logging.INFO)
423 | 
424 |     notebooks = quiver(args.library)
425 | 
426 |     if args.exclude_notebooks:
427 |         notebooks = [nb for nb in notebooks if not nb["uuid"] in args.exclude_notebooks]
428 | 
429 |     if args.notebooks:
430 |         notebooks = [nb for nb in notebooks if nb["uuid"] in args.notebooks]
431 | 
432 |     if args.list:
433 |         print(
434 |             ",\n".join(
435 |                 [
436 |                     str({k: nb[k] for k in nb if k != "notes"})
437 |                     for nb in notebooks
438 |                     if re.match(args.query, nb["name"])
439 |                 ]
440 |             )
441 |         )
442 |     elif args.export:
443 |         md_export(notebooks, args.export, index=args.index)
444 |     else:
445 |         notes = searchin_notebook(notebooks, args.query)
446 |         if notes:
447 |             print(
448 |                 ",\n".join(
449 |                     [
450 |                         str(
451 |                             {
452 |                                 "uuid": n["uuid"],
453 |                                 "title": n["title"],
454 |                                 "notebook": n["nb"],
455 |                             }
456 |                         )
457 |                         for n in notes
458 |                     ]
459 |                 )
460 |             )
461 |             # print "\n".join([n['title'] + ':\n' + n['uuid'] for n in notes])
462 |         else:
463 |             print("Nothing found")
464 | 
465 | 
466 | if __name__ == "__main__":
467 |     main()
468 |     sys.exit(0)
469 | 


--------------------------------------------------------------------------------