├── dendron-links-filters ├── dendron_links_md.py ├── dendron_links_pdf.py └── _dendron_link_tools.py ├── LICENSE ├── .gitignore ├── README.md └── update_frontmatter.py /dendron-links-filters/dendron_links_md.py: -------------------------------------------------------------------------------- 1 | from pandocfilters import toJSONFilter 2 | from _dendron_link_tools import dendron_to_markdown_factory 3 | 4 | if __name__ == "__main__": 5 | toJSONFilter(dendron_to_markdown_factory( 6 | linktext_gen = lambda x : x.split('.')[-1], 7 | pref = '', 8 | ext = 'md', 9 | )) -------------------------------------------------------------------------------- /dendron-links-filters/dendron_links_pdf.py: -------------------------------------------------------------------------------- 1 | from pandocfilters import toJSONFilter 2 | from _dendron_link_tools import dendron_to_markdown_factory 3 | 4 | if __name__ == "__main__": 5 | toJSONFilter(dendron_to_markdown_factory( 6 | linktext_gen = lambda x : x.split('.')[-1], 7 | pref = '', 8 | ext = 'pdf', 9 | )) -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 mivanit 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | -------------------------------------------------------------------------------- /dendron-links-filters/_dendron_link_tools.py: -------------------------------------------------------------------------------- 1 | from typing import * 2 | import re 3 | 4 | from pandocfilters import toJSONFilter, Str, Link 5 | 6 | class LinkRegexPattern(object): 7 | def __init__( 8 | self, 9 | name : str, 10 | regex_raw : str, 11 | link_group_idx : int, 12 | ) -> None: 13 | 14 | self.name = name 15 | self.regex_raw = regex_raw 16 | self.link_group_idx = link_group_idx 17 | self.pattern = re.compile(regex_raw) 18 | 19 | LINK_REGEXS : Dict[str,LinkRegexPattern] = { 20 | 'md' : LinkRegexPattern( 21 | name = 'md', 22 | regex_raw = r'\[(.*?)\]\((.*?)\)', 23 | link_group_idx = 2, 24 | ), 25 | 'dendron' : LinkRegexPattern( 26 | name = 'dendron', 27 | regex_raw = r'\[\[(.*?)\]\]', 28 | link_group_idx = 1, 29 | ), 30 | # 'html' : LinkRegexPattern( 31 | # name = 'html', 32 | # regex_raw = r'(.*?)', 33 | # link_group_idx = 1, 34 | # ), 35 | } 36 | 37 | DENDRON_LINK_REGEX : LinkRegexPattern = LINK_REGEXS['dendron'] 38 | 39 | def get_dendron_link(data : str) -> Union[str,None]: 40 | """check if the given data is a dendron link 41 | 42 | if no link exists, return `None` 43 | otherwise, return the link text 44 | """ 45 | 46 | match : re.Match = DENDRON_LINK_REGEX.pattern.match(data) 47 | if match: 48 | return match.group(DENDRON_LINK_REGEX.link_group_idx) 49 | else: 50 | return None 51 | 52 | def convert_dlink_factory( 53 | linktext_gen : Callable = lambda x : x.split('.')[-1], 54 | pref : str = '', 55 | ext : str = 'md', 56 | ) -> Callable: 57 | 58 | def convert(link : str) -> str: 59 | return Link( 60 | [ "", [], [] ], 61 | [Str(linktext_gen(link))], 62 | [ 63 | f"{pref}{link}.{ext}", 64 | "", 65 | ], 66 | ) 67 | 68 | return convert 69 | 70 | 71 | def dendron_to_markdown_factory( 72 | linktext_gen : Callable = lambda x : x.split('.')[-1], 73 | pref : str = '', 74 | ext : str = 'md', 75 | ) -> Callable: 76 | 77 | converter = convert_dlink_factory( 78 | linktext_gen = linktext_gen, 79 | pref = pref, 80 | ext = ext, 81 | ) 82 | 83 | def filter( 84 | key : Any, 85 | value : Any, 86 | format : Any, 87 | meta : Any, 88 | ) -> None: 89 | """convert dendron links to markdown links""" 90 | 91 | # if its a plain string 92 | if key == "Str": 93 | dlink : Optional[str] = get_dendron_link(value) 94 | if dlink: 95 | output = converter(dlink) 96 | 97 | return output 98 | else: 99 | return None 100 | 101 | return filter 102 | 103 | 104 | if __name__ == '__main__': 105 | raise Exception("this is a module") 106 | 107 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # dendron-pandoc 2 | 3 | [Dendron](https://wiki.dendron.so) is a markdown based note taking tool, and [Pandoc](https://pandoc.org/MANUAL.html) is a document conversion tool. However, they don't always play nice together. I wanted to be able to compile my Dendron vault into nice PDFs using pandoc, which is why this exists. 4 | 5 | ## Features: 6 | - Pandoc filters for making Dendron links work properly 7 | - a script for adding bibliography information (or other data) to all markdown files in a vault 8 | - also, you should use [`pandoc-mermaid-filter`](https://github.com/timofurrer/pandoc-mermaid-filter) 9 | 10 | 11 | # `dendron-links-filters` 12 | Dendron links look like `[[link.to.a.file]]`, which does not work by default with Pandoc. Pandoc filters are provided to convert the links. 13 | 14 | ## Files: 15 | 16 | - `dendron_links_md.py`: parses the links into the Pandoc AST. 17 | - `dendron_links_pdf.py`: parses the links, but will additionally link to pdfs files of the same name instead of raw markdown files. 18 | - `_dendron_link_tools.py` provides the utilities for working with the Pandoc AST. 19 | 20 | ## Requirements 21 | 22 | - [`pandocfilters`](https://pypi.org/project/pandocfilters/) (a python package) 23 | - [Pandoc](https://pandoc.org/MANUAL.html) itself 24 | 25 | ## Usage: 26 | 27 | To convert a markdown file to a `.tex` file with working links to the original markdown: 28 | ```bash 29 | pandoc my.dendron.file.md -o my.dendron.file.tex --filter dendron_links_md.py 30 | ``` 31 | Or, if you intent to compile every file into a pdf: 32 | ```bash 33 | pandoc my.dendron.file.md -o my.dendron.file.tex --filter dendron_links_pdf.py 34 | ``` 35 | 36 | If you intent to compile many files, you may create a `defaults.yaml` file with the following: 37 | ```yaml 38 | filters: 39 | - dendron_links_pdf.py 40 | ``` 41 | 42 | and invoke it with 43 | ```bash 44 | pandoc my.dendron.file.md -o my.dendron.file.tex --defaults defaults.yaml 45 | ``` 46 | 47 | > more information about pandoc filters: https://pandoc.org/filters.html 48 | 49 | # `frontmatter` 50 | 51 | To be able to cite things from a bibtex file in your notes, you need to set the bibliography in the frontmatter. `update_frontmatter.py` sets the same bibliography in the frontmatter in all files in a directory. 52 | 53 | ## Usage: 54 | First, in `update_frontmatter.py` set `MY_REFS` equal to a list of paths (absolute, or relative to the *markdown file itself, **not** the python script*). 55 | 56 | Then, run the script, pointing it to your Dendron vault 57 | ```bash 58 | python update_frontmatter.py vault 59 | ``` 60 | (note: there may be some weirdness for joining paths) 61 | 62 | > designed to work with the vscode extension [PandocCiter](https://github.com/notZaki/PandocCiter) 63 | 64 | # mermaid 65 | 66 | mermaid is a tool for making charts and diagrams that is used in Dendron. Thankfully, there already exists a filter for making this work with Pandoc: [`pandoc-mermaid-filter`](https://github.com/timofurrer/pandoc-mermaid-filter) 67 | 68 | 69 | # planned features: 70 | 71 | - somehow getting citations to look nice in the Dendron preview 72 | - resolving LaTeX macros into raw LaTeX for the Dendron preview 73 | 74 | -------------------------------------------------------------------------------- /update_frontmatter.py: -------------------------------------------------------------------------------- 1 | from typing import * 2 | import os 3 | 4 | import yaml 5 | 6 | 7 | MY_REFS : List[str] = ['../refs.bib'] 8 | 9 | def keylist_access_nested_dict( 10 | d : Dict[str,Any], 11 | keys : List[str], 12 | ) -> Tuple[dict,str]: 13 | """given a keylist `keys`, return (x,y) where x[y] is d[keys] 14 | by pretending that `d` can be accessed dotlist-style, with keys in the list being keys to successive nested dicts, we can provide both read and write access to the element of `d` pointed to by `keys` 15 | 16 | ### Parameters: 17 | - `d : Dict[str,Any]` 18 | dict to access 19 | - `keys : List[str]` 20 | list of keys to nested dict `d` 21 | 22 | ### Returns: 23 | - `Tuple[dict,str]` 24 | dict is the final layer dict which contains the element pointed to by `keys`, and the string is the last key in `keys` 25 | """ 26 | 27 | fin_dict : dict = d 28 | for k in keys[:-1]: 29 | if k in fin_dict: 30 | fin_dict = fin_dict[k] 31 | else: 32 | fin_dict[k] = {} 33 | fin_dict = fin_dict[k] 34 | fin_key = keys[-1] 35 | 36 | return (fin_dict,fin_key) 37 | 38 | def fm_add_to_list( 39 | data : dict, 40 | keylist : List[str], 41 | insert_data : list, 42 | ) -> dict: 43 | """add things to the frontmatter 44 | 45 | given `keylist`, append to `data[keylist[0]][keylist[1]][...]` if it exists and does not contain `insert_data` 46 | if `data[keylist[0]][keylist[1]][...]` does not exist, create it and set it to `insert_data` 47 | """ 48 | fin_dict,fin_key = keylist_access_nested_dict(data,keylist) 49 | if fin_key not in fin_dict: 50 | fin_dict[fin_key] = insert_data 51 | else: 52 | for item in insert_data: 53 | if item not in fin_dict[fin_key]: 54 | fin_dict[fin_key].append(item) 55 | 56 | return data 57 | 58 | 59 | def fm_add_bib( 60 | data : dict, 61 | bibfiles : List[str] = MY_REFS, 62 | ) -> dict: 63 | """add the bib files to the frontmatter 64 | 65 | we want it to look like 66 | ```yaml 67 | bibliography: [../refs.bib] 68 | ``` 69 | """ 70 | 71 | return fm_add_to_list( 72 | data = data, 73 | keylist = ['bibliography'], 74 | insert_data = bibfiles, 75 | ) 76 | 77 | def fm_add_filters( 78 | data : dict, 79 | filters : List[str] = ['$FILTERS$/get_markdown_links.py'], 80 | ) -> dict: 81 | """add the filters to the frontmatter 82 | 83 | NOTE: this is for a different tool which allows defaults to be set in the frontmatter, 84 | instead of a separate file. That tools is kind of a mess, but email me if you're interested. 85 | 86 | we want it to look like 87 | ```yaml 88 | __defaults__: 89 | filters: 90 | - $FILTERS$/get_markdown_links.py 91 | ``` 92 | """ 93 | 94 | return fm_add_to_list( 95 | data = data, 96 | keylist = ['__defaults__', 'filters'], 97 | insert_data = filters, 98 | ) 99 | 100 | 101 | DEFAULT_KEYORDER : List[str] = [ 102 | 'title', 103 | 'desc', 104 | 'id', 105 | 'created', 106 | 'updated', 107 | 'bibliography', 108 | '__defaults__', 109 | 'traitIds', 110 | ] 111 | 112 | class PandocMarkdown(object): 113 | def __init__( 114 | self, 115 | delim : str = '---', 116 | loader : Callable[[str],dict] = yaml.safe_load, 117 | keyorder : List[str] = DEFAULT_KEYORDER, 118 | writer : Callable[[dict],str] = lambda x : yaml.dump(x, default_flow_style = None, sort_keys = False), 119 | ) -> None: 120 | 121 | self.delim = delim 122 | self.loader = loader 123 | self.keyorder = keyorder 124 | self.writer = writer 125 | 126 | # get the first section and parse as yaml 127 | self.yaml_data : Dict[str, Any] = None 128 | # get the content 129 | self.content : str = None 130 | 131 | def load(self, filename : str) -> None: 132 | """load a file into the pandoc markdown object 133 | 134 | ### Parameters: 135 | - `filename : str` 136 | the filename to load 137 | """ 138 | 139 | with open(filename, "r") as f: 140 | # split the document by yaml file front matter 141 | sections : List[str] = f.read().split(self.delim) 142 | 143 | # check the zeroth section is empty 144 | if sections[0].strip(): 145 | raise ValueError(f"file does not start with yaml front matter, found at start of file: {sections[0]}") 146 | 147 | if len(sections) < 3: 148 | raise ValueError(f'missing sections in file {filename}, check delims') 149 | 150 | # get the first section and parse as yaml 151 | self.yaml_data : Dict[str, Any] = self.loader(sections[1]) 152 | # get the content 153 | self.content : str = self.delim.join(sections[2:]) 154 | 155 | def dumps(self) -> str: 156 | """dumps both the front matter and content to a string 157 | 158 | NOTE: we want this to be on a single line for compatibility with https://github.com/notZaki/PandocCiter, since that tool parses the bibliography in a weird way. hence, `self.writer` has `default_flow_style = None` 159 | """ 160 | if (self.yaml_data is None) or (self.content is None): 161 | raise Exception('') 162 | 163 | self.keyorder = self.keyorder + [ 164 | k for k in self.yaml_data 165 | if k not in self.keyorder 166 | ] 167 | 168 | # for k in self.keyorder: 169 | # if not (k in self.yaml_data): 170 | # raise KeyError(f'key {k} found in keyorder but not in yaml_data') 171 | 172 | self.yaml_data = { 173 | k : self.yaml_data[k] 174 | for k in self.keyorder 175 | if k in self.yaml_data 176 | } 177 | 178 | return '\n'.join([ 179 | self.delim, 180 | self.writer(self.yaml_data).strip(), 181 | self.delim, 182 | self.content.lstrip(), 183 | ]) 184 | 185 | 186 | def modify_file_fm(file : str, apply_funcs : List[Callable]) -> None: 187 | pdm : PandocMarkdown = PandocMarkdown() 188 | pdm.load(file) 189 | 190 | for func in apply_funcs: 191 | pdm.yaml_data = func(pdm.yaml_data) 192 | 193 | with open(file, "w") as f: 194 | f.write(pdm.dumps()) 195 | 196 | def update_all_files_fm( 197 | dir : str, 198 | apply_funcs : List[Callable] = [fm_add_bib, fm_add_filters], 199 | ) -> None: 200 | """update the frontmatter of all files in a directory 201 | 202 | ### Parameters: 203 | - `dir : str` 204 | the directory to update 205 | - `apply_funcs : List[Callable]` 206 | list of functions to apply to the frontmatter 207 | """ 208 | 209 | for file in os.listdir(dir): 210 | if file.endswith(".md"): 211 | modify_file_fm(f'{dir.rstrip("/")}/{file}', apply_funcs) 212 | 213 | if __name__ == "__main__": 214 | import sys 215 | if len(sys.argv) < 2: 216 | print("Usage: python update_frontmatter.py ") 217 | sys.exit(1) 218 | 219 | update_all_files_fm( 220 | dir = sys.argv[1], 221 | apply_funcs = [fm_add_bib], 222 | ) --------------------------------------------------------------------------------