├── README.md ├── LICENSE ├── test_frontmatter.py └── tiddly2md.py /README.md: -------------------------------------------------------------------------------- 1 | # TiddlyWiki converter 2 | 3 | Converts a TiddlyWiki to individual markdown files. 4 | 5 | ## Requirements 6 | 7 | - pandas (for reading the CSV files) 8 | 9 | ## Instructions 10 | 11 | 1. Install the [ExportTiddlersPlugin](http://www.tiddlytools.com/#ExportTiddlersPlugin). 12 | 2. Export wiki as CSV file. 13 | 3. Run `tiddly2md.py` on the CSV file. 14 | 15 | python tiddly2md.py export.csv 16 | 17 | See `-h` for options. 18 | 19 | python tiddly2md.py -h 20 | 21 | ## Options worth discussing 22 | 23 | You can export only specific tags by using the `-t TAG` option. The tiddlers will be exported if the `TAG` text is part of the tiddler's tags. For example, the option `-t author` would export tiddlers that have the tag "author:name". Multiple values of `-t` options can be used to export multiple tags at once: 24 | 25 | python tiddly2md.py -t author -t concept export.csv 26 | 27 | 28 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Alexandre Chabot-Leclerc 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /test_frontmatter.py: -------------------------------------------------------------------------------- 1 | from typing import Union, List 2 | 3 | import pytest 4 | 5 | from tiddly2md import list_from_tags, extract_special_tags, add_quotation_marks, unnumbered_lists 6 | 7 | 8 | @pytest.mark.parametrize("raw, outlist", 9 | [ 10 | ('[[BaSe Session 14.07.2020: PWR_V3_2 aufgebaut, BPi aufsetzen]] python', ['BaSe Session 14.07.2020: PWR_V3_2 aufgebaut, BPi aufsetzen', 'python']), 11 | ('[[BaSe HW rev3b]] BaSe_PWR_V3_2 [[BaSe SBC-Funktionalität]]', ['BaSe HW rev3b', 'BaSe_PWR_V3_2', 'BaSe SBC-Funktionalität']), 12 | ('[[BaSe Platinen]]', ['BaSe Platinen']), 13 | ('protokoll [[BaSe Protokolle]]', ['protokoll', 'BaSe Protokolle']), 14 | ('oneword', ['oneword']), 15 | ('oneword anotheroneword', ['oneword', 'anotheroneword']), 16 | (float('nan'), [""]), 17 | ] 18 | ) 19 | def test_tags(raw: Union[str, float], outlist: List[str]): 20 | assert set(list_from_tags(raw)) == set(outlist) 21 | 22 | 23 | @pytest.mark.parametrize("raw, proc", [ 24 | (['a b'], ['"a b"']), 25 | (['a'], ['a']), 26 | (['a b', 'ab'], ['"a b"', 'ab']) 27 | ]) 28 | def test_add_quotation_marks(raw, proc): 29 | assert add_quotation_marks(raw) == proc 30 | 31 | 32 | @pytest.mark.parametrize("tags, frontmatter", [( 33 | ['c'], '---\nup: c\nprogrammiersprache: c\n---\n\n' 34 | )]) 35 | def test_extract_special_tags(tags: List[str], frontmatter: str): 36 | assert extract_special_tags(tags) == frontmatter 37 | 38 | 39 | @pytest.mark.parametrize("tid_list, md_list", [ 40 | ("* list item1", "- list item1"), 41 | ("*list item1", "- list item1"), 42 | ("**list item2", " - list item2") 43 | ]) 44 | def test_unnumbered_lists(tid_list: str, md_list: str): 45 | assert unnumbered_lists(tid_list) == md_list -------------------------------------------------------------------------------- /tiddly2md.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import re 3 | import os 4 | import sys 5 | from typing import Union, List, Dict 6 | 7 | import pandas as pd 8 | 9 | 10 | def good_tag(tag, valid_tags): 11 | """Returns True if tag contains of the valid tags. 12 | 13 | :param tag: string 14 | :param valid_tags: list of valid tags 15 | """ 16 | for each in valid_tags: 17 | if each in str(tag): 18 | return True 19 | return False 20 | 21 | 22 | def numbered_lists(line: str) -> str: 23 | list_level = 0 24 | while line.startswith('#'): 25 | line = line[1:] 26 | list_level += 1 27 | if list_level: 28 | prefix = " "*(list_level - 1)*2 + "0. " 29 | line = prefix + line.strip() 30 | return line 31 | 32 | 33 | def unnumbered_lists(line: str) -> str: 34 | list_level = 0 35 | while line.startswith('*'): 36 | line = line[1:] 37 | list_level += 1 38 | if list_level: 39 | prefix = " "*(list_level - 1)*2 + "- " 40 | line = prefix + line.strip() 41 | return line 42 | 43 | 44 | def lists(line: str) -> str: 45 | line = line.strip() 46 | line = numbered_lists(line) 47 | line = unnumbered_lists(line) 48 | return line 49 | 50 | 51 | def wiki_to_md(text): 52 | """Convert wiki formatting to markdown formatting. 53 | 54 | :param text: string of text to process 55 | 56 | :return: processed string 57 | """ 58 | if isinstance(text, float): 59 | return "" 60 | fn = [] 61 | new_text = [] 62 | fn_n = 1 # Counter for footnotes 63 | for line in text.split('\n'): 64 | # lists (has to be first!) 65 | line = lists(line) 66 | # Replace wiki headers with markdown headers 67 | match = re.match('(!+)(\\s?)[^\\[]', line) 68 | if match: 69 | header, spaces = match.groups() 70 | new_str = '#' * len(header) 71 | line = re.sub('(!+)(\\s?)([^\\[])', new_str + ' ' + '\\3', line) 72 | # Underline (doesn't exist in MD) 73 | line = re.sub("__(.*?)__", "\\1", line) 74 | # Bold 75 | line = re.sub("''(.*?)''", "**\\1**", line) 76 | # Italics 77 | line = re.sub("//(.*?)//", "_\\1_", line) 78 | # Remove wiki links 79 | line = re.sub("\\[\\[(\\w+?)\\]\\]", "\\1", line) 80 | # Change links to markdown format 81 | line = re.sub("\\[\\[(.*)\\|(.*)\\]\\]", "[\\1](\\2)", line) 82 | # Code 83 | line = re.sub("\\{\\{\\{(.*?)\\}\\}\\}", "`\\1`", line) 84 | # Footnotes 85 | match = re.search("```(.*)```", line) 86 | if match: 87 | text = match.groups()[0] 88 | fn.append(text) 89 | line = re.sub("```(.*)```", '[^{}]'.format(fn_n), line) 90 | fn_n += 1 91 | new_text.append(line) 92 | 93 | # Append footnotes 94 | for i, each in enumerate(fn): 95 | new_text.append('[^{}]: {}'.format(i+1, each)) 96 | return '\n'.join(new_text) 97 | 98 | 99 | def sanitize(value): 100 | """Makes sure filenames are valid by replacing illegal characters 101 | 102 | :param value: string 103 | """ 104 | value = str(value) 105 | #value = strdata.normalize('NFKD', value).encode('ascii', 'ignore') 106 | value = str(re.sub('[^\w\s-]', '', value).strip()) 107 | return value 108 | 109 | 110 | def list_from_tags(raw: Union[float, str]) -> List[str]: 111 | if isinstance(raw, float): 112 | return [""] 113 | tags = [] 114 | while "[[" in raw: 115 | parts = raw.split('[[', 1) 116 | remainder = parts[0] 117 | parts = parts[1].split(']]', 1) 118 | remainder += parts[1] 119 | raw = remainder 120 | tags.append(parts[0]) 121 | raw = raw.strip() 122 | while " " in raw: 123 | parts = raw.split(' ', 1) 124 | remainder = parts[0] 125 | parts = parts[1].split(' ', 1) 126 | if len(parts) > 1: 127 | remainder += parts[1] 128 | raw = remainder 129 | tags.append(parts[0]) 130 | if raw: 131 | tags.append(raw) 132 | return tags 133 | 134 | 135 | def extract_special_tags(tags: List[str]) -> str: 136 | fm: Dict[str, str] = {"tags": "", "up": ", ".join(tags)} 137 | langs = ['c','c++','eagle: ulp','html','javascript','nodejs','php','python','Verilog','VHDL'] 138 | for tag in tags: 139 | if tag in langs: 140 | fm["programmiersprache"] = tag 141 | else: 142 | fm["tags"] += tag + ", " 143 | if fm["tags"].endswith(", "): 144 | fm["tags"] = fm["tags"][:-2] 145 | if fm["tags"] == "": 146 | fm.pop("tags") 147 | fm_string = "---\n" 148 | for key, value in fm.items(): 149 | fm_string += f"{key}: {value}\n" 150 | fm_string += "---\n\n" 151 | return fm_string 152 | 153 | 154 | def add_quotation_marks(tags: List[str]) -> List[str]: 155 | outlist = [] 156 | for tag in tags: 157 | if " " in tag: 158 | outlist.append('"'+tag+'"') 159 | else: 160 | outlist.append(tag) 161 | return outlist 162 | 163 | 164 | def frontmatter(raw_tags: Union[float, str]) -> str: 165 | tags = list_from_tags(raw_tags) 166 | tags = add_quotation_marks(tags) 167 | return extract_special_tags(tags) 168 | 169 | 170 | def main(args): 171 | 172 | output_path = args.outdir 173 | try: 174 | os.mkdir(output_path) 175 | except OSError: 176 | pass 177 | 178 | df = pd.read_csv(args.input_file) 179 | if args.tags: 180 | df = df[df.tags.apply(lambda x: good_tag(x, args.tags))] 181 | 182 | for row_id, row in df.iterrows(): 183 | filename = sanitize(row.title) 184 | filename = '{}.{}'.format(filename, args.ext) 185 | with open(os.path.join(output_path, filename), 'w') as f: 186 | try: 187 | f.write(frontmatter(row.tags)) 188 | f.write(wiki_to_md(row.text)) 189 | except Exception as e: 190 | print(f"{e} in {row['title']}") 191 | f.write('') 192 | 193 | 194 | if __name__ == '__main__': 195 | parser = argparse.ArgumentParser( 196 | description="Convert TiddlyWiki tiddlers exported as CSV to " 197 | "individual files.") 198 | parser.add_argument('--ext', '-e', dest='ext', 199 | default='md', 200 | help='File extension (defaults to "md")') 201 | parser.add_argument('--outdir', '-o', 202 | default='output', 203 | dest='outdir', 204 | help='Output folder (defaults to "output")') 205 | parser.add_argument('--tags', '-t', dest='tags', 206 | action='append', 207 | help='Valid tag to export, can have multiple') 208 | parser.add_argument('input_file', 209 | help='Exported CSV file') 210 | args = parser.parse_args() 211 | 212 | sys.exit(main(args)) 213 | --------------------------------------------------------------------------------