├── __init__.py ├── filterCSV.py ├── images ├── importable.png └── importedMarkdown.png ├── iThoughts ├── importable.itmz └── importedMarkdown.itmz ├── tests ├── iThoughts-OPML.itmz ├── badLevels.csv ├── test1.csv ├── test2.md ├── expected │ ├── A2AX_keep_test1_csv_err.txt │ ├── markdown_2_3_mdTest3_csv_err.txt │ ├── promote_2_promotion_csv_err.txt │ ├── test2_md_err.txt │ ├── A1_3_note_test1_csv_err.txt │ ├── check_repairsubtree_badLevels_csv_out.txt │ ├── A2AX_keep_test1_csv_out.txt │ ├── markdown_2_3_mdTest3_csv_out.txt │ ├── A1_3_note_test1_csv_out.txt │ ├── xml_freemind_mdTest3_csv_err.txt │ ├── test2_md_out.txt │ ├── promote_2_promotion_csv_out.txt │ ├── check_repairsubtree_badLevels_csv_err.txt │ └── xml_freemind_mdTest3_csv_out.txt ├── mdTest3.csv ├── iThoughts-OPML.opml ├── promotion.csv └── README.md ├── flatfiles ├── importable.csv └── importedMarkdown.md ├── LICENSE ├── .travis.yml ├── .github └── workflows │ └── lint_python.yml ├── test_filterCSV.py ├── README.md └── filterCSV /__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /filterCSV.py: -------------------------------------------------------------------------------- 1 | filterCSV -------------------------------------------------------------------------------- /images/importable.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MartinPacker/filterCSV/HEAD/images/importable.png -------------------------------------------------------------------------------- /iThoughts/importable.itmz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MartinPacker/filterCSV/HEAD/iThoughts/importable.itmz -------------------------------------------------------------------------------- /tests/iThoughts-OPML.itmz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MartinPacker/filterCSV/HEAD/tests/iThoughts-OPML.itmz -------------------------------------------------------------------------------- /images/importedMarkdown.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MartinPacker/filterCSV/HEAD/images/importedMarkdown.png -------------------------------------------------------------------------------- /iThoughts/importedMarkdown.itmz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MartinPacker/filterCSV/HEAD/iThoughts/importedMarkdown.itmz -------------------------------------------------------------------------------- /tests/badLevels.csv: -------------------------------------------------------------------------------- 1 | "shape","level","level0","level1","level2" 2 | ,1,,A 3 | ,2,,,A1 4 | ,2,,,A2 5 | ,3,,,,A2A 6 | square,0,X -------------------------------------------------------------------------------- /tests/test1.csv: -------------------------------------------------------------------------------- 1 | "shape","level","level0","level1","level2" 2 | ,0,A 3 | ,1,,A1 4 | ,1,,A2 5 | ,2,,,A2A 6 | ,3,,,,A2A1 7 | square,0,X -------------------------------------------------------------------------------- /tests/test2.md: -------------------------------------------------------------------------------- 1 | * A 2 | * A1 3 | * A1A 4 | * A1B 5 | * B 6 | * B1 7 | * B1A 8 | * B1A1 9 | * C 10 | * D 11 | -------------------------------------------------------------------------------- /flatfiles/importable.csv: -------------------------------------------------------------------------------- 1 | "colour","shape","level","level0","level1","level2" 2 | 00FFFF,,0,"A" 3 | ,triangle,1,,A1 4 | ,square,1,,A2 5 | FF0000,square,2,,,A2A -------------------------------------------------------------------------------- /flatfiles/importedMarkdown.md: -------------------------------------------------------------------------------- 1 | # Fruit 2 | 3 | ## Citrus 4 | 5 | * Lemon 6 | * Orange 7 | * Vaguely orange-like 8 | * Mandarin 9 | * Satsuma 10 | -------------------------------------------------------------------------------- /tests/expected/A2AX_keep_test1_csv_err.txt: -------------------------------------------------------------------------------- 1 | Criterion Actions 2 | --------- ------- 3 | A2A|X keep 4 | -------------------------------------------------------------------------------- /tests/expected/markdown_2_3_mdTest3_csv_err.txt: -------------------------------------------------------------------------------- 1 | Criterion Actions 2 | --------- ------- 3 | markdown 2,3 4 | -------------------------------------------------------------------------------- /tests/expected/promote_2_promotion_csv_err.txt: -------------------------------------------------------------------------------- 1 | Criterion Actions 2 | --------- ------- 3 | promote 2 4 | -------------------------------------------------------------------------------- /tests/expected/test2_md_err.txt: -------------------------------------------------------------------------------- 1 | Criterion Actions 2 | --------- ------- 3 | 4 | 5 | Input type detected as 'markdown'. 6 | 7 | Indentation detected: 8 | -------------------------------------------------------------------------------- /tests/expected/A1_3_note_test1_csv_err.txt: -------------------------------------------------------------------------------- 1 | Criterion Actions 2 | --------- ------- 3 | ^A1$ 3,note 4 | 5 | 6 | Input type detected as 'iThoughtsCSV'. 7 | -------------------------------------------------------------------------------- /tests/expected/check_repairsubtree_badLevels_csv_out.txt: -------------------------------------------------------------------------------- 1 | "colour","note","position","shape","level","level0","level1","level2" 2 | "","","","","0","A" 3 | "","","","","1","","A1" 4 | "","","","","1","","A2" 5 | "","","","","2","","","A2A" 6 | "","","","square","0","X" 7 | -------------------------------------------------------------------------------- /tests/expected/A2AX_keep_test1_csv_out.txt: -------------------------------------------------------------------------------- 1 | "colour","note","position","shape","level","level0","level1","level2","level3" 2 | "","","","","0","A" 3 | "","","","","1","","A2" 4 | "","","","","2","","","A2A" 5 | "","","","","3","","","","A2A1" 6 | "","","","square","0","X" 7 | -------------------------------------------------------------------------------- /tests/mdTest3.csv: -------------------------------------------------------------------------------- 1 | "note",shape","level","level0","level1","level2" 2 | ,,0,A 3 | ,,1,,A1 4 | ,,1,,A2 5 | ,,2,,,A2A 6 | This is a test note - to see how it gets rendered.,,3,,,,A2A1 7 | ,,3,,,,A2A2 8 | ,,4,,,,,A2A21 9 | "This is another test note with a second line",square,0,X -------------------------------------------------------------------------------- /tests/expected/markdown_2_3_mdTest3_csv_out.txt: -------------------------------------------------------------------------------- 1 | ### A 2 | 3 | 4 | #### A1 5 | 6 | 7 | #### A2 8 | 9 | * A2A 10 | * A2A1 11 |

This is a test note - to see how it gets rendered. 12 | 13 | * A2A2 14 | * A2A21 15 | 16 | ### X 17 | 18 | This is another test note with a second line 19 | -------------------------------------------------------------------------------- /tests/expected/A1_3_note_test1_csv_out.txt: -------------------------------------------------------------------------------- 1 | "colour","note","position","shape","level","level0","level1","level2","level3" 2 | "","","","","0","A" 3 | "FFFFB2","Matched ^A1$","","","1","","A1" 4 | "","","","","1","","A2" 5 | "","","","","2","","","A2A" 6 | "","","","","3","","","","A2A1" 7 | "","","","square","0","X" 8 | -------------------------------------------------------------------------------- /tests/expected/xml_freemind_mdTest3_csv_err.txt: -------------------------------------------------------------------------------- 1 | Criterion Actions 2 | --------- ------- 3 | xml freemind 4 | 5 | 6 | Input type detected as 'iThoughtsCSV'. 7 | 8 | Exported XML will have more than 1 root node. Some programs will get confused by this. Continuing. 9 | -------------------------------------------------------------------------------- /tests/iThoughts-OPML.opml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Central Idea - Really Level 0 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /tests/promotion.csv: -------------------------------------------------------------------------------- 1 | "colour","level","level0","level1","level2","level3","note" 2 | "","0","A" 3 | "DDDDDD","1","","B" 4 | "DDDDFF","2","","","C" 5 | "DDDDFF","2","","","D" 6 | "DDDDFF","3","","","","D1" 7 | "DDDDFF","3","","","","D2" 8 | "DDDDFF","2","","","C1" 9 | "DDDDFF","2","","","C2" 10 | "DDDDDD","1","","X" 11 | "DDDDFF","2","","","Y" 12 | "DDDDFF","3","","","","Z" 13 | -------------------------------------------------------------------------------- /tests/expected/test2_md_out.txt: -------------------------------------------------------------------------------- 1 | "colour","note","position","shape","level","level0","level1","level2","level3" 2 | "","","","","0","A" 3 | "","","","","1","","A1" 4 | "","","","","2","","","A1A" 5 | "","","","","2","","","A1B" 6 | "","","","","0","B" 7 | "","","","","1","","B1" 8 | "","","","","2","","","B1A" 9 | "","","","","3","","","","B1A1" 10 | "","","","","0","C" 11 | "","","","","0","D" 12 | -------------------------------------------------------------------------------- /tests/expected/promote_2_promotion_csv_out.txt: -------------------------------------------------------------------------------- 1 | "colour","note","position","shape","level","level0","level1","level2","level3","level4","level5","level6","level7","level8","level9","level10","level11","level12","level13","level14","level15","level16","level17","level18","level19","level20" 2 | "","","","","0","A" 3 | "DDDDFF","","","","1","","C" 4 | "DDDDFF","","","","1","","D" 5 | "DDDDFF","","","","2","","","D1" 6 | "DDDDFF","","","","2","","","D2" 7 | "DDDDFF","","","","1","","C1" 8 | "DDDDFF","","","","1","","C2" 9 | "DDDDFF","","","","1","","Y" 10 | "DDDDFF","","","","2","","","Z" 11 | -------------------------------------------------------------------------------- /tests/README.md: -------------------------------------------------------------------------------- 1 | ## Test Files 2 | 3 | This directory contains test files that you can test against and study to become familiar with filterCSV: 4 | 5 | * **test1.csv** is a simple CSV file in a format that can be imported into iThoughts. 6 | * **test2.md** is a simple nested-list Markdown file. Here indentation level is denoted by two spaces for the first level, and so on. 7 | * **mdtest3.csv** is a file with 5 levels of nodes, designed to test Markdown export. 8 | * **badlevels.csv** contains a level error - for use with `check`. 9 | 10 | There is also an **expected** directory that is used by pytest to verify that our file tests are sending the expected output to stdout and stderr. 11 | -------------------------------------------------------------------------------- /tests/expected/check_repairsubtree_badLevels_csv_err.txt: -------------------------------------------------------------------------------- 1 | Criterion Actions 2 | --------- ------- 3 | check repairsubtree 4 | 5 | 6 | Input type detected as 'iThoughtsCSV'. 7 | 8 | Beginning Level Check 9 | --------------------- 10 | A Error: Expected level 0. Found level 1. Repaired this node (setting its level to 0) and all its child nodes. 11 | A1 OK: Found level 1. 12 | A2 OK: Found level 1. 13 | A2A OK: Found level 2. 14 | X OK: Found level 0. 15 | --------------------- 16 | Completed Level Check 17 | -------------------------------------------------------------------------------- /tests/expected/xml_freemind_mdTest3_csv_out.txt: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 |

12 | This is a test note - to see how it gets rendered. 13 |

14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 |

29 | This is another test note with a second line 30 |

31 | 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Martin Packer 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | os: linux 2 | dist: bionic 3 | language: python 4 | python: 3.8 5 | install: pip install flake8 6 | jobs: 7 | include: 8 | - script: cat tests/badLevels.csv | ./filterCSV check repairsubtree # Test the ability to repair a file 9 | - script: cat tests/test1.csv | ./filterCSV '^A1$' '3 note' # Basic colouring and noting test 10 | - script: cat tests/test2.md | ./filterCSV # Test of Markdown list import 11 | - script: cat tests/mdTest3.csv | ./filterCSV markdown '2 3' # Test of Markdown export 12 | - script: cat tests/mdTest3.csv | ./filterCSV xml freemind # Test of Freemind XML export 13 | - script: cat tests/test1.csv | ./filterCSV 'A2A|X' 'keep' # Test of 'keep' - with two surviving trees 14 | # - script: cat tests/promotion.csv | ./filterCSV promote 2 # Test of promote 15 | - script: pydoc ./filterCSV.py 16 | before_script: 17 | - flake8 . --max-complexity=76 --max-line-length=88 --show-source --statistics 18 | - # ln -s filterCSV filterCSV.py 19 | - ls -la ; ls -la tests 20 | - pytest --doctest-modules 21 | notifications: 22 | email: false 23 | -------------------------------------------------------------------------------- /.github/workflows/lint_python.yml: -------------------------------------------------------------------------------- 1 | name: lint_python 2 | on: 3 | pull_request: 4 | push: 5 | branches: [master] 6 | jobs: 7 | lint_python: 8 | runs-on: ubuntu-latest 9 | # strategy: 10 | # matrix: 11 | # os: [ubuntu-latest, macos-latest, windows-latest] 12 | # python-version: [2.7, 3.5, 3.6, 3.7, 3.8] # , pypy3] 13 | steps: 14 | - uses: actions/checkout@master 15 | - uses: actions/setup-python@master 16 | - run: pip install black codespell flake8 isort pytest 17 | - run: black --check . || true 18 | - run: black --diff . || true 19 | # - if: matrix.python-version >= 3.6 20 | # run: | 21 | # pip install black 22 | # black --check . 23 | - run: codespell --quiet-level=2 # --ignore-words-list="" --skip="" 24 | - run: flake8 . --max-complexity=76 --max-line-length=88 --show-source --statistics 25 | - run: isort --recursive . || true 26 | - run: pip install -r requirements.txt || true 27 | # - run: ln -s filterCSV filterCSV.py 28 | - run: pytest --doctest-modules 29 | - run: | 30 | cat tests/badLevels.csv | ./filterCSV check repairsubtree # Test the ability to repair a file 31 | cat tests/test1.csv | ./filterCSV '^A1$' '3 note' # Basic colouring and noting test 32 | cat tests/test2.md | ./filterCSV # Test of Markdown list import 33 | cat tests/mdTest3.csv | ./filterCSV markdown '2 3' # Test of Markdown export 34 | cat tests/mdTest3.csv | ./filterCSV xml freemind # Test of Freemind XML export 35 | cat tests/test1.csv | ./filterCSV 'A2A|X' 'keep' # Test of 'keep' - with two surviving trees 36 | # cat tests/promotion.csv | ./filterCSV promote 2 # Test of promote 37 | -------------------------------------------------------------------------------- /test_filterCSV.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # To make it easy to import ../filterCSV even though it has no .py extension, we did: 4 | # ln -s ./filterCSV filterCSV.py # make a symlink from ./filterCSV to ./filterCSV.py 5 | 6 | import os 7 | import string 8 | import subprocess 9 | 10 | import pytest 11 | 12 | from . import filterCSV 13 | 14 | data_fields = ("shape", "colour", "note", "level", "position", "cell") 15 | 16 | 17 | def test_CSVTree(): 18 | csv_tree = filterCSV.CSVTree(*data_fields) 19 | assert csv_tree.childNodes == [] 20 | for field in data_fields: 21 | assert csv_tree.data[field] == field 22 | assert csv_tree.parent is None 23 | assert csv_tree.matched is False 24 | 25 | 26 | def test_CSVTree_add_get_delete(): 27 | csv_tree = filterCSV.CSVTree(*data_fields) 28 | child = filterCSV.CSVTree(*["child"] * 6) 29 | csv_tree.addChild(child) 30 | assert len(csv_tree.childNodes) == 1 31 | assert child in csv_tree.getChildren() 32 | csv_tree.deleteChild(child) 33 | assert csv_tree.childNodes == [] 34 | assert child not in csv_tree.getChildren() 35 | 36 | 37 | def test_CSVTree_isMatch(): 38 | csv_tree = filterCSV.CSVTree(*data_fields) 39 | for criterion in "shape ape colour col r note no cell ell".split(): 40 | assert csv_tree.isMatch(criterion), criterion 41 | for criterion in "level position x X y Y z Z 1 2 3 4 5 6 7 8 9 0".split(): 42 | assert not csv_tree.isMatch(criterion), criterion 43 | 44 | 45 | expected = ( 46 | "shape colour note 0 position cell", 47 | " child child child 1 child child", 48 | " grandchild grandchild grandchild 3 grandchild grandchild", 49 | ) 50 | 51 | 52 | def test_CSVTree_dump(): 53 | csv_tree = filterCSV.CSVTree(*data_fields) 54 | csv_tree.data["level"] = 0 55 | actual = csv_tree.dump() 56 | assert expected[0] in actual, (expected[0] in actual) 57 | child = filterCSV.CSVTree(*["child"] * 6) 58 | child.data["level"] = 1 59 | csv_tree.addChild(child) 60 | actual = csv_tree.dump() 61 | assert "\n".join(expected[0:1]) in actual, ("\n".join(expected[0:1]) in actual) 62 | child.addChild(filterCSV.CSVTree(*["grandchild"] * 6)).data["level"] = 3 63 | actual = csv_tree.dump() 64 | assert "\n".join(expected) in actual, ("\n".join(expected) in actual) 65 | 66 | 67 | def test_calculateMaximumLevel(): 68 | csv_tree = filterCSV.CSVTree(*data_fields) 69 | csv_tree.data["level"] = 1 # test as int 70 | assert csv_tree.calculateMaximumLevel() == 1 71 | csv_tree.data["level"] = "0" # test as str 72 | assert csv_tree.calculateMaximumLevel() == 0 73 | # make a child 74 | child = filterCSV.CSVTree(*["child"] * 6) 75 | child.data["level"] = "2" 76 | csv_tree.addChild(child) 77 | assert csv_tree.calculateMaximumLevel() == 2 78 | # make a grandchild 79 | grandchild = filterCSV.CSVTree(*["grandchild"] * 6) 80 | grandchild.data["level"] = "4" # test as str 81 | child.addChild(grandchild) 82 | assert csv_tree.calculateMaximumLevel() == 4 83 | assert child.calculateMaximumLevel() == 4 84 | assert grandchild.calculateMaximumLevel() == 4 85 | grandchild.data["level"] = 5 # test as int 86 | assert csv_tree.calculateMaximumLevel() == 5 87 | csv_tree.deleteChild(child) 88 | assert csv_tree.calculateMaximumLevel() == 0 89 | 90 | 91 | def test_writeCSVTree(): 92 | outputArray = ["a", "b", "c"] 93 | csv_tree = filterCSV.CSVTree(*data_fields) 94 | csv_tree.data["level"] = 1 95 | assert csv_tree.writeCSVTree(outputArray) == ['a', 'b', 'c', 96 | ['colour', 'note', 'position', 'shape', 1, '', 'cell'] # noqa: E128 97 | ] 98 | csv_tree.data["level"] = 0 99 | assert csv_tree.writeCSVTree(outputArray) == ['a', 'b', 'c', 100 | ['colour', 'note', 'position', 'shape', 1, '', 'cell'], # noqa: E128 101 | ['colour', 'note', 'position', 'shape', 0, 'cell'] 102 | ] 103 | csv_tree.data["level"] = -1 104 | assert csv_tree.writeCSVTree(outputArray) == ['a', 'b', 'c', 105 | ['colour', 'note', 'position', 'shape', 1, '', 'cell'], # noqa: E128 106 | ['colour', 'note', 'position', 'shape', 0, 'cell'] 107 | ] 108 | 109 | 110 | testdata = { 111 | " \t \t \t": "", 112 | "a b \tc\t\td": "", 113 | " filterCSV ": "", 114 | } 115 | 116 | 117 | @pytest.mark.parametrize("whitespace,expected", testdata.items()) 118 | def test_formatWhitespaceCharacters(whitespace, expected): 119 | assert filterCSV.formatWhitespaceCharacters(whitespace) == expected 120 | 121 | 122 | def test_no_spaces( 123 | whitespace=string.ascii_letters + string.digits + string.punctuation 124 | ): 125 | s = filterCSV.formatWhitespaceCharacters(whitespace) 126 | assert "s" not in s 127 | assert "p" not in s 128 | # "a" is in both and 129 | assert "c" not in s 130 | assert "e" not in s 131 | 132 | 133 | # cat tests/badLevels.csv | ./filterCSV check repairsubtree 134 | # cat tests/test1.csv | ./filterCSV '^A1$' '3 note' 135 | testdata = { 136 | "check repairsubtree": "tests/badLevels.csv", 137 | "^A1$ 3_note": "tests/test1.csv", 138 | "": "tests/test2.md", 139 | "markdown 2_3": "tests/mdTest3.csv", 140 | "xml freemind": "tests/mdTest3.csv", 141 | "A2A|X keep": "tests/test1.csv", 142 | # "promote 2": "tests/promotion.csv", 143 | } 144 | 145 | 146 | @pytest.mark.parametrize("args,stdin_file", testdata.items()) 147 | def test_file_processing(args, stdin_file): 148 | args = args.split() 149 | dirname, basename = os.path.split(stdin_file) 150 | file_base = os.path.join(dirname, "expected", "_".join(args + [basename.replace(".", "_")]).replace("^", "").replace("$", "").replace("|", "")) 151 | # tests/expected/check_repairsubtree_badLevels_csv 152 | # tests/expected/A2AX_keep_test1_csv 153 | 154 | args = [arg.replace("_", " ") for arg in ["./filterCSV"] + args] 155 | with open(stdin_file) as in_file: 156 | cp = subprocess.run(args, capture_output=True, text=True, stdin=in_file) 157 | assert cp.returncode == 0, cp 158 | 159 | with open(f"{file_base}_err.txt") as in_file: 160 | expected_err = in_file.read().strip() 161 | for line in expected_err.splitlines(): 162 | assert line in cp.stderr, f"\n{line}\n***is not in***\n{cp.stderr}" 163 | assert expected_err in cp.stderr, f"{expected_err}\n***is not in***\n{cp.stderr}" 164 | 165 | with open(f"{file_base}_out.txt") as in_file: 166 | expected_out = in_file.read().strip() 167 | for line in expected_out.splitlines(): 168 | assert line in cp.stdout, f"\n{line}\n***is not in***\n{cp.stdout}" 169 | assert expected_out in cp.stdout, f"{expected_out}\n***is not in***\n{cp.stdout}" 170 | 171 | 172 | if __name__ == "__main__": 173 | import doctest 174 | 175 | doctest.testmod() 176 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # filterCSV 2 | 3 | iThoughts is a third-party application for creating and managing mind maps. It runs on iOS, iPad OS, Mac OS and Windows. 4 | 5 | You can create a mind map either in the application itself or by importing files in a number of other formats. The most complete format is Comma-Separated Value (CSV). Being a text format, CSV can be programmatically created in a number of programming languages, such as Python. 6 | 7 | The CSV format that iThoughts understands has a tree-like structure. A tree consists of nodes, which contain data as well as potentially child nodes. A node with no parent is called a root node. A node with no children is called a leaf node. 8 | 9 | There can be multiple root nodes - and hence multiple trees - in an iThoughts CSV file. In which case it's better to call the ensemble a forest of trees. 10 | 11 | As well as the nodes' tree structure, an iThoughts' CSV file can store for each node its colour, its position, its shape and other attributes. 12 | To a very limited extent the format is documented [here](https://www.toketaware.com/ithoughts-howto-csv). A better way to understand the format is to export a mind map from iThoughts as CSV and look at the resulting file. 13 | 14 | * [An Introduction To The iThoughts CSV File Format](#an-introduction-to-the-ithoughts-csv-file-format) 15 | * [About filterCSV](#about-filtercsv) 16 | * [Using filterCSV](#using-filtercsv) 17 | * [Specifiers](#specifiers) 18 | * [Actions](#actions) 19 | * [Colour Numbers](#colour-numbers) 20 | * [Colour RGB Values](#colour-rgb-values) 21 | * [Automatic Colouring](#automatic-colouring) 22 | * [Delete](#delete) 23 | * [Keep](#keep) 24 | * [Shapes](#shapes) 25 | * [Positions](#positions) 26 | * [Icons](#icons) 27 | * [Priority](#priority) 28 | * [Progress](#progress) 29 | * [Removing Notes, Shapes, Colours, Positions, Icons, Progress, And Priority](#removing-notes-shapes-colours-positions-icons-progress-and-priority) 30 | * [Eliminating A Level](#eliminating-a-level) 31 | * [Computing Statistics About A Mind Map](#computing-statistics-about-a-mind-map) 32 | * [Merging Nodes Into Their Parent Node](#merging-nodes-into-their-parent-node) 33 | * [Sorting Child Nodes](#sorting-child-nodes) 34 | * [Reversing The Order Of Child Nodes](#reversing-the-order-of-child-nodes) 35 | * [Spreading Out Level 0 (Root) Nodes](#spreading-out-level-0-root-nodes) 36 | * [Replacing Strings](#replacing-strings) 37 | * [Match Statistics](#match-statistics) 38 | * [Input Files](#input-files) 39 | * [Nesting Level Detection](#nesting-level-detection) 40 | * [Metadata](#metadata) 41 | * [Checking](#checking) 42 | * [Handling CSV Files Not In The Format iThoughts Expects](#handling-csv-files-not-in-the-format-ithoughts-expects) 43 | * [Output Formats](#output-formats) 44 | * [Markdown Output](#markdown-output) 45 | * [HTML Output](#html-output) 46 | * [Freemind And OPML XML Output](#freemind-and-opml-xml-output) 47 | * [GraphViz .dot Format](#graphviz-dot-format) 48 | * [Indented Text](#indented-text) 49 | * [iThoughts CSV File Format](#ithoughts-csv-file-format) 50 | * [Command Files](#command-files) 51 | * [Test Files](#test-files) 52 | * [iThoughts Shape Names](#ithoughts-shape-names) 53 | * [iThoughts Icon Names](#ithoughts-icon-names) 54 | 55 | ## An Introduction To The iThoughts CSV File Format 56 | 57 | Here is a sample CSV file in the format required by iThoughts: 58 | 59 | "colour","shape","level","level0","level1","level2" 60 | 00FFFF,,0,"A" 61 | ,triangle,1,,A1 62 | ,square,1,,A2 63 | FF0000,square,2,,,A2A 64 | 65 | and here is how it looks when imported into iThoughts: 66 | 67 | ![](images/importable.png) 68 | 69 | This is obviously a very simple example, but it illustrates some features of the file format: 70 | 71 | * Each line is a node, apart from the first one. 72 | * All lines have a cell in the "level" column, with the level number filled in. 73 | * Some nodes have colours, in RGB format. (Nodes "A2" and "A1" inherit the turquoise colour from node "A".) 74 | * Some nodes have shapes associated with them. 75 | 76 | A more detailed description of the file format is given in [iThoughts CSV File Format](#ithoughts-csv-file-format) but this brief description should be enough to get you started. 77 | 78 | In more complex cases other columns come into play. 79 | 80 | ## About filterCSV 81 | 82 | filterCSV is a set of tools to automatically edit a CSV file in the form used in iThoughts. filterCSV is written in Python 3.6+. It has been tested on a Raspberry Pi and a machine running macOS. 83 | 84 | Based on matching regular expressions, plus a few other criteria, you can do things for matching nodes such as: 85 | 86 | * Set colours for nodes 87 | * Change their shape 88 | * Delete them 89 | * Set their positions 90 | * Set icons for nodes 91 | 92 | You can check the structure of the input CSV file is good for importing into iThoughts. 93 | 94 | You can export the CSV file as a Markdown file consisting of headings and bulleted lists, and in a number of other formats. 95 | 96 | **NOTE:** In this document we will use terms such as "mind map" and "tree". Structurally the data represents a tree. \ 97 | It might or might not be used for mapping your mind. 98 | 99 | ## Using filterCSV 100 | 101 | filterCSV reads from stdin and writes to stdout, with messages (including error messages) written to stderr. For example: 102 | 103 | filterCSV '^A1$' 'triangle' < input.csv > output.csv 104 | 105 | It's designed for use in a pipeline, where the input of one program can be the output of another. 106 | 107 | Do not specify the input and output files as command parameters. Instead 108 | 109 | * Code the input file as an input stream using `<`. 110 | * Code the output file as an output stream using `>`. 111 | * You can code stderr as an output stream using `2>` or let it default to the terminal session. 112 | 113 | Command line parameters instruct filterCSV on how to process the parsed input file to create the output file. The parameters are specified in pairs. 114 | Each pair consists of: 115 | 116 | 1. A specifier. This is a regular expression to match. (A special value `all` matches any value.) 117 | 1. An action or sequence of actions. 118 | 119 | In the case where no action is expected you can code anything you like for the second parameter. A useful suggestion would be to code `.` for it. 120 | 121 | Instead of using command line parameters you can code the commands in a file read in from Stream 3. See [Command Files](#commandfiles) for more information on this, potentially more flexible, way of controlling filterCSV. 122 | 123 | You can get some basic help by invoking filterCSV with no parameters. That help points to this README and the project on GitHub. 124 | 125 | ### Specifiers 126 | 127 | Specifiers are used to specify which nodes to operate on and can be in one of the following forms. 128 | 129 | * Regular expressions in a format the Python `re` module understands. 130 | * A special value of `all`, matching all nodes. 131 | * A special value of `none`, matching no nodes. 132 | * A level specifier of the form `@level:n` - where `n` is an integer, referring to the level number. 133 | * A priority specifier of the form `@priority:n` - where `n` is an integer between 1 and 5. You can use `@prio:n` for short. You can use `@nopriority` or `@noprio` to match nodes where the priority has not been set. 134 | * A progress specifier of the form `@progress:n` - where `n` is an integer between 0 and 100, representing percent complete. You can use `@prog:n` for short. You can use `@noprogress` or `@noprog` to match nodes where the progress has not been set. 135 | * A shape specifier of the form `@shape:myshape` - where `myshape` is one of the shapes described in [iThoughts Shape Names](#ithoughtsshapenames). You can use `@noshape` to specify nodes where the shape hasn't been set. 136 | * an icon specifier of the form `@icon:icon` - where `icon` is the name of the icon. It is one of the icons described in [iThoughts Icon Names](#ithoughtsiconnames). If the node's icon set contains this icon then the node matches. `@noicon` matches if the node has no icons. 137 | 138 | **Notes:** 139 | 140 | * If you want to match a cell's text exactly you can code something like `^A1$` where `^` means 'the start of the text' and `$` means 'the end of the text'. 141 | * In the `level:n` form of the specifier the level of a node is taken from how it was read in - though that could be modified by `check repairsubtree`. \ 142 | * For `@priority:n`, `@progress:n`, `@shape:s`, `@icon:i` an empty value in the node's attribute means it's not been set. 143 | * You can use values for shape, icon, etc, that iThoughts doesn't support, perhaps to tag a node. It is unpredictable what iThoughts will do if it encounters them. 144 | * If you're not sure the levels are properly numbered you should run `check repairsubtree` first. For example 145 | 146 | ~ 147 | ``` 148 | filterCSV < input_file.csv > output_file.csv \ 149 | check repairsubtree \ 150 | @level:1 'triangle note' 151 | ``` 152 | 153 | ### Actions 154 | 155 | Actions you can take include: 156 | 157 | * Specify a colour number 158 | * Specify a colour RGB value 159 | * Automate colouring nodes based on a regular expression's capturing group 160 | * `delete` 161 | * `keep` 162 | * Specify a shape 163 | * Specify a position 164 | * Specify an icon 165 | * Removing colour, shape, and icon specifications 166 | * Promote all subtrees at a certain level by 1 level 167 | * Computing basic statistics about the mind map 168 | 169 | In the following action specifications are case-insensitive; If you specify, for example, an action in upper case it will be converted to lower case before being applied to matching nodes. 170 | 171 | You can, in most cases, specify a sequence of actions. You can separate them by spaces or commas. If you specify multiple actions you probably need to surround them with a pair of single quotes. 172 | 173 | #### Colour Numbers 174 | 175 | A colour number is a 1- or 2-digit number. It is specified relative to the top left of iThoughts' colour palette. (`1` is the first colour in the palette.) 176 | 177 | You can also specify `nextcolour`, `nextcolor` or even `nc` and filterCSV will select the next colour in iThoughts' colour palette. `samecolour`, `samecolor` or `sc` can be used to specify the same colour again. 178 | 179 | #### Colour RGB Values 180 | 181 | This is a hexadecimal 6-character representation of the colour, in Red-Green-Blue (RGB) format. For example `FFAAFF`. 182 | 183 | 184 | #### Automatic Colouring 185 | 186 | Rather than either using [Colour Numbers](#colour-numbers) or [Colour RGB Values](#colour-rgb-values) you might be able to automate colouring nodes. 187 | 188 | Automatic node colouring requires you to code a capturing group inside a regular expression. 189 | 190 | Here is an example: 191 | 192 | filterCSV '-(.?)-' autocolour < test1.csv > test2.csv 193 | 194 | The capturing group is the portion of the regular expression inside the round brackets. When filterCSV processes this command it keeps track of the values that match the capturing group and uses them to consistently colour the nodes. 195 | 196 | **Notes** 197 | 198 | 1. You can specify `autocolour`, `autocolor`, or even `ac`. 199 | 1. You can use multiple capturing groups, for example `RC: (.*) SC: (.*)`. filterCSV uses all the groups to form a key; When **any** of the capturing groups' values changes a new colour is selected. 200 | #### Delete 201 | 202 | `delete` deletes the matching node and all its children. 203 | 204 | #### Keep 205 | 206 | `keep` retains the matching node, all of its children, and its parent, grandparent, great-grandparent, etc. The idea is to retain a workable tree. 207 | 208 | For example 209 | 210 | filterCSV 'EXCPs' keep < input.csv > output.csv 211 | 212 | would retain any nodes which match the string "EXCPs", and all the nodes below them. In addition, to ensure the tree remained valid (for import into iThoughts) any nodes leading from the root (level 0) to the matching nodes would be retained. 213 | 214 | **Note:** You can use regular expression alternation to keep multiple subtrees. For example: 215 | 216 | filterCSV 'A1|X' keep < input.csv > output.csv 217 | 218 | where `|` means either the term to the left (`A1`) or the term to the right (`X`) can be used to match. 219 | 220 | If you use `keep` in a filterCSV action you can't use anything else. For example, you can't use `triangle`. You can use another specifier, perhaps `all`with `triangle` to get the same effect. 221 | 222 | #### Shapes 223 | 224 | You can specify a shape for matching nodes using one of the names in the list in [iThoughts Shape Names](#ithoughtsshapenames). 225 | 226 | 227 | For example: 228 | 229 | filterCSV '^CF' triangle < input.csv > output.csv 230 | 231 | would change the shape of any nodes which match the string "CF" (but having no characters preceding "CF") to a triangle. 232 | 233 | You can also specify `nextshape`, or `ns` and filterCSV will select the next shape in iThoughts' set of shapes. `sameshape` or `ss` can be used to specify the same shape again. 234 | 235 | #### Positions 236 | 237 | Positions are specified in the form `{x,y}` where the braces are necessary. 238 | 239 | At present setting the position only seems to work for Level 0 (root) nodes. You can have as many Level 0 nodes as you like. 240 | 241 | For example: 242 | 243 | filterCSV 'A Root Node' '{100,200}' < input.csv > output.csv 244 | 245 | would move a level 0 whose name including the string 'A Root Node' to position (100,200). 246 | 247 | #### Icons 248 | 249 | You can add an icon to matching nodes using one of the names in the list in [iThoughts Icon Names](#ithoughtsiconnames). 250 | 251 | For example: 252 | 253 | filterCSV 'Done' tick < input.csv > output.csv 254 | 255 | would add a tick icon to any nodes which match the string "Done". 256 | 257 | **Note:** A node can have more than one icon so specifying `tick` in the above example would not replace any other icon; It would add a tick icon to any existing ones. 258 | 259 | #### Priority 260 | 261 | You can set a node's priority with `priority:n` or `prio:n`. You can unset it with `nopriority` or `noprio`. 262 | 263 | For example: 264 | 265 | filterCSV 'Unimportant' prio:5 < input.csv > output.csv 266 | 267 | will set the priority for any node matching "Unimportant" to 5. 268 | 269 | #### Progress 270 | 271 | You can set a node's progress with `progress:n` or `prog:n`. You can unset it with `noprogress` or `noprog`. 272 | 273 | For example: 274 | 275 | filterCSV 'Got nowhere' prog:0 < input.csv > output.csv 276 | 277 | will set the progress for any node matching "Got nowhere" to 0%. 278 | 279 | #### Removing Notes, Shapes, Colours, Positions, Icons, Progress, And Priority 280 | 281 | If you specify `noshape`, `nocolour`, `nonote`, `noposition`, or `noicons` the corresponding attribute is removed from matching nodes. 282 | 283 | Similarly priority or progress can be unset with `nopriority`, `noprio`, `noprogress` or `noprog`. 284 | 285 | Most usefully you could specify this with a match condition of `all` to reset an entire column. For example, `nonote` could clear all the notes from a mind map - to prepare it for exporting from iThoughts. 286 | 287 | #### Eliminating A Level 288 | 289 | Suppose you have some nodes at level 1 and you want to make them all level 0, retaining their subtrees. With `promote` you can do this. 290 | 291 | If you specify, for example 292 | 293 | filterCSV promote 2 < input.csv > output.csv 294 | 295 | The nodes at level 1 are deleted and all their direct children move up to level 1. These children might be the root of subtrees. All nodes in the subtrees are also promoted by 1 level. 296 | 297 | #### Computing Statistics About A Mind Map 298 | 299 | If you specify `stats` it will write basic statistics to an output file in one of the following forms: 300 | 301 | * Flat file - if you specify `stats text` 302 | * HTML table - if you specify `stats html` 303 | * Markdown table - if you specify `stats markdown` 304 | * Comma-Separated Value (CSV) - if you specify `stats csv` 305 | 306 | The statistics are (for each level): 307 | 308 | * Number of nodes at that level 309 | * Number of distinct text values at that level 310 | 311 | Here is an example - produced by specifying `stats text`: 312 | 313 | Level Nodes Distinct Nodes 314 | 0 2 1 315 | 1 2 2 316 | 2 1 1 317 | 3 1 1 318 | 319 | #### Merging Nodes Into Their Parent node 320 | 321 | You can merge a matching node into its parent as a bullet. 322 | 323 | To do this specify `asbullet`. For example 324 | 325 | filterCSV ^A1$ asbullet < input.csv > output.csv 326 | 327 | will merge any bullet whose text or note is 'A1' with its parent. The text of the note will be merged in, with the two characters '* ' denoting it's a bulleted item. 328 | 329 | #### Sorting Child Nodes 330 | 331 | You can sort the child nodes of selected nodes. The sort will be alphabetical and ascending. 332 | 333 | For example 334 | 335 | filterCSV ^A1$ sort < input.csv > output.csv 336 | 337 | will sort all the children of the nodes whose text is "A1". 338 | 339 | #### Reversing The Order Of Child Nodes 340 | 341 | You can reverse the order of the child nodes of selected nodes. 342 | 343 | For example 344 | 345 | filterCSV ^A1$ reverse < input.csv > output.csv 346 | 347 | will reverse the order of the children of the nodes whose text is "A1". 348 | 349 | You can use reverse after sort to make the sort effectively alphabetically descending. 350 | 351 | #### Spreading Out Level 0 (Root) Nodes 352 | 353 | If you import a CSV file into iThoughts without specifying positions in the file iThoughts will place all the Level 0 (root) nodes on top of each other. \ 354 | This is probably not what you want. \ 355 | filterCSV can spread out the Level 0 nodes - either horizontally or vertically. 356 | 357 | For example, if you specify `vspread 500` filterCSV will set the positions of the Level 0 nodes 500 units apart - one above the other. 358 | 359 | For example, if you specify `hspread 1000` filterCSV will set the positions of the Level 0 nodes 500 units apart - spaced out horizontally. 360 | 361 | In both cases the children will be arranged as normal, relative to the root nodes. 362 | 363 | vspread and hspread set the values for these nodes in the "position" column in the CSV file.\ 364 | Their format is of the form "{1000,0}". \ 365 | (In this example "1000" is the horizontal offset and "0" is the vertical offset.) \ 366 | If you specify vspread or hspread they will overwrite all Level 0 nodes' positions. 367 | 368 | #### Replacing Strings 369 | 370 | Regular expressions can be used for searching for and replacing strings - if these strings are in a format the Python `re` module's `sub` method understands. 371 | 372 | For example, the author's Production code emits iThoughts-friendly CSV files where the newline character ("\\n") is generated as a semicolon. \ 373 | filterCSV can readily replace every semicolon by a newline character. For example: 374 | 375 | filterCSV ';' sub:$'\n' < input.csv > output.csv 376 | 377 | Here the shell renders `$'\n'` as a newline character. \ 378 | iThoughts honours these newline characters in rendering the nodes. 379 | 380 | To indicate you want a matched string replaced code the action beginning with `sub:`. \ 381 | For example, if you want every occurrence of "A" replaced with "B" code: 382 | 383 | filterCSV 'A' 'sub:B' < input.csv > output.csv 384 | 385 | You can use references to matching groups. \ 386 | For example: 387 | 388 | filterCSV '(\d)' 'sub:\1\1' < input.csv > output.csv 389 | 390 | replaces every numeric digit with two copies of itself. \ 391 | Here the capturing group, marked by the bracketed expression `(\d)`, is referred to as "Capturing Group 1". \ 392 | The `\1` in the replacement refers to this capturing group. 393 | 394 | In general you can use the full flexibility of Python 3's `re.sub()` method. 395 | 396 | ### Match Statistics 397 | 398 | When filterCSV checks nodes against each [specifier](#specifiers) you get statistics for how many nodes matched the criterion. 399 | You also get the number of nodes that matched no criteria in the run. \ 400 | 401 | Here is a sample: 402 | 403 | ``` 404 | Match Statistics 405 | ---------------- 406 | Match count for RegEx 'A2': 3 407 | Match count for RegEx 'A1': 2 408 | Match count for RegEx '^A1': 1 409 | Match count for @level:1: 2 410 | Match count for @noshape: 0 411 | Match count for @shape:square: 1 412 | Match count for RegEx '^A$': 1 413 | Remaining unmatched: 0 414 | ---------------- 415 | ``` 416 | 417 | ### Input Files 418 | 419 | Input files can be in one of six formats: 420 | 421 | * A CSV file that is already in a format supported by iThoughts' Import function. 422 | * A flat file where each line is a new node. Spaces and tabs can be used to indent the text. Here the level of indentation is used to control what level the line is added at. 423 | * A Markdown nested list where each line is a new node. Spaces and tabs can be used to indent the text. Here the level of indentation is used to control what level the line is added at. Either an asterisk (`*`) or a minus sign followed by a space are supported as a list item marker. \ 424 | Further, Markdown headings of one type, described below, can be used. 425 | * An OPML XML file - with or without `head` or `body` elements. 426 | * An XML file, including one with namespaces (both default and named). 427 | * A CSV file where the hierarchy is described by how many empty cells are to the left of the first cell with text in. 428 | 429 | #### Nesting Level Detection 430 | 431 | When parsing a Markdown or tab/space-indented non-CSV file the first line with leading white space (spaces and/or tabs) is used to detect the indentation scheme: \ 432 | Any white space on this line is used as a single indentation level marker. \ 433 | It is expected that all lines are indented in the same way. 434 | 435 | For example, if the second line starts with two spaces this is taken to indicate that every line will have zero, two, four, etc spaces: 436 | 437 | * Zero spaces denotes a level 0 node 438 | * Two spaces denotes a level 1 node 439 | * Four spaces denotes a level 2 node 440 | * And so on 441 | 442 | If the file is detected to contain Markdown, leading dashes and asterisks followed by a single space are removed. \ 443 | (These are bulleted list markers in Markdown.) \ 444 | For example: 445 | 446 | * A 447 | * A1 448 | 449 | leads to the input file being interpreted as a level 0 node with text "A" and its child node with text "A1". 450 | 451 | You can also specify nesting levels using one of the two forms of heading that Markdown supports. \ 452 | Consider the following example: 453 | 454 | # Fruit 455 | 456 | ## Citrus 457 | 458 | * Lemon 459 | * Orange 460 | * Vaguely orange-like 461 | * Mandarin 462 | * Satsuma 463 | 464 | In this example: 465 | 466 | * "Fruit" is a Heading Level 0 node. 467 | * "Citrus" is a Heading Level 1 node. 468 | * "Lemon", "Orange" and "Vaguely orange-like" are Heading Level 2 nodes. 469 | * "Mandarin" and "Satsuma" are Heading Level 3 nodes. 470 | * The blank lines in between are ignored. 471 | * The leading `#` and `*` characters are removed, as are the first space after each occurrence. 472 | 473 | The resulting CSV file contains: 474 | 475 | "dueDate","startDate","effort(hours)","priority","progress","icons","colour","note","position","shape","level","level0","level1","level2","level3" 476 | "","","","","","","","","","","0","Fruit" 477 | "","","","","","","","","","","1","","Citrus" 478 | "","","","","","","","","","","2","","","Lemon" 479 | "","","","","","","","","","","2","","","Orange" 480 | "","","","","","","","","","","2","","","Vaguely orange-like" 481 | "","","","","","","","","","","3","","","","Mandarin" 482 | "","","","","","","","","","","3","","","","Satsuma" 483 | 484 | Imported into iThoughtsX, the resulting tree looks like: 485 | 486 | ![](images/importedMarkdown.png) 487 | 488 | For an XML input file the nesting level is in the data stream; Elements' children are at a deeper nesting level. \ 489 | On import child nodes are created for the element's value (if it has one) and any attributes. \ 490 | These are colour coded and free-standing nodes marked "Element", "Value", and "Attribute" are added as a legend. 491 | 492 | #### Metadata 493 | 494 | When importing Markdown text it can contain metadata that isn't part of the structure of the data. \ 495 | Here is an example of such a file: 496 | 497 | ``` 498 | font-size: 14 499 | creation-date: 2020-08-29 500 | 501 | * A 502 | * A1 503 | * A2 504 | ``` 505 | Metadata consists of key/value pairs, one per line, at the beginning of the file. After the last line of metadata is a blank line. \filterCSV performs two actions on encountering metadata: 506 | 507 | * Metadata key/value pairs are extracted and printed under the heading "Metadata". 508 | * The metadata, including the blank line, is removed and filterCSV processes the remainder of the file. 509 | 510 | #### Checking 511 | 512 | If the input data is not a CSV file filterCSV performs two checks on it: 513 | 514 | * If the indentation of an individual line contains a number of whitespace characters that is not a multiple of the first indented line's leading whitespace diagnostic messages are produced. This line's indentation level is rounded down. 515 | * If the indentation of an individual line contains a sequence that is not the first indented line's leading whitespace (perhaps repeated) diagnostic messages are produced. 516 | 517 | These checks are intended to help debug an indentation problem. 518 | 519 | If you specify `check` - in place of a regular specifier - filterCSV will check the level column of the data. It will detect bad levels in the following way: If the level of a node is greater than one more than the parent node it will consider this to be an error - which will be reported. 520 | 521 | If a level error is detected one of three things can happen: 522 | 523 | * If the action is specified as `repair` or `repairnode`, filterCSV will set the node's level to 1 more than its parent's.

524 | This is good for the case where you want all the gory details of badly leveled nodes. 525 | * If the action is specified as `repairsubtree`, filterCSV will set the node's level to 1 more than its parent's and then adjust the subtree below it in a similar fashion.

526 | This is good for the case where you want the tree repaired with a minimum of fuss. But you still get informed when repairs were necessary. 527 | 528 | * If the action is `stop`, filterCSV will terminate. 529 | 530 | As an example, you might code 531 | 532 | filterCSV check repair < input.csv > output.csv 533 | 534 | #### Handling CSV Files Not In The Format iThoughts Expects 535 | 536 | You can import a CSV file and the tree structure is described by how many empty cells are to the left of the first cell with text in. 537 | 538 | If there is more than one cell in the line with text in the last such cell is used to form a note for the node. 539 | 540 | Here is an example. 541 | 542 | "A" 543 | ,"A1" 544 | ,"A2","This is a note" 545 | ,,"A2A" 546 | 547 | In the above node "A" is at level 0, nodes "A1" and "A2" are at level 1 - and "A2" has a note ("This is a note"), and "A2A" is at level 2. 548 | 549 | ### Output Formats 550 | 551 | While generally you would write a CSV file for importing into iThoughts, you can also export the data to: 552 | 553 | * A Markdown file 554 | * A HTML table 555 | * A HTML nested list 556 | * A Freemind mind map 557 | * An OPML outline 558 | 559 | #### Markdown Output 560 | 561 | A CSV file can be exported to Markdown with each level rendered according to the following rules: 562 | 563 | * The first few levels are rendered as headings. For example the first level (level 0) might be rendered as `#` to denote "heading level 1". 564 | * Subsequent levels are rendered as nested bullets, with `*` as the indicator for a list item. Each level is indented by pairs of spaces. 565 | 566 | You can specify Markdown output by invocations such as 567 | 568 | filterCSV markdown 3 < myfile.csv > myfile.md 569 | 570 | or 571 | 572 | filterCSV markdown '2 3' < myfile.csv > myfile.md 573 | 574 | In the first case three levels of heading are required, starting with heading level 1. (Heading level 1 is the default). 575 | 576 | In the second case two levels of heading are required, starting with heading level 3. 577 | 578 | #### HTML Output 579 | 580 | You can export to HTML as either a nested list or a table. Colour is preserved on output, as are notes. 581 | 582 | You can specify HTML nested list output by invocations such as 583 | 584 | filterCSV html list < myfile.csv > myfile.md 585 | 586 | You can specify HTML table output by invocations such as 587 | 588 | filterCSV html table < myfile.csv > myfile.md 589 | 590 | The table will have extra columns if any of the nodes in the tree have any of the following attributes. \ 591 | Each column has its own `class` attribute - which can be used for styling with CSS: 592 | 593 | |Atribute|Class| 594 | |:-|:-| 595 | |Due Date|dueDate| 596 | |Start Date|startDate| 597 | |Effort|effort| 598 | |Priority|priority| 599 | |Progress|progress| 600 | 601 | 602 | #### Freemind And OPML XML Output 603 | 604 | filterCSV can output to Freemind XML format, including notes and colours: 605 | 606 | filterCSV xml freemind < myfile.csv > myfile.mm 607 | 608 | filterCSV can output to OPML XML format, but support for notes and colours by other programs is mixed. For example Omnifocus will create a custom "Notes" column: 609 | 610 | filterCSV xml opml < myfile.csv > myfile.opml 611 | 612 | #### GraphViz .dot Format 613 | 614 | filterCSV can export in a format compatible with the GraphViz .dot language. It creates a digraph (directed graph). Here is a sample output file: 615 | 616 | digraph { 617 | rankdir=TB 618 | N1[label="A"] 619 | N2[label="A1"] 620 | N1 -> N2 621 | N3[label="A2"] 622 | N1 -> N3 623 | N4[label="A2A",fillcolor="#00ff00",shape="rectangle",style="rounded,filled"] 624 | N3 -> N4 625 | N5[label="A2A1",fillcolor="#00ff00",shape="rectangle",style="rounded,filled"] 626 | N4 -> N5 627 | N6[label="X",shape="square"] 628 | } 629 | 630 | Here a number of nodes, whose names begin with "N", are defined. Additionally, directed links (arcs with arrows on them) are defined between them. \ 631 | filterCSV preserves colours and most shapes on export to .dot format. 632 | 633 | Here is a sample invocation: 634 | 635 | filterCSV digraph vertical < test.csv > test.dot 636 | 637 | You can use the dot command (part of GraphViz) to turn this into a PNG graphic: 638 | 639 | dot -Tpng test.dot > test.png 640 | 641 | In the above example the parameter `vertical` was used to align the root nodes next to each other, with descendants down the page. \ If you specify any other value, for example 'horizontal' or '.' the alignment will be horizontal. (You can use `v` for short, for `vertical`.) 642 | 643 | #### Indented Text 644 | 645 | You can write out the data as a text file where indentation is used to denote levels in the tree hierarchy. \ 646 | Use the `indented` command to write out the text in this format. \ 647 | There are numerous options for how to indent the text: 648 | 649 | * Code `original` to use the same indentation characters as on input. (This will only work if you read in an indented-text file; Otherwise the indentation will be an empty string.) 650 | 651 | * Code `tab` to use the tab character to indent. 652 | 653 | * Code `space:n` to use n spaces for each indentation level. 654 | 655 | * Code `.` as a shorthand to indent using two spaces for each level. 656 | 657 | * Code any other characters to use them for indentation. 658 | 659 | Here are two examples of usage: 660 | 661 | filterCSV < input.csv > output.txt indented space:4 662 | 663 | will write out a file where four space characters are used for each level of indentation. 664 | 665 | filterCSV < input.csv > output.txt indented "--" 666 | 667 | will write out a file where two dashes are used for each level of indentation. 668 | 669 | ### iThoughts CSV File Format 670 | 671 | The CSV format that iThoughts understands has a tree-like structure, within a table. As well as the tree of nodes, colour, position, node shape and other attributes of a node can be specified in the CSV file. To a very limited extent the format is documented [here](https://www.toketaware.com/ithoughts-howto-csv). A better way to understand the format is to export a mind map from iThoughts as CSV and look at the resulting file. 672 | 673 | The first row of the table contains headings iThoughts uses to understand the layout of the following rows. Each subsequent row represents a node. 674 | 675 | In summary, the iThoughts CSV file format has, at a minimum, the following columns: 676 | 677 | * level 678 | * level0 679 | 680 | This would be for a mind map with only (isolated) top-level nodes. An example like this would be 681 | 682 | level,level0 683 | 0,Text for this sole node 684 | 685 | The iThoughts CSV format is tabular. 686 | 687 | But usually you want more than one level of node: 688 | 689 | level,level0,level1,level2 690 | 0,Top-level node 691 | 1,,Next-level node 692 | 2,,,Leaf node at level 2 693 | 1,,Another intermediate-level node 694 | 695 | Here the structure is more apparent: 696 | 697 | * Each node has its level in the hierarchy (starting with 0) filled in in the "level" cell in the first row. 698 | * Each node's text is in the correct cell for the level. For example, the level 2 node's text is in the same column as the "level2" cell in the first row. 699 | 700 | filterCSV ensures the "level" and "level*n*" columns are present - to the extent needed by the tree. It also always adds the following columns, before the "level" column: 701 | 702 | * priority 703 | * progress 704 | * icons 705 | * position 706 | * colour 707 | * shape 708 | 709 | These extra columns are filled in to allow filterCSV to do interesting things with the attributes they represent, such as 710 | 711 | * Filtering on attributes - such as "Priority 1" 712 | * Setting attributes - such as setting the shape to a triangle. 713 | 714 | While iThoughts can tolerate CSV files where trailing empty cells are suppressed, filterCSV includes them. 715 | 716 | ## Command Files 717 | 718 | Instead of specifying commands as pairs of parameters on the command line you can use Stream 3 to point to a file containing the commands. 719 | 720 | For example: 721 | 722 | filterCSV < input.csv > output.csv 3< commands.txt 723 | 724 | Here the `3< commands.txt` specifies the commands will be read in from the file commands.txt. 725 | 726 | The format is very similar to the command line format for specifiers and actions. For example: 727 | 728 | '^A1$' 'triangle FF0000' // A1 nodes get the red triangle treatment 729 | 730 | In the above any characters before the first space are treated as the specifier. They do not have to be in quotation marks. 731 | Any characters after the first space are treated as the actions - up to just before the double slash. 732 | 733 | **Note:** The specifier and the actions must be on the same line. 734 | 735 | In the example above a comment was introduced by `//`. Any characters after this on the same line are treated as a comment and ignored. \ 736 | You can comment out a whole line with `//` - which might be useful for exploration purposes. Blank lines are also ignored. \ 737 | Comments aren't feasible with command line parameters - so using a command file like this might be preferred. 738 | 739 | ## Test Files 740 | 741 | [tests/README.md](./tests/README.md) describes test files that you can study to become familiar with filterCSV. 742 | 743 | 744 | ## iThoughts Shape Names 745 | 746 | The following shape names are defined by iThoughts. 747 | 748 | auto 749 | rectangle 750 | square 751 | rounded 752 | pill 753 | parallelogram 754 | diamond 755 | triangle 756 | oval 757 | circle 758 | underline 759 | none 760 | square bracket 761 | curved bracket 762 | 763 | 764 | ## iThoughts Icon Names 765 | 766 | The following icon names are defined by iThoughts. You can use them in two places 767 | 768 | * For matching nodes on - for example 'all nodes whose icon is "tick".' 769 | * For adding as an icon to a node - for example 'add the "p0" icon to all nodes that match "^A1$".' 770 | 771 | The names below are in the sequence they appear in the iThoughts icon palette. 772 | 773 | tick 774 | tickbox 775 | p0 776 | p1 777 | p2 778 | p3 779 | p4 780 | p5 781 | p6 782 | p7 783 | p8 784 | p9 785 | signal-flag-red 786 | signal-flag-yellow 787 | signal-flag-green 788 | icon-signal-flag-black 789 | icon-signal-flag-blue 790 | icon-signal-flag-orange 791 | icon-signal-flag-purple 792 | icon-signal-flag-white 793 | icon-signal-flag-checkered 794 | icon-hat-black 795 | icon-hat-blue 796 | icon-hat-green 797 | icon-hat-red 798 | icon-hat-white 799 | icon-hat-yellow 800 | icon-calendar1 801 | icon-calendar7 802 | icon-calendar12 803 | icon-calendar31 804 | icon-calendar52 805 | arrow-down-blue 806 | arrow-left-blue 807 | arrow-right-blue 808 | arrow-up-blue 809 | arrow-up-green 810 | arrow-down-red 811 | stop 812 | prep 813 | go 814 | smiley_happy 815 | icon-smiley-neutral 816 | smiley_sad 817 | icon-money 818 | currency-dollar 819 | currency-euro 820 | currency-pound 821 | currency-yen 822 | icon-currency-won 823 | icon-currency-yuan 824 | hand-yellow-card 825 | hand-red-card 826 | hand-stop 827 | hand-thumb-down 828 | hand-thumb-up 829 | question 830 | icon-questionmark 831 | icon-information 832 | icon-exclamationmark 833 | alert 834 | icon-add 835 | cross 836 | sign-forbidden 837 | sign-stop 838 | idea 839 | icon-camera 840 | auction-hammer 841 | bell 842 | bomb 843 | dynamite 844 | fire 845 | hourglass 846 | target 847 | view 848 | icon-airplane 849 | icon-alarmclock 850 | icon-bug 851 | icon-businessmen 852 | icon-car 853 | icon-clients 854 | icon-cup 855 | icon-data 856 | icon-desktop 857 | icon-earth 858 | icon-flash 859 | icon-gear 860 | icon-heart 861 | icon-key 862 | icon-lock-open 863 | icon-lock 864 | icon-mail 865 | icon-pin 866 | icon-printer 867 | icon-scales 868 | icon-star 869 | icom-telephone 870 | icon-pencil 871 | icon-alarm 872 | icon-book 873 | icon-certificate 874 | icon-cloud 875 | icon-compasses 876 | icon-dice 877 | icon-folder 878 | icon-document 879 | icon-male 880 | icon-female 881 | icon-newspaper 882 | icon-paperclip 883 | icon-presentation 884 | icon-signpost 885 | icon-step 886 | 887 | 888 | -------------------------------------------------------------------------------- /filterCSV: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | filterCSV - Augments matching CSV files - according to iThoughts requirements 4 | 5 | It can set colours for nodes, change their shape, or delete them - if the node 6 | matches any one of a set of criteria the user specifies. 7 | 8 | Reads from stdin 9 | Writes to stdout 10 | Commentary to stderr 11 | 12 | 13 | Command line parameters are pairs of: 14 | 15 | 1) A specifier. This is a regular expression to match. (A special value 'all' matches 16 | any value) 17 | 2) An action or sequence of actions. 18 | 19 | Actions can be: 20 | 21 | * A 1 or 2 digit colour number - relative to the top left of iThoughts' colour palette. 22 | * A colour RGB hexadecimal value. Ex. FFFFFF is black, 000000 is white, FF0000 is red 23 | * 'delete'. 24 | * A shape - as named by iThoughts. 25 | """ 26 | 27 | import csv 28 | import re 29 | import sys 30 | import os 31 | from collections import Counter 32 | from string import hexdigits 33 | import xml.etree.ElementTree as ElementTree 34 | from datetime import datetime 35 | from functools import reduce 36 | 37 | # from CSVTree import CSVTree 38 | 39 | filterCSV_level = "2.1+" 40 | filterCSV_date = "2 May, 2022" 41 | 42 | inputIndentCharacters = "" 43 | 44 | 45 | class streamHandler: 46 | def streamIsAvailable(self, streamNumber): 47 | try: 48 | os.stat(streamNumber) 49 | return True 50 | except: 51 | return False 52 | 53 | 54 | class ParameterParser: 55 | def preprocessCriterion(self, matchCriterion): 56 | if matchCriterion == "all": 57 | # This regex is guaranteed to match anything 58 | return re.compile(".*") 59 | elif matchCriterion == "none": 60 | # This regex is guaranteed to match nothing - and to fail quickly 61 | return re.compile("a^") 62 | elif matchCriterion.startswith("@"): 63 | # Pass through a level match criterion 64 | return matchCriterion 65 | else: 66 | # Some other criterion 67 | return re.compile(matchCriterion) 68 | 69 | def getParameters(self): 70 | matchCriteria = [] 71 | actionsLists = [] 72 | 73 | output = [] 74 | 75 | output.append("\nfilterCSV " + filterCSV_level + " (" + filterCSV_date + ")\n") 76 | 77 | # figure out whether parameters are on command line or in a file 78 | useParameterFile = streamHandler.streamIsAvailable(3) 79 | if useParameterFile: 80 | output.append("Reading parameters from stream 3.\n") 81 | commandFile = open(3, "r") 82 | commands = commandFile.readlines() 83 | else: 84 | output.append("Reading parameters as command line arguments.\n") 85 | 86 | # Heading for parameters display 87 | output.append("Criterion".ljust(40, " ") + " Actions") 88 | output.append("---------".ljust(40, " ") + " -------") 89 | 90 | # Read in pairs of parameters from command line or file 91 | parmPairs = [] 92 | 93 | if useParameterFile: 94 | # Use parameter file 95 | for parmLine in commands: 96 | # Process a line as a criterion / actions pair 97 | 98 | splitAtComment = parmLine.rstrip().split("//", 1) 99 | substantialLine = splitAtComment[0] 100 | if substantialLine != "": 101 | parmPair = substantialLine.split(" ", 1) 102 | 103 | matchCriterion = parmPair[0] 104 | if matchCriterion[0:1] == "'": 105 | matchCriterion = matchCriterion[1:] 106 | if matchCriterion[-1] == "'": 107 | matchCriterion = matchCriterion[:-1] 108 | 109 | actionsString = parmPair[1].rstrip() 110 | if actionsString[0:1] == "'": 111 | actionsString = actionsString[1:] 112 | if actionsString[-1] == "'": 113 | actionsString = actionsString[:-1] 114 | actionsString = actionsString.replace(" ", ",") 115 | 116 | parmPair = [matchCriterion, actionsString] 117 | parmPairs.append(parmPair) 118 | 119 | else: 120 | # Use command line parameters 121 | parmNumber = 1 122 | for parmPair in range((len(sys.argv) - 1) // 2): 123 | matchCriterion = sys.argv[parmNumber] 124 | actionsString = sys.argv[parmNumber + 1].lower().replace(" ", ",") 125 | 126 | parmPairs.append([matchCriterion, actionsString]) 127 | parmNumber += 2 128 | 129 | for parmPair in parmPairs: 130 | # Handle match criterion 131 | matchCriterion = parmPair[0] 132 | matchCriteria.append(self.preprocessCriterion(matchCriterion)) 133 | 134 | # Handle the actions that go with this match criterion 135 | actionsString = parmPair[1] 136 | actionList = actionsString.split(",") 137 | 138 | # For nextcolour/samecolour and nextshape/sameshape etc rewrite action as 139 | # the colour number or shape 140 | for actionNumber in range(len(actionList)): 141 | action = actionList[actionNumber] 142 | if action in ("nextcolour", "nextcolor", "nc"): 143 | actionList[actionNumber] = str( 144 | iThoughtsColours.getNextColourNumber() 145 | ) 146 | elif action in ("samecolour", "samecolor", "sc"): 147 | actionList[actionNumber] = str( 148 | iThoughtsColours.getSameColourNumber() 149 | ) 150 | elif action in ("nextshape", "ns"): 151 | actionList[actionNumber] = iThoughtsShapes[ 152 | iThoughtsShapes.getNextShapeNumber() 153 | ] 154 | elif action in ("sameshape", "ss"): 155 | actionList[actionNumber] = iThoughtsShapes[ 156 | iThoughtsShapes.getSameShapeNumber() 157 | ] 158 | 159 | # Add actions list to list of actions lists 160 | actionsLists.append(actionList) 161 | 162 | output.append(matchCriterion.ljust(40, " ") + " " + actionsString) 163 | 164 | output.append("\n") 165 | 166 | return matchCriteria, actionsLists, output 167 | 168 | 169 | class TreeReader: 170 | def detectInputStreamType(self, inputFile): 171 | firstLine = inputFile[0] 172 | firstChar = firstLine[0] 173 | 174 | if firstLine.lower().find("level0") > -1: 175 | return "iThoughtsCSV" 176 | else: 177 | if firstLine.find(",") > -1: 178 | return "CSV" 179 | elif firstChar == "<": 180 | return "XML" 181 | elif (firstChar in ["#", "*", "-"]) or (firstLine.find(":") > -1): 182 | return "markdown" 183 | else: 184 | return "text" 185 | 186 | @staticmethod 187 | def injectColumn(CSVArray, columnName): 188 | # Injects the named column at the beginning of each line, 189 | # with the heading given by columnName 190 | for rowNumber, _ in enumerate(CSVArray): 191 | if rowNumber == 0: 192 | CSVArray[0].insert(0, columnName) 193 | else: 194 | CSVArray[rowNumber].insert(0, "") 195 | return CSVArray 196 | 197 | def ensureMandatoryColumns(self, csvRows): 198 | """ 199 | Add columns if they are not already present 200 | """ 201 | for ( 202 | column 203 | ) in "dueDate startDate effort(hours) priority progress icons shape position note colour".split(): 204 | if column not in csvRows[0]: 205 | csvRows = self.injectColumn(csvRows, column) 206 | 207 | # Return the rows and the position of each column 208 | return ( 209 | csvRows, 210 | csvRows[0].index("colour"), 211 | csvRows[0].index("level"), 212 | csvRows[0].index("note"), 213 | csvRows[0].index("shape"), 214 | csvRows[0].index("position"), 215 | csvRows[0].index("icons"), 216 | csvRows[0].index("progress"), 217 | csvRows[0].index("priority"), 218 | csvRows[0].index("effort(hours)"), 219 | csvRows[0].index("startDate"), 220 | csvRows[0].index("dueDate"), 221 | ) 222 | 223 | def ensureColumnsPopulated(self, csvRows): 224 | # Ensures every column in the header row is present in each subsequent row 225 | minimumColumnCount = len(csvRows[0]) 226 | for rowNumber, row in enumerate(csvRows): 227 | extraCells = minimumColumnCount - len(row) 228 | if extraCells > 0: 229 | csvRows[rowNumber] += [""] * extraCells 230 | 231 | return csvRows 232 | 233 | def readiThoughtsCSVTree(self, inputFile): 234 | output = [] 235 | # Build a list of rows 236 | csvRows = [row for row in csv.reader(inputFile)] 237 | return self.ensureMandatoryColumns(csvRows) + (output,) 238 | 239 | def readNormalCSVTree(self, inputFile): 240 | output = [] 241 | 242 | # Build a list of rows 243 | csvRows = [] 244 | noteCells = [] 245 | for row in csv.reader(inputFile): 246 | csvRows.append(row) 247 | # See if there's another non-blank cell in the row 248 | nonBlankCells = 0 249 | lastNonBlankCell = 0 250 | cellNumber = 0 251 | for cell in row: 252 | if cell != "": 253 | nonBlankCells += 1 254 | lastNonBlankCell = cellNumber 255 | cellNumber += 1 256 | 257 | # If there's another non-blank cell last such becomes a note 258 | if nonBlankCells > 1: 259 | noteCells.append(row[lastNonBlankCell]) 260 | else: 261 | noteCells.append("") 262 | 263 | # Add a header line 264 | csvRows.insert( 265 | 0, 266 | [ 267 | "level", 268 | "level0", 269 | "level1", 270 | "level2", 271 | "level3", 272 | "level4", 273 | "level5", 274 | "level6", 275 | "level7", 276 | "level8", 277 | "level9", 278 | "level10", 279 | "level11", 280 | "level12", 281 | "level13", 282 | "level14", 283 | "level15", 284 | "level16", 285 | "level17", 286 | "level18", 287 | "level19", 288 | "level20", 289 | ], 290 | ) 291 | 292 | # Add level column to each row 293 | rowNumber = 0 294 | for row in csvRows: 295 | if rowNumber > 0: 296 | cellNumber = 0 297 | for cell in row: 298 | if cell != "": 299 | break 300 | cellNumber += 1 301 | row.insert(0, cellNumber) 302 | rowNumber += 1 303 | 304 | # Inject any notes (last blank cell) 305 | csvRows = self.injectColumn(csvRows, "note") 306 | noteColumn = csvRows[0].index("note") 307 | 308 | rowNumber = 0 309 | for row in csvRows: 310 | if rowNumber > 0: 311 | row[noteColumn] = noteCells[rowNumber - 1] 312 | rowNumber += 1 313 | 314 | return self.ensureMandatoryColumns(csvRows) + (output,) 315 | 316 | def readMarkdownOrTextTree(self, inputFileWithPossibleMetadata): 317 | global inputIndentCharacters 318 | 319 | output = [] 320 | 321 | inputFile = [] 322 | 323 | metadata = {} 324 | 325 | # See if there's a blank line to terminate Metadata 326 | lineNumber = 0 327 | firstBlankLineAt = -1 328 | for line in inputFileWithPossibleMetadata: 329 | if line == "\n": 330 | firstBlankLineAt = lineNumber 331 | break 332 | lineNumber += 1 333 | 334 | if firstBlankLineAt == -1: 335 | # There is no Metadata 336 | for line in inputFileWithPossibleMetadata: 337 | inputFile.append(line) 338 | else: 339 | # Check everything before the blank line contains at least 1 colon 340 | hasMetadata = True 341 | lineNumber = 0 342 | for line in inputFileWithPossibleMetadata: 343 | if lineNumber >= firstBlankLineAt: 344 | break 345 | if line.find(":") == -1: 346 | hasMetadata = False 347 | lineNumber += 1 348 | if hasMetadata: 349 | # Is metadata so keep everything after the blank line 350 | 351 | output.append("\nMetadata\n--------") 352 | 353 | lineNumber = 0 354 | for line in inputFileWithPossibleMetadata: 355 | if lineNumber > firstBlankLineAt: 356 | inputFile.append(line) 357 | elif lineNumber < firstBlankLineAt: 358 | splitLine = line.split(":", 1) 359 | metadataKey = splitLine[0] 360 | metadataValue = splitLine[1].strip() 361 | output.append(metadataKey + " -->" + metadataValue) 362 | metadata[metadataKey] = metadataValue 363 | lineNumber += 1 364 | output.append("") 365 | else: 366 | # Is not metadata so keep the whole file 367 | for line in inputFileWithPossibleMetadata: 368 | inputFile.append(line) 369 | 370 | maxHeadingLevel = -1 371 | # Detect heading levels 372 | headingLevels = [False, False, False, False, False, False] 373 | for line in inputFile: 374 | if line.startswith("# "): 375 | maxHeadingLevel = max(maxHeadingLevel, 0) 376 | headingLevels[0] = True 377 | elif line.startswith("## "): 378 | maxHeadingLevel = max(maxHeadingLevel, 1) 379 | headingLevels[1] = True 380 | elif line.startswith("### "): 381 | maxHeadingLevel = max(maxHeadingLevel, 2) 382 | headingLevels[2] = True 383 | elif line.startswith("#### "): 384 | maxHeadingLevel = max(maxHeadingLevel, 3) 385 | headingLevels[3] = True 386 | elif line.startswith("##### "): 387 | maxHeadingLevel = max(maxHeadingLevel, 4) 388 | headingLevels[4] = True 389 | elif line.startswith("###### "): 390 | maxHeadingLevel = max(maxHeadingLevel, 5) 391 | headingLevels[5] = True 392 | 393 | headingLevelCount = 0 394 | headingLevelString = "" 395 | for h in range(6): 396 | if headingLevels[h]: 397 | headingLevelString += " " + str(h + 1) 398 | headingLevelCount += 1 399 | 400 | output.append("Heading levels detected: " + headingLevelString + "\n") 401 | 402 | # Build array of rows 403 | csvRows = [] 404 | 405 | # Work out what an indent would be - in numbers of characters per level, 406 | # using the first indented line as the template. 407 | indentLength = 0 408 | for line in inputFile: 409 | if line == "\n": 410 | continue 411 | indentLength = len(line) - len(line.lstrip()) 412 | if indentLength > 0: 413 | # This is the first line with whitespace at the beginning 414 | 415 | # Save the indentation whitespace 416 | indentCharacters = line[0:indentLength] 417 | inputIndentCharacters = indentCharacters 418 | 419 | # Print the detected indentation - so user can debug 420 | output.append( 421 | "Indentation detected: " 422 | + formatWhitespaceCharacters(indentCharacters) 423 | ) 424 | 425 | break 426 | 427 | # Insert a header row, with attribute columns and a level0 column plus other 428 | # levels 429 | csvRows.append( 430 | [ 431 | "level", 432 | "level0", 433 | "level1", 434 | "level2", 435 | "level3", 436 | "level4", 437 | "level5", 438 | "level6", 439 | "level7", 440 | "level8", 441 | "level9", 442 | "level10", 443 | "level11", 444 | "level12", 445 | "level13", 446 | "level14", 447 | "level15", 448 | "level16", 449 | "level17", 450 | "level18", 451 | "level19", 452 | "level20", 453 | ], 454 | ) 455 | 456 | # Add a level column if one is not already present, and move the text into the 457 | # right level - for each data line 458 | for lineNumber, line in enumerate(inputFile): 459 | # Remove any blank lines 460 | if line == "\n": 461 | continue 462 | 463 | # A data line so work out how many levels deep it is indented 464 | if indentLength > 0: 465 | lineIndentLength = len(line) - len(line.lstrip()) 466 | if lineIndentLength % indentLength > 0: 467 | if lineIndentLength == 1: 468 | output.append("Bad indentation: 1 white space character.") 469 | else: 470 | output.append( 471 | "Bad indentation:" 472 | + str(lineIndentLength) 473 | + " white space characters." 474 | ) 475 | 476 | output.append( 477 | "Should be multiple of " 478 | + str(indentLength) 479 | + ". Rounding level down to " 480 | + str(lineIndentLength / indentLength) 481 | + "." 482 | ) 483 | output.append("Line in error (" + str(lineNumber) + ") is: " + line) 484 | output.append( 485 | "Leading white space characters: " 486 | + formatWhitespaceCharacters(line[0:lineIndentLength]) 487 | ) 488 | else: 489 | lineIndentCharacters = line[0:lineIndentLength] 490 | if lineIndentCharacters != indentCharacters * ( 491 | lineIndentLength // indentLength 492 | ): 493 | output.append("Bad indentation characters:") 494 | output.append( 495 | "Line in error (" + str(lineNumber) + ") is: " + line 496 | ) 497 | output.append( 498 | "Leading white space characters: " 499 | + formatWhitespaceCharacters(lineIndentCharacters) 500 | ) 501 | 502 | newRow = [] 503 | 504 | if line.startswith("# "): 505 | level = 0 506 | elif line.startswith("## "): 507 | level = 1 508 | elif line.startswith("### "): 509 | level = 2 510 | elif line.startswith("#### "): 511 | level = 3 512 | elif line.startswith("##### "): 513 | level = 4 514 | elif indentLength == 0: 515 | level = maxHeadingLevel + 1 516 | else: 517 | level = maxHeadingLevel + 1 + (lineIndentLength // indentLength) 518 | 519 | newRow.append(str(level)) 520 | 521 | # Insert blank cells - according to level 522 | for lev in range(level): 523 | newRow.append("") 524 | 525 | # Clean the line text to remove any list item marker or heading text 526 | cleanedLine = line.lstrip().rstrip() 527 | if cleanedLine[0:2] == "# ": 528 | cleanedLine = cleanedLine[2:] 529 | if cleanedLine[0:3] == "## ": 530 | cleanedLine = cleanedLine[3:] 531 | if cleanedLine[0:4] == "### ": 532 | cleanedLine = cleanedLine[4:] 533 | if cleanedLine[0:5] == "#### ": 534 | cleanedLine = cleanedLine[5:] 535 | if cleanedLine[0:6] == "##### ": 536 | cleanedLine = cleanedLine[6:] 537 | if cleanedLine[0:7] == "###### ": 538 | cleanedLine = cleanedLine[7:] 539 | elif cleanedLine[0:2] in ["* ", "- "]: 540 | cleanedLine = cleanedLine[2:] 541 | 542 | # Add the cleaned line at the appropriate level 543 | newRow.append(cleanedLine) 544 | 545 | # Add this new row 546 | csvRows.append(newRow) 547 | 548 | return self.ensureMandatoryColumns(csvRows) + (output,) 549 | 550 | def readOPMLTree(self, tree): 551 | self.XMLNamespaces = {} 552 | 553 | output = ["XML is specifically 'OPML'.\n"] 554 | 555 | if tree[0].tag == "head": 556 | # Level 0 node will be the contents of the title element within the 557 | # head element 558 | titleText = tree[0][0].text.strip() 559 | haveHead = True 560 | 561 | if (len(tree) > 1) & (tree[1].tag == "body"): 562 | haveBody = True 563 | bodyElement = tree[1] 564 | else: 565 | haveBody = False 566 | else: 567 | haveHead = False 568 | 569 | if tree[0].tag == "body": 570 | haveBody = True 571 | bodyElement = tree[0] 572 | else: 573 | haveBody = False 574 | 575 | # Build array of rows 576 | csvRows = [] 577 | 578 | # Insert a header row, with attribute columns and a level0 column plus other 579 | # levels 580 | csvRows.append( 581 | [ 582 | "dueDate", 583 | "startDate", 584 | "effort(hours)", 585 | "priority", 586 | "progress", 587 | "icons", 588 | "position", 589 | "colour", 590 | "shape", 591 | "level", 592 | "level0", 593 | "level1", 594 | "level2", 595 | "level3", 596 | "level4", 597 | "level5", 598 | "level6", 599 | "level7", 600 | "level8", 601 | "level9", 602 | "level10", 603 | "level11", 604 | "level12", 605 | "level13", 606 | "level14", 607 | "level15", 608 | "level16", 609 | "level17", 610 | "level18", 611 | "level19", 612 | "level20", 613 | ], 614 | ) 615 | 616 | if haveHead: 617 | headCSVRow = ["", "", "", "", "", "", "", "", "", "0", titleText] 618 | 619 | csvRows.append(headCSVRow) 620 | 621 | if haveBody: 622 | # Any level 1+ elements are children of the body element 623 | for child in bodyElement: 624 | csvRows += self._readOPMLTree(child, 1) 625 | else: 626 | # All top-level children of tree, except head, are level 1 627 | for child in tree: 628 | if child.tag != "head": 629 | csvRows += self._readOPMLTree(child, 1) 630 | else: 631 | # Don't have a head row so have to look for body or top-level outline 632 | # elements 633 | if haveBody: 634 | # All top-level children of body element are level 0 635 | for child in bodyElement: 636 | csvRows += self._readOPMLTree(child, 0) 637 | else: 638 | # All top-level children of tree are level 0 639 | for child in tree: 640 | csvRows += self._readOPMLTree(child, 0) 641 | 642 | return self.ensureMandatoryColumns(csvRows) + (output,) 643 | 644 | def _readOPMLTree(self, XMLNode, level): 645 | csvRows = [] 646 | nodeText = XMLNode.attrib["text"] 647 | 648 | nodeRow = ["", "", "", "", "", "", "", "", "", str(level)] 649 | 650 | levelBlankCells = [""] * (level) 651 | nodeRow += levelBlankCells 652 | 653 | nodeRow.append(nodeText) 654 | csvRows.append(nodeRow) 655 | 656 | for child in XMLNode: 657 | csvRows += self._readOPMLTree(child, level + 1) 658 | 659 | return csvRows 660 | 661 | def readXMLTree(self, inputFile): 662 | output = [] 663 | 664 | # Prepare the input text for namespace parsing and XML parsing 665 | XMLText = "\n".join(inputFile) 666 | 667 | # Create the XML parse tree 668 | self.XMLTree = ElementTree.fromstring(XMLText) 669 | 670 | # Check if OPML 671 | if self.XMLTree.tag == "opml": 672 | # Is OPML so treat separately from other XML 673 | return self.readOPMLTree(self.XMLTree) 674 | else: 675 | # Is not OPML 676 | # Hunt for the default namespace 677 | split1 = XMLText.split('xmlns="') 678 | if len(split1) == 1: 679 | output.append("No default namespace specification.") 680 | self.defaultXMLNamespace = "" 681 | else: 682 | self.defaultXMLNamespace = split1[1].split('"')[0] 683 | output.append(f"Default namespace is '{self.defaultXMLNamespace}'") 684 | 685 | # Hunt for other namespaces 686 | self.XMLNamespaces = {} 687 | split3 = XMLText.split("xmlns:") 688 | for fragment in range(len(split3)): 689 | if fragment > 0: 690 | split4 = split3[fragment].split('="') 691 | key = split4[0] 692 | split5 = split4[1].split('"') 693 | value = split5[0] 694 | self.XMLNamespaces[key] = value 695 | 696 | # Build array of rows 697 | csvRows = [] 698 | 699 | # Insert a header row, with attribute columns and a level0 column plus other 700 | # levels 701 | csvRows.append( 702 | [ 703 | "dueDate", 704 | "startDate", 705 | "effort(hours)", 706 | "priority", 707 | "progress", 708 | "icons", 709 | "position", 710 | "colour", 711 | "shape", 712 | "level", 713 | "level0", 714 | "level1", 715 | "level2", 716 | "level3", 717 | "level4", 718 | "level5", 719 | "level6", 720 | "level7", 721 | "level8", 722 | "level9", 723 | "level10", 724 | "level11", 725 | "level12", 726 | "level13", 727 | "level14", 728 | "level15", 729 | "level16", 730 | "level17", 731 | "level18", 732 | "level19", 733 | "level20", 734 | ], 735 | ) 736 | 737 | csvRows += self._readXMLTree(self.XMLTree, 0) 738 | 739 | return self.ensureMandatoryColumns(csvRows) + (output,) 740 | 741 | def resolveNamespaces(self, textToEdit): 742 | editedText = textToEdit.replace("{" + self.defaultXMLNamespace + "}", "") 743 | for key in self.XMLNamespaces: 744 | editedText = editedText.replace( 745 | "{" + self.XMLNamespaces[key] + "}", key + ":" 746 | ) 747 | return editedText 748 | 749 | def _readXMLTree(self, XMLNode, level): 750 | csvRows = [] 751 | 752 | # Edit the element tag - in case of namespaces 753 | editedElementName = self.resolveNamespaces(XMLNode.tag) 754 | 755 | # Create beginning of the row for the node itself - without attributes 756 | 757 | if XMLNode is self.XMLTree: 758 | # Is root node so create legend rows before it 759 | csvRows.append( 760 | [ 761 | "", 762 | "", 763 | "", 764 | "", 765 | "", 766 | "", 767 | "{-300,-120}", 768 | iThoughtsColours.getColour(2), 769 | "rectangle", 770 | "0", 771 | "element", 772 | ] 773 | ) 774 | csvRows.append( 775 | [ 776 | "", 777 | "", 778 | "", 779 | "", 780 | "", 781 | "", 782 | "{-300,-85}", 783 | iThoughtsColours.getColour(8), 784 | "rectangle", 785 | "0", 786 | "value", 787 | ] 788 | ) 789 | csvRows.append( 790 | [ 791 | "", 792 | "", 793 | "", 794 | "", 795 | "", 796 | "", 797 | "{-300,-50}", 798 | iThoughtsColours.getColour(6), 799 | "rectangle", 800 | "0", 801 | "attribute", 802 | ] 803 | ) 804 | 805 | # Set position of root node to {0,0} 806 | elementCSVRow = [ 807 | "", 808 | "", 809 | "", 810 | "", 811 | "", 812 | "", 813 | "{0,0}", 814 | iThoughtsColours.getColour(2), 815 | "rounded", 816 | f"{str(level)}", 817 | ] 818 | else: 819 | # Set position to blank for all but the root node 820 | elementCSVRow = [ 821 | "", 822 | "", 823 | "", 824 | "", 825 | "", 826 | "", 827 | "", 828 | iThoughtsColours.getColour(2), 829 | "rounded", 830 | f"{str(level)}", 831 | ] 832 | 833 | # Add blank cells to position the cell with the element in 834 | if level > 0: 835 | levelBlankCells = [""] * level 836 | elementCSVRow += levelBlankCells 837 | 838 | # Add the (edited) element name 839 | elementCSVRow.append(editedElementName) 840 | 841 | # Add the row to the list of rows 842 | csvRows.append(elementCSVRow) 843 | 844 | for key, value in XMLNode.attrib.items(): 845 | attributeCSVRow = [ 846 | "", 847 | "", 848 | "", 849 | "", 850 | "", 851 | "", 852 | "", 853 | iThoughtsColours.getColour(6), 854 | "rounded", 855 | f"{str(level + 1)}", 856 | ] 857 | levelBlankCells = [""] * (level + 1) 858 | attributeCSVRow += levelBlankCells 859 | 860 | editedAttributeName = self.resolveNamespaces(key) 861 | attributeCSVRow += [f'{editedAttributeName}="{value}"'] 862 | csvRows.append(attributeCSVRow) 863 | if XMLNode.text is not None: 864 | if XMLNode.text.lstrip().rstrip() != "": 865 | textCSVRow = [ 866 | "", 867 | "", 868 | "", 869 | "", 870 | "", 871 | "", 872 | "", 873 | iThoughtsColours.getColour(8), 874 | "rounded", 875 | f"{str(level + 1)}", 876 | ] 877 | levelBlankCells = [""] * (level + 1) 878 | textCSVRow += levelBlankCells 879 | textCSVRow += [f"{XMLNode.text}"] 880 | csvRows.append(textCSVRow) 881 | 882 | for child in XMLNode: 883 | csvRows += self._readXMLTree(child, level + 1) 884 | 885 | return csvRows 886 | 887 | def createCSVArray(self): 888 | output = [] 889 | 890 | # Read in whatever's in stdin - which might not be CSV 891 | inputFile = sys.stdin.readlines() 892 | 893 | # Detect the input stream type 894 | inputType = self.detectInputStreamType(inputFile) 895 | 896 | output.append(f"Input type detected as '{inputType}'.\n") 897 | 898 | # Depending on detected input type create a create CSV-like array, 899 | # including mandatory columns 900 | if inputType == "iThoughtsCSV": 901 | ( 902 | csvRows, 903 | colourColumn, 904 | levelColumn, 905 | noteColumn, 906 | shapeColumn, 907 | positionColumn, 908 | iconsColumn, 909 | progressColumn, 910 | priorityColumn, 911 | effortColumn, 912 | startColumn, 913 | dueColumn, 914 | output1, 915 | ) = self.readiThoughtsCSVTree(inputFile) 916 | 917 | output += output1 918 | 919 | csvRows = self.ensureColumnsPopulated(csvRows) 920 | 921 | return ( 922 | csvRows, 923 | colourColumn, 924 | levelColumn, 925 | noteColumn, 926 | shapeColumn, 927 | positionColumn, 928 | iconsColumn, 929 | progressColumn, 930 | priorityColumn, 931 | effortColumn, 932 | startColumn, 933 | dueColumn, 934 | output, 935 | ) 936 | elif inputType == "CSV": 937 | ( 938 | csvRows, 939 | colourColumn, 940 | levelColumn, 941 | noteColumn, 942 | shapeColumn, 943 | positionColumn, 944 | iconsColumn, 945 | progressColumn, 946 | priorityColumn, 947 | effortColumn, 948 | startColumn, 949 | dueColumn, 950 | output1, 951 | ) = self.readNormalCSVTree(inputFile) 952 | 953 | output += output1 954 | 955 | csvRows = self.ensureColumnsPopulated(csvRows) 956 | 957 | return ( 958 | csvRows, 959 | colourColumn, 960 | levelColumn, 961 | noteColumn, 962 | shapeColumn, 963 | positionColumn, 964 | iconsColumn, 965 | progressColumn, 966 | priorityColumn, 967 | effortColumn, 968 | startColumn, 969 | dueColumn, 970 | output, 971 | ) 972 | elif inputType in ["markdown", "text"]: 973 | ( 974 | csvRows, 975 | colourColumn, 976 | levelColumn, 977 | noteColumn, 978 | shapeColumn, 979 | positionColumn, 980 | iconsColumn, 981 | progressColumn, 982 | priorityColumn, 983 | effortColumn, 984 | startColumn, 985 | dueColumn, 986 | output1, 987 | ) = self.readMarkdownOrTextTree(inputFile) 988 | 989 | output += output1 990 | 991 | csvRows = self.ensureColumnsPopulated(csvRows) 992 | 993 | return ( 994 | csvRows, 995 | colourColumn, 996 | levelColumn, 997 | noteColumn, 998 | shapeColumn, 999 | positionColumn, 1000 | iconsColumn, 1001 | progressColumn, 1002 | priorityColumn, 1003 | effortColumn, 1004 | startColumn, 1005 | dueColumn, 1006 | output, 1007 | ) 1008 | elif inputType == "XML": 1009 | ( 1010 | csvRows, 1011 | colourColumn, 1012 | levelColumn, 1013 | noteColumn, 1014 | shapeColumn, 1015 | positionColumn, 1016 | iconsColumn, 1017 | progressColumn, 1018 | priorityColumn, 1019 | effortColumn, 1020 | startColumn, 1021 | dueColumn, 1022 | output1, 1023 | ) = self.readXMLTree(inputFile) 1024 | 1025 | output += output1 1026 | 1027 | csvRows = self.ensureColumnsPopulated(csvRows) 1028 | if len(self.XMLNamespaces) > 0: 1029 | output.append("Namespace".ljust(20, " ") + " Namespace URL") 1030 | output.append("---------".ljust(20, " ") + " -------------") 1031 | for key in self.XMLNamespaces: 1032 | output.append(key.ljust(20, " ") + " " + self.XMLNamespaces[key]) 1033 | output.append("") 1034 | 1035 | return ( 1036 | csvRows, 1037 | colourColumn, 1038 | levelColumn, 1039 | noteColumn, 1040 | shapeColumn, 1041 | positionColumn, 1042 | iconsColumn, 1043 | progressColumn, 1044 | priorityColumn, 1045 | effortColumn, 1046 | startColumn, 1047 | dueColumn, 1048 | output, 1049 | ) 1050 | 1051 | 1052 | class TreeWriter: 1053 | def writeTreeAsCSV(self): 1054 | # Write the tree out as a CSV file in iThoughts format 1055 | CSVwriter = csv.writer(sys.stdout, quoting=csv.QUOTE_ALL) 1056 | 1057 | # Write the header row 1058 | headerRow = [ 1059 | "dueDate", 1060 | "startDate", 1061 | "effort(hours)", 1062 | "priority", 1063 | "progress", 1064 | "icons", 1065 | "colour", 1066 | "note", 1067 | "position", 1068 | "shape", 1069 | "level", 1070 | ] 1071 | 1072 | # Establish how many levels to put out in the heading 1073 | levels = csvTree.calculateMaximumLevel() + 1 1074 | 1075 | for level in range(levels): 1076 | headerRow.append("level" + str(level)) 1077 | 1078 | CSVwriter.writerow(headerRow) 1079 | 1080 | # Write out the resulting CSV data 1081 | outputArray = [] 1082 | outputArray = csvTree.writeCSVTree(outputArray) 1083 | 1084 | for row in outputArray: 1085 | CSVwriter.writerow(row) 1086 | 1087 | 1088 | class iThoughtsColours: 1089 | light_pastel = """FFB2B2 FFD8B2 FFFFB2 D8FFB2 B2FFB2 B2FFD8 1090 | B2FFFF B2D8FF B2B2FF D8B2FF FFB2FF FFB2D8""".split() 1091 | 1092 | dark_pastel = """B24747 B27C47 B2B247 7CB247 47B247 47B27C 1093 | 47B2B2 477CB2 4747B2 7C47B2 B247B2 B2477C""".split() 1094 | 1095 | saturated = """FF0000 FF7F00 FFFF00 7FFF00 00FF00 00FF7F 1096 | 00FFFF 007FFF 0000FF 7F00FF FF00FF FF007F""".split() 1097 | 1098 | grayscale = """000000 929292 a9a9a9 C0C0C0 D6D6D6 FFFFFF""".split() 1099 | 1100 | solarized = """002b36 073642 586e75 657b83 839496 93a1a1 1101 | eee8d5 fdf6e3 b58900 cb4b16 dc322f d33682 1102 | 6c71c4 268bd2 2aa198 859900""".split() 1103 | colours = light_pastel + dark_pastel + saturated + grayscale + solarized 1104 | 1105 | colourNumber = 0 1106 | 1107 | def __getitem__(self, key): 1108 | """ 1109 | >>> ithoughts_colours = iThoughtsColours() 1110 | >>> ithoughts_colours[0] 1111 | 'FFB2B2' 1112 | >>> ithoughts_colours[-1] 1113 | '859900' 1114 | """ 1115 | return self.colours[key] 1116 | 1117 | def getColour(self, colourNumber): 1118 | """Get colour based on colour number (which starts at 1) 1119 | >>> ithoughts_colours = iThoughtsColours() 1120 | >>> ithoughts_colours.getColour(1) 1121 | 'FFB2B2' 1122 | >>> ithoughts_colours.getColour(58) 1123 | '859900' 1124 | """ 1125 | return self[colourNumber - 1] 1126 | 1127 | def __len__(self): 1128 | """ 1129 | >>> len(iThoughtsColours()) 1130 | 58 1131 | """ 1132 | return len(self.colours) 1133 | 1134 | def getNextColourNumber(self): 1135 | """Get the next colour number from the iThoughts palette 1136 | >>> ithoughts_colours = iThoughtsColours() 1137 | >>> ithoughts_colours.getNextColourNumber() 1138 | 1 1139 | """ 1140 | self.colourNumber += 1 1141 | return self.colourNumber 1142 | 1143 | def getSameColourNumber(self): 1144 | """Get the same colour number from the iThoughts palette 1145 | >>> ithoughts_colours = iThoughtsColours() 1146 | >>> ithoughts_colours.getSameColourNumber() 1147 | 0 1148 | """ 1149 | return self.colourNumber 1150 | 1151 | 1152 | class iThoughtsShapes: 1153 | # Shapes as understood by iThoughts 1154 | shapes = ( 1155 | "auto", 1156 | "rectangle", 1157 | "square", 1158 | "rounded", 1159 | "pill", 1160 | "parallelogram", 1161 | "diamond", 1162 | "triangle", 1163 | "oval", 1164 | "circle", 1165 | "underline", 1166 | "none", 1167 | "square bracket", 1168 | "curved bracket", 1169 | ) 1170 | 1171 | shapeNumber = 0 1172 | 1173 | def __contains__(self, item): 1174 | """Is item in self.shapes 1175 | >>> ithoughts_shapes = iThoughtsShapes() 1176 | >>> all(shape in ithoughts_shapes for shape 1177 | ... in ('auto', 'triangle', 'none', 'curved bracket')) 1178 | True 1179 | """ 1180 | return item in self.shapes 1181 | 1182 | def __getitem__(self, key): 1183 | """Get shape based on shape number (which starts at 0) 1184 | >>> ithoughts_shapes = iThoughtsShapes() 1185 | >>> ithoughts_shapes[0] 1186 | 'auto' 1187 | >>> ithoughts_shapes[-1] 1188 | 'curved bracket' 1189 | """ 1190 | return self.shapes[key] 1191 | 1192 | def __len__(self): 1193 | """ 1194 | >>> len(iThoughtsShapes()) 1195 | 14 1196 | """ 1197 | return len(self.shapes) 1198 | 1199 | def getNextShapeNumber(self): 1200 | """Get the next shape number from the iThoughts list 1201 | >>> ithoughts_shapes = iThoughtsShapes() 1202 | >>> ithoughts_shapes.getNextShapeNumber() 1203 | 1 1204 | """ 1205 | self.shapeNumber += 1 1206 | return self.shapeNumber 1207 | 1208 | def getSameShapeNumber(self): 1209 | """Get the same shape number from the iThoughts list 1210 | >>> ithoughts_shapes = iThoughtsShapes() 1211 | >>> ithoughts_shapes.getSameShapeNumber() 1212 | 0 1213 | """ 1214 | return self.shapeNumber 1215 | 1216 | 1217 | class iThoughtsIcons: 1218 | # icons as understood by iThoughts 1219 | icons = ( 1220 | "tick", 1221 | "tickbox", 1222 | "p0", 1223 | "p1", 1224 | "p2", 1225 | "p3", 1226 | "p4", 1227 | "p5", 1228 | "p6", 1229 | "p7", 1230 | "p8", 1231 | "p9", 1232 | "signal-flag-red", 1233 | "signal-flag-yellow", 1234 | "signal-flag-green", 1235 | "icon-signal-flag-black", 1236 | "icon-signal-flag-blue", 1237 | "icon-signal-flag-orange", 1238 | "icon-signal-flag-purple", 1239 | "icon-signal-flag-white", 1240 | "icon-signal-flag-checkered", 1241 | "icon-hat-black", 1242 | "icon-hat-blue", 1243 | "icon-hat-green", 1244 | "icon-hat-red", 1245 | "icon-hat-white", 1246 | "icon-hat-yellow", 1247 | "icon-calendar1", 1248 | "icon-calendar7", 1249 | "icon-calendar12", 1250 | "icon-calendar31", 1251 | "icon-calendar52", 1252 | "arrow-down-blue", 1253 | "arrow-left-blue", 1254 | "arrow-right-blue", 1255 | "arrow-up-blue", 1256 | "arrow-up-green", 1257 | "arrow-down-red", 1258 | "stop", 1259 | "prep", 1260 | "go", 1261 | "smiley_happy", 1262 | "icon-smiley-neutral", 1263 | "smiley_sad", 1264 | "icon-money", 1265 | "currency-dollar", 1266 | "currency-euro", 1267 | "currency-pound", 1268 | "currency-yen", 1269 | "icon-currency-won", 1270 | "icon-currency-yuan", 1271 | "hand-yellow-card", 1272 | "hand-red-card", 1273 | "hand-stop", 1274 | "hand-thumb-down", 1275 | "hand-thumb-up", 1276 | "question", 1277 | "icon-questionmark", 1278 | "icon-information", 1279 | "icon-exclamationmark", 1280 | "alert", 1281 | "icon-add", 1282 | "cross", 1283 | "sign-forbidden", 1284 | "sign-stop", 1285 | "idea", 1286 | "icon-camera", 1287 | "auction-hammer", 1288 | "bell", 1289 | "bomb", 1290 | "dynamite", 1291 | "fire", 1292 | "hourglass", 1293 | "target", 1294 | "view", 1295 | "icon-airplane", 1296 | "icon-alarmclock", 1297 | "icon-bug", 1298 | "icon-businessmen", 1299 | "icon-car", 1300 | "icon-clients", 1301 | "icon-cup", 1302 | "icon-data", 1303 | "icon-desktop", 1304 | "icon-earth", 1305 | "icon-flash", 1306 | "icon-gear", 1307 | "icon-heart", 1308 | "icon-key", 1309 | "icon-lock-open", 1310 | "icon-lock", 1311 | "icon-mail", 1312 | "icon-pin", 1313 | "icon-printer", 1314 | "icon-scales", 1315 | "icon-star", 1316 | "icom-telephone", 1317 | "icon-pencil", 1318 | "icon-alarm", 1319 | "icon-book", 1320 | "icon-certificate", 1321 | "icon-cloud", 1322 | "icon-compasses", 1323 | "icon-dice", 1324 | "icon-folder", 1325 | "icon-document", 1326 | "icon-male", 1327 | "icon-female", 1328 | "icon-newspaper", 1329 | "icon-paperclip", 1330 | "icon-presentation", 1331 | "icon-signpost", 1332 | "icon-step", 1333 | ) 1334 | 1335 | def __contains__(self, item): 1336 | return item in self.icons 1337 | 1338 | 1339 | class iThoughtsSpread: 1340 | horizontalPosition = 0 1341 | horizontalIncrement = 0 1342 | verticalPosition = 0 1343 | verticalIncrement = 0 1344 | 1345 | def resetHorizontalSpread(self, horizontalIncrement): 1346 | self.horizontalPosition = 0 1347 | self.horizontalIncrement = horizontalIncrement 1348 | 1349 | def nextHorizontal(self): 1350 | x = self.horizontalPosition 1351 | self.horizontalPosition += self.horizontalIncrement 1352 | return x 1353 | 1354 | def resetVerticalSpread(self, verticalIncrement): 1355 | self.verticalPosition = 0 1356 | self.verticalIncrement = verticalIncrement 1357 | 1358 | def nextVertical(self): 1359 | y = self.verticalPosition 1360 | self.verticalPosition += self.verticalIncrement 1361 | return y 1362 | 1363 | def getNextPosition(self): 1364 | return self.nextHorizontal(), self.nextVertical() 1365 | 1366 | 1367 | class CSVTree: 1368 | def __init__( 1369 | self, 1370 | dueDate, 1371 | startDate, 1372 | effort, 1373 | priority, 1374 | progress, 1375 | icons, 1376 | shape, 1377 | colour, 1378 | note, 1379 | level, 1380 | position, 1381 | cell, 1382 | ): 1383 | """ 1384 | >>> csv_tree = CSVTree("startDate", "dueDate", "effort(hours)", "priority", "progress", "icons","shape", "colour", "note", "level", 1, "cell") 1385 | >>> csv_tree.data["shape"] 1386 | 'shape' 1387 | """ 1388 | self.toBeDeleted = False 1389 | self.childNodes = [] 1390 | 1391 | if dueDate == "": 1392 | parsedDueDate = None 1393 | else: 1394 | parsedDueDate = datetime.strptime(dueDate, "%Y-%m-%dT%H:%M:%S%z") 1395 | 1396 | if startDate == "": 1397 | parsedStartDate = None 1398 | else: 1399 | parsedStartDate = datetime.strptime(startDate, "%Y-%m-%dT%H:%M:%S%z") 1400 | 1401 | if effort == "": 1402 | parsedEffortHours = None 1403 | else: 1404 | parsedEffortHours = float(effort) 1405 | 1406 | self.data = { 1407 | "dueDate": parsedDueDate, 1408 | "startDate": parsedStartDate, 1409 | "effort(hours)": parsedEffortHours, 1410 | "priority": priority, 1411 | "progress": progress, 1412 | "icons": icons, 1413 | "shape": shape, 1414 | "colour": colour, 1415 | "note": note, 1416 | "level": level, 1417 | "position": position, 1418 | "cell": cell, 1419 | } 1420 | self.parent = None 1421 | self.matched = False 1422 | self.matches = 0 1423 | 1424 | def addChild(self, childNode): 1425 | self.childNodes.append(childNode) 1426 | childNode.parent = self 1427 | return childNode 1428 | 1429 | def deleteChild(self, childNode): 1430 | self.childNodes.remove(childNode) 1431 | 1432 | def cleanUpDeleted(self): 1433 | # Actually remove nodes marked for deletion 1434 | # Propagate to children in reverse order - for tree traversal to work 1435 | for c in range(len(self.childNodes) - 1, -1, -1): 1436 | self.childNodes[c].cleanUpDeleted() 1437 | 1438 | # Now remove this node - if to be deleted 1439 | if self.toBeDeleted: 1440 | self.parent.deleteChild(self) 1441 | 1442 | def getChildren(self): 1443 | return self.childNodes 1444 | 1445 | def replaceChild(self, childNode, replacementChildren): 1446 | # Remove the child node having replaced it with the replacement 1447 | # children, which are a list. (List could contain 1 item, of course.) 1448 | if childNode in self.childNodes: 1449 | # Child node is indeed a child so find its index 1450 | childIndex = self.childNodes.index(childNode) 1451 | 1452 | # Insert each item from the replacement list before the child 1453 | for i in range(len(replacementChildren)): 1454 | self.childNodes.insert(i + childIndex - 1, replacementChildren[i]) 1455 | 1456 | # Mark the node that's been replaced to be deleted 1457 | childNode.toBeDeleted = True 1458 | 1459 | # Fix up the tree levels - from root downwards 1460 | # self.repairSubtreeLevels(int(self.data["level"])-1) 1461 | else: 1462 | # Supposed child node isn't a child of this CSVTree object 1463 | sys.stderr.write( 1464 | childNode.data["cell"] 1465 | + " is not a child of " 1466 | + self.data["cell"] 1467 | + "\n" 1468 | ) 1469 | 1470 | def isMatch(self, criterion): 1471 | """ 1472 | Common method for establishing if a node matches some criterion, usually a regex 1473 | """ 1474 | return ( 1475 | (re.search(criterion, self.data["icons"]) is not None) 1476 | | (re.search(criterion, self.data["shape"]) is not None) 1477 | | (re.search(criterion, self.data["colour"]) is not None) 1478 | | (re.search(criterion, self.data["note"]) is not None) 1479 | | (re.search(criterion, self.data["cell"]) is not None) 1480 | ) 1481 | 1482 | def applyAction(self, criterion, action, propagateToChildren): 1483 | if action == "delete": 1484 | # Mark the node for deletion 1485 | self.toBeDeleted = True 1486 | 1487 | # Don't propagate 1488 | propagateToChildren = False 1489 | 1490 | elif action == "asbullet": 1491 | self.makeAsBulletOfParent() 1492 | 1493 | elif action == "reverse": 1494 | self.reverseChildren() 1495 | 1496 | elif action == "sort": 1497 | self.sortChildren() 1498 | 1499 | elif action[0] == "{": 1500 | # position specified 1501 | self.data["position"] = action 1502 | 1503 | # Don't propagate 1504 | propagateToChildren = False 1505 | 1506 | elif action == "note": 1507 | # Document the match in the note field 1508 | if isinstance(criterion, str): 1509 | criterionString = criterion 1510 | else: 1511 | criterionString = criterion.pattern 1512 | 1513 | if self.data["note"] != "": 1514 | self.data["note"] += "\nMatched " + criterionString 1515 | else: 1516 | self.data["note"] = "Matched " + criterionString 1517 | 1518 | elif action == "noshape": 1519 | # Remove any shape specification from the matched node 1520 | self.data["shape"] = "" 1521 | 1522 | elif action == "nonote": 1523 | # Remove any note specification from the matched node 1524 | self.data["note"] = "" 1525 | 1526 | elif action == "noposition": 1527 | # Remove any position specification from the matched node 1528 | self.data["position"] = "" 1529 | 1530 | elif action == "nocolour": 1531 | # Remove any colour specification from the matched node 1532 | self.data["colour"] = "" 1533 | 1534 | elif len(action) == 6 and all(c in hexdigits for c in action): 1535 | # 6-digit hexadecimal so is colour RGB value 1536 | self.data["colour"] = action 1537 | 1538 | elif action in iThoughtsShapes: 1539 | # Is a shape 1540 | self.data["shape"] = action 1541 | 1542 | elif action[0:9] == "priority:": 1543 | if action[9:].isdigit(): 1544 | self.data["priority"] = action[9:] 1545 | else: 1546 | sys.stderr.write( 1547 | f"Erroneous priority value {action[9:]} " 1548 | f"(Pattern was: '{criterion.pattern}')." + "\n" 1549 | ) 1550 | 1551 | elif action[0:5] == "prio:": 1552 | if action[5].isdigit(): 1553 | self.data["priority"] = action[5] 1554 | else: 1555 | sys.stderr.write( 1556 | f"Erroneous priority value {action[5]} " 1557 | f"(Pattern was: '{criterion.pattern}')." + "\n" 1558 | ) 1559 | 1560 | elif action in ["noprio", "nopriority"]: 1561 | self.data["priority"] = "" 1562 | 1563 | elif action[0:9] == "progress:": 1564 | if action[9:].isdigit(): 1565 | self.data["progress"] = action[9:] 1566 | else: 1567 | sys.stderr.write( 1568 | f"Erroneous progress value {action[9:]} " 1569 | f"(Pattern was: '{criterion.pattern}')." + "\n" 1570 | ) 1571 | 1572 | elif action[0:5] == "prog:": 1573 | if action[5].isdigit(): 1574 | self.data["progress"] = action[5] 1575 | else: 1576 | sys.stderr.write( 1577 | f"Erroneous progress value {action[5]} " 1578 | f"(Pattern was: '{criterion.pattern}')." + "\n" 1579 | ) 1580 | elif action in ["noprog", "noprogress"]: 1581 | self.data["progress"] = "" 1582 | 1583 | elif action in iThoughtsIcons: 1584 | # Is an icons 1585 | if self.data["icons"] == "": 1586 | self.data["icons"] = action 1587 | else: 1588 | self.data["icons"] += "," + action 1589 | 1590 | elif action in ["noicons", "noicon"]: 1591 | self.data["icons"] = "" 1592 | 1593 | elif action[0:4] == "sub:": 1594 | self.data["cell"] = re.sub(criterion, action[4:], self.data["cell"]) 1595 | elif action == "justcount": 1596 | pass 1597 | 1598 | elif action.isdigit(): 1599 | # Attempt to parse as from the colour palette 1600 | colourNumber = int(action) 1601 | 1602 | # We have an integer. If it is too big but not 6 digits 1603 | # we flag an error and don't do the update 1604 | if colourNumber > len(iThoughtsColours): 1605 | sys.stderr.write( 1606 | f"Erroneous colour value {colourNumber} " 1607 | f"(Pattern was: '{criterion.pattern}')." + "\n" 1608 | ) 1609 | 1610 | else: 1611 | self.data["colour"] = iThoughtsColours.getColour(colourNumber) 1612 | 1613 | else: 1614 | sys.stderr.write( 1615 | f"Erroneous action value {action} " 1616 | f"(Pattern was: '{criterion.pattern}')." + "\n" 1617 | ) 1618 | return propagateToChildren 1619 | 1620 | def applyActions(self, criterion, actionsList): 1621 | """ 1622 | Apply filter to this node - if not tree root 1623 | """ 1624 | 1625 | matchCount = 0 1626 | propagateToChildren = True 1627 | if self.data["level"] != -1: 1628 | # Test string-based criteria 1629 | if isinstance(criterion, str): 1630 | # Level criterion? 1631 | if criterion.startswith("@level:"): 1632 | potentialLevelString = criterion[7:].rstrip() 1633 | if potentialLevelString.isdigit(): 1634 | wantedLevel = int(potentialLevelString) 1635 | if int(self.data["level"]) == wantedLevel: 1636 | # We have a node at the right level 1637 | self.matched = True 1638 | matchCount += 1 1639 | 1640 | for action in actionsList: 1641 | propagateToChildren = self.applyAction( 1642 | criterion, action, propagateToChildren 1643 | ) 1644 | else: 1645 | # The level specified wasn't an integer 1646 | print(f"Bad level string '{potentialLevelString}") 1647 | 1648 | # Priority criterion? 1649 | elif criterion.startswith("@priority:") | criterion.startswith( 1650 | "@prio:" 1651 | ): 1652 | if criterion.startswith("@priority:"): 1653 | potentialPriorityString = criterion[10:].rstrip() 1654 | else: 1655 | potentialPriorityString = criterion[6:].rstrip() 1656 | if potentialPriorityString.isdigit(): 1657 | wantedPriority = int(potentialPriorityString) 1658 | if self.data["priority"] == potentialPriorityString: 1659 | # We have a node with the right priority 1660 | self.matched = True 1661 | matchCount += 1 1662 | 1663 | for action in actionsList: 1664 | propagateToChildren = self.applyAction( 1665 | criterion, action, propagateToChildren 1666 | ) 1667 | 1668 | # No priority criterion? 1669 | elif criterion in ["@nopriority", "@noprio"]: 1670 | if self.data["priority"] == "": 1671 | # We have a node with no priority 1672 | self.matched = True 1673 | matchCount += 1 1674 | 1675 | for action in actionsList: 1676 | propagateToChildren = self.applyAction( 1677 | criterion, action, propagateToChildren 1678 | ) 1679 | 1680 | # Progress criterion? 1681 | elif criterion.startswith("@progress:") | criterion.startswith( 1682 | "@prog:" 1683 | ): 1684 | if criterion.startswith("@progress:"): 1685 | potentialProgressString = criterion[10:].rstrip() 1686 | else: 1687 | potentialProgressString = criterion[6:].rstrip() 1688 | if potentialProgressString.isdigit(): 1689 | wantedProgress = int(potentialProgressString) 1690 | if self.data["progress"] == potentialProgressString: 1691 | # We have a node with the right progress 1692 | self.matched = True 1693 | matchCount += 1 1694 | 1695 | for action in actionsList: 1696 | propagateToChildren = self.applyAction( 1697 | criterion, action, propagateToChildren 1698 | ) 1699 | 1700 | # No progress criterion? 1701 | elif criterion in ["@noprogress", "@noprog"]: 1702 | if self.data["progress"] == "": 1703 | # We have a node with no progress 1704 | self.matched = True 1705 | matchCount += 1 1706 | 1707 | for action in actionsList: 1708 | propagateToChildren = self.applyAction( 1709 | criterion, action, propagateToChildren 1710 | ) 1711 | 1712 | # Shape criterion? 1713 | elif criterion.startswith("@shape:"): 1714 | wantedShape = criterion[7:].rstrip() 1715 | if self.data["shape"] == wantedShape: 1716 | # We have a node with the right shape 1717 | self.matched = True 1718 | matchCount += 1 1719 | 1720 | for action in actionsList: 1721 | propagateToChildren = self.applyAction( 1722 | criterion, action, propagateToChildren 1723 | ) 1724 | 1725 | # No shape criterion? 1726 | elif criterion == "@noshape": 1727 | if self.data["shape"] == "": 1728 | # We have a node with no shape 1729 | self.matched = True 1730 | matchCount += 1 1731 | 1732 | for action in actionsList: 1733 | propagateToChildren = self.applyAction( 1734 | criterion, action, propagateToChildren 1735 | ) 1736 | 1737 | # Icon in set of icons criterion? 1738 | elif criterion.startswith("@icon:"): 1739 | wantedIcon = criterion[6:].rstrip() 1740 | iconArray =self.data["icons"].split(",") 1741 | if wantedIcon in iconArray: 1742 | # We have a node with the specigfied icon among all it has 1743 | self.matched = True 1744 | matchCount += 1 1745 | 1746 | for action in actionsList: 1747 | propagateToChildren = self.applyAction( 1748 | criterion, action, propagateToChildren 1749 | ) 1750 | 1751 | # Some icons match criterion? 1752 | elif criterion.startswith("@iconsmatch:"): 1753 | wantedIcon = criterion[12:].rstrip() 1754 | if self.data["icons"] == wantedIcon: 1755 | # We have a node with the right icon(s) 1756 | self.matched = True 1757 | matchCount += 1 1758 | 1759 | for action in actionsList: 1760 | propagateToChildren = self.applyAction( 1761 | criterion, action, propagateToChildren 1762 | ) 1763 | 1764 | # No icon criterion? 1765 | elif criterion == "@noicons": 1766 | if self.data["icons"] == "": 1767 | # We have a node with no icons 1768 | self.matched = True 1769 | matchCount += 1 1770 | 1771 | for action in actionsList: 1772 | propagateToChildren = self.applyAction( 1773 | criterion, action, propagateToChildren 1774 | ) 1775 | else: 1776 | # Criterion is a regular expression rather than a string 1777 | print(f"Bad criterion: '{criterion}'.") 1778 | 1779 | elif self.isMatch(criterion): 1780 | # Matched so apply all actions triggered by this match 1781 | self.matched = True 1782 | matchCount += 1 1783 | 1784 | for action in actionsList: 1785 | propagateToChildren = self.applyAction( 1786 | criterion, action, propagateToChildren 1787 | ) 1788 | 1789 | # Apply filter to children, recursively - if propagation is indicated 1790 | if propagateToChildren: 1791 | for child in self.childNodes: 1792 | matchCount += child.applyActions(criterion, actionsList) 1793 | 1794 | return matchCount 1795 | 1796 | def writeCSVTree(self, outputArray): 1797 | # Compose this node's line - if not root node 1798 | level = int(self.data["level"]) 1799 | if level > -1: 1800 | line = [] 1801 | for ( 1802 | key 1803 | ) in "dueDate startDate effort(hours) priority progress icons colour note position shape level".split(): 1804 | if key == "dueDate": 1805 | parsedDueDate = self.data["dueDate"] 1806 | if parsedDueDate == None: 1807 | line.append("") 1808 | else: 1809 | line.append(parsedDueDate.strftime("%Y-%m-%dT%H:%M:%S%z")) 1810 | elif key == "startDate": 1811 | parsedStartDate = self.data["startDate"] 1812 | if parsedStartDate == None: 1813 | line.append("") 1814 | else: 1815 | line.append(parsedStartDate.strftime("%Y-%m-%dT%H:%M:%S%z")) 1816 | elif key == "effort(hours)": 1817 | parsedEffortHours = self.data["effort(hours)"] 1818 | if parsedEffortHours == None: 1819 | line.append("") 1820 | else: 1821 | line.append(str(parsedEffortHours)) 1822 | else: 1823 | line.append(self.data[key]) 1824 | 1825 | line += [""] * int(self.data["level"]) 1826 | line.append(self.data["cell"]) 1827 | outputArray.append(line) 1828 | 1829 | # Print children, recursively 1830 | for child in self.childNodes: 1831 | child.writeCSVTree(outputArray) 1832 | 1833 | return outputArray 1834 | 1835 | def getCounter(self, counter=None): 1836 | """ 1837 | >>> csv_tree = CSVTree("icons", "shape", "colour", "note", 1, 1, "cell") 1838 | >>> csv_tree.getCounter() 1839 | Counter({1: 1}) 1840 | >>> _ = csv_tree.addChild(CSVTree("icons", "shape", "colour", "note", 2, 7, "cell")) 1841 | >>> csv_tree.getCounter() 1842 | Counter({1: 1, 2: 1}) 1843 | """ 1844 | counter = counter or Counter() 1845 | counter[self.data["level"]] += 1 1846 | for child in self.childNodes: 1847 | child.getCounter(counter) 1848 | return counter 1849 | 1850 | def checkHierarchy(self, actionsList): 1851 | sys.stderr.write("Beginning Level Check\n") 1852 | sys.stderr.write("---------------------\n") 1853 | 1854 | self._checkHierarchy(-2, actionsList) 1855 | 1856 | sys.stderr.write("---------------------\n") 1857 | sys.stderr.write("Completed Level Check\n") 1858 | 1859 | def _checkHierarchy(self, previousLevel, actionsList): 1860 | level = int(self.data["level"]) 1861 | if level > previousLevel + 1: 1862 | sys.stderr.write(self.data["cell"].ljust(40) + " Error: ") 1863 | sys.stderr.write(f"Expected level {previousLevel + 1}.") 1864 | sys.stderr.write(f" Found level {level}. ") 1865 | 1866 | firstAction = actionsList[0] 1867 | 1868 | if firstAction in ("repair", "repairnode"): 1869 | # Repair just this node 1870 | self.data["level"] = previousLevel + 1 1871 | sys.stderr.write( 1872 | f"Repaired, setting level to {previousLevel + 1}." + "\n" 1873 | ) 1874 | elif firstAction == "repairsubtree": 1875 | # Repair the whole subtree 1876 | sys.stderr.write( 1877 | "Repaired this node (setting its level to " 1878 | f"{previousLevel + 1}) and all its child nodes." + "\n" 1879 | ) 1880 | self.repairSubtreeLevels(previousLevel) 1881 | elif firstAction == "stop": 1882 | # Terminate the check 1883 | sys.stderr.write("Terminating.\n") 1884 | sys.exit() 1885 | else: 1886 | sys.stderr.write("\n") 1887 | else: 1888 | if previousLevel > -2: 1889 | sys.stderr.write(self.data["cell"].ljust(40) + " OK: ") 1890 | sys.stderr.write(f" Found level {level}." + "\n") 1891 | for childNode in self.childNodes: 1892 | childNode._checkHierarchy(previousLevel + 1, actionsList) 1893 | 1894 | def repairSubtreeLevels(self, parentLevel): 1895 | # Repair this level 1896 | self.data["level"] = str(parentLevel + 1) 1897 | 1898 | # Repair lower levels 1899 | for childNode in self.childNodes: 1900 | childNode.repairSubtreeLevels(parentLevel + 1) 1901 | 1902 | def exportToIndentedText(self, actionsList): 1903 | indentationType = actionsList[0].lower() 1904 | if indentationType == "tab": 1905 | indentationCharacters = "\t" 1906 | if indentationType == "original": 1907 | indentationCharacters = inputIndentCharacters 1908 | elif indentationType == ".": 1909 | indentationCharacters = " " 1910 | elif indentationType.startswith("space:"): 1911 | spaceCount = int(indentationType[6:]) 1912 | indentationCharacters = " " * spaceCount 1913 | else: 1914 | indentationCharacters = indentationType 1915 | 1916 | return self._exportToIndentedText(indentationCharacters) 1917 | 1918 | def _exportToIndentedText(self, indentationCharacters): 1919 | output = [] 1920 | 1921 | level = int(self.data["level"]) 1922 | 1923 | if level > -1: 1924 | indentationText = indentationCharacters * level 1925 | output.append(indentationText + self.data["cell"]) 1926 | 1927 | for childNode in self.childNodes: 1928 | output += childNode._exportToIndentedText(indentationCharacters) 1929 | 1930 | return output 1931 | 1932 | def exportToMarkdown(self, actionsList): 1933 | # Get the number of heading levels to allow before going to nested bulleted 1934 | # lists 1935 | headingLevels = int(actionsList[0]) 1936 | 1937 | if len(actionsList) > 1: 1938 | # Get first heading level 1939 | startingLevel = int(actionsList[1]) 1940 | else: 1941 | # Default to starting at heading level 1 1942 | startingLevel = 1 1943 | 1944 | return self._exportToMarkdown(headingLevels - 1, startingLevel) 1945 | 1946 | def _exportToMarkdown(self, maxHeadingLevel, startingLevel): 1947 | # Prime array of output lines 1948 | output = [] 1949 | 1950 | level = int(self.data["level"]) 1951 | if level > -1: 1952 | note = self.data["note"] 1953 | if level > maxHeadingLevel: 1954 | output.append( 1955 | " " * (level - maxHeadingLevel - startingLevel + 2) 1956 | + "* " 1957 | + self.data["cell"] 1958 | ) 1959 | if note: 1960 | output.append(f"

{note}" + "\n") 1961 | else: 1962 | output.append( 1963 | "\n" 1964 | + "#" * (level + startingLevel) 1965 | + " " 1966 | + self.data["cell"] 1967 | + "\n" 1968 | ) 1969 | if note: 1970 | output.append(note + "\n") 1971 | 1972 | for childNode in self.childNodes: 1973 | output += childNode._exportToMarkdown(maxHeadingLevel, startingLevel) 1974 | 1975 | return output 1976 | 1977 | def exportToDotDigraph(self, actionsList): 1978 | # Get the number of heading levels to allow before going to nested bulleted 1979 | output, dummy = self._exportToDotDigraph(-1, 0) 1980 | 1981 | if actionsList[0][0:1] == "v": 1982 | graphAttributes = "\n rankdir=TB" 1983 | else: 1984 | graphAttributes = "\n rankdir=LR" 1985 | 1986 | output = ["digraph {" + graphAttributes] + output + ["}"] 1987 | return output 1988 | 1989 | def _exportToDotDigraph(self, parentItemNumber, thisItemNumber): 1990 | # Prime array of output lines 1991 | output = [] 1992 | 1993 | if thisItemNumber > 0: 1994 | attributes = 'label="' + self.data["cell"] + '"' 1995 | 1996 | colour = self.data["colour"] or "FFFFFF" 1997 | if colour == "FFFFFF": 1998 | wantFilled = False 1999 | else: 2000 | wantFilled = True 2001 | attributes = attributes + ',fillcolor="#' + colour + '"' 2002 | 2003 | shape = self.data["shape"] 2004 | wantRounded = False 2005 | if shape != "": 2006 | if shape == "auto": 2007 | attributes = attributes + ',shape="rectangle"' 2008 | wantRounded = True 2009 | elif shape == "rectangle": 2010 | attributes = attributes + ',shape="rectangle"' 2011 | elif shape == "square": 2012 | attributes = attributes + ',shape="square"' 2013 | elif shape == "rounded": 2014 | attributes = attributes + ',shape="rectangle"' 2015 | wantRounded = True 2016 | elif shape == "pill": 2017 | attributes = attributes + ',shape="rectangle"' 2018 | wantRounded = True 2019 | elif shape == "parallelogram": 2020 | attributes = attributes + ',shape="parallelogram"' 2021 | elif shape == "diamond": 2022 | attributes = attributes + ',shape="diamond"' 2023 | elif shape == "triangle": 2024 | attributes = attributes + ',shape="triangle"' 2025 | elif shape == "oval": 2026 | attributes = attributes + ',shape="oval"' 2027 | elif shape == "circle": 2028 | attributes = attributes + ',shape="circle"' 2029 | elif shape == "underline": 2030 | attributes = attributes + ',shape="underline"' 2031 | elif shape == "none": 2032 | attributes = attributes + ',shape="none"' 2033 | elif shape == "square bracket": 2034 | attributes = attributes + ',shape="rectangle"' 2035 | elif shape == "round bracket": 2036 | attributes = attributes + ',shape="ellipse"' 2037 | else: 2038 | attributes = attributes + ',shape="rectangle"' 2039 | 2040 | if wantRounded | wantFilled: 2041 | styles = [] 2042 | if wantRounded: 2043 | styles.append("rounded") 2044 | if wantFilled: 2045 | styles.append("filled") 2046 | attributes = attributes + ',style="' + ",".join(styles) + '"' 2047 | 2048 | output.append(" N" + str(thisItemNumber) + "[" + attributes + "]") 2049 | if parentItemNumber > 0: 2050 | output.append( 2051 | " N" + str(parentItemNumber) + " -> N" + str(thisItemNumber) 2052 | ) 2053 | 2054 | nextItemNumber = thisItemNumber + 1 2055 | for childNode in self.childNodes: 2056 | returnedOutput, nextItemNumber = childNode._exportToDotDigraph( 2057 | thisItemNumber, nextItemNumber 2058 | ) 2059 | output += returnedOutput 2060 | 2061 | return output, nextItemNumber 2062 | 2063 | def calculateMaximumLevel(self, level=0): 2064 | """ 2065 | maximum level is the max of this level and the levels of all childNodes, 2066 | recursively through their descendents 2067 | """ 2068 | thisLevel = int(self.data["level"]) 2069 | if thisLevel > level: 2070 | level = thisLevel 2071 | for childNode in self.childNodes: 2072 | level = childNode.calculateMaximumLevel(level) 2073 | return level 2074 | 2075 | def exportToHTML(self, actionsList): 2076 | action = actionsList[0] 2077 | 2078 | # Prime array of output lines 2079 | output = [] 2080 | 2081 | if action == "table": 2082 | 2083 | # Work out how many levels are needed 2084 | maximumLevel = self.calculateMaximumLevel() 2085 | 2086 | # Write table start 2087 | output.append("") 2088 | 2089 | # Get tree properties to figure out what columns to write 2090 | treeProperties = self.getTreeProperties() 2091 | 2092 | # Put out headings - if needed 2093 | if reduce(lambda a, b: bool(a or b), treeProperties): 2094 | # At least one heading wanted 2095 | ( 2096 | hasNote, 2097 | hasPosition, 2098 | hasDueDate, 2099 | hasStartDate, 2100 | hasEffort, 2101 | hasPriority, 2102 | hasProgress, 2103 | hasIcon, 2104 | hasShape, 2105 | ) = treeProperties 2106 | 2107 | output.append("") 2108 | 2109 | if hasDueDate: 2110 | output.append("") 2111 | 2112 | if hasStartDate: 2113 | output.append("") 2114 | 2115 | if hasEffort: 2116 | output.append("") 2117 | 2118 | if hasPriority: 2119 | output.append("") 2120 | 2121 | if hasProgress: 2122 | output.append("") 2123 | 2124 | output.append("") 2125 | # Write table rows 2126 | output += self._exportToHTMLTable(maximumLevel, action, treeProperties) 2127 | 2128 | # Write table end 2129 | output.append("
Due DateStart DateEffort
(Hours)		PriorityProgress
%
") 2130 | else: 2131 | # Write top-level list start 2132 | output.append("
    ") 2133 | 2134 | # Write nested list 2135 | level, freshOutput = self._exportToHTMLList(action) 2136 | 2137 | output += freshOutput 2138 | 2139 | # Write top-level list stop 2140 | output.append("
") 2141 | 2142 | return output 2143 | 2144 | def _exportToHTMLTable(self, maximumLevel, action, treeProperties): 2145 | # Prime array of output lines 2146 | output = [] 2147 | 2148 | level = int(self.data["level"]) 2149 | # Determine the cell background colour 2150 | colour = self.data["colour"] or "FFFFFF" 2151 | # HTML table 2152 | if level > -1: 2153 | # Print table row start 2154 | output.append("") 2155 | 2156 | ( 2157 | hasNote, 2158 | hasPosition, 2159 | hasDueDate, 2160 | hasStartDate, 2161 | hasEffort, 2162 | hasPriority, 2163 | hasProgress, 2164 | hasIcon, 2165 | hasShape, 2166 | ) = treeProperties 2167 | 2168 | # Print any required optional columns 2169 | if hasDueDate: 2170 | dueDate = self.data["dueDate"] 2171 | output.append( 2172 | "" + formatTaskpaperDatetime(dueDate) + "" 2173 | ) 2174 | 2175 | if hasStartDate: 2176 | startDate = self.data["startDate"] 2177 | output.append( 2178 | "" 2179 | + formatTaskpaperDatetime(startDate) 2180 | + "" 2181 | ) 2182 | 2183 | if hasEffort: 2184 | effort = self.data["effort(hours)"] 2185 | if effort is None: 2186 | output.append("") 2187 | else: 2188 | output.append("" + str(effort) + "") 2189 | 2190 | if hasPriority: 2191 | output.append("" + self.data["priority"] + "") 2192 | 2193 | if hasProgress: 2194 | output.append("" + self.data["progress"] + "") 2195 | 2196 | # Print padding empty columns after the cell's column 2197 | if level > 0: 2198 | output.append("" * level) 2199 | 2200 | # Print the cell itself, including styling 2201 | output.append( 2202 | "" 2206 | + self.data["cell"] 2207 | + "" 2208 | ) 2209 | 2210 | # Print padding empty columns after the cell's column 2211 | if level < maximumLevel: 2212 | output.append("" * (maximumLevel - level)) 2213 | 2214 | # Print any note as a final column 2215 | note = self.data["note"] 2216 | if note: 2217 | output.append("" + note + "") 2218 | 2219 | # Print table row end 2220 | output.append("") 2221 | 2222 | for childNode in self.childNodes: 2223 | output += childNode._exportToHTMLTable(maximumLevel, action, treeProperties) 2224 | 2225 | return output 2226 | 2227 | def _exportToHTMLList(self, action): 2228 | # Prime array of output lines 2229 | output = [] 2230 | 2231 | level = int(self.data["level"]) 2232 | 2233 | # Determine the cell background colour 2234 | colour = self.data["colour"] 2235 | if colour == "": 2236 | colour = "FFFFFF" 2237 | 2238 | # Determine if there is a note 2239 | note = self.data["note"] 2240 | 2241 | # HTML nested list 2242 | if level > -1: 2243 | indent = " " * (level + 1) 2244 | # print list item 2245 | output.append( 2246 | indent 2247 | + "
  • " 2251 | + self.data["cell"] 2252 | + "" 2253 | ) 2254 | 2255 | # Print any note 2256 | if note: 2257 | output.append(indent + "

    " + note) 2258 | 2259 | needListElements = len(self.childNodes) > 0 2260 | if needListElements is True: 2261 | output.append(indent + "
      ") 2262 | 2263 | returnedLevel = -2 2264 | for childNode in self.childNodes: 2265 | returnedLevel, freshOutput = childNode._exportToHTMLList(action) 2266 | 2267 | output += freshOutput 2268 | 2269 | if level > -1: 2270 | if needListElements is True: 2271 | output.append(indent + "
    ") 2272 | 2273 | output.append(indent + "
    • ") 2274 | 2275 | return [level, output] 2276 | 2277 | def exportToXML(self, actionsList): 2278 | action = actionsList[0] 2279 | 2280 | if action == "freemind": 2281 | # Freemind XML export 2282 | return self.exportToFreemindXML(actionsList) 2283 | elif action == "opml": 2284 | return self.exportToOPMLXML(actionsList) 2285 | 2286 | def exportToFreemindXML(self, actionsList): 2287 | # Export to XML in the format Freemind, MindNode and iThoughts accept 2288 | 2289 | # Prime array of output lines 2290 | output = [] 2291 | 2292 | # Warn if resulting XML would produce more than 1 Level 0 node 2293 | if len(self.childNodes) > 1: 2294 | sys.stderr.write( 2295 | "Exported XML will have more than 1 root node. Some programs will get " 2296 | "confused by this. Continuing.\n" 2297 | ) 2298 | 2299 | # Start the map 2300 | output.append("") 2301 | output.append("") 2302 | 2303 | # Recursively print the nodes 2304 | output += self._exportToFreemindXML(actionsList) 2305 | 2306 | # Finish the map 2307 | output.append("") 2308 | 2309 | return output 2310 | 2311 | def _exportToFreemindXML(self, actionslist): 2312 | # Prime array of output lines 2313 | output = [] 2314 | 2315 | level = int(self.data["level"]) 2316 | 2317 | if level > -1: 2318 | # Print this node 2319 | colour = self.data["colour"] 2320 | cell = self.data["cell"] 2321 | note = self.data["note"] 2322 | indent = " " * (level + 1) 2323 | if colour == "": 2324 | output.append(indent + "") 2325 | else: 2326 | output.append( 2327 | indent 2328 | + "" 2333 | ) 2334 | 2335 | if note: 2336 | output.append(indent + " ") 2337 | output.append(indent + " ") 2338 | output.append(indent + " ") 2339 | output.append(indent + "

    ") 2340 | output.append(indent + " " + note) 2341 | output.append(indent + "

    ") 2342 | output.append(indent + " ") 2343 | 2344 | output.append(indent + "     ") 2345 | 2346 | for childNode in self.childNodes: 2347 | output += childNode._exportToFreemindXML(actionsList) 2348 | 2349 | if level > -1: 2350 | output.append(indent + "    ") 2351 | 2352 | return output 2353 | 2354 | def exportToOPMLXML(self, actionsList): 2355 | # Export to OPML XML 2356 | 2357 | # Prime array of output lines 2358 | output = [] 2359 | 2360 | # Warn if resulting XML would produce more than 1 Level 0 node 2361 | if len(self.childNodes) > 1: 2362 | sys.stderr.write( 2363 | "Exported XML will have more than 1 root node. Some programs will get " 2364 | "confused by this. Continuing.\n" 2365 | ) 2366 | 2367 | # Start the map 2368 | output.append("") 2369 | output.append("") 2370 | output.append(" ") 2371 | output.append(" ") 2372 | output.append(" ") 2373 | 2374 | # Recursively output.append the nodes 2375 | output += self._exportToOPMLXML(actionsList) 2376 | 2377 | # Finish the map 2378 | output.append(" ") 2379 | output.append("") 2380 | 2381 | return output 2382 | 2383 | def _exportToOPMLXML(self, actionslist): 2384 | # Prime array of output lines 2385 | output = [] 2386 | 2387 | level = int(self.data["level"]) 2388 | 2389 | if level > -1: 2390 | # Print this node 2391 | colour = self.data["colour"] 2392 | cell = self.data["cell"] 2393 | note = self.data["note"] 2394 | indent = " " * (level + 2) 2395 | if colour == "": 2396 | printLine = indent + "") 2411 | 2412 | for childNode in self.childNodes: 2413 | output += childNode._exportToOPMLXML(actionsList) 2414 | 2415 | if level > -1: 2416 | output.append(indent + "") 2417 | 2418 | return output 2419 | 2420 | def exportToTaskpaper(self, actionsList): 2421 | # Get the number of heading levels to allow before going to nested bulleted 2422 | # lists 2423 | 2424 | return self._exportToTaskpaper() 2425 | 2426 | def _exportToTaskpaper(self): 2427 | # Prime array of output lines 2428 | output = [] 2429 | 2430 | level = int(self.data["level"]) 2431 | if level > -1: 2432 | if level == 0: 2433 | output.append(self.data["cell"] + ":") 2434 | else: 2435 | dueDate = self.data["dueDate"] 2436 | if dueDate == None: 2437 | dueString = "" 2438 | else: 2439 | dueString = " @due(" + formatTaskpaperDatetime(dueDate) + ")" 2440 | 2441 | startDate = self.data["startDate"] 2442 | if startDate == None: 2443 | deferString = "" 2444 | else: 2445 | deferString = " @defer(" + formatTaskpaperDatetime(startDate) + ")" 2446 | 2447 | effort = self.data["effort(hours)"] 2448 | if effort == None: 2449 | effortString = "" 2450 | else: 2451 | effortString = " @effort(" + str(effort) + ")" 2452 | 2453 | priority = self.data["priority"] 2454 | if priority == "": 2455 | priorityString = "" 2456 | else: 2457 | priorityString = " @priority(" + priority + ")" 2458 | 2459 | progress = self.data["progress"] 2460 | if progress == "": 2461 | progressString = "" 2462 | elif progress == "100": 2463 | progressString = " @done" 2464 | else: 2465 | progressString = " @started" 2466 | 2467 | output.append( 2468 | "\t" * level 2469 | + "- " 2470 | + self.data["cell"] 2471 | + dueString 2472 | + deferString 2473 | + effortString 2474 | + priorityString 2475 | + progressString 2476 | ) 2477 | 2478 | noteText = self.data["note"] 2479 | if noteText != "": 2480 | output.append("\t" * (level + 1) + noteText) 2481 | 2482 | for childNode in self.childNodes: 2483 | output += childNode._exportToTaskpaper() 2484 | 2485 | return output 2486 | 2487 | def processKeep(self, matchCriterion): 2488 | # Reset all nodes' matched flags 2489 | self._markUnmatched() 2490 | 2491 | # Apply tests to each node and mark it and all its children and ancestors 2492 | # matched 2493 | self._processKeep(matchCriterion) 2494 | 2495 | # Delete any unmatched nodes 2496 | self._deleteUnmarked() 2497 | 2498 | def _processKeep(self, matchCriterion): 2499 | if self.isMatch(matchCriterion) is True: 2500 | # mark self and all the ancestors matched 2501 | self._markAncestorsMatched() 2502 | 2503 | # mark self and its subtree matched 2504 | self._markSubtreeMatched() 2505 | else: 2506 | # Maybe children etc are matches 2507 | for childNode in self.childNodes: 2508 | childNode._processKeep(matchCriterion) 2509 | 2510 | def processAutocolour(self, matchCriterion): 2511 | self._processAutocolour(matchCriterion, []) 2512 | 2513 | def _processAutocolour(self, matchCriterion, matchValues): 2514 | if self.isMatch(matchCriterion) is True: 2515 | # Process node as a match 2516 | searchResult = matchCriterion.search(self.data["cell"]) 2517 | 2518 | # Get key - tuple with match values for each group 2519 | matchValue = searchResult.groups() 2520 | 2521 | # Handle whether key is a new key or existing 2522 | if matchValue not in matchValues: 2523 | # New colour 2524 | matchValues.append(matchValue) 2525 | 2526 | colourNumber = len(matchValues) 2527 | else: 2528 | # Existing colour 2529 | colourNumber = matchValues.index(matchValue) + 1 2530 | self.data["colour"] = iThoughtsColours.getColour(colourNumber) 2531 | 2532 | # Maybe children etc are matches 2533 | for childNode in self.childNodes: 2534 | childNode._processAutocolour(matchCriterion, matchValues) 2535 | 2536 | def _markUnmatched(self): 2537 | self.matched = False 2538 | 2539 | for childNode in self.childNodes: 2540 | childNode._markUnmatched() 2541 | 2542 | def _markAncestorsMatched(self): 2543 | if self.data["level"] > -1: 2544 | self.matched = True 2545 | self.parent._markAncestorsMatched() 2546 | 2547 | def _markSubtreeMatched(self): 2548 | self.matched = True 2549 | for childNode in self.childNodes: 2550 | childNode._markSubtreeMatched() 2551 | 2552 | def _deleteUnmarked(self): 2553 | for childNode in self.childNodes: 2554 | childNode._deleteUnmarked() 2555 | if self.data["level"] > -1: 2556 | if not self.matched: 2557 | self.parent.deleteChild(self) 2558 | 2559 | def promoteLevel(self, actionslist): 2560 | # Promote everything at the specified level, deleting parents 2561 | promotedLevel = int(actionslist[0]) 2562 | if promotedLevel < 1: 2563 | sys.stderr.write("Cannot promote level " + str(promotedLevel) + "\n") 2564 | sys.exit() 2565 | 2566 | # Get nodes to promote 2567 | nodesToPromote = self.findNodesAtLevel(promotedLevel) 2568 | 2569 | # Get their parents, removing duplicates 2570 | parentsToDelete = [] 2571 | for node in nodesToPromote: 2572 | if node.parent not in parentsToDelete: 2573 | parentsToDelete.append(node.parent) 2574 | 2575 | # Promote each parent's children 2576 | for parent in parentsToDelete: 2577 | # Insert each child in turn 2578 | parentsParent = parent.parent 2579 | for childNode in parent.childNodes: 2580 | parentsParent.addChild(childNode) 2581 | 2582 | # Delete the parent 2583 | parentsParent.deleteChild(parent) 2584 | 2585 | # Repair all the levels in the tree 2586 | self.repairSubtreeLevels(-2) 2587 | 2588 | def findNodesAtLevel(self, level): 2589 | # returns a list of nodes at a particular level 2590 | return self._findNodesAtLevel(level, []) 2591 | 2592 | def _findNodesAtLevel(self, level, nodes): 2593 | # recursive helper routine to search the tree for nodes at a certain level 2594 | if self.data["level"] == level: 2595 | nodes.append(self) 2596 | else: 2597 | for childNode in self.childNodes: 2598 | nodes = childNode._findNodesAtLevel(level, nodes) 2599 | 2600 | return nodes 2601 | 2602 | def countMatches(self, matchCriterion): 2603 | 2604 | # Clear the matches for this match criterion 2605 | #clearMatches() 2606 | 2607 | # Apply the matches, counting them 2608 | if str(matchCriterion)[:3] == "re.": 2609 | matchCount = self._countMatches(matchCriterion) 2610 | pattern = f"RegEx { str(matchCriterion)[11:-1]}" 2611 | else: 2612 | matchCount = self.applyActions(matchCriterion,["justcount"]) 2613 | pattern = matchCriterion 2614 | 2615 | sys.stderr.write(f"Match count for {pattern}: {str(matchCount)}\n") 2616 | 2617 | # Return match count - which would be -1 if invalid 2618 | return matchCount 2619 | 2620 | def _countMatches(self, matchCriterion): 2621 | 2622 | if self.isMatch(matchCriterion) is True: 2623 | matchCount = 1 2624 | self.matched = True 2625 | self.matches += 1 2626 | else: 2627 | matchCount = 0 2628 | 2629 | for childNode in self.childNodes: 2630 | matchCount += childNode._countMatches(matchCriterion) 2631 | 2632 | return matchCount 2633 | 2634 | def countUnmatched(self): 2635 | 2636 | # Clear the matches for this match criterion 2637 | #clearMatches() 2638 | 2639 | # Apply the matches, counting them 2640 | unmatchedCount = self._countUnmatched() 2641 | 2642 | if unmatchedCount < 0: 2643 | unmatchedCount = 0 2644 | sys.stderr.write(f"Remaining unmatched: {str(unmatchedCount)}\n") 2645 | 2646 | def _countUnmatched(self): 2647 | if self.matched is True: 2648 | unmatchedCount = 0 2649 | else: 2650 | unmatchedCount = 1 2651 | 2652 | for childNode in self.childNodes: 2653 | unmatchedCount += childNode._countUnmatched() 2654 | 2655 | return unmatchedCount 2656 | 2657 | def writeStatistics(self, actionslist): 2658 | # Write statistics in one of a number of formats 2659 | 2660 | # Prime array of output lines 2661 | output = [] 2662 | 2663 | # Get the statistics array 2664 | statistics = self.getStatistics() 2665 | 2666 | # Output the statistics in the right format 2667 | firstAction = actionslist[0] 2668 | if firstAction == "csv": 2669 | output.append("'Level','Nodes','Distinct Nodes'") 2670 | 2671 | for level in range(0, 21): 2672 | levelNodes = statistics[0][level] 2673 | if levelNodes == 0: 2674 | break 2675 | distinctLevelNodes = statistics[1][level] 2676 | output.append( 2677 | str(level) + "," + str(levelNodes) + "," + str(distinctLevelNodes) 2678 | ) 2679 | elif firstAction == "html": 2680 | output.append("") 2681 | output.append( 2682 | "\n\n\n\n" 2683 | ) 2684 | for level in range(0, 21): 2685 | levelNodes = statistics[0][level] 2686 | if levelNodes == 0: 2687 | break 2688 | distinctLevelNodes = statistics[1][level] 2689 | output.append("") 2690 | output.append("") 2691 | output.append("") 2692 | output.append("") 2693 | output.append("") 2694 | output.append("
    LevelNodesDistinct Nodes
    " + str(level) + "" + str(levelNodes) + "" + str(distinctLevelNodes) + "
    ") 2695 | elif firstAction == "markdown": 2696 | output.append("|Level|Nodes|Distinct Nodes|") 2697 | output.append("|-:|-:|-:|") 2698 | for level in range(0, 21): 2699 | levelNodes = statistics[0][level] 2700 | if levelNodes == 0: 2701 | break 2702 | distinctLevelNodes = statistics[1][level] 2703 | output.append( 2704 | "|" 2705 | + str(level) 2706 | + "|" 2707 | + str(levelNodes) 2708 | + "|" 2709 | + str(distinctLevelNodes) 2710 | + "|" 2711 | ) 2712 | elif firstAction == "text": 2713 | output.append("Level Nodes Distinct Nodes") 2714 | for level in range(0, 21): 2715 | levelNodes = statistics[0][level] 2716 | if levelNodes == 0: 2717 | break 2718 | distinctLevelNodes = statistics[1][level] 2719 | output.append( 2720 | str(level).rjust(5, " ") 2721 | + " " 2722 | + str(levelNodes).rjust(5, " ") 2723 | + " " 2724 | + str(distinctLevelNodes).rjust(14, " ") 2725 | ) 2726 | else: 2727 | sys.stderr.write("Invalid format for 'stats': " + firstAction + "\n") 2728 | 2729 | return output 2730 | 2731 | def getStatistics(self): 2732 | # Prime statistics arrays 2733 | nodesAtLevel = [] 2734 | distinctNodeValuesAtLevel = [] 2735 | for level in range(0, 21): 2736 | nodesAtLevel.append(0) 2737 | distinctNodeValuesAtLevel.append([]) 2738 | 2739 | # Walk the tree, updating statistics 2740 | [nodesAtLevel, distinctNodeValuesAtLevel] = self._getStatistics( 2741 | nodesAtLevel, distinctNodeValuesAtLevel 2742 | ) 2743 | 2744 | # Coalesce the sets of node values into counts 2745 | distinctNodesAtLevel = [] 2746 | for level in range(0, 21): 2747 | distinctNodesAtLevel.append(len(distinctNodeValuesAtLevel[level])) 2748 | 2749 | # Return the statistics: 2750 | # Count of nodes at each level 2751 | # Count of unique node values at each level 2752 | return [nodesAtLevel, distinctNodesAtLevel] 2753 | 2754 | def _getStatistics(self, nodesAtLevel, distinctNodeValuesAtLevel): 2755 | level = int(self.data["level"]) 2756 | if level > -1: 2757 | # Increment count of nodes at this level 2758 | nodesAtLevel[level] += 1 2759 | 2760 | # Add cell value to the list for this level - if it's not already in it 2761 | cellValue = self.data["cell"] 2762 | if cellValue not in distinctNodeValuesAtLevel[level]: 2763 | distinctNodeValuesAtLevel[level].append(cellValue) 2764 | 2765 | for childNode in self.childNodes: 2766 | [nodesAtLevel, distinctNodeValuesAtLevel] = childNode._getStatistics( 2767 | nodesAtLevel, distinctNodeValuesAtLevel 2768 | ) 2769 | 2770 | return [nodesAtLevel, distinctNodeValuesAtLevel] 2771 | 2772 | def doHorizontalSpread(self, actionsList): 2773 | self._doSpread("horizontal", actionsList) 2774 | 2775 | def doVerticalSpread(self, actionsList): 2776 | self._doSpread("vertical", actionsList) 2777 | 2778 | def _doSpread(self, direction, actionsList): 2779 | # Spread the level 0 nodes - either vertically or horizontally 2780 | 2781 | # Get the increment 2782 | action = actionsList[0] 2783 | if action.isdigit(): 2784 | increment = int(action) 2785 | else: 2786 | sys.stderr.write("Increment value is not a positive integer.\n") 2787 | sys.exit() 2788 | 2789 | # Set iThoughtsSpread variables up 2790 | if direction == "horizontal": 2791 | iThoughtsSpread.resetHorizontalSpread(increment) 2792 | iThoughtsSpread.resetVerticalSpread(0) 2793 | else: 2794 | iThoughtsSpread.resetHorizontalSpread(0) 2795 | iThoughtsSpread.resetVerticalSpread(increment) 2796 | 2797 | for childNode in self.childNodes: 2798 | childNode.data["position"] = ( 2799 | "{" 2800 | + str(iThoughtsSpread.nextHorizontal()) 2801 | + "," 2802 | + str(iThoughtsSpread.nextVertical()) 2803 | + "}" 2804 | ) 2805 | 2806 | def dump(self, actionsList=None) -> str: 2807 | """ 2808 | >>> csv_tree = CSVTree("", "", "", "", "-1", "", "") 2809 | >>> csv_child = CSVTree("", "square", "", "", "0", "", "Foo") 2810 | >>> _ = csv_tree.addChild(csv_child) 2811 | >>> print(csv_tree.dump([]).rstrip()) 2812 | -1 2813 | square 0 Foo 2814 | """ 2815 | s = "".join( 2816 | f"{str(value)[:10]:<11}".replace("\n", "\\n") 2817 | for value in self.data.values() 2818 | ) 2819 | s = f"{' ' * int(self.data['level'])}{s.rstrip()}" + "\n" 2820 | return s + "".join(child.dump(actionsList) for child in self.getChildren()) 2821 | 2822 | def dump2(self, treeLevel) -> str: 2823 | s = ( 2824 | str(treeLevel) 2825 | + " " 2826 | + str(self.data["level"]) 2827 | + " " 2828 | + self.data["cell"] 2829 | + "\n" 2830 | ) 2831 | return s + "".join(child.dump2(treeLevel + 1) for child in self.getChildren()) 2832 | 2833 | def dumpAll(self): 2834 | # Find root Node 2835 | rootNode = self.parent 2836 | while rootNode.parent is not None: 2837 | rootNode = rootNode.parent 2838 | 2839 | # Dump from the root downwards 2840 | return rootNode.dump2(-1) 2841 | 2842 | def makeAsBulletOfParent(self): 2843 | if self.toBeDeleted: 2844 | return 2845 | 2846 | # Acquire the parent 2847 | if self.parent.data["level"] == -1: 2848 | # Can't make this node a bullet under parent if parent is root i.e. level -1 2849 | sys.stderr.write(f"Node {self.data['cell']} has no parent.\n") 2850 | else: 2851 | # Get parent cell value - to append to 2852 | parentCellValue = self.parent.data["cell"] 2853 | 2854 | # Acquire this cell's value and add as a bullet to parent's cell value 2855 | cellValue = self.data["cell"] 2856 | newParentCellValue = parentCellValue + "\n" + "* " + cellValue 2857 | 2858 | # set parent's cell value to the newly augmented one 2859 | self.parent.data["cell"] = newParentCellValue 2860 | 2861 | # Promote the nodes under this one, replacing it 2862 | self.parent.replaceChild(self, self.childNodes) 2863 | 2864 | def sortKey(self, node): 2865 | return node.data["cell"] 2866 | 2867 | def sortChildren(self): 2868 | newChildren = self.childNodes.copy() 2869 | newChildren.sort(reverse=False, key=self.sortKey) 2870 | self.childNodes = newChildren 2871 | 2872 | def reverseChildren(self): 2873 | newChildren = self.childNodes.copy() 2874 | newChildren.reverse() 2875 | self.childNodes = newChildren 2876 | 2877 | # Prints list of optional tree properties 2878 | def printTreeProperties(self): 2879 | ( 2880 | hasNote, 2881 | hasPosition, 2882 | hasDueDate, 2883 | hasStartDate, 2884 | hasEffort, 2885 | hasPriority, 2886 | hasProgress, 2887 | hasIcon, 2888 | hasShape, 2889 | ) = self.getTreeProperties() 2890 | 2891 | sys.stderr.write("Properties\n----------\n") 2892 | sys.stderr.write("Note: " + str(hasNote) + "\n") 2893 | sys.stderr.write("Position: " + str(hasPosition) + "\n") 2894 | sys.stderr.write("Due Date: " + str(hasDueDate) + "\n") 2895 | sys.stderr.write("Start Date: " + str(hasStartDate) + "\n") 2896 | sys.stderr.write("Effort: " + str(hasEffort) + "\n") 2897 | sys.stderr.write("Priority: " + str(hasPriority) + "\n") 2898 | sys.stderr.write("Progress: " + str(hasProgress) + "\n") 2899 | sys.stderr.write("Icon: " + str(hasIcon) + "\n") 2900 | sys.stderr.write("Shape: " + str(hasShape) + "\n") 2901 | 2902 | sys.stderr.write("\n") 2903 | 2904 | # Returns list of whether tree has any nodes with this property set in 2905 | # any way - for each optional property 2906 | def getTreeProperties(self): 2907 | if self.data["note"] == "": 2908 | hasNote = False 2909 | else: 2910 | hasNote = True 2911 | 2912 | if self.data["position"] == "": 2913 | hasPosition = False 2914 | else: 2915 | hasPosition = True 2916 | 2917 | if self.data["dueDate"] == None: 2918 | hasDueDate = False 2919 | else: 2920 | hasDueDate = True 2921 | 2922 | if self.data["startDate"] == None: 2923 | hasStartDate = False 2924 | else: 2925 | hasStartDate = True 2926 | 2927 | if self.data["effort(hours)"] == None: 2928 | hasEffort = False 2929 | else: 2930 | hasEffort = True 2931 | 2932 | if self.data["priority"] == "": 2933 | hasPriority = False 2934 | else: 2935 | hasPriority = True 2936 | 2937 | if self.data["progress"] == "": 2938 | hasProgress = False 2939 | else: 2940 | hasProgress = True 2941 | 2942 | if self.data["icons"] == "": 2943 | hasIcon = False 2944 | else: 2945 | hasIcon = True 2946 | 2947 | if self.data["shape"] == "": 2948 | hasShape = False 2949 | else: 2950 | hasShape = True 2951 | 2952 | for child in self.childNodes: 2953 | ( 2954 | childHasNote, 2955 | childHasPosition, 2956 | childHasDueDate, 2957 | childHasStartDate, 2958 | childHasEffort, 2959 | childHasPriority, 2960 | childHasProgress, 2961 | childHasIcon, 2962 | childHasShape, 2963 | ) = child.getTreeProperties() 2964 | hasNote = hasNote | childHasNote 2965 | hasPosition = hasPosition | childHasPosition 2966 | hasDueDate = hasDueDate | childHasDueDate 2967 | hasStartDate = hasStartDate | childHasStartDate 2968 | hasEffort = hasEffort | childHasEffort 2969 | hasPriority = hasPriority | childHasPriority 2970 | hasProgress = hasProgress | childHasProgress 2971 | hasIcon = hasIcon | childHasIcon 2972 | hasShape = hasShape | childHasShape 2973 | 2974 | return ( 2975 | hasNote, 2976 | hasPosition, 2977 | hasDueDate, 2978 | hasStartDate, 2979 | hasEffort, 2980 | hasPriority, 2981 | hasProgress, 2982 | hasIcon, 2983 | hasShape, 2984 | ) 2985 | 2986 | 2987 | def formatWhitespaceCharacters(whitespace): 2988 | """ 2989 | >>> formatWhitespaceCharacters("a b \\t c d") 2990 | '' 2991 | """ 2992 | # return "".join("" if c == " " else "" for c in whitespace) 2993 | return "".join( 2994 | "" if c == " " else "" if c == "\n" else "" 2995 | for c in whitespace 2996 | ) 2997 | 2998 | 2999 | def writeOutput(printLines): 3000 | for line in printLines: 3001 | print(line, file=sys.stderr) 3002 | 3003 | 3004 | def formatTaskpaperDatetime(date): 3005 | if date == None: 3006 | return "" 3007 | time = date.time() 3008 | 3009 | if time.hour + time.minute == 0: 3010 | return date.strftime("%Y-%m-%d") 3011 | else: 3012 | return date.strftime("%Y-%m-%d %H:%M") 3013 | 3014 | 3015 | if __name__ == "__main__": 3016 | if len(sys.argv) == 1: 3017 | # No parameters so print help information and quit 3018 | title = f"\nfilterCSV {filterCSV_level} - {filterCSV_date}" 3019 | print(title) 3020 | print("-" * (len(title) - 1)) 3021 | print( 3022 | """ 3023 | filterCSV is a tool for manipulating files for import into and export 3024 | from the iThoughtsX (Mac) and iThoughts (iOS / iPad OS app). 3025 | 3026 | It can import and export other types of file as well. For details see 3027 | the README.md file on GitHub. This file is here: 3028 | 3029 | https://github.com/MartinPacker/filterCSV/blob/master/README.md 3030 | 3031 | The filterCSV project is Open Sourced under the MIT licence. Its URL 3032 | is here: 3033 | 3034 | https://github.com/MartinPacker/filterCSV 3035 | 3036 | In basic use filterCSV expects the following streams: 3037 | 3038 | stdin (0) - The source file to be processed. 3039 | 3040 | stdout (1) - The resulting processed file. 3041 | 3042 | stderr (2) - Informational and error messages. 3043 | 3044 | filterCSV also expects some commands. These are pairs of parameters. 3045 | Here is an example: 3046 | 3047 | filterCSV < input.csv > output.csv 'A1' 'triangle FF0000' 3048 | 3049 | The first parameter specifies that any node whose text contains "A1" 3050 | is to be processed according to the second parameter. 3051 | 3052 | The second parameter, which is enclosed in quotes because it contains 3053 | spaces, says to make the node's shape be a triangle and to colour it 3054 | red. ("FF0000" is the RGB value for red.) 3055 | 3056 | For much more information see the README.md - which serves as the 3057 | manual. 3058 | """ 3059 | ) 3060 | sys.exit() 3061 | 3062 | iThoughtsColours = iThoughtsColours() 3063 | 3064 | iThoughtsShapes = iThoughtsShapes() 3065 | 3066 | iThoughtsSpread = iThoughtsSpread() 3067 | 3068 | iThoughtsIcons = iThoughtsIcons() 3069 | 3070 | streamHandler = streamHandler() 3071 | 3072 | matchCriteria, actionsLists, output = ParameterParser().getParameters() 3073 | 3074 | writeOutput(output) 3075 | 3076 | # Convert stdin data into CSV array - in whatever form it was 3077 | ( 3078 | csvRows, 3079 | colourColumn, 3080 | levelColumn, 3081 | noteColumn, 3082 | shapeColumn, 3083 | positionColumn, 3084 | iconsColumn, 3085 | progressColumn, 3086 | priorityColumn, 3087 | effortColumn, 3088 | startColumn, 3089 | dueColumn, 3090 | output, 3091 | ) = TreeReader().createCSVArray() 3092 | 3093 | writeOutput(output) 3094 | 3095 | # Build the tree from the CSV array 3096 | currentLevel = -1 3097 | csvTree = CSVTree("", "", "", "", "", "", "", "", "", currentLevel, "", "") 3098 | currentNode = csvTree 3099 | 3100 | for rowNumber, row in enumerate(csvRows[1:]): 3101 | # Extract information from this row 3102 | level = int(row[levelColumn]) 3103 | shape = row[shapeColumn] 3104 | colour = row[colourColumn] 3105 | note = row[noteColumn] 3106 | position = row[positionColumn] 3107 | icons = row[iconsColumn] 3108 | progress = row[progressColumn] 3109 | priority = row[priorityColumn] 3110 | effort = row[effortColumn] 3111 | start = row[startColumn] 3112 | due = row[dueColumn] 3113 | 3114 | cellValue = row[levelColumn + level + 1] 3115 | 3116 | if level > currentLevel: 3117 | # New child of previous node 3118 | currentLevel = level 3119 | currentNode = currentNode.addChild( 3120 | CSVTree( 3121 | due, 3122 | start, 3123 | effort, 3124 | priority, 3125 | progress, 3126 | icons, 3127 | shape, 3128 | colour, 3129 | note, 3130 | level, 3131 | position, 3132 | cellValue, 3133 | ) 3134 | ) 3135 | elif level == currentLevel: 3136 | # New sibling of previous node 3137 | currentNode = currentNode.parent.addChild( 3138 | CSVTree( 3139 | due, 3140 | start, 3141 | effort, 3142 | priority, 3143 | progress, 3144 | icons, 3145 | shape, 3146 | colour, 3147 | note, 3148 | level, 3149 | position, 3150 | cellValue, 3151 | ) 3152 | ) 3153 | else: 3154 | # Not a sibling or child of previous node 3155 | currentLevel = level 3156 | 3157 | # Look for the true parent by backing up the tree 3158 | parentNode = currentNode 3159 | while int(parentNode.data["level"]) >= int(level): 3160 | parentNode = parentNode.parent 3161 | 3162 | # Add the new node - now we've found the parent to add it to 3163 | currentNode = parentNode.addChild( 3164 | CSVTree( 3165 | due, 3166 | start, 3167 | effort, 3168 | priority, 3169 | progress, 3170 | icons, 3171 | shape, 3172 | colour, 3173 | note, 3174 | level, 3175 | position, 3176 | cellValue, 3177 | ) 3178 | ) 3179 | 3180 | currentNode.data["row"] = row 3181 | 3182 | # Statistics Header - For Matches and remaining unmatched 3183 | sys.stderr.write("Match Statistics\n") 3184 | sys.stderr.write("----------------\n") 3185 | 3186 | # Apply battery of parameter pairs to do the colouring etc. 3187 | # (A row could match more than one and a later one overrides an earlier one) 3188 | # In some cases the "match criterion" is a command and the "actions list" 3189 | # contains parameters for that command. e.g. "markdown" 3190 | matchesCount = -1 3191 | for parmPair, matchCriterion in enumerate(matchCriteria): 3192 | actionsList = actionsLists[parmPair] 3193 | 3194 | if (isinstance(matchCriterion, str)) and (matchCriterion.startswith("@")): 3195 | csvTree.applyActions(matchCriterion, actionsList) 3196 | 3197 | # generate statistics 3198 | matchesCount = csvTree.countMatches(matchCriterion) 3199 | 3200 | elif matchCriterion.pattern.lower() == "dump": 3201 | sys.stderr.write(csvTree.dump(actionsList)) 3202 | 3203 | elif actionsList[0] == "keep": 3204 | csvTree.processKeep(matchCriterion) 3205 | 3206 | elif actionsList[0] in ["autocolour", "autocolor", "ac"]: 3207 | csvTree.processAutocolour(matchCriterion) 3208 | 3209 | elif matchCriterion.pattern.lower() == "properties": 3210 | csvTree.printTreeProperties() 3211 | 3212 | else: 3213 | func = { 3214 | "check": csvTree.checkHierarchy, 3215 | "digraph": csvTree.exportToDotDigraph, 3216 | "hspread": csvTree.doHorizontalSpread, 3217 | "html": csvTree.exportToHTML, 3218 | "markdown": csvTree.exportToMarkdown, 3219 | "indented": csvTree.exportToIndentedText, 3220 | "promote": csvTree.promoteLevel, 3221 | "stats": csvTree.writeStatistics, 3222 | "vspread": csvTree.doVerticalSpread, 3223 | "xml": csvTree.exportToXML, 3224 | "taskpaper": csvTree.exportToTaskpaper, 3225 | }.get(matchCriterion.pattern.lower()) 3226 | 3227 | if func: 3228 | output = func(actionsList) 3229 | if output: 3230 | print("\n".join(output)) 3231 | sys.exit() 3232 | # commands that generate no output fall through 3233 | 3234 | else: 3235 | # This is where a node-level regular expression match would end up 3236 | csvTree.applyActions(matchCriterion, actionsList) 3237 | 3238 | # It's possible some nodes were deleted so actually remove them 3239 | csvTree.cleanUpDeleted() 3240 | 3241 | csvTree.repairSubtreeLevels(-2) 3242 | 3243 | # generate statistics 3244 | matchesCount = csvTree.countMatches(matchCriterion) 3245 | 3246 | if matchesCount > -1: 3247 | # Valid to print unmatched 3248 | csvTree.countUnmatched() 3249 | 3250 | # Statistics Trailer - For Matches and remaining unmatched 3251 | sys.stderr.write("----------------\n\n") 3252 | 3253 | 3254 | TreeWriter().writeTreeAsCSV() 3255 | --------------------------------------------------------------------------------