46 |
47 |
48 |
49 |
50 |
51 |
52 |
53 |
54 |
55 |
56 |
200 |
201 |
202 |
--------------------------------------------------------------------------------
/vimwikigraph/vimwikigraph.py:
--------------------------------------------------------------------------------
1 | import copy
2 | import os
3 | import re
4 | import traceback
5 | import logging
6 | from logging import critical, debug, error, info, warning
7 |
8 | import networkx as nx
9 | import numpy as np
10 | from pyvis.network import Network
11 |
12 |
13 | class VimwikiGraph:
14 |
15 | # {{{ Private
16 | def __init__(self, root_dir: str, file_extensions: list = ['wiki'], graph_name: str = 'vimwikigraph', **args):
17 | # logging.basicConfig(level=logging.DEBUG)
18 | self.graph_name = graph_name
19 | self.graph = nx.DiGraph(name=graph_name)
20 | self.root_dir = root_dir
21 | self.file_extensions = file_extensions
22 | self.lines = dict()
23 | node_dict = self.__create_nodes()
24 | self.__parse_and_add_edges(node_dict)
25 | self.original_graph = copy.deepcopy(self.graph)
26 |
27 | def __normalize_path(self, root, link):
28 | if re.match(r'https?://', link):
29 | link = re.sub(r'.*:/+', '', link)
30 | link = re.sub(r'/.*', '', link)
31 | return link
32 | if re.match('file:/', link):
33 | link = re.sub(r'.*:/+', '', link)
34 | link = re.sub(r'.*/', '', link)
35 | return link
36 | path = os.path.join(root, link)
37 | path = re.sub(r':', '', path)
38 | if not path.endswith(".wiki"):
39 | path += ".wiki"
40 | return re.sub(r'[^/]+/\.\./', '', path)
41 |
42 | def __resolve_relative_path(self, path):
43 | if path[0] == '/':
44 | return path
45 | return os.path.join(self.root_dir, path)
46 |
47 | def __add_suffix_to_node(self, path):
48 | if not path.endswith(".wiki"):
49 | return path + ".wiki"
50 |
51 | def __create_nodes(self):
52 | node_dict = dict()
53 | for root, dir, files in os.walk(self.root_dir):
54 | for file in files:
55 | split = file.split('.')
56 | if split[-1] in self.file_extensions:
57 | name = os.path.join(root, file)
58 | node_dict[name] = root
59 | self.graph.add_node(name, label='.'.join(split[:-1]))
60 | return node_dict
61 |
62 | def __parse_and_add_edges(self, node_dict):
63 | for name, root in node_dict.items():
64 | with open(name, 'r') as f:
65 | lines = f.readlines()
66 | self.lines[name] = lines
67 | for line in lines:
68 | links = re.findall(r'\[\[([^#|\[\]]+)(#[^|\[\]]*)?(|[^\]]*)?\]\]', line)
69 | for link in links:
70 | child_node = self.__normalize_path(root, link[0])
71 | self.graph.add_edge(name, child_node)
72 |
73 | def __filter_lines(self, regexes: list, lines: list):
74 | """
75 | Returns the number of regexes that matched any of the lines in lines.
76 |
77 | Args:
78 | regexes (list): List of regular expressions.
79 | lines (list): List of lines to match.
80 | """
81 | matches = 0
82 | try:
83 | for regex in regexes:
84 | for line in lines:
85 | if re.search(regex, line.lower()):
86 | matches += 1
87 | break
88 | except Exception as e:
89 | print(e)
90 | return matches
91 |
92 | def __filter_lines_all(self, regexes: list, lines: list, invert: bool = False):
93 | if invert:
94 | return self.__filter_lines(regexes, lines) == 0
95 | return self.__filter_lines(regexes, lines) == len(regexes)
96 |
97 | def __filter_lines_any(self, regexes: list, lines: list):
98 | return self.__filter_lines(regexes, lines) > 0
99 | # }}}
100 |
101 | # {{{ Graph Operations
102 | def reset_graph(self):
103 | self.graph = copy.deepcopy(self.original_graph)
104 | return self
105 |
106 | def reload_graph(self):
107 | del self.graph, self.original_graph, self.lines
108 | self.graph = nx.DiGraph(name=self.graph_name)
109 | self.lines = dict()
110 | node_dict = self.__create_nodes()
111 | self.__parse_and_add_edges(node_dict)
112 | self.original_graph = copy.deepcopy(self.graph)
113 |
114 | def add_attribute_by_regex(self, regexes: list, attribute: list, value: list):
115 | """
116 | Add attributes with the corresponding values to documents whose contents is matched by a
117 | conjunction of the specified regexes.
118 |
119 | Args:
120 | regexes (list)
121 | attribute (list): list of graphviz attribute names
122 | value (list): list of corresponding values
123 | """
124 | for node in self.graph.nodes:
125 | lines = self.lines.get(node, list())
126 | if self.__filter_lines_all(regexes, lines):
127 | for attr, val in zip(attribute, value):
128 | self.graph.nodes[node][attr] = val
129 | return self
130 |
131 | def weight_attribute(self, attribute: str = 'fontsize', min_val: int = 20, max_val: int = 100):
132 | """
133 | Adds and scales a DOC attribute according to a node's scaled betweenness centrality.
134 | It will assign the maximum value to the node with the highest betweenness centrality.
135 |
136 | Args:
137 | attribute: The attribute to add and scale.
138 | min_val: Minimum value.
139 | max_val: Maximum value.
140 | """
141 | try:
142 | exponent = np.log(max_val)/np.log(min_val)
143 | centrality = nx.betweenness_centrality(self.graph)
144 | max_centrality = max(centrality.values())
145 | if not max_centrality:
146 | return self
147 | for key, value in centrality.items():
148 | val = max(min((min_val*value/max_centrality)**exponent, max_val), min_val)
149 | self.graph.nodes[key][attribute] = val
150 | except Exception as e:
151 | print(e)
152 | finally:
153 | return self
154 |
155 | def filter_filenames(self, regexes: list, invert: bool = False):
156 | """
157 | Filter filenames by regexes. All node labels that do not match all of the regular expressions in 'regexes' will be removed. If invert then all
158 | nodes that match any of the regular expressions will be removed.
159 |
160 | Args:
161 | regexes (list)
162 | """
163 | nodes_to_remove = list()
164 | for node, data in self.graph.nodes(data=True):
165 | label = data.get('label')
166 | debug(f"{node} label: {label} regexes: {regexes}")
167 | if label and not self.__filter_lines_all(regexes, [data.get('label')], invert=invert):
168 | nodes_to_remove.append(node)
169 | self.graph.remove_nodes_from(nodes_to_remove)
170 | return self
171 |
172 | def filter_nodes(self, regexes: list, invert: bool = False):
173 | """
174 | Filters nodes by regexes. All nodes that do not match all of the regular expressions in 'regexes' will be removed. If invert then all nodes that
175 | match any of the regular expressions will be removed.
176 |
177 | Args:
178 | regexes (list)
179 | invert (bool)
180 | """
181 | nodes_to_remove = list()
182 | for node in self.graph.nodes:
183 | lines = self.lines.get(node, list())
184 | if not self.__filter_lines_all(regexes, lines, invert=invert):
185 | nodes_to_remove.append(node)
186 | self.graph.remove_nodes_from(nodes_to_remove)
187 | return self
188 |
189 | def expand_node(self, node: str):
190 | if self.graph[node]['is_collapsed']:
191 | edges_to_remove = []
192 | for neighbor in self.graph.neighbors(node):
193 | edges_to_remove.append((node, neighbor))
194 | self.graph.remove_edges_from(edges_to_remove)
195 |
196 | edges_to_add = []
197 | nodes_to_add = []
198 | for neighbor in self.original_graph.neighbors(node):
199 | nodes_to_add.append(neighbor)
200 | for neighbor2 in self.original_graph.neighbors(neighbor):
201 | edges_to_add.append((neighbor, neighbor2))
202 | self.graph.add_nodes_from(nodes_to_add)
203 | self.graph.add_edges_from(edges_to_add)
204 | self.graph.nodes[node]['is_collapsed'] = False
205 | return self
206 |
207 | def collapse_children(self, nodes: list, depth: int = 1):
208 | """
209 | All child nodes will be collapsed into this node. Edges from and to children will go to the parent instead.
210 |
211 | Args:
212 | node (str): Full or relative path of a vimwiki document.
213 | depth (int): Number of levels to collapse.
214 | """
215 | for node in nodes:
216 | try:
217 | node = self.__resolve_relative_path(node)
218 | node = self.__add_suffix_to_node(node)
219 | self.graph.nodes[node]['is_collapsed'] = True
220 | children = nx.dfs_successors(self.graph, node, depth)
221 | for child in np.concatenate(list(children.values())):
222 | nx.contracted_nodes(self.graph, node, child, self_loops=False, copy=False)
223 | del self.graph.nodes[node]['contraction']
224 | except Exception:
225 | traceback.print_exc()
226 | return self
227 |
228 | def remove_nonadjacent_nodes(self, node: str, depth: int = 1):
229 | """
230 | Removes all nodes that are not successors (up to a certain depth) of the specified node in an undirected copy of the graph.
231 |
232 | Args:
233 | node (str)
234 | depth (int)
235 | """
236 | node = self.__resolve_relative_path(node)
237 | adjacent_nodes = [node]
238 | successors = nx.dfs_successors(self.graph.to_undirected(), node, depth)
239 | if len(list(successors.values())):
240 | adjacent_nodes.extend(np.concatenate(list(successors.values())))
241 | adjacent_nodes = np.unique(adjacent_nodes)
242 | nodes_to_remove = list()
243 | for n in self.graph.nodes:
244 | if n not in adjacent_nodes:
245 | nodes_to_remove.append(n)
246 | self.graph.remove_nodes_from(nodes_to_remove)
247 | self.graph.nodes[node]['color'] = 'red'
248 | self.graph.nodes[node]['style'] = 'filled'
249 | return self
250 |
251 | def extend_node_label(self, regexes: list, join_str: str = '\n'):
252 | """
253 | Add additional information to node labels.
254 |
255 | Args:
256 | regexes (list): list of regexes to match data in the node's corresponding documents
257 | join_str (str): a string or character used to join any matches
258 | """
259 | try:
260 | for node in self.graph.nodes:
261 | lines = self.lines[node]
262 | all_matches = list()
263 | for regex in regexes:
264 | for line in lines:
265 | matches = re.findall(regex, line)
266 | all_matches.extend(matches)
267 | label = self.graph.nodes[node]['label']
268 | label = f'"{label}{join_str}{join_str.join(all_matches)}"'
269 | self.graph.nodes[node]['label'] = label
270 | except Exception as e:
271 | print(e)
272 | finally:
273 | return self
274 | # }}}
275 |
276 | # {{{ Output
277 | def write(self, name: str = "", filetype: str = 'png'):
278 | """
279 | Writes the current graph to one of name.png or name.dot
280 |
281 | Args:
282 | name (str): Name of the resulting file.
283 | filetype (str): File type. May be either 'png' or 'dot'.
284 | """
285 | if not name:
286 | name = self.graph.name
287 | PG = nx.drawing.nx_pydot.to_pydot(self.graph)
288 | PG.set_rankdir('LR')
289 | if filetype == 'png':
290 | PG.write_png(f"{name}.png")
291 | elif filetype == 'dot':
292 | PG.write_raw(f"{name}.dot")
293 | elif filetype == 'gml':
294 | nx.write_gml(self.graph, f"{name}.gml")
295 | else:
296 | raise Exception("Invalid file type")
297 |
298 | def write_pyviz(self, name: str = ""):
299 | if not name:
300 | name = self.graph.name
301 | nt = Network(
302 | height='80vh',
303 | width='95vw',
304 | neighborhood_highlight=True,
305 | filter_menu=True,
306 | cdn_resources='remote',
307 | )
308 | nt.from_nx(self.graph)
309 | with open(f'{name}.html', 'w') as f:
310 | f.write(nt.generate_html())
311 | # }}}
312 |
--------------------------------------------------------------------------------
/vimwikigraph/vimwikitags.py:
--------------------------------------------------------------------------------
1 | from ripgrepy import Ripgrepy
2 |
3 |
4 | class VimwikiTags:
5 |
6 | def __init__(self, root_dir, search_pattern=r'^:((\w+:)+)'):
7 | self.root_dir = root_dir
8 | self.search_pattern = search_pattern
9 | self.search_result = []
10 | self.count_dict = {}
11 |
12 | def _search(self):
13 | rg = Ripgrepy(self.search_pattern, self.root_dir)
14 | matches = rg.no_filename().only_matching().run().as_string
15 | matches = matches.split(':')
16 | self.search_result = [m for m in matches if m != '' and m != '\n']
17 |
18 | def _generate_counts_dict(self):
19 | counts = dict()
20 | for m in self.search_result:
21 | if m not in counts:
22 | counts[m] = 1
23 | else:
24 | counts[m] += 1
25 | self.count_dict = dict(sorted(counts.items(), key=lambda item: item[1], reverse=True))
26 |
27 | def _format_dict(self):
28 | counts_list = list()
29 | for k, v in self.count_dict.items():
30 | counts_list.append("* {:<3} {}".format(str(v), k))
31 | return counts_list
32 |
33 | def populate_tags(self) -> dict:
34 | if not self.search_result:
35 | self._search()
36 | if not self.count_dict:
37 | self._generate_counts_dict()
38 | return self.count_dict # type: ignore
39 |
40 | def reload(self):
41 | del self.search_result, self.count_dict
42 | self.search_result = []
43 | self.count_dict = {}
44 |
--------------------------------------------------------------------------------