├── watch ├── compile ├── LICENSE ├── infra.dot ├── .gitignore ├── postprocess.py ├── dotpy.py └── generate.py /watch: -------------------------------------------------------------------------------- 1 | #!/bin/bash -eux 2 | 3 | echo -e 'infra.dot\ndiagram.dot\ndotpy.py\npostprocess.py\ngenerate.py' | \ 4 | entr -s "echo refreshed && (./compile > myinfra.svg)" 5 | -------------------------------------------------------------------------------- /compile: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -eux -o pipefail 3 | 4 | ./generate.py 5 | # TODO tee for debug? 6 | m4 infra.dot > myinfra.post.dot 7 | cat myinfra.post.dot | dot -T svg | ./postprocess.py 8 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Dmitrii Gerasimov 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /infra.dot: -------------------------------------------------------------------------------- 1 | # http://www.graphviz.org/content/cluster 2 | # TODO https://www.graphviz.org/doc/info/colors.html 3 | // https://www.rff.com/flowchart_shapes.php 4 | // https://www.graphviz.org/doc/info/shapes.html 5 | 6 | 7 | // TODO highlight edges that we want to eliminate (mostly to/from the internet?) 8 | 9 | digraph G { 10 | // some hack for edges between clusters? 11 | // https://stackoverflow.com/a/2012106/706389 12 | compound=true; 13 | 14 | // otherwise edges overlap too much.. 15 | ranksep=1.1; 16 | 17 | // size="10,10" 18 | // ratio="fill" 19 | // ok, newrank doesn't respect clusters.. 20 | // newrank=true; 21 | 22 | // TODO eh? it just moves edges a bit, not sure how useful otherwise... 23 | // searchsize=500; 24 | 25 | node [ 26 | shape="box" 27 | // margin=0 28 | ] 29 | 30 | // TODO hmm maybe LR is not too bad? 31 | // ugh. quite bad with html tables though.. 32 | // rankdir="LR"; 33 | 34 | 35 | // ok, this is bad, messes up edges from html tables I think? 36 | // concentrate=true; 37 | 38 | // hmm. used to have this as 'max', but same works a bit neater.. 39 | rank=same; 40 | 41 | // TODO not sure which rank to choose? 42 | 43 | 44 | include(`diagram.dot') 45 | } 46 | 47 | // toblog: Demonstrates how much indirection is there if you want to own your data 48 | 49 | // todo trivial connections (e.g. twitter phone app -> twitter are omitted) 50 | // todo kindle (unused) 51 | // todo if I draw an edge from UI to phone.... gonna be fun 52 | 53 | 54 | # TODO hmm. how to still draw a frame around it? 55 | 56 | // TODO hmm red:green:blue could be useful.. 57 | 58 | // TODO PDF annotation software and pdf provider? 59 | 60 | // TODO as you can see not everything has data access layer 61 | // so there is still something to work on 62 | 63 | // TODO ugh. sometimes order of edges seems to matter... 64 | 65 | // TODO browser history? 66 | 67 | # TODO display google home and mention how useless it is 68 | 69 | # TODO motivation for blood 70 | # I'm planning on tracking this for several decades, so providers will change 71 | 72 | # TODO highlight that it's easy to hook to DAL? 73 | 74 | # TODO borg 75 | 76 | # TODO also provide dynamic version if someone wants to mess with in browser 77 | # wonder if could allow to show/hide nodes? 78 | 79 | # TODO these are read only; contribute to search 80 | 81 | # TODO link some of my blog posts? E.g. ones using endomondo 82 | 83 | # TODO show missing links? like HN 84 | 85 | # TODO for orger, give more specific examples for static (e.g. used for search) and interactive (e.g. used to process reddit/hn) 86 | 87 | # TODO I guess it's nice to mention where I mention certain bits of infrastructure? 88 | 89 | // TODO right. I think I need to add browser history and that's it. publish straigh away after that 90 | 91 | 92 | // TODO not sure what should be first class... e.g. it's nice to be able to change somthing in dot file and rerender immediately 93 | 94 | # todo cloudmacs? 95 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Created by https://www.gitignore.io/api/python,emacs 3 | # Edit at https://www.gitignore.io/?templates=python,emacs 4 | 5 | ### Emacs ### 6 | # -*- mode: gitignore; -*- 7 | *~ 8 | \#*\# 9 | /.emacs.desktop 10 | /.emacs.desktop.lock 11 | *.elc 12 | auto-save-list 13 | tramp 14 | .\#* 15 | 16 | # Org-mode 17 | .org-id-locations 18 | *_archive 19 | 20 | # flymake-mode 21 | *_flymake.* 22 | 23 | # eshell files 24 | /eshell/history 25 | /eshell/lastdir 26 | 27 | # elpa packages 28 | /elpa/ 29 | 30 | # reftex files 31 | *.rel 32 | 33 | # AUCTeX auto folder 34 | /auto/ 35 | 36 | # cask packages 37 | .cask/ 38 | dist/ 39 | 40 | # Flycheck 41 | flycheck_*.el 42 | 43 | # server auth directory 44 | /server/ 45 | 46 | # projectiles files 47 | .projectile 48 | 49 | # directory configuration 50 | .dir-locals.el 51 | 52 | # network security 53 | /network-security.data 54 | 55 | 56 | ### Python ### 57 | # Byte-compiled / optimized / DLL files 58 | __pycache__/ 59 | *.py[cod] 60 | *$py.class 61 | 62 | # C extensions 63 | *.so 64 | 65 | # Distribution / packaging 66 | .Python 67 | build/ 68 | develop-eggs/ 69 | downloads/ 70 | eggs/ 71 | .eggs/ 72 | lib/ 73 | lib64/ 74 | parts/ 75 | sdist/ 76 | var/ 77 | wheels/ 78 | pip-wheel-metadata/ 79 | share/python-wheels/ 80 | *.egg-info/ 81 | .installed.cfg 82 | *.egg 83 | MANIFEST 84 | 85 | # PyInstaller 86 | # Usually these files are written by a python script from a template 87 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 88 | *.manifest 89 | *.spec 90 | 91 | # Installer logs 92 | pip-log.txt 93 | pip-delete-this-directory.txt 94 | 95 | # Unit test / coverage reports 96 | htmlcov/ 97 | .tox/ 98 | .nox/ 99 | .coverage 100 | .coverage.* 101 | .cache 102 | nosetests.xml 103 | coverage.xml 104 | *.cover 105 | .hypothesis/ 106 | .pytest_cache/ 107 | 108 | # Translations 109 | *.mo 110 | *.pot 111 | 112 | # Scrapy stuff: 113 | .scrapy 114 | 115 | # Sphinx documentation 116 | docs/_build/ 117 | 118 | # PyBuilder 119 | target/ 120 | 121 | # pyenv 122 | .python-version 123 | 124 | # pipenv 125 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 126 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 127 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 128 | # install all needed dependencies. 129 | #Pipfile.lock 130 | 131 | # celery beat schedule file 132 | celerybeat-schedule 133 | 134 | # SageMath parsed files 135 | *.sage.py 136 | 137 | # Spyder project settings 138 | .spyderproject 139 | .spyproject 140 | 141 | # Rope project settings 142 | .ropeproject 143 | 144 | # Mr Developer 145 | .mr.developer.cfg 146 | .project 147 | .pydevproject 148 | 149 | # mkdocs documentation 150 | /site 151 | 152 | # mypy 153 | .mypy_cache/ 154 | .dmypy.json 155 | dmypy.json 156 | 157 | # Pyre type checker 158 | .pyre/ 159 | 160 | # End of https://www.gitignore.io/api/python,emacs 161 | -------------------------------------------------------------------------------- /postprocess.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | from pathlib import Path 3 | import sys 4 | 5 | from lxml import etree as ET # type: ignore 6 | 7 | NS = '{http://www.w3.org/2000/svg}' 8 | def ns(s): 9 | return NS + s 10 | 11 | 12 | def fix_edge(e) -> None: 13 | eid = e.attrib['id'] 14 | 15 | lid = 'label_' + eid 16 | 17 | p = e.find('.//' + ns('path')) 18 | 19 | if p is None: 20 | return 21 | 22 | t = e.find('.//' + ns('text')) 23 | if t is None: 24 | return 25 | 26 | p.attrib['id'] = lid 27 | 28 | del t.attrib['x'] 29 | del t.attrib['y'] 30 | del t.attrib['text-anchor'] 31 | label = t.text 32 | t.text = None 33 | tp = ET.SubElement(t, 'textPath') 34 | tp.attrib['href'] = '#' + lid 35 | 36 | # TODO FIXME horrible.. 37 | # NOTE careful if you change this.. it's fixed in the blog CSS 38 | offset = 40 # if label == 'DAL' else 7 39 | 40 | tp.attrib['startOffset'] = f'{offset}%' 41 | tp.attrib['side'] = 'right' 42 | tp.text = label 43 | # TODO would be nice to add some padding as well, but not sure if it's possible 44 | 45 | 46 | def fix_id(n) -> None: 47 | nid = n.attrib['id'] 48 | 49 | t = n.find('.//' + ns('text')) 50 | if t is None: 51 | return 52 | 53 | # this is a bit questionable, but it doesn't seem that fragments work for svg 54 | # it kind of jumps somewhere, but it doesn't jump at the start of the cluster/node 55 | # whereas on text nodes, it works fine 56 | t.attrib['id'] = nid 57 | del n.attrib['id'] 58 | 59 | 60 | def add_class(x, cls: str) -> None: 61 | c = x.attrib.get('class', '') 62 | c = cls if c == '' else f'{c} {cls}' 63 | x.attrib['class'] = c 64 | 65 | 66 | def fix_xlink_title(n) -> None: 67 | # delete useless autogenerated titles 68 | xtitle = '{http://www.w3.org/1999/xlink}title' 69 | title = n.attrib.get(xtitle, None) 70 | if title is None: 71 | return 72 | 73 | if title == '': 74 | del n.attrib[xtitle] 75 | return 76 | 77 | text = ''.join(n.itertext()).strip() 78 | 79 | # https://developer.mozilla.org/en-US/docs/Web/SVG/Attribute/xlink:title 80 | # 'New content should use a child element rather than a xlink:title attribute.' 81 | del n.attrib[xtitle] 82 | te = ET.SubElement(n, 'title') 83 | te.text = title 84 | 85 | boring = text == title 86 | # if not boring: 87 | # print(repr(text), repr(title), file=sys.stderr) 88 | 89 | if not boring: 90 | add_class(n, 'tooltip') 91 | 92 | 93 | def run(inp: bytes) -> ET.ElementTree: 94 | root = ET.fromstring(inp) 95 | st = ET.SubElement(root, 'style') 96 | st.text = STYLE 97 | 98 | ## make edge labels follow the curve 99 | # TODO not sure if should use xlabel? 100 | figures = root.findall(f'.//{NS}g') 101 | for fig in figures: 102 | classes = fig.attrib.get('class', '').split() 103 | if 'edge' not in classes: 104 | continue 105 | edge = fig 106 | fix_edge(edge) 107 | ## 108 | 109 | ## for some reason, svg figures are not nested in graphviz output 110 | figures = root.findall(f'.//{NS}g') 111 | for fig in figures: 112 | classes = fig.attrib.get('class', '').split() 113 | # right, seems lxml doesn't support contains(@class, "node") ??? 114 | if 'node' not in classes: 115 | continue 116 | node = fig 117 | 118 | # find the target clusrer id 119 | CLUST = '_clust_' # see dotpy subgraph() code 120 | classes = [c for c in classes if c.startswith(CLUST)] 121 | if len(classes) == 0: 122 | continue 123 | [cls] = classes 124 | clid = cls[len(CLUST):] 125 | # 126 | 127 | cluster = root.find(f'.//{NS}g[@id="{clid}"]') 128 | assert cluster is not None 129 | 130 | # reattach to become its child 131 | node.getparent().remove(node) 132 | cluster.append(node) 133 | ## 134 | 135 | ## for some reason, #fragment links don't work properly against clusters 136 | ## I think svg only likes them on text 137 | figures = root.findall(f'.//{NS}g') 138 | for fig in figures: 139 | classes = fig.attrib.get('class', '').split() 140 | if 'cluster' in classes or 'edge' in classes: 141 | fix_id(fig) 142 | ## 143 | 144 | as_ = root.findall(f'.//{NS}a') 145 | for a in as_: 146 | fix_xlink_title(a) 147 | 148 | return root 149 | 150 | 151 | def main() -> None: 152 | from argparse import ArgumentParser as P 153 | p = P() 154 | p.add_argument('path', nargs='?', type=str) 155 | args = p.parse_args() 156 | path = args.path 157 | 158 | inp = sys.stdin.read() if path is None else open(path).read() 159 | 160 | res = run(inp.encode('utf8')) 161 | ress = ET.tostring(res, pretty_print=True).decode('utf8') 162 | sys.stdout.write(ress) 163 | 164 | 165 | # TODO right, also node doesn't know its cluster... 166 | 167 | # apparently only these are allowed? https://www.w3.org/TR/SVG11/attindex.html#PresentationAttributes 168 | # TODO add a floating button to drop selection? 169 | # ok, that could be a good way of highlighting... 170 | 171 | # huh ok, nice so this sort of works.. 172 | STYLE = ''' 173 | /* relies on target hack in dotpy!!! */ 174 | text:target ~ .node text { 175 | fill: red; 176 | font-weight: bold; 177 | } 178 | 179 | /* hmm ok, maybe not needed with tooltip icons?.. 180 | .tooltip text { 181 | fill: yellow; 182 | } 183 | */ 184 | ''' 185 | 186 | 187 | if __name__ == '__main__': 188 | main() 189 | -------------------------------------------------------------------------------- /dotpy.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import sys 4 | from typing import Dict, Iterable, Iterator, List, NamedTuple, Optional, Sequence, Union, Any 5 | 6 | 7 | # TODO move this away from core? 8 | cylinder = 'cylinder' 9 | diamond = 'diamond' 10 | point = 'point' 11 | star = 'star' 12 | plaintext = 'plaintext' 13 | 14 | dashed = 'dashed' 15 | dotted = 'dotted' 16 | filled = 'filled' 17 | rounded = 'rounded' 18 | 19 | lightblue = 'lightblue' 20 | darkgreen = 'darkgreen' 21 | black = 'black' 22 | blue = 'blue' 23 | gray = 'gray' 24 | green = 'green' 25 | orange = 'orange' 26 | red = 'red' 27 | purple = 'purple' 28 | 29 | record = dict(shape='record') 30 | noconstraint = dict(constraint='false') 31 | invisible = dict(style='invisible') 32 | 33 | tapered = 'tapered' 34 | 35 | _MODULE_NAME: Optional[str] = None 36 | 37 | 38 | def _id2obj() -> Dict[int, Any]: 39 | # todo document/test 40 | assert _MODULE_NAME is not None, 'Looks like you forgot to init()' 41 | globs = vars(sys.modules[_MODULE_NAME]) 42 | return {id(v): k for k, v in globs.items()} 43 | 44 | 45 | def init(mname: str): 46 | global _MODULE_NAME 47 | _MODULE_NAME = mname 48 | 49 | 50 | def get_name(obj: Any) -> str: 51 | m = _id2obj() 52 | oid = id(obj) 53 | assert oid in m, obj 54 | return m[oid] 55 | 56 | 57 | Data = Union[str, Iterable[str]] 58 | 59 | 60 | class Graph(NamedTuple): 61 | name_: Optional[str] 62 | cluster: bool 63 | raw: Data 64 | kind: str = 'subgraph' 65 | 66 | @property 67 | def name(self) -> str: 68 | sn = self.name_ 69 | if sn is not None: 70 | return sn 71 | else: 72 | return get_name(self) 73 | 74 | def render(self) -> Iterable[str]: 75 | name = self.name 76 | mcl = 'cluster_' if self.cluster else '' 77 | yield f'{self.kind} {mcl}{name}' + ' {' 78 | for x in [self.raw] if isinstance(self.raw, str) else self.raw: 79 | # TODO handle multiline stuff properly? 80 | yield ' ' + x 81 | yield '}' 82 | Subgraph = Graph # todo deprecate? 83 | 84 | 85 | Extra = Dict[str, str] 86 | class Node(NamedTuple): 87 | name_: Optional[str] 88 | extra: Dict 89 | set_class: bool = False # meh 90 | 91 | @property 92 | def name(self) -> str: 93 | sn = self.name_ 94 | if sn is not None: 95 | if ' ' in sn: 96 | # TODO check if already quoted first? 97 | sn = f'"{sn}"' 98 | return sn 99 | else: 100 | return get_name(self) 101 | 102 | def render(self) -> Iterable[str]: 103 | extra = {**self.extra} 104 | 105 | if self.set_class: 106 | assert 'class' not in extra, extra 107 | extra['class'] = self.name 108 | 109 | if len(extra) == 0: 110 | yield self.name 111 | return 112 | 113 | yield f'{self.name} [' 114 | for l in _render(**extra, self_=self): 115 | yield ' ' + l 116 | yield ']' 117 | # todo maybe just implement __str__? not sure... 118 | 119 | 120 | # todo label is actually name? 121 | Label = str 122 | Nodish = Union[Node, Label] 123 | class Edge(NamedTuple): 124 | f: Nodish 125 | t: Nodish 126 | extra: Dict 127 | 128 | def render(self) -> Iterable[str]: 129 | f = self.f 130 | t = self.t 131 | kwargs = self.extra 132 | 133 | fn = f if isinstance(f, str) else f.name 134 | tn = t if isinstance(t, str) else t.name 135 | extras = '' if len(kwargs) == 0 else ' [' + '\n'.join(_render(**kwargs)) + ']' 136 | yield f'{fn} -> {tn}' + extras 137 | 138 | from typing import Callable 139 | Prop = Union[str, Callable[[], str]] 140 | 141 | 142 | def _render(*args: str, self_=None, **kwargs): 143 | def handle(x: Prop) -> str: 144 | if isinstance(x, Callable): 145 | x = x(self_=self_) 146 | assert isinstance(x, str), x 147 | 148 | # meh. this is for HTML labels... 149 | if x[:1] + x[-1:] == '<>': 150 | return x 151 | else: 152 | return f'"{x}"' 153 | 154 | return list(args) + [f'{k}={handle(v)}' for k, v in kwargs.items()] 155 | 156 | 157 | SubgraphItem = Union[str, Dict, Node, Edge] 158 | def subgraph( 159 | *args: SubgraphItem, 160 | name: Optional[str]=None, 161 | cluster: bool=False, 162 | klass: Optional[str]=None, 163 | kind: Optional[str]=None, 164 | **kwargs, 165 | ) -> Graph: 166 | mclass = {} if klass is None else {'class': klass} 167 | kw = {**kwargs, **mclass} 168 | 169 | clid = kwargs.get('id') # TODO might be nice to make it automatic? 170 | mid = {} if clid is None else {'class': '_clust_' + clid} 171 | 172 | # TODO ugh. right, don't think that rendering during construction was a good idea... 173 | # perhaps need to write few tests and fix it properly... 174 | 175 | def it() -> Iterable[str]: 176 | for x in args: 177 | if isinstance(x, dict): 178 | # TODO a bit horrible.. 179 | # TODO also incorrect if we got inline bits of strings (e.g. adhoc 'subgraph whatever {') 180 | kw.update(x) 181 | elif isinstance(x, Graph): 182 | yield from x.render() 183 | elif isinstance(x, str): 184 | yield x 185 | elif isinstance(x, Node): 186 | # TODO right, this is pretty horrible... 187 | # I guess I need to 188 | # a) do not violate immutability 189 | # b) wrap str in Node anyway 190 | # c) pass this to Edge as well 191 | # x.extra.update(mclass) 192 | x.extra.update(mid) 193 | yield from x.render() 194 | elif isinstance(x, Edge): 195 | yield from x.render() 196 | else: 197 | raise RuntimeError(x) 198 | ag: Sequence[str] = list(it()) 199 | res = _render(*ag, **kw) # todo ??? what's going on here... 200 | return Graph(name_=name, cluster=cluster, raw=res, **maybe(kind=kind)) 201 | 202 | # todo not a great name? 203 | def maybe(**kwargs): 204 | return {k: v for k, v in kwargs.items() if v is not None} 205 | 206 | 207 | def cluster(*args, **kwargs) -> Graph: 208 | return subgraph(*args, cluster=True, **kwargs) 209 | 210 | 211 | def graph(*args, **kwargs) -> Graph: 212 | return subgraph(*args, **kwargs, kind='graph') 213 | 214 | 215 | def digraph(*args, **kwargs) -> Graph: 216 | return subgraph(*args, **kwargs, kind='digraph') 217 | 218 | 219 | def node(name: Optional[str]=None, set_class=False, **kwargs) -> Node: 220 | return Node(name_=name, set_class=set_class, extra=kwargs) 221 | 222 | 223 | EdgeArg = Union[Nodish, Dict] 224 | 225 | 226 | def edges(f: Nodish, t: Nodish, *args: EdgeArg, **kwargs) -> Iterator[Edge]: 227 | ee = [f, t] 228 | # TODO maybe allow multiedges? 229 | extra = {**kwargs} 230 | for a in args: 231 | if isinstance(a, dict): 232 | extra.update(a) 233 | else: 234 | ee.append(a) 235 | for ff, tt in zip(ee, ee[1:]): 236 | yield Edge(f=ff, t=tt, extra=extra) 237 | 238 | 239 | def edge(*args, **kwargs) -> Edge: 240 | [e] = edges(*args, **kwargs) 241 | return e 242 | 243 | 244 | EXTERNAL = blue 245 | INTERNAL = darkgreen 246 | def url(u: str, color=EXTERNAL) -> Extra: 247 | if u.startswith('#'): 248 | color = INTERNAL 249 | return { 250 | 'URL': u, 251 | 'fontcolor': color, # meh 252 | } 253 | 254 | 255 | def render(x) -> str: 256 | if isinstance(x, str): 257 | return x 258 | elif hasattr(x, 'render'): 259 | return '\n'.join(x.render()) 260 | elif isinstance(x, Iterable): 261 | return '\n'.join(render(c) for c in x) 262 | else: 263 | raise RuntimeError(f"Unsupported: {x}") 264 | 265 | 266 | def with_style(*, svg: str, style: str) -> str: 267 | from lxml import etree as ET # type: ignore 268 | root = ET.fromstring(svg.encode('utf8')) # eh? lxml wants it 269 | st = ET.SubElement(root, 'style') 270 | st.text = style 271 | return ET.tostring(root, pretty_print=True).decode('utf8') 272 | # todo not sure if defs thing is necessary? 273 | # <defs> </defs> 274 | 275 | NS = '{http://www.w3.org/2000/svg}' 276 | def svgns(s): 277 | return NS + s 278 | 279 | 280 | def group(*things): 281 | yield '{' 282 | yield from things 283 | yield '}' 284 | 285 | 286 | def html_attrs(**kwargs) -> str: 287 | attrs = ' '.join(f'{k.upper()}="{v}"' for k, v in kwargs.items()) 288 | if len(attrs) > 0: 289 | attrs = ' ' + attrs 290 | return attrs 291 | 292 | 293 | def td(x: str, **kwargs) -> str: 294 | x = x.replace('\n', '<BR/>') 295 | if 'tooltip' in kwargs: 296 | x = x + '💬' # not sure.. 297 | return f'<TD {html_attrs(**kwargs)}>{x}</TD>' 298 | 299 | 300 | def td_url(text: str, *, href: str, color=EXTERNAL, **kwargs) -> str: 301 | if href.startswith('#'): 302 | color = INTERNAL 303 | kwargs['href'] = href 304 | 305 | return td( 306 | f'<FONT COLOR="{color}">' + text + '</FONT>', 307 | **kwargs, 308 | ) 309 | 310 | Rows = List[List[str]] 311 | def raw_table(rows: Rows, **kwargs): 312 | rows_: list[str] = [] 313 | for row in rows: 314 | # todo would be nice to tabulate? 315 | row_ = '<TR>' + '\n'.join(row) + '</TR>' 316 | rows_.append(row_) 317 | # default shape is plaintext? 318 | label = '\n'.join([ 319 | f'<TABLE {html_attrs(**kwargs)}>', 320 | *rows_, 321 | '</TABLE>', 322 | ]) 323 | return label 324 | 325 | 326 | # TODO hmm tables have weird padding, seems to be only in svg mode; png is fine 327 | # takes either str or td ojbect? 328 | def table(rows: Rows, **kwargs): 329 | label = raw_table(rows=rows, **kwargs) 330 | label = '<' + label + '>' 331 | return node( 332 | shape='plain', # hmm differs from 'plaintext'; enforces width? 333 | label=label, 334 | ) 335 | 336 | 337 | def test_node() -> None: 338 | n = node(name='test', shape='star', label="Some node") 339 | # todo not sure about quotes for star? 340 | assert render(n) == ''' 341 | test [ 342 | shape="star" 343 | label="Some node" 344 | ] 345 | '''.strip() 346 | 347 | # TODO not sure about this... maybe should be a distinguished class or something 348 | n = node(name='test', label='< <b>HTML</b> >') 349 | assert render(n) == ''' 350 | test [ 351 | label=< <b>HTML</b> > 352 | ] 353 | '''.strip() 354 | 355 | n = node(name='test', **{'class': 'custom'}) 356 | assert render(n) == ''' 357 | test [ 358 | class="custom" 359 | ] 360 | '''.strip() 361 | 362 | n = node(name='test', set_class=True) 363 | assert render(n) == ''' 364 | test [ 365 | class="test" 366 | ] 367 | '''.strip() 368 | 369 | n = node(name='lazy', id=lambda self_: 'so_' + self_.name) 370 | assert render(n) == ''' 371 | lazy [ 372 | id="so_lazy" 373 | ] 374 | '''.strip() 375 | 376 | 377 | def test_edges() -> None: 378 | e = edges('node1', 'node2', 'node3') 379 | r = render(e) 380 | assert r == ''' 381 | node1 -> node2 382 | node2 -> node3 383 | '''.strip() 384 | 385 | 386 | def test_graph() -> None: 387 | n1 = node(name='nnn') 388 | # LR = dict(rankdir='LR') 389 | g = graph( 390 | 'rankdir="LR"', 391 | n1, 392 | name='G', 393 | ) 394 | assert render(g) == ''' 395 | graph G { 396 | rankdir="LR" 397 | nnn 398 | } 399 | '''.strip() 400 | 401 | 402 | def test_cluster() -> None: 403 | n1 = node(name='node1') 404 | n2 = node(name='node2') 405 | c = cluster( 406 | 'node [shape=point]', 407 | n1, 408 | n2, 409 | edge(n1, n2), 410 | name='testcluster', 411 | ) 412 | r = render(c) 413 | assert r == ''' 414 | subgraph cluster_testcluster { 415 | node [shape=point] 416 | node1 417 | node2 418 | node1 -> node2 419 | } 420 | '''.strip() 421 | -------------------------------------------------------------------------------- /generate.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | from __future__ import annotations 3 | 4 | from itertools import chain 5 | from pathlib import Path 6 | 7 | import dotpy 8 | dotpy.init(__name__) # TODO extremely meh 9 | 10 | from dotpy import * 11 | 12 | 13 | debug = False 14 | 15 | def gh(x: str) -> str: 16 | return f'https://github.com/{x}' 17 | 18 | def bb(x: str) -> str: 19 | return f'https://beepb00p.xyz/{x}' 20 | 21 | 22 | BLOG_COLOR = purple 23 | 24 | CLOUD = { 25 | 'style': 'dashed,rounded', 26 | } 27 | 28 | DEAD = dict( 29 | bgcolor='red:lightgrey', 30 | gradientangle='270', 31 | ) 32 | WARNING = dict( 33 | bgcolor='yellow:lightgrey', 34 | gradientangle='270', 35 | ) 36 | 37 | INVIS = { 38 | 'style': 'invis', 39 | } 40 | 41 | COLOR_DEVICE = '#aaaaaa' 42 | DEVICE = { 43 | 'style': filled, 44 | 'color': COLOR_DEVICE, 45 | } 46 | 47 | noarrow = { 48 | 'arrowhead': 'none', 49 | } 50 | 51 | BLOG_EDGE = { 52 | 'color': BLOG_COLOR, 53 | 'style': dashed, 54 | **noarrow, 55 | } 56 | 57 | 58 | COLOR_UI = 'pink' 59 | UI = { 60 | 'style': filled, 61 | 'color': COLOR_UI, 62 | } 63 | 64 | NOCONSTRAINT = { 65 | 'constraint': 'false', 66 | } 67 | 68 | 69 | CYLINDER = { 70 | # ok, cylinder doesn't take too much extra space 71 | 'shape': cylinder, 72 | } 73 | 74 | AUTO = { 75 | 'style': rounded, 76 | } 77 | 78 | 79 | MANUAL = { 80 | # ok, trapezium wastes a bit too much space 81 | # 'shape': 'invtrapezium', 82 | 'style': filled, 83 | 'color': '#d5b85a', 84 | } 85 | 86 | CLOUD_SYNC = dict( 87 | style=dashed, 88 | **CYLINDER, 89 | ) 90 | 91 | ### 92 | 93 | class C: 94 | runnerup = '#0069d0' 95 | pinboard = '#af0000' 96 | twitter = lightblue 97 | reddit = 'pink' 98 | vk = 'blue' # todo 99 | instapaper = 'lightgray' 100 | kobo = '#bf2026' 101 | jawbone = '#540baf' 102 | endomondo = green 103 | hackernews = orange 104 | google = orange # ?? 105 | remarkable = 'gray' 106 | bluemaestro = 'blue' 107 | telegram = '#0088cc' 108 | gpslogger = '#4c76bd' 109 | pocket = '#e83e53' 110 | github = '#f7ba9a' 111 | blood = red 112 | weight = 'brown' 113 | messenger = '#14a2f9' 114 | garmin = 'black' # eh, why not 115 | emfit = 'gray' 116 | discord = '#8697f6' 117 | 118 | materialistic = hackernews 119 | twitter_archive = twitter 120 | takeouts = google 121 | 122 | ###### 123 | 124 | 125 | def browser(for_, label='Browser', **kwargs): 126 | # returns new node deliberately, to prevent edge clutter 127 | return node( 128 | name=f'browser_for_{for_}', 129 | label=label, 130 | **UI, 131 | **kwargs, 132 | ) 133 | 134 | 135 | # TODO pipelines could link to sad state 136 | 137 | def blog_post(link: str, *args, **kwargs) -> Node: 138 | return node( # type: ignore 139 | *args, 140 | shape='component', 141 | color=BLOG_COLOR, 142 | **url(link, color=BLOG_COLOR), # TODO need extra attr to mark link.. 143 | **kwargs, 144 | ) # type: ignore[misc] 145 | 146 | 147 | 148 | def bbm(x: str): 149 | return bb(f'my-data.html#{x}') 150 | 151 | scales = node( 152 | label='scales', 153 | **url(bbm('weight')), 154 | **DEVICE, 155 | ) 156 | 157 | blood_tests = node( 158 | label='Blood tests\n(GP/Thriva/etc)', 159 | **url(bbm('blood')) 160 | ) 161 | 162 | sleep_subj = node( 163 | label='Sleep data\n(subjective)', 164 | **url(bbm('sleep')), 165 | ) 166 | 167 | exercise = node('Exercise') 168 | 169 | 170 | blog_orger = td_url( 171 | 'Orger:\nplaintext reflection\nof your digital self', 172 | href=bb('orger.html'), 173 | color=BLOG_COLOR, 174 | ) 175 | 176 | blog_orger_todos = td_url( 177 | 'Managing inbound digital content', 178 | href=bb('orger-todos.html'), 179 | color=BLOG_COLOR, 180 | ) 181 | 182 | blog_orger_roam = td_url( 183 | 'Orger\n+\nRoam Research', 184 | href=bb('myinfra-roam.html#orger'), 185 | color=BLOG_COLOR, 186 | ) 187 | 188 | def orger_mirrors() -> List[str]: 189 | return [ 190 | 'kobo', 191 | 'twitter', 192 | 'instapaper', 193 | 'youtube', 194 | 'hypothesis', 195 | 'github', 196 | 'polar', 197 | '...and more', 198 | # todo actually get them straight from orger modules? 199 | ] 200 | 201 | def orger_queues() -> List[str]: 202 | return [ 203 | 'kobo2org', 204 | 'ip2org', 205 | 'reddit', 206 | 'hackernews', 207 | '...and more', 208 | ] 209 | 210 | 211 | orger_posts = table([ 212 | [blog_orger], 213 | [blog_orger_todos], 214 | [blog_orger_roam] 215 | ]) 216 | 217 | orger = table([ 218 | [td_url('Github: orger', href=gh('karlicoss/orger'), colspan=2)], 219 | # todo make them small tables? so could include direct module links 220 | [ 221 | td('\n'.join(['Mirrors:'] + orger_mirrors()), port='mirrors'), 222 | td('\n'.join(['Queues:' ] + orger_queues() ), port='queues' ), 223 | ], 224 | ]) 225 | 226 | 227 | # TODO instead of orger, it should be 'Plaintext reflections' or smth like that 228 | # TODO reduce distance between edges... 229 | # TODO eh. maybe instead simply list/url modules and only split into interactive/static? 230 | orger_cl = cluster( 231 | orger_posts, 232 | orger, 233 | # 234 | label='Orger ¶', 235 | style=dashed, 236 | id='orger', 237 | **url('#orger'), 238 | ) 239 | 240 | pkm_search_post = blog_post( 241 | bb('pkm-search.html'), 242 | label='Building personal\nsearch engine', 243 | ) 244 | 245 | emacs = node( 246 | label='Emacs\n(Doom)', 247 | **UI, 248 | **url('https://github.com/hlissner/doom-emacs'), 249 | ) 250 | 251 | logseq = node( 252 | label='Logseq', 253 | **UI, 254 | **url('https://github.com/logseq/logseq#why-logseq'), 255 | ) 256 | 257 | orger_outputs = cluster( 258 | 'node [shape=cylinder]', 259 | emacs, 260 | logseq, 261 | edge('orger:mirrors', '"data mirrors\n(read only)"'), 262 | edge('orger:queues' , '"todo lists\ninteractive queues"'), 263 | pkm_search_post, 264 | label='Plaintext files', 265 | style=dashed, 266 | ) 267 | 268 | blog_promnesia = td_url( 269 | 'My journey in\nfixing browser history', 270 | href=bb('promnesia.html'), 271 | color=BLOG_COLOR, 272 | ) 273 | 274 | blog_promnesia_roam = td_url( 275 | 'Extending my\npersonal infrastructure', 276 | href=bb('myinfra-roam.html'), 277 | color=BLOG_COLOR, 278 | ) 279 | 280 | blog_scheduler = td_url( 281 | 'In search of\na friendlier\nscheduler', 282 | href=bb('scheduler.html'), 283 | color=BLOG_COLOR, 284 | ) 285 | 286 | blog_dataliberation = td_url( 287 | 'Building data\nliberation\ninfrastructure', 288 | href=bb('exports.html'), 289 | color=BLOG_COLOR, 290 | ) 291 | 292 | 293 | promnesia_posts = table([[blog_promnesia]]) 294 | 295 | 296 | promnesia_browser = browser('promnesia', label='Browser\n(extension)', **url('https://github.com/karlicoss/promnesia#demos')) 297 | archivebox = node(label='Archivebox\n(web preservation)', **url('https://github.com/ArchiveBox/ArchiveBox#readme')) 298 | 299 | promnesia = table([ 300 | [td_url('Github: promnesia', href=gh('karlicoss/promnesia#readme'))], 301 | ]) 302 | 303 | promnesia_cl = cluster( 304 | promnesia_posts, 305 | promnesia, 306 | 307 | label='Promnesia ¶', 308 | style=dashed, 309 | id='promnesia', 310 | **url('#promnesia'), 311 | ) 312 | 313 | 314 | 315 | # todo indicate that it's selfhosted? 316 | 317 | syncthing = node(**INVIS) 318 | syncthing_cl = cluster( 319 | syncthing, 320 | CLOUD, 321 | url('https://syncthing.net'), 322 | color=lightblue, # todo fill? 323 | label='Syncthing', 324 | ) 325 | 326 | 327 | # TODO "timeline" can be treated as poor man's api?? 328 | timeline = node( 329 | label='Timeline\n/Memex\n(🚧wip🚧)', 330 | shape=star, 331 | **url(bb('tags.html#lifelogging')), 332 | ) 333 | 334 | 335 | mydata = blog_post( 336 | bb('my-data.html'), 337 | label='What data I collect\nand why?', 338 | ) 339 | 340 | brain_coping = blog_post( 341 | bb('pkm-setup.html'), 342 | label='How to cope\nwith a human brain', 343 | ) 344 | 345 | sad_infra = blog_post( 346 | bb('sad-infra.html'), 347 | label='The sad state of\npersonal data\nand infrastructure', 348 | ) 349 | 350 | 351 | meta = cluster( 352 | brain_coping, 353 | sad_infra, 354 | mydata, 355 | # *edges(brain_coping, sad_infra, **INVIS), 356 | label="Meta\n(why I'm doing all this?)", 357 | style=dashed, 358 | ) 359 | 360 | # todo colored text is clickable links? 361 | legend = cluster( 362 | node( 363 | 'Device', 364 | **DEVICE 365 | ), 366 | node( 367 | 'Cloud service', 368 | **CLOUD, 369 | ), 370 | node( 371 | name='legend_auto', 372 | label='Automatic\nscript', 373 | **AUTO, 374 | ), 375 | node( 376 | name='legend_manual', 377 | label='Manual\nstep', 378 | **MANUAL, 379 | ), 380 | blog_post( 381 | bb(''), 382 | label='Entry from my blog\n(clickable)', 383 | name='legend_blog', 384 | ), 385 | node( 386 | name='legend_ui', 387 | label='User facing\ninterface', 388 | **UI, 389 | ), 390 | # TODO elaborate what's so special about files? 391 | node( 392 | 'Disk storage', 393 | **CYLINDER, 394 | ), 395 | 396 | node( 397 | label='Dead\nservice/product', 398 | name='legend_dead', 399 | style=filled, 400 | # ugh. 401 | fillcolor='red:lightgrey', 402 | gradientangle='270', 403 | ), 404 | 405 | # 'Device -> "Manual step" -> legend_blog', 406 | label='Legend', 407 | style=dashed, 408 | ) 409 | 410 | # todo maybe nodes should be service aware or something? to color etc 411 | class exp: 412 | twint = node(**AUTO , **url(gh('twintproject/twint'))) 413 | tw_manual = node(**MANUAL, label='manual request\n(periodic)') 414 | vkexport = node(**AUTO , **url(gh('Totktonada/vk_messages_backup'))) 415 | telegram_backup = node(**AUTO , **url(gh('fabianonline/telegram_backup'))) 416 | 417 | messenger = node(**AUTO , label='fbmessengerexport', **url(gh('karlicoss/fbmessengerexport'))) 418 | 419 | rexport = node(**AUTO , **url(gh('karlicoss/rexport'))) 420 | pushshift = node(**AUTO , label='pushshift_export', **url(gh('seanbreckenridge/pushshift_comment_export'))) 421 | 422 | pinbexport = node(**AUTO , **url(gh('karlicoss/pinbexport'))) 423 | 424 | discord_manual = node(**MANUAL, label='manual request\n(periodic)') 425 | 426 | ghexport = node(**AUTO , **url(gh('karlicoss/ghexport'))) 427 | github_manual = node(**MANUAL, **url('https://github.com/settings/admin'), label='manual\ndownload') 428 | 429 | pockexport = node(**AUTO , **url(gh('karlicoss/pockexport'))) 430 | instapexport = node(**AUTO , **url(gh('karlicoss/instapexport'))) 431 | 432 | takeout_manual = node(**MANUAL, **url('https://beepb00p.xyz/my-data.html#takeout'), label='semi-manual\n(periodic)') 433 | kobuddy = node(**AUTO , **url(gh('karlicoss/kobuddy'))) 434 | remarkable_sync = node(**AUTO , label='script') 435 | 436 | # TODO maybe manual exports do not belong to export layer? not sure... 437 | # just make a note bout it? 438 | inp_weight = node(**MANUAL, label='manual\ninput') 439 | inp_blood = node(**MANUAL, label='manual\ninput') 440 | 441 | emfitexport = node(**AUTO , **url(gh('karlicoss/backup-emfit'))) 442 | jbexport = node(**AUTO ) 443 | inp_sleep = node(**MANUAL, label='manual\ninput') 444 | 445 | garmindb = node(**AUTO , label='GarminDB', **url('https://github.com/tcgoetz/GarminDB/blob/38c2409335a90130b0ca4618b0127d377ee4fbb1/download_garmin.py')) 446 | # TODO just unpack dicts if they are in args? 447 | endoexport = node(**AUTO , **url(gh('karlicoss/endoexport'))) 448 | inp_exercise = node(**MANUAL, label='manual\ninput') 449 | 450 | exp_items = {k: v for k, v in vars(exp).items() if not k.startswith('_')} 451 | for k, v in exp_items.items(): 452 | ex = v.extra 453 | if 'label' not in ex: 454 | ex['label'] = k 455 | globals()['exp_' + k] = v 456 | 457 | 458 | 459 | exports_infra = table([ 460 | [td('Data export infrastructure', border='0')], 461 | [blog_dataliberation], 462 | [blog_scheduler], 463 | ]) 464 | 465 | 466 | exports = cluster( 467 | exports_infra, 468 | *exp_items.values(), 469 | label='Export layer ¶', 470 | style=dotted, 471 | 472 | id='exports', 473 | **url('#exports'), 474 | ) 475 | 476 | 477 | 478 | blog_against_db = td_url( 479 | 'Against\nunnecessary databases', 480 | href=bb('unnecessary-db.html'), 481 | color=BLOG_COLOR, 482 | ) 483 | 484 | blog_backup_safety = td_url( 485 | '🚧Ensuring backup safety', 486 | href=bb('backup-checker.html'), 487 | color=BLOG_COLOR, 488 | ) 489 | 490 | blog_bleanser = td_url( 491 | '🚧Data exports deduplication', 492 | href='https://beepb00p.xyz/exobrain/projects/bleanser.html', 493 | color=BLOG_COLOR, 494 | ) 495 | 496 | 497 | filesystem_blog = table([ 498 | [blog_against_db], 499 | [blog_backup_safety], 500 | [blog_bleanser], 501 | ]) 502 | 503 | 504 | class fs: 505 | telegram = node(label='sqlite' ) 506 | 507 | messenger = node(label='sqlite' ) 508 | 509 | reddit = node(label='json' ) 510 | pushshift = node(label='json' ) 511 | 512 | pinboard = node(label='json' ) 513 | 514 | github = node(label='json' ) 515 | github_archive = node(label='zip/json') 516 | 517 | pocket = node(label='json' ) 518 | twitter = node(label='sqlite' ) 519 | twitter_archive = node(label='json' ) 520 | 521 | discord_archive = node(label='zip/json') 522 | 523 | kobo = node(label='sqlite' ) 524 | materialistic = node(label='sqlite' , **CLOUD_SYNC) 525 | remarkable = node(label='custom\nformat') 526 | 527 | vk = node(label='json' ) 528 | 529 | instapaper = node(label='json') 530 | # ugh. seems like a bug, it should inherit cylinder spect from the cluster 531 | bluemaestro = node(label='sqlite' , **CLOUD_SYNC) 532 | 533 | blood = node(label='orgmode') 534 | weight = node(label='orgmode') 535 | 536 | emfit = node(label='json' ) 537 | jawbone = node(label='json' ) 538 | sleep = node(label='orgmode') 539 | 540 | garmin = node(label='sqlite\njson fit') 541 | endomondo = node(label='json') 542 | exercise = node(label='orgmode') 543 | runnerup = node(label='tcx\nworkouts', **CLOUD_SYNC) 544 | 545 | gpslogger = node(label='gpx\ntracks', **CLOUD_SYNC) 546 | takeouts = node(label='json\nhtml') 547 | 548 | # TODO when I hover, highlight both? 549 | 550 | 551 | # todo mmm... hacky.. 552 | fs_items = {k: v for k, v in vars(fs).items() if not k.startswith('_')} 553 | for k, v in fs_items.items(): 554 | globals()['fs_' + k] = v 555 | # mm. hacky but sort of works.. nice 556 | col = getattr(C, k, None) 557 | if col is not None: 558 | v.extra['color'] = col 559 | 560 | 561 | filesystem = cluster( 562 | 'node [shape=cylinder]', 563 | filesystem_blog, 564 | *fs_items.values(), 565 | 566 | # wtf.. somehow this needs to be in the end in order to be rendered leftmost?? 567 | style=dotted, 568 | color=black, 569 | label='Filesystem ¶', 570 | id='fs', # ok, relying on ids makes sense 571 | **url("#fs"), 572 | ) 573 | 574 | # TODO add reference to data access layer to the graph 575 | 576 | blog_hpi = td_url( 577 | 'HPI: My life in a Python package', 578 | href=bb('hpi.html'), 579 | color=BLOG_COLOR, 580 | ) 581 | 582 | blog_hb_kcals = td_url( 583 | "Making sense of\nEndomondo's\ncalorie estimation", 584 | href=bb('heartbeats_vs_kcals.html'), 585 | color=BLOG_COLOR, 586 | ) 587 | blog_mypy_err = td_url( 588 | 'Using mypy for\nerror handling', 589 | href=bb('mypy-error-handling.html'), 590 | color=BLOG_COLOR, 591 | ) 592 | 593 | blog_configs_suck = td_url( 594 | 'Configs suck', 595 | href=bb('configs-suck.html'), 596 | color=BLOG_COLOR, 597 | ) 598 | 599 | 600 | 601 | def _m(name: str, **kwargs): # module label 602 | fname = f'<font point-size="24">{name}</font>' 603 | how = td_url if 'href' in kwargs else td 604 | return [how(fname, **kwargs)] 605 | 606 | class hpi: 607 | class other: 608 | _node = raw_table([ 609 | [td(' ', border=0)], 610 | _m('and more...', href='https://github.com/karlicoss/HPI/tree/master/my/'), 611 | ]) 612 | 613 | class sleep: 614 | main = 'sleep' 615 | 616 | jawbone = 'jawbone' 617 | emfit = 'emfit' 618 | manual = 'sleep_manual' 619 | garmin = 'sleep_garmin' 620 | 621 | _node = raw_table([ 622 | [ 623 | td(' ', border=0, port=emfit ), 624 | td(' ', border=0, port=jawbone), 625 | td(' ', border=0, port=manual ), 626 | td(' ', border=0, port=garmin ), 627 | ], 628 | _m('body.sleep', colspan=4, port=main), 629 | ]) 630 | 631 | class exercise: 632 | main = 'exercise' 633 | 634 | garmin = 'ex_garmin' 635 | endomondo = 'endomondo' 636 | manual = 'exercise_manual' 637 | runnerup = 'runnerup' 638 | 639 | _node = raw_table([ 640 | [ 641 | td(' ', border=0, port=garmin ), 642 | td(' ', border=0, port=endomondo), 643 | td(' ', border=0, port=manual ), 644 | td(' ', border=0, port=runnerup ), 645 | ], 646 | _m('body.exercise', colspan=4, port=main) 647 | ]) 648 | 649 | class weight: 650 | main = 'weight' 651 | inc = main + '_in' 652 | 653 | _node = raw_table([ 654 | [td(' ', border=0, port=inc)], # just dummy for consistencty 655 | _m('body.weight', port=main), 656 | ]) 657 | 658 | class blood: 659 | main = 'blood' 660 | inc = main + '_in' 661 | 662 | _node = raw_table([ 663 | [td(' ', border=0, port=inc)], # just dummy for consistencty 664 | _m('body.blood', port=main), 665 | ]) 666 | 667 | class bluemaestro: 668 | main = 'bluemaestro' 669 | inc = main + '_in' 670 | 671 | _node = raw_table([ 672 | [td(' ', border=0, port=inc)], 673 | _m('bluemaestro', port=main), 674 | ]) 675 | 676 | class github: 677 | main = 'github' 678 | api = main + '_api' 679 | archive = main + '_archive' 680 | 681 | _node = raw_table([ 682 | [ 683 | td(' ', border=0, port=api ), 684 | td(' ', border=0, port=archive), 685 | ], 686 | _m('github', colspan=2, port=main), 687 | ]) 688 | 689 | class twitter: 690 | main = 'twitter' 691 | 692 | api = 'twitter_api' 693 | archive = 'archive' 694 | 695 | _node = raw_table([ 696 | [ 697 | td(' ', border=0, port=api ), 698 | td(' ', border=0, port=archive), 699 | ], 700 | _m('twitter', colspan=2, port=main), 701 | ]) 702 | 703 | class discord: 704 | main = 'discord' 705 | inc = main + '_in' 706 | 707 | _node = raw_table([ 708 | [td(' ' , border=0, port=inc )], 709 | _m('discord<sup>sb</sup>', port=main, href='https://github.com/seanbreckenridge/HPI/blob/master/my/discord.py'), 710 | ]) 711 | 712 | class pinboard: 713 | main = 'pinboard' 714 | inc = main + '_in' 715 | 716 | # hmm ok, so with inc it's better for correctly doing incoming eldges.. 717 | _node = raw_table([ 718 | [td(' ' , border=0, port=inc )], 719 | _m('pinboard', port=main), 720 | ]) 721 | 722 | class reddit: 723 | main = 'reddit' 724 | 725 | api = main + '_api' 726 | pushshift = main + '_pushshift' 727 | 728 | _node = raw_table([ 729 | [ 730 | td(' ', border=0, port=api ), 731 | td(' ', border=0, port=pushshift), 732 | ], 733 | _m('reddit', port=main, colspan=2), 734 | ]) 735 | 736 | class pocket: 737 | main = 'pocket' 738 | inc = main + '_in' 739 | 740 | _node = raw_table([ 741 | [td(' ', border=0, port=inc)], 742 | _m('pocket', port=main), 743 | ]) 744 | 745 | 746 | class vk: 747 | main = 'vk' 748 | inc = main + '_in' 749 | 750 | _node = raw_table([ 751 | [td(' ', border=0, port=inc)], 752 | _m('vk', port=main), 753 | ]) 754 | 755 | class messenger: 756 | main = 'messenger' 757 | inc = main + '_in' 758 | _node = raw_table([ 759 | [td(' ', border=0, port=inc)], 760 | _m('messenger', port=main), 761 | ]) 762 | 763 | class instapaper: 764 | main = 'instapaper' 765 | inc = main + '_in' 766 | 767 | _node = raw_table([ 768 | [td(' ', border=0, port=inc)], 769 | _m('instapaper', port=main), 770 | ]) 771 | 772 | class kobo: 773 | main = 'kobo' 774 | inc = main + '_in' 775 | 776 | _node = raw_table([ 777 | [td(' ', border=0, port=inc)], 778 | _m('kobo', port=main), 779 | ]) 780 | 781 | class hackernews: 782 | main = 'hackernews' 783 | inc = main + '_in' 784 | 785 | _node = raw_table([ 786 | [td(' ', border=0, port=inc)], 787 | # TODO hmm currently named 'materialistic'? need to emphasize 788 | _m('hackernews', port=main), 789 | ]) 790 | 791 | _log_table = raw_table([ 792 | [ 793 | _m('location.google', port='loc_google', href='https://github.com/karlicoss/HPI/blob/master/my/location/google.py')[0], 794 | _m('gpslogger<sup>sb</sup>', port='gpslogger' , href='https://github.com/seanbreckenridge/HPI/blob/master/my/location/gpslogger.py')[0], 795 | ], 796 | _m("location&timezones\nfor other modules", colspan=2, border=0), 797 | ]) 798 | 799 | node = table([ 800 | # todo hmm, height/width a bit odd but kinda works.. 801 | [ 802 | '<td colspan="12" border="0"></td>', # todo would be nice to calc automatically 803 | '<td port="loctz">' + _log_table + '</td>', 804 | '<td colspan="5" border="0"></td>' 805 | ], 806 | [ 807 | '<td border="0">' + messenger ._node + '</td>', 808 | '<td border="0">' + vk ._node + '</td>', 809 | '<td border="0">' + twitter ._node + '</td>', 810 | '<td border="0">' + discord ._node + '</td>', 811 | '<td border="0">' + pinboard ._node + '</td>', # todo swap pinboard (it's only for promnesia?) 812 | '<td border="0">' + github ._node + '</td>', 813 | '<td border="0">' + pocket ._node + '</td>', 814 | '<td border="0">' + reddit ._node + '</td>', 815 | '<td border="0">' + instapaper ._node + '</td>', 816 | '<td border="0">' + hackernews ._node + '</td>', 817 | '<td border="0">' + kobo ._node + '</td>', 818 | '<td border="0">' + other ._node + '</td>', 819 | td_url('<font point-size="32">github/HPI</font>', port='main', href='https://github.com/karlicoss/HPI'), 820 | '<td border="0">' + bluemaestro._node + '</td>', 821 | '<td border="0">' + weight ._node + '</td>', 822 | '<td border="0">' + blood ._node + '</td>', 823 | '<td border="0">' + sleep ._node + '</td>', 824 | '<td border="0">' + exercise ._node + '</td>', 825 | ], 826 | ]) 827 | 828 | @staticmethod 829 | def p(x: str) -> str: 830 | return 'hpi_node:' + x 831 | 832 | main = 'main' 833 | # meh, just for node referencing... 834 | hpi_node = hpi.node 835 | 836 | # TODO could draw modules separately? 837 | # might be tricky to do vertical text orientation? 838 | 839 | # todo use different style 840 | cachew = td_url( 841 | 'cachew\npersistent cache/serialization', 842 | href='https://github.com/karlicoss/cachew', 843 | ) 844 | 845 | 846 | hpi_usecases = table([ 847 | [td("Usecases", border='0')], 848 | [blog_hb_kcals], 849 | [blog_promnesia_roam], 850 | ], style=dashed) 851 | 852 | hpi_tech = table([ 853 | [td('Libraries/patterns', border='0')], 854 | [cachew], 855 | [blog_configs_suck], 856 | [blog_mypy_err], 857 | ], style=dashed) 858 | 859 | # todo bring back 860 | def hpimodule_url(module: str) -> Optional[str]: 861 | if module in { 862 | # my.ex, 863 | # my.sleep, 864 | }: 865 | # private atm 866 | return None 867 | mp = module.replace('_', '/') 868 | pp = 'karlicoss/hpi/blob/master/' + mp 869 | addpy = module not in {} # meh, hacky 870 | if addpy: 871 | pp += '.py' 872 | return gh(pp) 873 | 874 | 875 | def hpi_module(*, module: str, lid: int): 876 | murl = hpimodule_url(module) 877 | aux = node(module, shape=point) 878 | yield aux 879 | label = module.replace('_', '.') 880 | # ok, multiple dotted -- impossible to see. 881 | # dashed a bit better but still not great.. 882 | yield edge(hpi.p(hpi.main), aux, label=label, style=dotted, **({} if murl is None else url(murl))) 883 | 884 | def hpi_out_edges(): 885 | def tedge(*args, **kwargs): 886 | kwargs = { 887 | 'style' : 'tapered', 888 | 'penwidth': '6', 889 | 'arrowhead': 'none', 890 | **kwargs 891 | } 892 | return edge(*args, **kwargs) 893 | 894 | # my.hyp, # todo add back.. 895 | # my.tg, 896 | # todo not so sure about color here? and styles? 897 | for m in { 898 | hpi.pinboard , 899 | hpi.reddit , 900 | hpi.twitter , 901 | hpi.instapaper, 902 | hpi.hackernews, 903 | hpi.pocket , 904 | hpi.github , 905 | hpi.discord , 906 | }: 907 | name = m.__name__ 908 | 909 | mwip = {} if m is not hpi.discord else dict(style=dashed) 910 | 911 | yield tedge(hpi.p(m.main), promnesia, color=getattr(C, name), **mwip) 912 | yield tedge(hpi.p(m.main), orger , color=getattr(C, name), **mwip) 913 | 914 | yield tedge(hpi.p(hpi.messenger .main), promnesia, color=C.messenger) 915 | yield tedge(hpi.p(hpi.vk .main), promnesia, color=C.vk ) 916 | 917 | yield tedge(hpi.p(hpi.kobo .main), orger , color=C.kobo ) 918 | 919 | yield tedge(hpi.p(hpi.bluemaestro.main), dashboard) 920 | yield tedge(hpi.p(hpi.blood .main), dashboard) 921 | yield tedge(hpi.p(hpi.weight .main), dashboard) 922 | yield tedge(hpi.p(hpi.exercise .main), dashboard) 923 | yield tedge(hpi.p(hpi.sleep .main), dashboard) 924 | 925 | 926 | 927 | # hmm ok so this is useful to display 'DAL' in a more controlled way... not sure how important is it tbh 928 | # NOTE: need to display 'DAL' in the middle... (hacked via postprocessing python script) 929 | def _mi(from_, **kwargs): 930 | fname = from_ if isinstance(from_, str) else from_.name 931 | pcol = kwargs.get('fillcolor') 932 | # TODO hacky.. 933 | auxcol = {} if pcol is None else dict(color=pcol) 934 | 935 | node_shape = point 936 | label = kwargs.get('label') 937 | if label != 'DAL': # TODO how to link to 'data access layer'?? 938 | # hacky... 939 | # TODO maybe do the opposite? if dal, then arrowhead 940 | if 'arrowhead' not in kwargs: 941 | kwargs['arrowhead'] = 'none' # todo get rid of this? 942 | auxcol.update(dict( 943 | style=point, 944 | # meh, but works... 945 | fixedsize='true', 946 | width='0.02', 947 | height='0.02', 948 | )) 949 | else: 950 | kwargs['class'] = 'dal_edge' 951 | if 'color' not in auxcol: 952 | col = getattr(C, fname[3:], None) 953 | if col is not None: 954 | auxcol['color'] = col 955 | aux = node('hpi_in_' + fname, shape=point, **auxcol) # TODO ugh. invis doesn't help here; it still takes space.. 956 | yield aux 957 | # TODO check first..arrowhead='none', 958 | kwargs = { 959 | 'penwidth': '3', 960 | 'arrowhead': 'none', 961 | **kwargs, 962 | } 963 | yield edge(from_, aux, **kwargs) 964 | # maybe incoming things all correspond to modules anyway?: 965 | 966 | 967 | def hpi_incoming_edges(): 968 | return chain.from_iterable([ 969 | _mi(fs.messenger , label='DAL' , color=C.messenger , **url(gh('karlicoss/fbmessengerexport'))), 970 | 971 | _mi(fs.reddit , label='DAL' , color=C.reddit , **url(gh('karlicoss/rexport'))), 972 | # todo hmm html labels arent' friendly to labels along the lines.. 973 | # tooltip? 974 | _mi(fs.pushshift , label='DAL' , **E.reddit , **url(gh('seanbreckenridge/pushshift_comment_export'))), 975 | 976 | _mi(fs.github , label='DAL' , color=C.github , **url(gh('karlicoss/github'))), 977 | _mi(fs.github_archive, color=C.github ), 978 | 979 | _mi(fs.pinboard , label='DAL' , color=C.pinboard , **url(gh('karlicoss/pinbexport'))), 980 | _mi(fs.pocket , label='DAL' , color=C.pocket , **url(gh('karlicoss/pockexport'))), 981 | _mi(fs.twitter , color=C.twitter ), 982 | 983 | _mi(fs.garmin , label='🚧WIP🚧', color=C.garmin , style=dotted, fontcolor='magenta'), 984 | _mi(fs.endomondo , label='DAL' , color=C.endomondo , **url(gh('karlicoss/endoexport')), id='dal'), # eh. id here is kinda arbitrary... 985 | _mi(fs.instapaper , label='DAL' , color=C.instapaper , **url(gh('karlicoss/instapexport'))), 986 | _mi(fs.kobo , label='DAL' , color=C.kobo , **url(gh('karlicoss/kobuddy'))), 987 | _mi(fs.remarkable , label='🚧WIP🚧', color=C.remarkable , style=dotted, fontcolor='magenta'), # todo maybe for wip ones plot a dot here or something? 988 | _mi(fs.bluemaestro , color=C.bluemaestro), 989 | _mi(fs.materialistic , color=C.hackernews ), 990 | _mi(fs.runnerup , color=C.runnerup ), 991 | 992 | _mi(fs.takeouts , color=C.google ), 993 | _mi(fs.twitter_archive , color=C.twitter ), 994 | _mi(fs.discord_archive, label='DAL' , color=C.discord , **url('https://github.com/seanbreckenridge/discord_data')), 995 | _mi(fs.jawbone , color=C.jawbone ), 996 | _mi(fs.emfit , label='DAL' , **url(gh('karlicoss/emfitexport'))), 997 | _mi(fs.vk , color=C.vk ), 998 | 999 | _mi(fs.weight , **E.weight), 1000 | _mi(fs.blood , **E.blood ), 1001 | _mi(fs.sleep ), 1002 | _mi(fs.exercise ), 1003 | _mi(fs.gpslogger , color=C.gpslogger ), 1004 | # TODO blog: note how this edge is still active despite the fact that jbexport isn't working anymore (same with endomondo) 1005 | ]) 1006 | 1007 | # TODO would be nice to add color; in that case node could be 'contaigious' and propagate color 1008 | 1009 | def hpicl(): 1010 | def iedge(name: str, *args, **kwargs): 1011 | kwargs = { 1012 | 'style' : tapered, 1013 | 'penwidth': '3', 1014 | **kwargs, 1015 | } 1016 | if 'color' not in kwargs: 1017 | service = name[3:] 1018 | col = getattr(C, service, None) 1019 | if col is not None: 1020 | kwargs['color'] = col 1021 | 1022 | return edge('hpi_in_' + name, *args, **kwargs) 1023 | return cluster( 1024 | cluster( 1025 | hpi_usecases, 1026 | hpi.node, 1027 | hpi_tech, 1028 | INVIS, 1029 | name='hpi_core', 1030 | ), 1031 | 1032 | *hpi_incoming_edges(), 1033 | 1034 | iedge('fs_messenger' , hpi.p(hpi.messenger.inc)), 1035 | 1036 | iedge('fs_gpslogger' , hpi.p('gpslogger')), 1037 | iedge('fs_takeouts' , hpi.p('loc_google')), 1038 | 1039 | iedge('fs_materialistic', hpi.p(hpi.hackernews.inc)), 1040 | 1041 | iedge('fs_kobo' , hpi.p(hpi.kobo .inc)), 1042 | 1043 | iedge('fs_vk' , hpi.p(hpi.vk .inc)), 1044 | 1045 | iedge('fs_reddit' , hpi.p(hpi.reddit .api)), 1046 | iedge('fs_pushshift' , hpi.p(hpi.reddit .pushshift), **E.reddit), 1047 | 1048 | iedge('fs_pinboard' , hpi.p(hpi.pinboard .inc)), # TODO maybe add a fake node for it? 1049 | 1050 | iedge('fs_github' , hpi.p(hpi.github .api)), 1051 | iedge('fs_github_archive', hpi.p(hpi.github .archive), **E.github), 1052 | 1053 | iedge('fs_pocket' , hpi.p(hpi.pocket .inc)), 1054 | iedge('fs_instapaper' , hpi.p(hpi.instapaper.inc)), 1055 | 1056 | iedge('fs_discord_archive', hpi.p(hpi.discord.inc), **E.discord), 1057 | 1058 | iedge('fs_twitter_archive', hpi.p(hpi.twitter.archive)), 1059 | iedge('fs_twitter' , hpi.p(hpi.twitter.api )), 1060 | 1061 | iedge('fs_bluemaestro', hpi.p(hpi.bluemaestro.inc)), 1062 | iedge('fs_weight' , hpi.p(hpi.weight .inc)), 1063 | iedge('fs_blood' , hpi.p(hpi.blood .inc)), 1064 | 1065 | iedge('fs_garmin' , hpi.p(hpi.exercise.garmin ), style=dotted), 1066 | iedge('fs_endomondo' , hpi.p(hpi.exercise.endomondo)), 1067 | iedge('fs_runnerup' , hpi.p(hpi.exercise.runnerup )), 1068 | iedge('fs_exercise' , hpi.p(hpi.exercise.manual )), 1069 | 1070 | iedge('fs_jawbone' , hpi.p(hpi.sleep .jawbone )), 1071 | iedge('fs_emfit' , hpi.p(hpi.sleep .emfit )), 1072 | iedge('fs_sleep' , hpi.p(hpi.sleep .manual )), 1073 | iedge('fs_garmin' , hpi.p(hpi.sleep .garmin ), style=dotted), 1074 | 1075 | url('#hpi'), 1076 | label='Human Programming Interface ¶', 1077 | id='hpi', 1078 | 1079 | 1080 | style=dotted, 1081 | name='hpicl', 1082 | # todo ugh, don't think height here controls anything? 1083 | ) 1084 | 1085 | def pipelines(): 1086 | items = [ 1087 | exports, 1088 | 1089 | # ... 1090 | filesystem, 1091 | 1092 | # TODO maybe, patch stroke in python? 1093 | edge(tw_website, exp.twint, E.twitter, 1094 | label='fragile', **url('https://github.com/twintproject/twint/issues?q=is%3Aissue+is%3Aopen+sort%3Acomments-desc'), 1095 | color=red, style=dashed, 1096 | ), 1097 | *edges( exp.twint , fs.twitter , E.twitter), 1098 | *edges(tw_archive, exp.tw_manual, fs.twitter_archive, E.twitter), 1099 | 1100 | *edges(discord_archive, exp.discord_manual, fs.discord_archive, E.discord), 1101 | 1102 | *edges(garmin_website, exp.garmindb , fs.garmin , E.garmin), 1103 | 1104 | *edges(endomondo_api , exp.endoexport , E.endomondo, style=dotted, color=red, label='dead'), 1105 | *edges( exp.endoexport , fs.endomondo , E.endomondo ), # todo use colors directly?? 1106 | *edges(tg_api , exp.telegram_backup, fs.telegram , E.telegram), 1107 | 1108 | *edges(messenger_api, exp.messenger , E.messenger, 1109 | label='fragile', **url('https://github.com/fbchat-dev/fbchat/issues'), 1110 | color=red, style=dashed, 1111 | ), 1112 | *edges( exp.messenger, fs.messenger , E.messenger), 1113 | 1114 | # todo ugh. unclear how to 'autostyle' it 1115 | *edges(reddit_api , exp.rexport , fs.reddit , E.reddit ), 1116 | *edges(pushshift , exp.pushshift , fs.pushshift , E.reddit ), 1117 | *edges(pinboard_api , exp.pinbexport , fs.pinboard , E.pinboard), 1118 | *edges(github_archive, exp.github_manual , fs.github_archive, E.github ), 1119 | *edges(github_api , exp.ghexport , fs.github , E.github ), 1120 | *edges(pocket_api , exp.pockexport , fs.pocket , E.pocket ), 1121 | *edges(kobo_sqlite , exp.kobuddy , fs.kobo , E.kobo ), 1122 | *edges(remarkable_ssh, exp.remarkable_sync, fs.remarkable , E.remarkable), 1123 | 1124 | edge(jawbone_api, exp.jbexport , E.jawbone, style=dotted, color=red, label='dead'), 1125 | edge(exp.jbexport , fs.jawbone , E.jawbone), 1126 | 1127 | *edges(google_takeout , exp.takeout_manual, fs.takeouts , E.takeouts ), 1128 | *edges(emfit_cloud_api, exp.emfitexport , fs.emfit , E.emfit ), 1129 | *edges(instapaper_api , exp.instapexport , fs.instapaper, E.instapaper), 1130 | 1131 | edge(vk_api, exp.vkexport, label='API closed?', style=dotted, color=red, **url('https://github.com/Totktonada/vk_messages_backup/pull/8#issuecomment-494582792')), 1132 | edge( exp.vkexport, fs.vk, E.vk), 1133 | 1134 | *edges(sleep_subj , exp.inp_sleep , fs.sleep ), 1135 | *edges(exercise , exp.inp_exercise , fs.exercise), 1136 | *edges(blood_tests , exp.inp_blood , fs.blood , **E.blood), 1137 | *edges(scales , exp.inp_weight , fs.weight , **E.weight), 1138 | 1139 | 1140 | # TODO hmm, margin look interesting.. 1141 | hpicl(), 1142 | 1143 | 'color=red\nstyle=dashed' if debug else 'style=invisible', 1144 | ] 1145 | return items 1146 | 1147 | # TODO remove nodes when there is no data access layer?? 1148 | 1149 | dashboard = node( 1150 | **url('https://github.com/karlicoss/dashboard'), 1151 | label='Dashboard\n(🚧wip🚧)', 1152 | shape=star, 1153 | ) 1154 | 1155 | 1156 | jupyter = node( 1157 | label='Jupyter\nIPython', 1158 | **url('https://github.com/karlicoss/HPI#ad-hoc-and-interactive'), 1159 | **UI, 1160 | ) 1161 | hpi_http = node( 1162 | label="HTTP API\n(🚧wip🚧)", 1163 | **url('https://github.com/karlicoss/HPI/issues/16'), 1164 | **UI, 1165 | ) 1166 | hpi_sqlite = node( 1167 | label='Sqlite\n(via cachew)', 1168 | **url('https://github.com/karlicoss/cachew#readme'), 1169 | **UI, 1170 | ) 1171 | hpi_influxdb = node( 1172 | label='Influxdb', 1173 | **url('https://github.com/influxdata/influxdb#influxdb-'), 1174 | **UI, 1175 | ) 1176 | hpi_datasette = node( 1177 | label='Datasette', 1178 | **url('https://simonwillison.net/2020/Nov/14/personal-data-warehouses'), 1179 | **UI, 1180 | ) 1181 | 1182 | hpi_memacs = node( 1183 | label='Memacs', 1184 | **url('https://github.com/novoid/Memacs'), 1185 | **UI, 1186 | ) 1187 | 1188 | hpi_grafana = node( 1189 | label='Grafana', 1190 | **url('https://github.com/grafana/grafana#readme'), 1191 | **UI, 1192 | ) 1193 | 1194 | hpi_metabase = node( 1195 | label='Metabase', 1196 | **url('https://github.com/metabase/metabase#features'), 1197 | **UI, 1198 | ) 1199 | 1200 | hpi_spreadsheet = node( 1201 | label='Spreadsheet-like\ninterface?', 1202 | **url('https://github.com/karlicoss/HPI/issues/104'), 1203 | **UI, 1204 | ) 1205 | 1206 | hpi_openhumans = node( 1207 | label='openhumans.org', 1208 | **url('https://www.openhumans.org'), 1209 | **UI, 1210 | ) 1211 | 1212 | hpi_ffi = table( 1213 | [ 1214 | [td('Other\nprogramming\nlanguages\n(FFI)', border=0)], 1215 | [ 1216 | td_url('Apache Arrow', href='https://arrow.apache.org/'), 1217 | ], 1218 | ], 1219 | bgcolor=COLOR_UI, 1220 | ) 1221 | 1222 | hpi_memri = node( 1223 | label='Memri', 1224 | **url('https://memri.io'), 1225 | **UI, 1226 | ) 1227 | 1228 | hpi_solid = node( 1229 | label='Solid project', 1230 | **url('https://solidproject.org'), 1231 | **UI, 1232 | ) 1233 | 1234 | 1235 | jupyter2 = node( # meh, but not sure if worth reusing 1236 | label='Jupyter\nIPython', 1237 | **UI, 1238 | ) 1239 | 1240 | 1241 | wip = dict(style=dashed , penwidth='2', color='grey', label='🚧 WIP 🚧') # todo annoying, too close to graph 1242 | 1243 | def post(): 1244 | tbro = browser('timeline' , label='Browser\n(HTML)') 1245 | 1246 | taper = dict(style=tapered, penwidth='8', arrowhead='none') 1247 | 1248 | items = [ 1249 | # edge('exp_takeouts', promnesia, label='Browsing history'), 1250 | # edge('exp_telegram', promnesia, label='Telegram'), 1251 | 1252 | hpi_memacs, 1253 | jupyter, 1254 | hpi_http, 1255 | hpi_spreadsheet, 1256 | hpi_influxdb, 1257 | hpi_ffi, 1258 | hpi_sqlite, 1259 | hpi_metabase, 1260 | hpi_datasette, 1261 | hpi_solid, 1262 | hpi_memri, 1263 | hpi_grafana, 1264 | 1265 | edge(hpi.p(hpi.main), timeline , {**taper, 'penwidth': '20'}, color='#a0e98999'), 1266 | edge(hpi.p(hpi.main), jupyter , taper, color='pink'), 1267 | edge(hpi.p(hpi.main), hpi_sqlite , taper, color='grey'), 1268 | edge(hpi.p(hpi.main), hpi_influxdb , wip, **url('https://github.com/karlicoss/HPI/blob/20585a313023f9cccd5f05abf9f7e3663e9264f3/my/core/influxdb.py')), 1269 | edge(hpi_sqlite , hpi_datasette , wip, **url('https://news.ycombinator.com/item?id=25090643')), 1270 | edge(hpi_sqlite , hpi_grafana , **url('https://grafana.com/grafana/plugins/frser-sqlite-datasource'), label='plugin'), 1271 | edge(hpi_influxdb , hpi_grafana , label='see demo', **url('https://twitter.com/karlicoss/status/1361100437332590593')), 1272 | edge(hpi.p(hpi.main), hpi_http , taper, color='brown'), 1273 | edge(hpi.p(hpi.main), hpi_memacs , wip), 1274 | edge(hpi_sqlite , hpi_metabase , wip), 1275 | edge(hpi.p(hpi.main), hpi_spreadsheet, wip), 1276 | edge(hpi.p(hpi.main), hpi_ffi , wip), 1277 | edge(hpi.p(hpi.main), hpi_memri , wip), 1278 | edge(hpi_http , hpi_solid , wip), 1279 | 1280 | # TODO ugh. why they stick out of the cluster?? 1281 | edge(hpi_ffi , hpi_http , **NOCONSTRAINT, style=dashed, arrowhead='none'), 1282 | edge(hpi_memri , hpi_http , **NOCONSTRAINT, style=dashed, arrowhead='none'), 1283 | edge(hpi_spreadsheet, hpi_metabase, style=dashed, arrowhead='none'), 1284 | 1285 | # mm, looks a bit ugly?.. 1286 | edge(hpi_memri , hpi_ffi , **NOCONSTRAINT, style=dashed, arrowhead='none'), 1287 | ] 1288 | return items 1289 | 1290 | 1291 | # TODO actually make node attributes accessible in runtime? so I could do smth.color? 1292 | app_fs_bluemaestro = node(label='sqlite' , **CLOUD_SYNC, color=C.bluemaestro) 1293 | app_fs_materialistic = node(label='sqlite' , **CLOUD_SYNC, color=C.hackernews ) 1294 | app_fs_runnerup = node(label='tcx\nworkouts', **CLOUD_SYNC, color=C.runnerup ) 1295 | app_fs_gpslogger = node(label='gpx\ntracks' , **CLOUD_SYNC, color=C.gpslogger ) 1296 | # phone_syncthing = node(label='Syncthing', **CLOUD) # TODO fill with lightblue? 1297 | 1298 | phone_fss = cluster( 1299 | app_fs_bluemaestro, 1300 | app_fs_materialistic, 1301 | app_fs_runnerup, 1302 | app_fs_gpslogger, 1303 | # phone_syncthing, 1304 | label='Filesystem', 1305 | ) 1306 | 1307 | gps = node(label='GPS', shape='plain', style='solid') 1308 | app_bm = node( 1309 | label='Bluemaestro\napp', 1310 | ) 1311 | app_runnerup = node( 1312 | label='Runnerup\napp', 1313 | **url('https://github.com/jonasoreland/runnerup#runnerup'), 1314 | ) 1315 | app_gpslogger = node( 1316 | label='Gpslogger\napp', 1317 | **url('https://github.com/mendhak/gpslogger'), 1318 | ) 1319 | app_garmin = node( 1320 | label='Garmin\napp', 1321 | ) 1322 | app_endomondo = node(**INVIS, shape=point) 1323 | app_jawbone = node(**INVIS, shape=point) 1324 | 1325 | app_materialistic = node( 1326 | label='Materialistic\n(Hackernews app)', 1327 | **url('https://github.com/hidroh/materialistic#materialistic-for-hacker-news'), 1328 | # todo how to mark that it stopped updating? 1329 | ) 1330 | 1331 | phone = cluster( 1332 | gps, 1333 | app_jawbone, 1334 | app_endomondo, 1335 | 1336 | # ugh. to make sure e.g. endomondo app appears on the next level after phone 1337 | 'qqq [shape=point, style=invis]', 1338 | 'qqq -> gps [style=invis]', 1339 | 'qqq -> app_endomondo [style=invis]', 1340 | 'qqq -> app_jawbone [style=invis]', 1341 | 'qqq -> app_garmin [style=invis]', 1342 | 1343 | phone_fss, 1344 | 1345 | app_runnerup, 1346 | app_materialistic, 1347 | app_gpslogger, 1348 | app_bm, 1349 | app_garmin, 1350 | 1351 | # TODO demonstrate that root is necessary?? 1352 | edge(app_bm , app_fs_bluemaestro , style=dashed, **noarrow), 1353 | edge(app_materialistic, app_fs_materialistic, style=dashed, **noarrow), 1354 | edge(app_runnerup , app_fs_runnerup , style=dashed, **noarrow), 1355 | edge(app_gpslogger , app_fs_gpslogger , style=dashed, **noarrow), 1356 | edge(gps , app_gpslogger , style=dotted, **NOCONSTRAINT), 1357 | edge(gps , app_runnerup , style=dotted, **NOCONSTRAINT), 1358 | edge(gps , app_garmin , style=dotted, **NOCONSTRAINT), 1359 | DEVICE, 1360 | label='Android\nphone', 1361 | ) 1362 | 1363 | 1364 | google_loc = node(name='Google Location') 1365 | google_takeout = node( 1366 | name='Takeout', 1367 | **url('https://takeout.google.com'), 1368 | ) 1369 | 1370 | google = table( 1371 | [ 1372 | [td('Google', border='0', colspan='2')], 1373 | [td('Browser\nhistory'), td('Location')], 1374 | [td_url('Takeout', port='takeout', colspan='2', href='https://en.wikipedia.org/wiki/Google_Takeout', **WARNING, tooltip='Unclear retention rules https://beepb00p.xyz/takeout-data-gone.html')], 1375 | #TODO no apis? 1376 | ], 1377 | **CLOUD, 1378 | color=C.google, 1379 | ) 1380 | google_takeout = 'google:takeout' 1381 | 1382 | # TODO map manual steps without intermediate nodes? 1383 | 1384 | 1385 | telegram = table( 1386 | [ 1387 | [td('Telegram', border=0)], 1388 | [td('API', port='api')], 1389 | ], 1390 | color=C.telegram, 1391 | **CLOUD, 1392 | ) 1393 | tg_api = 'telegram:api' # todo would be nice to make static... 1394 | 1395 | messenger = table( 1396 | [ 1397 | [td('FB Messenger', border=0)], 1398 | [td('API\n(private)', port='api', tooltip='API is private', **WARNING)], 1399 | ], 1400 | color=C.messenger, 1401 | **CLOUD, 1402 | ) 1403 | messenger_api = 'messenger:api' 1404 | 1405 | twitter = table( 1406 | [ 1407 | [td('Twitter', border=0, colspan=2)], 1408 | [ 1409 | # todo annoying, colors the text instead of border... 1410 | td_url('API' , port='api' , color=red, style=dashed, 1411 | href='https://redfern.me/banned-from-twitter', 1412 | tooltip='Twitter is getting more and more hostile to hobbyist project and 3rd party clients' 1413 | ), 1414 | td_url('website\n(scraping)', port='website', href='', style=dashed, **WARNING, tooltip='scraping Twitter is extremely fragile'), 1415 | td_url('archive', port='archive', href='https://help.twitter.com/en/managing-your-account/how-to-download-your-twitter-archive'), 1416 | ], 1417 | ], 1418 | color=C.twitter, 1419 | **CLOUD, 1420 | # todo color needed only for outer border? 1421 | ) 1422 | tw_api = 'twitter:api' 1423 | tw_website = 'twitter:website' 1424 | tw_archive = 'twitter:archive' 1425 | 1426 | 1427 | discord = table( 1428 | [ 1429 | [td('Discord', border=0, colspan=2)], 1430 | [ 1431 | td_url('API', port='api', color=red, style=dashed, 1432 | href='https://github.com/Tyrrrz/DiscordChatExporter/issues/171', 1433 | tooltip="Hostile against alternative clients, e.g. can't retrieve DMs with api", 1434 | ), 1435 | td_url('archive', port='archive', href='https://support.discord.com/hc/en-us/articles/360004027692-Requesting-a-Copy-of-your-Data'), 1436 | ], 1437 | ], 1438 | color=C.discord, 1439 | **CLOUD, 1440 | ) 1441 | discord_api = 'discord:api' 1442 | discord_archive = 'discord:archive' 1443 | 1444 | reddit = table( 1445 | [ 1446 | [td('Reddit', border='0')], 1447 | [ 1448 | td_url('API', port='api', href="https://github.com/karlicoss/rexport#api-limitations", **WARNING, tooltip='only 1000 latest items via API'), 1449 | td_url('GDPR\nexport', port='gdpr', href='https://www.redditinc.com/policies/privacy-policy#data-subject-and-consumer-information-requests', **WARNING, tooltip='Only on email request'), 1450 | td_url('pushshift', port='pushshift', href='https://pushshift.io'), 1451 | ] 1452 | ], 1453 | color=C.reddit, 1454 | **CLOUD, 1455 | ) 1456 | reddit_api = 'reddit:api' 1457 | pushshift = 'reddit:pushshift' 1458 | 1459 | pinboard = table( 1460 | [ 1461 | [td('Pinboard', border=0)], 1462 | [td('API', port='api')] 1463 | ], 1464 | color=C.pinboard, 1465 | **CLOUD, 1466 | ) 1467 | pinboard_api = 'pinboard:api' 1468 | 1469 | github = table( 1470 | [ 1471 | [td('Github', border=0)], 1472 | [ 1473 | td_url('API' , port='api' , href='https://github.com/karlicoss/ghexport#api-limitations', **WARNING, tooltip='only 300 latest events via API'), 1474 | td_url('archive', port='archive', href='https://github.com/settings/admin'), 1475 | ], 1476 | ], 1477 | color=C.github, 1478 | **CLOUD, 1479 | ) 1480 | github_api = 'github:api' 1481 | github_archive = 'github:archive' 1482 | 1483 | pocket = table( 1484 | [ 1485 | [td('Pocket', border=0)], 1486 | [td('API', port='api')], # todo mention the hack? 1487 | ], 1488 | color=C.pocket, 1489 | **CLOUD, 1490 | ) 1491 | pocket_api = 'pocket:api' 1492 | 1493 | instapaper = table( 1494 | [ 1495 | [td_url('Instapaper', href='https://www.instapaper.com', border=0)], 1496 | [td('API', port='api')], 1497 | ], 1498 | color=C.instapaper, 1499 | **CLOUD, 1500 | ) 1501 | instapaper_api = 'instapaper:api' 1502 | 1503 | endomondo = table( 1504 | [ 1505 | [td_url('Endomondo\n(dead)', border=0, 1506 | href='https://web.archive.org/web/20210127215914/https://support.endomondo.com/hc/en-us/articles/360016251359-Endomondo-Is-Retired', 1507 | tooltip='discontinued in December 2020', 1508 | )], 1509 | [td('API', port='api', color=red)], 1510 | ], 1511 | color=C.endomondo, 1512 | **CLOUD, 1513 | **DEAD, 1514 | ) 1515 | endomondo_api = 'endomondo:api' 1516 | 1517 | garmin = table( 1518 | [ 1519 | [td_url('Garmin Connect', border=0, href='https://connect.garmin.com')], 1520 | [td_url('website\n(scraping)', port='website', href='', style=dashed, **WARNING, tooltip='Scraping is inherently fragile')], 1521 | ], 1522 | color=C.garmin, 1523 | **CLOUD, 1524 | ) 1525 | garmin_website = 'garmin:website' 1526 | 1527 | 1528 | vk = table( 1529 | [ 1530 | [td_url('VK.com', href='https://vk.com', border=0)], 1531 | [td_url('API', port='api', color=red, style=dashed, href='https://vk.com/wall-1_390510', tooltip='Messages API locked down')], 1532 | ], 1533 | **CLOUD, 1534 | ) 1535 | vk_api = 'vk:api' 1536 | 1537 | 1538 | emfit_cloud = table( 1539 | [ 1540 | [td('Emfit', border='0')], 1541 | [td('API', port='api')], 1542 | ], 1543 | **CLOUD, 1544 | ) 1545 | emfit_cloud_api = 'emfit_cloud:api' 1546 | 1547 | 1548 | jawbone = table( 1549 | [ 1550 | [td_url('Jawbone\n(dead)', border=0, 1551 | href='https://en.wikipedia.org/wiki/Jawbone_(company)#UP24', 1552 | tooltip='Discontinued in 2017', 1553 | )], 1554 | [td('API', port='api')], 1555 | ], 1556 | color=C.jawbone, 1557 | **CLOUD, 1558 | **DEAD, 1559 | ) 1560 | jawbone_api = 'jawbone:api' 1561 | 1562 | 1563 | kobo = table( 1564 | [ 1565 | [td_url('Kobo reader', href='https://us.kobobooks.com/products/kobo-aura-one-limited-edition', border=0)], 1566 | [td('sqlite', port='sqlite')], 1567 | ], 1568 | bgcolor=COLOR_DEVICE, 1569 | ) 1570 | kobo_sqlite = 'kobo:sqlite' 1571 | 1572 | remarkable = table( 1573 | [ 1574 | [td_url('Remarkable 2 tablet', href='https://remarkable.com/store/remarkable-2', border=0)], 1575 | [td('ssh', port='ssh')], 1576 | ], 1577 | bgcolor=COLOR_DEVICE, 1578 | ) 1579 | remarkable_ssh = 'remarkable:ssh' 1580 | 1581 | 1582 | class E: 1583 | endomondo = {} 1584 | twitter = {} 1585 | telegram = {} 1586 | kobo = {} 1587 | jawbone = {} 1588 | blood = {} 1589 | weight = {} 1590 | reddit = {} 1591 | discord = {} 1592 | instapaper = {} 1593 | pinboard = {} 1594 | pocket = {} 1595 | github = {} 1596 | messenger = {} 1597 | remarkable = {} 1598 | garmin = {} 1599 | takeouts = {} 1600 | vk = {} 1601 | emfit = {} 1602 | remarkable = {} 1603 | 1604 | 1605 | for k, v in vars(E).items(): 1606 | if not k.startswith('_'): 1607 | color = getattr(C, k) 1608 | setattr(E, k, { 1609 | 'color' : color, # arrow outline 1610 | 'fillcolor': color, # arrow tip 1611 | }) 1612 | 1613 | 1614 | # TODO also could show how data gets _into_ the services, i.e. clients? 1615 | 1616 | emfit_wifi = node( 1617 | label='wifi\n(local API)', 1618 | **url('https://gist.github.com/harperreed/9d063322eb84e88bc2d0580885011bdd'), 1619 | ) 1620 | 1621 | emfit = table( 1622 | [ 1623 | [td_url('Emfit QS\nsleep tracker', href='https://www.emfit.com/why-choose-emfit-for-sleep-analysis', border=0, colspan=2)], 1624 | [ 1625 | td('wifi\n(local API)', port='local'), 1626 | td('wifi\n(cloud API)', port='cloud'), 1627 | ] 1628 | ], 1629 | bgcolor=COLOR_DEVICE, 1630 | ) 1631 | 1632 | wahoo = node( 1633 | label='Wahoo Tickr X\n(HR monitor)', 1634 | **DEVICE, 1635 | **url('https://uk.wahoofitness.com/devices/heart-rate-monitors/wahoo-tickr-x-heart-rate-strap'), 1636 | ) 1637 | 1638 | garmin_watch = node( 1639 | label='Garmin watch', 1640 | **DEVICE, 1641 | # todo not sure if need url 1642 | ) 1643 | 1644 | jawbone_band = node( 1645 | label='Jawbone\nsleep tracker', 1646 | **DEVICE, 1647 | ) 1648 | 1649 | bluemaestro = node( 1650 | label='Bluemaestro\n(environment\nsensor)', 1651 | **DEVICE, 1652 | **url('https://bluemaestro.com/products/product-details/bluetooth-environmental-monitor-and-logger'), 1653 | ) 1654 | 1655 | # todo not sure what's the point of clustering? 1656 | devices = cluster( 1657 | wahoo, 1658 | jawbone_band, 1659 | bluemaestro, 1660 | garmin_watch, 1661 | 1662 | # TODO the fuck? coloring these to blue 1663 | edge(wahoo , app_endomondo, label='BT', style=dotted), 1664 | edge(wahoo , app_runnerup , label='BT', style=dotted), 1665 | edge(jawbone_band, app_jawbone , label='BT', style=dotted), 1666 | edge(bluemaestro , app_bm , label='BT', style=dotted), 1667 | edge(garmin_watch, app_garmin , label='BT', style=dotted), 1668 | 1669 | # label='Devices', 1670 | # style=dashed, 1671 | INVIS, 1672 | ) 1673 | 1674 | def meh(x): 1675 | # weird flex with curlys but ok.. puts these nodes in the right place? 1676 | # od they all need a shared cluster? 1677 | return ['{', x, '}'] 1678 | # meh indeed.. 1679 | 1680 | dbro = browser('dashboard', label='Browser\n(HTML)') 1681 | def generate() -> str: 1682 | items = [ 1683 | cluster( 1684 | legend, 1685 | meta, 1686 | edge('legend_ui', sad_infra, INVIS), 1687 | INVIS, 1688 | name='group', 1689 | ), 1690 | 1691 | phone, 1692 | 1693 | 1694 | meh(telegram), 1695 | meh(messenger), 1696 | 1697 | meh(google), 1698 | 1699 | devices, 1700 | emfit, 1701 | 1702 | meh(vk), 1703 | meh(twitter), 1704 | meh(discord), 1705 | meh(pinboard), 1706 | meh(github), 1707 | meh(pocket), 1708 | meh(reddit), 1709 | meh(instapaper), 1710 | meh(kobo), 1711 | meh(remarkable), 1712 | 1713 | meh(scales), 1714 | meh(blood_tests), 1715 | 1716 | # TODO fuck. kind of a mess with arrows 1717 | emfit, 1718 | meh(emfit_cloud), 1719 | meh(jawbone), 1720 | meh(sleep_subj), 1721 | 1722 | meh(garmin), 1723 | meh(endomondo), 1724 | meh(exercise), 1725 | 1726 | # syncthing_cl, # TODO maybe should't be a sluster?? 1727 | 1728 | edge(gps, google), 1729 | edge(app_endomondo, endomondo), 1730 | edge(app_jawbone , jawbone), 1731 | edge('emfit:cloud', emfit_cloud), 1732 | # TODO hmm, syncthing could be an edge 1733 | 1734 | edges(app_garmin, garmin), 1735 | 1736 | # *edges(app_bm, syncthing, exp_bluemaestro), # TODO here, a rooted script is involved 1737 | # TODO not sure about that.. 1738 | 1739 | promnesia_cl, 1740 | orger_cl, 1741 | orger_outputs, 1742 | 1743 | promnesia_browser, 1744 | archivebox, 1745 | edge(promnesia, promnesia_browser), 1746 | edge(promnesia, archivebox, style=dashed), 1747 | 1748 | 'subgraph cluster_pipelines {', 1749 | *pipelines(), 1750 | '}', 1751 | 1752 | 'subgraph cluster_for_hpi {', 1753 | 'style=invisible', 1754 | *post(), 1755 | '}', 1756 | 1757 | 'subgraph cluster_for_dashboard {', 1758 | 'style=invisible', 1759 | dashboard, 1760 | dbro, 1761 | jupyter2, 1762 | hpi_openhumans, 1763 | edge(dashboard, dbro), 1764 | edge(dashboard, jupyter2), 1765 | edge(dashboard, hpi_openhumans , wip), 1766 | '}', 1767 | edge(dashboard, hpi_grafana , wip), 1768 | 1769 | 'subgraph cluster_for_timeline {', 1770 | 'style=invisible', 1771 | timeline, 1772 | '}', 1773 | 1774 | # TODO hmm. instead, add 'ports' and mention that ports are attached to syncthing? 1775 | # *edges(app_bm , exp_bluemaestro , label='Syncthing', **url('https://syncthing.net')), 1776 | # *edges(app_materialistic, exp_materialistic, label='Syncthing', **url('https://synchting.get')), 1777 | *hpi_out_edges(), 1778 | 1779 | ] 1780 | return '\n'.join(map(render, items)) 1781 | 1782 | 1783 | 1784 | def main(): 1785 | Path('diagram.dot').write_text(generate()) 1786 | 1787 | 1788 | if __name__ == '__main__': 1789 | main() 1790 | 1791 | 1792 | # TODO meh. okay, I might need some hackery to properly display edge labels... 1793 | # https://developer.mozilla.org/en-US/docs/Web/SVG/Element/textPath 1794 | 1795 | # TODO dot allows comments? 1796 | 1797 | # TODO maybe, use html table? not sure.. 1798 | # https://renenyffenegger.ch/notes/tools/Graphviz/examples/index 1799 | 1800 | # TODO add hover anchors everywhere 1801 | 1802 | # TODO hmm, use xlabel? it doesn't impact layout, might be useful for edges? 1803 | 1804 | # TODO add emacs as org-mode interface? 1805 | 1806 | # TODO mention that you can follow diamonds to see how the data flows through the system 1807 | --------------------------------------------------------------------------------