├── README.md └── src ├── app └── graph-query-cli.hoon ├── gen └── graph-query.hoon └── lib ├── graph-query.hoon └── pal.hoon /README.md: -------------------------------------------------------------------------------- 1 | # graph-query 2 | 3 | ## installation 4 | 5 | `rsync -av src/ $URBIT_SHIP_DIR/home/` 6 | or your specified desk instead of `home` 7 | or from dojo 8 | `|sync %home ~nortex-ramnyd %graph-query` 9 | 10 | ## generator input arguments 11 | 12 | ``` 13 | resource=resource:g :: group to query from 14 | sq=(unit tape) :: search query word :: TODO make case insensitive 15 | author=(unit @p) :: author of post 16 | before=(unit @da) :: cutoff posts after a timepoint 17 | after=(unit @da) :: cutoff posts before a timepoint 18 | page=@ud :: one page of 420 posts from graph, page 1: queries 420 nodes, page 2: next 420 nodes etc. 19 | ``` 20 | 21 | ## example query with generator 22 | 23 | ``+graph-query [~bitbet-bolbel %urbit-community] ~ `~tinnus-napbus `~2021.4.28 `~2021.4.10 1`` 24 | 25 | ## shoe app usage 26 | 27 | make sure to `|commit %home` (or other specified desk) inside dojo and then `|start %graph-query-cli` 28 | 29 | then press `?` to see query options 30 | -------------------------------------------------------------------------------- /src/app/graph-query-cli.hoon: -------------------------------------------------------------------------------- 1 | /- g=graph-store 2 | /+ shoe, verb, dbug, default-agent, resource, pal 3 | /= query /gen/graph-query 4 | :: 5 | |% 6 | +$ state-0 7 | $: %0 8 | query-input=gen-input 9 | dms-disabled=_| 10 | posts=(list [resource:g (list node:g)]) 11 | current-page=@ud 12 | render-input-state=[resource=tape text=tape author=tape before=tape after=tape] 13 | == 14 | :: 15 | +$ gen-input [=resource:resource search-text=(unit tape) author=(unit @p) before=(unit @da) after=(unit @da) page=_1 ~] 16 | :: 17 | :: 18 | +$ command 19 | $% 20 | [%run-query ~] 21 | [%next-page ~] 22 | [%page page=@ud] 23 | [%show-query-state ~] 24 | [%clear-query-state ~] 25 | [%show-options ~] 26 | [%show-resource ~] 27 | [%select-resource id=@ud] 28 | [%disable-dms ~] 29 | [%search-text text=tape] 30 | [%author author=(unit @p)] 31 | [%after after=@da] 32 | [%before before=@da] 33 | == 34 | :: 35 | +$ card card:shoe 36 | -- 37 | =| state-0 38 | =* state - 39 | :: 40 | %+ verb | 41 | %- agent:dbug 42 | ^- agent:gall 43 | %- (agent:shoe command) 44 | ^- (shoe:shoe command) 45 | =< 46 | |_ =bowl:gall 47 | +* this . 48 | def ~(. (default-agent this %|) bowl) 49 | des ~(. (default:shoe this command) bowl) 50 | qo ~(. +> bowl) 51 | :: 52 | ++ on-init on-init:def 53 | ++ on-save !>(state) 54 | ++ on-load 55 | |= old=vase 56 | ^- (quip card _this) 57 | [~ this] 58 | :: 59 | ++ on-poke on-poke:def 60 | ++ on-watch on-watch:def 61 | ++ on-leave on-leave:def 62 | ++ on-peek on-peek:def 63 | ++ on-agent on-agent:def 64 | ++ on-arvo on-arvo:def 65 | ++ on-fail on-fail:def 66 | :: 67 | ++ command-parser build-parser:qo 68 | :: 69 | ++ tab-list tab-list:des 70 | :: 71 | ++ on-command 72 | |= [sole-id=@ta =command] 73 | ^- (quip card _this) 74 | =^ cards state 75 | ?- -.command 76 | %run-query run-query:qo 77 | :: 78 | %next-page =.(page.query-input.+.state +(current-page) run-query:qo) 79 | :: 80 | %page =.(page.query-input.+.state +.command run-query:qo) 81 | :: 82 | %show-query-state render-query-state:render:qo 83 | :: 84 | %clear-query-state clear-query-state:qo 85 | :: 86 | %show-options render-options:render:qo 87 | :: 88 | %show-resource joined-groups:render:qo 89 | :: 90 | %select-resource (put-resource:build-query-generator-input:qo command) 91 | ::< 92 | %disable-dms disable-dms:qo 93 | :: 94 | %search-text (put-search-text:build-query-generator-input:qo command) 95 | :: 96 | %author (put-author:build-query-generator-input:qo command) 97 | :: 98 | %before (put-before:build-query-generator-input:qo command) 99 | :: 100 | %after (put-after:build-query-generator-input:qo command) 101 | == 102 | [cards this] 103 | :: 104 | ++ can-connect 105 | |= sole-id=@ta 106 | ^- ? 107 | (team:title [our src]:bowl) 108 | :: 109 | ++ on-connect 110 | |= sole-id=@ta 111 | ^- (quip card _this) 112 | :_ this 113 | [%shoe [sole-id]~ start:render:qo]~ 114 | :: 115 | ++ on-disconnect on-disconnect:des 116 | -- 117 | :: 118 | |_ =bowl:gall 119 | :: 120 | ++ build-parser 121 | |= sole-id=@ta 122 | ^+ |~(nail *(like [? command])) 123 | %+ pick 124 | ;~ pose 125 | (cold [%run-query ~] (just 'r')) 126 | (cold [%next-page ~] gar) 127 | (cold [%show-query-state ~] dot) 128 | (cold [%clear-query-state ~] hep) 129 | (cold [%show-options ~] wut) 130 | (cold [%show-resource ~] (just 'g')) 131 | (cold [%disable-dms ~] (just 'd')) 132 | == 133 | :: 134 | ;~ pose 135 | (stag %select-resource dem) 136 | (stag %search-text ;~(pfix fas (star next))) 137 | (stag %author ;~(pfix sig (punt fed:ag))) 138 | (stag %page ;~(pfix (just 'p') dem)) 139 | (stag %before ;~(pfix (just 'b') (cook year when:so))) 140 | (stag %after ;~(pfix (just 'a') (cook year when:so))) 141 | == 142 | :: 143 | ++ run-query 144 | ^- (quip card _state) 145 | =/ posts-from-gen-call 146 | =< + 147 | %- +:query 148 | [[now:bowl eny:bowl byk:bowl] query-input.+.state ~] 149 | :_ 150 | %= state 151 | posts posts-from-gen-call 152 | current-page +(current-page) 153 | == 154 | :~ 155 | :+ %shoe ~ 156 | :^ %table 157 | ~[t+'ship' t+'date' t+'post' t+'index' t+'channel'] 158 | ~[15 22 55 40 30] 159 | ^- (list (list dime)) 160 | %- zing 161 | %+ turn posts-from-gen-call 162 | |= i=[resource:g (list node:g)] 163 | %+ turn +.i 164 | |= i=node:g 165 | =/ text (get-text-content:destructure-node i) 166 | =/ table-text 167 | ?~ text ~ 168 | ?. (gth (lent text) 1) 169 | (crip i.text) 170 | %- crip 171 | ;;(tape (reel `(list tape)`(turn text |=(i=tape (weld i "\0a"))) weld)) 172 | ~[p+(get-author:destructure-node i) da+(get-time-sent:destructure-node i) t+table-text t+(get-index:destructure-node i) t+(get-channel:destructure-node -:^i)] 173 | == 174 | :: 175 | ++ disable-dms 176 | ^- (quip card _state) 177 | :_ state(dms-disabled &) 178 | [%shoe ~ %sole %txt "direct messages disabled\0a"]~ 179 | :: 180 | ++ clear-query-state 181 | ^- (quip card _state) 182 | :_ =. state *state-0 state 183 | [%shoe ~ %sole %txt "query cleared\0a"]~ 184 | :: 185 | ++ build-query-generator-input 186 | |% 187 | ++ put-resource 188 | |= i=[%select-resource @ud] 189 | ^- (quip card _state) 190 | :: if invalid group id input, don't proceed 191 | ?: (gth +.i (lent joined-groups-list)) [[%shoe ~ %sole %txt "invalid group id"]~ state] 192 | =/ current-resource (~(got by joined-groups-map) +.i) 193 | =/ render-current-resource "=group {(scow %p entity.current-resource)}/{(scow %tas name.current-resource)} " 194 | :_ %= state 195 | resource.query-input current-resource 196 | resource.render-input-state render-current-resource 197 | == 198 | :~ 199 | :+ %shoe ~ 200 | :+ %sole %txt 201 | render-current-resource 202 | == 203 | :: 204 | ++ put-search-text 205 | |= i=[%search-text tape] 206 | ^- (quip card _state) 207 | =/ render-current-text "=text {+.i} " 208 | :_ %= state 209 | search-text.query-input `+.i 210 | text.render-input-state render-current-text 211 | == 212 | :~ 213 | :+ %shoe ~ 214 | :+ %sole %txt 215 | render-current-text 216 | == 217 | ++ put-author 218 | |= i=[%author (unit @p)] 219 | ^- (quip card _state) 220 | =/ render-current-author "=author {(scow %p (need +.i))} " 221 | :_ %= state 222 | author.query-input +.i 223 | author.render-input-state render-current-author 224 | == 225 | :~ 226 | :+ %shoe ~ 227 | :+ %sole %txt 228 | render-current-author 229 | == 230 | ++ put-before 231 | |= i=[%before @da] 232 | ^- (quip card _state) 233 | =/ render-current-before "=before {(scow %da +.i)} " 234 | :_ %= state 235 | before.query-input `+.i 236 | before.render-input-state render-current-before 237 | == 238 | :~ 239 | :+ %shoe ~ 240 | :+ %sole %txt 241 | render-current-before 242 | == 243 | ++ put-after 244 | |= i=[%after @da] 245 | ^- (quip card _state) 246 | =/ render-current-after "=after {(scow %da +.i)} " 247 | :_ %= state 248 | after.query-input `+.i 249 | after.render-input-state render-current-after 250 | == 251 | :~ 252 | :+ %shoe ~ 253 | :+ %sole %txt 254 | render-current-after 255 | == 256 | -- 257 | :: 258 | ++ joined-groups-list 259 | %~ tap in 260 | .^((set resource:resource) %gy /(scot %p our:bowl)/group-store/(scot %da now:bowl)/groups) 261 | :: 262 | ++ joined-groups-listmap 263 | (fuse:pal (gulf 1 (lent joined-groups-list)) joined-groups-list) 264 | :: 265 | ++ joined-groups-map 266 | (malt joined-groups-listmap) 267 | :: 268 | ++ destructure-node 269 | |% 270 | ++ get-text-content 271 | |= i=node:g 272 | ?> ?=(%& -.post.i) 273 | %+ turn (skim contents.p.post.i |=(i=content:g |(?=([%text *] i) ?=([%url *] i)))) 274 | |= i=content:g 275 | ?+ -.i ~ 276 | %text 277 | ?> ?=([%text *] i) (trip text.i) 278 | %url 279 | ?> ?=([%url *] i) (trip url.i) 280 | == 281 | ++ get-author 282 | |= i=node:g 283 | ?> ?=(%& -.post.i) 284 | author.p.post.i 285 | :: 286 | ++ get-time-sent 287 | |= i=node:g 288 | ?> ?=(%& -.post.i) 289 | =/ time-sent time-sent.p.post.i 290 | (sub time-sent (mod time-sent ~s1)) 291 | :: 292 | ++ get-index 293 | |= i=node:g 294 | ?> ?=(%& -.post.i) 295 | ;;(cord p.+:(numb:enjs:format -.index.p.post.i)) 296 | :: 297 | ++ get-channel 298 | |= i=resource:g 299 | (crip "{(scow %p entity.i)}/{(scow %tas name.i)}") 300 | -- 301 | :: 302 | ++ render 303 | |% 304 | :: 305 | ++ start [%sole %txt "\0agraph-store query: build and run a query by composing query options. press ? to see query options\0a"] 306 | :: 307 | ++ render-options 308 | ^- (quip card _state) 309 | :- 310 | :~ 311 | :+ %shoe ~ 312 | :+ %sole %mor 313 | =- (turn - (lead %txt)) 314 | :~ 315 | "\0ag show groups" 316 | "1 select a group to query from and press enter" 317 | "d exclude direct messages in query, by default included" 318 | "/ enter a search term after /, e.g. /sample" 319 | "~ enter ship to see all posts from this ship, e.g. ~sonsum" 320 | "b enter a date to see posts before that date. date must be in format bYYYY.M.D, e.g. b2021.5.30 or b2020.11.5..13.03.00" 321 | "a enter a date to see posts after that date. date must be in format aYYYY.M.D, e.g. a2021.5.30 or a2020.11.5..13.03.00" 322 | "r run query" 323 | "> go to next page of search results" 324 | "p go to specified page of search results" 325 | ". show currently applied query options" 326 | "- clear query\0a" 327 | == 328 | == 329 | state 330 | :: 331 | ++ render-query-state 332 | ^- (quip card _state) 333 | :- 334 | :~ 335 | :+ %shoe ~ 336 | :+ %sole %txt 337 | ;: weld 338 | "\0a" 339 | resource.render-input-state 340 | text.render-input-state 341 | author.render-input-state 342 | before.render-input-state 343 | after.render-input-state 344 | "\0a" 345 | == 346 | == 347 | state 348 | :: 349 | ++ joined-groups 350 | ^- (quip card _state) 351 | :- 352 | :~ 353 | :+ %shoe ~ 354 | :+ %sole %mor 355 | =- (turn - (lead %txt)) 356 | ^- wall 357 | ?. =(& dms-disabled) 358 | :: dms enabled 359 | %+ turn joined-groups-listmap 360 | |= i=[id=@ud =resource:resource] 361 | "{(scow %ud id.i)} {(scow %p -.resource.i)}/{(scow %tas +.resource.i)}\0a" 362 | :: dms disabled 363 | %+ turn (skip joined-groups-listmap |=(i=[@ =resource:resource] =("dm" `tape`(scag 2 (scow %tas name.resource.i))))) 364 | |= i=[id=@ud =resource:resource] 365 | "{(scow %ud id.i)} {(scow %p -.resource.i)}/{(scow %tas +.resource.i)}\0a" 366 | == 367 | state 368 | :: 369 | -- 370 | -- -------------------------------------------------------------------------------- /src/gen/graph-query.hoon: -------------------------------------------------------------------------------- 1 | /- g=graph-store 2 | /- metadata=metadata-store 3 | /+ gra=graph-query 4 | 5 | :- %say 6 | |= $: [now=@da * bec=beak] 7 | $: 8 | resource=resource:g :: group to query from 9 | sq=(unit tape) :: search query word :: TODO make case insensitive 10 | author=(unit @p) :: author of post 11 | before=(unit @da) :: cutoff posts after a timepoint 12 | after=(unit @da) :: cutoff posts before a timepoint 13 | page=@ud :: cutoff value of amount of nodes to query to back in time 14 | ~ 15 | == 16 | ~ 17 | == 18 | :- %noun 19 | =* our p.bec 20 | |^ 21 | (skip channel-reducer |=(i=[resource:g (list node:g)] =(~ +.i))) 22 | :: 23 | +$ query-filter (unit $-(node:g ?(~ [~ node:g]))) 24 | :: 25 | ++ get-nodes 26 | |= [ship=@p name=cord] 27 | =/ graph-srcy .^(update:g %gx /(scot %p our)/graph-store/(scot %da now)/graph/(scot %p ship)/(scot %tas name)/noun) 28 | =/ graph 29 | ?> ?=([%add-graph *] q.graph-srcy) graph.q.graph-srcy 30 | (tap-deep-time:gra [*index:g graph page]) 31 | :: 32 | ++ get-text-content 33 | |= c=content:g 34 | ?.(?=([%text *] c) ~ `c) 35 | :: 36 | ++ get-match 37 | |= c=content:g 38 | =/ gah2 (mask ~[`@`10 `@`9 ' ']) 39 | ?> ?=([%text *] c) 40 | =/ search 41 | %+ skim 42 | `wall`(rash text.c (more gah2 (star ;~(less gah2 prn)))) :: tokenize text and skip nulls 43 | |=(i=tape =(i (need sq))) 44 | ?~ search ~ 45 | `c 46 | :: 47 | ++ matched-contents-reducer 48 | |= i=node:g 49 | ?> ?=(%& -.post.i) 50 | %+ reel 51 | (limo ~[get-match get-text-content]) 52 | |= [f=$-(content:g ?(~ [~ content:g])) l=_contents.p.post.i] 53 | (murn l f) 54 | :: 55 | ++ query-reducer 56 | |= [ship=@p name=cord] 57 | =/ search-f=query-filter ?~(sq ~ `|=(i=node:g ?.(!=(~ (matched-contents-reducer i)) ~ `i))) 58 | =/ author-f=query-filter ?~(author ~ `|=(i=node:g ?.(=((need author) ?>(?=(%& -.post.i) author.p.post.i)) ~ `i))) 59 | =/ before-f=query-filter ?~(before ~ `|=(i=node:g ?.((lth ?>(?=(%& -.post.i) time-sent.p.post.i) (need before)) ~ `i))) 60 | =/ after-f=query-filter ?~(after ~ `|=(i=node:g ?.((gth ?>(?=(%& -.post.i) time-sent.p.post.i) (need after)) ~ `i))) 61 | =/ deleted-f=query-filter `|=(i=node:g ?.(?=(%& -.post.i) ~ `i)) 62 | =/ composite-f=(list $-(node:g ?(~ [~ node:g]))) 63 | %+ murn 64 | `(list query-filter)`~[search-f after-f before-f author-f deleted-f] 65 | |=(i=query-filter ?~(i ~ i)) 66 | :: 67 | %+ reel 68 | composite-f 69 | |= [f=$-(node:g ?(~ [~ node:g])) l=_(turn (get-nodes [ship name]) |=(i=[p=index:g q=node:g] q.i))] 70 | (murn l f) 71 | :: 72 | ++ channel-reducer 73 | ^- (list [resource:g (list node:g)]) 74 | ;; (list [resource:g (list node:g)]) 75 | =/ group-channels .^(associations:metadata %gx /(scot %p our)/metadata-store/(scot %da now)/app-name/graph/noun) 76 | =/ joined-channels q:.^(update:g %gx /(scot %p our)/graph-store/(scot %da now)/keys/noun) 77 | ?> ?=([%keys *] joined-channels) 78 | %+ skip 79 | %+ turn ~(tap by group-channels) 80 | |= i=[p=md-resource:metadata q=association:metadata] 81 | ?. ?& =([entity.group.q.i name.group.q.i] resource) 82 | (~(has in resources.joined-channels) resource.p.i) 83 | == 84 | ~ 85 | [`resource:g`[entity.resource.p.i name.resource.p.i] (query-reducer [entity.resource.p.i `cord`name.resource.p.i])] 86 | |= i=?(~ [resource:g *]) 87 | =(~ i) 88 | -- -------------------------------------------------------------------------------- /src/lib/graph-query.hoon: -------------------------------------------------------------------------------- 1 | /+ graph-lib=graph 2 | /+ store=graph-store 3 | |_ =bowl:gall 4 | ++ tap-deep-time 5 | |= [=index:store =graph:store page=@ud] 6 | ^- (list [index:store node:store]) 7 | =/ many 420 8 | =/ nodelist 9 | =/ nodes (tap:orm:store graph) 10 | =| lis=(list [index:store node:store]) 11 | =| counter=@ud 12 | |- 13 | ?~ nodes lis 14 | =/ child-index (snoc index key.i.nodes) 15 | =/ childless-node val.i.nodes(children [%empty ~]) 16 | ?: ?=(%empty -.children.val.i.nodes) 17 | ?: =(counter (mul many page)) lis 18 | ^- (list [index:store node:store]) 19 | $(lis (snoc lis [child-index childless-node]), nodes t.nodes, counter +(counter)) 20 | ^- (list [index:store node:store]) 21 | %= $ 22 | lis 23 | %+ weld 24 | (snoc lis [child-index childless-node]) 25 | (tap-deep:graph-lib child-index p.children.val.i.nodes) 26 | nodes t.nodes 27 | counter +(counter) 28 | == 29 | ?: =(page 1) nodelist 30 | %+ slag (mul many (sub page 1)) nodelist 31 | -- -------------------------------------------------------------------------------- /src/lib/pal.hoon: -------------------------------------------------------------------------------- 1 | :: /lib/pal: friendly helper library 2 | :: by courtesy of ~palfun-foslup 3 | :: 4 | |% 5 | :: list operations 6 | :: 7 | ++ fuse :: cons contents 8 | |* [a=(list) b=(list)] 9 | :: ^- (list _?>(?=([^ ^] [a b]) [i.a i.b])) ::TODO why does this not work? 10 | ^- (list [_?>(?=(^ a) i.a) _?>(?=(^ b) i.b)]) 11 | ?~ a ~ 12 | ?~ b ~ 13 | :- [i.a i.b] 14 | $(a t.a, b t.b) 15 | :: 16 | ++ snip :: drop tail off list 17 | |* a=(list) 18 | ^+ a 19 | ?~ a ~ 20 | ?: =(~ t.a) ~ ::NOTE avoiding tmi 21 | [i.a $(a t.a)] 22 | :: 23 | ++ rear :: last item of list 24 | |* a=(list) 25 | ^- _?>(?=(^ a) i.a) 26 | ?~ a !! 27 | ?: =(~ t.a) i.a ::NOTE avoiding tmi 28 | $(a t.a) 29 | :: 30 | :: data structures 31 | :: 32 | ++ mip :: map of maps 33 | |$ [kex key value] 34 | (map kex (map key value)) 35 | :: 36 | ++ bi :: mip engine 37 | =| a=(map * (map)) 38 | |@ 39 | ++ del 40 | |* [b=* c=*] 41 | =+ d=(~(gut by a) b ~) 42 | =+ e=(~(del by d) c) 43 | ?~ e 44 | (~(del by a) b) 45 | (~(put by a) b e) 46 | :: 47 | ++ get 48 | |* [b=* c=*] 49 | (~(get by (~(gut by a) b ~)) c) 50 | :: 51 | ++ got 52 | |* [b=* c=*] 53 | (need (get b c)) 54 | :: 55 | ++ gut 56 | |* [b=* c=* d=*] 57 | (~(gut by (~(gut by a) b ~)) c d) 58 | :: 59 | ++ put 60 | |* [b=* c=* d=*] 61 | %+ ~(put by a) b 62 | %. [c d] 63 | %~ put by 64 | (~(gut by a) b ~) 65 | :: 66 | ++ tap 67 | ::NOTE naive turn-based implementation find-errors ): 68 | =< $ 69 | =+ b=`_?>(?=(^ a) *(list [x=_p.n.a _?>(?=(^ q.n.a) [y=p v=q]:n.q.n.a)]))`~ 70 | |. ^+ b 71 | ?~ a 72 | b 73 | $(a r.a, b (welp (turn ~(tap by q.n.a) (lead p.n.a)) $(a l.a))) 74 | -- 75 | :: 76 | :: parsers 77 | :: 78 | ++ opt :: rule or ~ 79 | |* =rule 80 | (may rule ~) 81 | :: 82 | ++ may :: rule or constant 83 | |* [=rule else=*] 84 | ;~(pose rule (easy else)) 85 | :: 86 | ++ qfix :: singular prefix 87 | |* [p=rule r=rule] 88 | ;~(pfix p r) 89 | :: 90 | ++ lean :: measure to parse 91 | |* [measure=rule separate=rule parse=$-(* rule)] 92 | |= =nail 93 | =/ edge=(like @) (;~(sfix measure separate) nail) 94 | ?~ q.edge edge 95 | =* size p.u.q.edge 96 | =* rest q.u.q.edge 97 | ((parse size) rest) 98 | -- 99 | --------------------------------------------------------------------------------