├── .dir-locals.el ├── .github └── workflows │ └── tests.yml ├── .gitignore ├── CHANGELOG.md ├── LICENSE ├── README.md ├── bb.edn ├── benchmark.clj ├── bin └── kaocha ├── deps.edn ├── docs └── GUIDE.md ├── package-lock.json ├── package.json ├── project.clj ├── scratch.clj ├── site ├── .clj-kondo │ └── config.edn ├── .gitignore ├── deps.edn ├── package-lock.json ├── package.json ├── postcss.config.js ├── public │ ├── css │ │ └── app.css │ └── index.html ├── shadow-cljs.edn ├── src │ ├── helix │ │ └── core │ │ │ └── alpha.cljs │ └── pyramid │ │ └── site │ │ ├── codemirror.cljs │ │ ├── components.cljs │ │ ├── core.cljs │ │ └── tree.cljs └── tailwind.config.js ├── src └── pyramid │ ├── core.cljc │ ├── ident.cljc │ ├── pull.cljc │ └── query.cljc ├── test ├── bb_test_runner.clj └── pyramid │ ├── core_test.cljc │ ├── pull_test.cljc │ └── query_test.cljc └── tests.edn /.dir-locals.el: -------------------------------------------------------------------------------- 1 | ((nil 2 | (cider-clojure-cli-global-options . "-A:dev:test:benchmark:profile"))) 3 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: Unit tests 2 | on: [push] 3 | jobs: 4 | run-tests: 5 | runs-on: ubuntu-latest 6 | steps: 7 | - name: checkout 8 | uses: actions/checkout@v2 9 | - uses: actions/setup-java@v1 10 | with: 11 | java-version: '11' 12 | - uses: DeLaGuardo/setup-clojure@master 13 | with: 14 | cli: '1.10.1.727' 15 | - name: Download bb master 16 | run: bash <(curl -s https://raw.githubusercontent.com/babashka/babashka/master/install) --version 0.8.157-SNAPSHOT 17 | - name: Setup Node.js environment 18 | uses: actions/setup-node@v1 19 | - name: Install npm deps 20 | run: npm install 21 | - name: Run tests 22 | run: bin/kaocha 23 | - name: Run bb tests 24 | run: bb test-bb 25 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /node_modules/ 2 | /out/ 3 | /.cljs_node_repl/ 4 | /.cpcache/ 5 | /.nrepl-port 6 | /target/ 7 | /pom.xml.asc 8 | /pom.xml 9 | .idea 10 | *.iml 11 | /site/public/css/app.compiled.css 12 | /site/.clj-kondo/.cache/ 13 | /.clj-kondo/ 14 | /.lsp/ 15 | /.log/ 16 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # CHANGELOG.md 2 | 3 | ## 4.1.0 4 | 5 | ### Fixed 6 | 7 | - `:visitor` on a join will take the full value if the value is a map, or be 8 | called for each child if it's a collection. This matches the behavior that the 9 | README says. 10 | 11 | ## 4.0.0 12 | 13 | ### Breaking 14 | 15 | * You can now mix entities and non-entities in collections when using `pull` and 16 | `pull-report`. 17 | 18 | ### Fixed 19 | 20 | * Unions on lists now return results in correct order 21 | 22 | ## 3.4.0 23 | 24 | ### Added 25 | 26 | * New `pyramid.core/identify` function which takes a DB and entity map and returns the lookup ID of the map if it were added to the DB. 27 | * New visitor pattern support 28 | 29 | Visitor pattern: you can now annotate parts of an EQL query with 30 | `{:visitor (fn visit [db data] ,,,)}` metadata, which will replace the location 31 | with the return value of the `visit` function in the final result of the `pull` 32 | or `pull-report` call. 33 | 34 | It is similar to doing a postwalk on the results of `pull` or `pull-report`, but 35 | is done in the same pass as pulling data out of the DB - so less traversals - and 36 | annotated directly on the query. 37 | 38 | See [pull_test.cljc](https://github.com/lilactown/pyramid/blob/839306f374ddb888b9e06f0f9dfd14a5f943b0ee/test/pyramid/pull_test.cljc#L53) for example usage. 39 | 40 | ### Fixed 41 | 42 | - `merge-entity` in babashka 43 | 44 | ## 3.3.0 45 | 46 | This update includes a significant rewrite of the algorithm which traverses & 47 | pulls data out of the db based on an EQL query. It has a minor if any performance 48 | impact, but allows working with arbitrarily nested data. 49 | 50 | ### Added 51 | 52 | - Pull queries now use protocol `IPullable` to resolve entities, allowing 53 | pyramid to query arbitrary data stores using EQL. 54 | - Datalog queries now use protocol `IQueryable` to get all entities from an 55 | object, allowing pyramid to query arbitrary data stores in memory. 56 | - `pull-report` now returns an `:indices` key with a set of top-level indices 57 | that are used in the query. 58 | - `add-report` now returns an `:indices` key with a set of top-level indices 59 | that are modified through adding the data to the db 60 | - Support for babashka 61 | 62 | ### Fixed 63 | 64 | - Fix error when a reference to an entity that doesn't exist in the db is queried. 65 | Now returns a map with the ident conj'd. 66 | 67 | 68 | ## 3.2.0 69 | 70 | This update includes a significant rewrite of the normalization algorithm which 71 | in the real world results in a 50% reduction in time spent normalization and 72 | also supports arbitrary levels of nesting (up to computer memory limits). 73 | 74 | ### Fixed 75 | 76 | - Maintain correct order of sequences of entities when being normalized 77 | - Fix #14: data loss in p/add with nested maps 78 | 79 | ## 3.1.4 80 | 81 | ### Fixed 82 | 83 | - Fixed a bug in adding data where joins with params whose keys look like an entity 84 | was being replaced with a ref incorrectly 85 | 86 | ## 3.1.3 87 | 88 | ### Fixed 89 | 90 | - Fixed a bug in adding data where lists (such as joins with params) got reversed 91 | 92 | 93 | ## 3.0.0 to 3.1.2 94 | 95 | Renamed to pyramid. 96 | Experimental datalog-like query engine. 97 | Internal refactor to use zippers. 98 | 99 | ## 2.0.0 100 | 101 | ### Breaking 102 | 103 | In 1.2.0, we try to figure out how to identify a map by running a fn on each key 104 | without context. This precludes more complex logic, such as preferring one ID 105 | over another or using a composite of multiple keys to produce an ident. 106 | 107 | Now, when you create a new db value, the optional second argument should be a 108 | function that takes the entire map as a value and returns either an ident (a 109 | tuple [key val] that uniquely identifies the entity in your system) or nil. 110 | 111 | 112 | ### Added 113 | 114 | Additionally, a new namespace `autonormal.ident` is available for composing 115 | functions for identifying entities. 116 | 117 | ```clojure 118 | (require '[autonormal.core :as a] 119 | '[autonormal.ident :as ident]) 120 | 121 | ;; creates a new db that will use `person/id` and `:food/name` 122 | ;; to identify entities 123 | (a/db 124 | [] 125 | (fn [entity] 126 | ;; first, check for :person/id 127 | (if-let [person-id (:person/id entity)] 128 | [:person/id person-id] 129 | ;; else, check for :food/name. return nil otherwise 130 | (when-let [food-name (:food/name entity)] 131 | [:food/name food-name])))) 132 | 133 | 134 | ;; autonormal.ident/by is a helper to compose functions that take an entity 135 | ;; and return either an ident or nil 136 | (a/db [] (ident/by 137 | (fn [entity] 138 | (when-let [person-id (:person/id entity)] 139 | [:person/id person-id])) 140 | (fn [entity] 141 | (when-let [food-name (:food/name entity)] 142 | [:food/name food-name])))) 143 | 144 | 145 | ;; autonormal.ident/by-keys is a helper that handles the specific case of 146 | ;; composing keys 147 | (a/db [] (ident/by-keys :person/id :food/name)) 148 | ``` 149 | 150 | ## 1.2.0 151 | 152 | ### Added 153 | 154 | - `db` now takes a second argument, `identify`, which is a function used to determine whether a key is 155 | used to identify the entity or not 156 | 157 | ## 1.1.1 158 | 159 | ### Fixed 160 | 161 | - `:autonormal.core/not-found` values were still present in union entries, are now filtered out appropriately 162 | 163 | ## 1.1.0 164 | 165 | This is a minor bump to better reflect changes made in 1.0.3 and 1.0.2. 166 | 167 | ## Added 168 | 169 | * More docstrings 170 | 171 | ## 1.0.3 172 | 173 | ### Added 174 | 175 | * `delete`, which dissocs an entity from the db and removes all references to it 176 | 177 | ### Fixed 178 | 179 | * Adding and creating databases with entities that have non-entities inside collections 180 | 181 | ## 1.0.2 182 | 183 | ### Added 184 | 185 | * `add-report`, which returns a map with keys `:db`, containing the updated map, 186 | and `:entities`, which contains the set of lookup refs modified 187 | 188 | * `pull-report`, which returns a map with keys `:data`, containing the result of 189 | the EQL query, and `:entities`, which contains the set of lookup refs 190 | queried in `:data` 191 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Eclipse Public License - v 2.0 2 | 3 | THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE 4 | PUBLIC LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION 5 | OF THE PROGRAM CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. 6 | 7 | 1. DEFINITIONS 8 | 9 | "Contribution" means: 10 | 11 | a) in the case of the initial Contributor, the initial content 12 | Distributed under this Agreement, and 13 | 14 | b) in the case of each subsequent Contributor: 15 | i) changes to the Program, and 16 | ii) additions to the Program; 17 | where such changes and/or additions to the Program originate from 18 | and are Distributed by that particular Contributor. A Contribution 19 | "originates" from a Contributor if it was added to the Program by 20 | such Contributor itself or anyone acting on such Contributor's behalf. 21 | Contributions do not include changes or additions to the Program that 22 | are not Modified Works. 23 | 24 | "Contributor" means any person or entity that Distributes the Program. 25 | 26 | "Licensed Patents" mean patent claims licensable by a Contributor which 27 | are necessarily infringed by the use or sale of its Contribution alone 28 | or when combined with the Program. 29 | 30 | "Program" means the Contributions Distributed in accordance with this 31 | Agreement. 32 | 33 | "Recipient" means anyone who receives the Program under this Agreement 34 | or any Secondary License (as applicable), including Contributors. 35 | 36 | "Derivative Works" shall mean any work, whether in Source Code or other 37 | form, that is based on (or derived from) the Program and for which the 38 | editorial revisions, annotations, elaborations, or other modifications 39 | represent, as a whole, an original work of authorship. 40 | 41 | "Modified Works" shall mean any work in Source Code or other form that 42 | results from an addition to, deletion from, or modification of the 43 | contents of the Program, including, for purposes of clarity any new file 44 | in Source Code form that contains any contents of the Program. Modified 45 | Works shall not include works that contain only declarations, 46 | interfaces, types, classes, structures, or files of the Program solely 47 | in each case in order to link to, bind by name, or subclass the Program 48 | or Modified Works thereof. 49 | 50 | "Distribute" means the acts of a) distributing or b) making available 51 | in any manner that enables the transfer of a copy. 52 | 53 | "Source Code" means the form of a Program preferred for making 54 | modifications, including but not limited to software source code, 55 | documentation source, and configuration files. 56 | 57 | "Secondary License" means either the GNU General Public License, 58 | Version 2.0, or any later versions of that license, including any 59 | exceptions or additional permissions as identified by the initial 60 | Contributor. 61 | 62 | 2. GRANT OF RIGHTS 63 | 64 | a) Subject to the terms of this Agreement, each Contributor hereby 65 | grants Recipient a non-exclusive, worldwide, royalty-free copyright 66 | license to reproduce, prepare Derivative Works of, publicly display, 67 | publicly perform, Distribute and sublicense the Contribution of such 68 | Contributor, if any, and such Derivative Works. 69 | 70 | b) Subject to the terms of this Agreement, each Contributor hereby 71 | grants Recipient a non-exclusive, worldwide, royalty-free patent 72 | license under Licensed Patents to make, use, sell, offer to sell, 73 | import and otherwise transfer the Contribution of such Contributor, 74 | if any, in Source Code or other form. This patent license shall 75 | apply to the combination of the Contribution and the Program if, at 76 | the time the Contribution is added by the Contributor, such addition 77 | of the Contribution causes such combination to be covered by the 78 | Licensed Patents. The patent license shall not apply to any other 79 | combinations which include the Contribution. No hardware per se is 80 | licensed hereunder. 81 | 82 | c) Recipient understands that although each Contributor grants the 83 | licenses to its Contributions set forth herein, no assurances are 84 | provided by any Contributor that the Program does not infringe the 85 | patent or other intellectual property rights of any other entity. 86 | Each Contributor disclaims any liability to Recipient for claims 87 | brought by any other entity based on infringement of intellectual 88 | property rights or otherwise. As a condition to exercising the 89 | rights and licenses granted hereunder, each Recipient hereby 90 | assumes sole responsibility to secure any other intellectual 91 | property rights needed, if any. For example, if a third party 92 | patent license is required to allow Recipient to Distribute the 93 | Program, it is Recipient's responsibility to acquire that license 94 | before distributing the Program. 95 | 96 | d) Each Contributor represents that to its knowledge it has 97 | sufficient copyright rights in its Contribution, if any, to grant 98 | the copyright license set forth in this Agreement. 99 | 100 | e) Notwithstanding the terms of any Secondary License, no 101 | Contributor makes additional grants to any Recipient (other than 102 | those set forth in this Agreement) as a result of such Recipient's 103 | receipt of the Program under the terms of a Secondary License 104 | (if permitted under the terms of Section 3). 105 | 106 | 3. REQUIREMENTS 107 | 108 | 3.1 If a Contributor Distributes the Program in any form, then: 109 | 110 | a) the Program must also be made available as Source Code, in 111 | accordance with section 3.2, and the Contributor must accompany 112 | the Program with a statement that the Source Code for the Program 113 | is available under this Agreement, and informs Recipients how to 114 | obtain it in a reasonable manner on or through a medium customarily 115 | used for software exchange; and 116 | 117 | b) the Contributor may Distribute the Program under a license 118 | different than this Agreement, provided that such license: 119 | i) effectively disclaims on behalf of all other Contributors all 120 | warranties and conditions, express and implied, including 121 | warranties or conditions of title and non-infringement, and 122 | implied warranties or conditions of merchantability and fitness 123 | for a particular purpose; 124 | 125 | ii) effectively excludes on behalf of all other Contributors all 126 | liability for damages, including direct, indirect, special, 127 | incidental and consequential damages, such as lost profits; 128 | 129 | iii) does not attempt to limit or alter the recipients' rights 130 | in the Source Code under section 3.2; and 131 | 132 | iv) requires any subsequent distribution of the Program by any 133 | party to be under a license that satisfies the requirements 134 | of this section 3. 135 | 136 | 3.2 When the Program is Distributed as Source Code: 137 | 138 | a) it must be made available under this Agreement, or if the 139 | Program (i) is combined with other material in a separate file or 140 | files made available under a Secondary License, and (ii) the initial 141 | Contributor attached to the Source Code the notice described in 142 | Exhibit A of this Agreement, then the Program may be made available 143 | under the terms of such Secondary Licenses, and 144 | 145 | b) a copy of this Agreement must be included with each copy of 146 | the Program. 147 | 148 | 3.3 Contributors may not remove or alter any copyright, patent, 149 | trademark, attribution notices, disclaimers of warranty, or limitations 150 | of liability ("notices") contained within the Program from any copy of 151 | the Program which they Distribute, provided that Contributors may add 152 | their own appropriate notices. 153 | 154 | 4. COMMERCIAL DISTRIBUTION 155 | 156 | Commercial distributors of software may accept certain responsibilities 157 | with respect to end users, business partners and the like. While this 158 | license is intended to facilitate the commercial use of the Program, 159 | the Contributor who includes the Program in a commercial product 160 | offering should do so in a manner which does not create potential 161 | liability for other Contributors. Therefore, if a Contributor includes 162 | the Program in a commercial product offering, such Contributor 163 | ("Commercial Contributor") hereby agrees to defend and indemnify every 164 | other Contributor ("Indemnified Contributor") against any losses, 165 | damages and costs (collectively "Losses") arising from claims, lawsuits 166 | and other legal actions brought by a third party against the Indemnified 167 | Contributor to the extent caused by the acts or omissions of such 168 | Commercial Contributor in connection with its distribution of the Program 169 | in a commercial product offering. The obligations in this section do not 170 | apply to any claims or Losses relating to any actual or alleged 171 | intellectual property infringement. In order to qualify, an Indemnified 172 | Contributor must: a) promptly notify the Commercial Contributor in 173 | writing of such claim, and b) allow the Commercial Contributor to control, 174 | and cooperate with the Commercial Contributor in, the defense and any 175 | related settlement negotiations. The Indemnified Contributor may 176 | participate in any such claim at its own expense. 177 | 178 | For example, a Contributor might include the Program in a commercial 179 | product offering, Product X. That Contributor is then a Commercial 180 | Contributor. If that Commercial Contributor then makes performance 181 | claims, or offers warranties related to Product X, those performance 182 | claims and warranties are such Commercial Contributor's responsibility 183 | alone. Under this section, the Commercial Contributor would have to 184 | defend claims against the other Contributors related to those performance 185 | claims and warranties, and if a court requires any other Contributor to 186 | pay any damages as a result, the Commercial Contributor must pay 187 | those damages. 188 | 189 | 5. NO WARRANTY 190 | 191 | EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, AND TO THE EXTENT 192 | PERMITTED BY APPLICABLE LAW, THE PROGRAM IS PROVIDED ON AN "AS IS" 193 | BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR 194 | IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF 195 | TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR 196 | PURPOSE. Each Recipient is solely responsible for determining the 197 | appropriateness of using and distributing the Program and assumes all 198 | risks associated with its exercise of rights under this Agreement, 199 | including but not limited to the risks and costs of program errors, 200 | compliance with applicable laws, damage to or loss of data, programs 201 | or equipment, and unavailability or interruption of operations. 202 | 203 | 6. DISCLAIMER OF LIABILITY 204 | 205 | EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, AND TO THE EXTENT 206 | PERMITTED BY APPLICABLE LAW, NEITHER RECIPIENT NOR ANY CONTRIBUTORS 207 | SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 208 | EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION LOST 209 | PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 210 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 211 | ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE 212 | EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE 213 | POSSIBILITY OF SUCH DAMAGES. 214 | 215 | 7. GENERAL 216 | 217 | If any provision of this Agreement is invalid or unenforceable under 218 | applicable law, it shall not affect the validity or enforceability of 219 | the remainder of the terms of this Agreement, and without further 220 | action by the parties hereto, such provision shall be reformed to the 221 | minimum extent necessary to make such provision valid and enforceable. 222 | 223 | If Recipient institutes patent litigation against any entity 224 | (including a cross-claim or counterclaim in a lawsuit) alleging that the 225 | Program itself (excluding combinations of the Program with other software 226 | or hardware) infringes such Recipient's patent(s), then such Recipient's 227 | rights granted under Section 2(b) shall terminate as of the date such 228 | litigation is filed. 229 | 230 | All Recipient's rights under this Agreement shall terminate if it 231 | fails to comply with any of the material terms or conditions of this 232 | Agreement and does not cure such failure in a reasonable period of 233 | time after becoming aware of such noncompliance. If all Recipient's 234 | rights under this Agreement terminate, Recipient agrees to cease use 235 | and distribution of the Program as soon as reasonably practicable. 236 | However, Recipient's obligations under this Agreement and any licenses 237 | granted by Recipient relating to the Program shall continue and survive. 238 | 239 | Everyone is permitted to copy and distribute copies of this Agreement, 240 | but in order to avoid inconsistency the Agreement is copyrighted and 241 | may only be modified in the following manner. The Agreement Steward 242 | reserves the right to publish new versions (including revisions) of 243 | this Agreement from time to time. No one other than the Agreement 244 | Steward has the right to modify this Agreement. The Eclipse Foundation 245 | is the initial Agreement Steward. The Eclipse Foundation may assign the 246 | responsibility to serve as the Agreement Steward to a suitable separate 247 | entity. Each new version of the Agreement will be given a distinguishing 248 | version number. The Program (including Contributions) may always be 249 | Distributed subject to the version of the Agreement under which it was 250 | received. In addition, after a new version of the Agreement is published, 251 | Contributor may elect to Distribute the Program (including its 252 | Contributions) under the new version. 253 | 254 | Except as expressly stated in Sections 2(a) and 2(b) above, Recipient 255 | receives no rights or licenses to the intellectual property of any 256 | Contributor under this Agreement, whether expressly, by implication, 257 | estoppel or otherwise. All rights in the Program not expressly granted 258 | under this Agreement are reserved. Nothing in this Agreement is intended 259 | to be enforceable by any entity that is not a Contributor or Recipient. 260 | No third-party beneficiary rights are created under this Agreement. 261 | 262 | Exhibit A - Form of Secondary Licenses Notice 263 | 264 | "This Source Code may also be made available under the following 265 | Secondary Licenses when the conditions for such availability set forth 266 | in the Eclipse Public License, v. 2.0 are satisfied: {name license(s), 267 | version(s), and exceptions or additional permissions here}." 268 | 269 | Simply including a copy of this Agreement, including this Exhibit A 270 | is not sufficient to license the Source Code under Secondary Licenses. 271 | 272 | If it is not possible or desirable to put the notice in a particular 273 | file, then You may include the notice in a location (such as a LICENSE 274 | file in a relevant directory) where a recipient would be likely to 275 | look for such a notice. 276 | 277 | You may add additional accurate notices of copyright ownership. 278 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # pyramid 2 | 3 | A library for storing and querying graph data in Clojure. 4 | 5 | Features: 6 | * Graph query engine that works on any in-memory data store 7 | * Algorithm for taking trees of data and storing them in normal form in Clojure 8 | data. 9 | 10 | ## Install & docs 11 | 12 | [![Clojars Project](https://img.shields.io/clojars/v/town.lilac/pyramid.svg)](https://clojars.org/town.lilac/pyramid) [![cljdoc badge](https://cljdoc.org/badge/town.lilac/pyramid)](https://cljdoc.org/d/town.lilac/pyramid/CURRENT) 13 | 14 | 15 | ## Why 16 | 17 | Clojure is well known for its graph databases like datomic and datascript which 18 | implement a datalog-like query language. There are contexts where the power vs 19 | performance tradeoffs of these query languages don't make sense, which is where 20 | pyramid can shine. 21 | 22 | Pyramid focuses on doing essential things like selecting data from and 23 | traversing relationships between entities, while eschewing arbitrary logic like 24 | what SQL and datalog provide. What it lacks in features it makes up for in read 25 | performance when combined with a data store that has fast in-memory look ups of 26 | entities by key, such as Clojure maps. It can also be extended to databases like 27 | Datomic, DataScript and Asami. 28 | 29 | 30 | ## What 31 | 32 | Pyramid can be useful at each evolutionary stage of a program where one needs 33 | to traverse and make selections out of graphs of data. 34 | 35 | ### Selection 36 | 37 | Pyramid starts by working with Clojure data structures. A simple program that 38 | uses pyramid can use a query to select specific data out of a large, deeply 39 | nested tree, like a supercharged `select-keys`. 40 | 41 | ```clojure 42 | (def data 43 | {:people [{:given-name "Bob" :surname "Smith" :age 29} 44 | {:given-name "Alice" :surname "Meyer" :age 43}] 45 | :items {}}) 46 | 47 | (def query [{:people [:given-name]}]) 48 | 49 | (pyramid.core/pull data query) 50 | ;; => {:people [{:given-name "Bob"} {:given-name "Alice"}]} 51 | ``` 52 | 53 | ### Transformation 54 | 55 | Pyramid combines querying with the [Visitor pattern](https://en.wikipedia.org/wiki/Visitor_pattern) 56 | in a powerful way, allowing one to easily perform transformations of selections 57 | of data. Simply annotate parts of your query with metadata 58 | `{:visitor (fn visit [data selection] ,,,)}` and the `visit` function will be 59 | used to transform the data in a depth-first, post-order traversal (just like 60 | `clojure.walk/postwalk`). 61 | 62 | ```clojure 63 | (def data 64 | {:people [{:given-name "Bob" :surname "Smith" :age 29} 65 | {:given-name "Alice" :surname "Meyer" :age 43}] 66 | :items {}}) 67 | 68 | (defn fullname 69 | [_db {:keys [given-name surname] :as person}] 70 | (str given-name " " surname)) 71 | 72 | (def query [{:people ^{:visitor fullname} [:given-name :surname]}]) 73 | 74 | (pyramid.core/pull data query) 75 | ;; => {:people ["Bob Smith" "Alice Meyer"]} 76 | ``` 77 | 78 | ### Accretion 79 | 80 | A more complex program may need to keep track of that data over time, or query 81 | data that contains cycles, which can be done by creating a `pyramid.core/db`. 82 | "Pyramid dbs" are maps that have a particular structure: 83 | 84 | ```clojure 85 | ;; for any entity identified by `[key id]`, it follows the shape: 86 | {key {id {,,,}} 87 | ``` 88 | 89 | Adding data to a db will [normalize](https://en.wikipedia.org/wiki/Database_normalization) 90 | the data into a flat structure allowing for easy updating of entities as new 91 | data is obtained and allow relationships that are hard to represent in trees. 92 | Queries can traverse the references inside this data. 93 | 94 | See [docs/GUIDE.md](docs/GUIDE.md). 95 | 96 | ### Durability 97 | 98 | A program may grow to need durable storage and other features that more full 99 | featured in-memory databases provide. Pyramid provides a protocol, `IPullable`, 100 | which can be extended to allow queries to run over any store that data can be 101 | looked up by a tuple, `[primary-key value]`. This is generalizable to most 102 | databases like Datomic, DataScript, Asami and SQLite. 103 | 104 | ### Full stack 105 | 106 | The above shows the evolution of a single program, but many programs never grow 107 | beyond the accretion stage. Pyramid has been used primarily in user interfaces 108 | where data is stored in a data structure and queried over time to show different 109 | views on a large graph. Through its protocols, it can now be extended to be used 110 | with durable storage on the server as well. 111 | 112 | ## Concepts 113 | 114 | **Query**: A query is written using [EQL](https://edn-query-language.org/eql/1.0.0/what-is-eql.html), 115 | a query language implemented inside Clojure. It provides the ability to select 116 | data in a nested, recursive way, making it ideal for traversing graphs of data. 117 | It does not provide arbitrary logic like SQL or Datalog. 118 | 119 | **Entity map**: a Clojure map which contains information that uniquely identifies 120 | the domain entity it is about. E.g. `{:person/id 1234 :person/name "Bill" 121 | :person/age 67}` could be uniquely identified by it's `:person/id` key. By 122 | default, any map which contains a key which `(= "id" (name key))` is true, is an 123 | entity map and can be normalized using `pyramid.core/db`. 124 | 125 | **Ident function**: a function that takes a map and returns a tuple `[:key val]` 126 | that uniquely identifies the entity the map describes. 127 | 128 | **Lookup ref**: a 2-element Clojure vector that has a keyword and a value which 129 | together act as a pointer to a domain entity. E.g. `[:person/id 1234]`. 130 | `pyramid.core/pull` will attempt to look them up in the db value if they appear 131 | in the result at a location where the query expects to do a join. 132 | 133 | ## Usage 134 | 135 | See [docs/GUIDE.md](docs/GUIDE.md) 136 | 137 | ## Prior art 138 | 139 | - [Fulcro](https://fulcro.fulcrologic.com/) 140 | - @souenzzo's POC [eql-refdb](https://github.com/souenzzo/eql-refdb) 141 | - [MapGraph](https://github.com/stuartsierra/mapgraph/blob/master/test/com/stuartsierra/mapgraph/compare.clj) 142 | - [EntityDB](https://keechma.com/guides/entitydb/) 143 | - [DataScript](https://github.com/tonsky/datascript/) and derivatives 144 | - [Pathom](https://github.com/wilkerlucio/pathom) 145 | - [juxt/pull](https://github.com/juxt/pull) 146 | - [ribelo/doxa](https://github.com/ribelo/doxa) 147 | 148 | ## Copyright 149 | 150 | Copyright © 2023 Will Acton. Distributed under the EPL 2.0. 151 | -------------------------------------------------------------------------------- /bb.edn: -------------------------------------------------------------------------------- 1 | {:paths ["src" "test"] 2 | :deps {edn-query-language/eql {:mvn/version "1.0.1"} 3 | town.lilac/cascade {:mvn/version "2.0.1"} 4 | org.babashka/spec.alpha {:git/url "https://github.com/babashka/spec.alpha" 5 | :git/sha "1a841c4cc1d4f6dab7505a98ed2d532dd9d56b78"}} 6 | :tasks {test-bb 7 | {:doc "Run Babashka tests" 8 | :extra-deps {org.babashka/spec.alpha {:git/url "https://github.com/babashka/spec.alpha" 9 | :git/sha "1a841c4cc1d4f6dab7505a98ed2d532dd9d56b78"}} 10 | :extra-paths ["src" "test"] 11 | :task bb-test-runner/run-tests}}} 12 | -------------------------------------------------------------------------------- /benchmark.clj: -------------------------------------------------------------------------------- 1 | (ns benchmark 2 | (:require 3 | [pyramid.core :as p] 4 | [criterium.core :as c] 5 | [clj-async-profiler.core :as prof])) 6 | 7 | 8 | (prof/serve-files 8080) 9 | 10 | 11 | (prof/profile (dotimes [i 10000] 12 | (p/db [{:person/id 123 13 | :person/name "Will" 14 | :contact {:phone "000-000-0001"} 15 | :best-friend 16 | {:person/id 456 17 | :person/name "Jose" 18 | :account/email "asdf@jkl"} 19 | :friends 20 | [{:person/id 9001 21 | :person/name "Georgia"} 22 | {:person/id 456 23 | :person/name "Jose"} 24 | {:person/id 789 25 | :person/name "Frank"} 26 | {:person/id 1000 27 | :person/name "Robert"}]} 28 | {:person/id 456 29 | :best-friend {:person/id 123}}]))) 30 | 31 | 32 | (c/quick-bench 33 | (p/db [{:person/id 123 34 | :person/name "Will" 35 | :contact {:phone "000-000-0001"} 36 | :best-friend 37 | {:person/id 456 38 | :person/name "Jose" 39 | :account/email "asdf@jkl"} 40 | :friends 41 | [{:person/id 9001 42 | :person/name "Georgia"} 43 | {:person/id 456 44 | :person/name "Jose"} 45 | {:person/id 789 46 | :person/name "Frank"} 47 | {:person/id 1000 48 | :person/name "Robert"}]} 49 | {:person/id 456 50 | :best-friend {:person/id 123}}])) 51 | 52 | 53 | (def big-data 54 | [{:foo/data 55 | (vec 56 | (for [i (range 1000)] 57 | {:foo/id (str "id" i) 58 | :foo/name (str "bar" i) 59 | :foo/metadata {:some ["dumb" "data"]}})) } ]) 60 | 61 | 62 | (c/quick-bench (p/db big-data)) 63 | 64 | 65 | (prof/profile 66 | (dotimes [i 1000] 67 | (p/db big-data))) 68 | 69 | 70 | ;; this throws w/ StackOverflow on my computer when running p/db 71 | ;; numbers 10000 and up throw when generating the tree 72 | (def limit 7946) 73 | 74 | (def nested-data 75 | (letfn [(create [i] 76 | (if (zero? i) 77 | nil 78 | {:id i 79 | :child (create (dec i))}))] 80 | {:foo (create limit) })) 81 | 82 | 83 | (do (p/db [nested-data]) 84 | nil) 85 | 86 | 87 | (c/quick-bench (p/db [nested-data])) 88 | 89 | 90 | ;; testing out different trampoline strategies 91 | (defn create [k i] 92 | (if (zero? i) 93 | (k) 94 | (fn [] 95 | (create 96 | (fn [] 97 | {:id i 98 | :child (k)}) 99 | (dec i))))) 100 | 101 | 102 | (def really-nested-data 103 | {:foo (trampoline create (constantly {:id 0}) 50000) }) 104 | 105 | 106 | (do (p/db [really-nested-data]) 107 | nil) 108 | 109 | (c/quick-bench (p/db [really-nested-data])) 110 | 111 | 112 | (prof/profile (dotimes [i 1000] 113 | (p/db [really-nested-data]))) 114 | 115 | 116 | 117 | (def ppl-db 118 | (p/db [{:person/id 123 119 | :person/name "Will" 120 | :contact {:phone "000-000-0001"} 121 | :best-friend 122 | {:person/id 456 123 | :person/name "Jose" 124 | :account/email "asdf@jkl"} 125 | :friends 126 | [{:person/id 9001 127 | :person/name "Georgia"} 128 | {:person/id 456 129 | :person/name "Jose"} 130 | {:person/id 789 131 | :person/name "Frank"} 132 | {:person/id 1000 133 | :person/name "Robert"}]} 134 | {:person/id 456 135 | :best-friend {:person/id 123}}])) 136 | 137 | 138 | (p/pull ppl-db [{[:person/id 123] [:person/id 139 | :person/name 140 | {:friends [:person/id :person/name]}]}]) 141 | 142 | (def query [{[:person/id 123] [:person/id 143 | :person/name 144 | {:friends [:person/id :person/name]}]}]) 145 | 146 | (c/quick-bench 147 | (p/pull ppl-db query)) 148 | 149 | (prof/profile (dotimes [i 5000] 150 | (p/pull ppl-db query))) 151 | -------------------------------------------------------------------------------- /bin/kaocha: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | clojure -A:test -m kaocha.runner "$@" -------------------------------------------------------------------------------- /deps.edn: -------------------------------------------------------------------------------- 1 | {:paths ["src"] 2 | :deps {edn-query-language/eql {:mvn/version "1.0.1"} 3 | town.lilac/cascade {:mvn/version "2.0.1"}} 4 | :aliases {:test 5 | {:extra-paths ["test"] 6 | :extra-deps 7 | {lambdaisland/kaocha {:mvn/version "1.0.732"} 8 | lambdaisland/kaocha-cljs {:mvn/version "0.0-71"}}} 9 | :dev {:extra-deps 10 | {com.wsscode/pathom {:mvn/version "2.3.1"} 11 | datascript/datascript {:mvn/version "1.2.5"} 12 | org.clojars.quoll/asami {:mvn/version "2.1.1"} 13 | meander/epsilon {:mvn/version "0.0.602"} 14 | #_#_ town.lilac/cascade {:local/root "../cascade"}}} 15 | :benchmark {:extra-deps 16 | {criterium/criterium {:mvn/version "0.4.6"}}} 17 | :profile {:extra-deps 18 | {com.clojure-goes-fast/clj-async-profiler {:mvn/version "0.5.1"}} 19 | :jvm-opts ["-Djdk.attach.allowAttachSelf" 20 | "-XX:+UnlockDiagnosticVMOptions" 21 | "-XX:+DebugNonSafepoints"]}}} 22 | -------------------------------------------------------------------------------- /docs/GUIDE.md: -------------------------------------------------------------------------------- 1 | # Guide 2 | 3 | A guided walk through of how to use Pyramid with simple clojure data. 4 | 5 | ## Normalizing 6 | 7 | A `db` is simply a map with a tabular structure of entities, potentially with 8 | references to other entities. 9 | 10 | **Pyramid** currently makes a default conventional assumption: your entities 11 | are identified by a keyword whose name is `"id"`, e.g. `:id`, `:person/id`, 12 | `:my.corp.product/id`, etc. 13 | 14 | ```clojure 15 | (require '[pyramid.core :as p]) 16 | 17 | (def data 18 | {:person/id 0 :person/name "Rachel" 19 | :friend/list [{:person/id 1 :person/name "Marco"} 20 | {:person/id 2 :person/name "Cassie"} 21 | {:person/id 3 :person/name "Jake"} 22 | {:person/id 4 :person/name "Tobias"} 23 | {:person/id 5 :person/name "Ax"}]}) 24 | 25 | ;; you can pass in multiple entities to instantiate a db, so `p/db` gets a vector 26 | (def animorphs (p/db [data])) 27 | ;; => {:person/id {0 {:person/id 0 28 | ;; :person/name "Rachel" 29 | ;; :friend/list [[:person/id 1] 30 | ;; [:person/id 2] 31 | ;; [:person/id 3] 32 | ;; [:person/id 4] 33 | ;; [:person/id 5]]} 34 | ;; 1 {:person/id 1 :person/name "Marco"} 35 | ;; 2 {:person/id 2 :person/name "Cassie"} 36 | ;; 3 {:person/id 3 :person/name "Jake"} 37 | ;; 4 {:person/id 4 :person/name "Tobias"} 38 | ;; 5 {:person/id 5 :person/name "Ax"}}} 39 | ``` 40 | 41 | The map structure of a db is very efficient for getting info about any 42 | particular entity; it's just a `get-in` away: 43 | 44 | ```clojure 45 | (get-in animorphs [:person/id 1]) 46 | ;; => {:person/id 1 :person/name "Marco"} 47 | ``` 48 | 49 | You can `assoc`/`dissoc`/`update`/etc. this map in whatever way you would like. 50 | However, if you want to accrete more potentially nested data, there's a helpful 51 | `add` function to normalize it for you: 52 | 53 | ```clojure 54 | ;; Marco and Jake are each others best friend 55 | (def animorphs-2 56 | (p/add animorphs {:person/id 1 57 | :friend/best {:person/id 3 58 | :friend/best {:person/id 1}}})) 59 | ;; => {:person/id {0 {:person/id 0 60 | ;; :person/name "Rachel" 61 | ;; :friend/list [[:person/id 1] 62 | ;; [:person/id 2] 63 | ;; [:person/id 3] 64 | ;; [:person/id 4] 65 | ;; [:person/id 5]]} 66 | ;; 1 {:person/id 1 67 | ;; :person/name "Marco" 68 | ;; :friend/best [:person/id 3]} 69 | ;; 2 {:person/id 2 :person/name "Cassie"} 70 | ;; 3 {:person/id 3 71 | ;; :person/name "Jake" 72 | ;; :friend/best [:person/id 1]} 73 | ;; 4 {:person/id 4 :person/name "Tobias"} 74 | ;; 5 {:person/id 5 :person/name "Ax"}}} 75 | ``` 76 | 77 | Note that our `animorphs` db is an immutable hash map; `add` simply returns the 78 | new value. It's up to you to decide how to track its value and keep it up to 79 | date in your system, e.g. in an atom. 80 | 81 | ## Adding non-entities 82 | 83 | Maps that are `add`ed are typically entities, but you can also add arbitrary 84 | maps and `add` will merge any non-entities with the database, normalizing and 85 | referencing any nested entities. 86 | 87 | Using this capability, you can create additional indexes on your entities. 88 | Example: 89 | 90 | ```clojure 91 | (def animorphs-3 92 | (p/add animorphs-2 {:species {:andalites [{:person/id 5 93 | :person/species "andalite"}]}})) 94 | ;; => {:person/id {,,, 95 | ;; 5 {:person/id 5 96 | ;; :person/name "Ax" 97 | ;; :person/species "andalite"}} 98 | ;; :species {:andalites [[:person/id 5]]}} 99 | ``` 100 | 101 | ## Pull queries 102 | 103 | This library implements a fast EQL engine for Clojure data. 104 | 105 | ```clojure 106 | (p/pull animorphs-3 [[:person/id 1]]) 107 | ;; => {[:person/id 1] {:person/id 1 108 | ;; :person/name "Macro" 109 | ;; :friend/best {:person/id 3}}} 110 | ``` 111 | 112 | You can join on idents and keys within entities, and it will resolve any 113 | references found in order to continue the query: 114 | 115 | ```clojure 116 | (p/pull animorphs-3 [{[:person/id 1] [:person/name 117 | {:friend/best [:person/name]}]}]) 118 | ;; => {[:person/id 1] {:person/name "Marco" 119 | ;; :friend/best {:person/name "Jake"}}} 120 | ``` 121 | 122 | Top-level keys in the db can also be joined on. 123 | 124 | ```clojure 125 | (p/pull animorphs-3 [{:species [{:andalites [:person/name]}]}]) 126 | ;; => {:species {:andalites [{:person/name "Ax"}]}} 127 | ``` 128 | 129 | Recursion is supported: 130 | 131 | ```clojure 132 | (def query '[{[:person/id 0] [:person/id 133 | :person/name 134 | {:friend/list ...}]}]) 135 | 136 | (= (-> (p/pull animorphs-3 query) 137 | (get [:person/id 0])) 138 | data) 139 | ;; => true 140 | ``` 141 | 142 | See the EQL docs and tests in this repo for more examples of what's possible! 143 | 144 | 145 | ## More details 146 | 147 | Collections are recursively walked to find entities. 148 | 149 | To get meta-information about what entities were added or queried, use the 150 | `add-report` and `pull-report` functions. 151 | 152 | To delete an entity and all references to it, use the `delete` function. 153 | 154 | ## Tips & Tricks 155 | 156 | ### Replacing an entity 157 | 158 | Data that is `add`ed about an existing entity are merged with whatever is in the 159 | db. To replace an entity, `dissoc` it first: 160 | 161 | ```clojure 162 | (-> (p/db [{:person/id 0 :foo "bar"}]) 163 | (update :person/id dissoc 0) 164 | (p/add {:person/id 0 :bar "baz"})) 165 | ;; => {:person/id {0 {:person/id 0 :bar "baz"}}} 166 | ``` 167 | 168 | ### Getting data for a specific entity 169 | 170 | Since a db is a simple map, you can always use `get-in` to get basic info 171 | regarding an entity. However, if your entity contains references, it will not 172 | resolve those for you. Enter EQL! 173 | 174 | To write an EQL query to get info about a specific entity, you can use an _ident_ 175 | to begin your query: 176 | 177 | ```clojure 178 | (p/pull animorphs-3 [[:person/id 1]]) 179 | ;; => {[:person/id 1] 180 | ;; {:person/id 1, :person/name "Marco", :friend/best #:person{:id 3}}} 181 | ``` 182 | 183 | You can add to the query to resolve references and get information about, e.g. 184 | Marco's best friend: 185 | 186 | ```clojure 187 | (p/pull animorphs-3 [{[:person/id 1] [:person/name 188 | {:friend/best [:person/name]}]}]) 189 | ;; => {[:person/id 1] {:person/name "Marco", :friend/best #:person{:name "Jake"}}} 190 | ``` 191 | 192 | ## Datalog queries 193 | 194 | The latest version of pyramid includes an experimental datomic/datascript-style 195 | query engine in `pyramid.query`. It is not production ready, but is good enough 196 | to explore a pyramid db for developer inspection and troubleshooting. 197 | 198 | 199 | ```clojure 200 | (require '[pyramid.query :refer [q]]) 201 | 202 | 203 | ;; find the names of people with best friends, and their best friends' name 204 | (q [:find ?name ?best-friend 205 | :where 206 | [?e :friend/best ?bf] 207 | [?e :person/name ?name] 208 | [?bf :person/name ?best-friend]] 209 | animorphs-3) 210 | ;; => (["Marco" "Jake"] ["Jake" "Marco"]) 211 | ``` 212 | 213 | -------------------------------------------------------------------------------- /package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pyramid", 3 | "version": "1.0.0", 4 | "lockfileVersion": 2, 5 | "requires": true, 6 | "packages": { 7 | "": { 8 | "name": "pyramid", 9 | "version": "1.0.0", 10 | "license": "ISC", 11 | "dependencies": { 12 | "ws": "^7.3.1" 13 | } 14 | }, 15 | "node_modules/ws": { 16 | "version": "7.3.1", 17 | "resolved": "https://registry.npmjs.org/ws/-/ws-7.3.1.tgz", 18 | "integrity": "sha512-D3RuNkynyHmEJIpD2qrgVkc9DQ23OrN/moAwZX4L8DfvszsJxpjQuUq3LMx6HoYji9fbIOBY18XWBsAux1ZZUA==", 19 | "engines": { 20 | "node": ">=8.3.0" 21 | }, 22 | "peerDependencies": { 23 | "bufferutil": "^4.0.1", 24 | "utf-8-validate": "^5.0.2" 25 | }, 26 | "peerDependenciesMeta": { 27 | "bufferutil": { 28 | "optional": true 29 | }, 30 | "utf-8-validate": { 31 | "optional": true 32 | } 33 | } 34 | } 35 | }, 36 | "dependencies": { 37 | "ws": { 38 | "version": "7.3.1", 39 | "resolved": "https://registry.npmjs.org/ws/-/ws-7.3.1.tgz", 40 | "integrity": "sha512-D3RuNkynyHmEJIpD2qrgVkc9DQ23OrN/moAwZX4L8DfvszsJxpjQuUq3LMx6HoYji9fbIOBY18XWBsAux1ZZUA==", 41 | "requires": {} 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pyramid", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "directories": { 7 | "test": "test" 8 | }, 9 | "scripts": { 10 | "test": "echo \"Error: no test specified\" && exit 1" 11 | }, 12 | "author": "", 13 | "license": "ISC", 14 | "dependencies": { 15 | "ws": "^7.3.1" 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /project.clj: -------------------------------------------------------------------------------- 1 | (defproject town.lilac/pyramid "4.0.0" 2 | :description "A library for storing and querying graph data in a Clojure map" 3 | :url "https://github.com/lilactown/pyramid" 4 | :scm {:name "git" :url "https://github.com/lilactown/pyramid"} 5 | :license {:name "Eclipse Public License" 6 | :url "http://www.eclipse.org/legal/epl-v20.html"} 7 | :source-paths ["src"] 8 | :dependencies [[edn-query-language/eql "1.0.1"] 9 | [town.lilac/cascade "2.0.1"]] 10 | :deploy-repositories [["snapshots" {:sign-releases false 11 | :url "https://clojars.org" 12 | :creds :gpg}]]) 13 | -------------------------------------------------------------------------------- /scratch.clj: -------------------------------------------------------------------------------- 1 | (require '[meander.epsilon :as m]) 2 | (require '[pyramid.core :as p]) 3 | 4 | (def data 5 | {:person/id 0 :person/name "Rachel" 6 | :friend/list [{:person/id 1 :person/name "Marco"} 7 | {:person/id 2 :person/name "Cassie"} 8 | {:person/id 3 :person/name "Jake"} 9 | {:person/id 4 :person/name "Tobias"} 10 | {:person/id 5 :person/name "Ax"}]}) 11 | 12 | (def data-2 13 | {:person/id 1 14 | :friend/best {:person/id 3 15 | :friend/best {:person/id 1}}}) 16 | 17 | 18 | (def data-3 19 | {:species {:andalites [{:person/id 5 20 | :person/species "andalite"}]}}) 21 | 22 | (def animorphs (p/db [data])) 23 | 24 | (def animorphs-2 25 | (p/add animorphs data-2)) 26 | (def animorphs-3 27 | (p/add animorphs-2 data-3)) 28 | 29 | (p/pull animorphs-3 [{[:person/id 1] [:person/name 30 | {:friend/best [:person/name]}]}]) 31 | ;; => {[:person/id 1] {:person/name "Marco", :friend/best #:person{:name "Jake"}}} 32 | 33 | 34 | (m/search animorphs-3 35 | {:person/id {?id {:person/name "Rachel"}}} 36 | [?id]) 37 | 38 | 39 | (require '[datascript.core :as ds]) 40 | 41 | (require '[datascript.db]) 42 | 43 | (require '[pyramid.pull :as pull]) 44 | 45 | 46 | (def ds-animorphs 47 | (-> {:person/id {:db/unique :db.unique/identity} 48 | :friend/best {:db/valueType :db.type/ref} 49 | :friend/list {:db/valueType :db.type/ref 50 | :db/cardinality :db.cardinality/many}} 51 | (ds/empty-db) 52 | (ds/db-with [data]) 53 | (ds/db-with [data-2]) 54 | (ds/db-with [data-3]))) 55 | 56 | 57 | (-> (ds/entity ds-animorphs [:person/id 1]) 58 | (get :person/name)) 59 | ;; => "Marco" 60 | 61 | (-> (ds/entity ds-animorphs [:person/id 1]) 62 | (get :friend/best) 63 | (get :person/name)) 64 | ;; => "Jake" 65 | 66 | 67 | (datascript.db/entid ds-animorphs [:person/id 1]) 68 | ;; => 2 69 | 70 | (datascript.db/-search ds-animorphs [(datascript.db/entid ds-animorphs [:person/id 1])]) 71 | ;; => (#datascript/Datom [2 :friend/best 4 536870914 true] #datascript/Datom [2 :person/id 1 536870913 true] #datascript/Datom [2 :person/name "Marco" 536870913 true]) 72 | 73 | 74 | (ds/pull ds-animorphs '[*] [:person/id 1]) 75 | ;; => {:db/id 2, :friend/best #:db{:id 4}, :person/id 1, :person/name "Marco"} 76 | 77 | 78 | (defn- resolve-attr [db a datoms] 79 | (if (datascript.db/multival? db a) 80 | (if (datascript.db/ref? db a) 81 | (reduce #(conj %1 [:db/id (:v %2)]) 82 | #{} datoms) 83 | (reduce #(conj %1 (:v %2)) #{} datoms)) 84 | (if (datascript.db/ref? db a) 85 | [:db/id (:v (first datoms))] 86 | (:v (first datoms))))) 87 | 88 | 89 | (defn resolve-datascript-entity 90 | [db lookup-ref] 91 | (let [eid (if (= :db/id (first lookup-ref)) 92 | (second lookup-ref) 93 | (datascript.db/entid db lookup-ref)) 94 | attrs (datascript.db/-search db [eid])] 95 | (into {} 96 | (for [attr (map second attrs) 97 | :let [datoms (datascript.db/-search db [eid attr])]] 98 | [attr (resolve-attr db attr datoms)])))) 99 | 100 | 101 | (resolve-datascript-entity ds-animorphs [:person/id 1]) 102 | ;; => {:friend/best [:db/id 4], :person/id 1, :person/name "Marco"} 103 | 104 | (resolve-datascript-entity ds-animorphs [:db/id 2]) 105 | ;; => {:friend/best [:db/id 4], :person/id 1, :person/name "Marco"} 106 | 107 | (extend-protocol pull/IPullable 108 | datascript.db.DB 109 | (resolve-ref 110 | ([db lookup-ref] (resolve-datascript-entity db lookup-ref)) 111 | ([db lookup-ref not-found] 112 | (or (pull/resolve-ref db lookup-ref) not-found)))) 113 | 114 | 115 | (p/pull ds-animorphs [[:person/id 0]]) 116 | ;; => {[:person/id 0] {:friend/list #{#:person{:id 5} #:person{:id 1} #:person{:id 2} #:person{:id 3} #:person{:id 4}}, :person/id 0, :person/name "Rachel"}} 117 | 118 | (p/pull ds-animorphs [{[:person/id 1] [:person/name 119 | {:friend/best [:person/name]}]}]) 120 | ;; => {[:person/id 1] {:person/name "Marco", :friend/best #:person{:name "Jake"}}} 121 | 122 | 123 | (def query-1 124 | '[:find ?name ?best-friend 125 | :where 126 | [?e :friend/best ?friend] 127 | [?e :person/name ?name] 128 | [?friend :person/name ?best-friend]]) 129 | 130 | (time 131 | (ds/q 132 | query-1 133 | ds-animorphs)) 134 | 135 | (require '[pyramid.query :as p.q]) 136 | 137 | 138 | (time (p.q/q query-1 animorphs-3)) 139 | 140 | 141 | (def query-2 142 | '[:find ?name ?friend 143 | :where 144 | [?e :friend/list ^:many ?friends] 145 | [?e :person/name ?name] 146 | [?friends :person/name ?friend]]) 147 | 148 | (time (ds/q query-2 ds-animorphs)) 149 | 150 | (time (p.q/q query-2 animorphs-3)) 151 | 152 | 153 | (time 154 | (p.q/q 155 | '[:find ?e ?v 156 | :where 157 | [?e :friend/list ^:many ?v]] 158 | animorphs-3)) 159 | 160 | 161 | (p.q/q 162 | '[:find ?name ?best-friend 163 | :where 164 | [?e :friend/best ?bf] 165 | [?e :person/name ?name] 166 | [?bf :person/name ?best-friend]] 167 | animorphs-3) 168 | 169 | 170 | (require '[asami.core :as a]) 171 | 172 | 173 | (a/create-database "asami:mem://pyramid-example2") 174 | 175 | 176 | (def conn (a/connect "asami:mem://pyramid-example2")) 177 | 178 | 179 | (require '[asami.graph :as ag]) 180 | 181 | 182 | (a/transact 183 | conn {:tx-data 184 | [(-> data 185 | (assoc :db/id -1) 186 | (update :friend/list 187 | (partial map-indexed #(assoc %2 :db/id (- -2 %1))))) 188 | (-> data-2 189 | (assoc :db/id -2) 190 | (update :friend/best assoc :db/id -4)) 191 | (assoc-in data-3 [:species :andalites 0 :db/id] -6)]}) 192 | 193 | (a/db conn) 194 | 195 | (ag/resolve-triple (a/graph (a/db conn)) '?e :person/id 0) 196 | ;; => ([:tg/node-26575]) 197 | 198 | (ag/resolve-triple (a/graph (a/db conn)) :tg/node-26575 '?a '?v) 199 | ;; => ([:person/id 0] [:person/name "Rachel"] [:tg/owns :tg/node-26581] [:tg/owns :tg/node-26583] [:tg/owns :tg/node-26585] [:tg/owns :tg/node-26577] [:tg/owns :tg/node-26576] [:tg/owns :tg/node-26579] [:friend/list :tg/node-26576] [:db/ident :tg/node-26575] [:tg/entity true]) 200 | 201 | (->> (ag/resolve-triple (a/graph (a/db conn)) :tg/node-26576 '?a '?v) 202 | (filter #(= :tg/contains (first %))) 203 | (map #(ag/resolve-triple 204 | (a/graph (a/db conn)) 205 | (second %) '?a '?v))) 206 | 207 | (require '[clojure.string :as string]) 208 | 209 | (defn- resolve-asami-node 210 | [graph eid] 211 | #_(prn eid) 212 | (let [datoms (ag/resolve-triple graph eid '?a '?v)] 213 | (cond 214 | (some #(= :tg/contains (first %)) datoms) 215 | (->> datoms 216 | (keep #(when (= :tg/contains (first %)) 217 | (second %))) 218 | (map #(resolve-asami-node graph %))) 219 | 220 | #_(some #(= [:tg/entity true] %) datoms) 221 | :else 222 | (into 223 | {} 224 | (comp 225 | (remove #(= :tg/owns (first %))) 226 | (map (fn [[k v]] 227 | (if (and (keyword? v) 228 | (not= :db/ident k) 229 | (string/starts-with? (str v) ":tg/node-")) 230 | [k (resolve-asami-node graph v)] 231 | [k v])))) 232 | datoms)))) 233 | 234 | (defn resolve-asami-entity 235 | [db lookup-ref] 236 | (let [graph (a/graph db) 237 | [[eid]] (ag/resolve-triple graph '?e (nth lookup-ref 0) (nth lookup-ref 1)) 238 | #_#_datoms (ag/resolve-triple graph eid '?a '?v)] 239 | #_(into {} (remove #(= :tg/owns (first %))) datoms) 240 | (resolve-asami-node graph eid))) 241 | 242 | (resolve-asami-entity (a/db conn) [:person/id 0]) 243 | 244 | (a/q '[:find ?e 245 | :where [?e :person/id 1]] 246 | (a/db conn)) 247 | 248 | 249 | (def data [{:book/id 1 250 | :book/title "Title 1" 251 | :book/author {:author/id 1 252 | :author/name "Author 1"}} 253 | {:book/id 2 254 | :book/title "Title 2" 255 | :book/author {:author/id 2 256 | :author/name "Author 2"}}]) 257 | 258 | (def db (apply p/add {} data)) 259 | 260 | 261 | (defn pull-books 262 | [book-ids] 263 | (for [book-id book-ids] 264 | (-> db 265 | (p/pull [{[:book/id book-id] 266 | [:book/id :book/title 267 | {:book/author [:author/id :author/name]}]}]) 268 | (get [:book/id book-id])))) 269 | 270 | (pull-books [1 2]) 271 | ;; => (#:book{:id 1, :title "Title 1", :author #:author{:id 1, :name "Author 1"}} #:book{:id 2, :title "Title 2", :author #:author{:id 2, :name "Author 2"}}) 272 | 273 | 274 | (def data {:books [{:book/id 1 275 | :book/title "Title 1" 276 | :book/author {:author/id 1 277 | :author/name "Author 1"}} 278 | {:book/id 2 279 | :book/title "Title 2" 280 | :book/author {:author/id 2 281 | :author/name "Author 2"}}]}) 282 | 283 | (def db (p/add {} data)) 284 | 285 | (p/pull db [{:books [:book/id :book/title 286 | {:book/author [:author/id :author/name]}]}]) 287 | ;; => {:books [#:book{:id 1, :title "Title 1", :author #:author{:id 1, :name "Author 1"}} #:book{:id 2, :title "Title 2", :author #:author{:id 2, :name "Author 2"}}]} 288 | 289 | 290 | -------------------------------------------------------------------------------- /site/.clj-kondo/config.edn: -------------------------------------------------------------------------------- 1 | {:lint-as {helix.core/defnc clojure.core/defn 2 | helix.core/defhook clojure.core/defn 3 | helix.core/fnc clojure.core/fn}} 4 | -------------------------------------------------------------------------------- /site/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | public/js 3 | .cpcache 4 | .shadow-cljs 5 | /site/.clj-kondo/.cache/ 6 | -------------------------------------------------------------------------------- /site/deps.edn: -------------------------------------------------------------------------------- 1 | {:paths ["src"] 2 | :deps {binaryage/devtools {:mvn/version "1.0.3"} 3 | town.lilac/pyramid {:local/root "../"} 4 | lilactown/helix {:mvn/version "0.1.1"} 5 | nextjournal.clojure-mode/nextjournal.clojure-mode 6 | {:git/url "https://github.com/nextjournal/clojure-mode" 7 | :sha "a83c87cd2bd2049b70613f360336a096d15c5518"} 8 | org.clojure/tools.reader {:mvn/version "1.3.6"} 9 | thheller/shadow-cljs {:mvn/version "2.15.3"}}} 10 | -------------------------------------------------------------------------------- /site/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "site", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1", 8 | "css": "postcss public/css/app.css -o public/css/app.compiled.css", 9 | "watch:css": "postcss public/css/app.css -o public/css/app.compiled.css -w", 10 | "release": "shadow-cljs release --verbose app && NODE_ENV=production npm run css" 11 | }, 12 | "author": "", 13 | "license": "ISC", 14 | "devDependencies": { 15 | "autoprefixer": "^10.3.1", 16 | "postcss": "^8.3.6", 17 | "postcss-cli": "^8.3.1", 18 | "shadow-cljs": "^2.14.4", 19 | "tailwindcss": "^2.2.7" 20 | }, 21 | "dependencies": { 22 | "@codemirror/autocomplete": "0.18.8", 23 | "@codemirror/closebrackets": "0.18.0", 24 | "@codemirror/commands": "0.18.3", 25 | "@codemirror/comment": "0.18.1", 26 | "@codemirror/fold": "0.18.2", 27 | "@codemirror/gutter": "0.18.4", 28 | "@codemirror/highlight": "0.18.4", 29 | "@codemirror/history": "0.18.1", 30 | "@codemirror/language": "0.18.2", 31 | "@codemirror/lint": "0.18.6", 32 | "@codemirror/matchbrackets": "0.18.0", 33 | "@codemirror/rectangular-selection": "0.18.1", 34 | "@codemirror/search": "0.18.4", 35 | "@codemirror/state": "0.18.7", 36 | "@codemirror/view": "0.18.19", 37 | "lezer-clojure": "0.1.10", 38 | "lezer-generator": "^0.12.0", 39 | "react": "^18.0.0-alpha-88d121899-20210811", 40 | "react-dom": "^18.0.0-alpha-88d121899-20210811", 41 | "react-refresh": "^0.11.0-alpha-82583617b-20210812" 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /site/postcss.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | plugins: { 3 | tailwindcss: {}, 4 | autoprefixer: {}, 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /site/public/css/app.css: -------------------------------------------------------------------------------- 1 | @tailwind base; 2 | @tailwind components; 3 | @tailwind utilities; 4 | 5 | 6 | .code { 7 | font-family: "Fira Code"; 8 | } 9 | -------------------------------------------------------------------------------- /site/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 |
10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /site/shadow-cljs.edn: -------------------------------------------------------------------------------- 1 | {:deps true 2 | :dev-http{8000 "public"} 3 | :builds {:app {:target :browser 4 | :output-dir "public/js" 5 | :asset-path "/js" 6 | :modules {:main {:entries [pyramid.site.core] 7 | :init-fn pyramid.site.core/start}}}}} 8 | -------------------------------------------------------------------------------- /site/src/helix/core/alpha.cljs: -------------------------------------------------------------------------------- 1 | (ns helix.core.alpha 2 | (:require 3 | ["react" :as react])) 4 | 5 | 6 | (defn with-transition 7 | ([f] (with-transition react/startTransition f)) 8 | ([start-transition f] 9 | (fn [& args] 10 | (start-transition #(apply f args))))) 11 | 12 | 13 | (def use-transition 14 | react/useTransition) 15 | -------------------------------------------------------------------------------- /site/src/pyramid/site/codemirror.cljs: -------------------------------------------------------------------------------- 1 | (ns pyramid.site.codemirror 2 | (:require 3 | ["@codemirror/closebrackets" :refer [closeBrackets]] 4 | ["@codemirror/fold" :as fold] 5 | ["@codemirror/gutter" :refer [lineNumbers]] 6 | ["@codemirror/highlight" :as highlight] 7 | ["@codemirror/history" :refer [history historyKeymap]] 8 | ["@codemirror/state" :refer [EditorState]] 9 | ["@codemirror/view" :as view :refer [EditorView]] 10 | ["lezer" :as lezer] 11 | ["lezer-generator" :as lg] 12 | ["lezer-tree" :as lz-tree] 13 | [clojure.string :as string] 14 | [helix.core :refer [defnc $]] 15 | [helix.dom :as d] 16 | [helix.hooks :as hooks] 17 | [nextjournal.clojure-mode :as cm-clj] 18 | [nextjournal.clojure-mode.extensions.close-brackets :as close-brackets] 19 | [nextjournal.clojure-mode.extensions.formatting :as format] 20 | [nextjournal.clojure-mode.extensions.selection-history :as sel-history] 21 | [nextjournal.clojure-mode.keymap :as keymap] 22 | [nextjournal.clojure-mode.live-grammar :as live-grammar] 23 | [nextjournal.clojure-mode.node :as n] 24 | [nextjournal.clojure-mode.selections :as sel])) 25 | 26 | 27 | (def theme 28 | (.theme 29 | EditorView 30 | #js {".cm-content" #js {:white-space "pre-wrap" 31 | :padding "10px 8px" 32 | :min-height "100%"} 33 | ".cm-line" #js {:font-family "Fira Code" 34 | :font-size "0.8rem"} 35 | "&.cm-focused" #js {:outline "none"} 36 | ".cm-gutters" #js {:background "transparent" 37 | :border "none"} 38 | ".cm-gutterElement" #js {:margin-left "5px"}})) 39 | 40 | 41 | (def extensions 42 | #js [theme 43 | (history) 44 | highlight/defaultHighlightStyle 45 | (view/drawSelection) 46 | #_(fold/foldGutter) 47 | (.. EditorState -allowMultipleSelections (of true)) 48 | cm-clj/default-extensions 49 | (.of view/keymap cm-clj/complete-keymap) 50 | (.of view/keymap historyKeymap)]) 51 | 52 | 53 | (defn new-cm 54 | [{:keys [parent initial-value on-change]}] 55 | (EditorView. 56 | #js {:state 57 | (.create 58 | EditorState 59 | #js {:doc initial-value 60 | :extensions 61 | (cond-> extensions 62 | ;; readonly 63 | (nil? on-change) 64 | (.concat 65 | (-> EditorView 66 | (.-editable) 67 | (.of false))) 68 | 69 | (some? on-change) 70 | (.concat 71 | (-> EditorView 72 | (.-updateListener) 73 | (.of 74 | (fn [^js update] 75 | (when (.-docChanged update) 76 | (-> (.. update -state -doc toString) 77 | (on-change))))))))}) 78 | :parent parent})) 79 | 80 | 81 | (defnc editor 82 | [{:keys [initial-value on-change value style]}] 83 | (let [cm-instance (hooks/use-ref nil) 84 | cm-mount (hooks/use-callback 85 | :once 86 | #(when % 87 | (reset! 88 | cm-instance 89 | (new-cm 90 | {:parent % 91 | :initial-value (or initial-value "") 92 | :on-change on-change}))))] 93 | (hooks/use-effect 94 | :once 95 | ;; on unmount 96 | #(when-let [^js cm @cm-instance] 97 | (.destroy cm))) 98 | 99 | (hooks/use-layout-effect 100 | [value] 101 | (when value 102 | (when-let [cm @cm-instance] 103 | (when (not= (string/join "\n" (.. cm -state -doc -text)) 104 | value) 105 | (let [tx (-> (.-state cm) 106 | (.update 107 | #js {:changes 108 | ;; replace entire text with value 109 | #js {:from 0 110 | :to (.. cm -state -doc -length) 111 | :insert value}}))] 112 | (.dispatch cm tx)))))) 113 | (d/div 114 | {:class "min-h-full" 115 | :style style 116 | :ref cm-mount}))) 117 | -------------------------------------------------------------------------------- /site/src/pyramid/site/components.cljs: -------------------------------------------------------------------------------- 1 | (ns pyramid.site.components 2 | (:require 3 | [helix.core :refer [defnc $]] 4 | [helix.dom :as d])) 5 | 6 | 7 | (def ^:private vconj (fnil conj [])) 8 | 9 | (defnc button 10 | [{:keys [disabled] :as props}] 11 | (d/button 12 | {:& (update 13 | props 14 | :class 15 | vconj 16 | (if disabled 17 | "bg-gray-400" 18 | "bg-blue-400") 19 | "px-2" 20 | "py-1" 21 | (when-not disabled "shadow") 22 | "text-white")})) 23 | 24 | 25 | (defnc tab 26 | [{:keys [active? class on-click children]}] 27 | (d/div 28 | {:on-click on-click 29 | :class (into ["border-b-2 border-solid" 30 | (if active? 31 | "border-blue-400" 32 | "border-gray-300") 33 | "cursor-pointer"] class)} 34 | children)) 35 | 36 | 37 | (defnc title-bar 38 | [props] 39 | (d/div 40 | {:& (-> props 41 | (update :class vconj 42 | "p-1 px-3 bg-gray-200 border-l-4 border-solid border-gray-300"))})) 43 | 44 | 45 | (defnc pane 46 | [{:keys [title title-class class style children]}] 47 | (d/div 48 | {:class class} 49 | ($ title-bar 50 | {:class title-class} 51 | (d/h3 52 | {:class ["text-lg"] 53 | :style style} 54 | title)) 55 | children)) 56 | 57 | 58 | (defnc writable-pane 59 | [props] 60 | ($ pane 61 | {:& (-> props 62 | (update :class vconj "shadow" "border" "border-solid" "border-blue-300") 63 | (update :title-class vconj "bg-blue-100 border-blue-300 text-blue-500"))})) 64 | 65 | 66 | (defnc read-only-pane 67 | [props] 68 | ($ pane 69 | {:& (-> props 70 | (update :class vconj "border" "border-solid" "border-gray-400") 71 | (update :title-class vconj "border-gray-400 text-gray-500"))})) 72 | -------------------------------------------------------------------------------- /site/src/pyramid/site/core.cljs: -------------------------------------------------------------------------------- 1 | (ns pyramid.site.core 2 | (:require 3 | ["react-dom" :as rdom] 4 | [pyramid.core :as a] 5 | [pyramid.query :as a.query] 6 | [pyramid.site.codemirror :as site.cm] 7 | [pyramid.site.components :as c] 8 | [pyramid.site.tree :as tree] 9 | [cljs.repl :as repl] 10 | [clojure.pprint :as pp] 11 | [clojure.string :as string] 12 | [clojure.tools.reader.edn :as edn] 13 | [goog.functions :as gfn] 14 | [helix.core :refer [defnc $ <>]] 15 | [helix.core.alpha :as hx.alpha] 16 | [helix.dom :as d] 17 | [helix.hooks :as hooks])) 18 | 19 | 20 | (def initial-data 21 | (let [friends '([:person/id 0] 22 | [:person/id 1] 23 | [:person/id 2] 24 | [:person/id 3] 25 | [:person/id 4] 26 | [:person/id 5])] 27 | [{:person/id 0 :person/name "Rachel" 28 | :friend/list (remove #{[:person/id 0]} friends)} 29 | {:person/id 1 30 | :person/name "Marco" 31 | :friend/best [:person/id 3] 32 | :friend/list (remove #{[:person/id 1]} friends)} 33 | {:person/id 2 :person/name "Cassie" 34 | :friend/list (remove #{[:person/id 2]} friends)} 35 | {:person/id 3 36 | :person/name "Jake" 37 | :friend/best [:person/id 1] 38 | :friend/list (remove #{[:person/id 3]} friends)} 39 | {:person/id 4 :person/name "Tobias" 40 | :friend/best {:person/id 5} 41 | :friend/list (remove #{[:person/id 4]} friends)} 42 | {:person/id 5 :person/name "Ax" 43 | :friend/best {:person/id 4} 44 | :friend/list (remove #{[:person/id 5]} friends)} 45 | {:species {:andalites [[:person/id 5]]}} 46 | {:a 47 | {:deeply 48 | {:nested 49 | {:map 50 | {:of 51 | {:very 52 | {:important 53 | {:data [:person/id 0]}}}}}}}}])) 54 | 55 | 56 | (defnc db-add-input 57 | [{:keys [on-add]}] 58 | (let [[data set-data] (hooks/use-state "")] 59 | (d/div 60 | ($ c/writable-pane 61 | ($ site.cm/editor 62 | {:value data 63 | :on-change set-data})) 64 | ($ c/button 65 | {:on-click (fn [_] 66 | (set-data "") 67 | (on-add (edn/read-string 68 | {:default tagged-literal} 69 | data)))} 70 | "Add")))) 71 | 72 | 73 | (defnc query-explorer 74 | [{:keys [db query set-query query-type set-query-type]}] 75 | (let [[result-pending? start-result] (hx.alpha/use-transition) 76 | set-query (hooks/use-memo 77 | :once 78 | (gfn/debounce set-query 400)) 79 | result (hooks/use-memo 80 | [db query] 81 | (-> (try 82 | (let [query-string (edn/read-string 83 | {:default tagged-literal} 84 | query)] 85 | (with-out-str 86 | (cond-> query-string 87 | (= :pull query-type) (->> (a/pull db)) 88 | (= :datalog query-type) (a.query/q db) 89 | true (pp/pprint)))) 90 | (catch js/Error e 91 | (with-out-str 92 | (pp/pprint (cljs.repl/Error->map e))))) 93 | (string/trim)))] 94 | ;; simulate long render 95 | ;; (doseq [x (range 10000) y (range 10000)] (* x y)) 96 | (d/div 97 | (d/div 98 | {:class "flex gap-2" 99 | :style {:height "75vh"}} 100 | ($ c/writable-pane 101 | {:title (d/div 102 | {:class "flex items-center"} 103 | (d/div {:class "flex-1"} "Query") 104 | (d/div 105 | {:class "text-sm"} 106 | (d/select 107 | {:on-change #(-> (.. % -target -value) 108 | (keyword) 109 | (set-query-type)) 110 | :value (name query-type)} 111 | (d/option {:value "pull"} "Pull") 112 | (d/option {:value "datalog"} "Datalog")))) 113 | :class ["flex-1 min-h-full overflow-scroll"]} 114 | ($ site.cm/editor 115 | {:initial-value query 116 | :on-change (hx.alpha/with-transition 117 | start-result 118 | set-query)})) 119 | ($ c/read-only-pane 120 | {:title "Result" 121 | :class ["flex-1 transition-opacity delay-200 duration-400" 122 | (if result-pending? 123 | "opacity-20" 124 | "opacity-100") 125 | "overflow-scroll"]} 126 | ($ site.cm/editor 127 | {:value result})))))) 128 | 129 | 130 | (defnc database-explorer 131 | [{:keys [db]}] 132 | ($ c/read-only-pane 133 | {:title "Database explorer"} 134 | ($ tree/data-tree {:data db}))) 135 | 136 | 137 | (defnc database-editor 138 | [{:keys [db set-db]}] 139 | ;; simulate long render 140 | ;; (doseq [x (range 10000) y (range 10000)] (* x y)) 141 | (let [db-string (hooks/use-memo 142 | [db] 143 | (string/trim 144 | (with-out-str 145 | (pp/pprint db)))) 146 | dbnc-set-db (hooks/use-memo 147 | [set-db] 148 | (gfn/debounce set-db 400)) 149 | ;; this is used to remount the editor when we transact changes 150 | ;; to the db data 151 | [editor-inst set-inst] (hooks/use-state 0)] 152 | ($ c/writable-pane 153 | {:title "Database"} 154 | (d/div 155 | {:class "overflow-scroll" 156 | :style {:max-height "65vh"}} 157 | ($ site.cm/editor 158 | {:key editor-inst 159 | :initial-value db-string 160 | :on-change 161 | #(when-not (= db-string %) 162 | (try 163 | (dbnc-set-db (edn/read-string {:default tagged-literal} %)) 164 | (catch js/Error e 165 | (js/console.error e))))})) 166 | ($ c/button 167 | {:on-click #(do 168 | (set-db {}) 169 | (set-inst inc)) 170 | :class ["m-1"]} 171 | "Reset") 172 | ($ c/button 173 | {:on-click (fn [_] 174 | (set-db #(a/db [%])) 175 | (set-inst inc))} 176 | "Normalize") 177 | #_(d/div 178 | {:class "py-2"} 179 | ($ db-add-input {:on-add #(do 180 | (set-db a/add %) 181 | (set-inst inc))}))))) 182 | 183 | 184 | (defnc app 185 | [] 186 | (let [[screen set-screen] (hooks/use-state :query) 187 | [db set-db] (hooks/use-state (a/db initial-data)) 188 | [query set-query] (hooks/use-state "[]") 189 | [query-type set-query-type] (hooks/use-state :pull) 190 | [nav-pending? start-nav] (hx.alpha/use-transition)] 191 | (d/div 192 | {:class "container mx-auto p-3"} 193 | (d/div 194 | {:class "pb-1 flex gap-1"} 195 | (d/h1 196 | {:class "text-xl ml-1 mr-8 my-2 border-solid border-blue-400"} 197 | "Pyramid " 198 | (d/small {:class "italic"} "Playground")) 199 | ($ c/tab 200 | {:on-click (hx.alpha/with-transition 201 | start-nav 202 | #(set-screen :query)) 203 | :active? (= :query screen) 204 | :class ["mx-1 my-2"]} 205 | "Query") 206 | ($ c/tab 207 | {:on-click (hx.alpha/with-transition 208 | start-nav 209 | #(set-screen :db-explorer)) 210 | :active? (= :db-explorer screen) 211 | :class ["mx-1 my-2"]} 212 | "Database Explorer") 213 | ($ c/tab 214 | {:on-click (hx.alpha/with-transition 215 | start-nav 216 | #(set-screen :db-editor)) 217 | :active? (= :db-editor screen) 218 | :class ["mx-1 my-2"]} 219 | "Database Editor") 220 | (d/span 221 | {:class ["transition-opacity delay-200 duration-400 select-none" 222 | (if nav-pending? "opacity-70" "opacity-0")]} 223 | "Loading...")) 224 | (d/div 225 | {:class "p-1"} 226 | (case screen 227 | :query ($ query-explorer 228 | {:db db 229 | :query query 230 | :set-query set-query 231 | :query-type query-type 232 | :set-query-type set-query-type}) 233 | :db-explorer ($ database-explorer 234 | {:db db}) 235 | :db-editor ($ database-editor 236 | {:db db 237 | :set-db set-db}) 238 | nil))))) 239 | 240 | 241 | (defonce root (rdom/createRoot (js/document.getElementById "app"))) 242 | 243 | 244 | (defn ^:dev/after-load start 245 | [] 246 | (.render root ($ app))) 247 | -------------------------------------------------------------------------------- /site/src/pyramid/site/tree.cljs: -------------------------------------------------------------------------------- 1 | (ns pyramid.site.tree 2 | (:require 3 | [pyramid.ident :as a.ident] 4 | [pyramid.site.components :as c] 5 | [clojure.pprint :as pp] 6 | [helix.core :refer [defnc $]] 7 | [helix.core.alpha :as hx.alpha] 8 | [helix.dom :as d] 9 | [helix.hooks :as hooks])) 10 | 11 | 12 | (defnc expandable-key 13 | [{:keys [data expanded? on-click]}] 14 | (d/div 15 | {:class ["px-2 py-0.5" 16 | (when-not expanded? 17 | "shadow") 18 | (if expanded? 19 | "bg-blue-300" 20 | "bg-gray-300") 21 | "hover:bg-blue-200" 22 | "cursor-pointer"] 23 | :on-click on-click} 24 | (d/code {:class "code"} (pr-str data)))) 25 | 26 | 27 | (defnc leaf 28 | [{:keys [data set-top-expanded]}] 29 | (let [[expanded set-expanded] (hooks/use-state nil)] 30 | (d/div 31 | {:class "flex flex-col gap-1"} 32 | (if-not (map? data) 33 | (cond 34 | (not (seqable? data)) 35 | (d/pre 36 | (d/code 37 | {:class "code bg-yellow-100"} 38 | (with-out-str (pp/pprint data)))) 39 | 40 | (a.ident/ident? data) 41 | ($ expandable-key 42 | {:on-click #(set-top-expanded data) 43 | :data data}) 44 | 45 | (and (sequential? data) 46 | (every? a.ident/ident? data)) 47 | (for [ident data] 48 | ($ expandable-key 49 | {:key (str ident) 50 | :on-click #(set-top-expanded ident) 51 | :data ident})) 52 | 53 | :else (d/pre 54 | (d/code 55 | {:class "code bg-yellow-100"} 56 | (with-out-str (pp/pprint data))))) 57 | (for [[k v] data] 58 | (d/div 59 | {:class "flex" 60 | :key (str k)} 61 | (d/div 62 | ($ expandable-key 63 | {:data k 64 | :expanded? (= k expanded) 65 | :on-click (if (= k expanded) 66 | #(set-expanded nil) 67 | #(set-expanded k))})) 68 | (d/div 69 | {:class "px-2"} 70 | (if (= k expanded) 71 | ($ leaf {:data v :set-top-expanded set-top-expanded}) 72 | (d/pre 73 | (d/code 74 | {:class "code"} 75 | (with-out-str (pp/pprint v)))))))))))) 76 | 77 | 78 | (defnc data-tree 79 | [{:keys [data]}] 80 | (let [[expanded set-expanded] (hooks/use-state nil) 81 | [expand-pending? start-expand] (hx.alpha/use-transition)] 82 | (d/div 83 | {:class ["flex p-2 gap-1 flex-col"]} 84 | (if (empty? data) 85 | (d/span {:class "italic"} "No data") 86 | (let [entities (for [[k v] data 87 | :when (map? v) 88 | [k2 v2] v 89 | :when (and (a.ident/ident? [k k2]) 90 | (map? v2))] 91 | [[k k2] v2]) 92 | non-entities (for [[k v] data 93 | :when (not 94 | (contains? (set (map ffirst entities)) k))] 95 | [k v])] 96 | (helix.core/<> 97 | ($ c/title-bar {:class ["mb-1"]} "Entities") 98 | (for [[ident v] entities 99 | :let [pstr (pr-str v) 100 | expanded? (= expanded ident)]] 101 | (d/div 102 | {:key (str ident) 103 | :class "flex"} 104 | ($ expandable-key 105 | {:expanded? expanded? 106 | :on-click (hx.alpha/with-transition 107 | start-expand 108 | (if expanded? 109 | #(set-expanded nil) 110 | #(set-expanded ident))) 111 | :data ident}) 112 | (d/div 113 | {:class ["flex-1 px-2" 114 | "overflow-scroll"]} 115 | (if expanded? 116 | ($ leaf {:data v 117 | :set-top-expanded (hx.alpha/with-transition 118 | start-expand 119 | set-expanded)}) 120 | (d/code 121 | {:class "max-w-full code"} 122 | (d/pre (d/code {:class "code"} pstr))))))) 123 | ($ c/title-bar {:class ["my-2"]} "Other") 124 | ($ leaf {:data (into {} non-entities) 125 | :set-top-expanded (hx.alpha/with-transition 126 | start-expand 127 | set-expanded)}))))))) 128 | -------------------------------------------------------------------------------- /site/tailwind.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | purge: [ 3 | './src/**/*.cljs' 4 | ], 5 | darkMode: false, // or 'media' or 'class' 6 | theme: { 7 | extend: {}, 8 | }, 9 | variants: { 10 | extend: {}, 11 | }, 12 | plugins: [], 13 | } 14 | -------------------------------------------------------------------------------- /src/pyramid/core.cljc: -------------------------------------------------------------------------------- 1 | (ns pyramid.core 2 | "A library for storing graph data in a Clojure map that automatically 3 | normalizes nested data and allows querying via EQL. 4 | 5 | Create a new db with `db`. db values are normal maps with a tabular structure. 6 | 7 | Add new ones using `add`. 8 | 9 | Entities are identified by the first key with the name \"id\", e.g. 10 | :person/id. Adding data about the same entity will merge them together in 11 | order of addition. To replace an entity, `dissoc` it first. 12 | 13 | Query them w/ EQL using `pull`. 14 | 15 | To get meta-information about what entities were added or queried, use the 16 | `add-report` and `pull-report` functions." 17 | (:require 18 | [cascade.hike :as h] 19 | [pyramid.ident :as ident] 20 | [pyramid.pull :as pull] 21 | [pyramid.query :as query] 22 | [clojure.set] 23 | [edn-query-language.core :as eql])) 24 | 25 | 26 | (def default-ident 27 | (fn ident-by-id [entity] 28 | (loop [kvs entity] 29 | (when-some [[k v] (first kvs)] 30 | (if (and 31 | (keyword? k) 32 | (= "id" (name k))) 33 | (ident/ident k v) 34 | (recur (rest kvs))))))) 35 | 36 | 37 | (defn- lookup-ref-of 38 | [identify entity] 39 | (let [identify (if-some [ident-key (get (meta entity) :db/ident)] 40 | (ident/by-keys ident-key) 41 | identify)] 42 | (identify entity))) 43 | 44 | 45 | (defn- entity-map? 46 | [identify x] 47 | (and (map? x) 48 | (some? (lookup-ref-of identify x)))) 49 | 50 | 51 | (defn- map-entry 52 | [mk mv] 53 | #?(:bb (first {mk mv}) 54 | :clj (clojure.lang.MapEntry/create mk mv) 55 | :cljs (cljs.core/MapEntry. mk mv nil))) 56 | 57 | 58 | ;; 59 | ;; performance-specific code ahead 60 | ;; 61 | 62 | 63 | #?(:clj 64 | (defn- fast-assoc 65 | {:inline 66 | (fn [m k v] 67 | (if (symbol? m) 68 | `(.assoc ~(with-meta m {:tag "clojure.lang.Associative"}) ~k ~v) 69 | `(let [m# ~m] (fast-assoc m# ~k ~v))))} 70 | [^clojure.lang.Associative m k v] 71 | (.assoc m k v))) 72 | 73 | 74 | (defn- update-ref 75 | ([m [k ek] f x] 76 | (let [em (or (get m k) {}) 77 | v (get em ek)] 78 | #?(:bb (assoc m k 79 | (assoc em ek 80 | (f v x))) 81 | :clj (fast-assoc m k 82 | (fast-assoc em ek 83 | (f v x))) 84 | :cljs (assoc m k 85 | (assoc em ek 86 | (f v x))))))) 87 | 88 | 89 | (defn- merge-entity 90 | [e #?(:clj ^clojure.lang.IKVReduce m :cljs ^IKVReduce m)] 91 | (if (nil? e) 92 | m 93 | (if (nil? m) 94 | e 95 | #?(:bb (reduce-kv assoc e m) 96 | :clj (.kvreduce m fast-assoc e) 97 | :cljs (-kv-reduce m assoc e))))) 98 | 99 | 100 | (defn add-report* 101 | "For normal usage, see `pyramid.core/add-report`. 102 | 103 | Takes a normalized map `db` and some new `data`. 104 | Returns a 0-arity function (thunk) which, when called, will return either 105 | another thunk or a map containing the keys: 106 | - :db - the data normalized and merged into `db` 107 | - :entities - a set of entities found in `data` 108 | 109 | Each thunk should be called to continue the process until the result is 110 | returned. See `clojure.core/trampoline`." 111 | [db data] 112 | (let [identify (:db/ident (meta db) default-ident) 113 | *db (volatile! db) 114 | *entities (volatile! (transient #{})) 115 | process! (fn process! [x] 116 | (if (map? x) 117 | (if-some [lookup-ref (lookup-ref-of identify x)] 118 | (do 119 | #_(prn :lookup-ref lookup-ref) 120 | (vswap! *db update-ref lookup-ref merge-entity x) 121 | #_(prn :*db) 122 | (vswap! *entities conj! lookup-ref) 123 | #_(prn :*entities) 124 | lookup-ref) 125 | x) 126 | x))] 127 | #_(prn :add-report*) 128 | (h/walk 129 | (fn inner [k x] 130 | (if (map-entry? x) 131 | ;; skip processing map keys 132 | (h/walk 133 | inner 134 | (fn outer-map-entry 135 | [v] 136 | (k (map-entry 137 | (key x) 138 | (process! v)))) 139 | (val x)) 140 | ;; regular c/postwalk 141 | (h/walk inner (comp k process!) x))) 142 | ;; we've traversed all other elements, now process the top-level map 143 | ;; and return the results. 144 | (fn outer [d] 145 | #_(prn :outer d) 146 | (let [data' (process! d) 147 | em? (entity-map? identify data)] 148 | #_(prn :done-process!) 149 | {:entities (persistent! @*entities) 150 | :indices (if em? 151 | #{} 152 | (set (keys data'))) 153 | :db (if em? 154 | @*db 155 | ;; data isn't an entity map, so we assoc each key in data. 156 | ;; they act like one-off custom indexes and can be pulled later 157 | (merge @*db data'))})) 158 | data))) 159 | 160 | 161 | (defn add-report 162 | "Takes a normalized map `db`, and some new `data`. 163 | Returns a map containing keys: 164 | :db - the data normalized and merged into `db`. 165 | :entities - a set of entities found in `data`" 166 | [db data] 167 | (trampoline add-report* db data)) 168 | 169 | 170 | #_(add-report 171 | {} 172 | {:foo {:id 1 173 | '(:foo {:id "bar"}) {:id 2}} 174 | :bar {:baz [{:id 3}]}}) 175 | 176 | 177 | (defn add 178 | "Takes a normalized map `db`, and some new `data`. 179 | 180 | Returns a new map with the data normalized and merged into `db`." 181 | ([db data] 182 | (:db (add-report db data))) 183 | ([db data & more] 184 | (reduce add (add db data) more))) 185 | 186 | 187 | (defn entities 188 | "Returns a lazy seq of all entity maps in the DB" 189 | [db] 190 | (let [identify (get (meta db) :db/ident default-ident)] 191 | (->> (vals db) ; skip the first layer of table IDs 192 | (mapcat #(when (satisfies? query/IQueryable %) 193 | (query/entities %))) 194 | (filter identify)))) 195 | 196 | 197 | (defn db 198 | "Takes an optional collection of `entities`. 199 | 200 | Returns a new map with the `entities` normalized." 201 | ([] {}) 202 | ([entities] 203 | (db entities default-ident)) 204 | ([entities identify] 205 | (reduce 206 | add 207 | (with-meta {} {:db/ident identify 208 | `query/entities entities}) 209 | entities))) 210 | 211 | 212 | (defn identify 213 | [db data] 214 | (let [identify (:db/ident (meta db) default-ident)] 215 | (lookup-ref-of identify data))) 216 | 217 | 218 | (defn pull-report 219 | "Executes an EQL query against a normalized map `db`. Returns a map with the 220 | following keys: 221 | :data - the result of the query 222 | :entities - a set of lookup refs that were visited during the query" 223 | [db query] 224 | (trampoline pull/pull-report db query)) 225 | 226 | 227 | (defn pull 228 | "Executes an EQL query against a normalized map `db` and returns the result." 229 | [db query] 230 | (:data (pull-report db query))) 231 | 232 | 233 | (defn- delete-nested-entity 234 | [lookup-ref v] 235 | (cond 236 | (map? v) (into 237 | (empty v) 238 | (comp 239 | (filter #(not= lookup-ref (val %))) 240 | (map 241 | (juxt key #(delete-nested-entity lookup-ref (val %))))) 242 | v) 243 | 244 | (coll? v) (into 245 | (empty v) 246 | (comp 247 | (filter #(not= lookup-ref %)) 248 | (map #(delete-nested-entity lookup-ref %))) 249 | v) 250 | 251 | :else v)) 252 | 253 | 254 | (defn delete 255 | "Deletes an entity from the db, removing all references to it. A lookup-ref is 256 | a vector of [keyword id], e.g. [:person/id \"a123\"]" 257 | [db lookup-ref] 258 | (delete-nested-entity 259 | lookup-ref 260 | (update db (first lookup-ref) dissoc (second lookup-ref)))) 261 | 262 | 263 | (defn data->ast 264 | "Like pyramid.core/data->query, but returns the AST." 265 | [data] 266 | (cond 267 | (map? data) {:type :root 268 | :children (for [[k v] data 269 | :let [node (data->ast v)]] 270 | (if node 271 | (assoc node 272 | :type :join 273 | :key k 274 | :query (eql/ast->query node) 275 | :dispatch-key (if (coll? k) 276 | (first k) 277 | k)) 278 | {:type :prop 279 | :key k 280 | :dispatch-key k}))} 281 | ;; pathom uses `sequential?` here. 282 | (coll? data) (transduce (map data->ast) 283 | (fn 284 | ([] {:type :root :children []}) 285 | ([result] result) 286 | ([result el] (eql/merge-asts result el))) 287 | data))) 288 | 289 | (defn data->query 290 | "Returns an EQL query that matches the shape of `data` passed to it. 291 | Useful when you have some data already and want to see what an EQL query that 292 | returns that data would look like." 293 | [data] 294 | (-> data 295 | data->ast 296 | eql/ast->query)) 297 | -------------------------------------------------------------------------------- /src/pyramid/ident.cljc: -------------------------------------------------------------------------------- 1 | (ns pyramid.ident 2 | "Tools for identifying entity maps & creating ident functions. 3 | 4 | An 'ident function' is a function that takes a map and returns a tuple 5 | [:key val] that uniquely identifies the entity the map describes." 6 | (:refer-clojure :exclude [ident?])) 7 | 8 | 9 | (defn ident 10 | "Takes a key and value and returns an ident." 11 | [k v] 12 | [k v]) 13 | 14 | 15 | (defn ident? 16 | [x] 17 | (and (vector? x) 18 | (= 2 (count x)) 19 | (keyword? (first x)))) 20 | 21 | 22 | (defn by* 23 | "Takes a collection of functions. Returns an ident function that calls the 24 | first function on a map, then the second, and so on until one of the functions 25 | returns a non-nil value, which should be an ident. 26 | 27 | Returns nil if all functions return nil." 28 | [fns] 29 | (fn ident-by 30 | [entity] 31 | (some #(% entity) fns))) 32 | 33 | 34 | (defn by 35 | "Takes a number of functions. Returns an ident function that calls the 36 | first function on a map, then the second, and so on until one of the functions 37 | returns a non-nil value, which should be an ident. 38 | 39 | Returns nil if all functions return nil." 40 | ([f] 41 | (fn ident-by-1 42 | [entity] 43 | (f entity))) 44 | ([f & fns] 45 | (by* (cons f fns)))) 46 | 47 | 48 | (comment 49 | (defn person 50 | [e] 51 | (when-some [pid (:person/id e)] 52 | [:person/id pid])) 53 | 54 | (defn item 55 | [e] 56 | (when-some [id (:item/id e)] 57 | [:item/id id])) 58 | 59 | (def identify (by person item)) 60 | 61 | (identify {:person/id 1}) 62 | 63 | (identify {:item/id 1}) 64 | 65 | (identify {:food/id 1}) 66 | 67 | ;; composes 68 | (def identify2 (by identify (fn static-ident [_e] [:foo 1]))) 69 | 70 | (identify2 {:person/id 1}) 71 | 72 | (identify2 {:item/id 1}) 73 | 74 | (identify2 {:asdf 1})) 75 | 76 | 77 | (defn identify-by-key 78 | "Takes an entity and a key, and returns an ident using key and value in entity 79 | if found. Otherwise returns nil." 80 | [entity key] 81 | (let [v (get entity key ::not-found)] 82 | (when (not= v ::not-found) 83 | (ident key v)))) 84 | 85 | 86 | (defn by-keys* 87 | "Takes a collection of keys. Returns an ident function that looks for the 88 | first key in a map, then the second, and so on until it finds a matching 89 | key, then returns an ident using that key and the value found. 90 | 91 | Returns nil if no keys are found." 92 | [keys] 93 | (fn identify-by-keys 94 | [entity] 95 | (some #(identify-by-key entity %) keys))) 96 | 97 | 98 | (defn by-keys 99 | "Takes a number of keys. Returns an ident function that looks for the 100 | first key in a map, then the second, and so on until it finds a matching 101 | key, then returns an ident using that key and the value found. 102 | 103 | Returns nil if no keys are found." 104 | ([key] 105 | #(identify-by-key % key)) 106 | ([key & keys] 107 | (by-keys* (cons key keys)))) 108 | 109 | 110 | (comment 111 | (def identify-keys (by-keys :person/id :item/id)) 112 | 113 | (identify-keys {:person/id 1}) 114 | 115 | (identify-keys {:item/id 1}) 116 | 117 | (identify-keys {:food/id 1})) 118 | -------------------------------------------------------------------------------- /src/pyramid/pull.cljc: -------------------------------------------------------------------------------- 1 | (ns pyramid.pull 2 | (:require 3 | [cascade.core :as cc] 4 | [pyramid.ident :as ident] 5 | [edn-query-language.core :as eql])) 6 | 7 | 8 | (defprotocol IPullable 9 | (resolve-ref [p lookup-ref] [p lookup-ref not-found] 10 | "Given a ref [:key val], return the entity map it refers to")) 11 | 12 | 13 | (extend-protocol IPullable 14 | #?(:clj clojure.lang.IPersistentMap :cljs IMap) 15 | (resolve-ref 16 | ([m lookup-ref] (get-in m lookup-ref)) 17 | ([m lookup-ref not-found] 18 | (get-in m lookup-ref not-found))) 19 | 20 | #?@(:cljs [PersistentArrayMap 21 | (resolve-ref ([m ref] (get-in m ref)) 22 | ([m ref nf] (get-in m ref nf)))]) 23 | #?@(:cljs [PersistentHashMap 24 | (resolve-ref ([m ref] (get-in m ref)) 25 | ([m ref nf] (get-in m ref nf)))]) 26 | #?@(:cljs [PersistentTreeMap 27 | (resolve-ref ([m ref] (get-in m ref)) 28 | ([m ref nf] (get-in m ref nf)))]) 29 | #?@(:cljs [default 30 | (resolve-ref 31 | ([o ref] 32 | (if (map? o) 33 | (get-in o ref) 34 | (throw (ex-info "no resolve-ref implementation found" {:value o})))) 35 | ([o ref nf] 36 | (if (map? o) 37 | (get-in o ref nf) 38 | (throw (ex-info "no resolve-ref implementation found" {:value o})))))])) 39 | 40 | 41 | (def not-found ::not-found) 42 | 43 | 44 | #?(:clj 45 | (defn- fast-assoc 46 | {:inline 47 | (fn [m k v] 48 | (if (symbol? m) 49 | `(.assoc ~(with-meta m {:tag "clojure.lang.Associative"}) ~k ~v) 50 | `(let [m# ~m] (fast-assoc m# ~k ~v))))} 51 | [^clojure.lang.Associative m k v] 52 | (.assoc m k v))) 53 | 54 | 55 | (defn- found? 56 | [x] 57 | (not (identical? not-found x))) 58 | 59 | 60 | (defn- lookup-ref? 61 | [x] 62 | (ident/ident? x)) 63 | 64 | 65 | (defn- node->key 66 | [node] 67 | (if-some [params (:params node)] 68 | (list (:key node) params) 69 | (:key node))) 70 | 71 | 72 | (defn- replace-all-nested-lookups 73 | "Converts all lookup-refs like [:foo \"bar\"] to maps {:foo \"bar\"}" 74 | [k x] 75 | (cond 76 | (map? x) 77 | (cc/into 78 | k 79 | {} 80 | (cc/map (fn [k x] 81 | (replace-all-nested-lookups 82 | #(k (vector (key x) %)) 83 | (val x)))) 84 | x) 85 | 86 | (lookup-ref? x) 87 | #(k (#?(:bb assoc :clj fast-assoc :cljs assoc) {} (first x) (second x))) 88 | 89 | (coll? x) 90 | (cc/into 91 | k 92 | (empty x) 93 | (cc/map (fn [k x] (replace-all-nested-lookups k x))) x) 94 | 95 | :else #(k x))) 96 | 97 | 98 | (defn- visit 99 | [k db node {:keys [data parent entities]}] 100 | (case (:type node) 101 | :union 102 | (cc/some k (fn [k x] 103 | (visit 104 | (fn [x] 105 | (when (found? x) (k x))) 106 | db x {:data data :entities entities})) 107 | (:children node)) 108 | 109 | :union-entry 110 | (let [union-key (:union-key node)] 111 | (if (contains? data union-key) 112 | (cc/into 113 | (if-let [visitor (-> node :query meta :visitor)] 114 | (comp k #(visitor db %)) 115 | k) 116 | {} 117 | (comp 118 | (cc/map (fn [k x] 119 | #(visit k db x {:data data :entities entities}))) 120 | (cc/filter (cc/cont-with (comp found? second)))) 121 | (:children node)) 122 | #(k nil))) 123 | 124 | :prop 125 | (let [key (node->key node)] 126 | (cond 127 | (map? data) (let [result (if (lookup-ref? key) 128 | ;; ident query 129 | (do 130 | (conj! entities key) 131 | (resolve-ref db key not-found)) 132 | (get data key not-found))] 133 | ;; lookup-ref result 134 | (if (lookup-ref? result) 135 | (do 136 | (conj! entities result) 137 | #(k [(:key node) (resolve-ref db result not-found)])) 138 | (replace-all-nested-lookups 139 | #(k [(:key node) %]) 140 | result))) 141 | 142 | ;; handle ordering of lists by using map/filter directly instead of into 143 | (or (list? data) (seq? data)) 144 | (cc/map 145 | ;; k 146 | (fn [s] 147 | (cc/filter 148 | k 149 | (cc/cont-with (comp found? second)) 150 | s)) 151 | ;; f 152 | (cc/cont-with 153 | #(vector key (get % key not-found))) 154 | data) 155 | 156 | (coll? data) (cc/into 157 | k 158 | (empty data) 159 | (comp 160 | (cc/map (cc/cont-with 161 | #(vector key (get % key not-found)))) 162 | (cc/filter (cc/cont-with (comp found? second)))) 163 | data) 164 | :else #(k nil))) 165 | 166 | :join 167 | (let [key (node->key node) 168 | key-result (if (lookup-ref? key) 169 | (do 170 | (conj! entities key) 171 | (resolve-ref db key not-found)) 172 | (get data key not-found)) 173 | data (cond 174 | (lookup-ref? key-result) 175 | (do 176 | (conj! entities key-result) 177 | (resolve-ref db key-result)) 178 | 179 | ;; not a coll 180 | (map? key-result) 181 | key-result 182 | 183 | (or (list? key-result) (seq? key-result)) 184 | (map #(if (lookup-ref? %) 185 | (do (conj! entities %) 186 | (resolve-ref db % (conj {} %))) 187 | %) 188 | key-result) 189 | 190 | (coll? key-result) 191 | (into 192 | (empty key-result) 193 | (map #(if (lookup-ref? %) 194 | (do (conj! entities %) 195 | (resolve-ref db % (conj {} %))) 196 | %)) 197 | key-result) 198 | 199 | :else key-result) 200 | [children new-parent] (cond 201 | (contains? node :children) 202 | [(:children node) node] 203 | ;; infinite recursion 204 | ;; repeat this query with the new data 205 | (= (:query node) '...) 206 | [(:children parent) parent] 207 | 208 | (pos-int? (:query node)) 209 | (let [parent (assoc 210 | parent :children 211 | (mapv 212 | (fn [node'] 213 | (if (= key 214 | (node->key node')) 215 | (update 216 | node' :query 217 | dec) 218 | node')) 219 | (:children parent)))] 220 | [(:children parent) 221 | parent])) 222 | k' (comp k #(vector (:key node) %)) 223 | node-visitor (if-let [visitor (some-> node :query meta :visitor)] 224 | (fn [x] 225 | (visitor db x)) 226 | identity) 227 | union-child? (and (= 1 (count (:children node))) 228 | (= :union (:type (first (:children node)))))] 229 | (cond 230 | (map? data) 231 | ;; handle union, which might have a visitor 232 | (if (and union-child? (map? data)) 233 | #(visit (comp k' node-visitor) 234 | db (first (:children node)) 235 | {:data data 236 | :parent new-parent 237 | :entities entities}) 238 | (cc/into 239 | (comp k' node-visitor) 240 | (with-meta {} (:meta node)) 241 | (comp 242 | (cc/map (fn [k x] 243 | (visit k db x {:data data 244 | :parent new-parent 245 | :entities entities}))) 246 | (cc/filter (cc/cont-with seq)) 247 | (cc/filter (cc/cont-with (comp found? second)))) 248 | children)) 249 | 250 | ;; handle ordering of lists by using map/filter directly instead of into 251 | (or (list? data) (seq? data)) 252 | (cc/map 253 | ;; k 254 | (fn [s] 255 | (cc/filter 256 | k' 257 | (cc/cont-with seq) ;pred 258 | s)) 259 | ;; f 260 | (fn [k datum] 261 | (if union-child? 262 | #(visit (comp k 263 | node-visitor 264 | (fn [x] 265 | (with-meta x (:meta node)))) 266 | db (first children) 267 | {:data datum 268 | :parent new-parent 269 | :entities entities}) 270 | (cc/into 271 | (comp k node-visitor) 272 | (with-meta (empty datum) (:meta node)) 273 | (comp 274 | (cc/map (fn [k x] 275 | (visit k db x {:data datum 276 | :parent new-parent 277 | :entities entities}))) 278 | (cc/filter (cc/cont-with (comp found? second)))) 279 | children))) 280 | data) 281 | 282 | (coll? data) (cc/into 283 | k' 284 | (empty data) 285 | (comp 286 | (cc/map 287 | (fn [k datum] 288 | (if union-child? 289 | #(visit (comp k 290 | node-visitor 291 | (fn [x] 292 | (with-meta x (:meta node)))) 293 | db (first children) 294 | {:data datum 295 | :parent new-parent 296 | :entities entities}) 297 | (cc/into 298 | (comp k node-visitor) 299 | (with-meta (empty datum) (:meta node)) 300 | (comp 301 | (cc/map (fn [k x] 302 | (visit k db x {:data datum 303 | :parent new-parent 304 | :entities entities}))) 305 | (cc/filter (cc/cont-with (comp found? second)))) 306 | children)))) 307 | (cc/filter (cc/cont-with seq))) 308 | data) 309 | :else #(k nil))))) 310 | 311 | 312 | (defn pull-report 313 | "Executes an EQL query against a normalized map `db`. Returns a map with the 314 | following keys: 315 | :data - the result of the query 316 | :entities - a set of lookup refs that were visited during the query" 317 | [db query] 318 | (let [root (eql/query->ast query) 319 | entities (transient #{}) 320 | indices (into 321 | #{} 322 | (comp (map :key) 323 | (remove ident/ident?)) 324 | (:children root))] 325 | (cc/into 326 | (fn [data] 327 | {:data data 328 | :entities (persistent! entities) 329 | :indices indices}) 330 | (with-meta {} (:meta root)) 331 | (comp 332 | (cc/map (fn [k x] 333 | (visit k db x {:data db :entities entities}))) 334 | (cc/filter (cc/cont-with (comp found? second)))) 335 | (:children root)))) 336 | -------------------------------------------------------------------------------- /src/pyramid/query.cljc: -------------------------------------------------------------------------------- 1 | (ns pyramid.query 2 | "Experimental! 3 | 4 | A datalog query engine for normalized maps in the pyramid.core/db fashion" 5 | (:require 6 | [clojure.string :as string])) 7 | 8 | ;; TODO 9 | ;; * wildcards 10 | ;; * boolean logic / filters 11 | ;; * pull in :find 12 | ;; * single result in :find 13 | 14 | 15 | (def ? '?) 16 | (def $ '$) 17 | (def _ '_) 18 | 19 | 20 | (defn parse 21 | [query db & params] 22 | (let [[bindings clauses] (split-with #(not= % :where) query) 23 | find (->> bindings 24 | (take-while #(not= % :in)) 25 | (drop 1)) 26 | inputs (->> bindings 27 | (drop-while #(not= % :in)) 28 | ;; users have to pass in $ 29 | (drop 2) 30 | (cons $)) 31 | clauses (drop 1 clauses) 32 | anomalies (cond-> [] 33 | (not= (count inputs) (inc (count params))) 34 | (conj {:query/anomaly 35 | "Inputs in query do not match those passed in"}) 36 | 37 | (empty? clauses) 38 | (conj {:query/anomaly 39 | "No :where clauses given"}))] 40 | {:find find 41 | :in (zipmap inputs (cons db params)) 42 | :where clauses 43 | :anomalies anomalies})) 44 | 45 | 46 | (comment 47 | ;; simple 48 | (parse '[:find ?id ?value 49 | :where 50 | [?e :foo/id ?id] 51 | [?e :foo/value ?value]] 52 | {:foo/id {"1234" {:foo/id "1234" 53 | :foo/value "asdf"}}}) 54 | 55 | (parse '[:find ?id ?value 56 | :where 57 | [?e :foo/id ?id] 58 | [?e :foo/value]] 59 | {:foo/id {"1234" {:foo/id "1234" 60 | :foo/value "asdf"}}}) 61 | 62 | 63 | ;; invalid 64 | (parse '[:find ?id ?value 65 | :where 66 | [:foo/id ?id]] 67 | {:foo/id {"1234" {:foo/id "1234" 68 | :foo/value "asdf"}}}) 69 | 70 | ;; inputs 71 | (parse '[:find ?id ?value 72 | :in $ ?a ?b 73 | :where 74 | [?e :foo/id ?id] 75 | [?e :foo/value ?value]] 76 | {:foo/id {"1234" {:foo/id "1234" 77 | :foo/value "asdf"}}} 78 | "a" 79 | "b") 80 | 81 | ;; anomaly, mismatch inputs 82 | (parse '[:find ?id ?value 83 | :in $ ?a ?b 84 | :where 85 | [?e :foo/id ?id] 86 | [?e :foo/value ?value]] 87 | {:foo/id {"1234" {:foo/id "1234" 88 | :foo/value "asdf"}}}) 89 | ) 90 | 91 | 92 | (defn variable? 93 | [x] 94 | (and (symbol? x) (string/starts-with? (name x) "?"))) 95 | 96 | 97 | (defn pattern 98 | [x] 99 | (if (variable? x) ? :v)) 100 | 101 | 102 | (defprotocol IQueryable 103 | :extend-via-metadata true 104 | (entities [o] "Returns a seq of all entities which can be queried.")) 105 | 106 | 107 | (defn- map-entities 108 | [m] 109 | (->> m 110 | (tree-seq coll? #(if (map? %) (vals %) (seq %))) 111 | (filter map?))) 112 | 113 | 114 | (defn- coll-entities 115 | [c] 116 | (mapcat #(when (satisfies? IQueryable %) (entities %)) c)) 117 | 118 | 119 | (extend-protocol IQueryable 120 | #?(:clj clojure.lang.IPersistentMap :cljs IMap) 121 | (entities [m] (map-entities m)) 122 | 123 | #?(:clj clojure.lang.IPersistentCollection :cljs ICollection) 124 | (entities [c] (coll-entities c)) 125 | 126 | #?@(:cljs [default 127 | (entities 128 | [o] 129 | (cond 130 | (satisfies? IMap o) (map-entities o) 131 | (satisfies? ICollection o) (coll-entities o) 132 | :else nil))])) 133 | 134 | 135 | (defn- contains-in? 136 | [m ks] 137 | (let [path (drop-last ks) 138 | k (last ks)] 139 | (contains? (get-in m path) k))) 140 | 141 | 142 | (defn- resolve-entity 143 | ([db e] 144 | (if (map? e) 145 | e 146 | (get-in db e))) 147 | ([db e a] (resolve-entity db e a nil)) 148 | ([db e a nf] 149 | (if (map? e) 150 | (get e a nf) 151 | (get-in db (conj e a) nf)))) 152 | 153 | 154 | (defn- entity-exists? 155 | ([db e] 156 | (or (map? e) (contains-in? db e))) 157 | ([db e a] 158 | (if (map? e) 159 | (contains? e a) 160 | (contains-in? db (conj e a))))) 161 | 162 | 163 | (defn- resolve-triple 164 | [db triple] 165 | (let [[e a v] triple 166 | entities (entities db)] 167 | #_(prn triple (map pattern triple)) 168 | (case (map pattern triple) 169 | [:v :v :v] 170 | (if (= v (resolve-entity db e a)) 171 | [[]] 172 | []) 173 | 174 | [? :v :v] 175 | (into 176 | [] 177 | (comp 178 | (filter #(= v (resolve-entity db % a ::not-found))) 179 | (map vector)) 180 | entities) 181 | 182 | [:v ? :v] 183 | (into 184 | [] 185 | (comp 186 | (filter #(= v (val %))) 187 | (map key)) 188 | (resolve-entity db e)) 189 | 190 | [:v :v ?] 191 | (if (entity-exists? db e a) 192 | (if (:many (meta v)) 193 | (map vector (resolve-entity db e a)) 194 | [[(resolve-entity db e a)]]) 195 | []) 196 | 197 | [? ? :v] 198 | (mapcat 199 | (fn [entity] 200 | (->> (resolve-entity db entity) 201 | (filter #(= v (val %))) 202 | (map (fn [entry] 203 | [entity (key entry)])))) 204 | entities) 205 | 206 | [? :v ?] 207 | (into 208 | [] 209 | (comp 210 | (filter #(entity-exists? db % a)) 211 | (if (:many (meta v)) 212 | (mapcat 213 | (fn [entity] 214 | (map 215 | (fn [value] [entity value]) 216 | (resolve-entity db entity a)))) 217 | (map 218 | (fn [entity] 219 | [entity (resolve-entity db entity a)])))) 220 | entities) 221 | 222 | [:v ? ?] 223 | (if (:many (meta v)) 224 | (into 225 | [] 226 | (mapcat 227 | (fn [entry] 228 | (let [k (key entry)] 229 | (map #(vector k %) (val entry)))) 230 | (resolve-entity db e))) 231 | (mapv 232 | (fn [entry] 233 | [(key entry) (val entry)]) 234 | (resolve-entity db e))) 235 | 236 | [? ? ?] 237 | (if (:many (meta v)) 238 | (for [entity-or-ident entities 239 | :let [entity (resolve-entity db entity-or-ident)] 240 | entry entity 241 | values (val entry)] 242 | [entity-or-ident (key entry) values]) 243 | (for [entity-or-ident entities 244 | :let [entity (resolve-entity db entity-or-ident)] 245 | entry entity] 246 | [entity-or-ident (key entry) (val entry)]))))) 247 | 248 | 249 | (comment 250 | (def db {:foo/id {"123" {:foo/id "123" 251 | :foo/bar "baz"} 252 | "456" {:foo/id "456" 253 | :foo/bar "asdf"}} 254 | :foo {:bar "baz"} 255 | :asdf "jkl"}) 256 | 257 | ;; [:v :v :v] found 258 | (resolve-triple db [[:foo/id "123"] :foo/bar "baz"]) 259 | 260 | ;; [:v :v :v] not-found 261 | (resolve-triple db ['[:foo/id "456"] :foo/bar "bar"]) 262 | 263 | (resolve-triple db ['[:foo/id "123"] :foo/bat "bar"]) 264 | 265 | (resolve-triple db ['[:foo/id "123"] :foo/bar "bat"]) 266 | 267 | (resolve-triple db ['[:foo/id "asdf"] :foo/bar "bar"]) 268 | 269 | 270 | ;; [? :v :v] found 271 | (resolve-triple db '[?e :foo/bar "baz"]) 272 | 273 | (resolve-triple db (with-meta '[?e :foo/bar "baz"] 274 | {:original '[?e :foo/bar ?bar]})) 275 | 276 | ;; [? :v :v] not-found 277 | (resolve-triple db '[?e :foo/bar "bat"]) 278 | 279 | (resolve-triple db '[?e :foo/bat "baz"]) 280 | 281 | 282 | ;; [:v ? :v] found 283 | (resolve-triple db '[[:foo/id "123"] ?a "baz"]) 284 | 285 | ;; [:v ? :v] not-found 286 | (resolve-triple db '[[:foo/id "123"] ?a "bat"]) 287 | 288 | (resolve-triple db '[[:foo/id "456"] ?a "bar"]) 289 | 290 | (resolve-triple db '[[:foo/id "asdf"] ?a "bar"]) 291 | 292 | 293 | ;; [:v :v ?] found 294 | (resolve-triple db '[[:foo/id "123"] :foo/bar ?v]) 295 | 296 | ;; [:v :v ?] not-found 297 | (resolve-triple db '[[:foo/id "asdf"] :foo/bar ?v]) 298 | 299 | (resolve-triple db '[[:foo/id "123"] :foo/bat ?v]) 300 | 301 | 302 | ;; [? ? :v] found 303 | (resolve-triple db '[?e ?a "baz"]) 304 | 305 | ;; [? ? :v] not-found 306 | (resolve-triple db '[?e ?a "bat"]) 307 | 308 | 309 | ;; [? :v ?] found 310 | (resolve-triple db '[?e :foo/id ?id]) 311 | 312 | (resolve-triple db '[?e :foo/bar ?id]) 313 | 314 | ;; [? :v ?] not-found 315 | (resolve-triple db '[?e :foo/bat ?bat]) 316 | 317 | 318 | ;; [:v ? ?] found 319 | (resolve-triple db '[[:foo/id "123"] ?a ?v]) 320 | 321 | ;; [:v ? ?] not-found 322 | (resolve-triple db '[[:foo/id "asdf"] ?a ?v]) 323 | 324 | 325 | ;; [? ? ?] 326 | (resolve-triple db '[?e ?a ?v]) 327 | ) 328 | 329 | 330 | 331 | (defn- rewrite-and-resolve-triple 332 | [db results [e a v :as triple]] 333 | ;; TODO for some reason this is nesting too many collections 334 | (for [left results 335 | :let [pattern (:pattern (meta left)) 336 | ;; get mapping of var to result index 337 | var->idx (into 338 | {} 339 | (comp 340 | (map-indexed #(vector %2 %1)) 341 | (filter #(variable? (first %)))) 342 | pattern) 343 | [ei ai vi] (map var->idx triple) 344 | triple' [(if ei (nth left ei) e) 345 | (if ai (nth left ai) a) 346 | (if vi (nth left vi) v)] 347 | pattern' (->> triple' 348 | (remove (complement variable?)) 349 | (remove (set pattern)) 350 | (concat pattern) 351 | (vec))] 352 | right (resolve-triple db triple')] 353 | (with-meta 354 | (concat left right) 355 | {:pattern pattern'}))) 356 | 357 | 358 | (comment 359 | (rewrite-and-resolve-triple 360 | {:foo/id {1 {:foo/id 1 361 | :foo/name "bar"} 362 | 2 {:foo/id 2 363 | :foo/name "baz"}}} 364 | [(with-meta [[:foo/id 1] 1] 365 | {:pattern '[?e ?id]}) 366 | (with-meta [[:foo/id 2] 2] 367 | {:pattern '[?e ?id]})] 368 | '[?e :foo/name ?name]) 369 | 370 | (map meta 371 | (rewrite-and-resolve-triple 372 | {:foo/id {1 {:foo/id 1 373 | :foo/name "bar"} 374 | 2 {:foo/id 2 375 | :foo/name "baz"}}} 376 | [(with-meta [[:foo/id 1] 1] 377 | {:pattern '[?e ?id]})] 378 | '[?e :foo/name ?name])) 379 | 380 | (rewrite-and-resolve-triple 381 | {:foo/id {"123" #:foo{:id "123", :bar "baz"} 382 | "456" #:foo{:id "456", :bar "asdf"} 383 | "789" #:foo{:id "456"}} 384 | :foo {:bar "baz"}, :asdf "jkl"} 385 | nil 386 | '[?e :foo/id ?id]) 387 | 388 | (rewrite-and-resolve-triple 389 | {:foo/id {"123" #:foo{:id "123", :bar "baz"} 390 | "456" #:foo{:id "456", :bar "asdf"} 391 | "789" #:foo{:id "456"}}, :foo {:bar "baz"}, :asdf "jkl"} 392 | [(with-meta [[:foo/id "123"] "123"] 393 | '{:pattern (?e ?id)}) 394 | (with-meta [[:foo/id "456"] "456"] 395 | '{:pattern (?e ?id)}) 396 | (with-meta [[:foo/id "789"] "456"] 397 | '{:pattern (?e ?id)})] 398 | '[?e :foo/bar ?bar])) 399 | 400 | 401 | (defn- project-results 402 | [results bindings] 403 | (for [result results 404 | :let [pattern (:pattern (meta result)) 405 | var->idx (into 406 | {} 407 | (comp 408 | (map-indexed #(vector %2 %1)) 409 | (filter #(variable? (first %)))) 410 | pattern) 411 | indices (->> bindings 412 | (map var->idx))] 413 | :when (some some? indices)] 414 | (mapv #(when (some? %) (nth result %)) indices))) 415 | 416 | 417 | (defn in->results 418 | [in] 419 | #_(prn in) 420 | ;; convert map {?var value|[value]} to ([?value] [?value]) with pattern meta 421 | (let [expanded-in (for [[k v] in] 422 | (map 423 | #(with-meta [%] {:pattern [k]}) 424 | (if (:many (meta k)) 425 | v 426 | [v])))] 427 | (loop [results [[]] 428 | in expanded-in] 429 | (if-let [hd (first in)] 430 | (recur 431 | (for [left results 432 | right hd] 433 | (with-meta 434 | (concat left right) 435 | {:pattern (concat (:pattern (meta left)) 436 | (:pattern (meta right)))})) 437 | (rest in)) ;; peace 438 | results)))) 439 | 440 | 441 | (comment 442 | (in->results '{$ {} ?foo "bar"}) 443 | 444 | (in->results '{$ {} ^:many ?foo ["bar" "baz"] ?asdf [1 2 3]}) 445 | 446 | (in->results '{$ {} ^:many ?foo ["bar" "baz"] ^:many ?asdf [1 2 3]}) 447 | ) 448 | 449 | 450 | (defn execute 451 | [{:keys [find in where]}] 452 | (let [db (get in $)] 453 | (loop [clauses where 454 | results (in->results in)] 455 | ;; (prn (first results)) 456 | ;; (prn (meta (first results))) 457 | (if-let [clause (first clauses)] 458 | (recur 459 | (rest clauses) 460 | (rewrite-and-resolve-triple db results clause)) 461 | (do 462 | #_(prn (meta (first results))) 463 | #_(clojure.pprint/pprint results) 464 | (project-results results find)))))) 465 | 466 | 467 | (comment 468 | ;; found 469 | (-> (parse 470 | '[:find ?id ?bar 471 | :where 472 | [?e :foo/id ?id] 473 | [?e :foo/bar ?bar]] 474 | {:foo/id {"123" {:foo/id "123" 475 | :foo/bar "baz"} 476 | "456" {:foo/id "456" 477 | :foo/bar "asdf"} 478 | "789" {:foo/id "456"}} 479 | :foo {:bar "baz"} 480 | :asdf "jkl"}) 481 | (execute) 482 | #_(->> (map meta))) 483 | 484 | (-> (parse 485 | '[:find ?id 486 | :in $ 487 | :where 488 | [?e :foo/id ?id] 489 | [?e :foo/bar "baz"]] 490 | {:foo/id {"123" {:foo/id "123" 491 | :foo/bar "baz"} 492 | "456" {:foo/id "456" 493 | :foo/bar "asdf"}} 494 | :foo {:bar "baz"} 495 | :asdf "jkl"}) 496 | (execute) 497 | #_(->> (map meta))) 498 | 499 | (-> (parse 500 | '[:find ?e0 ?friend ?bar 501 | :where 502 | [?e0 :foo/id ?id] 503 | [?e0 :foo/friend ?friend] 504 | [?friend :foo/bar ?bar]] 505 | {:foo/id {"123" {:foo/id "123" 506 | :foo/friend [:foo/id "456"] 507 | :foo/bar "asdf"} 508 | "456" {:foo/id "456" 509 | :foo/bar "baz"}} 510 | :foo {:bar "baz"} 511 | :asdf "jkl"}) 512 | (execute) 513 | #_(->> (map meta))) 514 | 515 | ;; not-found 516 | (execute 517 | '{:find [?e ?id] 518 | :in {$ [{:foo/id {"123" {:foo/id "123" 519 | :foo/bar "baz"} 520 | "456" {:foo/id "456" 521 | :foo/bar "asdf"}} 522 | :foo {:bar "baz"} 523 | :asdf "jkl"}] 524 | ?bar ["bat"]} 525 | :where 526 | ([?e :foo/id ?id] 527 | [?e :foo/bar ?bar])}) 528 | 529 | ) 530 | 531 | 532 | (defn q 533 | [query db & params] 534 | (-> (apply parse query db params) 535 | (execute))) 536 | -------------------------------------------------------------------------------- /test/bb_test_runner.clj: -------------------------------------------------------------------------------- 1 | (ns bb-test-runner 2 | (:require 3 | [clojure.test :as t] 4 | [pyramid.core-test] 5 | [pyramid.pull-test] 6 | [pyramid.query-test])) 7 | 8 | (defn run-tests 9 | [& _args] 10 | (let [{:keys [fail error]} 11 | (t/run-tests 12 | 'pyramid.core-test 13 | 'pyramid.pull-test 14 | 'pyramid.query-test)] 15 | (when (or (pos? fail) 16 | (pos? error)) 17 | (System/exit 1)))) 18 | -------------------------------------------------------------------------------- /test/pyramid/core_test.cljc: -------------------------------------------------------------------------------- 1 | (ns pyramid.core-test 2 | (:require 3 | [clojure.test :refer [deftest is testing]] 4 | [pyramid.core :as p] 5 | [pyramid.ident :as ident])) 6 | 7 | 8 | (deftest normalization 9 | (is (= {:person/id {0 {:person/id 0}}} 10 | (p/db [{:person/id 0}])) 11 | "a single entity") 12 | (is (= {:person/id {0 {:person/id 0 13 | :person/name "asdf"} 14 | 1 {:person/id 1 15 | :person/name "jkl"}}} 16 | (p/db [{:person/id 0 17 | :person/name "asdf"} 18 | {:person/id 1 19 | :person/name "jkl"}])) 20 | "multiple entities with attributes") 21 | (is (= {:person/id {0 {:person/id 0 22 | :person/name "asdf"} 23 | 1 {:person/id 1 24 | :person/name "jkl"}} 25 | :people [[:person/id 0] 26 | [:person/id 1]]} 27 | (p/db [{:people [{:person/id 0 28 | :person/name "asdf"} 29 | {:person/id 1 30 | :person/name "jkl"}]}])) 31 | "nested under a key") 32 | (is (= {:person/id {0 {:person/id 0 33 | :some-data {1 "hello" 34 | 3 "world"}}}} 35 | (p/db [{:person/id 0 36 | :some-data {1 "hello" 37 | 3 "world"}}])) 38 | "Map with numbers as keys") 39 | (is (= {:a/id {1 {:a/id 1 40 | :b [{:c [:d/id 1]}]}} 41 | :d/id {1 {:d/id 1 42 | :d/txt "a"}}} 43 | (p/db [{:a/id 1 44 | :b [{:c {:d/id 1 45 | :d/txt "a"}}]}])) 46 | "Collections of non-entities still get normalized") 47 | (is (= {:person/id {0 {:person/id 0 48 | :person/name "Bill" 49 | :person/friends [{:person/name "Bob"} 50 | [:person/id 2]]} 51 | 2 {:person/id 2 52 | :person/name "Alice"}}} 53 | (p/db [{:person/id 0 54 | :person/name "Bill" 55 | :person/friends [{:person/name "Bob"} 56 | {:person/name "Alice" 57 | :person/id 2}]}])) 58 | "heterogeneous collections") 59 | (is (= {:person/id 60 | {123 61 | {:person/id 123, 62 | :person/name "Will", 63 | :contact {:phone "000-000-0001"}, 64 | :best-friend [:person/id 456], 65 | :friends 66 | [[:person/id 9001] 67 | [:person/id 456] 68 | [:person/id 789] 69 | [:person/id 1000]]}, 70 | 456 71 | {:person/id 456, 72 | :person/name "Jose", 73 | :account/email "asdf@jkl", 74 | :best-friend [:person/id 123]}, 75 | 9001 #:person{:id 9001, :name "Georgia"}, 76 | 789 #:person{:id 789, :name "Frank"}, 77 | 1000 #:person{:id 1000, :name "Robert"}}} 78 | (p/db [{:person/id 123 79 | :person/name "Will" 80 | :contact {:phone "000-000-0001"} 81 | :best-friend 82 | {:person/id 456 83 | :person/name "Jose" 84 | :account/email "asdf@jkl"} 85 | :friends 86 | [{:person/id 9001 87 | :person/name "Georgia"} 88 | {:person/id 456 89 | :person/name "Jose"} 90 | {:person/id 789 91 | :person/name "Frank"} 92 | {:person/id 1000 93 | :person/name "Robert"}]} 94 | {:person/id 456 95 | :best-friend {:person/id 123}}])) 96 | "refs")) 97 | 98 | 99 | (deftest non-entities 100 | (is (= {:foo ["bar"]} (p/db [{:foo ["bar"]}]))) 101 | (is (= {:person/id {0 {:person/id 0 102 | :foo ["bar"]}}} 103 | (p/db [{:person/id 0 104 | :foo ["bar"]}])))) 105 | 106 | 107 | (deftest custom-schema 108 | (is (= {:color {"red" {:color "red" :hex "#ff0000"}}} 109 | (p/db [{:color "red" :hex "#ff0000"}] 110 | (ident/by-keys :color))) 111 | "ident/by-keys") 112 | (is (= {:color {"red" {:color "red" :hex "#ff0000"}}} 113 | (p/db [^{:db/ident :color} 114 | {:color "red" :hex "#ff0000"}])) 115 | "local schema") 116 | (testing "complex schema" 117 | (let [db (p/db [{:type "person" 118 | :id "1234" 119 | :purchases [{:type "item" 120 | :id "1234"}]} 121 | {:type "item" 122 | :id "5678"} 123 | {:type "foo"} 124 | {:id "bar"}] 125 | (fn [entity] 126 | (let [{:keys [type id]} entity] 127 | (when (and (some? type) (some? id)) 128 | [(keyword type "id") id]))))] 129 | (is (= {:person/id 130 | {"1234" {:type "person", :id "1234", :purchases [[:item/id "1234"]]}}, 131 | :item/id 132 | {"1234" {:type "item", :id "1234"}, "5678" {:type "item", :id "5678"}}, 133 | :type "foo", 134 | :id "bar"} 135 | db) 136 | "correctly identifies entities") 137 | (is (= {[:person/id "1234"] 138 | {:type "person", :id "1234", :purchases [{:type "item", :id "1234"}]}} 139 | (p/pull db [{[:person/id "1234"] [:type :id {:purchases [:type :id]}]}])) 140 | "pull")))) 141 | 142 | 143 | (deftest add 144 | (is (= {:person/id {0 {:person/id 0}}} 145 | (p/add {} {:person/id 0}))) 146 | (is (= {:person/id {0 {:person/id 0 :person/name "Gill"} 147 | 1 {:person/id 1}}} 148 | (p/add 149 | {} 150 | {:person/id 0 :person/name "Alice"} 151 | {:person/id 1} 152 | {:person/id 0 :person/name "Gill"})))) 153 | 154 | 155 | (deftest add-report 156 | (is (= {:db {:person/id {0 {:person/id 0}}} 157 | :entities #{[:person/id 0]} 158 | :indices #{}} 159 | (p/add-report {} {:person/id 0}))) 160 | (is (= {:db {:person/id {0 {:person/id 0 161 | :person/name "Gill" 162 | :best-friend [:person/id 1]} 163 | 1 {:person/id 1 164 | :person/name "Uma"}} 165 | :me [:person/id 0]} 166 | :entities #{[:person/id 0] 167 | [:person/id 1]} 168 | :indices #{:me}} 169 | (p/add-report {} {:me {:person/id 0 170 | :person/name "Gill" 171 | :best-friend {:person/id 1 172 | :person/name "Uma"}}}))) 173 | #_(is (= {:db {:person/id {0 {:person/id 0 :person/name "Gill"} 174 | 1 {:person/id 1}}} 175 | :entities #{{:person/id 0 :person/name "Gill"} 176 | {:person/id 1}}} 177 | (p/add-report 178 | {} 179 | {:person/id 0} 180 | {:person/id 1} 181 | {:person/id 0 :person/name "Gill"})))) 182 | 183 | 184 | (defrecord Thing [a b c]) 185 | 186 | 187 | (deftest records 188 | (is (= (->Thing "foo" "bar" "baz") 189 | (-> [{:id 0 190 | :thing (->Thing "foo" "bar" "baz")}] 191 | (p/db) 192 | (get-in [:id 0 :thing])))) 193 | #_(is (= (->Thing "foo" "bar" "baz") 194 | (-> [{:id 0 195 | :thing (->Thing "foo" "bar" "baz")}] 196 | (p/db) 197 | (p/pull [{[:id 0] [:thing]}]) 198 | (get-in [[:id 0] :thing]))) 199 | "pulling a record returns the right type")) 200 | 201 | 202 | (def data 203 | {:people/all [{:person/id 0 204 | :person/name "Alice" 205 | :person/age 25 206 | :best-friend {:person/id 1} 207 | :person/favorites 208 | {:favorite/ice-cream "vanilla"}} 209 | {:person/id 1 210 | :person/name "Bob" 211 | :person/age 23}]}) 212 | 213 | 214 | (def db 215 | (p/db [data])) 216 | 217 | 218 | (deftest pull 219 | (is (= #:people{:all [{:person/id 0} {:person/id 1}]} 220 | (p/pull db [:people/all])) 221 | "simple key") 222 | (is (= {:people/all [{:person/name "Alice" 223 | :person/id 0} 224 | {:person/name "Bob" 225 | :person/id 1}]} 226 | (p/pull db [{:people/all [:person/name :person/id]}])) 227 | "basic join + prop") 228 | (is (= #:people{:all [{:person/name "Alice" 229 | :person/id 0 230 | :best-friend #:person{:name "Bob", :id 1 :age 23}} 231 | #:person{:name "Bob", :id 1}]} 232 | (p/pull db [#:people{:all [:person/name :person/id :best-friend]}])) 233 | "join + prop + join ref lookup") 234 | (is (= #:people{:all [{:person/name "Alice" 235 | :person/id 0 236 | :best-friend #:person{:name "Bob"}} 237 | #:person{:name "Bob", :id 1}]} 238 | (p/pull db [#:people{:all [:person/name 239 | :person/id 240 | {:best-friend [:person/name]}]}])) 241 | "join + prop, ref as prop resolver") 242 | (is (= {[:person/id 1] #:person{:id 1, :name "Bob", :age 23}} 243 | (p/pull db [[:person/id 1]])) 244 | "ident acts as ref lookup") 245 | (is (= {[:person/id 0] {:person/id 0 246 | :person/name "Alice" 247 | :person/age 25 248 | :best-friend {:person/id 1} 249 | :person/favorites #:favorite{:ice-cream "vanilla"}}} 250 | (p/pull db [[:person/id 0]])) 251 | "ident does not resolve nested refs") 252 | (is (= {[:person/id 0] #:person{:id 0 253 | :name "Alice" 254 | :favorites #:favorite{:ice-cream "vanilla"}}} 255 | (p/pull db [{[:person/id 0] [:person/id 256 | :person/name 257 | :person/favorites]}])) 258 | "join on ident") 259 | (is (= {:people/all [{:person/name "Alice" 260 | :person/id 0 261 | :best-friend #:person{:name "Bob", :id 1 :age 23}} 262 | #:person{:name "Bob", :id 1}] 263 | [:person/id 1] #:person{:age 23}} 264 | (p/pull db [{:people/all [:person/name :person/id :best-friend]} 265 | {[:person/id 1] [:person/age]}])) 266 | "multiple joins") 267 | 268 | (testing "includes params" 269 | (is (= #:people{:all [#:person{:name "Bob", :id 1}]} 270 | (p/pull (-> db 271 | (p/add {'(:people/all {:with "params"}) [[:person/id 1]]})) 272 | '[{(:people/all {:with "params"}) 273 | [:person/name :person/id]}]))) 274 | (is (= '{:person/foo {:person/id 1 275 | :person/name "Bob"}} 276 | (p/pull (-> db 277 | (p/add {'(:person/foo {:person/id 2}) 278 | {:person/id 1}})) 279 | '[{(:person/foo {:person/id 2}) 280 | [:person/name :person/id]}])) 281 | "params that include an entity-looking thing should not be normalized") 282 | (is (= {} 283 | (p/pull db '[([:person/id 1] {:with "params"})]))) 284 | (is (= {} 285 | (p/pull db '[{(:people/all {:with "params"}) 286 | [:person/name :person/id]}])))) 287 | 288 | (testing "union" 289 | (let [data {:foo {:bar/id 2 :asdf 123 :jkl 456 :qux 789}} 290 | db (p/db [data]) 291 | query [{:foo {:bar/id [:bar/id :asdf :jkl] 292 | :baz/id [:baz/id :arst :nei]}}]] 293 | (is (= {:foo {:bar/id 2 :asdf 123 :jkl 456}} 294 | (p/pull db query)))) 295 | (let [data {:chat/entries 296 | [{:message/id 0 297 | :message/text "foo" 298 | :chat.entry/timestamp "1234"} 299 | {:message/id 1 300 | :message/text "bar" 301 | :chat.entry/timestamp "1235"} 302 | {:audio/id 0 303 | :audio/url "audio://asdf.jkl" 304 | :audio/duration 1234 305 | :chat.entry/timestamp "4567"} 306 | {:photo/id 0 307 | :photo/url "photo://asdf_10x10.jkl" 308 | :photo/height 10 309 | :photo/width 10 310 | :chat.entry/timestamp "7890"}]} 311 | db1 (p/db [data]) 312 | query [{:chat/entries 313 | {:message/id 314 | [:message/id :message/text :chat.entry/timestamp] 315 | 316 | :audio/id 317 | [:audio/id :audio/url :audio/duration :chat.entry/timestamp] 318 | 319 | :photo/id 320 | [:photo/id :photo/url :photo/width :photo/height :chat.entry/timestamp] 321 | 322 | :asdf/jkl [:asdf/jkl]}}]] 323 | (is (= #:chat{:entries [{:message/id 0 324 | :message/text "foo" 325 | :chat.entry/timestamp "1234"} 326 | {:message/id 1 327 | :message/text "bar" 328 | :chat.entry/timestamp "1235"} 329 | {:audio/id 0 330 | :audio/url "audio://asdf.jkl" 331 | :audio/duration 1234 332 | :chat.entry/timestamp "4567"} 333 | {:photo/id 0 334 | :photo/url "photo://asdf_10x10.jkl" 335 | :photo/width 10 336 | :photo/height 10 337 | :chat.entry/timestamp "7890"}]} 338 | (p/pull db1 query))))) 339 | 340 | (testing "not found" 341 | (is (= {} (p/pull {} [:foo]))) 342 | (is (= {} (p/pull {} [:foo :bar :baz]))) 343 | (is (= {} (p/pull {} [:foo {:bar [:asdf]} :baz]))) 344 | 345 | (is (= {:foo "bar"} 346 | (p/pull {:foo "bar"} [:foo {:bar [:asdf]} :baz]))) 347 | (is (= {:bar {:asdf 123}} 348 | (p/pull 349 | {:bar {:asdf 123}} 350 | [:foo {:bar [:asdf :jkl]} :baz]))) 351 | (is (= {:bar {}} 352 | (p/pull 353 | (p/db [{:bar {:bar/id 0}} 354 | {:bar/id 0 355 | :qwerty 1234}]) 356 | [:foo {:bar [:asdf :jkl]} :baz]))) 357 | (is (= {:bar {:asdf "jkl"}} 358 | (p/pull 359 | (p/db [{:bar {:bar/id 0}} 360 | {:bar/id 0 361 | :asdf "jkl"}]) 362 | [:foo {:bar [:asdf :jkl]} :baz]))) 363 | (is (= {:bar {}} 364 | (p/pull 365 | (p/db [{:bar {:bar/id 0}} 366 | {:bar/id 1 367 | :asdf "jkl"}]) 368 | [:foo {:bar [:asdf :jkl]} :baz]))) 369 | 370 | (is (= {:foo [{:bar/id 1 371 | :bar/name "asdf"} 372 | {:baz/id 1 373 | :baz/name "jkl"}]} 374 | (p/pull 375 | (p/db [{:foo [{:bar/id 1 376 | :bar/name "asdf"} 377 | {:baz/id 1 378 | :baz/name "jkl"}]}]) 379 | [{:foo {:bar/id [:bar/id :bar/name] 380 | :baz/id [:baz/id :baz/name]}}]))) 381 | 382 | (is (= {:foo [{:bar/id 1 383 | :bar/name "asdf"} 384 | {:bar/id 2} 385 | {:baz/id 1 386 | :baz/name "jkl"}]} 387 | (p/pull 388 | (p/db [{:foo [{:bar/id 1 389 | :bar/name "asdf"} 390 | {:bar/id 2} 391 | {:baz/id 1 392 | :baz/name "jkl"}]}]) 393 | [{:foo {:bar/id [:bar/id :bar/name] 394 | :baz/id [:baz/id :baz/name]}}])))) 395 | 396 | (testing "bounded recursion" 397 | (let [data {:entries 398 | {:entry/id "foo" 399 | :entry/folders 400 | [{:entry/id "bar"} 401 | {:entry/id "baz" 402 | :entry/folders 403 | [{:entry/id "asdf" 404 | :entry/folders 405 | [{:entry/id "qwerty"}]} 406 | {:entry/id "jkl" 407 | :entry/folders 408 | [{:entry/id "uiop"}]}]}]}} 409 | db (p/db [data])] 410 | (is (= {:entries 411 | {:entry/id "foo" 412 | :entry/folders 413 | []}} 414 | (p/pull db '[{:entries [:entry/id 415 | {:entry/folders 0}]}]))) 416 | (is (= {:entries 417 | {:entry/id "foo" 418 | :entry/folders 419 | [{:entry/id "bar"} 420 | {:entry/id "baz" 421 | :entry/folders []}]}} 422 | (p/pull db '[{:entries [:entry/id 423 | {:entry/folders 1}]}]))) 424 | (is (= {:entries 425 | {:entry/id "foo" 426 | :entry/folders 427 | [{:entry/id "bar"} 428 | {:entry/id "baz" 429 | :entry/folders 430 | [{:entry/id "asdf" 431 | :entry/folders []} 432 | {:entry/id "jkl" 433 | :entry/folders []}]}]}} 434 | (p/pull db '[{:entries [:entry/id 435 | {:entry/folders 2}]}]))) 436 | (is (= {:entries 437 | {:entry/id "foo" 438 | :entry/folders 439 | [{:entry/id "bar"} 440 | {:entry/id "baz" 441 | :entry/folders 442 | [{:entry/id "asdf" 443 | :entry/folders 444 | [{:entry/id "qwerty"}]} 445 | {:entry/id "jkl" 446 | :entry/folders 447 | [{:entry/id "uiop"}]}]}]}} 448 | (p/pull db '[{:entries [:entry/id 449 | {:entry/folders 3}]}]))) 450 | (is (= {:entries 451 | {:entry/id "foo" 452 | :entry/folders 453 | [{:entry/id "bar"} 454 | {:entry/id "baz" 455 | :entry/folders 456 | [{:entry/id "asdf" 457 | :entry/folders 458 | [{:entry/id "qwerty"}]} 459 | {:entry/id "jkl" 460 | :entry/folders 461 | [{:entry/id "uiop"}]}]}]}} 462 | (p/pull db '[{:entries [:entry/id 463 | {:entry/folders 10}]}]))))) 464 | 465 | (testing "infinite recursion" 466 | (let [data {:entries 467 | {:entry/id "foo" 468 | :entry/folders 469 | [{:entry/id "bar"} 470 | {:entry/id "baz" 471 | :entry/folders 472 | [{:entry/id "asdf" 473 | :entry/folders 474 | [{:entry/id "qwerty"}]} 475 | {:entry/id "jkl" 476 | :entry/folders 477 | [{:entry/id "uiop"}]}]}]}} 478 | db (p/db [data])] 479 | (is (= data 480 | (p/pull db '[{:entries [:entry/id 481 | {:entry/folders ...}]}]))))) 482 | 483 | (testing "query metadata" 484 | (is (-> db 485 | (p/pull ^:foo []) 486 | (meta) 487 | (:foo)) 488 | "root") 489 | (is (-> db 490 | (p/pull [^:foo {[:person/id 0] [:person/name]}]) 491 | (get [:person/id 0]) 492 | (meta) 493 | (:foo)) 494 | "join") 495 | (let [data {:chat/entries 496 | [{:message/id 0 497 | :message/text "foo" 498 | :chat.entry/timestamp "1234"} 499 | {:message/id 1 500 | :message/text "bar" 501 | :chat.entry/timestamp "1235"} 502 | {:audio/id 0 503 | :audio/url "audio://asdf.jkl" 504 | :audio/duration 1234 505 | :chat.entry/timestamp "4567"} 506 | {:photo/id 0 507 | :photo/url "photo://asdf_10x10.jkl" 508 | :photo/height 10 509 | :photo/width 10 510 | :chat.entry/timestamp "7890"}]} 511 | db1 (p/db [data]) 512 | query ^:foo [^:bar 513 | {:chat/entries 514 | {:message/id 515 | [:message/id :message/text :chat.entry/timestamp] 516 | 517 | :audio/id 518 | [:audio/id :audio/url :audio/duration :chat.entry/timestamp] 519 | 520 | :photo/id 521 | [:photo/id :photo/url :photo/width :photo/height :chat.entry/timestamp] 522 | 523 | :asdf/jkl [:asdf/jkl]}}] 524 | result (p/pull db1 query)] 525 | (is (= #:chat{:entries [{:message/id 0 526 | :message/text "foo" 527 | :chat.entry/timestamp "1234"} 528 | {:message/id 1 529 | :message/text "bar" 530 | :chat.entry/timestamp "1235"} 531 | {:audio/id 0 532 | :audio/url "audio://asdf.jkl" 533 | :audio/duration 1234 534 | :chat.entry/timestamp "4567"} 535 | {:photo/id 0 536 | :photo/url "photo://asdf_10x10.jkl" 537 | :photo/width 10 538 | :photo/height 10 539 | :chat.entry/timestamp "7890"}]} 540 | result)) 541 | (is (-> result meta :foo)) 542 | (is (every? #(:bar (meta %)) (get result :chat/entries))))) 543 | (testing "dangling entities" 544 | (is (= {[:id 0] {:friends [{:id 1} {:id 2}]}} 545 | (p/pull 546 | {:id {0 {:id 0 :name "asdf" :friends [[:id 1] [:id 2]]} 547 | 1 {:id 1 :name "jkl"}}} 548 | [{[:id 0] [:friends]}])) 549 | "dangling entity shows up in queries that do not select any props") 550 | (is (= {[:id 0] {:friends [{:id 1, :name "jkl"} {:id 2}]}} 551 | (p/pull 552 | {:id {0 {:id 0 :name "asdf" :friends [[:id 1] [:id 2]]} 553 | 1 {:id 1 :name "jkl"}}} 554 | [{[:id 0] [{:friends [:id :name]}]}])) 555 | "dangling entity shows up in queries that include ID") 556 | (is (= {[:id 0] {:friends [{:name "jkl"}]}} 557 | (p/pull 558 | {:id {0 {:id 0 :name "asdf" :friends [[:id 1] [:id 2]]} 559 | 1 {:id 1 :name "jkl"}}} 560 | [{[:id 0] [{:friends [:name]}]}])) 561 | "dangling entity does not show up in queries that do not include ID"))) 562 | 563 | 564 | (deftest pull-report 565 | (is (= {:data {:people/all [{:person/name "Alice"} 566 | {:person/name "Bob"}]} 567 | :entities #{[:person/id 0] [:person/id 1]} 568 | :indices #{:people/all}} 569 | (p/pull-report db [{:people/all [:person/name]}])) 570 | "basic join + prop") 571 | (is (= {:data #:people{:all [{:person/name "Alice" 572 | :best-friend #:person{:name "Bob", :id 1 :age 23}} 573 | #:person{:name "Bob"}]} 574 | :entities #{[:person/id 0] [:person/id 1]} 575 | :indices #{:people/all}} 576 | (p/pull-report db [#:people{:all [:person/name :best-friend]}])) 577 | "join + prop + join ref lookup") 578 | (is (= {:data {[:person/id 1] #:person{:id 1, :name "Bob", :age 23}} 579 | :entities #{[:person/id 1]} 580 | :indices #{}} 581 | (p/pull-report db [[:person/id 1]])) 582 | "ident acts as ref lookup") 583 | (is (= {:data {[:person/id 0] {:person/id 0 584 | :person/name "Alice" 585 | :person/age 25 586 | :best-friend {:person/id 1} 587 | :person/favorites #:favorite{:ice-cream "vanilla"}}} 588 | :entities #{[:person/id 0]} 589 | :indices #{}} 590 | (p/pull-report db [[:person/id 0]])) 591 | "ident does not resolve nested refs")) 592 | 593 | 594 | (deftest delete 595 | (is (= {:people/all [[:person/id 0]] 596 | :person/id {0 {:person/id 0 597 | :person/name "Alice" 598 | :person/age 25 599 | :person/favorites #:favorite{:ice-cream "vanilla"}}}} 600 | (p/delete db [:person/id 1]))) 601 | (is (= (-> {} 602 | (p/delete [:person/id 1]) 603 | (p/add {:person/id 1 :person/name "Alice"})) 604 | {:person/id {1 {:person/id 1 :person/name "Alice"}}}))) 605 | 606 | 607 | 608 | (deftest data->query 609 | (is (= [:a] 610 | (p/data->query {:a 42}))) 611 | (is (= [{:a [:b]}] 612 | (p/data->query {:a {:b 42}}))) 613 | (is (= [{:a [:b :c]}] 614 | (p/data->query {:a [{:b 42} {:c :d}]}))) 615 | (is (= [{[:a 42] [:b]}] 616 | (p/data->query {[:a 42] {:b 33}})))) 617 | 618 | (comment 619 | (clojure.test/run-tests)) 620 | -------------------------------------------------------------------------------- /test/pyramid/pull_test.cljc: -------------------------------------------------------------------------------- 1 | (ns pyramid.pull-test 2 | (:require 3 | [clojure.test :refer [deftest is testing]] 4 | [pyramid.pull :as p])) 5 | 6 | 7 | (def entities 8 | (for [i (range 1000)] 9 | [:id i])) 10 | 11 | 12 | (deftest many-entities 13 | (is (= (set entities) 14 | (:entities 15 | (trampoline 16 | p/pull-report 17 | {:id (into 18 | {} 19 | (map #(vector 20 | (second %) 21 | (hash-map (first %) (second %)))) 22 | entities) 23 | :all (vec entities)} 24 | [{:all [:id]}]))))) 25 | 26 | 27 | (deftest list-order 28 | (is (= 29 | '({:thing {:id 1}} {:thing {:id 2}} {:thing {:id 3}}), 30 | (let [db {:id {1 {:id 1} 31 | 2 {:id 2} 32 | 3 {:id 3} 33 | 9 {:id 9 34 | :my-list '({:thing [:id 1]} 35 | {:thing [:id 2]} 36 | {:thing [:id 3]})}}} 37 | query [{[:id 9] [:id {:my-list [{:thing [:id]}]}]}]] 38 | (-> (trampoline p/pull-report db query) 39 | (get-in [:data [:id 9] :my-list])))))) 40 | 41 | 42 | (deftest heterogeneous-colls 43 | (let [db {:person/id {0 {:person/id 0 44 | :person/name "Bill" 45 | :person/friends [{:person/name "Bob"} 46 | [:person/id 2]]} 47 | 2 {:person/id 2 48 | :person/name "Alice"}}} 49 | query [{[:person/id 0] [:person/name {:person/friends [:person/name]}]}]] 50 | (is 51 | (= {[:person/id 0] {:person/name "Bill" 52 | :person/friends [{:person/name "Bob"} 53 | {:person/name "Alice"}]}} 54 | (:data (trampoline p/pull-report db query)))))) 55 | 56 | 57 | (defrecord Visited [query result]) 58 | 59 | 60 | (defn visit 61 | [q] 62 | (with-meta q {:visitor #(->Visited (doto q prn) %2)})) 63 | 64 | 65 | (deftype Opaque [result]) 66 | 67 | 68 | (deftest visitors 69 | (testing "simple join" 70 | (let [query [{:foo (visit [:bar :baz])}] 71 | data {:foo {:bar 123 :baz 456}}] 72 | (is (= {:foo (->Visited 73 | [:bar :baz] 74 | {:bar 123 :baz 456})} 75 | (:data (trampoline p/pull-report data query))) 76 | "single item")) 77 | (let [query [{:foo (visit [:bar :baz])}] 78 | data {:foo [{:bar 123 :baz 456} 79 | {:bar 789 :baz "qux"}]}] 80 | (is (= {:foo [(->Visited [:bar :baz] {:bar 123 :baz 456}) 81 | (->Visited [:bar :baz] {:bar 789 :baz "qux"})]} 82 | (:data (trampoline p/pull-report data query))) 83 | "multiple items"))) 84 | (testing "nested join" 85 | (let [query [{:foo (visit [{:bar [:baz]}])}] 86 | data {:foo {:bar {:baz 123}}}] 87 | (is (= {:foo (->Visited 88 | [{:bar [:baz]}] 89 | {:bar {:baz 123}})} 90 | (:data (trampoline p/pull-report data query)))))) 91 | (testing "nested visitors" 92 | (let [query [{:foo (visit [{:bar (visit [:baz])}])}] 93 | data {:foo [{:bar {:baz 123}} 94 | {:bar {:baz 456}}]}] 95 | (is (= {:foo [(->Visited 96 | [{:bar [:baz]}] 97 | {:bar (->Visited 98 | [:baz] 99 | {:baz 123})}) 100 | (->Visited 101 | [{:bar [:baz]}] 102 | {:bar (->Visited 103 | [:baz] 104 | {:baz 456})})]} 105 | (:data (trampoline p/pull-report data query)))))) 106 | (testing "union" 107 | (let [query [{:foo (visit 108 | {:bar [:bar :asdf :jkl] 109 | :baz [:baz :arst :nei] 110 | :qux [:qux :qwfp :luy]})}] 111 | data {:foo {:bar 2 :asdf 123 :jkl 456}}] 112 | (is (= {:foo (->Visited {:bar [:bar :asdf :jkl] 113 | :baz [:baz :arst :nei] 114 | :qux [:qux :qwfp :luy]} 115 | {:bar 2 :asdf 123 :jkl 456})} 116 | (:data (trampoline p/pull-report data query))) 117 | "whole union in visitor")) 118 | (let [query [{:foo {:bar (visit [:bar :asdf :jkl]) 119 | :baz (visit [:baz :arst :nei]) 120 | :qux [:qux :qwfp :luy]}}] 121 | data {:foo {:bar 2 :asdf 123 :jkl 456}}] 122 | (is (= {:foo (->Visited [:bar :asdf :jkl] 123 | {:bar 2 :asdf 123 :jkl 456})} 124 | (:data (trampoline p/pull-report data query))) 125 | "single item union entry")) 126 | (let [query [{:foo {:bar (visit [:bar :asdf :jkl]) 127 | :baz (visit [:baz :arst :nei]) 128 | :qux [:qux :qwfp :luy]}}] 129 | data {:foo [{:qux 1 :qwfp 123 :luy 456} 130 | {:bar 2 :asdf 123 :jkl 456} 131 | {:baz 3 :arst 123 :nei 457}]}] 132 | (is (= {:foo [{:qux 1 :qwfp 123 :luy 456} 133 | (->Visited [:bar :asdf :jkl] 134 | {:bar 2 :asdf 123 :jkl 456}) 135 | (->Visited [:baz :arst :nei] 136 | {:baz 3 :arst 123 :nei 457})]} 137 | (:data (trampoline p/pull-report data query))) 138 | "vector union entry") 139 | (is (= {:foo [{:qux 1 :qwfp 123 :luy 456} 140 | (->Visited [:bar :asdf :jkl] 141 | {:bar 2 :asdf 123 :jkl 456}) 142 | (->Visited [:baz :arst :nei] 143 | {:baz 3 :arst 123 :nei 457})]} 144 | (:data (trampoline p/pull-report (update data :foo seq) query))) 145 | "seq union entry"))) 146 | (testing "opaque transform" 147 | (let [query [{:foo 148 | ^{:visitor #(->Opaque %2)} 149 | [{:bar 150 | ^{:visitor #(->Opaque %2)} 151 | [:baz]}]}] 152 | data {:foo {:bar {:baz 123}}}] 153 | (is (instance? 154 | Opaque 155 | (-> (trampoline p/pull-report data query) 156 | (:data) 157 | (:foo)))) 158 | (is (instance? 159 | Opaque 160 | (-> (trampoline p/pull-report data query) 161 | (:data) 162 | (:foo) 163 | (.-result) 164 | (:bar))))))) 165 | 166 | 167 | (deftest issue-36 168 | (let [db '{:zones ([:zone/id "b3d91e54-75e0-424b-b4c6-363cbd0ff06a"] 169 | [:zone/id "4fba72c3-97ec-4458-9e78-b75bd6b323d0"] 170 | [:zone/id "5a3f1bf5-aee1-40f1-9ea3-313ea3123c90"])} 171 | q [{:zones [:zone/id]}]] 172 | (is (= '{:zones 173 | (#:zone{:id "b3d91e54-75e0-424b-b4c6-363cbd0ff06a"} 174 | #:zone{:id "4fba72c3-97ec-4458-9e78-b75bd6b323d0"} 175 | #:zone{:id "5a3f1bf5-aee1-40f1-9ea3-313ea3123c90"})} 176 | (:data (trampoline p/pull-report db q)))))) 177 | -------------------------------------------------------------------------------- /test/pyramid/query_test.cljc: -------------------------------------------------------------------------------- 1 | (ns pyramid.query-test 2 | (:require 3 | [clojure.test :refer [deftest is]] 4 | [pyramid.core :as core] 5 | [pyramid.query :as p.q :refer [q]])) 6 | 7 | 8 | (def db 9 | ^{`p.q/entities core/entities} 10 | {:person/id {"123" {:person/id "123" 11 | :person/name "foo" 12 | :person/best-friend [:person/id "789"] 13 | :person/friends [[:person/id "456"] 14 | [:person/id "789"]]} 15 | "456" {:person/id "456" 16 | :person/name "bar" 17 | :person/friends [[:person/id "123"] 18 | [:person/id "789"]]} 19 | "789" {:person/id "789" 20 | :person/name "baz" 21 | :person/friends [[:person/id "123"] 22 | [:person/id "456"]]} 23 | "1011" {:person/id "1011" 24 | :meta {:asdf [{:jkl 42} 25 | {:jkl 84} 26 | {:jkl 128}]}}} 27 | :person {:bar "baz"} 28 | :asdf "jkl"}) 29 | 30 | 31 | (deftest joins 32 | (is (= '([42] [84] [128]) 33 | (q '[:find ?jkl 34 | :where 35 | [?e :person/id "1011"] 36 | [?e :meta ?meta] 37 | [?meta :asdf ^:many ?asdf] 38 | [?asdf :jkl ?jkl]] 39 | db)) 40 | "map entities") 41 | 42 | (is (= '(["123"] ["456"] ["789"] ["1011"]) 43 | (q '[:find ?id 44 | :where 45 | [?e :person/id ?id]] 46 | db))) 47 | 48 | (is (= '(["123" "foo"] ["456" "bar"] ["789" "baz"]) 49 | (q '[:find ?id ?name 50 | :where 51 | [?e :person/id ?id] 52 | [?e :person/name ?name]] 53 | db))) 54 | 55 | (is (= '(["123" "foo" [:person/id "789"]]) 56 | (q '[:find ?id ?name ?friend 57 | :where 58 | [?e :person/id ?id] 59 | [?e :person/name ?name] 60 | [?e :person/best-friend ?friend]] 61 | db))) 62 | 63 | (is (= '(["foo" "baz"]) 64 | (q '[:find ?name ?friend-name 65 | :where 66 | [?e :person/name ?name] 67 | [?e :person/best-friend ?friend] 68 | [?friend :person/name ?friend-name]] 69 | db))) 70 | 71 | (is (= '() 72 | (q '[:find ?id 73 | :where 74 | [?e :person/name "asdf"] 75 | [?e :person/id ?id]] 76 | db)) 77 | "not found") 78 | 79 | (is (= '(["123" "foo"]) 80 | (q '[:find ?id ?name 81 | :in $ ?name 82 | :where 83 | [?e :person/name ?name] 84 | [?e :person/id ?id]] 85 | db 86 | "foo")) 87 | "join on :in") 88 | 89 | (is (= '(["foo" "bar"] 90 | ["foo" "baz"] 91 | ["bar" "foo"] 92 | ["bar" "baz"] 93 | ["baz" "foo"] 94 | ["baz" "bar"]) 95 | (q '[:find ?name ?friend-name 96 | :where 97 | [?e :person/name ?name] 98 | [?e :person/friends ^:many ?friend] 99 | [?friend :person/name ?friend-name]] 100 | db)) 101 | "multiple cardinality value") 102 | 103 | (is (= '(["123" "foo"] 104 | ["456" "bar"]) 105 | (q '[:find ?id ?name 106 | :in $ ^:many ?name 107 | :where 108 | [?e :person/name ?name] 109 | [?e :person/id ?id]] 110 | db 111 | ["foo" "bar"])) 112 | "multi cardinality join on :in") 113 | 114 | (is (= '(["foo" "foo"] 115 | ["foo" "bar"] 116 | ["foo" "baz"] 117 | ["bar" "foo"] 118 | ["bar" "bar"] 119 | ["bar" "baz"] 120 | ["baz" "foo"] 121 | ["baz" "bar"] 122 | ["baz" "baz"]) 123 | (q '[:find ?name1 ?name2 124 | :where 125 | [?e1 :person/name ?name1] 126 | [?e2 :person/name ?name2]] 127 | db)) 128 | "cross product")) 129 | 130 | 131 | (deftest query-map 132 | (is (= [[42]] 133 | (q '[:find ?baz 134 | :where 135 | [[:foo] :bar ?bar] 136 | [?bar :baz ?baz]] 137 | {:foo {:bar {:baz 42}}}))) 138 | (is (= [[:foo {:bar {:baz {:asdf 42}}}] 139 | [:bar {:baz {:asdf 42}}] 140 | [:baz {:asdf 42}] 141 | [:asdf 42]] 142 | (q '[:find ?a ?v 143 | :where 144 | [?e ?a ?v]] 145 | {:foo {:bar {:baz {:asdf 42}}}}))) 146 | (is (= [[{:asdf "jkl"}] [{:asdf "qwerty"}] [{:asdf "uiop"}]] 147 | (q '[:find ?bar 148 | :where 149 | [[:foo] :bar ^:many ?bar] 150 | [?bar ?a ?v]] 151 | {:foo {:bar [{:asdf "jkl"} 152 | {:asdf "qwerty"} 153 | {:asdf "uiop"}]}})))) 154 | -------------------------------------------------------------------------------- /tests.edn: -------------------------------------------------------------------------------- 1 | #kaocha/v1 2 | #meta-merge [{:tests [{:id :clj 3 | :type :kaocha.type/clojure.test} 4 | {:id :cljs 5 | :type :kaocha.type/cljs}]} ] 6 | --------------------------------------------------------------------------------