├── .circleci └── config.yml ├── .dir-locals.el ├── .dockerignore ├── .gitignore ├── CHANGELOG.md ├── Dockerfile ├── LICENSE.txt ├── README.md ├── bin ├── cider_connect ├── cider_connect.el ├── feature_table ├── feature_table.ed ├── kaocha └── start_metabase ├── deps.edn ├── dev ├── user.clj └── user │ ├── repl.clj │ └── setup.clj ├── doc └── architecture_decision_log.org ├── features.yml ├── project.clj ├── repl_sessions ├── navigation.clj ├── query_checks.clj ├── schema.clj ├── test_data.clj └── undo_special_types.clj ├── resources └── metabase-plugin.yaml ├── src └── metabase │ └── driver │ ├── datomic.clj │ └── datomic │ ├── fix_types.clj │ ├── monkey_patch.clj │ ├── query_processor.clj │ └── util.clj ├── test ├── metabase │ ├── driver │ │ ├── datomic │ │ │ ├── aggregation_test.clj │ │ │ ├── breakout_test.clj │ │ │ ├── datomic_test.clj │ │ │ ├── fields_test.clj │ │ │ ├── filter_test.clj │ │ │ ├── joins_test.clj │ │ │ ├── order_by_test.clj │ │ │ ├── query_processor_test.clj │ │ │ ├── source_query_test.clj │ │ │ ├── test.clj │ │ │ └── test_data.clj │ │ └── datomic_test.clj │ └── test │ │ └── data │ │ └── datomic.clj └── metabase_datomic │ └── kaocha_hooks.clj └── tests.edn /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2.1 2 | 3 | orbs: 4 | kaocha: lambdaisland/kaocha@dev:first 5 | clojure: lambdaisland/clojure@dev:first 6 | 7 | commands: 8 | checkout_and_run: 9 | parameters: 10 | clojure_version: 11 | type: string 12 | steps: 13 | - checkout 14 | - clojure/with_cache: 15 | cache_version: << parameters.clojure_version >> 16 | steps: 17 | - kaocha/execute: 18 | args: "--reporter documentation --plugin cloverage --codecov" 19 | clojure_version: << parameters.clojure_version >> 20 | - kaocha/upload_codecov 21 | 22 | jobs: 23 | java-11-clojure-1_10: 24 | executor: clojure/openjdk11 25 | steps: [{checkout_and_run: {clojure_version: "1.10.0-RC5"}}] 26 | 27 | java-11-clojure-1_9: 28 | executor: clojure/openjdk11 29 | steps: [{checkout_and_run: {clojure_version: "1.9.0"}}] 30 | 31 | java-9-clojure-1_10: 32 | executor: clojure/openjdk9 33 | steps: [{checkout_and_run: {clojure_version: "1.10.0-RC5"}}] 34 | 35 | java-9-clojure-1_9: 36 | executor: clojure/openjdk9 37 | steps: [{checkout_and_run: {clojure_version: "1.9.0"}}] 38 | 39 | java-8-clojure-1_10: 40 | executor: clojure/openjdk8 41 | steps: [{checkout_and_run: {clojure_version: "1.10.0-RC5"}}] 42 | 43 | java-8-clojure-1_9: 44 | executor: clojure/openjdk8 45 | steps: [{checkout_and_run: {clojure_version: "1.9.0"}}] 46 | 47 | workflows: 48 | kaocha_test: 49 | jobs: 50 | - java-11-clojure-1_10 51 | - java-11-clojure-1_9 52 | - java-9-clojure-1_10 53 | - java-9-clojure-1_9 54 | - java-8-clojure-1_10 55 | - java-8-clojure-1_9 56 | -------------------------------------------------------------------------------- /.dir-locals.el: -------------------------------------------------------------------------------- 1 | ((nil . ((cider-clojure-cli-global-options . "-A:dev:test:datomic-free") 2 | (cider-redirect-server-output-to-repl . nil)))) 3 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | .cpcache 2 | .nrepl-port 3 | target 4 | repl 5 | scratch.clj 6 | *.db 7 | .lein-repl-history 8 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .cpcache 2 | .nrepl-port 3 | target 4 | repl 5 | scratch.clj 6 | *.db 7 | .lein-repl-history 8 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Unreleased 2 | 3 | ## Added 4 | 5 | ## Fixed 6 | 7 | ## Changed 8 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | ARG METABASE_VERSION=v0.32.8 2 | 3 | FROM metabase/metabase:${METABASE_VERSION} as builder 4 | ARG CLOJURE_CLI_VERSION=1.10.1.447 5 | 6 | WORKDIR /app/metabase-datomic 7 | 8 | RUN apk add --no-cache curl 9 | 10 | ADD https://download.clojure.org/install/linux-install-${CLOJURE_CLI_VERSION}.sh ./clojure-cli-linux-install.sh 11 | RUN chmod 744 clojure-cli-linux-install.sh 12 | RUN ./clojure-cli-linux-install.sh 13 | RUN rm clojure-cli-linux-install.sh 14 | 15 | ADD https://raw.github.com/technomancy/leiningen/stable/bin/lein /usr/local/bin/lein 16 | RUN chmod 744 /usr/local/bin/lein 17 | RUN lein upgrade 18 | 19 | COPY . ./ 20 | 21 | ENV CLASSPATH=/app/metabase.jar 22 | RUN lein with-profiles +datomic-free uberjar 23 | 24 | FROM metabase/metabase:${METABASE_VERSION} as runner 25 | 26 | COPY --from=builder /app/metabase-datomic/target/uberjar/datomic.metabase-driver.jar /plugins 27 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Mozilla Public License Version 2.0 2 | ================================== 3 | 4 | 1. Definitions 5 | -------------- 6 | 7 | 1.1. "Contributor" 8 | means each individual or legal entity that creates, contributes to 9 | the creation of, or owns Covered Software. 10 | 11 | 1.2. "Contributor Version" 12 | means the combination of the Contributions of others (if any) used 13 | by a Contributor and that particular Contributor's Contribution. 14 | 15 | 1.3. "Contribution" 16 | means Covered Software of a particular Contributor. 17 | 18 | 1.4. "Covered Software" 19 | means Source Code Form to which the initial Contributor has attached 20 | the notice in Exhibit A, the Executable Form of such Source Code 21 | Form, and Modifications of such Source Code Form, in each case 22 | including portions thereof. 23 | 24 | 1.5. "Incompatible With Secondary Licenses" 25 | means 26 | 27 | (a) that the initial Contributor has attached the notice described 28 | in Exhibit B to the Covered Software; or 29 | 30 | (b) that the Covered Software was made available under the terms of 31 | version 1.1 or earlier of the License, but not also under the 32 | terms of a Secondary License. 33 | 34 | 1.6. "Executable Form" 35 | means any form of the work other than Source Code Form. 36 | 37 | 1.7. "Larger Work" 38 | means a work that combines Covered Software with other material, in 39 | a separate file or files, that is not Covered Software. 40 | 41 | 1.8. "License" 42 | means this document. 43 | 44 | 1.9. "Licensable" 45 | means having the right to grant, to the maximum extent possible, 46 | whether at the time of the initial grant or subsequently, any and 47 | all of the rights conveyed by this License. 48 | 49 | 1.10. "Modifications" 50 | means any of the following: 51 | 52 | (a) any file in Source Code Form that results from an addition to, 53 | deletion from, or modification of the contents of Covered 54 | Software; or 55 | 56 | (b) any new file in Source Code Form that contains any Covered 57 | Software. 58 | 59 | 1.11. "Patent Claims" of a Contributor 60 | means any patent claim(s), including without limitation, method, 61 | process, and apparatus claims, in any patent Licensable by such 62 | Contributor that would be infringed, but for the grant of the 63 | License, by the making, using, selling, offering for sale, having 64 | made, import, or transfer of either its Contributions or its 65 | Contributor Version. 66 | 67 | 1.12. "Secondary License" 68 | means either the GNU General Public License, Version 2.0, the GNU 69 | Lesser General Public License, Version 2.1, the GNU Affero General 70 | Public License, Version 3.0, or any later versions of those 71 | licenses. 72 | 73 | 1.13. "Source Code Form" 74 | means the form of the work preferred for making modifications. 75 | 76 | 1.14. "You" (or "Your") 77 | means an individual or a legal entity exercising rights under this 78 | License. For legal entities, "You" includes any entity that 79 | controls, is controlled by, or is under common control with You. For 80 | purposes of this definition, "control" means (a) the power, direct 81 | or indirect, to cause the direction or management of such entity, 82 | whether by contract or otherwise, or (b) ownership of more than 83 | fifty percent (50%) of the outstanding shares or beneficial 84 | ownership of such entity. 85 | 86 | 2. License Grants and Conditions 87 | -------------------------------- 88 | 89 | 2.1. Grants 90 | 91 | Each Contributor hereby grants You a world-wide, royalty-free, 92 | non-exclusive license: 93 | 94 | (a) under intellectual property rights (other than patent or trademark) 95 | Licensable by such Contributor to use, reproduce, make available, 96 | modify, display, perform, distribute, and otherwise exploit its 97 | Contributions, either on an unmodified basis, with Modifications, or 98 | as part of a Larger Work; and 99 | 100 | (b) under Patent Claims of such Contributor to make, use, sell, offer 101 | for sale, have made, import, and otherwise transfer either its 102 | Contributions or its Contributor Version. 103 | 104 | 2.2. Effective Date 105 | 106 | The licenses granted in Section 2.1 with respect to any Contribution 107 | become effective for each Contribution on the date the Contributor first 108 | distributes such Contribution. 109 | 110 | 2.3. Limitations on Grant Scope 111 | 112 | The licenses granted in this Section 2 are the only rights granted under 113 | this License. No additional rights or licenses will be implied from the 114 | distribution or licensing of Covered Software under this License. 115 | Notwithstanding Section 2.1(b) above, no patent license is granted by a 116 | Contributor: 117 | 118 | (a) for any code that a Contributor has removed from Covered Software; 119 | or 120 | 121 | (b) for infringements caused by: (i) Your and any other third party's 122 | modifications of Covered Software, or (ii) the combination of its 123 | Contributions with other software (except as part of its Contributor 124 | Version); or 125 | 126 | (c) under Patent Claims infringed by Covered Software in the absence of 127 | its Contributions. 128 | 129 | This License does not grant any rights in the trademarks, service marks, 130 | or logos of any Contributor (except as may be necessary to comply with 131 | the notice requirements in Section 3.4). 132 | 133 | 2.4. Subsequent Licenses 134 | 135 | No Contributor makes additional grants as a result of Your choice to 136 | distribute the Covered Software under a subsequent version of this 137 | License (see Section 10.2) or under the terms of a Secondary License (if 138 | permitted under the terms of Section 3.3). 139 | 140 | 2.5. Representation 141 | 142 | Each Contributor represents that the Contributor believes its 143 | Contributions are its original creation(s) or it has sufficient rights 144 | to grant the rights to its Contributions conveyed by this License. 145 | 146 | 2.6. Fair Use 147 | 148 | This License is not intended to limit any rights You have under 149 | applicable copyright doctrines of fair use, fair dealing, or other 150 | equivalents. 151 | 152 | 2.7. Conditions 153 | 154 | Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted 155 | in Section 2.1. 156 | 157 | 3. Responsibilities 158 | ------------------- 159 | 160 | 3.1. Distribution of Source Form 161 | 162 | All distribution of Covered Software in Source Code Form, including any 163 | Modifications that You create or to which You contribute, must be under 164 | the terms of this License. You must inform recipients that the Source 165 | Code Form of the Covered Software is governed by the terms of this 166 | License, and how they can obtain a copy of this License. You may not 167 | attempt to alter or restrict the recipients' rights in the Source Code 168 | Form. 169 | 170 | 3.2. Distribution of Executable Form 171 | 172 | If You distribute Covered Software in Executable Form then: 173 | 174 | (a) such Covered Software must also be made available in Source Code 175 | Form, as described in Section 3.1, and You must inform recipients of 176 | the Executable Form how they can obtain a copy of such Source Code 177 | Form by reasonable means in a timely manner, at a charge no more 178 | than the cost of distribution to the recipient; and 179 | 180 | (b) You may distribute such Executable Form under the terms of this 181 | License, or sublicense it under different terms, provided that the 182 | license for the Executable Form does not attempt to limit or alter 183 | the recipients' rights in the Source Code Form under this License. 184 | 185 | 3.3. Distribution of a Larger Work 186 | 187 | You may create and distribute a Larger Work under terms of Your choice, 188 | provided that You also comply with the requirements of this License for 189 | the Covered Software. If the Larger Work is a combination of Covered 190 | Software with a work governed by one or more Secondary Licenses, and the 191 | Covered Software is not Incompatible With Secondary Licenses, this 192 | License permits You to additionally distribute such Covered Software 193 | under the terms of such Secondary License(s), so that the recipient of 194 | the Larger Work may, at their option, further distribute the Covered 195 | Software under the terms of either this License or such Secondary 196 | License(s). 197 | 198 | 3.4. Notices 199 | 200 | You may not remove or alter the substance of any license notices 201 | (including copyright notices, patent notices, disclaimers of warranty, 202 | or limitations of liability) contained within the Source Code Form of 203 | the Covered Software, except that You may alter any license notices to 204 | the extent required to remedy known factual inaccuracies. 205 | 206 | 3.5. Application of Additional Terms 207 | 208 | You may choose to offer, and to charge a fee for, warranty, support, 209 | indemnity or liability obligations to one or more recipients of Covered 210 | Software. However, You may do so only on Your own behalf, and not on 211 | behalf of any Contributor. You must make it absolutely clear that any 212 | such warranty, support, indemnity, or liability obligation is offered by 213 | You alone, and You hereby agree to indemnify every Contributor for any 214 | liability incurred by such Contributor as a result of warranty, support, 215 | indemnity or liability terms You offer. You may include additional 216 | disclaimers of warranty and limitations of liability specific to any 217 | jurisdiction. 218 | 219 | 4. Inability to Comply Due to Statute or Regulation 220 | --------------------------------------------------- 221 | 222 | If it is impossible for You to comply with any of the terms of this 223 | License with respect to some or all of the Covered Software due to 224 | statute, judicial order, or regulation then You must: (a) comply with 225 | the terms of this License to the maximum extent possible; and (b) 226 | describe the limitations and the code they affect. Such description must 227 | be placed in a text file included with all distributions of the Covered 228 | Software under this License. Except to the extent prohibited by statute 229 | or regulation, such description must be sufficiently detailed for a 230 | recipient of ordinary skill to be able to understand it. 231 | 232 | 5. Termination 233 | -------------- 234 | 235 | 5.1. The rights granted under this License will terminate automatically 236 | if You fail to comply with any of its terms. However, if You become 237 | compliant, then the rights granted under this License from a particular 238 | Contributor are reinstated (a) provisionally, unless and until such 239 | Contributor explicitly and finally terminates Your grants, and (b) on an 240 | ongoing basis, if such Contributor fails to notify You of the 241 | non-compliance by some reasonable means prior to 60 days after You have 242 | come back into compliance. Moreover, Your grants from a particular 243 | Contributor are reinstated on an ongoing basis if such Contributor 244 | notifies You of the non-compliance by some reasonable means, this is the 245 | first time You have received notice of non-compliance with this License 246 | from such Contributor, and You become compliant prior to 30 days after 247 | Your receipt of the notice. 248 | 249 | 5.2. If You initiate litigation against any entity by asserting a patent 250 | infringement claim (excluding declaratory judgment actions, 251 | counter-claims, and cross-claims) alleging that a Contributor Version 252 | directly or indirectly infringes any patent, then the rights granted to 253 | You by any and all Contributors for the Covered Software under Section 254 | 2.1 of this License shall terminate. 255 | 256 | 5.3. In the event of termination under Sections 5.1 or 5.2 above, all 257 | end user license agreements (excluding distributors and resellers) which 258 | have been validly granted by You or Your distributors under this License 259 | prior to termination shall survive termination. 260 | 261 | ************************************************************************ 262 | * * 263 | * 6. Disclaimer of Warranty * 264 | * ------------------------- * 265 | * * 266 | * Covered Software is provided under this License on an "as is" * 267 | * basis, without warranty of any kind, either expressed, implied, or * 268 | * statutory, including, without limitation, warranties that the * 269 | * Covered Software is free of defects, merchantable, fit for a * 270 | * particular purpose or non-infringing. The entire risk as to the * 271 | * quality and performance of the Covered Software is with You. * 272 | * Should any Covered Software prove defective in any respect, You * 273 | * (not any Contributor) assume the cost of any necessary servicing, * 274 | * repair, or correction. This disclaimer of warranty constitutes an * 275 | * essential part of this License. No use of any Covered Software is * 276 | * authorized under this License except under this disclaimer. * 277 | * * 278 | ************************************************************************ 279 | 280 | ************************************************************************ 281 | * * 282 | * 7. Limitation of Liability * 283 | * -------------------------- * 284 | * * 285 | * Under no circumstances and under no legal theory, whether tort * 286 | * (including negligence), contract, or otherwise, shall any * 287 | * Contributor, or anyone who distributes Covered Software as * 288 | * permitted above, be liable to You for any direct, indirect, * 289 | * special, incidental, or consequential damages of any character * 290 | * including, without limitation, damages for lost profits, loss of * 291 | * goodwill, work stoppage, computer failure or malfunction, or any * 292 | * and all other commercial damages or losses, even if such party * 293 | * shall have been informed of the possibility of such damages. This * 294 | * limitation of liability shall not apply to liability for death or * 295 | * personal injury resulting from such party's negligence to the * 296 | * extent applicable law prohibits such limitation. Some * 297 | * jurisdictions do not allow the exclusion or limitation of * 298 | * incidental or consequential damages, so this exclusion and * 299 | * limitation may not apply to You. * 300 | * * 301 | ************************************************************************ 302 | 303 | 8. Litigation 304 | ------------- 305 | 306 | Any litigation relating to this License may be brought only in the 307 | courts of a jurisdiction where the defendant maintains its principal 308 | place of business and such litigation shall be governed by laws of that 309 | jurisdiction, without reference to its conflict-of-law provisions. 310 | Nothing in this Section shall prevent a party's ability to bring 311 | cross-claims or counter-claims. 312 | 313 | 9. Miscellaneous 314 | ---------------- 315 | 316 | This License represents the complete agreement concerning the subject 317 | matter hereof. If any provision of this License is held to be 318 | unenforceable, such provision shall be reformed only to the extent 319 | necessary to make it enforceable. Any law or regulation which provides 320 | that the language of a contract shall be construed against the drafter 321 | shall not be used to construe this License against a Contributor. 322 | 323 | 10. Versions of the License 324 | --------------------------- 325 | 326 | 10.1. New Versions 327 | 328 | Mozilla Foundation is the license steward. Except as provided in Section 329 | 10.3, no one other than the license steward has the right to modify or 330 | publish new versions of this License. Each version will be given a 331 | distinguishing version number. 332 | 333 | 10.2. Effect of New Versions 334 | 335 | You may distribute the Covered Software under the terms of the version 336 | of the License under which You originally received the Covered Software, 337 | or under the terms of any subsequent version published by the license 338 | steward. 339 | 340 | 10.3. Modified Versions 341 | 342 | If you create software not governed by this License, and you want to 343 | create a new license for such software, you may create and use a 344 | modified version of this License if you rename the license and remove 345 | any references to the name of the license steward (except to note that 346 | such modified license differs from this License). 347 | 348 | 10.4. Distributing Source Code Form that is Incompatible With Secondary 349 | Licenses 350 | 351 | If You choose to distribute Source Code Form that is Incompatible With 352 | Secondary Licenses under the terms of this version of the License, the 353 | notice described in Exhibit B of this License must be attached. 354 | 355 | Exhibit A - Source Code Form License Notice 356 | ------------------------------------------- 357 | 358 | This Source Code Form is subject to the terms of the Mozilla Public 359 | License, v. 2.0. If a copy of the MPL was not distributed with this 360 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 361 | 362 | If it is not possible or desirable to put the notice in a particular 363 | file, then You may include the notice in a location (such as a LICENSE 364 | file in a relevant directory) where a recipient would be likely to look 365 | for such a notice. 366 | 367 | You may add additional accurate notices of copyright ownership. 368 | 369 | Exhibit B - "Incompatible With Secondary Licenses" Notice 370 | --------------------------------------------------------- 371 | 372 | This Source Code Form is "Incompatible With Secondary Licenses", as 373 | defined by the Mozilla Public License, v. 2.0. 374 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # metabase-datomic 2 | 3 | 4 | 5 | 6 | 7 | A Metabase driver for Datomic. 8 | 9 | Commercial support is provided by [Gaiwan](http://gaiwan.co). 10 | 11 | ## Try it! 12 | 13 | ``` 14 | docker run -p 3000:3000 lambdaisland/metabase-datomic 15 | ``` 16 | 17 | ## Design decisions 18 | 19 | See the [Architecture Decision Log](doc/architecture_decision_log.org) 20 | 21 | ## Developing 22 | 23 | To get a REPL based workflow, do a git clone of both `metabase` and 24 | `metabase-datomic`, such that they are in sibling directories 25 | 26 | ``` shell 27 | $ git clone git@github.com:metabase/metabase.git 28 | $ git clone git@github.com:plexus/metabase-datomic.git 29 | 30 | $ tree -L 1 31 | . 32 | │ 33 | ├── metabase 34 | └── metabase-datomic 35 | ``` 36 | 37 | Before you can use Metabase you need to build the frontend. This step you only 38 | need to do once. 39 | 40 | ``` shell 41 | cd metabase 42 | yarn build 43 | ``` 44 | 45 | And install metabase locally 46 | 47 | ``` shell 48 | lein install 49 | ``` 50 | 51 | Now `cd` into the `metabase-datomic` directory, and run `bin/start_metabase` to 52 | lauch the process including nREPL running on port 4444. 53 | 54 | ``` shell 55 | cd metabase-datomic 56 | bin/start_metabase 57 | ``` 58 | 59 | Now you can connect from Emacs/CIDER to port 4444, or use the 60 | `bin/cider_connect` script to automatically connect, and to associate the REPL 61 | session with both projects, so you can easily evaluate code in either, and 62 | navigate back and forth. 63 | 64 | Once you have a REPL you can start the web app. This also opens a browser at 65 | `localhost:3000` 66 | 67 | ``` clojure 68 | user=> (go) 69 | ``` 70 | 71 | The first time it will ask you to create a user and do some other initial setup. 72 | To skip this step, invoke `setup!`. This will create a user with username 73 | `arne@example.com` and password `dev`. It will also create a Datomic database 74 | with URL `datomic:free://localhost:4334/mbrainz`. You are encouraged to run a 75 | datomic-free transactor, and 76 | [import the MusicBrainz](https://github.com/Datomic/mbrainz-sample) 77 | database for testing. 78 | 79 | ``` clojure 80 | user=> (setup!) 81 | ``` 82 | 83 | ## Installing 84 | 85 | The general process is to build an uberjar, and copy the result into 86 | your Metabase `plugins/` directory. You can build a jar based on 87 | datomic-free, or datomic-pro (assuming you have a license). Metabase 88 | must be available as a local JAR. 89 | 90 | ``` shell 91 | cd metabase 92 | lein install 93 | mkdir plugins 94 | cd ../metabase-datomic 95 | lein with-profiles +datomic-free uberjar 96 | # lein with-profiles +datomic-pro uberjar 97 | cp target/uberjar/datomic.metabase-driver.jar ../metabase/plugins 98 | ``` 99 | 100 | Now you can start Metabase, and start adding Datomic databases 101 | 102 | ``` shell 103 | cd ../metabase 104 | lein run -m metabase.core 105 | ``` 106 | 107 | ## Configuration EDN 108 | 109 | When you configure a Datomic Database in Metabase you will notice a config field 110 | called "Configuration EDN". Here you can paste a snippet of EDN which will 111 | influence some of the Driver's behavior. 112 | 113 | The EDN needs to represent a Clojure map. These keys are currently understood 114 | 115 | - `:inclusion-clauses` 116 | - `:tx-filter` 117 | - `:relationships` 118 | 119 | Other keys are ignored. 120 | 121 | ### `:inclusion-clauses` 122 | 123 | Datomic does not have tables, but nevertheless the driver will map your data to 124 | Metabase tables based on the attribute names in your schema. To limit results to 125 | the right entities it needs to do a check to see if a certain entity logically 126 | belongs to such a table. 127 | 128 | By default these look like this 129 | 130 | ``` clojure 131 | [(or [?eid :user/name] 132 | [?eid :user/password] 133 | [?eid :user/roles])] 134 | ``` 135 | 136 | In other words we look for entities that have any attribute starting with the 137 | given prefix. This can be both suboptimal (there might be a single attribute 138 | with an index that is faster to check), and it may be wrong, depending on your 139 | setup. 140 | 141 | So we allow configuring this clause per table. The configured value should be a 142 | vector of datomic clauses. You have the full power of datalog available. Use the 143 | special symbol `?eid` for the entity that is being filtered. 144 | 145 | ``` clojure 146 | {:inclusion-clauses {"user" [[?eid :user/handle]]}} 147 | ``` 148 | 149 | ### `:tx-filter` 150 | 151 | The `datomic.api/filter` function allows you to get a filtered view of the 152 | database. A common use case is to select datoms based on metadata added to 153 | transaction entities. 154 | 155 | You can set `:tx-filter` to any form that evaluates to a Clojure function. Make 156 | sure any namespaces like `datomic.api` are fully qualified. 157 | 158 | ``` clojure 159 | {:tx-filter 160 | (fn [db ^datomic.Datom datom] 161 | (let [tx-user (get-in (datomic.api/entity db (.tx datom)) [:tx/user :db/id])] 162 | (or (nil? tx-tenant) (= 17592186046521 tx-user))))} 163 | ``` 164 | 165 | ### `:rules` 166 | 167 | This allows you to configure Datomic rules. These then become available in the 168 | native query editor, as well in `:inclusion-clauses` and `:relationships`. 169 | 170 | ``` clojure 171 | {:rules 172 | [[(sub-accounts ?p ?c) 173 | [?p :account/children ?c]] 174 | [(sub-accounts ?p ?d) 175 | [?p :account/children ?c] 176 | (sub-accounts ?c ?d)]]} 177 | ``` 178 | 179 | ### `:relationships` 180 | 181 | This features allows you to add "synthetic foreign keys" to tables. These are 182 | fields that Metabase will consider to be foreign keys, but in reality they are 183 | backed by an arbitrary lookup path in Datomic. This can include reverse 184 | reference (`:foo/_bar`) and rules. 185 | 186 | To set up an extra relationship you start from the table where you want to add 187 | the relationship, then give it a name, give the path of attributes and rules 188 | needed to get to the other entity, and specifiy which table the resulting entity 189 | belongs to. 190 | 191 | ``` clojure 192 | {:relationships 193 | {;; foreign keys added to the account table 194 | :account 195 | {:journal-entry-lines 196 | {:path [:journal-entry-line/_account] 197 | :target :journal-entry-line} 198 | 199 | :subaccounts 200 | {:path [sub-accounts] 201 | :target :account} 202 | 203 | :parent-accounts 204 | {:path [_sub-accounts] ;; apply a rule in reverse 205 | :target :account}} 206 | 207 | ;; foreign keys added to the journal-entry-line table 208 | :journal-entry-line 209 | {:fiscal-year 210 | {:path [:journal-entry/_journal-entry-lines 211 | :ledger/_journal-entries 212 | :fiscal-year/_ledgers] 213 | :target :fiscal-year}}}} 214 | ``` 215 | 216 | ## Status 217 | 218 | 219 |
FeatureSupported?
Basics
    {:source-table integer-literal}Yes
    {:fields [& field]}Yes
        [:field-id field-id]Yes
        [:datetime-field local-field | fk unit]Yes
    {:breakout [& concrete-field]}Yes
        [:field-id field-id]Yes
        [:aggregation 0]Yes
        [:datetime-field local-field | fk unit]Yes
    {:filter filter-clause}
        [:and & filter-clause]Yes
        [:or & filter-clause]Yes
        [:not filter-clause]Yes
        [:= concrete-field value & value]Yes
        [:!= concrete-field value & value]Yes
        [:< concrete-field orderable-value]Yes
        [:> concrete-field orderable-value]Yes
        [:<= concrete-field orderable-value]Yes
        [:>= concrete-field orderable-value]Yes
        [:is-null concrete-field]Yes
        [:not-null concrete-field]Yes
        [:between concrete-field min orderable-value max orderable-value]Yes
        [:inside lat concrete-field lon concrete-field lat-max numeric-literal lon-min numeric-literal lat-min numeric-literal lon-max numeric-literal]Yes
        [:starts-with concrete-field string-literal]Yes
        [:contains concrete-field string-literal]Yes
        [:does-not-contain concrete-field string-literal]Yes
        [:ends-with concrete-field string-literal]Yes
        [:time-interval field concrete-field n :current|:last|:next|integer-literal unit relative-datetime-unit]
    {:limit integer-literal}Yes
    {:order-by [& order-by-clause]}Yes
:basic-aggregations
    {:aggregation aggregation-clause}
        [:count]Yes
        [:count concrete-field]Yes
        [:cum-count concrete-field]Yes
        [:cum-sum concrete-field]Yes
        [:distinct concrete-field]Yes
        [:sum concrete-field]Yes
        [:min concrete-field]Yes
        [:max concrete-field]Yes
        [:share filter-clause]
:standard-deviation-aggregationsYes
    {:aggregation aggregation-clause}Yes
        [:stddev concrete-field]Yes
:foreign-keysYes
    {:fields [& field]}Yes
        [:fk-> fk-field-id dest-field-id]Yes
:nested-fields
:set-timezone
:expressions
    {:fields [& field]}
        [:expression]
    {:breakout [& concrete-field]}
        [:expression]
    {:expressions {expression-name expression}}
:native-parameters
:expression-aggregations
:nested-queriesYes
    {:source-query query}Yes
:binning
:case-sensitivity-string-filter-optionsYes
220 | 221 | 222 | ## License 223 | 224 | Copyright © 2019 Arne Brasseur 225 | 226 | Licensed under the term of the Mozilla Public License 2.0, see LICENSE. 227 | -------------------------------------------------------------------------------- /bin/cider_connect: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | emacsclient -e '(load "'`pwd`'/bin/cider_connect.el")' 4 | -------------------------------------------------------------------------------- /bin/cider_connect.el: -------------------------------------------------------------------------------- 1 | (let ((repl-buffer 2 | (cider-connect-clj (list :host "localhost" 3 | :port "4444" 4 | :project-dir "/home/arne/github/metabase")))) 5 | 6 | (sesman-link-session 'CIDER 7 | (list "github/metabase:localhost:4444" repl-buffer) 8 | 'project 9 | "/home/arne/github/metabase-datomic/")) 10 | 11 | ;;(sesman-browser) 12 | -------------------------------------------------------------------------------- /bin/feature_table: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # coding: utf-8 3 | 4 | require 'yaml' 5 | 6 | class Object 7 | def to_html 8 | inspect.to_html 9 | end 10 | end 11 | 12 | class String 13 | def to_html 14 | gsub("&", "&") 15 | .gsub("<", "<") 16 | .gsub(">", ">") 17 | .gsub('"', """) 18 | .gsub("'", "'") 19 | end 20 | end 21 | 22 | class Symbol 23 | def to_html 24 | ":#{self}" 25 | end 26 | end 27 | 28 | class Array 29 | def __html_props 30 | if Hash === self[1] 31 | self[1].map do |k,v| 32 | " #{k.to_s.to_html}='#{v.to_html}'" 33 | end.join 34 | end 35 | end 36 | 37 | def __html_children 38 | if Hash === self[1] 39 | drop(2) 40 | else 41 | drop(1) 42 | end 43 | end 44 | 45 | def to_html 46 | "<#{first}#{__html_props}>#{__html_children.map(&:to_html).join}" 47 | end 48 | end 49 | 50 | class TrueClass 51 | def to_html 52 | "Yes" 53 | end 54 | end 55 | 56 | class NilClass 57 | def to_html 58 | "" 59 | end 60 | end 61 | 62 | def raw(s) 63 | class << s 64 | def to_html 65 | self 66 | end 67 | end 68 | s 69 | end 70 | 71 | def indent(n) 72 | raw(" " * n) 73 | end 74 | 75 | def render_value(value) 76 | if value.is_a?(Array) 77 | if value.empty? 78 | nil 79 | else 80 | if value.all? {|v| render_value(v.first.last)} 81 | true 82 | else 83 | nil 84 | end 85 | end 86 | else 87 | value 88 | end 89 | end 90 | 91 | def subfeature_table(key, value) 92 | [[:tr, 93 | [:td, {align: "left"}, indent(4), key], 94 | [:td, {align: "center"}, render_value(value).to_html]], 95 | *(value.is_a?(Array) ? value : []).map do |v| 96 | key = v.first.first 97 | value = v.first.last 98 | [:tr, 99 | [:td, {align: "left"}, indent(8), key], 100 | [:td, {align: "center"}, render_value(value).to_html]] 101 | end 102 | ] 103 | end 104 | 105 | def feature_table(yaml) 106 | yaml.flat_map do |feature| 107 | value = feature.first.last 108 | feature = feature.first.first 109 | [ 110 | [:tr, 111 | [:th, {align: "left"}, feature], 112 | [:th, {align: "center"}, render_value(value).to_html] 113 | ], 114 | *(value.is_a?(Array) ? value : []).flat_map do |s| 115 | subfeature_table(s.first.first, s.first.last) 116 | end 117 | ] 118 | end 119 | end 120 | 121 | puts [:table, 122 | [:tr, 123 | [:th, {align: "left"}, "Feature"], 124 | [:th, {align: "center"}, "Supported?"]], 125 | *feature_table(YAML.load(IO.read('features.yml'))) 126 | ].to_html 127 | -------------------------------------------------------------------------------- /bin/feature_table.ed: -------------------------------------------------------------------------------- 1 | /-- feature-table --/+,/-- \/feature-table --/- d 2 | - r ! bin/feature_table 3 | wq 4 | -------------------------------------------------------------------------------- /bin/kaocha: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd ../metabase 4 | 5 | clojure -Sdeps "$(cat ../metabase-datomic/deps.edn)" -A:dev:test:metabase:datomic-free -m kaocha.runner --no-capture-output --config-file ../metabase-datomic/tests.edn "$@" 6 | 7 | cd ../metabase-datomic 8 | -------------------------------------------------------------------------------- /bin/start_metabase: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This script expects Metabase and Metabase-datomic to be checked out in two 4 | # sibling directories named "metabase" and "metabase-datomic". Run this script 5 | # from the latter as bin/start_datomic. 6 | # 7 | # Alternatively you can set METABASE_HOME and METABASE_DATOMIC_HOME before 8 | # invoking the script. 9 | 10 | export METABASE_HOME="${METABASE_HOME:-../metabase}" 11 | export METABASE_DATOMIC_HOME="${METABASE_DATOMIC_HOME:-`pwd`}" 12 | 13 | cd "$METABASE_HOME" 14 | 15 | ln -fs ../metabase-datomic/deps.edn . 16 | ln -fs ../metabase-datomic/tests.edn . 17 | 18 | # Delete the :repl profile to stop metabase from loading all drivers 19 | if grep -q ':repl$' project.clj ; 20 | then 21 | ed project.clj <<-EOF 22 | H 23 | /:profiles/ 24 | i 25 | :plugins 26 | [[refactor-nrepl "2.4.1-SNAPSHOT"] 27 | [cider/cider-nrepl "0.21.2-SNAPSHOT"] 28 | [lein-tools-deps "0.4.3"]] 29 | 30 | :middleware [lein-tools-deps.plugin/resolve-dependencies-with-deps-edn] 31 | 32 | :lein-tools-deps/config 33 | {:config-files [:install :project] 34 | :aliases [:dev :test :datomic-free]} 35 | 36 | . 37 | /:repl$/ 38 | .,+2d 39 | wq 40 | EOF 41 | fi 42 | 43 | lein update-in :repl-options assoc :init-ns user -- repl :headless :port 4444 44 | -------------------------------------------------------------------------------- /deps.edn: -------------------------------------------------------------------------------- 1 | {:paths ["../metabase-datomic/src" 2 | "../metabase-datomic/resources" 3 | "src" 4 | "resources"] 5 | :deps {} 6 | 7 | :mvn/repos 8 | {"my.datomic.com" {:url "https://my.datomic.com/repo"}} 9 | 10 | :aliases 11 | {:metabase 12 | {:extra-deps {org.clojure/clojure {:mvn/version "1.10.0"} 13 | 14 | ;; Apache Commons Lang, this seems to be a missing dependency of 15 | ;; the current master of Metabase (2019-05-09) 16 | commons-lang {:mvn/version "2.4"} 17 | metabase-core {:mvn/version "1.0.0-SNAPSHOT" 18 | :scope "provided"}}} 19 | 20 | :dev 21 | {:extra-paths ["../metabase-datomic/dev"] 22 | :extra-deps {nrepl {:mvn/version "0.6.0"} 23 | vvvvalvalval/scope-capture {:mvn/version "0.3.2"}}} 24 | 25 | :datomic-free 26 | {:extra-deps {com.datomic/datomic-free {:mvn/version "0.9.5697" 27 | :exclusions [org.slf4j/jcl-over-slf4j 28 | org.slf4j/jul-to-slf4j 29 | org.slf4j/log4j-over-slf4j 30 | org.slf4j/slf4j-nop]}}} 31 | 32 | :datomic-pro 33 | {:extra-deps {com.datomic/datomic-pro {:mvn/version "0.9.5927" 34 | :exclusions [org.slf4j/jcl-over-slf4j 35 | org.slf4j/jul-to-slf4j 36 | org.slf4j/log4j-over-slf4j 37 | org.slf4j/slf4j-nop]}}} 38 | 39 | :test 40 | {:extra-paths ["../metabase/src" 41 | "../metabase/test" 42 | "../metabase-datomic/test"] 43 | :jvm-opts ["-Dmb.db.file=metabase.datomic.test" 44 | "-Dmb.jetty.port=3999"] 45 | :extra-deps {lambdaisland/kaocha {:mvn/version "0.0-418"} 46 | expectations {:mvn/version "2.2.0-beta2"} 47 | nubank/matcher-combinators {:mvn/version "0.9.0"}}}}} 48 | -------------------------------------------------------------------------------- /dev/user.clj: -------------------------------------------------------------------------------- 1 | (ns user 2 | (:require [clojure.java.io :as io])) 3 | 4 | (defmacro jit 5 | "Just in time loading of dependencies." 6 | [sym] 7 | `(do 8 | (require '~(symbol (namespace sym))) 9 | (find-var '~sym))) 10 | 11 | (defn plugin-yaml-path [] 12 | (some #(when (.exists %) %) 13 | [(io/file "resources/metabase-plugin.yaml") 14 | (io/file "../metabase-datomic/resources/metabase-plugin.yaml") 15 | (io/file "metabase-datomic/resource/metabase-plugin.yaml")])) 16 | 17 | (defn setup-driver! [] 18 | (-> (plugin-yaml-path) 19 | ((jit yaml.core/from-file)) 20 | ((jit metabase.plugins.initialize/init-plugin-with-info!)))) 21 | 22 | (defn setup-db! [] 23 | ((jit metabase.db/setup-db!))) 24 | 25 | (defn open-metabase [] 26 | ((jit clojure.java.browse/browse-url) "http://localhost:3000")) 27 | 28 | (defn go [] 29 | (let [start-web-server! (jit metabase.server/start-web-server!) 30 | app (jit metabase.handler/app) 31 | init! (jit metabase.core/init!) 32 | clean-up-in-mem-dbs (jit user.repl/clean-up-in-mem-dbs)] 33 | (setup-db!) 34 | (clean-up-in-mem-dbs) 35 | (start-web-server! app) 36 | (init!) 37 | (setup-driver!) 38 | (open-metabase) 39 | ((jit user.repl/clean-up-in-mem-dbs)))) 40 | 41 | (defn setup! [] 42 | ((jit user.setup/setup-all))) 43 | 44 | (defn refresh [] 45 | ((jit clojure.tools.namespace.repl/set-refresh-dirs) 46 | "../metabase/src" 47 | "../metabase-datomic/src" 48 | "../metabase-datomic/dev" 49 | "../metabase-datomic/test") 50 | ((jit clojure.tools.namespace.repl/refresh))) 51 | 52 | (defn refresh-all [] 53 | ((jit clojure.tools.namespace.repl/set-refresh-dirs) 54 | "../metabase/src" 55 | "../metabase-datomic/src" 56 | "../metabase-datomic/dev" 57 | "../metabase-datomic/test") 58 | ((jit clojure.tools.namespace.repl/refresh-all))) 59 | 60 | (defn refer-repl [] 61 | (require '[user.repl :refer :all] 62 | '[user.setup :refer :all] 63 | '[user :refer :all] 64 | '[clojure.repl :refer :all] 65 | '[sc.api :refer :all])) 66 | -------------------------------------------------------------------------------- /dev/user/repl.clj: -------------------------------------------------------------------------------- 1 | (ns user.repl 2 | (:require clojure.tools.namespace.repl 3 | datomic.api 4 | metabase.driver.datomic 5 | metabase.query-processor 6 | [toucan.db :as db] 7 | [metabase.models.database :refer [Database]] 8 | [datomic.api :as d] 9 | [clojure.string :as str])) 10 | 11 | (def mbrainz-url "datomic:free://localhost:4334/mbrainz") 12 | (def eeleven-url "datomic:free://localhost:4334/eeleven") 13 | 14 | (defn conn 15 | ([] 16 | (conn mbrainz-url)) 17 | ([url] 18 | (datomic.api/connect url))) 19 | 20 | (defn db 21 | ([] 22 | (db mbrainz-url)) 23 | ([url] 24 | (datomic.api/db (conn url)))) 25 | 26 | (defn mbql-history [] 27 | @metabase.driver.datomic/mbql-history) 28 | 29 | (defn query-history [] 30 | @metabase.driver.datomic/query-history) 31 | 32 | (defn qry 33 | ([] 34 | (first (mbql-history))) 35 | ([n] 36 | (nth (mbql-history) n))) 37 | 38 | (defn nqry 39 | ([] 40 | (first (query-history))) 41 | ([n] 42 | (nth (query-history) n))) 43 | 44 | (defn query->native [q] 45 | (metabase.query-processor/query->native q)) 46 | 47 | (defn clean-up-in-mem-dbs [] 48 | (doseq [{{uri :db} :details id :id} 49 | (db/select Database :engine "datomic") 50 | :when (str/includes? uri ":mem:")] 51 | (try 52 | (d/connect uri) 53 | (catch Exception e 54 | (db/delete! Database :id id))))) 55 | 56 | ;; (d/delete-database "datomic:mem:test-data") 57 | ;; (clean-up-in-mem-dbs) 58 | 59 | (defn bind-driver! [] 60 | (alter-var-root #'metabase.driver/*driver* (constantly :datomic))) 61 | -------------------------------------------------------------------------------- /dev/user/setup.clj: -------------------------------------------------------------------------------- 1 | (ns user.setup 2 | (:require [toucan.db :as db] 3 | [metabase.setup :as setup] 4 | [metabase.models.user :as user :refer [User]] 5 | [metabase.models.database :as database :refer [Database]] 6 | [metabase.models.field :as field :refer [Field]] 7 | [metabase.models.table :as table :refer [Table]] 8 | [metabase.public-settings :as public-settings] 9 | [metabase.sync :as sync] 10 | [clojure.java.io :as io] 11 | [datomic.api :as d])) 12 | 13 | (defn setup-first-user [] 14 | (let [new-user (db/insert! User 15 | :email "admin@example.com" 16 | :first_name "dev" 17 | :last_name "dev" 18 | :password (str (java.util.UUID/randomUUID)) 19 | :is_superuser true)] 20 | (user/set-password! (:id new-user) "dev"))) 21 | 22 | (defn setup-site [] 23 | (public-settings/site-name "Acme Inc.") 24 | (public-settings/admin-email "arne@example.com") 25 | (public-settings/anon-tracking-enabled false) 26 | (setup/clear-token!)) 27 | 28 | (defn setup-database 29 | ([] 30 | (setup-database "MusicBrainz" "datomic:free://localhost:4334/mbrainz" {})) 31 | ([name url config] 32 | (let [dbinst (db/insert! Database 33 | {:name name 34 | :engine :datomic 35 | :details {:db url 36 | :config (pr-str config)} 37 | :is_on_demand false 38 | :is_full_sync true 39 | :cache_field_values_schedule "0 50 0 * * ? *" 40 | :metadata_sync_schedule "0 50 * * * ? *"})] 41 | (sync/sync-database! dbinst)))) 42 | 43 | (defn remove-database 44 | ([] 45 | (remove-database "MusicBrainz")) 46 | ([name] 47 | (remove-database name false)) 48 | ([name remove-datomic-db?] 49 | (let [dbinst (db/select-one Database :name name) 50 | tables (db/select Table :db_id (:id dbinst)) 51 | fields (db/select Field {:where [:in :table_id (map :id tables)]})] 52 | (when (= :datomic (:engine dbinst)) 53 | (d/delete-database (-> dbinst :details :db))) 54 | (db/delete! Field {:where [:in :id (map :id fields)]}) 55 | (db/delete! Table {:where [:in :id (map :id tables)]}) 56 | (db/delete! Database :id (:id dbinst))))) 57 | 58 | (defn reset-database! 59 | ([] 60 | (remove-database) 61 | (setup-database)) 62 | ([name] 63 | (remove-database name) 64 | (setup-database name))) 65 | 66 | (defn setup-all [] 67 | (setup-first-user) 68 | (setup-site) 69 | (setup-database)) 70 | -------------------------------------------------------------------------------- /doc/architecture_decision_log.org: -------------------------------------------------------------------------------- 1 | #+TITLE: Metabase-Datomic: Architecture Decision Log 2 | 3 | For information on ADRs (Architecture Decision Records) see [[http://thinkrelevance.com/blog/2011/11/15/documenting-architecture-decisions][Documenting 4 | Architecture Decisions]] by Michael Nygard. 5 | 6 | * 001: The Metabase schema is inferred from the Datomic schema 7 | 8 | Status: ACCEPTED 9 | 10 | ** Context 11 | 12 | Metabase views the world as consisting of Tables and Fields. Datomic thinks 13 | in Entities with Attributes. We need some way to map the latter model onto 14 | the former that corresponds with people's intuition about their data. 15 | 16 | While in theory you can mix and match attributes freely in Datomic, in 17 | practice people's data tends to be neatly divided into types of entities, 18 | each with their own distinct set of attributes. This tends to roughly map to 19 | how people use attribute namespaces in their schemas, so while there is no 20 | explicit modeling of a entity "type", you can informally derive that entities 21 | mostly consisting of attributes named ~:user/*~ are "user"-type entities. 22 | 23 | ** Decision 24 | 25 | Attributes in Datomic are namespaced keywords, we treat each namespace 26 | (a.k.a. prefix) that occurs in the Datomic schema as a metabase Table, with 27 | the exception of some internal prefixes, namely ~["db" "db.alter" "db.excise" 28 | "db.install" "db.sys" "fressian"]~. 29 | 30 | An entity that has any attributes with the given prefix is mapped to a Row in 31 | this Table. 32 | 33 | Any attribute with the given prefix, or any attribute that co-occurs with 34 | attributes with the given prefix, is mapped to a Field (column) of this 35 | Table. For attributes which have the same prefix as the table name, the field 36 | name is the attribute name without the prefix. For other attributes we use 37 | the full attribute name as the field name. 38 | 39 | Example: 40 | 41 | #+begin_src clojure 42 | {:artist/name "Beatles" 43 | :artist/startYear 1960 44 | :group/location "Liverpool"} 45 | #+end_src 46 | 47 | This becomes a row in an "artist" table, with fields "name", "startYear", and "group/location". 48 | 49 | It also becomes a row in the "group" table. 50 | 51 | ** Consequences 52 | 53 | If entities have attributes with multiple namespaces, then these entities 54 | occur in multiple "tables". 55 | 56 | When deriving Tables and Fields from a Datomic database that has a schema but 57 | no data, then only the prefixes can be examined, so each attribute only 58 | occurs as a Field in a single Table. Once data is added an attribute could 59 | become a Field in other Tables as well, and a re-sync is necessary. 60 | 61 | To test "table membership", we need to check if any of the given attributes 62 | is present, so the equivalent of ~SELECT * FROM artists;~ becomes: 63 | 64 | #+begin_src clojure 65 | {:find [?eid] 66 | :where [[(or [?eid :artist/name] [?eid :artist/startYear])]]} 67 | #+end_src 68 | 69 | ** Future considerations 70 | 71 | This derived schema may be sub-optimal, and it might be necessary to provide 72 | people with a way to edit the mapping. 73 | 74 | * 002: Use Datalog in map format as "native" format 75 | 76 | Status: ACCEPTED 77 | 78 | ** Context 79 | 80 | Metabase drivers perform their queries in two steps, first they convert the 81 | MBQL (Metabase Query Language) into a "native" format (typically SQL), then 82 | they execute this native query and return the results. 83 | 84 | A Metabase user can at any point switch to a "native" query mode, where the 85 | query can be edited by hand, so a driver does not only need to support the 86 | queries it generates, but any query a user can reasonably pass it. 87 | 88 | ** Decision 89 | 90 | As the "native" representation for Datomic queries we use the map format of 91 | Datomic's Datalog, with certain restrictions. E.g. we do not allow pull 92 | queries, as they can lead to arbitrary nesting, which isn't suitable for the 93 | table-based representation that Metabase works with. 94 | 95 | ** Consequences 96 | 97 | We need to not only support the queries we generate, but other arbitrary 98 | datalog queries as well. We need to decide and define which (classes of) 99 | queries we accept, so that the user knows which features are available when 100 | writing queries. 101 | 102 | * 003: Use an "extended Datalog" format 103 | 104 | Status: ACCEPTED / Partially superseded by 007 105 | 106 | ** Context 107 | We are challenged with the task of converting Metabase's internal query 108 | language MBQL to something Datomic understands: Datalog. MBQL by and large 109 | follows SQL semantics, which is in some areas quite different from Datalog. 110 | 111 | Consider this query: 112 | 113 | #+begin_src sql 114 | SELECT first_name, last_name FROM users WHERE age > 18; 115 | #+end_src 116 | 117 | Naively this would translate to 118 | 119 | #+begin_quote clojure 120 | [:find ?first ?last 121 | :where [?u :user/first-name ?first] 122 | [?u :user/last-name ?last] 123 | [?u :user/age ?age] 124 | [(< 18 ?age)]] 125 | #+end_quote 126 | 127 | But this won't find entities where ~:user/first-name~ or ~:user/last-name~ 128 | aren't present, whereas the SQL will. You could address this with a pull query 129 | in the ~:find~ clause instead, but these are harder to construct 130 | algorithmically, and harder to process, since results will now have arbitrary 131 | nesting. 132 | 133 | Another example is ~ORDER BY~, a functionality that Datlog does not provide 134 | and must instead be performed in application code. 135 | 136 | We need to capture these requirements in a "native" query format that the user 137 | is able to manipulate, since Metabase allows to convert any query it generates 138 | to "native" for direct editing. 139 | 140 | ** Decision 141 | 142 | In order to stick to MBQL/SQL semantics we process queries in two parts: we 143 | perform a Datalog query to fetch all entities under consideration, then do a 144 | second pass in application code, to pull out the necessary fields, and do 145 | sorting. 146 | 147 | To this end we add two extra fields to Datalog: ~:select~, and ~:order-by~. 148 | The first determines which fields each returned row has, so the main query 149 | only returns entity ids and aggregates like ~count~, the second determines the 150 | sorting of the result. 151 | 152 | #+begin_src clojure 153 | {:find [?eid] 154 | 155 | :where [[(or [?eid :user/first-name] 156 | [?eid :user/last-name] 157 | [?eid :user/age])] 158 | [?eid :user/age ?age] 159 | [(< 18 ?age)]] 160 | 161 | :select [(:user/first-name ?eid) 162 | (:user/last-name ?eid)] 163 | 164 | :order-by [(:user/last-name ?eid) :desc]} 165 | #+end_src 166 | 167 | ** Consequences 168 | 169 | We will still have to be able to handle native queries that don't have a 170 | ~:select~ clause. 171 | 172 | * 004: Use specific conventions for Datalog logic variable names 173 | 174 | Status: ACCEPTED 175 | 176 | ** Context 177 | 178 | Datalog uses logic variables (the ones that start with a question mark) in 179 | its queries. These names are normally arbitrary, but since we need to analyze 180 | the query after the fact (because of the MBQL/Native query split in 181 | Metabase), we need to be able to re-interpret these names. 182 | 183 | In Datalog adding a field to the ~:find~ clause has an impact on how grouping 184 | is handled, and so we treat MBQL ~:fields~ references differently from 185 | ~:breakout~ references. Fields from ~:breakouts~ are directly put in the 186 | ~:find~ clause, for the ones in ~:fields~ we only look up the entity id. 187 | 188 | When sorting afterwards this becomes problematic, because we no longer have a 189 | unified way of looking up field references. The solution is to have enough 190 | information in the names, so that we can look up field references, either by 191 | finding them directly in the result, or by going through the Datomic entity. 192 | 193 | ** Decision 194 | 195 | A name like ~?artist~ refers to an entity that logically belongs to the 196 | "artist" table. 197 | 198 | A name like ~?artist|artist|name~ refers to the ~:artist/name~ attribute, 199 | used on an entity that logically belongs to the ~"artist"~ table. The form 200 | here is ~?table|attribute-namespace|attribute-name~. 201 | 202 | Another example: ~?venue|db|id~, refers to the ~:db/id~ of a venue. 203 | 204 | In the ~:select~ and ~:order-by~ clauses the form ~(:foo/bar ?eid)~ can also 205 | be used, which is equivalent to ~?eid|foo|bar~. 206 | 207 | ** Consequences 208 | 209 | We'll have to see how this continues to behave when foreign keys and other 210 | constructs are added to the mix, but so far this handles the use cases of 211 | sorting with both ~:fields~ and ~:breakouts~ based queries, and it seems a 212 | more future proof approach in general. 213 | 214 | * 005: Mimic SQL left outer join when dealing with foreign keys 215 | Status: ACCEPTED 216 | 217 | ** Context 218 | 219 | References in Datomic are similar to foreign keys in RDBMS systems, but 220 | datomic supports many-to-many references (~:db.cardinality/many~), whereas 221 | SQL requires a JOIN table to model the same thing. 222 | 223 | This means that values in a result row can actually be sets, something 224 | Metabase is not able to handle well. 225 | 226 | ** Decision 227 | 228 | Expand ~cardinality/many~ to its cartesian product, in other words for every 229 | referenced entity we emit a new "row" in the result set. 230 | 231 | If the result set is empty then we emit a single row, with the referenced 232 | field being ~nil~, this mimics the behavior of an SQL left (outer) join. 233 | 234 | ** Consequences 235 | 236 | This decision impacts post-processing, as we need to loop over all result 237 | rows to expand sets. It also impacts sorting, as we can only really sort 238 | after expansion. 239 | 240 | * 006: If referenced entities have a ~:db/ident~, then display that instead of the ~:db/id~ 241 | Status: ACCEPTED 242 | 243 | ** Context 244 | 245 | In Datomic any entity can have a ~:db/ident~, a symbolic identifier that can 246 | be used interchangably with its ~:db/id~. 247 | 248 | In practice this is mainly used for attributes (part of the database schema), 249 | and for "enums" or categories. E.g. gender, currency, country, etc. 250 | 251 | In these cases showing the symbolic identifier to the user is preferable, as 252 | it carries much more information. Compare: 253 | 254 | ~["Jonh" "Doe" 19483895]~ 255 | 256 | vs 257 | 258 | ~["Jonh" "Doe" :gender/male]~ 259 | 260 | ** Decision 261 | 262 | If a referenced entity has a ~:db/ident~, then return that (as a string), 263 | rather than the ~:db/id~. This way enum-type fields are always shown with 264 | their symbolic identifier rather than their numeric ~:db/id~. 265 | 266 | ** Consequences 267 | 268 | This is mainly a post-processing concern. For fields that are looked via the 269 | entity API (i.e. breakout) this is straightforward, for other cases we do 270 | some schema introspection to see if we're dealing with a ~:db/id~, and then 271 | check if we have a ~:db/ident~ for the given entity. 272 | 273 | When we return these values as foreign key values, then Metabase will also 274 | hand them back to us when constructing filter clauses, so we need to convert 275 | them back to keywords so Datomic recognizes them as idents. 276 | 277 | * 007: Use ~get-else~ to bind fields, to mimic ~nil~ semantics 278 | 279 | Status: ACCEPTED 280 | 281 | ** Context 282 | 283 | Datomic does not have ~nil~ values for attributes, either an attribute is 284 | present on an entity or it is not, but it can not be present with a value of 285 | ~nil~. Metabase and MBQL are modeled on RDBMS/SQL semantics, where fields can 286 | be ~NULL~. We want to mimic this by treating missing attributes as if they 287 | are attributes with ~nil~ values. 288 | 289 | A datalog ~:where~ form like ~[?venue :venue/name ?venue|name]~ does two 290 | things, it *binds* the ~?venue~ and ~?venue|name~ logical variables (lvars), 291 | and it filters the result set (unification). This means that entities where 292 | the ~:venue/name~ is absent (conceptually ~nil~), these entities will not 293 | show up in the result. 294 | 295 | We partially worked around this in 003 by avoiding using binding forms to 296 | pull out attribute values, and instead adopted a two step process where we 297 | first query for entity ids only, then use the entity API to pull out the 298 | necessary attributes. 299 | 300 | This turned out to be less than ideal, because it made it hard to implement 301 | filtering operations correctly. 302 | 303 | ** Decision 304 | 305 | Instead we *do* bind attribute values to lvars, but we use ~get-else~ and 306 | ~nil~ placeholder value to deal with missing attributes. 307 | 308 | #+begin_src clojure 309 | [:where 310 | [(get-else $ ?venue :venue/name ::nil) ?venue/name]] 311 | #+end_src 312 | 313 | This way we get the binding inside the query, and can implement filtering 314 | operations inside the query based on those values. It also means ~:fields~ 315 | and ~:breakout~ based queries differ less than before, and so share more 316 | code and logic. 317 | 318 | For reference attributes with :cardinality/many ~get-else~ is not supported, 319 | in these cases we fall back to emulating ~get-else~ with an ~or-join~. 320 | 321 | In this case using a special keyword as a placeholder does not work, Datomic 322 | expects the lvar to be bound to an integer (a possible ~:db/id~), so instead 323 | of ~::nil~ as a placeholder, we use ~Long/MIN_VALUE~. 324 | 325 | #+begin_src clojure 326 | (or-join [?ledger ?ledger|ledger|tax-entries] 327 | [?ledger :ledger/tax-entries ?ledger|ledger|tax-entries] 328 | (and [(missing? $ ?ledger :ledger/tax-entries)] 329 | [(ground Long/MIN_VALUE) ?ledger|ledger|tax-entries])) 330 | #+end_src 331 | 332 | Note that the two-step process from 003 is still in place, we still honor the 333 | extra ~:select~ clause to perform post-processing on the result, but it isn't 334 | used as extensively as before, as most of the values we need are directly 335 | returned in the query result. 336 | 337 | ** Consequences 338 | 339 | Once this was implemented implemeting filter operations was a breeze. This 340 | adds some new post-processing because we need to replace the placeholders 341 | with actual ~nil~ values again. 342 | 343 | Query generating code needs to pay attention to use the necessary helper 344 | functions to set up lvar bindings, so these semantics are preserved. 345 | 346 | * 008: Use type-specific ~nil~ placeholders 347 | 348 | Status: ACCEPTED 349 | 350 | ** Context 351 | 352 | Datomic / Datalog does not explicitly handle ~nil~. To emulate the behavior 353 | of an RDBMS where fields and relationships can be ~NULL~ we use placeholder 354 | values in our queries, so that in the result set we can find these sentinels 355 | and replace them with ~nil~. 356 | 357 | So far we used ~Long/MIN_VALUE~ for refs, as they need to resemble Datomic 358 | ids, and the special keyword ~::nil~ (i.e. 359 | ~:metabase.driver.datomic.query-processor/nil~) elsewhere. 360 | 361 | This turned out to be too simplistic, because when combined with aggregate 362 | functions Datomic attempts to group values, which means it needs to be able 363 | to compare values, and when it does that with values of disparate types, 364 | things blow up. 365 | 366 | ** Decision 367 | 368 | We look at the database type of the attribute when choosing an appropriate 369 | substitute value for ~nil~. This can be the keyword used before, the same 370 | keyword turned into a string, ~Long/MIN_VALUE~ for numeric types and refs, 371 | and a specially crafted datetime for inst fields. 372 | 373 | ** Consequences 374 | 375 | There is one big downside to this approach, if an attribute in the database 376 | actually has the placeholder value as a value, it will be shown as 377 | ~nil~/absent. The keyword/string versions are namespaced and so unlikely to 378 | cause problems, the datetime is set at the start of the common era, so only 379 | potentially problematic when dealing with ancient history (and even then it 380 | needs to match to the millisecond), but number fields that contain 381 | ~Long/MIN_VALUE~ may actually occur in the wild. We currently don't have a 382 | solution for this. 383 | 384 | * 009: Use s-expression forms in ~:select~ to signal specific post-processing steps 385 | 386 | Status: ACCEPTED / SUPERSEDED BY XXX 387 | 388 | ** Context 389 | 390 | In 003 we introduced an "extended datalog" format, where a ~:select~ form can 391 | be added to specify how to turn the result from the ~:find~ clause into the 392 | final results. 393 | 394 | So far you could put the following things in ~:select~: 395 | 396 | - forms that are identical to those used in ~find~, the value will be copied verbatim 397 | - s-expression forms using keywords in function position, these will either 398 | be converted to a symbol that is looked up in the ~:find~ clause, or if not 399 | find the entity API will be used instead to look up the attribute value 400 | 401 | ** Decision 402 | 403 | Sometimes more specific post-processing is needed, in these cases we 404 | communicate this from the query generation step to the execution step by 405 | using s-expressions with namespaced symbols in function position. These 406 | symbols are used to dispatch to the ~select-field-form~ multimethod, which 407 | returns a function that extracts and tranforms a value from a result row. 408 | 409 | So far we implement two of these special cases 410 | 411 | - ~metabase.driver.datomic.query-processor/datetime~ which will do the 412 | necessary truncation on a datetime value 413 | 414 | - ~metabase.driver.datomic.query-processor/field~ this is used to communicate 415 | field metadata, so that we can do type specific coercions or 416 | transformations. This is used so far to transform ref ~:db/id~ values to 417 | idents. 418 | 419 | ** Consequences 420 | 421 | This elegantly provides a future-proof mechanism for communicating extra 422 | metadata to the query execution stage. 423 | 424 | * 010: Support custom relationships 425 | 426 | Status: ACCEPTED 427 | 428 | ** Context 429 | 430 | While Metabase's foreign key functionality is very handy, it is at the same 431 | time quite limited. You can only navigate one "hop" away from your source 432 | table, and can only navigate in the direction the relationship is declared. 433 | 434 | Datomic being (among other things) a graph database makes it very easy to 435 | traverse chains of relationships, but we are stuck with the Metabase UI. Can 436 | we "hack" the UI to support more general relationship traversal. 437 | 438 | ** Decision 439 | 440 | Allow configuring "custom" relationships. These have a source "table", a 441 | destination "table", a "path", and a "name". The UI will display these as 442 | regular foreign keys, allowing you to navigate from one table to the next, 443 | but in reality they are backed by an arbitrary path lookup, including reverse 444 | lookups and Datomic rules. 445 | 446 | To configure these you paste a snippet of EDN configuration into the admin 447 | UI. 448 | 449 | When syncing the schema we use this information to create synthetic foreign 450 | keys, which we give the special database type "metabase.driver.datomic/path" 451 | to distinguish them from "real" foreign keys. 452 | 453 | During query generation ~:field-id~ of ~:fk->~ clauses need to be checked 454 | first, and potentially handled by a separate logic which generates the 455 | necessary Datalog ~:where~ binding based on the path. 456 | 457 | ** Consequences 458 | 459 | This adds some special cases that need to be considered, but it also adds a 460 | tremendous amount of flexibility of use. 461 | 462 | * 011: Use false as the nil-placeholder for boolean columns 463 | ** Context 464 | We try to generate queries in such a way that when an attribute is missing on 465 | an entity that instead we returns placeholder value, which is then in 466 | post-processing replaced with actual ~nil~. This way we don't remove entities 467 | from a result set just because the entity doesn't have all the attributes we 468 | expected. 469 | 470 | We use a type-specific placeholder because Datomic internally compares result 471 | values to group results, and this fails if values returned for the same logic 472 | variable have types that can't be compared (e.g. keyword and boolean). 473 | 474 | The most common placeholders are 475 | ~:metabase.driver.datomic.query-processor/nil~, and ~Long/MIN_VALUE~. 476 | 477 | ** Decision 478 | For boolean columns we use ~false~ as the placeholder, so that we maintain 479 | the correct filtering semantics, and so that queries don't blow up because of 480 | type differences. 481 | 482 | ** Consequences 483 | This is different from the other cases, because it is no longer possible to 484 | discern which entities have an actual value of ~false~, and which ones are 485 | missing/~nil~. So in this case we leave the filled-in placeholders in the 486 | result, rather than replacing them with ~nil~. 487 | 488 | In other words: entities where a boolean attribute is missing will be shown 489 | as having ~false~ for that attribute. 490 | 491 | * Template 492 | 493 | Status: ACCEPTED / SUPERSEDED BY XXX 494 | 495 | ** Context 496 | ** Decision 497 | ** Consequences 498 | -------------------------------------------------------------------------------- /features.yml: -------------------------------------------------------------------------------- 1 | - Basics: 2 | - "{:source-table integer-literal}": Yes 3 | - "{:fields [& field]}": 4 | - "[:field-id field-id]": Yes 5 | - "[:datetime-field local-field | fk unit]": Yes 6 | - "{:breakout [& concrete-field]}": 7 | - "[:field-id field-id]": Yes 8 | - "[:aggregation 0]": Yes 9 | - "[:datetime-field local-field | fk unit]": Yes 10 | - "{:filter filter-clause}": 11 | - "[:and & filter-clause]": Yes 12 | - "[:or & filter-clause]": Yes 13 | - "[:not filter-clause]": Yes 14 | - "[:= concrete-field value & value]": Yes 15 | - "[:!= concrete-field value & value]": Yes 16 | - "[:< concrete-field orderable-value]": Yes 17 | - "[:> concrete-field orderable-value]": Yes 18 | - "[:<= concrete-field orderable-value]": Yes 19 | - "[:>= concrete-field orderable-value]": Yes 20 | - "[:is-null concrete-field]": Yes 21 | - "[:not-null concrete-field]": Yes 22 | - "[:between concrete-field min orderable-value max orderable-value]": Yes 23 | - "[:inside lat concrete-field lon concrete-field lat-max numeric-literal lon-min numeric-literal lat-min numeric-literal lon-max numeric-literal]": Yes 24 | - "[:starts-with concrete-field string-literal]": Yes 25 | - "[:contains concrete-field string-literal]": Yes 26 | - "[:does-not-contain concrete-field string-literal]": Yes 27 | - "[:ends-with concrete-field string-literal]": Yes 28 | - "[:time-interval field concrete-field n :current|:last|:next|integer-literal unit relative-datetime-unit]": 29 | - "{:limit integer-literal}": Yes 30 | - "{:order-by [& order-by-clause]}": Yes 31 | - :basic-aggregations: 32 | - "{:aggregation aggregation-clause}": 33 | - "[:count]": Yes 34 | - "[:count concrete-field]": Yes 35 | - "[:cum-count concrete-field]": Yes 36 | - "[:cum-sum concrete-field]": Yes 37 | - "[:distinct concrete-field]": Yes 38 | - "[:sum concrete-field]": Yes 39 | - "[:min concrete-field]": Yes 40 | - "[:max concrete-field]": Yes 41 | - "[:share filter-clause]": 42 | - :standard-deviation-aggregations: 43 | - "{:aggregation aggregation-clause}": 44 | - "[:stddev concrete-field]": Yes 45 | - :foreign-keys: 46 | - "{:fields [& field]}": 47 | - "[:fk-> fk-field-id dest-field-id]": Yes 48 | - :nested-fields: 49 | - :set-timezone: 50 | - :expressions: 51 | - "{:fields [& field]}": 52 | - "[:expression]": 53 | - "{:breakout [& concrete-field]}": 54 | - "[:expression]": 55 | - "{:expressions {expression-name expression}}": 56 | - :native-parameters: 57 | - :expression-aggregations: 58 | - :nested-queries: 59 | - "{:source-query query}": Yes 60 | - :binning: 61 | - :case-sensitivity-string-filter-options: Yes 62 | -------------------------------------------------------------------------------- /project.clj: -------------------------------------------------------------------------------- 1 | (defproject metabase/datomic-driver "1.0.0-SNAPSHOT-0.9.5697" 2 | :min-lein-version "2.5.0" 3 | 4 | :plugins [[lein-tools-deps "0.4.3"]] 5 | :middleware [lein-tools-deps.plugin/resolve-dependencies-with-deps-edn] 6 | :lein-tools-deps/config {:aliases [:metabase] 7 | :config-files [:install :project]} 8 | 9 | :profiles 10 | {:provided 11 | {:dependencies [[metabase-core "1.0.0-SNAPSHOT"]]} 12 | 13 | :datomic-free {:lein-tools-deps/config {:aliases [:datomic-free]}} 14 | :datomic-pro 15 | {:lein-tools-deps/config {:aliases [:datomic-pro]} 16 | :repositories 17 | {"my.datomic.com" {:url "https://my.datomic.com/repo"}}} 18 | 19 | :uberjar 20 | {:auto-clean true 21 | :aot :all 22 | :javac-options ["-target" "1.8", "-source" "1.8"] 23 | :target-path "target/%s" 24 | :uberjar-name "datomic.metabase-driver.jar"}}) 25 | -------------------------------------------------------------------------------- /repl_sessions/navigation.clj: -------------------------------------------------------------------------------- 1 | (ns navigation 2 | (:require [datomic.api :as d] 3 | [clojure.string :as str])) 4 | 5 | {:relationships {:journal-entry 6 | {:account 7 | {:path [:journal-entry/journal-entry-lines :journal-entry-line/account] 8 | :target :account}}} 9 | 10 | :tx-filter 11 | (fn [db ^datomic.Datom datom] 12 | (let [tx-tenant (get-in (datomic.api/entity db (.tx datom)) [:tx/tenant :db/id])] 13 | (or (nil? tx-tenant) (= 17592186046521 tx-tenant))))} 14 | 15 | 16 | 17 | {:rules 18 | [[(sub-accounts ?p ?c) 19 | [?p :account/children ?c]] 20 | [(sub-accounts ?p ?d) 21 | [?p :account/children ?c] 22 | (sub-accounts ?c ?d)] 23 | ] 24 | 25 | :relationships 26 | {:account 27 | {:journal-entry-lines 28 | {:path [:journal-entry-line/_account] 29 | :target :journal-entry-line} 30 | 31 | :subaccounts 32 | {:path [sub-accounts] 33 | :target :account} 34 | 35 | :parent-accounts 36 | {:path [_sub-accounts] 37 | :target :account} 38 | } 39 | 40 | :journal-entry-line 41 | {:fiscal-year 42 | {:path [:journal-entry/_journal-entry-lines 43 | :ledger/_journal-entries 44 | :fiscal-year/_ledgers] 45 | :target :fiscal-year}} 46 | 47 | }} 48 | 49 | 50 | 51 | (path-binding '?jel '?fj '[:journal-entry/_journal-entry-lines 52 | :ledger/_journal-entries 53 | :fiscal-year/_ledgers]) 54 | 55 | (d/q '{:find [?e] 56 | :where [[?e :db/ident]]} (db)) 57 | 58 | 59 | (->> (for [path (->> #_[:journal-entry/_journal-entry-lines 60 | :ledger/_journal-entries 61 | :fiscal-year/_ledgers 62 | :company/_fiscal-year] 63 | [:fiscal-year/_accounts 64 | :company/_fiscal-year] 65 | reverse 66 | (reductions conj []) 67 | (map reverse) 68 | next) 69 | :let [[rel _] path 70 | table (keyword (subs (name rel) 1))]] 71 | [table :company {:path (vec path) 72 | :target :company}]) 73 | (reduce (fn [c p] 74 | (assoc-in c (butlast p) (last p))) {})) 75 | 76 | 77 | {:fiscal-year 78 | {:company {:path [:company/_fiscal-year] 79 | :target :company}} 80 | :ledger 81 | {:company {:path [:fiscal-year/_ledgers 82 | :company/_fiscal-year] 83 | :target :company}} 84 | :journal-entry 85 | {:company {:path [:ledger/_journal-entries 86 | :fiscal-year/_ledgers 87 | :company/_fiscal-year] 88 | :target :company}} 89 | :journal-entry-line 90 | {:company {:path [:journal-entry/_journal-entry-lines 91 | :ledger/_journal-entries 92 | :fiscal-year/_ledgers 93 | :company/_fiscal-year] 94 | :target :company}} 95 | :account 96 | {:company {:path [:fiscal-year/_accounts 97 | :company/_fiscal-year] 98 | :target :company}}} 99 | 100 | (user/refer-repl) 101 | 102 | (vec (sort (d/q '{:find [?ident] 103 | :where [[?e :db/ident ?ident] 104 | [?e :db/valueType :db.type/ref] 105 | [?e :db/cardinality :db.cardinality/many]]} 106 | (db eeleven-url)))) 107 | 108 | 109 | (def hierarchy 110 | {:tenant/companies 111 | {:company/fiscal-years 112 | {:fiscal-year/account-matches 113 | {} 114 | 115 | :fiscal-year/accounts 116 | {:account/children 117 | {} 118 | 119 | :account/contact-cards 120 | {} 121 | 122 | ;; :account-match/journal-entry-lines 123 | ;; {} 124 | } 125 | 126 | :fiscal-year/currency-options 127 | {} 128 | 129 | :fiscal-year/fiscal-periods 130 | {} 131 | 132 | :fiscal-year/ledgers 133 | {:ledger/journal-entries 134 | {:journal-entry/journal-entry-lines 135 | {} 136 | 137 | } 138 | 139 | :ledger/tax-entries 140 | {:tax-entry/tax-lines 141 | {}}} 142 | 143 | :fiscal-year/tax-accounts 144 | {} 145 | 146 | :fiscal-year/analytical-year 147 | {:analytical-year/entries 148 | {:analytical-entry/analytical-entry-lines 149 | {}} 150 | 151 | :analytical-year/dimensions 152 | {} 153 | 154 | :analytical-year/tracked-accounts 155 | {}} 156 | } 157 | 158 | :company/bank-reconciliations 159 | {:bank-reconciliation/bank-statement-lines 160 | {} 161 | 162 | ;; :bank-reconciliation/journal-entry-lines 163 | ;; {} 164 | } 165 | 166 | :company/bank-statements 167 | {:bank-statement/bank-statement-lines {}} 168 | 169 | :company/contact-cards 170 | {} 171 | 172 | :company/documents 173 | {} 174 | 175 | :company/payments 176 | {:payment/payables 177 | {} 178 | 179 | :payment/receivables 180 | {}} 181 | 182 | :company/receipts 183 | {:receipt/payables 184 | {} 185 | 186 | :receipt/receivables 187 | {}} 188 | 189 | :company/sequences 190 | {}} 191 | 192 | :tenant/org-units 193 | {:org-unit/children 194 | {}} 195 | 196 | :tenant/users 197 | {:user/roles 198 | {:role/roles 199 | {}}}}) 200 | 201 | (defn maps->paths [m] 202 | (mapcat (fn [k] 203 | (cons [k] 204 | (map (partial cons k) (maps->paths (get m k))))) 205 | (keys m))) 206 | 207 | (defn reverse-path [p] 208 | (->> p 209 | reverse 210 | (mapv #(keyword (namespace %) (str "_" (name %)))))) 211 | 212 | (defn singularize [kw] 213 | ({:entries :entry 214 | :tax-entries :tax-entry 215 | :journal-entries :journal-entry 216 | :account-matches :account-match 217 | :analytical-year :analytical-year} 218 | kw 219 | (keyword (subs (name kw) 0 (dec (count (name kw))))))) 220 | 221 | {:relationships 222 | (-> 223 | (reduce 224 | (fn [rels path] 225 | (assoc-in rels [(singularize 226 | (keyword (name (last path)))) 227 | :company] {:path (reverse-path path) 228 | :target :company})) 229 | {} 230 | (maps->paths (:tenant/companies hierarchy))) 231 | (dissoc :childre :tracked-account :receivable :tax-line))} 232 | -------------------------------------------------------------------------------- /repl_sessions/schema.clj: -------------------------------------------------------------------------------- 1 | (ns schema 2 | (:require [metabase.driver.datomic :as datomic-driver] 3 | [metabase.driver.datomic.query-processor :as datomic.qp] 4 | [metabase.models.database :refer [Database]] 5 | [toucan.db :as db])) 6 | 7 | (user/refer-repl) 8 | 9 | (datomic.qp/table-columns (db eeleven-url) "journal-entry-line") 10 | 11 | (filter (comp #{"account"} :name) 12 | (:fields (datomic-driver/describe-table (db/select-one Database :name "Eleven SG") {:name "journal-entry-line"}))) 13 | -------------------------------------------------------------------------------- /repl_sessions/test_data.clj: -------------------------------------------------------------------------------- 1 | (ns test-data 2 | (:require [datomic.api :as d] 3 | [metabase.driver.datomic.test :refer [with-datomic]] 4 | [metabase.test.data :as data])) 5 | 6 | (user/refer-repl) 7 | 8 | (with-datomic 9 | (data/get-or-create-test-data-db!)) 10 | 11 | (def url "datomic:mem:test-data") 12 | (def conn (d/connect url)) 13 | (defn db [] (d/db conn)) 14 | 15 | (d/q 16 | '{:find [?checkins|checkins|user_id] 17 | :order-by [[:asc (:checkins/user_id ?checkins)]], 18 | :where [[?checkins :checkins/user_id ?checkins|checkins|user_id]], 19 | :select [(:checkins/user_id ?checkins)], 20 | } 21 | (db)) 22 | 23 | 24 | (d/q 25 | '{:find [?medium] 26 | :where 27 | [(or [?medium :medium/format] [?medium :medium/name] [?medium :medium/position] [?medium :medium/trackCount] [?medium :medium/tracks]) 28 | [?medium :medium/format ?medium|medium|format] 29 | [?medium|medium|format :db/ident ?i] 30 | [(= ?i :medium.format/vinyl)]], 31 | ;;:select [(:db/id ?medium) (:medium/name ?medium) (:medium/format ?medium) (:medium/position ?medium) (:medium/trackCount ?medium) (:medium/tracks ?medium)], 32 | :with ()} 33 | (db)) 34 | 35 | (d/q 36 | '{;;:order-by [[:asc (:medium/name ?medium)] [:asc (:medium/format ?medium)]], 37 | :find [?medium|medium|name #_ ?medium|medium|format], 38 | :where [[?medium :medium/name ?medium|medium|name] 39 | #_[?medium :medium/format ?medium|medium|format]], 40 | ;; :select [(:medium/name ?medium) (:medium/format ?medium)], 41 | :with ()} 42 | (db)) 43 | 44 | 45 | 46 | 47 | 48 | [[5 86 #inst "2015-04-02T07:00:00.000-00:00"] 49 | [7 98 #inst "2015-04-04T07:00:00.000-00:00"] 50 | [1 97 #inst "2015-04-05T07:00:00.000-00:00"] 51 | [11 74 #inst "2015-04-06T07:00:00.000-00:00"] 52 | [5 12 #inst "2015-04-07T07:00:00.000-00:00"] 53 | [10 80 #inst "2015-04-08T07:00:00.000-00:00"] 54 | [11 73 #inst "2015-04-09T07:00:00.000-00:00"] 55 | [9 20 #inst "2015-04-09T07:00:00.000-00:00"] 56 | [8 51 #inst "2015-04-10T07:00:00.000-00:00"] 57 | [11 88 #inst "2015-04-10T07:00:00.000-00:00"] 58 | [10 44 #inst "2015-04-11T07:00:00.000-00:00"] 59 | [10 12 #inst "2015-04-12T07:00:00.000-00:00"] 60 | [5 38 #inst "2015-04-15T07:00:00.000-00:00"] 61 | [14 8 #inst "2015-04-16T07:00:00.000-00:00"] 62 | [12 5 #inst "2015-04-16T07:00:00.000-00:00"] 63 | [1 1 #inst "2015-04-18T07:00:00.000-00:00"] 64 | [9 21 #inst "2015-04-18T07:00:00.000-00:00"] 65 | [12 49 #inst "2015-04-19T07:00:00.000-00:00"] 66 | [8 84 #inst "2015-04-20T07:00:00.000-00:00"] 67 | [12 7 #inst "2015-04-21T07:00:00.000-00:00"] 68 | [7 98 #inst "2015-04-21T07:00:00.000-00:00"] 69 | [7 10 #inst "2015-04-23T07:00:00.000-00:00"] 70 | [12 2 #inst "2015-04-23T07:00:00.000-00:00"] 71 | [11 71 #inst "2015-04-24T07:00:00.000-00:00"] 72 | [6 22 #inst "2015-04-24T07:00:00.000-00:00"] 73 | [1 46 #inst "2015-04-25T07:00:00.000-00:00"] 74 | [11 65 #inst "2015-04-30T07:00:00.000-00:00"] 75 | [15 52 #inst "2015-05-01T07:00:00.000-00:00"] 76 | [13 86 #inst "2015-05-01T07:00:00.000-00:00"]] 77 | -------------------------------------------------------------------------------- /repl_sessions/undo_special_types.clj: -------------------------------------------------------------------------------- 1 | (ns undo-special-types 2 | (:require [metabase.models.database :refer [Database]] 3 | [metabase.models.field :refer [Field]] 4 | [metabase.models.table :refer [Table]] 5 | [toucan.db :as db])) 6 | 7 | (db/update-where! Field 8 | {:table_id [:in {:select [:id] 9 | :from [:metabase_table] 10 | :where [:in :db_id {:select [:id] 11 | :from [:metabase_database] 12 | :where [:= :engine "datomic"]}]}] 13 | :database_type [:not= "db.type/ref"] 14 | :special_type "type/PK"} 15 | :special_type nil) 16 | -------------------------------------------------------------------------------- /resources/metabase-plugin.yaml: -------------------------------------------------------------------------------- 1 | info: 2 | name: Metabase Datomic Driver 3 | version: 1.0.0-SNAPSHOT-0.9.5697 4 | description: Allows Metabase to connect to Datomic databases. 5 | driver: 6 | name: datomic 7 | display-name: Datomic 8 | lazy-load: true 9 | connection-properties: 10 | - name: db 11 | display-name: URL 12 | placeholder: datomic:mem://metabase_dev 13 | required: true 14 | - name: config 15 | display-name: Configuration EDN 16 | placeholder: "{}" 17 | required: true 18 | init: 19 | - step: load-namespace 20 | namespace: metabase.driver.datomic 21 | -------------------------------------------------------------------------------- /src/metabase/driver/datomic.clj: -------------------------------------------------------------------------------- 1 | (ns metabase.driver.datomic 2 | (:require [datomic.api :as d] 3 | [metabase.driver :as driver] 4 | [metabase.driver.datomic.query-processor :as datomic.qp] 5 | [metabase.driver.datomic.util :as util] 6 | [toucan.db :as db] 7 | [clojure.tools.logging :as log])) 8 | 9 | (require 'metabase.driver.datomic.monkey-patch) 10 | 11 | (driver/register! :datomic) 12 | 13 | (def features 14 | {:basic-aggregations true 15 | :standard-deviation-aggregations true 16 | :case-sensitivity-string-filter-options true 17 | :foreign-keys true 18 | :nested-queries true 19 | :expressions false 20 | :expression-aggregations false 21 | :native-parameters false 22 | :binning false}) 23 | 24 | (doseq [[feature] features] 25 | (defmethod driver/supports? [:datomic feature] [_ _] 26 | (get features feature))) 27 | 28 | (defmethod driver/can-connect? :datomic [_ {db :db}] 29 | (try 30 | (d/connect db) 31 | true 32 | (catch Exception e 33 | false))) 34 | 35 | (defmethod driver/describe-database :datomic [_ instance] 36 | (let [url (get-in instance [:details :db]) 37 | table-names (datomic.qp/derive-table-names (d/db (d/connect url)))] 38 | {:tables 39 | (set 40 | (for [table-name table-names] 41 | {:name table-name 42 | :schema nil}))})) 43 | 44 | (derive :type/Keyword :type/Text) 45 | 46 | (def datomic->metabase-type 47 | {:db.type/keyword :type/Keyword ;; Value type for keywords. 48 | :db.type/string :type/Text ;; Value type for strings. 49 | :db.type/boolean :type/Boolean ;; Boolean value type. 50 | :db.type/long :type/Integer ;; Fixed integer value type. Same semantics as a Java long: 64 bits wide, two's complement binary representation. 51 | :db.type/bigint :type/BigInteger ;; Value type for arbitrary precision integers. Maps to java.math.BigInteger on Java platforms. 52 | :db.type/float :type/Float ;; Floating point value type. Same semantics as a Java float: single-precision 32-bit IEEE 754 floating point. 53 | :db.type/double :type/Float ;; Floating point value type. Same semantics as a Java double: double-precision 64-bit IEEE 754 floating point. 54 | :db.type/bigdec :type/Decimal ;; Value type for arbitrary precision floating point numbers. Maps to java.math.BigDecimal on Java platforms. 55 | :db.type/ref :type/FK ;; Value type for references. All references from one entity to another are through attributes with this value type. 56 | :db.type/instant :type/DateTime ;; Value type for instants in time. Stored internally as a number of milliseconds since midnight, January 1, 1970 UTC. Maps to java.util.Date on Java platforms. 57 | :db.type/uuid :type/UUID ;; Value type for UUIDs. Maps to java.util.UUID on Java platforms. 58 | :db.type/uri :type/URL ;; Value type for URIs. Maps to java.net.URI on Java platforms. 59 | :db.type/bytes :type/Array ;; Value type for small binary data. Maps to byte array on Java platforms. See limitations. 60 | :db.type/tuple :type/Array 61 | 62 | :db.type/symbol :type/Keyword 63 | #_:db.type/fn}) 64 | 65 | (defn column-name [table-name col] 66 | (if (= (namespace col) 67 | table-name) 68 | (name col) 69 | (util/kw->str col))) 70 | 71 | (defn describe-table [database {table-name :name}] 72 | (let [url (get-in database [:details :db]) 73 | config (datomic.qp/user-config database) 74 | db (d/db (d/connect url)) 75 | cols (datomic.qp/table-columns db table-name) 76 | rels (get-in config [:relationships (keyword table-name)]) 77 | xtra-fields (get-in config [:fields (keyword table-name)])] 78 | {:name table-name 79 | :schema nil 80 | 81 | ;; Fields *must* be a set 82 | :fields 83 | (-> #{{:name "db/id" 84 | :database-type "db.type/ref" 85 | :base-type :type/PK 86 | :pk? true}} 87 | (into (for [[col type] cols] 88 | {:name (column-name table-name col) 89 | :database-type (util/kw->str type) 90 | :base-type (datomic->metabase-type type) 91 | :special-type (datomic->metabase-type type)})) 92 | (into (for [[rel-name {:keys [_path target]}] rels] 93 | {:name (name rel-name) 94 | :database-type "metabase.driver.datomic/path" 95 | :base-type :type/FK 96 | :special-type :type/FK})) 97 | (into (for [[fname {:keys [type _rule]}] xtra-fields] 98 | {:name (name fname) 99 | :database-type "metabase.driver.datomic/computed-field" 100 | :base-type type 101 | :special-type type})))})) 102 | 103 | (defmethod driver/describe-table :datomic [_ database table] 104 | (describe-table database table)) 105 | 106 | (defn guess-dest-column [db table-names col] 107 | (let [table? (into #{} table-names) 108 | attrs (d/q (assoc '{:find [[?ident ...]]} 109 | :where [['_ col '?eid] 110 | '[?eid ?attr] 111 | '[?attr :db/ident ?ident]]) 112 | db)] 113 | (or (some->> attrs 114 | (map namespace) 115 | (remove #{"db"}) 116 | frequencies 117 | (sort-by val) 118 | last 119 | key) 120 | (table? (name col))))) 121 | 122 | (defn describe-table-fks [database {table-name :name}] 123 | (let [url (get-in database [:details :db]) 124 | db (d/db (d/connect url)) 125 | config (datomic.qp/user-config database) 126 | tables (datomic.qp/derive-table-names db) 127 | cols (datomic.qp/table-columns db table-name) 128 | rels (get-in config [:relationships (keyword table-name)])] 129 | 130 | (-> #{} 131 | (into (for [[col type] cols 132 | :when (= type :db.type/ref) 133 | :let [dest (guess-dest-column db tables col)] 134 | :when dest] 135 | {:fk-column-name (column-name table-name col) 136 | :dest-table {:name dest 137 | :schema nil} 138 | :dest-column-name "db/id"})) 139 | (into (for [[rel-name {:keys [path target]}] rels] 140 | {:fk-column-name (name rel-name) 141 | :dest-table {:name (name target) 142 | :schema nil} 143 | :dest-column-name "db/id"}))))) 144 | 145 | (defmethod driver/describe-table-fks :datomic [_ database table] 146 | (describe-table-fks database table)) 147 | 148 | (defonce mbql-history (atom ())) 149 | (defonce query-history (atom ())) 150 | 151 | (defmethod driver/mbql->native :datomic [_ query] 152 | (swap! mbql-history conj query) 153 | (datomic.qp/mbql->native query)) 154 | 155 | (defmethod driver/execute-query :datomic [_ native-query] 156 | (swap! query-history conj native-query) 157 | (let [result (datomic.qp/execute-query native-query)] 158 | (swap! query-history conj result) 159 | result)) 160 | -------------------------------------------------------------------------------- /src/metabase/driver/datomic/fix_types.clj: -------------------------------------------------------------------------------- 1 | (ns metabase.driver.datomic.fix-types 2 | (:require [metabase.models.database :refer [Database]] 3 | [metabase.models.field :refer [Field]] 4 | [metabase.models.table :refer [Table]] 5 | [toucan.db :as db])) 6 | 7 | (defn undo-invalid-primary-keys! 8 | "Only :db/id should ever be marked as PK, unfortunately Metabase heuristics can 9 | also mark other fields as primary, which leads to a mess. This changes the 10 | \"special_type\" of any field with special_type=PK back to nil, unless the 11 | field is called :db/id." 12 | [] 13 | (db/update-where! Field 14 | {:table_id [:in {:select [:id] 15 | :from [:metabase_table] 16 | :where [:in :db_id {:select [:id] 17 | :from [:metabase_database] 18 | :where [:= :engine "datomic"]}]}] 19 | :database_type [:not= "db.type/ref"] 20 | :special_type "type/PK"} 21 | :special_type nil)) 22 | -------------------------------------------------------------------------------- /src/metabase/driver/datomic/monkey_patch.clj: -------------------------------------------------------------------------------- 1 | (ns metabase.driver.datomic.monkey-patch 2 | (:require [toucan.models] 3 | [toucan.hydrate])) 4 | 5 | ;; Prevent Toucan from doing a keyword-call on every var in the systems, as this 6 | ;; causes some of Datomic's internal cache structures to throw an exception. 7 | ;; https://github.com/metabase/toucan/issues/55 8 | 9 | ;; We can drop this as soon as Metabase has upgraded to Toucan 1.12.0 10 | 11 | (defn- require-model-namespaces-and-find-hydration-fns [] 12 | (reduce 13 | (fn [coll ns] 14 | (reduce 15 | (fn [coll sym-var] 16 | (let [model (var-get (val sym-var))] 17 | (if (and (record? model) (toucan.models/model? model)) 18 | (reduce #(assoc %1 %2 model) 19 | coll 20 | (toucan.models/hydration-keys model)) 21 | coll))) 22 | coll 23 | (ns-publics ns))) 24 | {} 25 | (all-ns))) 26 | 27 | (alter-var-root #'toucan.hydrate/require-model-namespaces-and-find-hydration-fns 28 | (constantly require-model-namespaces-and-find-hydration-fns)) 29 | -------------------------------------------------------------------------------- /src/metabase/driver/datomic/query_processor.clj: -------------------------------------------------------------------------------- 1 | (ns metabase.driver.datomic.query-processor 2 | (:require [clojure.set :as set] 3 | [clojure.string :as str] 4 | [datomic.api :as d] 5 | [metabase.driver.datomic.util :as util :refer [pal par]] 6 | [metabase.mbql.util :as mbql.u] 7 | [metabase.models.field :as field :refer [Field]] 8 | [metabase.models.table :refer [Table]] 9 | [metabase.query-processor.store :as qp.store] 10 | [toucan.db :as db] 11 | [clojure.tools.logging :as log] 12 | [clojure.walk :as walk]) 13 | (:import java.net.URI 14 | java.util.UUID)) 15 | 16 | ;; Local variable naming conventions: 17 | 18 | ;; dqry : Datalog query 19 | ;; mbqry : Metabase (MBQL) query 20 | ;; db : Datomic DB instance 21 | ;; attr : A datomic attribute, i.e. a qualified keyword 22 | ;; ?foo / lvar : A logic variable, i.e. a symbol starting with a question mark 23 | 24 | (def connect #_(memoize d/connect) 25 | d/connect) 26 | 27 | (defn user-config 28 | ([] 29 | (user-config (qp.store/database))) 30 | ([database] 31 | (try 32 | (let [edn (get-in database [:details :config])] 33 | (read-string (or edn "{}"))) 34 | (catch Exception e 35 | (log/error e "Datomic EDN is not configured correctly.") 36 | {})))) 37 | 38 | (defn tx-filter [] 39 | (when-let [form (get (user-config) :tx-filter)] 40 | (eval form))) 41 | 42 | (defn db [] 43 | (let [db (-> (get-in (qp.store/database) [:details :db]) connect d/db)] 44 | (if-let [pred (tx-filter)] 45 | (d/filter db pred) 46 | db))) 47 | 48 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 49 | ;; SCHEMA 50 | 51 | (def reserved-prefixes 52 | #{"fressian" 53 | "db" 54 | "db.alter" 55 | "db.excise" 56 | "db.install" 57 | "db.sys"}) 58 | 59 | (defn attributes 60 | "Query db for all attribute entities." 61 | [db] 62 | (->> db 63 | (d/q '{:find [[?eid ...]] :where [[?eid :db/valueType] 64 | [?eid :db/ident]]}) 65 | (map (partial d/entity db)))) 66 | 67 | (defn attrs-by-table 68 | "Map from table name to collection of attribute entities." 69 | [db] 70 | (reduce #(update %1 (namespace (:db/ident %2)) conj %2) 71 | {} 72 | (attributes db))) 73 | 74 | (defn derive-table-names 75 | "Find all \"tables\" i.e. all namespace prefixes used in attribute names." 76 | [db] 77 | (remove reserved-prefixes 78 | (keys (attrs-by-table db)))) 79 | 80 | (defn table-columns 81 | "Given the name of a \"table\" (attribute namespace prefix), find all attribute 82 | names that occur in entities that have an attribute with this prefix." 83 | [db table] 84 | {:pre [(instance? datomic.db.Db db) 85 | (string? table)]} 86 | (let [attrs (get (attrs-by-table db) table)] 87 | (-> #{} 88 | (into (map (juxt :db/ident :db/valueType)) 89 | attrs) 90 | (into (d/q 91 | {:find '[?ident ?type] 92 | :where [(cons 'or 93 | (for [attr attrs] 94 | ['?eid (:db/ident attr)])) 95 | '[?eid ?attr] 96 | '[?attr :db/ident ?ident] 97 | '[?attr :db/valueType ?type-id] 98 | '[?type-id :db/ident ?type] 99 | '[(not= ?ident :db/ident)]]} 100 | db)) 101 | sort))) 102 | 103 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 104 | ;; QUERY->NATIVE 105 | 106 | (def ^:dynamic *settings* {}) 107 | 108 | (def ^:dynamic *mbqry* nil) 109 | 110 | (defn- timezone-id 111 | [] 112 | (or (:report-timezone *settings*) "UTC")) 113 | 114 | (defn source-table [] 115 | (:source-table *mbqry* (:source-table (:source-query *mbqry*)))) 116 | 117 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 118 | 119 | (def ^:dynamic 120 | *db* 121 | "Datomic db, for when we need to inspect the schema during query generation." 122 | nil) 123 | 124 | (defn cardinality-many? 125 | "Is the given keyword an reference attribute with cardinality/many?" 126 | [attr] 127 | (= :db.cardinality/many (:db/cardinality (d/entity *db* attr)))) 128 | 129 | (defn attr-type [attr] 130 | (get-in 131 | (d/pull *db* [{:db/valueType [:db/ident]}] attr) 132 | [:db/valueType :db/ident])) 133 | 134 | (defn entid [ident] 135 | (d/entid *db* ident)) 136 | 137 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 138 | ;; Datalog query helpers 139 | ;; 140 | ;; These functions all handle various parts of building up complex Datalog 141 | ;; queries based on the given MBQL query. 142 | 143 | (declare aggregation-clause) 144 | (declare field-lvar) 145 | 146 | (defn into-clause 147 | "Helper to build up datalog queries. Takes a partial query, a clause like :find 148 | or :where, and a collection, and appends the collection's elements into the 149 | given clause. 150 | 151 | Optionally takes a transducer." 152 | ([dqry clause coll] 153 | (into-clause dqry clause identity coll)) 154 | ([dqry clause xform coll] 155 | (if (seq coll) 156 | (update dqry clause (fn [x] (into (or x []) xform coll))) 157 | dqry))) 158 | 159 | (defn distinct-preseed 160 | "Like clojure.core distinct, but preseed the 'seen' collection." 161 | [coll] 162 | (fn [rf] 163 | (let [seen (volatile! (set coll))] 164 | (fn 165 | ([] (rf)) 166 | ([result] (rf result)) 167 | ([result input] 168 | (if (contains? @seen input) 169 | result 170 | (do (vswap! seen conj input) 171 | (rf result input)))))))) 172 | 173 | (defn into-clause-uniq 174 | "Like into-clause, but assures that the same clause is never inserted twice." 175 | ([dqry clause coll] 176 | (into-clause-uniq dqry clause identity coll)) 177 | ([dqry clause xform coll] 178 | (into-clause dqry 179 | clause 180 | (comp xform 181 | (distinct-preseed (get dqry clause))) 182 | coll))) 183 | 184 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 185 | 186 | ;; [:field-id 55] => :artist/name 187 | (defmulti ->attrib 188 | "Convert an MBQL field reference (vector) to a Datomic attribute name (qualified 189 | symbol)." 190 | mbql.u/dispatch-by-clause-name-or-class) 191 | 192 | (defmethod ->attrib (class Field) [{:keys [name table_id] :as field}] 193 | (if (some #{\/} name) 194 | (keyword name) 195 | (keyword (:name (qp.store/table table_id)) name))) 196 | 197 | (defmethod ->attrib :field-id [[_ field-id]] 198 | (->attrib (qp.store/field field-id))) 199 | 200 | (defmethod ->attrib :fk-> [[_ src dst]] 201 | (->attrib dst)) 202 | 203 | (defmethod ->attrib :aggregation [[_ field-id]] 204 | (->attrib (qp.store/field field-id))) 205 | 206 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 207 | 208 | (defmulti field-inst 209 | "Given an MBQL field reference, return the metabase.model.Field instance. May 210 | return nil." 211 | mbql.u/dispatch-by-clause-name-or-class) 212 | 213 | (defmethod field-inst :field-id [[_ id]] 214 | (qp.store/field id)) 215 | 216 | (defmethod field-inst :fk-> [[_ src _]] 217 | (field-inst src)) 218 | 219 | (defmethod field-inst :datetime-field [[_ field _]] 220 | (field-inst field)) 221 | 222 | (defmethod field-inst :field-literal [[_ literal]] 223 | (when-let [table (source-table)] 224 | (db/select-one Field :table_id table :name literal))) 225 | 226 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 227 | 228 | ;; [:field-id 55] => ?artist 229 | (defmulti table-lvar 230 | "Return a logic variable name (a symbol starting with '?') corresponding with 231 | the 'table' of the given field reference." 232 | mbql.u/dispatch-by-clause-name-or-class) 233 | 234 | (defmethod table-lvar (class Table) [{:keys [name]}] 235 | (symbol (str "?" name))) 236 | 237 | (defmethod table-lvar (class Field) [{:keys [table_id]}] 238 | (table-lvar (qp.store/table table_id))) 239 | 240 | (defmethod table-lvar Integer [table_id] 241 | (table-lvar (qp.store/table table_id))) 242 | 243 | (defmethod table-lvar :field-id [[_ field-id]] 244 | (table-lvar (qp.store/field field-id))) 245 | 246 | (defmethod table-lvar :fk-> [[_ src dst]] 247 | (field-lvar src)) 248 | 249 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 250 | 251 | (defmulti field-lookup 252 | "Turn an MBQL field reference into something we can stick into our 253 | pseudo-datalag :select or :order-by clauses. In some cases we stick the same 254 | thing in :find, in others we parse this form after the fact to pull the data 255 | out of the entity. 256 | 257 | Closely related to field-lvar, but the latter is always just a single 258 | symbol (logic variable), but with sections separated by | which can be parsed. 259 | 260 | [:field 15] ;;=> (:artist/name ?artist) 261 | [:datetime-field [:field 25] :hour] ;;=> (datetime (:user/logged-in ?user) :hour) 262 | [:aggregation 0] ;;=> (count ?artist)" 263 | (fn [_ field-ref] 264 | (mbql.u/dispatch-by-clause-name-or-class field-ref))) 265 | 266 | (defmethod field-lookup :default [_ field-ref] 267 | `(field ~(field-lvar field-ref) 268 | ~(select-keys (field-inst field-ref) [:database_type :base_type :special_type]))) 269 | 270 | (defmethod field-lookup :aggregation [mbqry [_ idx]] 271 | (aggregation-clause mbqry (nth (:aggregation mbqry) idx))) 272 | 273 | (defmethod field-lookup :datetime-field [mbqry [_ fref unit]] 274 | `(datetime ~(field-lvar fref) ~unit)) 275 | 276 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 277 | 278 | ;; Note that lvar in this context always means *logic variable* (in the 279 | ;; prolog/datalog sense), not to be confused with lval/rval in the C/C++ sense. 280 | 281 | (defn lvar 282 | "Generate a logic variable, a symbol starting with a question mark, by combining 283 | multiple pieces separated by pipe symbols. Parts are converted to string and 284 | leading question marks stripped, so you can combined lvars into one bigger 285 | lvar, e.g. `(lvar '?foo '?bar)` => `?foo|bar`" 286 | [& parts] 287 | (symbol 288 | (str "?" (str/join "|" (map (fn [p] 289 | (let [s (str p)] 290 | (cond-> s 291 | (= \: (first s)) 292 | (subs 1) 293 | (= \? (first s)) 294 | (subs 1)))) 295 | parts))))) 296 | 297 | ;; "[:field-id 45] ;;=> ?artist|artist|name" 298 | (defmulti field-lvar 299 | "Convert an MBQL field reference like [:field-id 45] into a logic variable named 300 | based on naming conventions (see Architecture Decision Log). 301 | 302 | Will look like this: 303 | 304 | ?table|attr-namespace|attr-name 305 | ?table|attr-namespace|attr-name|time-binning-unit 306 | ?table|attr-namespace|attr-name->fk-dest-table|fk-attr-ns|fk-attr-name" 307 | mbql.u/dispatch-by-clause-name-or-class) 308 | 309 | (defmethod field-lvar :field-id [field-ref] 310 | (let [attr (->attrib field-ref) 311 | eid (table-lvar field-ref)] 312 | (if (= :db/id attr) 313 | eid 314 | (lvar eid (namespace attr) (name attr))))) 315 | 316 | (defmethod field-lvar :datetime-field [[_ ref unit]] 317 | (lvar (field-lvar ref) (name unit))) 318 | 319 | (defmethod field-lvar :fk-> [[_ src dst]] 320 | (if (= "db/id" (:name (field-inst dst))) 321 | (field-lvar src) 322 | (lvar (str (field-lvar src) "->" (subs (str (field-lvar dst)) 1))))) 323 | 324 | (defmethod field-lvar :field-literal [[_ field-name]] 325 | (if (some #{\|} field-name) 326 | (lvar field-name) 327 | (if-let [table (source-table)] 328 | (let [?table (table-lvar table)] 329 | (if (some #{\/} field-name) 330 | (lvar ?table field-name) 331 | (lvar ?table ?table field-name))) 332 | (lvar field-name)))) 333 | 334 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 335 | 336 | (defn ident-lvar [field-ref] 337 | (symbol (str (field-lvar field-ref) ":ident"))) 338 | 339 | ;; Datomic function helpers 340 | (defmacro %get-else% [& args] `(list '~'get-else ~@args)) 341 | (defmacro %count% [& args] `(list '~'count ~@args)) 342 | (defmacro %count-distinct% [& args] `(list '~'count-distinct ~@args)) 343 | 344 | (def NIL ::nil) 345 | (def NIL-REF Long/MIN_VALUE) 346 | 347 | ;; Try to provide a type-appropriate placeholder, so that Datomic is able to 348 | ;; sort/group correctly 349 | (def NIL_VALUES 350 | {:db.type/string (str ::nil) 351 | :db.type/keyword ::nil 352 | :db.type/boolean false 353 | :db.type/bigdec Long/MIN_VALUE 354 | :db.type/bigint Long/MIN_VALUE 355 | :db.type/double Long/MIN_VALUE 356 | :db.type/float Long/MIN_VALUE 357 | :db.type/long Long/MIN_VALUE 358 | :db.type/ref Long/MIN_VALUE 359 | :db.type/instant #inst "0001-01-01T00:00:00" 360 | :db.type/uri (URI. (str "nil" ::nil)) 361 | :db.type/uuid #uuid "00000000-0000-0000-0000-000000000000"}) 362 | 363 | (def ^:dynamic *strict-bindings* false) 364 | 365 | (defmacro with-strict-bindings [& body] 366 | `(binding [*strict-bindings* true] 367 | ~@body)) 368 | 369 | (defn- bind-attr-reverse 370 | "When dealing with reverse attributes (:foo/_bar) either missing? or get-else 371 | don't work, and we need a different approach to get correct join semantics in 372 | the presence of missing data. 373 | 374 | Note that the caller takes care of removing the underscore from the attribute 375 | name and swapping the ?e and ?v." 376 | [?e a ?v] 377 | (if *strict-bindings* 378 | [?e a ?v] 379 | (list 'or-join [?e ?v] 380 | [?e a ?v] 381 | (list 'and (list 'not ['_ a ?v]) 382 | [(list 'ground NIL-REF) ?e])))) 383 | 384 | (defn reverse-attr? [attr] 385 | (when (= \_ (first (name attr))) 386 | (keyword (namespace attr) (subs (name attr) 1)))) 387 | 388 | (defn bind-attr 389 | "Datalog EAV binding that unifies to NIL if the attribute is not present, the 390 | equivalent of an outer join, so we can look up attributes without filtering 391 | the result at the same time. 392 | 393 | Will do a simple [e a v] binding when *strict-bindings* is true." 394 | [?e a ?v] 395 | (if-let [a (reverse-attr? a)] 396 | (bind-attr-reverse ?v a ?e) 397 | (cond 398 | *strict-bindings* 399 | [?e a ?v] 400 | 401 | ;; get-else is not supported on cardinality/many 402 | (cardinality-many? a) 403 | (list 'or-join [?e ?v] 404 | [?e a ?v] 405 | (list 'and 406 | [(list 'missing? '$ ?e a)] 407 | [(list 'ground NIL-REF) ?v])) 408 | 409 | :else 410 | [(%get-else% '$ ?e a (NIL_VALUES (attr-type a) ::nil)) ?v]))) 411 | 412 | (defn date-trunc-or-extract-some [unit date] 413 | (if (= NIL date) 414 | NIL 415 | (metabase.util.date/date-trunc-or-extract unit date (timezone-id)))) 416 | 417 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 418 | 419 | (defn path-bindings 420 | "Given a start and end point (two lvars) and a path, a sequence of Datomic 421 | attributes (kw) and rule names (sym), generates a sequence of :where binding 422 | forms to navigate from start to symbol via the given path." 423 | [?from ?to path] 424 | (let [segment-binding 425 | (fn [?from ?to seg] 426 | (cond 427 | (keyword? seg) 428 | (bind-attr ?from seg ?to) 429 | 430 | (symbol? seg) 431 | (if (= \_ (first (name seg))) 432 | (list (symbol (subs (str seg) 1)) ?to ?from) 433 | (list seg ?from ?to))))] 434 | (loop [[p & path] path 435 | binding [] 436 | ?from ?from] 437 | (if (seq path) 438 | (let [?next-from (lvar ?from (namespace p) (name p))] 439 | (recur path 440 | (conj binding (segment-binding ?from ?next-from p)) 441 | ?next-from)) 442 | (conj binding (segment-binding ?from ?to p)))))) 443 | 444 | (defn custom-relationship? 445 | "Is tis field reference a custom relationship, i.e. configured via the admin UI 446 | and backed by a custom path traversal." 447 | [field-ref] 448 | (-> field-ref 449 | field-inst 450 | :database_type 451 | (= "metabase.driver.datomic/path"))) 452 | 453 | (defn computed-field? 454 | "Is tis field reference a custom relationship, i.e. configured via the admin UI 455 | and backed by a custom path traversal." 456 | [field-ref] 457 | (-> field-ref 458 | field-inst 459 | :database_type 460 | (= "metabase.driver.datomic/computed-field"))) 461 | 462 | ;;=> [:field-id 45] ;;=> [[?artist :artist/name ?artist|artist|name]] 463 | (defmulti field-bindings 464 | "Given a field reference, return the necessary Datalog bindings (as used 465 | in :where) to bind the entity-eid to [[table-lvar]], and the associated value 466 | to [[field-lvar]]. 467 | 468 | This uses Datomic's `get-else` to prevent filtering, in other words this will 469 | bind logic variables, but does not restrict the result." 470 | mbql.u/dispatch-by-clause-name-or-class) 471 | 472 | (defmethod field-bindings :field-id [field-ref] 473 | (cond 474 | (custom-relationship? field-ref) 475 | (let [src-field (field-inst field-ref) 476 | src-name (keyword (:name (qp.store/table (:table_id src-field)))) 477 | rel-name (keyword (:name src-field)) 478 | {:keys [path target]} (get-in (user-config) [:relationships src-name rel-name])] 479 | (path-bindings (table-lvar field-ref) 480 | (field-lvar field-ref) 481 | path)) 482 | 483 | (computed-field? field-ref) 484 | (let [field (field-inst field-ref) 485 | table-name (keyword (:name (qp.store/table (:table_id field)))) 486 | field-name (keyword (:name field)) 487 | {:keys [rule]} (get-in (user-config) [:fields table-name field-name])] 488 | [(list rule (table-lvar field-ref) (field-lvar field-ref))]) 489 | 490 | :else 491 | (let [attr (->attrib field-ref)] 492 | (when-not (= :db/id attr) 493 | [(bind-attr (table-lvar field-ref) attr (field-lvar field-ref))])))) 494 | 495 | (defmethod field-bindings :field-literal [[_ literal :as field-ref]] 496 | ;; This is dodgy, as field literals contain no source or schema information, 497 | ;; but this is used in native queries, so if we have a source table, and the 498 | ;; field name seems to correspond with an actual attribute with that prefix, 499 | ;; then we'll go for it. If not this retuns an empty seq i.e. doesn't bind 500 | ;; anything, and you will likely end up with Datomic complaining about 501 | ;; insufficient bindings. 502 | (if-let [table (source-table)] 503 | (let [attr (keyword (:name (qp.store/table table)) literal)] 504 | (if (attr-type attr) 505 | [(bind-attr (table-lvar table) attr (field-lvar field-ref))] 506 | [])) 507 | [])) 508 | 509 | (defmethod field-bindings :fk-> [[_ src dst :as field]] 510 | (cond 511 | (custom-relationship? src) 512 | (let [src-field (field-inst src) 513 | src-name (keyword (:name (qp.store/table (:table_id src-field)))) 514 | rel-name (keyword (:name src-field)) 515 | {:keys [path target]} (get-in (user-config) [:relationships src-name rel-name]) 516 | attrib (->attrib dst) 517 | path (if (= :db/id attrib) 518 | path 519 | (conj path attrib))] 520 | (path-bindings (table-lvar src) 521 | (field-lvar field) 522 | path)) 523 | 524 | (computed-field? dst) 525 | (let [dst-field (field-inst dst) 526 | table-name (keyword (:name (qp.store/table (:table_id dst-field)))) 527 | field-name (keyword (:name dst-field)) 528 | {:keys [rule]} (get-in (user-config) [:fields table-name field-name])] 529 | [(bind-attr (table-lvar src) (->attrib src) (field-lvar src)) 530 | (list rule (field-lvar src) (field-lvar field))]) 531 | 532 | (= :db/id (->attrib field)) 533 | [[(table-lvar src) (->attrib src) (table-lvar field)]] 534 | 535 | :else 536 | [(bind-attr (table-lvar src) (->attrib src) (field-lvar src)) 537 | (bind-attr (field-lvar src) (->attrib field) (field-lvar field))])) 538 | 539 | (defmethod field-bindings :aggregation [_] 540 | []) 541 | 542 | (defmethod field-bindings :datetime-field [[_ field-ref unit :as dt-field]] 543 | (conj (field-bindings field-ref) 544 | [`(date-trunc-or-extract-some ~unit ~(field-lvar field-ref)) 545 | (field-lvar dt-field)])) 546 | 547 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 548 | 549 | (defmulti constant-binding 550 | "Datalog bindings for filtering based on the value of an attribute (a constant 551 | value in MBQL), given a field reference and a value. 552 | 553 | At its simplest returns [[?table-lvar :attribute CONSTANT]], but can return 554 | more complex forms to deal with looking by :db/id, by :db/ident, or through 555 | foreign key." 556 | (fn [field-ref value] 557 | (mbql.u/dispatch-by-clause-name-or-class field-ref))) 558 | 559 | (defmethod constant-binding :field-id [field-ref value] 560 | (let [attr (->attrib field-ref) 561 | ?eid (table-lvar field-ref) 562 | ?val (field-lvar field-ref)] 563 | (if (= :db/id attr) 564 | (if (keyword? value) 565 | [[?eid :db/ident value]] 566 | [[(list '= ?eid value)]]) 567 | (conj (field-bindings field-ref) 568 | [(list 'ground value) ?val])))) 569 | 570 | (defmethod constant-binding :field-literal [field-name value] 571 | [[(list 'ground value) (field-lvar field-name)]]) 572 | 573 | (defmethod constant-binding :datetime-field [[_ field-ref unit :as dt-field] value] 574 | (let [?val (field-lvar dt-field)] 575 | (conj (field-bindings dt-field) 576 | [(list '= (date-trunc-or-extract-some unit value) ?val)]))) 577 | 578 | (defmethod constant-binding :fk-> [[_ src dst :as field-ref] value] 579 | (let [src-attr (->attrib src) 580 | dst-attr (->attrib dst) 581 | ?src (table-lvar src) 582 | ?dst (field-lvar src) 583 | ?val (field-lvar field-ref)] 584 | (if (= :db/id dst-attr) 585 | (if (keyword? value) 586 | (let [?ident (ident-lvar field-ref)] 587 | [[?src src-attr ?ident] 588 | [?ident :db/ident value]]) 589 | [[?src src-attr value]]) 590 | (conj (field-bindings src) 591 | (bind-attr ?dst dst-attr ?val) 592 | [(list 'ground value) ?val])))) 593 | 594 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 595 | 596 | (defmulti aggregation-clause 597 | "Return a Datalog clause for a given MBQL aggregation 598 | 599 | [:count [:field-id 45]] 600 | ;;=> (count ?foo|bar|baz)" 601 | (fn [mbqry aggregation] 602 | (first aggregation))) 603 | 604 | (defmethod aggregation-clause :default [mbqry [aggr-type field-ref]] 605 | (list (symbol (name aggr-type)) (field-lvar field-ref))) 606 | 607 | (defmethod aggregation-clause :count [mbqry [_ field-ref]] 608 | (cond 609 | field-ref 610 | (%count% (field-lvar field-ref)) 611 | 612 | (source-table) 613 | (%count% (table-lvar (source-table))) 614 | 615 | :else 616 | (assert false "Count without field is not supported on native sub-queries."))) 617 | 618 | (defmethod aggregation-clause :distinct [mbqry [_ field-ref]] 619 | (%count-distinct% (field-lvar field-ref))) 620 | 621 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 622 | 623 | (defmulti value-literal 624 | "Extracts the value literal out of and coerce to the DB type. 625 | 626 | [:value 40 627 | {:base_type :type/Float 628 | :special_type :type/Latitude 629 | :database_type \"db.type/double\"}] 630 | ;;=> 40" 631 | (fn [[type :as clause]] type)) 632 | 633 | (defmethod value-literal :default [clause] 634 | (assert false (str "Unrecognized value clause: " clause))) 635 | 636 | (defmethod value-literal :value [[t v f]] 637 | (if (nil? v) 638 | (NIL_VALUES (keyword (:database_type f)) NIL) 639 | (case (:database_type f) 640 | "db.type/ref" 641 | (cond 642 | (and (string? v) (some #{\/} v)) 643 | (entid (keyword v)) 644 | 645 | (string? v) 646 | (Long/parseLong v) 647 | 648 | :else 649 | v) 650 | 651 | "db.type/string" 652 | (str v) 653 | 654 | "db.type/long" 655 | (if (string? v) 656 | (Long/parseLong v) 657 | v) 658 | 659 | "db.type/float" 660 | (if (string? v) 661 | (Float/parseFloat v) 662 | v) 663 | 664 | "db.type/uri" 665 | (if (string? v) 666 | (java.net.URI. v) 667 | v) 668 | 669 | "db.type/uuid" 670 | (if (string? v) 671 | (UUID/fromString v) 672 | v) 673 | 674 | v))) 675 | 676 | (defmethod value-literal :absolute-datetime [[_ inst unit]] 677 | (metabase.util.date/date-trunc-or-extract unit inst (timezone-id))) 678 | 679 | (defmethod value-literal :relative-datetime [[_ offset unit]] 680 | (if (= :current offset) 681 | (java.util.Date.) 682 | (metabase.util.date/relative-date unit offset))) 683 | 684 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 685 | 686 | (defmulti filter-clauses 687 | "Convert an MBQL :filter form into Datalog :where clauses 688 | 689 | [:= [:field-id 45] [:value 20]] 690 | ;;=> [[?user :user/age 20]]" 691 | (fn [[clause-type _]] 692 | clause-type)) 693 | 694 | (defmethod filter-clauses := [[_ field-ref [_ _ {base-type :base_type 695 | :as field-inst} 696 | :as vclause]]] 697 | (constant-binding field-ref (value-literal vclause))) 698 | 699 | (defmethod filter-clauses :and [[_ & clauses]] 700 | (into [] (mapcat filter-clauses) clauses)) 701 | 702 | (defn logic-vars 703 | "Recursively find all logic var symbols starting with a question mark." 704 | [clause] 705 | (cond 706 | (coll? clause) 707 | (into #{} (mapcat logic-vars) clause) 708 | 709 | (and (symbol? clause) 710 | (= \? (first (name clause)))) 711 | [clause] 712 | 713 | :else 714 | [])) 715 | 716 | (defn or-join 717 | "Takes a sequence of sequence of datalog :where clauses, each inner sequence is 718 | considered a single logical group, where clauses within one group are 719 | considered logical conjunctions (and), and generates an or-join, unifying 720 | those lvars that are present in all clauses. 721 | 722 | (or-join '[[[?venue :venue/location ?loc] 723 | [?loc :location/city \"New York\"]] 724 | [[?venue :venue/size 1000]]]) 725 | ;;=> 726 | [(or-join [?venue] 727 | (and [?venue :venue/location ?loc] 728 | [?loc :location/city \"New York\"]) 729 | [?venue :venue/size 1000])] " 730 | [clauses] 731 | (let [lvars (apply set/intersection (map logic-vars clauses))] 732 | (assert (pos-int? (count lvars)) 733 | (str "No logic variables found to unify across [:or] in " (pr-str `[:or ~@clauses]))) 734 | 735 | ;; Only bind any logic vars shared by all clauses in the outer clause. This 736 | ;; will prevent Datomic from complaining, but could potentially be too 737 | ;; naive. Since typically all clauses filter the same entity though this 738 | ;; should generally be good enough. 739 | [`(~'or-join [~@lvars] 740 | ~@(map (fn [c] 741 | (if (= (count c) 1) 742 | (first c) 743 | (cons 'and c))) 744 | clauses))])) 745 | 746 | (defmethod filter-clauses :or [[_ & clauses]] 747 | (or-join (map filter-clauses clauses))) 748 | 749 | (defmethod filter-clauses :< [[_ field value]] 750 | (conj (field-bindings field) 751 | [`(util/lt ~(field-lvar field) ~(value-literal value))])) 752 | 753 | (defmethod filter-clauses :> [[_ field value]] 754 | (conj (field-bindings field) 755 | [`(util/gt ~(field-lvar field) ~(value-literal value))])) 756 | 757 | (defmethod filter-clauses :<= [[_ field value]] 758 | (conj (field-bindings field) 759 | [`(util/lte ~(field-lvar field) ~(value-literal value))])) 760 | 761 | (defmethod filter-clauses :>= [[_ field value]] 762 | (conj (field-bindings field) 763 | [`(util/gte ~(field-lvar field) ~(value-literal value))])) 764 | 765 | (defmethod filter-clauses :!= [[_ field value]] 766 | (conj (field-bindings field) 767 | [`(not= ~(field-lvar field) ~(value-literal value))])) 768 | 769 | (defmethod filter-clauses :between [[_ field min-val max-val]] 770 | (into (field-bindings field) 771 | [[`(util/lte ~(value-literal min-val) ~(field-lvar field))] 772 | [`(util/lte ~(field-lvar field) ~(value-literal max-val))]])) 773 | 774 | (defmethod filter-clauses :starts-with [[_ field value opts]] 775 | (conj (field-bindings field) 776 | [`(util/str-starts-with? ~(field-lvar field) 777 | ~(value-literal value) 778 | ~(merge {:case-sensitive true} opts))])) 779 | 780 | (defmethod filter-clauses :ends-with [[_ field value opts]] 781 | (conj (field-bindings field) 782 | [`(util/str-ends-with? ~(field-lvar field) 783 | ~(value-literal value) 784 | ~(merge {:case-sensitive true} opts))])) 785 | 786 | (defmethod filter-clauses :contains [[_ field value opts]] 787 | (conj (field-bindings field) 788 | [`(util/str-contains? ~(field-lvar field) 789 | ~(value-literal value) 790 | ~(merge {:case-sensitive true} opts))])) 791 | 792 | (defmethod filter-clauses :not [[_ [_ field :as pred]]] 793 | (let [negate (fn [[e a :as pred]] 794 | (if (= 'not e) 795 | a 796 | (list 'not pred))) 797 | pred-clauses (filter-clauses pred) 798 | {bindings true 799 | predicates false} (group-by (fn [[e a v :as clause]] 800 | (or (and (simple-symbol? e) 801 | (qualified-keyword? a) 802 | (simple-symbol? v)) 803 | (and (list? e) 804 | (= 'get-else (first e))))) 805 | pred-clauses)] 806 | (if (= 1 (count predicates)) 807 | (conj bindings (negate (first predicates))) 808 | (conj bindings (cons 'not predicates))))) 809 | 810 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 811 | ;; MBQL top level constructs 812 | ;; 813 | ;; Each of these functions handles a single top-level MBQL construct 814 | ;; like :source-table, :fields, or :order-by, converting it incrementally into 815 | ;; the corresponding Datalog. Most of the heavy lifting is done by the Datalog 816 | ;; helper functions above. 817 | 818 | (declare read-query) 819 | (declare mbqry->dqry) 820 | 821 | (defn apply-source-query 822 | "Nested query support. We don't actually 'nest' queries as Datalog doesn't have 823 | that, instead we merge queries, but keeping only the :find and :select parts 824 | of the outer query." 825 | [dqry {:keys [source-query fields breakout] :as mbqry}] 826 | (if source-query 827 | (cond-> (if-let [native (:native source-query)] 828 | (read-query native) 829 | (mbqry->dqry source-query)) 830 | (or (seq fields) (seq breakout)) 831 | (dissoc :find :select)) 832 | dqry)) 833 | 834 | (defn source-table-clause [dqry {:keys [source-table breakout] :as mbqry}] 835 | (let [table (qp.store/table source-table) 836 | eid (table-lvar table)] 837 | (if-let [custom-clause (get-in (user-config) [:inclusion-clauses (keyword (:name table))])] 838 | (walk/postwalk-replace {'?eid eid} custom-clause) 839 | (let [fields (db/select Field :table_id source-table) 840 | attribs (->> fields 841 | (remove (comp #{"metabase.driver.datomic/path" "metabase.driver.datomic/computed-field"} :database_type)) 842 | (map ->attrib) 843 | (remove (comp reserved-prefixes namespace)))] 844 | [`(~'or ~@(map #(vector eid %) attribs))])))) 845 | 846 | (defn apply-source-table 847 | "Convert an MBQL :source-table clause into the corresponding Datalog. This 848 | generates a clause of the form 849 | 850 | (or [?user :user/first-name] 851 | [?user :user/last-name] 852 | [?user :user/name]) 853 | 854 | In other words this binds the [[table-lvar]] to any entity that has any 855 | attributes corresponding with the given 'columns' of the given 'table'." 856 | [dqry {:keys [source-table] :as mbqry}] 857 | (if source-table 858 | (into-clause-uniq dqry :where (source-table-clause dqry mbqry)) 859 | dqry)) 860 | 861 | ;; Entries in the :fields clause can be 862 | ;; 863 | ;; | Concrete field refrences | [:field-id 15] | 864 | ;; | Expression references | [:expression :sales_tax] | 865 | ;; | Aggregates | [:aggregate 0] | 866 | ;; | Foreign keys | [:fk-> 10 20] | 867 | (defn apply-fields [dqry {:keys [source-table join-tables fields order-by] :as mbqry}] 868 | (if (seq fields) 869 | (-> dqry 870 | (into-clause-uniq :find (map field-lvar) fields) 871 | (into-clause-uniq :where (mapcat field-bindings) fields) 872 | (into-clause :select (map (partial field-lookup mbqry)) fields)) 873 | dqry)) 874 | 875 | ;; breakouts with aggregation = GROUP BY 876 | ;; breakouts without aggregation = SELECT DISTINCT 877 | (defn apply-breakouts [dqry {:keys [breakout order-by aggregation] :as mbqry}] 878 | (if (seq breakout) 879 | (-> dqry 880 | (into-clause-uniq :find (map field-lvar) breakout) 881 | (into-clause-uniq :where (mapcat field-bindings) breakout) 882 | (into-clause :select (map (partial field-lookup mbqry)) breakout)) 883 | dqry)) 884 | 885 | (defn apply-aggregation [mbqry dqry aggregation] 886 | (let [clause (aggregation-clause mbqry aggregation) 887 | [aggr-type field-ref] aggregation] 888 | (-> dqry 889 | (into-clause-uniq :find [clause]) 890 | (into-clause :select [clause]) 891 | (cond-> #_dqry 892 | (#{:avg :sum :stddev} aggr-type) 893 | (into-clause-uniq :with [(table-lvar field-ref)]) 894 | field-ref 895 | (into-clause-uniq :where 896 | (with-strict-bindings 897 | (field-bindings field-ref))))))) 898 | 899 | (defn apply-aggregations [dqry {:keys [aggregation] :as mbqry}] 900 | (reduce (partial apply-aggregation mbqry) dqry aggregation)) 901 | 902 | (defn apply-order-by [dqry {:keys [order-by aggregation] :as mbqry}] 903 | (if (seq order-by) 904 | (-> dqry 905 | (into-clause-uniq :find 906 | (map (fn [[_ field-ref]] 907 | (if (= :aggregation (first field-ref)) 908 | (aggregation-clause mbqry (nth aggregation (second field-ref))) 909 | (field-lvar field-ref)))) 910 | order-by) 911 | (into-clause-uniq :where (mapcat (comp field-bindings second)) order-by) 912 | (into-clause :order-by 913 | (map (fn [[dir field-ref]] 914 | [dir (field-lookup mbqry field-ref)])) 915 | order-by)) 916 | dqry)) 917 | 918 | (defn apply-filters [dqry {:keys [filter]}] 919 | (if (seq filter) 920 | (into-clause-uniq dqry :where (filter-clauses filter)) 921 | dqry)) 922 | 923 | (defn clean-up-with-clause 924 | "If a logic variable appears in both an aggregate in the :find clause, and in 925 | the :with clause, then this will seriously confuse the Datomic query engine. 926 | In this scenario having the logic variable in `:with' is superfluous, having 927 | it in an aggregate achieves the same thing. 928 | 929 | This scenario occurs when e.g. having both a [:count] and [:sum $field] 930 | aggregate. The count doesn't have a field, so it counts the entities, the sum 931 | adds a with clause for the entity id to prevent merging of duplicates, 932 | resulting in Datomic trying to unify the variable with itself. " 933 | [{:keys [find] :as dqry}] 934 | (update dqry 935 | :with 936 | (pal remove 937 | (set (concat 938 | find 939 | (keep (fn [clause] 940 | (and (list? clause) 941 | (second clause))) 942 | find)))))) 943 | 944 | (defn mbqry->dqry [mbqry] 945 | (-> '{:in [$ %]} 946 | (apply-source-query mbqry) 947 | (apply-source-table mbqry) 948 | (apply-fields mbqry) 949 | (apply-filters mbqry) 950 | (apply-order-by mbqry) 951 | (apply-breakouts mbqry) 952 | (apply-aggregations mbqry) 953 | (clean-up-with-clause))) 954 | 955 | (defn mbql->native [{database :database 956 | mbqry :query 957 | settings :settings}] 958 | (binding [*settings* settings 959 | *db* (db) 960 | *mbqry* mbqry] 961 | {:query (-> mbqry 962 | mbqry->dqry 963 | clojure.pprint/pprint 964 | with-out-str)})) 965 | 966 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 967 | ;; EXECUTE-QUERY 968 | 969 | (defn index-of [xs x] 970 | (loop [idx 0 971 | [y & xs] xs] 972 | (if (= x y) 973 | idx 974 | (if (seq xs) 975 | (recur (inc idx) xs))))) 976 | 977 | ;; Field selection 978 | ;; 979 | ;; This is the process where we take the :select clause, and compare it with 980 | ;; the :find clause, to see how to convert each row of data into the stuff 981 | ;; that's being asked for. 982 | ;; 983 | ;; Example 1: 984 | ;; {:find [?eid] :select [(:artist/name ?eid)]} 985 | ;; 986 | ;; Not too hard, look up the (d/entity ?eid) and get :artist/name attribute 987 | ;; 988 | ;; Example 2: 989 | ;; {:find [?artist|artist|name] :select [(:artist/name ?artist)]} 990 | ;; 991 | ;; The attribute is already fetched directly, so just use it. 992 | 993 | (defmulti select-field-form (fn [dqry entity-fn form] 994 | (first form))) 995 | 996 | (defn ref? [entity-fn attr] 997 | (let [attr-entity (entity-fn attr)] 998 | (when (= :db.type/ref (:db/valueType attr-entity)) 999 | attr-entity))) 1000 | 1001 | (defn entity-map? 1002 | "Is the object an EntityMap, i.e. the result of calling datomic.api/entity." 1003 | [x] 1004 | (instance? datomic.query.EntityMap x)) 1005 | 1006 | (defn unwrap-entity [val] 1007 | (if (entity-map? val) 1008 | (:db/id val) 1009 | val)) 1010 | 1011 | (defmethod select-field-form :default [{:keys [find]} entity-fn [attr ?eid]] 1012 | (assert (qualified-keyword? attr)) 1013 | (assert (symbol? ?eid)) 1014 | (if-let [idx (index-of find ?eid)] 1015 | (fn [row] 1016 | (let [entity (-> row (nth idx) entity-fn)] 1017 | (unwrap-entity (get entity attr)))) 1018 | (let [attr-sym (lvar ?eid (namespace attr) (name attr)) 1019 | idx (index-of find attr-sym)] 1020 | (assert idx) 1021 | (fn [row] 1022 | (let [value (nth row idx)] 1023 | ;; Try to convert enum-style ident references back to keywords 1024 | (if-let [attr-entity (and (integer? value) (ref? entity-fn attr))] 1025 | (or (d/ident (d/entity-db attr-entity) value) 1026 | value) 1027 | value)))))) 1028 | 1029 | (declare select-field) 1030 | 1031 | (defmethod select-field-form `datetime [dqry entity-fn [_ field-lvar unit]] 1032 | (if-let [row->field (select-field dqry entity-fn (lvar field-lvar unit))] 1033 | row->field 1034 | (let [row->field (select-field dqry entity-fn field-lvar)] 1035 | (fn [row] 1036 | (metabase.util.date/date-trunc-or-extract 1037 | unit 1038 | (row->field row) 1039 | (timezone-id)))))) 1040 | 1041 | (defmethod select-field-form `field [dqry entity-fn [_ field {:keys [database_type]}]] 1042 | (let [row->field (select-field dqry entity-fn field)] 1043 | (fn [row] 1044 | (let [value (row->field row)] 1045 | (cond 1046 | (and (= "db.type/ref" database_type) (integer? value)) 1047 | (:db/ident (entity-fn value) value) 1048 | 1049 | (and (= "metabase.driver.datomic/path" database_type) (integer? value)) 1050 | (:db/ident (entity-fn value) value) 1051 | 1052 | :else 1053 | value))))) 1054 | 1055 | (defn select-field 1056 | "Returns a function which, given a row of data fetched from datomic, will 1057 | extrect a single field. It will first check if the requested field was fetched 1058 | directly in the `:find` clause, if not it will resolve the entity and get the 1059 | attribute from there." 1060 | [{:keys [find] :as dqry} entity-fn field] 1061 | (if-let [idx (index-of find field)] 1062 | (par nth idx) 1063 | (if (list? field) 1064 | (select-field-form dqry entity-fn field)))) 1065 | 1066 | (defn select-fields [dqry entity-fn fields] 1067 | (apply juxt (map (pal select-field dqry entity-fn) fields))) 1068 | 1069 | (defn order-clause->comparator [dqry entity-fn order-by] 1070 | (fn [x y] 1071 | (reduce (fn [result [dir field]] 1072 | (if (= 0 result) 1073 | (* 1074 | (if (= :desc dir) -1 1) 1075 | (let [x ((select-field dqry entity-fn field) x) 1076 | y ((select-field dqry entity-fn field) y)] 1077 | (cond 1078 | (= x y) 0 1079 | (util/lt x y) -1 1080 | (util/gt x y) 1 1081 | :else (compare (str (class x)) 1082 | (str (class y)))))) 1083 | (reduced result))) 1084 | 0 1085 | order-by))) 1086 | 1087 | (defn order-by-attribs [dqry entity-fn order-by results] 1088 | (if (seq order-by) 1089 | (sort (order-clause->comparator dqry entity-fn order-by) results) 1090 | results)) 1091 | 1092 | (defn cartesian-product 1093 | "Expand any set results (references with cardinality/many) to their cartesian 1094 | products. Empty sets produce a single row with a nil value (similar to an 1095 | outer join). 1096 | 1097 | (cartesian-product [1 2 #{:a :b}]) 1098 | ;;=> ([1 2 :b] [1 2 :a]) 1099 | 1100 | (cartesian-product [1 #{:x :y} #{:a :b}]) 1101 | ;;=> ([1 :y :b] [1 :y :a] [1 :x :b] [1 :x :a]) 1102 | 1103 | (cartesian-product [1 2 #{}]) 1104 | ;;=> ([1 2 nil])" 1105 | [row] 1106 | (reduce (fn [res value] 1107 | (if (set? value) 1108 | (if (seq value) 1109 | (for [r res 1110 | v value] 1111 | (conj r v)) 1112 | (map (par conj nil) res)) 1113 | (map (par conj value) res))) 1114 | [[]] 1115 | row)) 1116 | 1117 | (defn entity->db-id [val] 1118 | (if (instance? datomic.Entity val) 1119 | (if-let [ident (:db/ident val)] 1120 | (str ident) 1121 | (:db/id val)) 1122 | val)) 1123 | 1124 | (defn resolve-fields [db result {:keys [select order-by] :as dqry}] 1125 | (let [nil-placeholder? (-> NIL_VALUES vals set 1126 | ;; false is used as a stand-in for nil in boolean 1127 | ;; fields, but we don't want to replace false with 1128 | ;; nil in results. 1129 | (disj false)) 1130 | entity-fn (memoize (fn [eid] (d/entity db eid)))] 1131 | (->> result 1132 | ;; TODO: This needs to be retought, we can only really order after 1133 | ;; expanding set references (cartesian-product). Currently breaks when 1134 | ;; sorting on cardinality/many fields. 1135 | (order-by-attribs dqry entity-fn order-by) 1136 | (map (select-fields dqry entity-fn select)) 1137 | (mapcat cartesian-product) 1138 | (map (pal map entity->db-id)) 1139 | (map (pal map #(if (nil-placeholder? %) 1140 | nil 1141 | %)))))) 1142 | 1143 | (defmulti col-name mbql.u/dispatch-by-clause-name-or-class) 1144 | 1145 | (defmethod col-name :field-id [[_ id]] 1146 | (:name (qp.store/field id))) 1147 | 1148 | (defmethod col-name :field-literal [[_ field]] 1149 | field) 1150 | 1151 | (defmethod col-name :datetime-field [[_ ref unit]] 1152 | (str (col-name ref) ":" (name unit))) 1153 | 1154 | (defmethod col-name :fk-> [[_ src dest]] 1155 | (col-name dest)) 1156 | 1157 | (def aggr-col-name nil) 1158 | (defmulti aggr-col-name first) 1159 | 1160 | (defmethod aggr-col-name :default [[aggr-type]] 1161 | (name aggr-type)) 1162 | 1163 | (defmethod aggr-col-name :distinct [_] 1164 | "count") 1165 | 1166 | (defn lvar->col [lvar] 1167 | (->> (str/split (subs (str lvar) 1) #"\|") 1168 | reverse 1169 | (take 2) 1170 | reverse 1171 | (str/join "_"))) 1172 | 1173 | (defn lookup->col [form] 1174 | (cond 1175 | (list? form) 1176 | (str (first form)) 1177 | 1178 | (symbol? form) 1179 | (lvar->col form) 1180 | 1181 | :else 1182 | (str form))) 1183 | 1184 | (defn result-columns [dqry {:keys [source-query source-table fields limit breakout aggregation]}] 1185 | (let [cols (concat (map col-name fields) 1186 | (map col-name breakout) 1187 | (map aggr-col-name aggregation))] 1188 | (cond 1189 | (seq cols) 1190 | cols 1191 | 1192 | source-query 1193 | (recur dqry source-query) 1194 | 1195 | (:select dqry) 1196 | (map lookup->col (:select dqry)) 1197 | 1198 | (:find dqry) 1199 | (map lvar->col (:select dqry))))) 1200 | 1201 | (defn result-map-mbql 1202 | "Result map for a query originating from Metabase directly. We have access to 1203 | the original MBQL query." 1204 | [db results dqry mbqry] 1205 | {:columns (result-columns dqry mbqry) 1206 | :rows (resolve-fields db results dqry)}) 1207 | 1208 | (defn result-map-native 1209 | "Result map for a 'native' query entered directly by the user." 1210 | [db results dqry] 1211 | {:columns (map str (:find dqry)) 1212 | :rows (seq results)}) 1213 | 1214 | (defn read-query [q] 1215 | #_(binding [*data-readers* (assoc *data-readers* 'metabase-datomic/nil (fn [_] NIL))]) 1216 | (let [qry (read-string q)] 1217 | (if (vector? qry) 1218 | (loop [key (first qry) 1219 | val (take-while (complement keyword?) (next qry)) 1220 | qry (drop (inc (count val)) qry) 1221 | res {key val}] 1222 | (if (seq qry) 1223 | (let [key (first qry) 1224 | val (take-while (complement keyword?) (next qry)) 1225 | qry (drop (inc (count val)) qry) 1226 | res (assoc res key val)] 1227 | (recur key val qry res)) 1228 | res)) 1229 | qry))) 1230 | 1231 | 1232 | (defn execute-query [{:keys [native query] :as native-query}] 1233 | (let [db (db) 1234 | dqry (read-query (:query native)) 1235 | results (d/q (dissoc dqry :fields) db (:rules (user-config))) 1236 | ;; Hacking around this is as it's so common in Metabase's automatic 1237 | ;; dashboards. Datomic never returns a count of zero, instead it just 1238 | ;; returns an empty result. 1239 | results (if (and (empty? results) 1240 | (empty? (:breakout query)) 1241 | (#{[[:count]] [[:sum]]} (:aggregation query))) 1242 | [[0]] 1243 | results)] 1244 | (if query 1245 | (result-map-mbql db results dqry query) 1246 | (result-map-native db results dqry)))) 1247 | -------------------------------------------------------------------------------- /src/metabase/driver/datomic/util.clj: -------------------------------------------------------------------------------- 1 | (ns metabase.driver.datomic.util 2 | (:import java.util.Date) 3 | (:require [clojure.string :as str])) 4 | 5 | (defn kw->str [s] 6 | (str (namespace s) "/" (name s))) 7 | 8 | (def pal 9 | "Partial-left (same as clojure.core/partial)" 10 | partial) 11 | 12 | (defn par 13 | "Partial-right, partially apply rightmost function arguments." 14 | [f & xs] 15 | (fn [& ys] 16 | (apply f (concat ys xs)))) 17 | 18 | ;; Polymorphic and type forgiving comparisons, for use in generated queries. 19 | (defprotocol Comparisons 20 | (lt [x y]) 21 | (gt [x y]) 22 | (lte [x y]) 23 | (gte [x y])) 24 | 25 | (extend-protocol Comparisons 26 | java.util.UUID 27 | (lt [x y] 28 | (and (instance? java.util.UUID y) (< (.compareTo x y) 0))) 29 | (gt [x y] 30 | (and (instance? java.util.UUID y) (> (.compareTo x y) 0))) 31 | (lte [x y] 32 | (and (instance? java.util.UUID y) (<= (.compareTo x y) 0))) 33 | (gte [x y] 34 | (and (instance? java.util.UUID y) (>= (.compareTo x y) 0))) 35 | 36 | java.lang.Number 37 | (lt [x y] 38 | (and (number? y) (< x y))) 39 | (gt [x y] 40 | (and (number? y) (> x y))) 41 | (lte [x y] 42 | (and (number? y) (<= x y))) 43 | (gte [x y] 44 | (and (number? y) (>= x y))) 45 | 46 | java.util.Date 47 | (lt [x y] 48 | (and (inst? y) (< (inst-ms x) (inst-ms y)))) 49 | (gt [x y] 50 | (and (inst? y) (> (inst-ms x) (inst-ms y)))) 51 | (lte [x y] 52 | (and (inst? y) (<= (inst-ms x) (inst-ms y)))) 53 | (gte [x y] 54 | (and (inst? y) (>= (inst-ms x) (inst-ms y)))) 55 | 56 | clojure.core.Inst 57 | (lt [x y] 58 | (and (inst? y) (< (inst-ms x) (inst-ms y)))) 59 | (gt [x y] 60 | (and (inst? y) (> (inst-ms x) (inst-ms y)))) 61 | (lte [x y] 62 | (and (inst? y) (<= (inst-ms x) (inst-ms y)))) 63 | (gte [x y] 64 | (and (inst? y) (>= (inst-ms x) (inst-ms y)))) 65 | 66 | java.lang.String 67 | (lt [x y] 68 | (and (string? y) (< (.compareTo x y) 0))) 69 | (gt [x y] 70 | (and (string? y) (> (.compareTo x y) 0))) 71 | (lte [x y] 72 | (and (string? y) (<= (.compareTo x y) 0))) 73 | (gte [x y] 74 | (and (string? y) (>= (.compareTo x y) 0))) 75 | 76 | clojure.lang.Keyword 77 | (lt [x y] 78 | (and (keyword? y) (< (.compareTo (name x) (name y)) 0))) 79 | (gt [x y] 80 | (and (keyword? y) (> (.compareTo (name x) (name y)) 0))) 81 | (lte [x y] 82 | (and (keyword? y) (<= (.compareTo (name x) (name y)) 0))) 83 | (gte [x y] 84 | (and (keyword? y) (>= (.compareTo (name x) (name y)) 0)))) 85 | 86 | (defn str-starts-with? [s prefix {case? :case-sensitive}] 87 | (if case? 88 | (str/starts-with? (str s) 89 | (str prefix)) 90 | (str/starts-with? (str/lower-case (str s)) 91 | (str/lower-case (str prefix))))) 92 | 93 | (defn str-ends-with? [s prefix {case? :case-sensitive}] 94 | (if case? 95 | (str/ends-with? (str s) 96 | (str prefix)) 97 | (str/ends-with? (str/lower-case (str s)) 98 | (str/lower-case (str prefix))))) 99 | 100 | (defn str-contains? [s prefix {case? :case-sensitive}] 101 | (if case? 102 | (str/includes? (str s) 103 | (str prefix)) 104 | (str/includes? (str/lower-case (str s)) 105 | (str/lower-case (str prefix))))) 106 | -------------------------------------------------------------------------------- /test/metabase/driver/datomic/aggregation_test.clj: -------------------------------------------------------------------------------- 1 | (ns metabase.driver.datomic.aggregation-test 2 | (:require [clojure.test :refer :all] 3 | [metabase.driver.datomic.test :refer :all] 4 | [metabase.driver.datomic.test-data :as test-data] 5 | [metabase.models.field :refer [Field]] 6 | [metabase.query-processor-test :refer [aggregate-col]] 7 | [metabase.test.data :as data] 8 | [metabase.test.util :as tu])) 9 | 10 | (deftest count-test 11 | (is (match? {:data {:columns ["f1" "count"], 12 | :rows [["xxx" 2] ["yyy" 1]]}} 13 | (with-datomic 14 | (data/with-temp-db [_ test-data/aggr-data] 15 | (data/run-mbql-query foo 16 | {:aggregation [[:count]] 17 | :breakout [$f1]}))))) 18 | 19 | (is (match? {:data {:columns ["count"] 20 | :rows [[100]]}} 21 | (with-datomic 22 | (data/run-mbql-query venues 23 | {:aggregation [[:count]]}))))) 24 | 25 | (deftest sum-test 26 | (is (match? {:data {:columns ["sum"] 27 | :rows [[203]]}} 28 | (with-datomic 29 | (data/run-mbql-query venues 30 | {:aggregation [[:sum $price]]}))))) 31 | 32 | (deftest avg-test 33 | (is (match? {:data {:columns ["avg"] 34 | :rows [[#(= 355058 (long (* 10000 %)))]]}} 35 | (with-datomic 36 | (data/run-mbql-query venues 37 | {:aggregation [[:avg $latitude]]}))))) 38 | 39 | (deftest distinct-test 40 | (is (match? {:data {:columns ["count"] 41 | :rows [[15]]}} 42 | (with-datomic 43 | (data/run-mbql-query checkins 44 | {:aggregation [[:distinct $user_id]]}))))) 45 | 46 | (deftest no-aggregation-test 47 | (is (match? {:data 48 | {:rows 49 | [[pos-int? "Red Medicine" pos-int? 10.0646 -165.374 3] 50 | [pos-int? "Stout Burgers & Beers" pos-int? 34.0996 -118.329 2] 51 | [pos-int? "The Apple Pan" pos-int? 34.0406 -118.428 2] 52 | [pos-int? "Wurstküche" pos-int? 33.9997 -118.465 2] 53 | [pos-int? "Brite Spot Family Restaurant" pos-int? 34.0778 -118.261 2] 54 | [pos-int? "The 101 Coffee Shop" pos-int? 34.1054 -118.324 2] 55 | [pos-int? "Don Day Korean Restaurant" pos-int? 34.0689 -118.305 2] 56 | [pos-int? "25°" pos-int? 34.1015 -118.342 2] 57 | [pos-int? "Krua Siri" pos-int? 34.1018 -118.301 1] 58 | [pos-int? "Fred 62" pos-int? 34.1046 -118.292 2]]}} 59 | 60 | (with-datomic 61 | (data/run-mbql-query venues 62 | {:limit 10 63 | :order-by [[:asc (data/id "venues" "db/id")]]}))))) 64 | 65 | (deftest stddev-test 66 | (is (match? {:data {:columns ["stddev"] 67 | :rows [[#(= 3417 (long (* 1000 %)))]]}} 68 | (with-datomic 69 | (data/run-mbql-query venues 70 | {:aggregation [[:stddev $latitude]]}))))) 71 | 72 | (deftest min-test 73 | (is (match? {:data {:columns ["min"] 74 | :rows [[1]]}} 75 | (with-datomic 76 | (data/run-mbql-query venues 77 | {:aggregation [[:min $price]]})))) 78 | 79 | (is (match? {:data 80 | {:columns ["price" "min"], 81 | :rows [[1 34.0071] [2 33.7701] [3 10.0646] [4 33.983]]}} 82 | (with-datomic 83 | (data/run-mbql-query venues 84 | {:aggregation [[:min $latitude]] 85 | :breakout [$price]}))))) 86 | 87 | (deftest max-test 88 | (is (match? {:data {:columns ["max"] 89 | :rows [[4]]}} 90 | (with-datomic 91 | (data/run-mbql-query venues 92 | {:aggregation [[:max $price]]})))) 93 | 94 | (is (match? {:data {:columns ["price" "max"] 95 | :rows [[1 37.8078] [2 40.7794] [3 40.7262] [4 40.7677]] }} 96 | (with-datomic 97 | (data/run-mbql-query venues 98 | {:aggregation [[:max $latitude]] 99 | :breakout [$price]}))))) 100 | 101 | (deftest multiple-aggregates-test 102 | (testing "two aggregations" 103 | (is (match? {:data {:columns ["count" "sum"] 104 | :rows [[100 203]]}} 105 | (with-datomic 106 | (data/run-mbql-query venues 107 | {:aggregation [[:count] [:sum $price]]}))))) 108 | 109 | (testing "three aggregations" 110 | (is (match? {:data {:columns ["avg" "count" "sum"] 111 | :rows [[2.03 100 203]]}} 112 | (with-datomic 113 | (data/run-mbql-query venues 114 | {:aggregation [[:avg $price] [:count] [:sum $price]]}))))) 115 | 116 | (testing "second aggregate has correct metadata" 117 | (is (match? {:data 118 | {:cols [(aggregate-col :count) 119 | (assoc (aggregate-col :count) :name "count_2")]}} 120 | (with-datomic 121 | (data/run-mbql-query venues 122 | {:aggregation [[:count] [:count]]})))))) 123 | 124 | (deftest edge-case-tests 125 | ;; Copied from the main metabase test base, these seem to have been added as 126 | ;; regression tests for specific issues. Can't hurt to run them for Datomic as 127 | ;; well. 128 | (testing "Field.settings show up for aggregate fields" 129 | (is (= {:base_type :type/Integer 130 | :special_type :type/Category 131 | :settings {:is_priceless false} 132 | :name "sum" 133 | :display_name "sum" 134 | :source :aggregation} 135 | (with-datomic 136 | (tu/with-temp-vals-in-db 137 | Field (data/id :venues :price) {:settings {:is_priceless false}} 138 | (let [results (data/run-mbql-query venues 139 | {:aggregation [[:sum [:field-id $price]]]})] 140 | (or (-> results :data :cols first) 141 | results))))))) 142 | 143 | (testing "handle queries that have more than one of the same aggregation?" 144 | (is (match? {:data {:rows [[#(< 3550.58 % 3550.59) 203]]}} 145 | (with-datomic 146 | (data/run-mbql-query venues 147 | {:aggregation [[:sum $latitude] [:sum $price]]})))))) 148 | 149 | ;; NOT YET IMPLEMENTED: cumulative count / cumulative sum 150 | (comment 151 | ;;; Simple cumulative sum where breakout field is same as cum_sum field 152 | (qp-expect-with-all-drivers 153 | {:rows [[ 1 1] 154 | [ 2 3] 155 | [ 3 6] 156 | [ 4 10] 157 | [ 5 15] 158 | [ 6 21] 159 | [ 7 28] 160 | [ 8 36] 161 | [ 9 45] 162 | [10 55] 163 | [11 66] 164 | [12 78] 165 | [13 91] 166 | [14 105] 167 | [15 120]] 168 | :columns [(data/format-name "id") 169 | "sum"] 170 | :cols [(breakout-col (users-col :id)) 171 | (aggregate-col :sum (users-col :id))] 172 | :native_form true} 173 | (->> (data/run-mbql-query users 174 | {:aggregation [[:cum-sum $id]] 175 | :breakout [$id]}) 176 | booleanize-native-form 177 | (format-rows-by [int int]))) 178 | 179 | 180 | ;;; Cumulative sum w/ a different breakout field 181 | (qp-expect-with-all-drivers 182 | {:rows [["Broen Olujimi" 14] 183 | ["Conchúr Tihomir" 21] 184 | ["Dwight Gresham" 34] 185 | ["Felipinho Asklepios" 36] 186 | ["Frans Hevel" 46] 187 | ["Kaneonuskatew Eiran" 49] 188 | ["Kfir Caj" 61] 189 | ["Nils Gotam" 70] 190 | ["Plato Yeshua" 71] 191 | ["Quentin Sören" 76] 192 | ["Rüstem Hebel" 91] 193 | ["Shad Ferdynand" 97] 194 | ["Simcha Yan" 101] 195 | ["Spiros Teofil" 112] 196 | ["Szymon Theutrich" 120]] 197 | :columns [(data/format-name "name") 198 | "sum"] 199 | :cols [(breakout-col (users-col :name)) 200 | (aggregate-col :sum (users-col :id))] 201 | :native_form true} 202 | (->> (data/run-mbql-query users 203 | {:aggregation [[:cum-sum $id]] 204 | :breakout [$name]}) 205 | booleanize-native-form 206 | (format-rows-by [str int]) 207 | tu/round-fingerprint-cols)) 208 | 209 | 210 | ;;; Cumulative sum w/ a different breakout field that requires grouping 211 | (qp-expect-with-all-drivers 212 | {:columns [(data/format-name "price") 213 | "sum"] 214 | :cols [(breakout-col (venues-col :price)) 215 | (aggregate-col :sum (venues-col :id))] 216 | :rows [[1 1211] 217 | [2 4066] 218 | [3 4681] 219 | [4 5050]] 220 | :native_form true} 221 | (->> (data/run-mbql-query venues 222 | {:aggregation [[:cum-sum $id]] 223 | :breakout [$price]}) 224 | booleanize-native-form 225 | (format-rows-by [int int]) 226 | tu/round-fingerprint-cols)) 227 | 228 | 229 | ;;; ------------------------------------------------ CUMULATIVE COUNT ------------------------------------------------ 230 | 231 | (defn- cumulative-count-col [col-fn col-name] 232 | (assoc (aggregate-col :count (col-fn col-name)) 233 | :base_type :type/Integer 234 | :special_type :type/Number)) 235 | 236 | ;;; cum_count w/o breakout should be treated the same as count 237 | (qp-expect-with-all-drivers 238 | {:rows [[15]] 239 | :columns ["count"] 240 | :cols [(cumulative-count-col users-col :id)] 241 | :native_form true} 242 | (->> (data/run-mbql-query users 243 | {:aggregation [[:cum-count $id]]}) 244 | booleanize-native-form 245 | (format-rows-by [int]))) 246 | 247 | ;;; Cumulative count w/ a different breakout field 248 | (qp-expect-with-all-drivers 249 | {:rows [["Broen Olujimi" 1] 250 | ["Conchúr Tihomir" 2] 251 | ["Dwight Gresham" 3] 252 | ["Felipinho Asklepios" 4] 253 | ["Frans Hevel" 5] 254 | ["Kaneonuskatew Eiran" 6] 255 | ["Kfir Caj" 7] 256 | ["Nils Gotam" 8] 257 | ["Plato Yeshua" 9] 258 | ["Quentin Sören" 10] 259 | ["Rüstem Hebel" 11] 260 | ["Shad Ferdynand" 12] 261 | ["Simcha Yan" 13] 262 | ["Spiros Teofil" 14] 263 | ["Szymon Theutrich" 15]] 264 | :columns [(data/format-name "name") 265 | "count"] 266 | :cols [(breakout-col (users-col :name)) 267 | (cumulative-count-col users-col :id)] 268 | :native_form true} 269 | (->> (data/run-mbql-query users 270 | {:aggregation [[:cum-count $id]] 271 | :breakout [$name]}) 272 | booleanize-native-form 273 | (format-rows-by [str int]) 274 | tu/round-fingerprint-cols)) 275 | 276 | 277 | ;; Cumulative count w/ a different breakout field that requires grouping 278 | (qp-expect-with-all-drivers 279 | {:columns [(data/format-name "price") 280 | "count"] 281 | :cols [(breakout-col (venues-col :price)) 282 | (cumulative-count-col venues-col :id)] 283 | :rows [[1 22] 284 | [2 81] 285 | [3 94] 286 | [4 100]] 287 | :native_form true} 288 | (->> (data/run-mbql-query venues 289 | {:aggregation [[:cum-count $id]] 290 | :breakout [$price]}) 291 | booleanize-native-form 292 | (format-rows-by [int int]) 293 | tu/round-fingerprint-cols))) 294 | -------------------------------------------------------------------------------- /test/metabase/driver/datomic/breakout_test.clj: -------------------------------------------------------------------------------- 1 | (ns metabase.driver.datomic.breakout-test 2 | (:require [cheshire.core :as json] 3 | [clojure.test :refer :all] 4 | [datomic.api :as d] 5 | [metabase.driver.datomic.query-processor :as datomic.qp] 6 | [metabase.driver.datomic.test :refer :all] 7 | [metabase.driver.datomic.util :refer [pal]] 8 | metabase.models.database 9 | [metabase.models.dimension :refer [Dimension]] 10 | [metabase.models.field :refer [Field]] 11 | [metabase.models.field-values :refer [FieldValues]] 12 | [metabase.test.data :as data] 13 | [metabase.test.data.dataset-definitions :as defs] 14 | [toucan.db :as db])) 15 | 16 | (deftest breakout-single-column-test 17 | (let [result (with-datomic 18 | (data/run-mbql-query checkins 19 | {:aggregation [[:count]] 20 | :breakout [$user_id] 21 | :order-by [[:asc $user_id]]}))] 22 | 23 | (is (match? {:data {:rows [[pos-int? 31] [pos-int? 70] [pos-int? 75] 24 | [pos-int? 77] [pos-int? 69] [pos-int? 70] 25 | [pos-int? 76] [pos-int? 81] [pos-int? 68] 26 | [pos-int? 78] [pos-int? 74] [pos-int? 59] 27 | [pos-int? 76] [pos-int? 62] [pos-int? 34]] 28 | :columns ["user_id" 29 | "count"]}} 30 | result)) 31 | 32 | (is (= (get-in result [:data :rows]) 33 | (sort-by first (get-in result [:data :rows])))))) 34 | 35 | (deftest breakout-aggregation-test 36 | (testing "This should act as a \"distinct values\" query and return ordered results" 37 | (is (match? {:data 38 | {:columns ["price"], 39 | :rows [[1] [2] [3] [4]] 40 | :cols [{:name "price"}]}} 41 | 42 | (with-datomic 43 | (data/run-mbql-query venues 44 | {:breakout [$price] 45 | :limit 10})))))) 46 | 47 | (deftest breakout-multiple-columns-implicit-order 48 | (testing "Fields should be implicitly ordered :ASC for all the fields in `breakout` that are not specified in `order-by`" 49 | (is (match? {:data 50 | {:columns ["user_id" "venue_id" "count"] 51 | :rows 52 | (fn [rows] 53 | (= rows (sort-by (comp vec (pal take 2)) rows)))}} 54 | (with-datomic 55 | (data/run-mbql-query checkins 56 | {:aggregation [[:count]] 57 | :breakout [$user_id $venue_id] 58 | :limit 10})))))) 59 | 60 | (deftest breakout-multiple-columns-explicit-order 61 | (testing "`breakout` should not implicitly order by any fields specified in `order-by`" 62 | (is (match? 63 | {:data 64 | {:columns ["name" "price" "count"] 65 | :rows [["bigmista's barbecue" 2 1] 66 | ["Zeke's Smokehouse" 2 1] 67 | ["Yuca's Taqueria" 1 1] 68 | ["Ye Rustic Inn" 1 1] 69 | ["Yamashiro Hollywood" 3 1] 70 | ["Wurstküche" 2 1] 71 | ["Two Sisters Bar & Books" 2 1] 72 | ["Tu Lan Restaurant" 1 1] 73 | ["Tout Sweet Patisserie" 2 1] 74 | ["Tito's Tacos" 1 1]]}} 75 | (with-datomic 76 | (data/run-mbql-query venues 77 | {:aggregation [[:count]] 78 | :breakout [$name $price] 79 | :order-by [[:desc $name]] 80 | :limit 10})))))) 81 | 82 | (defn test-data-categories [] 83 | (sort 84 | (d/q '{:find [?cat ?id] 85 | :where [[?id :categories/name ?cat]]} 86 | (d/db (d/connect "datomic:mem:test-data"))))) 87 | 88 | (deftest remapped-column 89 | (testing "breakout returns the remapped values for a custom dimension" 90 | (is (match? 91 | {:data 92 | {:rows [[pos-int? 8 "Blues"] 93 | [pos-int? 2 "Rock"] 94 | [pos-int? 2 "Swing"] 95 | [pos-int? 7 "African"] 96 | [pos-int? 2 "American"]] 97 | :columns ["category_id" "count" "Foo"] 98 | :cols [{:name "category_id" :remapped_to "Foo"} 99 | {:name "count"} 100 | {:name "Foo" :remapped_from "category_id"}]}} 101 | (with-datomic 102 | (data/with-data 103 | #(let [categories (test-data-categories) 104 | cat-names (->> categories 105 | (map first) 106 | (concat ["Jazz" "Blues" "Rock" "Swing"]) 107 | (take (count categories))) 108 | cat-ids (map last categories)] 109 | [(db/insert! Dimension 110 | {:field_id (data/id :venues :category_id) 111 | :name "Foo" 112 | :type :internal}) 113 | (db/insert! FieldValues 114 | {:field_id (data/id :venues :category_id) 115 | :values (json/generate-string cat-ids) 116 | :human_readable_values (json/generate-string cat-names)})]) 117 | (data/run-mbql-query venues 118 | {:aggregation [[:count]] 119 | :breakout [$category_id] 120 | :limit 5}))))))) 121 | 122 | (deftest order-by-custom-dimension 123 | (is (= [["Wine Bar" "Thai" "Thai" "Thai" "Thai" "Steakhouse" "Steakhouse" 124 | "Steakhouse" "Steakhouse" "Southern"] 125 | ["American" "American" "American" "American" "American" "American" 126 | "American" "American" "Artisan" "Artisan"]] 127 | (with-datomic 128 | (data/with-data 129 | (fn [] 130 | [(db/insert! Dimension 131 | {:field_id (data/id :venues :category_id) 132 | :name "Foo" 133 | :type :external 134 | :human_readable_field_id (data/id :categories :name)})]) 135 | [(->> (data/run-mbql-query venues 136 | {:order-by [[:desc $category_id]] 137 | :limit 10}) 138 | :data :rows 139 | (map last)) 140 | (->> (data/run-mbql-query venues 141 | {:order-by [[:asc $category_id]] 142 | :limit 10}) 143 | :data :rows 144 | (map last))]))))) 145 | 146 | 147 | (comment 148 | ;; We can convert more of these once we implement binning 149 | 150 | (datasets/expect-with-drivers (non-timeseries-drivers-with-feature :binning) 151 | [[10.0 1] [32.0 4] [34.0 57] [36.0 29] [40.0 9]] 152 | (format-rows-by [(partial u/round-to-decimals 1) int] 153 | (rows (data/run-mbql-query venues 154 | {:aggregation [[:count]] 155 | :breakout [[:binning-strategy $latitude :num-bins 20]]})))) 156 | 157 | (datasets/expect-with-drivers (non-timeseries-drivers-with-feature :binning) 158 | [[0.0 1] [20.0 90] [40.0 9]] 159 | (format-rows-by [(partial u/round-to-decimals 1) int] 160 | (rows (data/run-mbql-query venues 161 | {:aggregation [[:count]] 162 | :breakout [[:binning-strategy $latitude :num-bins 3]]})))) 163 | 164 | (datasets/expect-with-drivers (non-timeseries-drivers-with-feature :binning) 165 | [[10.0 -170.0 1] [32.0 -120.0 4] [34.0 -120.0 57] [36.0 -125.0 29] [40.0 -75.0 9]] 166 | (format-rows-by [(partial u/round-to-decimals 1) (partial u/round-to-decimals 1) int] 167 | (rows (data/run-mbql-query venues 168 | {:aggregation [[:count]] 169 | :breakout [[:binning-strategy $latitude :num-bins 20] 170 | [:binning-strategy $longitude :num-bins 20]]})))) 171 | 172 | ;; Currently defaults to 8 bins when the number of bins isn't 173 | ;; specified 174 | (datasets/expect-with-drivers (non-timeseries-drivers-with-feature :binning) 175 | [[10.0 1] [30.0 90] [40.0 9]] 176 | (format-rows-by [(partial u/round-to-decimals 1) int] 177 | (rows (data/run-mbql-query venues 178 | {:aggregation [[:count]] 179 | :breakout [[:binning-strategy $latitude :default]]})))) 180 | 181 | (datasets/expect-with-drivers (non-timeseries-drivers-with-feature :binning) 182 | [[10.0 1] [30.0 61] [35.0 29] [40.0 9]] 183 | (tu/with-temporary-setting-values [breakout-bin-width 5.0] 184 | (format-rows-by [(partial u/round-to-decimals 1) int] 185 | (rows (data/run-mbql-query venues 186 | {:aggregation [[:count]] 187 | :breakout [[:binning-strategy $latitude :default]]}))))) 188 | 189 | ;; Testing bin-width 190 | (datasets/expect-with-drivers (non-timeseries-drivers-with-feature :binning) 191 | [[10.0 1] [33.0 4] [34.0 57] [37.0 29] [40.0 9]] 192 | (format-rows-by [(partial u/round-to-decimals 1) int] 193 | (rows (data/run-mbql-query venues 194 | {:aggregation [[:count]] 195 | :breakout [[:binning-strategy $latitude :bin-width 1]]})))) 196 | 197 | ;; Testing bin-width using a float 198 | (datasets/expect-with-drivers (non-timeseries-drivers-with-feature :binning) 199 | [[10.0 1] [32.5 61] [37.5 29] [40.0 9]] 200 | (format-rows-by [(partial u/round-to-decimals 1) int] 201 | (rows (data/run-mbql-query venues 202 | {:aggregation [[:count]] 203 | :breakout [[:binning-strategy $latitude :bin-width 2.5]]})))) 204 | 205 | (datasets/expect-with-drivers (non-timeseries-drivers-with-feature :binning) 206 | [[33.0 4] [34.0 57]] 207 | (tu/with-temporary-setting-values [breakout-bin-width 1.0] 208 | (format-rows-by [(partial u/round-to-decimals 1) int] 209 | (rows (data/run-mbql-query venues 210 | {:aggregation [[:count]] 211 | :filter [:and 212 | [:< $latitude 35] 213 | [:> $latitude 20]] 214 | :breakout [[:binning-strategy $latitude :default]]}))))) 215 | 216 | (defn- round-binning-decimals [result] 217 | (let [round-to-decimal #(u/round-to-decimals 4 %)] 218 | (-> result 219 | (update :min_value round-to-decimal) 220 | (update :max_value round-to-decimal) 221 | (update-in [:binning_info :min_value] round-to-decimal) 222 | (update-in [:binning_info :max_value] round-to-decimal)))) 223 | 224 | ;;Validate binning info is returned with the binning-strategy 225 | (datasets/expect-with-drivers (non-timeseries-drivers-with-feature :binning) 226 | (assoc (breakout-col (venues-col :latitude)) 227 | :binning_info {:min_value 10.0, :max_value 50.0, :num_bins 4, :bin_width 10.0, :binning_strategy :bin-width}) 228 | (-> (data/run-mbql-query venues 229 | {:aggregation [[:count]] 230 | :breakout [[:binning-strategy $latitude :default]]}) 231 | tu/round-fingerprint-cols 232 | (get-in [:data :cols]) 233 | first)) 234 | 235 | (datasets/expect-with-drivers (non-timeseries-drivers-with-feature :binning) 236 | (assoc (breakout-col (venues-col :latitude)) 237 | :binning_info {:min_value 7.5, :max_value 45.0, :num_bins 5, :bin_width 7.5, :binning_strategy :num-bins}) 238 | (-> (data/run-mbql-query venues 239 | {:aggregation [[:count]] 240 | :breakout [[:binning-strategy $latitude :num-bins 5]]}) 241 | tu/round-fingerprint-cols 242 | (get-in [:data :cols]) 243 | first)) 244 | 245 | ;;Validate binning info is returned with the binning-strategy 246 | (datasets/expect-with-drivers (non-timeseries-drivers-with-feature :binning) 247 | {:status :failed 248 | :class Exception 249 | :error "Unable to bin Field without a min/max value"} 250 | (tu/with-temp-vals-in-db Field (data/id :venues :latitude) {:fingerprint {:type {:type/Number {:min nil, :max nil}}}} 251 | (-> (tu.log/suppress-output 252 | (data/run-mbql-query venues 253 | {:aggregation [[:count]] 254 | :breakout [[:binning-strategy $latitude :default]]})) 255 | (select-keys [:status :class :error])))) 256 | 257 | (defn- field->result-metadata [field] 258 | (select-keys field [:name :display_name :description :base_type :special_type :unit :fingerprint])) 259 | 260 | (defn- nested-venues-query [card-or-card-id] 261 | {:database metabase.models.database/virtual-id 262 | :type :query 263 | :query {:source-table (str "card__" (u/get-id card-or-card-id)) 264 | :aggregation [:count] 265 | :breakout [[:binning-strategy [:field-literal (data/format-name :latitude) :type/Float] :num-bins 20]]}}) 266 | 267 | ;; Binning should be allowed on nested queries that have result metadata 268 | (datasets/expect-with-drivers (non-timeseries-drivers-with-feature :binning :nested-queries) 269 | [[10.0 1] [32.0 4] [34.0 57] [36.0 29] [40.0 9]] 270 | (tt/with-temp Card [card {:dataset_query {:database (data/id) 271 | :type :query 272 | :query {:source-query {:source-table (data/id :venues)}}} 273 | :result_metadata (mapv field->result-metadata (db/select Field :table_id (data/id :venues)))}] 274 | (->> (nested-venues-query card) 275 | qp/process-query 276 | rows 277 | (format-rows-by [(partial u/round-to-decimals 1) int])))) 278 | 279 | ;; Binning is not supported when there is no fingerprint to determine boundaries 280 | (datasets/expect-with-drivers (non-timeseries-drivers-with-feature :binning :nested-queries) 281 | Exception 282 | (tu.log/suppress-output 283 | (tt/with-temp Card [card {:dataset_query {:database (data/id) 284 | :type :query 285 | :query {:source-query {:source-table (data/id :venues)}}}}] 286 | (-> (nested-venues-query card) 287 | qp/process-query 288 | rows)))) 289 | 290 | ;; if we include a Field in both breakout and fields, does the query still work? (Normalization should be taking care 291 | ;; of this) (#8760) 292 | (expect-with-non-timeseries-dbs 293 | :completed 294 | (-> (qp/process-query 295 | {:database (data/id) 296 | :type :query 297 | :query {:source-table (data/id :venues) 298 | :breakout [[:field-id (data/id :venues :price)]] 299 | :fields [["field_id" (data/id :venues :price)]]}}) 300 | :status)) 301 | ) 302 | -------------------------------------------------------------------------------- /test/metabase/driver/datomic/datomic_test.clj: -------------------------------------------------------------------------------- 1 | (ns metabase.driver.datomic.datomic-test 2 | "Checks for things that are particular to datomic, and regression tests." 3 | (:require [clojure.test :refer :all] 4 | [datomic.api :as d] 5 | [metabase.driver.datomic.fix-types :as fix-types] 6 | [metabase.driver.datomic.test :refer :all] 7 | [metabase.models.database :refer [Database]] 8 | [metabase.models.table :refer [Table]] 9 | [metabase.query-processor :as qp] 10 | [metabase.sync :as sync] 11 | [metabase.test.data :as data] 12 | [toucan.db :as db])) 13 | 14 | ;; Skip these tests if the eeleven sample DB does not exist. 15 | (try 16 | (d/connect "datomic:free://localhost:4334/eeleven") 17 | (catch Exception e 18 | (alter-meta! *ns* assoc :kaocha/skip true))) 19 | 20 | (defn get-or-create-eeleven-db! [] 21 | (if-let [db-inst (db/select-one Database :name "Eeleven")] 22 | db-inst 23 | (let [db (db/insert! Database 24 | :name "Eeleven" 25 | :engine :datomic 26 | :details {:db "datomic:free://localhost:4334/eeleven"})] 27 | (sync/sync-database! db) 28 | (fix-types/undo-invalid-primary-keys!) 29 | (Database (:id db))))) 30 | 31 | (defmacro with-datomic-db [& body] 32 | `(with-datomic 33 | (data/with-db (get-or-create-eeleven-db!) 34 | ~@body))) 35 | 36 | (deftest filter-cardinality-many 37 | (is (= [17592186046126] 38 | (->> (with-datomic-db 39 | (data/run-mbql-query ledger 40 | {:filter [:= [:field-id $journal-entries] 17592186046126] 41 | :fields [$journal-entries]})) 42 | :data 43 | :rows 44 | (map first))))) 45 | 46 | ;; check that idents are returned as such, and can be filtered 47 | (deftest ident-test 48 | (is (match? {:data {:rows [[:currency/CAD] [:currency/HKD] [:currency/SGD]]}} 49 | (with-datomic-db 50 | (data/run-mbql-query journal-entry 51 | {:fields [$currency] 52 | :order-by [[:asc $currency]]})))) 53 | 54 | (is (match? {:data {:rows [[3]]}} 55 | (with-datomic-db 56 | (data/run-mbql-query journal-entry 57 | {:aggregation [[:count]] 58 | :filter [:= $currency "currency/HKD"]}))))) 59 | 60 | ;; Do grouping across nil placeholders 61 | ;; {:find [(count ?foo) ?bar]} 62 | ;; where :bar can be "nil" 63 | (deftest group-across-nil 64 | (is (match? {:data {:columns ["id" "count"] 65 | :rows [["JE-1556-47-8585" 2] 66 | ["JE-2117-58-6345" 3] 67 | ["JE-5555-47-8584" 2]]}} 68 | (with-datomic-db 69 | (data/run-mbql-query journal-entry 70 | {:aggregation [[:count $journal-entry-lines]] 71 | :breakout [$id] 72 | :filter [:= $currency "currency/HKD"] 73 | :order-by [[:asc $id]]}))))) 74 | 75 | (deftest cum-sum-across-fk 76 | (is (match? {:row_count 48 77 | :data {:columns ["date:week" "sum"]}} 78 | (with-datomic-db 79 | (data/run-mbql-query journal-entry 80 | {:breakout [[:datetime-field $date :week]] 81 | :aggregation [[:cum-sum [:fk-> 82 | $journal-entry-lines 83 | [:field-id (data/id "journal-entry-line" "amount")]]]] 84 | :order-by [[:asc [:datetime-field $date :week]]]}))))) 85 | 86 | (deftest filter-on-foreign-key 87 | ;; specifically this checks that filtering on a foreign key works correctly 88 | ;; even if the foreign field is not returned in the result. In other words: it 89 | ;; checks that constant-binding works correctly for foreign fields. 90 | (is (match? {:data {:columns ["db/id" "id" "journal-entries" "name" "tax-entries"], 91 | :rows [[17592186045737 "LGR-5487-92-0122" 17592186045912 "SG-01-GL-01" nil]]}}, 92 | (with-datomic-db 93 | (data/run-mbql-query ledger 94 | {:filter [:= [:fk-> $journal-entries (data/id "journal-entry" "id")] "JE-2879-00-0055"] 95 | :fields [[:field-id (data/id "ledger" "db/id")] $id $journal-entries $name $tax-entries]}))))) 96 | 97 | (deftest ^:kaocha/pending filter-aggregate-test 98 | ;; This is possibly the biggest use case for sub queries, the ability to 99 | ;; filter on an aggregate (like an SQL HAVING clause), but we don't currently 100 | ;; support this. This isn't trivial as we can't do it inside the Datalog 101 | ;; clause, so we need to distinguish this case from a regular filter case, and 102 | ;; add metadata to filter the result set after query execution. 103 | 104 | (with-datomic-db 105 | (qp/process-query 106 | (let [journal-entry-line (data/id "journal-entry-line") 107 | $flow [:field-id (data/id "journal-entry-line" "flow")] 108 | $amount [:field-id (data/id "journal-entry-line" "amount")] 109 | $account [:field-id (data/id "journal-entry-line" "account")] 110 | $account__id [:field-id (data/id "account" "id")] 111 | $account__name [:field-id (data/id "account" "name")]] 112 | {:database (db/select-one-field :db_id Table, :id (data/id "journal-entry-line")) 113 | :type :query 114 | :query { 115 | :filter [:< [:field-literal "sum" :type/Decimal] [:value 1000 nil]], 116 | :source-query 117 | {:source-table journal-entry-line 118 | :filter [:= $flow "flow/debit"], 119 | :breakout [[:fk-> $account $account__name] 120 | [:fk-> $account $account__id]] 121 | :aggregation [[:sum $amount]], 122 | ;;:order-by [[:asc [:fk-> [:field-id 245] [:field-id 46]]] [:asc [:fk-> [:field-id 245] [:field-id 38]]]], 123 | }}})))) 124 | -------------------------------------------------------------------------------- /test/metabase/driver/datomic/fields_test.clj: -------------------------------------------------------------------------------- 1 | (ns metabase.driver.datomic.fields-test 2 | (:require [clojure.test :refer :all] 3 | [metabase.driver.datomic.test :refer :all] 4 | [metabase.driver.datomic.test-data :as test-data] 5 | [metabase.test.data :as data] 6 | [toucan.db :as db] 7 | [matcher-combinators.matchers :as m])) 8 | 9 | (deftest fields-test 10 | (let [result 11 | (with-datomic 12 | (let [id [:field-id (data/id :venues "db/id")]] 13 | (data/run-mbql-query venues 14 | {:fields [$name id] 15 | :limit 10 16 | :order-by [[:asc id]]})))] 17 | 18 | (testing "we get the right fields back" 19 | (is (match? 20 | {:row_count 10 21 | :status :completed 22 | :data 23 | {:rows [["Red Medicine" pos-int?] 24 | ["Stout Burgers & Beers" pos-int?] 25 | ["The Apple Pan" pos-int?] 26 | ["Wurstküche" pos-int?] 27 | ["Brite Spot Family Restaurant" pos-int?] 28 | ["The 101 Coffee Shop" pos-int?] 29 | ["Don Day Korean Restaurant" pos-int?] 30 | ["25°" pos-int?] 31 | ["Krua Siri" pos-int?] 32 | ["Fred 62" pos-int?]] 33 | :columns ["name" "db/id"] 34 | :cols [{:base_type :type/Text 35 | :special_type :type/Name 36 | :name "name" 37 | :display_name "Name" 38 | :source :fields 39 | :visibility_type :normal} 40 | {:base_type :type/PK 41 | :special_type :type/PK 42 | :name "db/id" 43 | :display_name "Db/id" 44 | :source :fields 45 | :visibility_type :normal}] 46 | :native_form {:query string?}}} 47 | result))) 48 | 49 | (testing "returns all ids in order" 50 | (let [ids (map last (get-in result [:data :rows]))] 51 | (is (= (sort ids) ids)))))) 52 | 53 | (deftest basic-query-test 54 | (with-datomic 55 | (is (match? {:columns ["db/id" "name" "code"] 56 | :rows (m/in-any-order 57 | [[pos-int? "Belgium" "BE"] 58 | [pos-int? "Germany" "DE"] 59 | [pos-int? "Finnland" "FI"]])} 60 | (test-data/rows+cols 61 | (data/with-temp-db [_ test-data/countries] 62 | (data/run-mbql-query country))))) 63 | 64 | (is (match? 65 | {:data {:columns ["name" "db/id"], 66 | :rows [["20th Century Cafe" pos-int?] 67 | ["25°" pos-int?] 68 | ["33 Taps" pos-int?] 69 | ["800 Degrees Neapolitan Pizzeria" pos-int?] 70 | ["BCD Tofu House" pos-int?] 71 | ["Baby Blues BBQ" pos-int?] 72 | ["Barney's Beanery" pos-int?] 73 | ["Beachwood BBQ & Brewing" pos-int?] 74 | ["Beyond Sushi" pos-int?] 75 | ["Bludso's BBQ" pos-int?]]}} 76 | (data/run-mbql-query venues 77 | {:fields [$name [:field-id (data/id :venues "db/id")]] 78 | :limit 10 79 | :order-by [[:asc $name]]}))))) 80 | -------------------------------------------------------------------------------- /test/metabase/driver/datomic/filter_test.clj: -------------------------------------------------------------------------------- 1 | (ns metabase.driver.datomic.filter-test 2 | (:require [clojure.test :refer :all] 3 | [metabase.driver.datomic.test :refer :all] 4 | [metabase.test.data :as data] 5 | [metabase.test.util.timezone :as tu.tz] 6 | [metabase.api.dataset :as dataset] 7 | [metabase.driver.datomic.test-data :as test-data])) 8 | 9 | (deftest and-gt-gte 10 | (is (match? {:data {:rows [[pos-int? "Sushi Nakazawa" pos-int? 40.7318 -74.0045 4] 11 | [pos-int? "Sushi Yasuda" pos-int? 40.7514 -73.9736 4] 12 | [pos-int? "Tanoshi Sushi & Sake Bar" pos-int? 40.7677 -73.9533 4]]}} 13 | (with-datomic 14 | (data/run-mbql-query venues 15 | {:filter [:and [:> $latitude 40] [:>= $price 4]] 16 | :order-by [[:asc [:field-id (data/id "venues" "db/id")]]]}))))) 17 | 18 | 19 | (deftest and-lt-gt-ne 20 | (is (match? {:data {:rows 21 | [[pos-int? "Red Medicine" pos-int? 10.0646 -165.374 3] 22 | [pos-int? "Jones Hollywood" pos-int? 34.0908 -118.346 3] 23 | [pos-int? "Boneyard Bistro" pos-int? 34.1477 -118.428 3] 24 | [pos-int? "Marlowe" pos-int? 37.7767 -122.396 3] 25 | [pos-int? "Hotel Biron" pos-int? 37.7735 -122.422 3] 26 | [pos-int? "Empress of China" pos-int? 37.7949 -122.406 3] 27 | [pos-int? "Tam O'Shanter" pos-int? 34.1254 -118.264 3] 28 | [pos-int? "Yamashiro Hollywood" pos-int? 34.1057 -118.342 3] 29 | [pos-int? "Musso & Frank Grill" pos-int? 34.1018 -118.335 3] 30 | [pos-int? "Taylor's Prime Steak House" pos-int? 34.0579 -118.302 3] 31 | [pos-int? "Pacific Dining Car" pos-int? 34.0555 -118.266 3] 32 | [pos-int? "Polo Lounge" pos-int? 34.0815 -118.414 3] 33 | [pos-int? "Blue Ribbon Sushi" pos-int? 40.7262 -74.0026 3]]}} 34 | (with-datomic 35 | (data/run-mbql-query venues 36 | {:filter [:and [:< $price 4] [:> $price 1] [:!= $price 2]] 37 | :order-by [[:asc [:field-id (data/id "venues" "db/id")]]]}))))) 38 | 39 | (deftest filter-with-false-value 40 | ;; Check that we're checking for non-nil values, not just logically true ones. 41 | ;; There's only one place (out of 3) that I don't like 42 | (is (match? {:data {:rows [[1]]}} 43 | (with-datomic 44 | (data/dataset places-cam-likes 45 | (data/run-mbql-query places 46 | {:aggregation [[:count]] 47 | :filter [:= $liked false]})))))) 48 | 49 | (deftest filter-equals-true 50 | (is (match? {:data {:rows [[pos-int? "Tempest" true] 51 | [pos-int? "Bullit" true]]}} 52 | (with-datomic 53 | (data/dataset places-cam-likes 54 | (data/run-mbql-query places 55 | {:filter [:= $liked true] 56 | :order-by [[:asc [:field-id (data/id "places" "db/id")]]]})))))) 57 | 58 | (deftest filter-not-is-false 59 | (is (match? {:data {:rows [[pos-int? "Tempest" true] 60 | [pos-int? "Bullit" true]]}} 61 | (with-datomic 62 | (data/dataset places-cam-likes 63 | (data/run-mbql-query places 64 | {:filter [:!= $liked false] 65 | :order-by [[:asc [:field-id (data/id "places" "db/id")]]]})))))) 66 | 67 | (deftest filter-not-is-true 68 | (is (match? {:data {:rows [[pos-int? "The Dentist" false]]}} 69 | (with-datomic 70 | (data/dataset places-cam-likes 71 | (data/run-mbql-query places 72 | {:filter [:!= $liked true] 73 | :order-by [[:asc [:field-id (data/id "places" "db/id")]]]})))))) 74 | 75 | (deftest between-test 76 | (is (match? {:data {:rows [[pos-int? "Beachwood BBQ & Brewing" pos-int? 33.7701 -118.191 2] 77 | [pos-int? "Bludso's BBQ" pos-int? 33.8894 -118.207 2] 78 | [pos-int? "Dal Rae Restaurant" pos-int? 33.983 -118.096 4] 79 | [pos-int? "Wurstküche" pos-int? 33.9997 -118.465 2]]}} 80 | (with-datomic 81 | (data/run-mbql-query venues 82 | {:filter [:between $latitude 33 34] 83 | :order-by [[:asc $latitude]]}))))) 84 | 85 | 86 | (deftest between-dates 87 | (is (match? {:data {:rows [[29]]}} 88 | (with-datomic 89 | (tu.tz/with-jvm-tz "UTC" 90 | (data/run-mbql-query checkins 91 | {:aggregation [[:count]] 92 | :filter [:between [:datetime-field $date :day] "2015-04-01" "2015-05-01"]})))))) 93 | 94 | (deftest or-lte-eq 95 | (is (match? 96 | {:data {:rows [[pos-int? "Red Medicine" pos-int? 10.0646 -165.374 3] 97 | [pos-int? "The Apple Pan" pos-int? 34.0406 -118.428 2] 98 | [pos-int? "Bludso's BBQ" pos-int? 33.8894 -118.207 2] 99 | [pos-int? "Beachwood BBQ & Brewing" pos-int? 33.7701 -118.191 2]]}} 100 | 101 | (with-datomic 102 | (data/run-mbql-query venues 103 | {:filter [:or [:<= $latitude 33.9] [:= $name "The Apple Pan"]] 104 | :order-by [[:asc [:field-id (data/id "venues" "db/id")]]]}))))) 105 | 106 | (deftest starts-with 107 | (is (match? {:data {:columns ["name"], 108 | :rows []}} 109 | (with-datomic 110 | (data/run-mbql-query venues 111 | {:fields [$name] 112 | :filter [:starts-with $name "CHE"] 113 | :order-by [[:asc [:field-id (data/id "venues" "db/id")]]]})))) 114 | 115 | (is (match? {:data {:columns ["name"], 116 | :rows [["Cheese Steak Shop"] ["Chez Jay"]]}} 117 | (with-datomic 118 | (data/run-mbql-query venues 119 | {:fields [$name] 120 | :filter [:starts-with $name "CHE" {:case-sensitive false}] 121 | :order-by [[:asc [:field-id (data/id "venues" "db/id")]]]}))))) 122 | 123 | (deftest ends-with 124 | (is (match? {:data {:columns ["name" "latitude" "longitude" "price"] 125 | :rows [["Brite Spot Family Restaurant" 34.0778 -118.261 2] 126 | ["Don Day Korean Restaurant" 34.0689 -118.305 2] 127 | ["Ruen Pair Thai Restaurant" 34.1021 -118.306 2] 128 | ["Tu Lan Restaurant" 37.7821 -122.41 1] 129 | ["Dal Rae Restaurant" 33.983 -118.096 4]]}} 130 | (with-datomic 131 | (data/run-mbql-query venues 132 | {:fields [$name $latitude $longitude $price] 133 | :filter [:ends-with $name "Restaurant"] 134 | :order-by [[:asc (data/id "venues" "db/id")]]})))) 135 | 136 | (is (match? {:data {:columns ["name" "latitude" "longitude" "price"] 137 | :rows []}} 138 | (with-datomic 139 | (data/run-mbql-query venues 140 | {:fields [$name $latitude $longitude $price] 141 | :filter [:ends-with $name "RESTAURANT" {:case-sensitive true}] 142 | :order-by [[:asc (data/id "venues" "db/id")]]})))) 143 | 144 | (is (match? {:data {:columns ["name" "latitude" "longitude" "price"] 145 | :rows [["Brite Spot Family Restaurant" 34.0778 -118.261 2] 146 | ["Don Day Korean Restaurant" 34.0689 -118.305 2] 147 | ["Ruen Pair Thai Restaurant" 34.1021 -118.306 2] 148 | ["Tu Lan Restaurant" 37.7821 -122.41 1] 149 | ["Dal Rae Restaurant" 33.983 -118.096 4]]}} 150 | (with-datomic 151 | (data/run-mbql-query venues 152 | {:fields [$name $latitude $longitude $price] 153 | :filter [:ends-with $name "RESTAURANT" {:case-sensitive false}] 154 | :order-by [[:asc (data/id "venues" "db/id")]]}))))) 155 | 156 | (deftest contains-test 157 | (is (match? {:data {:rows []}} 158 | (with-datomic 159 | (data/run-mbql-query venues 160 | {:filter [:contains $name "bbq"] 161 | :order-by [[:asc (data/id "venues" "db/id")]]})))) 162 | 163 | (is (match? {:data {:rows [["Bludso's BBQ" 2] 164 | ["Beachwood BBQ & Brewing" 2] 165 | ["Baby Blues BBQ" 2]]}} 166 | (with-datomic 167 | (data/run-mbql-query venues 168 | {:fields [$name $price] 169 | :filter [:contains $name "BBQ"] 170 | :order-by [[:asc (data/id "venues" "db/id")]]})))) 171 | 172 | (is (match? {:data {:rows [["Bludso's BBQ" 2] 173 | ["Beachwood BBQ & Brewing" 2] 174 | ["Baby Blues BBQ" 2]]}} 175 | (with-datomic 176 | (data/run-mbql-query venues 177 | {:fields [$name $price] 178 | :filter [:contains $name "bbq" {:case-sensitive false}] 179 | :order-by [[:asc (data/id "venues" "db/id")]]}))))) 180 | 181 | (deftest nestend-and-or 182 | (is (match? {:data {:rows [[81]]}} 183 | (with-datomic 184 | (data/run-mbql-query venues 185 | {:aggregation [[:count]] 186 | :filter [:and 187 | [:!= $price 3] 188 | [:or 189 | [:= $price 1] 190 | [:= $price 2]]]}))))) 191 | 192 | (deftest =-!=-multiple-values 193 | (is (match? {:data {:rows [[81]]}} 194 | (with-datomic 195 | (data/run-mbql-query venues 196 | {:aggregation [[:count]] 197 | :filter [:= $price 1 2]})))) 198 | 199 | (is (match? {:data {:rows [[19]]}} 200 | (with-datomic 201 | (data/run-mbql-query venues 202 | {:aggregation [[:count]] 203 | :filter [:!= $price 1 2]}))))) 204 | 205 | (deftest not-filter 206 | (is (match? {:data {:rows [[41]]}} 207 | (with-datomic 208 | (data/run-mbql-query venues 209 | {:aggregation [[:count]] 210 | :filter [:not [:= $price 2]]})))) 211 | 212 | (is (match? {:data {:rows [[59]]}} 213 | (with-datomic 214 | (data/run-mbql-query venues 215 | {:aggregation [[:count]] 216 | :filter [:not [:!= $price 2]]})))) 217 | 218 | (is (match? {:data {:rows [[78]]}} 219 | (with-datomic 220 | (data/run-mbql-query venues 221 | {:aggregation [[:count]] 222 | :filter [:not [:< $price 2]]})))) 223 | 224 | (is (match? {:data {:rows [[81]]}} 225 | (with-datomic 226 | (data/run-mbql-query venues 227 | {:aggregation [[:count]] 228 | :filter [:not [:> $price 2]]})))) 229 | 230 | (is (match? {:data {:rows [[19]]}} 231 | (with-datomic 232 | (data/run-mbql-query venues 233 | {:aggregation [[:count]] 234 | :filter [:not [:<= $price 2]]})))) 235 | 236 | (is (match? {:data {:rows [[22]]}} 237 | (with-datomic 238 | (data/run-mbql-query venues 239 | {:aggregation [[:count]] 240 | :filter [:not [:>= $price 2]]})))) 241 | 242 | (is (match? {:data {:rows [[22]]}} 243 | (with-datomic 244 | (data/run-mbql-query venues 245 | {:aggregation [[:count]] 246 | :filter [:not [:between $price 2 4]]})))) 247 | 248 | (is (match? {:data {:rows [[80]]}} 249 | (with-datomic 250 | (data/run-mbql-query venues 251 | {:aggregation [[:count]] 252 | :filter [:not [:starts-with $name "T"]]})))) 253 | 254 | (is (match? {:data {:rows [[3]]}} 255 | (with-datomic 256 | (data/run-mbql-query venues 257 | {:aggregation [[:count]] 258 | :filter [:not [:not [:contains $name "BBQ"]]]})))) 259 | 260 | (is (match? {:data {:rows [[97]]}} 261 | (with-datomic 262 | (data/run-mbql-query venues 263 | {:aggregation [[:count]] 264 | :filter [:not [:contains $name "BBQ"]]})))) 265 | 266 | (is (match? {:data {:rows [[97]]}} 267 | (with-datomic 268 | (data/run-mbql-query venues 269 | {:aggregation [[:count]] 270 | :filter [:does-not-contain $name "BBQ"]})))) 271 | 272 | (is (match? {:data {:rows [[3]]}} 273 | (with-datomic 274 | (data/run-mbql-query venues 275 | {:aggregation [[:count]] 276 | :filter [:and 277 | [:not [:> $price 2]] 278 | [:contains $name "BBQ"]]})))) 279 | 280 | (is (match? {:data {:rows [[87]]}} 281 | (with-datomic 282 | (data/run-mbql-query venues 283 | {:aggregation [[:count]] 284 | :filter [:not [:ends-with $name "a"]]})))) 285 | 286 | (is (match? {:data {:rows [[84]]}} 287 | (with-datomic 288 | (data/run-mbql-query venues 289 | {:aggregation [[:count]] 290 | :filter [:not [:and 291 | [:contains $name "ar"] 292 | [:< $price 4]]]})))) 293 | 294 | (is (match? {:data {:rows [[4]]}} 295 | (with-datomic 296 | (data/run-mbql-query venues 297 | {:aggregation [[:count]] 298 | :filter [:not [:or 299 | [:contains $name "ar"] 300 | [:< $price 4]]]})))) 301 | 302 | (is (match? {:data {:rows [[24]]}} 303 | (with-datomic 304 | (data/run-mbql-query venues 305 | {:aggregation [[:count]] 306 | :filter [:not [:or 307 | [:contains $name "ar"] 308 | [:and 309 | [:> $price 1] 310 | [:< $price 4]]]]}))))) 311 | 312 | (deftest is-null 313 | (is (match? {:data {:rows [["DE" nil] 314 | ["FI" nil]]}} 315 | (with-datomic 316 | (data/with-temp-db [_ test-data/with-nulls] 317 | (data/run-mbql-query country 318 | {:fields [$code $population] 319 | :filter [:is-null $population] 320 | :order-by [[:asc $code]]}))))) 321 | 322 | (is (match? {:data {:rows [["BE" 11000000]]}} 323 | (with-datomic 324 | (data/with-temp-db [_ test-data/with-nulls] 325 | (data/run-mbql-query country 326 | {:fields [$code $population] 327 | :filter [:not [:is-null $population]]})))))) 328 | 329 | (deftest inside 330 | ;; This is converted to [:and [:between ..] [:between ..]] so we get this one for free 331 | (is (match? {:data {:rows [["Red Medicine" 10.0646 -165.374 3]]}} 332 | (with-datomic 333 | (data/run-mbql-query venues 334 | {:fields [$name $latitude $longitude $price] 335 | :filter [:inside $latitude $longitude 10.0649 -165.379 10.0641 -165.371]})))) 336 | 337 | (is (match? {:data {:rows [[39]]}} 338 | (with-datomic 339 | (data/run-mbql-query venues 340 | {:aggregation [[:count]] 341 | :filter [:not [:inside $latitude $longitude 40 -120 30 -110]]}))))) 342 | 343 | 344 | ;; These are related to historical bugs in Metabase itself, we simply copied 345 | ;; these over to make sure we can handle these cases. 346 | (deftest edge-cases 347 | ;; make sure that filtering with dates truncating to minutes works (#4632) 348 | (is (match? {:data {:rows [[107]]}} 349 | (with-datomic 350 | (data/run-mbql-query checkins 351 | {:aggregation [[:count]] 352 | :filter [:between [:datetime-field $date :minute] "2015-01-01T12:30:00" "2015-05-31"]})))) 353 | 354 | ;; make sure that filtering with dates bucketing by weeks works (#4956) 355 | (is (match? {:data {:rows [[7]]}} 356 | (with-datomic 357 | (tu.tz/with-jvm-tz "UTC" 358 | (data/run-mbql-query checkins 359 | {:aggregation [[:count]] 360 | :filter [:= [:datetime-field $date :week] "2015-06-21T07:00:00.000000000-00:00"]}))))) 361 | 362 | ;; FILTER - `is-null` & `not-null` on datetime columns 363 | (is (match? {:data {:rows [[1000]]}} 364 | (with-datomic 365 | (data/run-mbql-query checkins 366 | {:aggregation [[:count]] 367 | :filter [:not-null $date]})))) 368 | 369 | (is (match? {:data {:rows [[1000]]}} 370 | (with-datomic 371 | (data/run-mbql-query checkins 372 | {:aggregation [[:count]] 373 | :filter ["NOT_NULL" 374 | ["field-id" 375 | ["field-literal" (data/format-name "date") "type/DateTime"]]]})))) 376 | 377 | (is (match? {:data {:rows [[0]]}} 378 | (with-datomic 379 | (data/run-mbql-query checkins 380 | {:aggregation [[:count]] 381 | :filter [:is-null $date]}))))) 382 | -------------------------------------------------------------------------------- /test/metabase/driver/datomic/joins_test.clj: -------------------------------------------------------------------------------- 1 | (ns metabase.driver.datomic.joins-test 2 | (:require [clojure.test :refer :all] 3 | [metabase.driver.datomic.test :refer [with-datomic]] 4 | [metabase.models.database :refer [Database]] 5 | [metabase.test.data :as data] 6 | [toucan.db :as db])) 7 | 8 | (deftest breakout-fk-test 9 | (testing "breakout FK Field is returned in the results" 10 | ;; The top 10 cities by number of Tupac sightings 11 | (is (match? {:data {:columns ["name" "count"] 12 | :rows [["Arlington" 16] 13 | ["Albany" 15] 14 | ["Portland" 14] 15 | ["Louisville" 13] 16 | ["Philadelphia" 13] 17 | ["Anchorage" 12] 18 | ["Lincoln" 12] 19 | ["Houston" 11] 20 | ["Irvine" 11] 21 | ["Lakeland" 11]]}} 22 | (with-datomic 23 | (data/dataset tupac-sightings 24 | (data/run-mbql-query sightings 25 | {:aggregation [[:count]] 26 | :breakout [$city_id->cities.name] 27 | :order-by [[:desc [:aggregation 0]]] 28 | :limit 10}))))))) 29 | 30 | (deftest filter-on-fk-field-test 31 | ;; Number of Tupac sightings in the Expa office 32 | (testing "we can filter on an FK field" 33 | (is (match? {:data {:columns ["count"] 34 | :rows [[60]]}} 35 | (with-datomic 36 | (data/dataset tupac-sightings 37 | (data/run-mbql-query sightings 38 | {:aggregation [[:count]] 39 | :filter [:= $category_id->categories.name "In the Expa Office"]}))))))) 40 | 41 | (deftest fk-field-in-fields-clause-test 42 | ;; THE 10 MOST RECENT TUPAC SIGHTINGS (!) 43 | (testing "we can include the FK field in the fields clause" 44 | (is (match? {:data {:columns ["name"] 45 | :rows [["In the Park"] 46 | ["Working at a Pet Store"] 47 | ["At the Airport"] 48 | ["At a Restaurant"] 49 | ["Working as a Limo Driver"] 50 | ["At Starbucks"] 51 | ["On TV"] 52 | ["At a Restaurant"] 53 | ["Wearing a Biggie Shirt"] 54 | ["In the Expa Office"]]}} 55 | (with-datomic 56 | (data/dataset tupac-sightings 57 | (data/run-mbql-query sightings 58 | {:fields [$category_id->categories.name] 59 | :order-by [[:desc $timestamp]] 60 | :limit 10}))))))) 61 | 62 | (deftest order-by-multiple-foreign-keys-test 63 | ;; 1. Check that we can order by Foreign Keys 64 | ;; (this query targets sightings and orders by cities.name and categories.name) 65 | ;; 2. Check that we can join MULTIPLE tables in a single query 66 | ;; (this query joins both cities and categories) 67 | (is (match? {:data {:columns ["name" "name"], 68 | :rows [["Akron" "Working at a Pet Store"] 69 | ["Akron" "Working as a Limo Driver"] 70 | ["Akron" "Working as a Limo Driver"] 71 | ["Akron" "Wearing a Biggie Shirt"] 72 | ["Akron" "In the Mall"] 73 | ["Akron" "At a Restaurant"] 74 | ["Albany" "Working as a Limo Driver"] 75 | ["Albany" "Working as a Limo Driver"] 76 | ["Albany" "Wearing a Biggie Shirt"] 77 | ["Albany" "Wearing a Biggie Shirt"]]}} 78 | (with-datomic 79 | (data/dataset tupac-sightings 80 | (data/run-mbql-query sightings 81 | {:fields [$city_id->cities.name 82 | $category_id->categories.name] 83 | :order-by [[:asc $city_id->cities.name] 84 | [:desc $category_id->categories.name] 85 | [:asc (data/id "sightings" "db/id")]] 86 | :limit 10})))))) 87 | 88 | (deftest multi-fk-single-table-test 89 | (testing "join against the same table twice via different paths" 90 | (is (match? {:data {:columns ["name" "count"] 91 | :rows [["Bob the Sea Gull" 2] 92 | ["Brenda Blackbird" 2] 93 | ["Lucky Pigeon" 2] 94 | ["Peter Pelican" 5] 95 | ["Ronald Raven" 1]]}} 96 | 97 | (with-datomic 98 | (data/dataset avian-singles 99 | (data/run-mbql-query messages 100 | {:aggregation [[:count]] 101 | :breakout [$sender_id->users.name] 102 | :filter [:= $reciever_id->users.name "Rasta Toucan"]}))))))) 103 | -------------------------------------------------------------------------------- /test/metabase/driver/datomic/order_by_test.clj: -------------------------------------------------------------------------------- 1 | (ns metabase.driver.datomic.order-by-test 2 | (:require [clojure.test :refer :all] 3 | [metabase.driver :as driver] 4 | [metabase.driver.datomic.test :refer :all] 5 | [metabase.driver.datomic.test-data :as test-data] 6 | [metabase.test.data :as data])) 7 | 8 | (def transpose (partial apply map vector)) 9 | 10 | (defn ranks 11 | "Convert a sequence of values into a sequence of 'ranks', a rank in this case 12 | being the position that each value has in the sorted input sequence." 13 | [xs] 14 | (let [idxs (into {} (reverse (map vector (sort xs) (next (range)))))] 15 | (mapv idxs xs))) 16 | 17 | (defn mranks 18 | "Like ranks but applied to a matrix, where each element is replaced by its rank 19 | in its column. 20 | 21 | When sorting on id fields we don't know the exact values to be returned, since 22 | Datomic assigns the ids, but we know their relative ordering, i.e. rank. " 23 | [rows] 24 | (->> rows transpose (map ranks) transpose)) 25 | 26 | (deftest order-by-test 27 | (with-datomic 28 | (is (match? {:data {:columns ["db/id" "name" "code"], 29 | :rows 30 | [[pos-int? "Germany" "DE"] 31 | [pos-int? "Finnland" "FI"] 32 | [pos-int? "Belgium" "BE"]]}} 33 | (data/with-temp-db [_ test-data/countries] 34 | (data/run-mbql-query country 35 | {:order-by [[:desc $name]]})))))) 36 | 37 | (deftest order-by-foreign-keys 38 | (is (= [[1 8 5] 39 | [1 4 3] 40 | [1 1 1] 41 | [4 10 2] 42 | [4 8 7] 43 | [4 7 4] 44 | [4 4 8] 45 | [4 4 10] 46 | [4 3 6] 47 | [4 2 9]] 48 | 49 | (with-datomic 50 | (-> (data/run-mbql-query checkins 51 | {:fields [$venue_id $user_id [:field-id (data/id "checkins" "db/id")]] 52 | :order-by [[:asc $venue_id] 53 | [:desc $user_id] 54 | [:asc [:field-id (data/id "checkins" "db/id")]]] 55 | :limit 10}) 56 | (get-in [:data :rows]) 57 | mranks))))) 58 | 59 | (deftest order-by-aggregate-count 60 | (is (match? {:data {:columns ["price" "count"] 61 | :rows [[4 6] [3 13] [1 22] [2 59]]}} 62 | (with-datomic 63 | (data/run-mbql-query venues 64 | {:aggregation [[:count]] 65 | :breakout [$price] 66 | :order-by [[:asc [:aggregation 0]]]}))))) 67 | 68 | (deftest order-by-aggregate-sum 69 | (is (match? {:data {:columns ["price" "sum"] 70 | :rows 71 | [[2 #(< 2102.52379 % 2102.52381)] 72 | [1 786.8249000000002] 73 | [3 436.9022] 74 | [4 224.33829999999998]]}} 75 | 76 | (with-datomic 77 | (data/run-mbql-query venues 78 | {:aggregation [[:sum $latitude]] 79 | :breakout [$price] 80 | :order-by [[:desc [:aggregation 0]]]}))))) 81 | 82 | (deftest order-by-distinct 83 | (is (match? {:data {:columns ["price" "count"] 84 | :rows [[4 6] [3 13] [1 22] [2 59]]}} 85 | (with-datomic 86 | (data/run-mbql-query venues 87 | {:aggregation [[:distinct [:field-id (data/id "venues" "db/id")]]] 88 | :breakout [$price] 89 | :order-by [[:asc [:aggregation 0]]]}))))) 90 | 91 | (deftest order-by-avg 92 | (is (match? {:data {:columns ["price" "avg"] 93 | :rows 94 | [[3 #(< 33.6078615 % 33.6078616)] 95 | [2 #(< 35.6359966 % 35.6359967)] 96 | [1 #(< 35.7647681 % 35.7647682)] 97 | [4 #(< 37.3897166 % 37.3897167)]]}} 98 | (with-datomic 99 | (data/run-mbql-query venues 100 | {:aggregation [[:avg $latitude]] 101 | :breakout [$price] 102 | :order-by [[:asc [:aggregation 0]]]}))))) 103 | 104 | (deftest order-by-stddev-test 105 | (is (match? 106 | {:data {:columns ["price" "stddev"] 107 | :rows [[3 25.16407695964168] 108 | [1 23.55773642142646] 109 | [2 21.23640212816769] 110 | [4 13.5]]}} 111 | 112 | (with-datomic 113 | (data/run-mbql-query venues 114 | {:aggregation [[:stddev $category_id]] 115 | :breakout [$price] 116 | :order-by [[:desc [:aggregation 0]]]}))))) 117 | -------------------------------------------------------------------------------- /test/metabase/driver/datomic/query_processor_test.clj: -------------------------------------------------------------------------------- 1 | (ns metabase.driver.datomic.query-processor-test 2 | (:require [metabase.driver.datomic.query-processor :as datomic.qp] 3 | [metabase.driver.datomic.test :refer :all] 4 | [metabase.driver.datomic.test-data :as test-data] 5 | [metabase.query-processor :as qp] 6 | [metabase.test.data :as data] 7 | [clojure.test :refer :all])) 8 | 9 | (deftest select-field-test 10 | (let [select-name (datomic.qp/select-field 11 | '{:find [?foo ?artist]} 12 | {1 {:artist/name "abc"}} 13 | '(:artist/name ?artist))] 14 | (is (= "abc" 15 | (select-name [2 1]))))) 16 | 17 | (deftest select-fields-test 18 | (let [select-fn (datomic.qp/select-fields 19 | '{:find [?artist (count ?artist)]} 20 | {1 {:artist/name "abc"} 21 | 7 {:artist/name "foo"}} 22 | '[(count ?artist) (:artist/name ?artist)])] 23 | (is (= [42 "foo"] 24 | (select-fn [7 42]))))) 25 | 26 | (deftest execute-native-query-test 27 | (with-datomic 28 | (is (= {:columns ["?code" "?country"] 29 | :rows [["BE" "Belgium"] 30 | ["FI" "Finnland"] 31 | ["DE" "Germany"]]} 32 | (test-data/rows+cols 33 | (data/with-temp-db [db test-data/countries] 34 | (qp/process-query 35 | {:type :native 36 | :database (:id db) 37 | :native 38 | {:query 39 | (pr-str '{:find [?code ?country] 40 | :where [[?eid :country/code ?code] 41 | [?eid :country/name ?country]]})}}))))))) 42 | -------------------------------------------------------------------------------- /test/metabase/driver/datomic/source_query_test.clj: -------------------------------------------------------------------------------- 1 | (ns metabase.driver.datomic.source-query-test 2 | (:require [clojure.test :refer :all] 3 | [metabase.driver.datomic.test :refer :all] 4 | [metabase.driver.datomic.test-data :as test-data] 5 | [metabase.models.table :refer [Table]] 6 | [metabase.query-processor :as qp] 7 | [metabase.test.data :as data] 8 | [metabase.test.util.timezone :as tu.tz] 9 | [toucan.db :as db])) 10 | 11 | (defn table->db [t] 12 | (db/select-one-field :db_id Table, :id (data/id t))) 13 | 14 | (deftest only-source-query 15 | (is (match? {:data {:columns ["db/id" "name" "code"], 16 | :rows 17 | [[17592186045418 "Belgium" "BE"] 18 | [17592186045420 "Finnland" "FI"] 19 | [17592186045419 "Germany" "DE"]]}} 20 | 21 | (with-datomic 22 | (data/with-temp-db [_ test-data/countries] 23 | (qp/process-query 24 | {:database (table->db "country") 25 | :type :query 26 | :query {:source-query {:source-table (data/id "country") 27 | :order-by [[:asc [:field-id (data/id "country" "name")]]]}}})))))) 28 | 29 | (deftest source-query+aggregation 30 | (is (match? {:data {:columns ["price" "count"] 31 | :rows [[1 22] [2 59] [3 13] [4 6]]}} 32 | (with-datomic 33 | (qp/process-query 34 | (data/dataset test-data 35 | {:database (table->db "venues") 36 | :type :query 37 | :query {:aggregation [[:count]] 38 | :breakout [[:field-literal "price" :type/Integer]] 39 | :source-query {:source-table (data/id :venues)}}})))))) 40 | 41 | 42 | (deftest native-source-query+aggregation 43 | (is (match? {:data {:columns ["price" "count"] 44 | :rows [[1 22] [2 59] [3 13] [4 6]]}} 45 | (with-datomic 46 | (qp/process-query 47 | (data/dataset test-data 48 | {:database (table->db "venues") 49 | :type :query 50 | :query {:aggregation [[:count [:field-literal "venue" :type/Integer]]] 51 | :breakout [[:field-literal "price" :type/Integer]] 52 | :source-query {:native 53 | (pr-str 54 | '{:find [?price] 55 | :where [[?venue :venues/price ?price]]})}}})))))) 56 | 57 | 58 | (deftest filter-source-query 59 | (is (match? {:data {:columns ["date"], 60 | :rows 61 | [["2015-01-04T08:00:00.000Z"] 62 | ["2015-01-14T08:00:00.000Z"] 63 | ["2015-01-15T08:00:00.000Z"]]}} 64 | (with-datomic 65 | (tu.tz/with-jvm-tz "UTC" 66 | (qp/process-query 67 | (data/dataset test-data 68 | {:database (table->db "checkins") 69 | :type :query 70 | :query {:filter [:between [:field-literal "date" :type/Date] "2015-01-01" "2015-01-15"] 71 | :order-by [[:asc [:field-literal "date" :type/Date]]] 72 | :source-query {:fields [[:field-id (data/id "checkins" "date")]] 73 | :source-table (data/id :checkins)}}}))))))) 74 | 75 | (deftest nested-source-query 76 | (is (match? {:row_count 25 77 | :data {:columns ["db/id" "name" "category_id" "latitude" "longitude" "price"]}} 78 | (with-datomic 79 | (qp/process-query 80 | (data/dataset test-data 81 | {:database (table->db "venues") 82 | :type :query 83 | :query {:limit 25 84 | :source-query {:limit 50 85 | :source-query {:source-table (data/id :venues) 86 | :limit 100}}} })))))) 87 | -------------------------------------------------------------------------------- /test/metabase/driver/datomic/test.clj: -------------------------------------------------------------------------------- 1 | (ns metabase.driver.datomic.test 2 | (:require [metabase.driver :as driver] 3 | [metabase.query-processor :as qp])) 4 | 5 | (require 'metabase.driver.datomic) 6 | (require 'metabase.driver.datomic.test-data) 7 | (require 'matcher-combinators.test) 8 | 9 | (defmacro with-datomic [& body] 10 | `(driver/with-driver :datomic 11 | ~@body)) 12 | 13 | (def query->native 14 | (comp read-string :query qp/query->native)) 15 | -------------------------------------------------------------------------------- /test/metabase/driver/datomic/test_data.clj: -------------------------------------------------------------------------------- 1 | (ns metabase.driver.datomic.test-data 2 | (:require [metabase.test.data.interface :as tx])) 3 | 4 | (defn rows+cols [result] 5 | (select-keys (:data result) [:columns :rows])) 6 | 7 | (tx/def-database-definition aggr-data 8 | [["foo" 9 | [{:field-name "f1" :base-type :type/Text} 10 | {:field-name "f2" :base-type :type/Text}] 11 | [["xxx" "a"] 12 | ["xxx" "b"] 13 | ["yyy" "c"]]]]) 14 | 15 | (tx/def-database-definition countries 16 | [["country" 17 | [{:field-name "code" :base-type :type/Text} 18 | {:field-name "name" :base-type :type/Text}] 19 | [["BE" "Belgium"] 20 | ["DE" "Germany"] 21 | ["FI" "Finnland"]]]]) 22 | 23 | (tx/def-database-definition with-nulls 24 | [["country" 25 | [{:field-name "code" :base-type :type/Text} 26 | {:field-name "name" :base-type :type/Text} 27 | {:field-name "population" :base-type :type/Integer} 28 | ] 29 | [["BE" "Belgium" 11000000] 30 | ["DE" "Germany"] 31 | ["FI" "Finnland"]]]]) 32 | 33 | #_(user.setup/remove-database "countries") 34 | -------------------------------------------------------------------------------- /test/metabase/driver/datomic_test.clj: -------------------------------------------------------------------------------- 1 | (ns metabase.driver.datomic-test) 2 | -------------------------------------------------------------------------------- /test/metabase/test/data/datomic.clj: -------------------------------------------------------------------------------- 1 | (ns metabase.test.data.datomic 2 | (:require [metabase.test.data.interface :as tx] 3 | [metabase.test.data.sql :as sql.tx] 4 | [metabase.driver.datomic :as datomic-driver] 5 | [metabase.driver :as driver] 6 | [datomic.api :as d])) 7 | 8 | (driver/add-parent! :datomic ::tx/test-extensions) 9 | 10 | (def metabase->datomic-type 11 | (into {:type/DateTime :db.type/instant 12 | :type/Date :db.type/instant} 13 | (map (fn [[k v]] [v k])) 14 | datomic-driver/datomic->metabase-type)) 15 | 16 | (defmethod sql.tx/field-base-type->sql-type [:datomic :type/*] [_ t] 17 | (metabase->datomic-type t)) 18 | 19 | (defn- db-url [dbdef] 20 | (str "datomic:mem:" (tx/escaped-name dbdef))) 21 | 22 | (defn field->attributes 23 | [table-name {:keys [field-name base-type fk field-comment]}] 24 | (cond-> {:db/ident (keyword table-name field-name) 25 | :db/valueType (if fk 26 | :db.type/ref 27 | (metabase->datomic-type base-type)) 28 | :db/cardinality :db.cardinality/one} 29 | field-comment 30 | (assoc :db/doc field-comment))) 31 | 32 | (defn table->attributes 33 | [{:keys [table-name field-definitions rows table-comment]}] 34 | (map (partial field->attributes table-name) field-definitions)) 35 | 36 | (defn table->entities 37 | [{:keys [table-name field-definitions rows]}] 38 | (let [attrs (map (comp (partial keyword table-name) 39 | :field-name) 40 | field-definitions)] 41 | (->> rows 42 | (map (partial map (fn [field value] 43 | (if-let [dest-table (:fk field)] 44 | ;; Turn foreign keys ints into temp-id strings 45 | (str (name dest-table) "-" value) 46 | (if (= :type/BigInteger (:base-type field)) 47 | (java.math.BigInteger/valueOf value) 48 | value))) 49 | field-definitions)) 50 | (map (partial zipmap attrs)) 51 | (map (fn [attr-map] 52 | (into {} (filter (comp some? val)) attr-map))) 53 | (map (fn [idx attr-map] 54 | ;; Add temp-id strings to resolve foreign keys 55 | (assoc attr-map :db/id (str table-name "-" idx))) 56 | (next (range)))))) 57 | 58 | (defmethod tx/create-db! :datomic 59 | [_ {:keys [table-definitions] :as dbdef} & {:keys [skip-drop-db?] :as opts}] 60 | (let [url (db-url dbdef)] 61 | (when-not skip-drop-db? 62 | (d/delete-database url)) 63 | (d/create-database url) 64 | (let [conn (d/connect url) 65 | schema (mapcat table->attributes table-definitions) 66 | data (mapcat table->entities table-definitions)] 67 | @(d/transact conn schema) 68 | @(d/transact conn data)))) 69 | 70 | (defmethod tx/dbdef->connection-details :datomic [_ context dbdef] 71 | {:db (db-url dbdef)}) 72 | 73 | (comment 74 | (letsc 9 (mapcat table->attributes (:table-definitions dbdef))) 75 | (def dbdef (letsc 7 dbdef)) 76 | (table->attributes (first (:table-definitions dbdef))) 77 | ) 78 | -------------------------------------------------------------------------------- /test/metabase_datomic/kaocha_hooks.clj: -------------------------------------------------------------------------------- 1 | (ns metabase-datomic.kaocha-hooks 2 | (:require [clojure.java.io :as io] 3 | [clojure.edn :as edn] 4 | [metabase.models.database :as database :refer [Database]] 5 | [metabase.util :as util] 6 | [toucan.db :as db] 7 | [metabase.test-setup :as test-setup] 8 | [clojure.string :as str] 9 | [datomic.api :as d])) 10 | 11 | (defonce startup-once 12 | (delay 13 | (test-setup/test-startup) 14 | (require 'user) 15 | (require 'user.repl) 16 | ((resolve 'user.repl/clean-up-in-mem-dbs)) 17 | ((resolve 'user/setup-driver!)))) 18 | 19 | (defn pre-load [test-plan] 20 | @startup-once 21 | test-plan) 22 | 23 | (defn post-run [test-plan] 24 | #_(doseq [{:keys [id name details]} (db/select Database {:engine :datomic}) 25 | :when (str/includes? (:db details) ":mem:")] 26 | (util/ignore-exceptions 27 | (d/delete-database (:db details))) 28 | (db/delete! Database {:id id})) 29 | #_(test-setup/test-teardown) 30 | test-plan) 31 | -------------------------------------------------------------------------------- /tests.edn: -------------------------------------------------------------------------------- 1 | #kaocha/v1 2 | {:plugins [:kaocha.plugin/notifier 3 | :kaocha.plugin/print-invocations 4 | :kaocha.plugin/profiling 5 | :kaocha.plugin/hooks] 6 | 7 | :tests 8 | [{:source-paths ["../metabase-datomic/src"] 9 | :test-paths ["../metabase-datomic/test"]}] 10 | 11 | :kaocha.hooks/pre-load [metabase-datomic.kaocha-hooks/pre-load] 12 | :kaocha.hooks/post-run [metabase-datomic.kaocha-hooks/post-run]} 13 | --------------------------------------------------------------------------------