├── .gitignore ├── .gitlab-ci.yml ├── .travis.yml ├── LICENSE ├── README.md ├── config └── config.exs ├── lib ├── builtins.ex ├── datetime.ex ├── executor.ex ├── exosql.ex ├── expr.ex ├── extractors │ ├── csv.ex │ ├── http.ex │ └── node.ex ├── format.ex ├── parser.ex ├── planner.ex └── utils.ex ├── mix.exs ├── mix.lock ├── src ├── .gitignore ├── sql_lexer.xrl └── sql_parser.yrl └── test ├── builtins_test.exs ├── data └── csv │ ├── campaigns.csv │ ├── family.csv │ ├── json.csv │ ├── products.csv │ ├── purchases.csv │ ├── unnestarray.csv │ ├── urls.csv │ └── users.csv ├── debug_test.exs ├── esql_test.exs ├── executor_test.exs ├── nested_select_test.exs ├── parser_test.exs ├── planner_test.exs ├── query_test.exs └── test_helper.exs /.gitignore: -------------------------------------------------------------------------------- 1 | # The directory Mix will write compiled artifacts to. 2 | /_build/ 3 | 4 | # If you run "mix test --cover", coverage assets end up here. 5 | /cover/ 6 | 7 | # The directory Mix downloads your dependencies sources to. 8 | /deps/ 9 | 10 | # Where 3rd-party dependencies like ExDoc output generated docs. 11 | /doc/ 12 | 13 | # Ignore .fetch files in case you like to edit your project deps locally. 14 | /.fetch 15 | 16 | # If the VM crashes, it generates a dump, let's ignore it too. 17 | erl_crash.dump 18 | 19 | # Also ignore archive artifacts (built via "mix archive.build"). 20 | *.ez 21 | 22 | .elixir_ls 23 | -------------------------------------------------------------------------------- /.gitlab-ci.yml: -------------------------------------------------------------------------------- 1 | stages: 2 | - test 3 | 4 | test_upm: 5 | stage: test 6 | script: 7 | - mix deps.get 8 | - mix test 9 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: elixir 2 | sudo: false 3 | env: 4 | - MIX_ENV=test 5 | elixir: 6 | - 1.5.1 7 | otp_release: 8 | - 20.0 9 | before_script: 10 | - mix compile 11 | script: 12 | - mix test 13 | cache: 14 | directories: 15 | - ~/.mix 16 | - ~/.hex 17 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ExoSQL 2 | 3 | [![Build Status](https://travis-ci.org/serverboards/exosql.svg?branch=master)](https://travis-ci.org/serverboards/exosql) 4 | 5 | Universal SQL engine for Elixir. 6 | 7 | This library implements the SQL engine to perform queries on user provided 8 | databases using a simple interface based on Foreign Data Wrappers from 9 | PostgreSQL. 10 | 11 | This allows to use SQL on your own data and virtual tables. 12 | 13 | For example it includes a CSV reader and an HTTP client, so that you can 14 | do queries as: 15 | 16 | ```SQL 17 | SELECT url, status_code 18 | FROM urls 19 | INNER JOIN request 20 | ON urls.url = request.url 21 | ``` 22 | 23 | There is a simple repl to be able to test ExoSQL: 24 | 25 | ```elixir 26 | iex> ExoSQL.repl() 27 | exosql> SELECT m, SUM(price) FROM generate_series(10) as m LEFT JOIN (SELECT width_bucket(price, 0, 200, 10) AS n, price FROM products) ON n = m GROUP BY m 28 | tmp.m.m | tmp.tmp.col_2 29 | -------------------------- 30 | 1 | 31 31 | 2 | 30 32 | 3 | 0 33 | 4 | 0 34 | 5 | 0 35 | 6 | 0 36 | 7 | 0 37 | 8 | 0 38 | 9 | 0 39 | 10 | 0 40 | ``` 41 | 42 | ## Origin 43 | 44 | The origin of the library is as a SQL layer to all the services connected to you 45 | [Serverboards](https://serverboards.io). 46 | 47 | Each service can export tables to be accessed via SQL and then can show the data 48 | in the Dashboards, the notebook, or used in the rules. 49 | 50 | ## Installation 51 | 52 | The package can be installed by adding `exosql` to your list of dependencies in 53 | `mix.exs`: 54 | 55 | ```elixir 56 | def deps do 57 | [ 58 | {:exosql, "~> 0.2"} 59 | ] 60 | end 61 | ``` 62 | 63 | ## Features 64 | 65 | * **SELECT over external databases (CSV, HTTP endpoints... Programmable)** 66 | * `SELECT` over several tables 67 | * `WHERE` 68 | * `INNER JOIN` 69 | * `LEFT JOIN` 70 | * `RIGHT JOIN` 71 | * `GROUP BY` 72 | * `ORDER BY` 73 | * `OFFSET` and `LIMIT` 74 | * `DISTINCT` and `DISTINCT ON` 75 | * `LIKE` and `ILIKE` 76 | * `CASE` `WHEN` `THEN` `ELSE` `END` / `IF` `THEN` `ELIF` `ELSE` `END`. 77 | * `UNION` and `UNION ALL`. 78 | * `RANGE` 79 | * `WITH` common table expressions 80 | * `CROSS JOIN LATERAL` 81 | * `CROSSTAB` / `CROSSTAB ON` 82 | * table and column alias with `AS` 83 | * nested `SELECT`: At `FROM`, `SELECT`, `WHERE`... 84 | * `generate_series` function tables 85 | * Aggregation functions: `COUNT`, `SUM`, `AVG`... 86 | * Builtin functions and operators: * / + - || `or` `and` `in` `not`; `round` `concat`... [See all](#builtins). 87 | * Builtin `format`, `strftime`, `regex` and more string and time formatting functions. 88 | * Basic Reflection over `self.tables` 89 | * JSON support via [json pointer](#jp). 90 | * Array support: `[1, 2, 3, 4]` 91 | * Variables 92 | * Comments (-- SQL style) 93 | 94 | Check the tests for current available features. 95 | 96 | ## Variables 97 | 98 | Variables can be passed as a dictionary at `__vars__` inside the context, and 99 | referenced as `$name` at the SQL expression. This may change in the future 100 | to streamline it more with standard SQL (no need for `$`). 101 | 102 | There is a special variable `debug` that, when true, enables extra debug to the 103 | logger, for example each step of the execution of the query. 104 | 105 | ## INNER JOIN 106 | 107 | Because some columns may need to be autogenerated depending on the query, 108 | if you want to access those columns you may need to use INNER JOINS. This 109 | way the planner asks for those specific column values. 110 | 111 | For example: 112 | 113 | ```SQL 114 | SELECT * FROM request 115 | ``` 116 | 117 | does not know to which URL you want to access, but: 118 | 119 | ```SQL 120 | SELECT * FROM request WHERE url = 'http://serverboards.io' 121 | ``` 122 | 123 | knows the URL and can get the data. 124 | 125 | Then same way, on INNER JOINS this can be used to access to auto generated data: 126 | 127 | ```SQL 128 | SELECT url, status_code 129 | FROM urls 130 | INNER JOIN request 131 | ON urls.url = request.url 132 | ``` 133 | 134 | ## CROSS JOIN LATERAL 135 | 136 | ExoSQL can do limited lateral joins on table expressions. This allow for example 137 | to use a JSON array to be unnested as several columns: 138 | 139 | ```sql 140 | SELECT id, email, name FROM json CROSS JOIN LATERAL unnest(json, 'email', 'name') 141 | ``` 142 | 143 | Currently only support for expressions is ready, nested queries is not funcional 144 | yet. 145 | 146 | ## CROSSTAB 147 | 148 | Crosstab is a custom extension, not existing on the SQL standard, and other 149 | implementations may use other syntax: 150 | 151 | * [crosstab](https://www.postgresql.org/message-id/20031102232246.6a3c6088.veramente%40libero.it) extension at PostgreSQL 152 | * [concat](https://stackoverflow.com/questions/12382771/mysql-pivot-crosstab-query) at MySQL 153 | * [pivot](https://www.mssqltips.com/sqlservertip/1019/crosstab-queries-using-pivot-in-sql-server/) as MsSQL 154 | 155 | Serverboards tries another option: 156 | 157 | ```sql 158 | SELECT CROSSTAB ON (sugus, lollipop) 159 | user, product, sum(amount) 160 | FROM 161 | productsales 162 | ``` 163 | 164 | It will result in a table with this form: 165 | 166 | | user | sugus | lollipop| 167 | |-------|-------|---------| 168 | | David | 10 | 20 | 169 | | Anna | NULL | 15 | 170 | | ... | ... | ... | 171 | 172 | 173 | It follows the syntax of `DISTINCT`. 174 | 175 | If `ON` is provided only extracts those columns, which in some cases may be 176 | completely empty. If it is not present it returns all possible columns, in 177 | alphabetical order. 178 | 179 | Crosstabs have the following caveats: 180 | 181 | * Column names may not be known at plan time, so if you need any specific column 182 | for subqueries, you need to use the "ON" version. The first column name is 183 | always known. 184 | * Crosstab is performed just after the select. If you need to order 185 | the rows it has to be by column number. See [`ORDER BY`](#order_by) for 186 | the full rationale. As a workaround you can use a nested select. 187 | 188 | ## ORDER BY 189 | 190 | Sort can be used, per SQL standard, by column name or result column number. 191 | 192 | Due to the way ExoSQL works the ORDER operation is done in two steps: 193 | 194 | 1. Before select, orders by expressions. 195 | 2. After select, orders by the resulting column number. 196 | 197 | This is like this as after the select we do not have access to all the column 198 | names, only the resulting ones. And before there is no access to the column 199 | number results. 200 | 201 | This have two important implications: 202 | 203 | 1. There is a bug when you mix `ORDER BY` expression and column number. The 204 | second order by number will be always more important than by expression. 205 | 2. At `CROSSTAB` the `ORDER BY` name just don't work. When data gets into 206 | the crosstab algorithm the order is not specified for rows, and is 207 | alphabetical for columns. It is possible to order by number, as it happens 208 | after the crosstab. 209 | 210 | 211 | ## Builtins 212 | 213 | ### String operations 214 | 215 | #### `debug(str)` 216 | 217 | Prints to the Elixir Logger.debug the value of that string. Returns the same 218 | value. 219 | 220 | Can be useful on some debugging. 221 | 222 | #### `format(format_str, args...)` 223 | 224 | Formats a String using C sprintf-like parameters. Known placeholders are: 225 | 226 | * `%s` -- String 227 | * `%10s` -- String. String at the right, add spaces until 10 chars. (padleft) 228 | * `%-10s` -- String. String at the left, add spaces until 10 chars. (padright) 229 | * `%d` -- Integer number 230 | * `+%d` -- Integer number, always add sign 231 | * `%02d` -- Number padded with 0 to fill 2 chars 232 | * `%f` -- Float 233 | * `%.2f` -- Float with precision 234 | * `+%f` -- Float, always add sign. 2 chars of precision. 235 | * `%k` -- Metric System suffix: k, M, G, T. Try to show most relevant information. 236 | * `%.2k` -- Metric System suffix with precision 237 | * `%,2k` -- Metric System, using `.` to separate thousands and `,` for decimals. Follow Spanish numbering system. 238 | 239 | #### `lower(str)` 240 | 241 | Lower case a full string. 242 | 243 | #### `join(str, sep=",")` 244 | 245 | Joins all elements from a list into a string, using the given separator. 246 | 247 | ```sql 248 | join([1,2,3,4], "/") 249 | "1/2/3/4" 250 | ``` 251 | 252 | #### `split(str, sep=[", ", ",", " "])` 253 | 254 | Splits a string into a list using the given separator. 255 | 256 | ```sql 257 | split("1, 2,3 4") 258 | ["1", "2", "3", "4"] 259 | ``` 260 | 261 | 262 | #### `substr(str, start, end=10000)` / `substr(str, end)` 263 | 264 | Extracts a substring from the first argument. 265 | 266 | Can use negative indexes to start to count from the end. 267 | 268 | ```sql 269 | substr('#test#', 1, -1) 270 | "test" 271 | ``` 272 | 273 | #### `to_string(arg)` 274 | 275 | Converts the given argument into a string. 276 | 277 | ```sql 278 | to_string(1) 279 | "1" 280 | ``` 281 | 282 | #### `upper(str)` 283 | 284 | Upper cases a full string 285 | 286 | ### Date time functions 287 | 288 | #### `datediff(start, end, unit \\ "days")` / `datediff(range, unit \\ "days")` 289 | 290 | Returns how many `unit` has passed since start to end. 291 | 292 | #### `now()` 293 | 294 | Returns current datetime. 295 | 296 | #### `strftime(datetime, format_str)` 297 | 298 | Convert a datetime to a string. Can be used also to extract some parts of a 299 | date, as the day, year and so on. 300 | 301 | Normally `strftime` can be used directly with a string or an integer as it does 302 | the conversion to datetime implicitly. 303 | 304 | It is based on [Timex](https://github.com/bitwalker/timex) 305 | [formatting](https://hexdocs.pm/timex/Timex.Format.DateTime.Formatters.Strftime.html). 306 | 307 | Most common markers: 308 | 309 | * `%Y` -- Year four digits 310 | * `%y` -- Year two digits 311 | * `%m` -- Month number 312 | * `%d` -- Day of month 313 | * `%H` -- Hour 314 | * `%M` -- Minute 315 | * `%S` -- Second 316 | * `%V` -- ISO Week (01-53) 317 | * `%s` -- Unix time 318 | * `%F` -- ISO year: yyyy-mm-dd 319 | * `%H` -- Time: HH:MM:SS 320 | 321 | #### `to_datetime(str | int, mod \\ nil)` / `to_datetime(str | int, timezone)` 322 | 323 | Converts the given string or integer to a date. 324 | 325 | The string must be in ISO8859 sub string format: 326 | 327 | * `YYYY-mm-dd` 328 | * `YYYY-mm-ddTHH:MM` 329 | * `YYYY-mm-dd HH:MM` 330 | * `YYYY-mm-ddTHH:MM:SS` 331 | * `YYYY-mm-dd HH:MM:SS` 332 | * or an Unix epoch integer. 333 | 334 | This is called implicitly on `strftime` calls, and normally is not needed. 335 | 336 | Last argument can be a modifier to add or subtract time, or a timezone shifter. 337 | 338 | ##### Datetime modifier 339 | 340 | `mod` MUST start with `+` or `-` 341 | 342 | If `mod` is given it is a duration modifier as defined by 343 | [ISO8601](https://en.wikipedia.org/wiki/ISO_8601#Durations), with the following 344 | changes: 345 | 346 | * Must start with `+` or `-` 347 | * A subsequent `P` is optional 348 | 349 | For example: 350 | 351 | * Subtract one month `to_datetime(NOW(), "-1M")` 352 | * Add 30 minutes: `to_datetime(NOW(), "+T30M")` 353 | * One year and a half and 6 minutes ago: `to_datetime(NOW(), "-1Y1MT6M")` 354 | 355 | 356 | ##### Datetime timezone change 357 | 358 | If a timezone is provided the datetime will be changed from the current timezone 359 | to the provided one, making the appropiate changes to the datetime. 360 | 361 | For example, `to_datetime(0, 'Europe/Madrid')` gives the Madrid time for unix 362 | epoch 0: `1970-01-01 01:00:00+01:00 CET Europe/Madrid`. 363 | 364 | Check https://en.wikipedia.org/wiki/List_of_tz_database_time_zones for the 365 | timezone names. 366 | 367 | ### Boolean functions 368 | 369 | #### `bool(arg)` 370 | 371 | Converts to boolean. Equivalent to `NOT NOT arg` 372 | 373 | ### Aggregation functions 374 | 375 | #### `avg(expr)` 376 | 377 | Calculates the average of the calculated expression on the group rows. 378 | Equivalent to `sum(expr) / count(expr)`. 379 | 380 | If no rows, returns `NULL`. 381 | 382 | #### `count(*)` 383 | 384 | Counts the number of rows of the aggregates expression. 385 | 386 | #### `max(expr)` 387 | 388 | Returns the maximum value of the given expression for the group. 389 | 390 | #### `min` 391 | 392 | Returns the minimum value of the given expression for the group. 393 | 394 | #### `sum(expr)` 395 | 396 | For each of the grouped rows, calculates the expression and returns the sum. If 397 | there are no rows, returns 0. 398 | 399 | ### Math functions 400 | 401 | #### `abs(number)` 402 | 403 | Returns the absolute value a number 404 | 405 | #### `ceil(number)` 406 | 407 | Rounds up a number 408 | 409 | #### `floor(number)` 410 | 411 | Rounds down a number 412 | 413 | #### `ln(number)` 414 | 415 | Returns the natural logarithm of the given number. 416 | 417 | #### `log(number)` 418 | 419 | Returns the base 10 logarithm of the given number. 420 | 421 | #### `power(number, number)` 422 | 423 | Returns the power of the first number to the second. 424 | 425 | For example `power(2,2)` return `4`. 426 | 427 | #### `random()` 428 | 429 | Return a random float between 0 and 1. 430 | 431 | #### `randint(max)` / `RANDINT(min, max)` 432 | 433 | Returns a random integer between `min` and `max`. 434 | 435 | #### `round(number, precision=0)` 436 | 437 | Returns the number rounded to the given precision. May be convert to integer if 438 | precision is 0. 439 | 440 | #### `sign(number)` 441 | 442 | Returns `-1`, `0` or `1` depending if the number is `<0`, `=0` or `>0` 443 | 444 | #### `sqrt(number)` 445 | 446 | Returns the square root of the given number. See the `power` function for more 447 | advanced roots. 448 | 449 | #### `trunc(number, precision=0)` 450 | 451 | Returns the number rounded to the given precision. May be convert to integer if 452 | precision is 0. It is an alias to round. 453 | 454 | ### Miscellaneous functions 455 | 456 | #### `coalesce(a, b, ...)` 457 | 458 | Returns the first not NULL value. 459 | 460 | #### `generate_series(end)` / `generate_series(start, end, step=0)` 461 | 462 | This function generates a virtual table with one column and on each row a value of the series. 463 | 464 | Can be reverse with a larger start than end and negative step. 465 | 466 | It can be used to for example fill all holes in a temporal serie: 467 | 468 | ```sql 469 | SELECT month, SUM(value) 470 | FROM generate_series(12) AS month 471 | LEFT JOIN purchases 472 | ON strftime(purchases.datetime, "%m") == month 473 | GROUP BY month 474 | ``` 475 | 476 | This will return 0 for empty months on the purchases table. 477 | 478 | #### `greatest(a, b[, c, d, e])` 479 | 480 | Get the greatest value of all the given ones. 481 | 482 | It's similar to `max` in other languages, but in SQL can not use `max` as it is 483 | an aggregation function with different semantics. 484 | 485 | #### `json(str)` 486 | 487 | Parses a string into a json object. NULL is nil. 488 | 489 | #### `jp(json, selector)` 490 | 491 | Does [JSON Pointer](https://tools.ietf.org/html/rfc6901) selection: 492 | 493 | * Use / to navigate through the object keys or array indexes. 494 | * If no data found, return `NULL` 495 | 496 | #### `least(a, b[, c, d, e])` 497 | 498 | Get the lowest value of all the given ones. 499 | 500 | It's similar to `min` in other languages, but in SQL can not use `min` as it is 501 | an aggregation function with different semantics. 502 | 503 | #### `lower(range)` 504 | 505 | Get the lower bound of a range. 506 | 507 | #### `nullif(a,b)` 508 | 509 | Returns NULL if A and B are equal. Else returns A. 510 | 511 | This is used for example to set a default value: 512 | 513 | ```sql 514 | SELECT coalesce(nullif(name, ''), 'John Doe') FROM users 515 | ``` 516 | 517 | ### `range(start, end)` 518 | 519 | Returns a halt open interval `[start, end)` that later can be used to get 520 | intersection `*` or membership. 521 | 522 | The range includes the `start` but not the `end` (`all X | start >= X < end`). 523 | This is important for later datediff and similar. 524 | 525 | For example, the following query will check if `NOW()` is in the intersection of 526 | some range given by parameters `$start` and `$end` and the range set by the columns 527 | `start` and `end`. 528 | 529 | ```sql 530 | SELECT NOW() IN (range(start, end) * range($start, $end)) 531 | FROM dates 532 | ``` 533 | 534 | It works for both dates, texts and numbers. 535 | 536 | Ranges can be decomposed with `lower(range)` and `upper(range)`. 537 | 538 | #### `regex(str, regex, query \\ nil)` 539 | 540 | Performs a regex search on the string and returns the first match. 541 | 542 | It uses elixir regex, so use it as reference. 543 | 544 | Can use groups and named groups for matching and it will return a list of a map 545 | with the result. It can optionally use directly JSON pointer queries. See 546 | `jp` function. 547 | 548 | If matches the result will be "trueish" (or "falsy" if doesn't) so can be used 549 | as a boolean. 550 | 551 | #### `regex_all(str, regex, query \\ nil)` 552 | 553 | Similar to `regex(str, regex, query \\ nil)` but returns all matches in a list. 554 | 555 | #### `unnest(json, col1...)` 556 | 557 | Expands a json or an array to be used on `LATERAL` joins. 558 | 559 | It converts the array or json representation of an array to a list of lists, 560 | as required by the `LATERAL` joins. Optionally, if column names are given the 561 | items are expanded as such columns. 562 | 563 | For example: 564 | 565 | ```sql 566 | SELECT id, email, name FROM json CROSS JOIN LATERAL unnest(json, 'email', 'name') 567 | ``` 568 | 569 | For each column expands the `json` array, getting only the `email` and `name` of 570 | each item, so the final result has all the emails and names for all the arrays 571 | at the json table. 572 | 573 | 574 | A.json.id | tmp.unnest.email | tmp.unnest.name | A.json.json 575 | ----------|------------------|-----------------|--------- 576 | 1 | one@example.com | uno | [{"email": "one@example.com", "name": "uno"}, {"email": "two@example.com", "name": "dos"}] 577 | 1 | two@example.com | dos | [{"email": "one@example.com", "name": "uno"}, {"email": "two@example.com", "name": "dos"}] 578 | 2 | three@example.com | tres | [{"email": "three@example.com", "name": "tres"}, {"email": "four@example.com", "name": "cuatro"}] 579 | 2 | four@example.com | cuatro | [{"email": "three@example.com", "name": "tres"}, {"email": "four@example.com", "name": "cuatro"}] 580 | 581 | 582 | The last column has the original json fo the each json line (2 lines), but it is 583 | expanded to four lines. 584 | 585 | 586 | #### `upper(range)` 587 | 588 | Get the upper bound of a range. 589 | 590 | #### `urlparse(string, sel="")` 591 | 592 | Parses an URL and returns a JSON. 593 | 594 | If selector is given it does the equivalent of callong `jp` with that selector. 595 | 596 | #### `width_bucket(n, start, end, buckets)` 597 | 598 | Given a `n` value it is assigned a bucket between 0 and `buckets`, that correspond to the full width between `start` and `end`. 599 | 600 | If a value is out of bounds it is set either to 0 or to `buckets - 1`. 601 | 602 | This helper eases the generation of histograms. 603 | 604 | For example an histogram of prices: 605 | 606 | ```sql 607 | SELECT n, SUM(price) 608 | FROM (SELECT width_bucket(price, 0, 200, 10) AS n, price 609 | FROM products) 610 | GROUP BY n 611 | ``` 612 | 613 | or more complete, with filling zeroes: 614 | 615 | ```sql 616 | SELECT m, SUM(price) 617 | FROM generate_series(10) AS m 618 | LEFT JOIN ( 619 | SELECT width_bucket(price, 0, 200, 10) AS n, price 620 | FROM products 621 | ) 622 | ON n = m 623 | GROUP BY m 624 | ``` 625 | 626 | ## Included extractors 627 | 628 | ExoSQL has been developed with the idea of connecting to Serverboards services, 629 | and as such it does not provide more than some test extractors: 630 | 631 | * CSV files 632 | * HTTP requests 633 | 634 | Creating new ones is a very straightforward process. The HTTP example can be 635 | followed. 636 | 637 | This is not intended a full database system, but to be embedded into other 638 | Elixir programs and accessible from them by end users. As such it does contain 639 | only some basic extractors that are needed for proper testing. 640 | 641 | ## Using ExoSQL 642 | 643 | There is no formal documentation yet, but you can check the `esql_test.exs` file 644 | to get an idea of how to use ExoSQL. 645 | 646 | Example: 647 | 648 | ```elixir 649 | context = %{ 650 | "A" => {ExoSQL.Csv, path: "test/data/csv/"}, 651 | "B" => {ExoSQL.HTTP, []}. 652 | "__vars__" => %{ "start" => "2018-01-01" } 653 | } 654 | {:ok, result} = ExoSQL.query(" 655 | SELECT urls.url, request.status_code 656 | FROM urls 657 | INNER JOIN request 658 | ON urls.url = request.url 659 | ", context) 660 | ``` 661 | 662 | ```elixir 663 | %ExoSQL.Result{ 664 | columns: [{"A", "urls", "url"}, {"B", "request", "status_code"}], 665 | rows: [ 666 | ["https://serverboards.io/e404", 404], 667 | ["http://www.facebook.com", 302], 668 | ["https://serverboards.io", 200], 669 | ["http://www.serverboards.io", 301], 670 | ["http://www.google.com", 302] 671 | ]} 672 | ``` 673 | 674 | A Simple extractor can be: 675 | ```elixir 676 | defmodule MyExtractor do 677 | def schema(_config), do: {:ok, ["week"]} 678 | def schema(_config, "week"), do: {:ok, %{ columns: ["id", "nr", "name", "weekend"] }} 679 | def execute(_config, "week", _quals, _columns) do 680 | {:ok, %{ 681 | columns: ["id", "nr", "name", "weekend"], 682 | rows: [ 683 | [1, 0, "Sunday", true], 684 | [2, 1, "Monday", false], 685 | [3, 2, "Tuesday", false], 686 | [4, 3, "Wednesday", false], 687 | [5, 4, "Thursday", false], 688 | [6, 5, "Friday", false], 689 | [7, 6, "Saturday", true], 690 | ] 691 | }} 692 | end 693 | end 694 | ``` 695 | 696 | And then a simple query: 697 | 698 | ```elixir 699 | {:ok, res} = ExoSQL.query("SELECT * FROM week WHERE weekend", %{ "A" => {MyExtractor, []}}) 700 | ExoSQL.format_result(res) 701 | ``` 702 | 703 | |A.week.id | A.week.nr | A.week.name | A.week.weekend| 704 | |----------|-----------|-------------|---------------| 705 | |1 | 0 | Sunday | true | 706 | |7 | 6 | Saturday | true | 707 | 708 | ## Related libraries 709 | 710 | There are other implementations of this very same idea: 711 | 712 | * [Postgres Foreign Data Wrappers] (FDW). Integrates any external 713 | source with a postgres database. Can be programmed in C and Python. Postgres 714 | FDW gave me the initial inspiration for ExoSQL. 715 | * [Apache Foundation's Drill]. Integrates NoSQL database and SQL databases. 716 | * [Apache Foundation's Calcite]. Java based library, very similar to ExoSQL, 717 | with many many adapters. Many projects use parts of calcite, for example 718 | Drill uses the SQL parser. 719 | 720 | If you know any other, please ping me and I will add it here. 721 | 722 | I develop ExoSQL as I needed an elixir solution for an existing project, and 723 | to learn how to create an SQL engine. ExoSQL is currently used in 724 | [Serverboards] KPI. 725 | 726 | [Postgres Foreign Data Wrappers]: https://wiki.postgresql.org/wiki/Foreign_data_wrappers 727 | [Apache Foundation's Drill]: https://drill.apache.org 728 | [Apache Foundation's Calcite]: https://calcite.apache.org 729 | [Serverboards]: https://serverboards.io 730 | 731 | ## Known BUGS 732 | 733 | * When doing `ORDER BY [column id], [column name]`, it reverses the order. To 734 | avoid use one or the other, dont mix order by column name and result column 735 | position. 736 | 737 | This is because the planner does the ordering on column name first, then 738 | the select which limits the columns and reorder them and then the ordering 739 | by column position. 740 | 741 | * Can not use variables inside aggregation functions. 742 | 743 | ## Future improvements 744 | 745 | * BEAM compile: expressions and functions first, then full plan. Now uses an AST 746 | executor, which may be slower 747 | 748 | * Bytecode compile. Check also if compiling to a bytecode may improve 749 | performance 750 | 751 | * `LIMIT`/`OFFSET` before `SELECT`. No need to do the full select over all the 752 | data, specially if there are format, expressions, aggregations and other 753 | expensive operations, if we only want some results. As it may be used with 754 | `SORT BY` extra care is needed to calculate the required columns, and then 755 | calculate the rest. 756 | 757 | * Streaming extractors. Now all is stored in memory, which can be a lot. 758 | 759 | * Multiprocessing. If possible calculate in other processes part of the 760 | operations. This must be taking care of dependencies as `WITH` may depend on 761 | previous queries, or not. 762 | 763 | * Allow user functions 764 | -------------------------------------------------------------------------------- /config/config.exs: -------------------------------------------------------------------------------- 1 | # This file is responsible for configuring your application 2 | # and its dependencies with the aid of the Mix.Config module. 3 | use Mix.Config 4 | 5 | # This configuration is loaded before any dependency and is restricted 6 | # to this project. If another project depends on this project, this 7 | # file won't be loaded nor affect the parent project. For this reason, 8 | # if you want to provide default values for your application for 9 | # 3rd-party users, it should be done in your "mix.exs" file. 10 | 11 | # You can configure your application as: 12 | # 13 | # config :exosql, key: :value 14 | # 15 | # and access this configuration in your application as: 16 | # 17 | # Application.get_env(:exosql, :key) 18 | # 19 | # You can also configure a 3rd-party app: 20 | # 21 | # config :logger, level: :info 22 | # 23 | 24 | # It is also possible to import configuration files, relative to this 25 | # directory. For example, you can emulate configuration per environment 26 | # by uncommenting the line below and defining dev.exs, test.exs and such. 27 | # Configuration from the imported file will override the ones defined 28 | # here (which is why it is important to import them last). 29 | # 30 | # import_config "#{Mix.env}.exs" 31 | -------------------------------------------------------------------------------- /lib/builtins.ex: -------------------------------------------------------------------------------- 1 | require Logger 2 | require Integer 3 | 4 | defmodule ExoSQL.Builtins do 5 | @moduledoc """ 6 | Builtin functions. 7 | 8 | There are two categories, normal functions and aggregate functions. Aggregate 9 | functions receive as first parameter a ExoSQL.Result with a full table, 10 | and the rest of parameters are the function calling parameters, unsolved. 11 | 12 | These expressions must be first simplified with 13 | `ExoSQL.Expr.simplify` and then executed on the rows with 14 | `ExoSQL.Expr.run_expr`. 15 | """ 16 | import ExoSQL.Utils, only: [to_number: 1, to_float: 1] 17 | 18 | @functions %{ 19 | "bool" => {ExoSQL.Builtins, :bool}, 20 | "lower" => {ExoSQL.Builtins, :lower}, 21 | "upper" => {ExoSQL.Builtins, :upper}, 22 | "split" => {ExoSQL.Builtins, :split}, 23 | "join" => {ExoSQL.Builtins, :join}, 24 | "trim" => {ExoSQL.Builtins, :trim}, 25 | "to_string" => {ExoSQL.Builtins, :to_string_}, 26 | "to_datetime" => {ExoSQL.Builtins, :to_datetime}, 27 | "to_timestamp" => {ExoSQL.Builtins, :to_timestamp}, 28 | "to_number" => {ExoSQL.Utils, :to_number!}, 29 | "substr" => {ExoSQL.Builtins, :substr}, 30 | "now" => {ExoSQL.Builtins, :now}, 31 | "strftime" => {ExoSQL.Builtins, :strftime}, 32 | "format" => {ExoSQL.Builtins, :format}, 33 | "debug" => {ExoSQL.Builtins, :debug}, 34 | "width_bucket" => {ExoSQL.Builtins, :width_bucket}, 35 | "generate_series" => {ExoSQL.Builtins, :generate_series}, 36 | "urlparse" => {ExoSQL.Builtins, :urlparse}, 37 | "jp" => {ExoSQL.Builtins, :jp}, 38 | "json" => {ExoSQL.Builtins, :json}, 39 | "unnest" => {ExoSQL.Builtins, :unnest}, 40 | "regex" => {ExoSQL.Builtins, :regex}, 41 | "regex_all" => {ExoSQL.Builtins, :regex_all}, 42 | "random" => {ExoSQL.Builtins, :random}, 43 | "randint" => {ExoSQL.Builtins, :randint}, 44 | "range" => {ExoSQL.Builtins, :range}, 45 | "greatest" => {ExoSQL.Builtins, :greatest}, 46 | "lowest" => {ExoSQL.Builtins, :lowest}, 47 | "coalesce" => {ExoSQL.Builtins, :coalesce}, 48 | "nullif" => {ExoSQL.Builtins, :nullif}, 49 | "datediff" => {ExoSQL.DateTime, :datediff}, 50 | 51 | ## Math 52 | "round" => {ExoSQL.Builtins, :round}, 53 | "trunc" => {ExoSQL.Builtins, :trunc}, 54 | "floor" => {ExoSQL.Builtins, :floor_}, 55 | "ceil" => {ExoSQL.Builtins, :ceil_}, 56 | "power" => {ExoSQL.Builtins, :power}, 57 | "sqrt" => {ExoSQL.Builtins, :sqrt}, 58 | "log" => {ExoSQL.Builtins, :log}, 59 | "ln" => {ExoSQL.Builtins, :ln}, 60 | "abs" => {ExoSQL.Builtins, :abs}, 61 | "mod" => {ExoSQL.Builtins, :mod}, 62 | "sign" => {ExoSQL.Builtins, :sign}, 63 | 64 | ## Aggregates 65 | "count" => {ExoSQL.Builtins, :count}, 66 | "sum" => {ExoSQL.Builtins, :sum}, 67 | "avg" => {ExoSQL.Builtins, :avg}, 68 | "max" => {ExoSQL.Builtins, :max_}, 69 | "min" => {ExoSQL.Builtins, :min_} 70 | } 71 | def call_function({mod, fun, name}, params) do 72 | try do 73 | apply(mod, fun, params) 74 | rescue 75 | _excp -> 76 | # Logger.debug("Exception #{inspect _excp}: #{inspect {{mod, fun}, params}}") 77 | throw({:function, {name, params}}) 78 | end 79 | end 80 | 81 | def call_function(name, params) do 82 | case @functions[name] do 83 | nil -> 84 | raise {:unknown_function, name} 85 | 86 | {mod, fun} -> 87 | try do 88 | apply(mod, fun, params) 89 | rescue 90 | _excp -> 91 | # Logger.debug("Exception #{inspect(_excp)}: #{inspect({{mod, fun}, params})}") 92 | throw({:function, {name, params}}) 93 | end 94 | end 95 | end 96 | 97 | def cant_simplify(f) do 98 | is_aggregate(f) or f in ["random", "randint", "debug"] 99 | end 100 | 101 | def is_projectable(f) do 102 | f in ["unnest", "generate_series"] 103 | end 104 | 105 | def round(nil), do: nil 106 | 107 | def round(n) do 108 | {:ok, n} = to_float(n) 109 | 110 | Kernel.round(n) 111 | end 112 | 113 | def round(n, 0) do 114 | {:ok, n} = to_float(n) 115 | 116 | Kernel.round(n) 117 | end 118 | 119 | def round(n, "0") do 120 | {:ok, n} = to_float(n) 121 | 122 | Kernel.round(n) 123 | end 124 | 125 | def round(n, r) do 126 | {:ok, n} = to_float(n) 127 | {:ok, r} = to_number(r) 128 | 129 | Float.round(n, r) 130 | end 131 | 132 | def power(nil, _), do: nil 133 | def power(_, nil), do: nil 134 | def power(_, 0), do: 1 135 | 136 | # To allow big power. From https://stackoverflow.com/questions/32024156/how-do-i-raise-a-number-to-a-power-in-elixir#32024157 137 | def power(x, n) when Integer.is_odd(n) do 138 | {:ok, x} = to_number(x) 139 | {:ok, n} = to_number(n) 140 | x * :math.pow(x, n - 1) 141 | end 142 | 143 | def power(x, n) do 144 | {:ok, x} = to_number(x) 145 | {:ok, n} = to_number(n) 146 | result = :math.pow(x, n / 2) 147 | result * result 148 | end 149 | 150 | def sqrt(nil), do: nil 151 | 152 | def sqrt(n) do 153 | {:ok, n} = to_number(n) 154 | :math.sqrt(n) 155 | end 156 | 157 | def log(nil), do: nil 158 | 159 | def log(n) do 160 | {:ok, n} = to_number(n) 161 | :math.log10(n) 162 | end 163 | 164 | def ln(nil), do: nil 165 | 166 | def ln(n) do 167 | {:ok, n} = to_number(n) 168 | :math.log(n) 169 | end 170 | 171 | def abs(nil), do: nil 172 | 173 | def abs(n) do 174 | {:ok, n} = to_number(n) 175 | :erlang.abs(n) 176 | end 177 | 178 | def mod(nil, _), do: nil 179 | def mod(_, nil), do: nil 180 | 181 | def mod(n, m) do 182 | {:ok, n} = to_number(n) 183 | {:ok, m} = to_number(m) 184 | :math.fmod(n, m) 185 | end 186 | 187 | def sign(nil), do: nil 188 | 189 | def sign(n) do 190 | {:ok, n} = to_number(n) 191 | 192 | cond do 193 | n < 0 -> -1 194 | n == 0 -> 0 195 | true -> 1 196 | end 197 | end 198 | 199 | def random(), do: :rand.uniform() 200 | 201 | def randint(max_) do 202 | :rand.uniform(max_ - 1) 203 | end 204 | 205 | def randint(min_, max_) do 206 | :rand.uniform(max_ - min_ - 1) + min_ - 1 207 | end 208 | 209 | def bool(nil), do: false 210 | def bool(0), do: false 211 | def bool(""), do: false 212 | def bool(false), do: false 213 | def bool(_), do: true 214 | 215 | def lower(nil), do: nil 216 | def lower({:range, {a, _b}}), do: a 217 | def lower(s), do: String.downcase(s) 218 | 219 | def upper(nil), do: nil 220 | def upper({:range, {_a, b}}), do: b 221 | def upper(s), do: String.upcase(s) 222 | 223 | def to_string_(%DateTime{} = d), do: DateTime.to_iso8601(d) 224 | 225 | def to_string_(%{} = d) do 226 | {:ok, e} = Poison.encode(d) 227 | e 228 | end 229 | 230 | def to_string_(s), do: to_string(s) 231 | 232 | def now(), do: Timex.local() 233 | def now(tz), do: Timex.now(tz) 234 | def to_datetime(nil), do: nil 235 | def to_datetime(other), do: ExoSQL.DateTime.to_datetime(other) 236 | def to_datetime(other, mod), do: ExoSQL.DateTime.to_datetime(other, mod) 237 | def to_timestamp(%DateTime{} = d), do: DateTime.to_unix(d) 238 | 239 | def substr(nil, _skip, _len) do 240 | "" 241 | end 242 | 243 | def substr(str, skip, len) do 244 | # force string 245 | str = to_string_(str) 246 | 247 | {:ok, skip} = to_number(skip) 248 | {:ok, len} = to_number(len) 249 | 250 | if len < 0 do 251 | String.slice(str, skip, max(0, String.length(str) + len - skip)) 252 | else 253 | String.slice(str, skip, len) 254 | end 255 | end 256 | 257 | def substr(str, skip) do 258 | # A upper limit on what to return, should be enought 259 | substr(str, skip, 10_000) 260 | end 261 | 262 | def trim(nil), do: nil 263 | 264 | def trim(str) do 265 | String.trim(str) 266 | end 267 | 268 | def join(str), do: join(str, ",") 269 | 270 | def join(nil, _), do: nil 271 | 272 | def join(str, sep) do 273 | Enum.join(str, sep) 274 | end 275 | 276 | def split(nil, _sep), do: [] 277 | 278 | def split(str, sep) do 279 | String.split(str, sep) 280 | end 281 | 282 | def split(str) do 283 | split(str, [", ", ",", " "]) 284 | end 285 | 286 | @doc ~S""" 287 | Convert datetime to string. 288 | 289 | If no format is given, it is as to_string, which returns the ISO 8601. 290 | Format allows all substitutions from 291 | [Timex.format](https://hexdocs.pm/timex/Timex.Format.DateTime.Formatters.Strftime.html), 292 | for example: 293 | 294 | %d day of month: 00 295 | %H hour: 00-24 296 | %m month: 01-12 297 | %M minute: 00-59 298 | %s seconds since 1970-01-01 299 | %S seconds: 00-59 300 | %Y year: 0000-9999 301 | %i ISO 8601 format 302 | %V Week number 303 | %% % 304 | """ 305 | def strftime(%DateTime{} = d), do: to_string_(d) 306 | def strftime(%DateTime{} = d, format), do: ExoSQL.DateTime.strftime(d, format) 307 | def strftime(other, format), do: strftime(to_datetime(other), format) 308 | 309 | @doc ~S""" 310 | sprintf style formatting. 311 | 312 | Known interpolations: 313 | 314 | %d - Integer 315 | %f - Float, 2 digits 316 | %.Nf - Float N digits 317 | %k - integer with k, M sufix 318 | %.k - float with k, M sufix, uses float part 319 | """ 320 | def format(str), do: ExoSQL.Format.format(str, []) 321 | 322 | def format(str, args) when is_list(args) do 323 | ExoSQL.Format.format(str, args) 324 | end 325 | 326 | @doc ~S""" 327 | Very simple sprintf formatter. Knows this formats: 328 | 329 | * %% 330 | * %s 331 | * %d 332 | * %f (only two decimals) 333 | * %.{ndec}f 334 | """ 335 | def format(str, arg1), do: format(str, [arg1]) 336 | def format(str, arg1, arg2), do: format(str, [arg1, arg2]) 337 | def format(str, arg1, arg2, arg3), do: format(str, [arg1, arg2, arg3]) 338 | def format(str, arg1, arg2, arg3, arg4), do: format(str, [arg1, arg2, arg3, arg4]) 339 | def format(str, arg1, arg2, arg3, arg4, arg5), do: format(str, [arg1, arg2, arg3, arg4, arg5]) 340 | 341 | def format(str, arg1, arg2, arg3, arg4, arg5, arg6), 342 | do: format(str, [arg1, arg2, arg3, arg4, arg5, arg6]) 343 | 344 | def format(str, arg1, arg2, arg3, arg4, arg5, arg6, arg7), 345 | do: format(str, [arg1, arg2, arg3, arg4, arg5, arg6, arg7]) 346 | 347 | def format(str, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8), 348 | do: format(str, [arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8]) 349 | 350 | def format(str, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9), 351 | do: format(str, [arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9]) 352 | 353 | @doc ~S""" 354 | Print some value to the log 355 | """ 356 | def debug(str) do 357 | Logger.debug("SQL DEBUG: #{inspect(str)}") 358 | str 359 | end 360 | 361 | @doc ~S""" 362 | Returns to which bucket it belongs. 363 | 364 | Only numbers, but datetimes can be transformed to unix datetime. 365 | """ 366 | def width_bucket(n, start_, end_, nbuckets) do 367 | import ExoSQL.Utils, only: [to_float!: 1, to_number!: 1] 368 | 369 | n = to_float!(n) 370 | start_ = to_float!(start_) 371 | end_ = to_float!(end_) 372 | nbuckets = to_number!(nbuckets) 373 | 374 | bucket = (n - start_) * nbuckets / (end_ - start_) 375 | bucket = bucket |> Kernel.round() 376 | 377 | cond do 378 | bucket < 0 -> 0 379 | bucket >= nbuckets -> nbuckets - 1 380 | true -> bucket 381 | end 382 | end 383 | 384 | @doc ~S""" 385 | Performs a regex match 386 | 387 | May return a list of groups, or a dict with named groups, depending on 388 | the regex. 389 | 390 | As an optional third parameter it performs a jp query. 391 | 392 | Returns NULL if no match (which is falsy, so can be used for expressions) 393 | """ 394 | def regex(str, regexs) do 395 | # slow. Should have been precompiled (simplify) 396 | regex_real(str, regexs) 397 | end 398 | 399 | def regex(str, regexs, query) do 400 | jp(regex_real(str, regexs), query) 401 | end 402 | 403 | def regex_real(str, {regex, captures}) do 404 | if captures do 405 | Regex.named_captures(regex, str) 406 | else 407 | Regex.run(regex, str) 408 | end 409 | end 410 | 411 | def regex_real(str, regexs) when is_binary(regexs) do 412 | captures = String.contains?(regexs, "(?<") 413 | regex = Regex.compile!(regexs) 414 | regex_real(str, {regex, captures}) 415 | end 416 | 417 | @doc ~S""" 418 | Performs a regex scan 419 | 420 | Returns all the matches, if groups are used, then its a list of groups matching. 421 | 422 | As an optional third parameter it performs a jp query. 423 | 424 | Returns NULL if no match (which is falsy, so can be used for expressions) 425 | """ 426 | def regex_all(str, regexs) do 427 | # slow. Should have been precompiled (simplify) 428 | regex_all_real(str, regexs) 429 | end 430 | 431 | def regex_all(str, regexs, query) do 432 | regex_all_real(str, regexs) 433 | |> Enum.map(&jp(&1, query)) 434 | end 435 | 436 | def regex_all_real(str, regexs) when is_binary(regexs) do 437 | regex = Regex.compile!(regexs) 438 | regex_all_real(str, regex) 439 | end 440 | 441 | def regex_all_real(str, regex) do 442 | Regex.scan(regex, str) 443 | end 444 | 445 | @doc ~S""" 446 | Generates a table with the series of numbers as given. Use for histograms 447 | without holes. 448 | """ 449 | def generate_series(end_), do: generate_series(1, end_, 1) 450 | def generate_series(start_, end_), do: generate_series(start_, end_, 1) 451 | 452 | def generate_series(%DateTime{} = start_, %DateTime{} = end_, days) when is_number(days) do 453 | generate_series(start_, end_, "#{days}D") 454 | end 455 | 456 | def generate_series(%DateTime{} = start_, %DateTime{} = end_, mod) when is_binary(mod) do 457 | duration = 458 | case ExoSQL.DateTime.Duration.parse(mod) do 459 | {:error, other} -> 460 | throw({:error, other}) 461 | 462 | %ExoSQL.DateTime.Duration{seconds: 0, days: 0, months: 0, years: 0} -> 463 | throw({:error, :invalid_duration}) 464 | 465 | {:ok, other} -> 466 | other 467 | end 468 | 469 | cmp = 470 | if ExoSQL.DateTime.Duration.is_negative(duration) do 471 | :lt 472 | else 473 | :gt 474 | end 475 | 476 | rows = 477 | ExoSQL.Utils.generate(start_, fn value -> 478 | cmpr = DateTime.compare(value, end_) 479 | 480 | if cmpr == cmp do 481 | :halt 482 | else 483 | next = ExoSQL.DateTime.Duration.datetime_add(value, duration) 484 | {[value], next} 485 | end 486 | end) 487 | 488 | %ExoSQL.Result{ 489 | columns: [{:tmp, :tmp, "generate_series"}], 490 | rows: rows 491 | } 492 | end 493 | 494 | def generate_series(start_, end_, step) 495 | when is_number(start_) and is_number(end_) and is_number(step) do 496 | if step == 0 do 497 | raise ArgumentError, "Step invalid. Will never reach end." 498 | end 499 | 500 | if step < 0 and start_ < end_ do 501 | raise ArgumentError, "Start, end and step invalid. Will never reach end." 502 | end 503 | 504 | if step > 0 and start_ > end_ do 505 | raise ArgumentError, "Start, end and step invalid. Will never reach end." 506 | end 507 | 508 | %ExoSQL.Result{ 509 | columns: [{:tmp, :tmp, "generate_series"}], 510 | rows: generate_series_range(start_, end_, step) 511 | } 512 | end 513 | 514 | def generate_series(start_, end_, step) do 515 | import ExoSQL.Utils, only: [to_number!: 1, to_number: 1] 516 | 517 | # there are two options: numbers or dates. Check if I can convert the start_ to a number 518 | # and if so, do the generate_series for numbers 519 | 520 | case to_number(start_) do 521 | {:ok, start_} -> 522 | generate_series(start_, to_number!(end_), to_number!(step)) 523 | 524 | # maybe a date 525 | {:error, _} -> 526 | generate_series(to_datetime(start_), to_datetime(end_), step) 527 | end 528 | end 529 | 530 | defp generate_series_range(current, stop, step) do 531 | cond do 532 | step > 0 and current > stop -> 533 | [] 534 | 535 | step < 0 and current < stop -> 536 | [] 537 | 538 | true -> 539 | [[current] | generate_series_range(current + step, stop, step)] 540 | end 541 | end 542 | 543 | @doc ~S""" 544 | Parses an URL and return some part of it. 545 | 546 | If not what is provided, returns a JSON object with: 547 | * host 548 | * port 549 | * scheme 550 | * path 551 | * query 552 | * user 553 | 554 | If what is passed, it performs a JSON Pointer search (jp function). 555 | 556 | It must receive a url with scheme://server or the result may not be well 557 | formed. 558 | 559 | For example, for emails, just use "email://connect@serverboards.io" or 560 | similar. 561 | 562 | """ 563 | def urlparse(url), do: urlparse(url, nil) 564 | def urlparse(nil, what), do: urlparse("", what) 565 | 566 | def urlparse(url, what) do 567 | parsed = URI.parse(url) 568 | 569 | query = 570 | case parsed.query do 571 | nil -> nil 572 | q -> URI.decode_query(q) 573 | end 574 | 575 | json = %{ 576 | "host" => parsed.host, 577 | "port" => parsed.port, 578 | "scheme" => parsed.scheme, 579 | "path" => parsed.path, 580 | "query" => query, 581 | "user" => parsed.userinfo, 582 | "domain" => get_domain(parsed.host) 583 | } 584 | 585 | if what do 586 | jp(json, what) 587 | else 588 | json 589 | end 590 | end 591 | 592 | @doc ~S""" 593 | Gets the domain from the domain name. 594 | 595 | This means "google" from "www.google.com" or "google" from "www.google.co.uk" 596 | 597 | The algorithm disposes the tld (.uk) and the skips unwanted names (.co). 598 | Returns the first thats rest, or a default that is originally the full domain 599 | name or then each disposed part. 600 | """ 601 | def get_domain(nil), do: nil 602 | 603 | def get_domain(hostname) do 604 | [_tld | rparts] = hostname |> String.split(".") |> Enum.reverse() 605 | 606 | # always remove last part 607 | get_domainr(rparts, hostname) 608 | end 609 | 610 | # list of strings that are never domains. 611 | defp get_domainr([head | rest], candidate) do 612 | nodomains = ~w(com org net www co) 613 | 614 | if head in nodomains do 615 | get_domainr(rest, candidate) 616 | else 617 | head 618 | end 619 | end 620 | 621 | defp get_domainr([], candidate), do: candidate 622 | 623 | @doc ~S""" 624 | Performs a JSON Pointer search on JSON data. 625 | 626 | It just uses / to separate keys. 627 | """ 628 | def jp(nil, _), do: nil 629 | def jp(json, idx) when is_list(json) and is_number(idx), do: Enum.at(json, idx) 630 | def jp(json, str) when is_binary(str), do: jp(json, String.split(str, "/")) 631 | 632 | def jp(json, [head | rest]) when is_list(json) do 633 | n = ExoSQL.Utils.to_number!(head) 634 | jp(Enum.at(json, n), rest) 635 | end 636 | 637 | def jp(json, ["" | rest]), do: jp(json, rest) 638 | def jp(json, [head | rest]), do: jp(Map.get(json, head, nil), rest) 639 | def jp(json, []), do: json 640 | 641 | @doc ~S""" 642 | Convert from a string to a JSON object 643 | """ 644 | def json(nil), do: nil 645 | 646 | def json(str) when is_binary(str) do 647 | Poison.decode!(str) 648 | end 649 | 650 | def json(js) when is_map(js), do: js 651 | def json(arr) when is_list(arr), do: arr 652 | 653 | @doc ~S""" 654 | Extracts some keys from each value on an array and returns the array of 655 | those values 656 | """ 657 | def unnest(array) do 658 | array = json(array) || [] 659 | 660 | %ExoSQL.Result{ 661 | columns: [{:tmp, :tmp, "unnest"}], 662 | rows: Enum.map(array, &[&1]) 663 | } 664 | end 665 | 666 | def unnest(array, cols) when is_list(cols) do 667 | array = json(array) || [] 668 | 669 | rows = 670 | Enum.map(array, fn row -> 671 | Enum.map(cols, &Map.get(row, &1)) 672 | end) 673 | 674 | columns = Enum.map(cols, &{:tmp, :tmp, &1}) 675 | 676 | %ExoSQL.Result{ 677 | columns: columns, 678 | rows: rows 679 | } 680 | end 681 | 682 | def unnest(array, col1), do: unnest(array, [col1]) 683 | def unnest(array, col1, col2), do: unnest(array, [col1, col2]) 684 | def unnest(array, col1, col2, col3), do: unnest(array, [col1, col2, col3]) 685 | def unnest(array, col1, col2, col3, col4), do: unnest(array, [col1, col2, col3, col4]) 686 | 687 | def unnest(array, col1, col2, col3, col4, col5), 688 | do: unnest(array, [col1, col2, col3, col4, col5]) 689 | 690 | def unnest(array, col1, col2, col3, col4, col5, col5), 691 | do: unnest(array, [col1, col2, col3, col4, col5, col5]) 692 | 693 | def unnest(array, col1, col2, col3, col4, col5, col5, col6), 694 | do: unnest(array, [col1, col2, col3, col4, col5, col5, col6]) 695 | 696 | def unnest(array, col1, col2, col3, col4, col5, col5, col6, col7), 697 | do: unnest(array, [col1, col2, col3, col4, col5, col5, col6, col7]) 698 | 699 | def unnest(array, col1, col2, col3, col4, col5, col5, col6, col7, col8), 700 | do: unnest(array, [col1, col2, col3, col4, col5, col5, col6, col7, col8]) 701 | 702 | @doc ~S""" 703 | Creates a range, which can later be used in: 704 | 705 | * `IN` -- Subset / element contains 706 | * `*` -- Interesection -> nil if no intersection, the intersected range if any. 707 | """ 708 | def range(a, b), do: {:range, {a, b}} 709 | 710 | @doc ~S""" 711 | Get the greatest of arguments 712 | """ 713 | def greatest(a, nil), do: a 714 | def greatest(nil, b), do: b 715 | 716 | def greatest(a, b) do 717 | if a > b do 718 | a 719 | else 720 | b 721 | end 722 | end 723 | 724 | def greatest(a, b, c), do: Enum.reduce([a, b, c], nil, &greatest/2) 725 | def greatest(a, b, c, d), do: Enum.reduce([a, b, c, d], nil, &greatest/2) 726 | def greatest(a, b, c, d, e), do: Enum.reduce([a, b, c, d, e], nil, &greatest/2) 727 | 728 | @doc ~S""" 729 | Get the least of arguments. 730 | 731 | Like min, for not for aggregations. 732 | """ 733 | def least(a, nil), do: a 734 | def least(nil, b), do: b 735 | 736 | def least(a, b) do 737 | if a < b do 738 | a 739 | else 740 | b 741 | end 742 | end 743 | 744 | def least(a, b, c), do: Enum.reduce([a, b, c], nil, &least/2) 745 | def least(a, b, c, d), do: Enum.reduce([a, b, c, d], nil, &least/2) 746 | def least(a, b, c, d, e), do: Enum.reduce([a, b, c, d, e], nil, &least/2) 747 | 748 | @doc ~S""" 749 | Returns the first not NULL 750 | """ 751 | def coalesce(a, b), do: Enum.find([a, b], &(&1 != nil)) 752 | def coalesce(a, b, c), do: Enum.find([a, b, c], &(&1 != nil)) 753 | def coalesce(a, b, c, d), do: Enum.find([a, b, c, d], &(&1 != nil)) 754 | def coalesce(a, b, c, d, e), do: Enum.find([a, b, c, d, e], &(&1 != nil)) 755 | 756 | @doc ~S""" 757 | Returns NULL if both equal, first argument if not. 758 | """ 759 | def nullif(a, a), do: nil 760 | def nullif(a, _), do: a 761 | 762 | def floor_(n) when is_float(n), do: trunc(Float.floor(n)) 763 | def floor_(n) when is_number(n), do: n 764 | def floor_(n), do: floor_(ExoSQL.Utils.to_number!(n)) 765 | 766 | def ceil_(n) when is_float(n), do: trunc(Float.ceil(n)) 767 | def ceil_(n) when is_number(n), do: n 768 | def ceil_(n), do: ceil_(ExoSQL.Utils.to_number!(n)) 769 | 770 | ### Aggregate functions 771 | def is_aggregate(x) do 772 | x in ["count", "avg", "sum", "max", "min"] 773 | end 774 | 775 | def count(data, {:lit, '*'}) do 776 | Enum.count(data.rows) 777 | end 778 | 779 | def count(data, {:distinct, expr}) do 780 | expr = ExoSQL.Expr.simplify(expr, %{columns: data.columns}) 781 | 782 | Enum.reduce(data.rows, MapSet.new(), fn row, acc -> 783 | case ExoSQL.Expr.run_expr(expr, %{row: row}) do 784 | nil -> acc 785 | val -> MapSet.put(acc, val) 786 | end 787 | end) 788 | |> Enum.count() 789 | end 790 | 791 | def count(data, expr) do 792 | expr = ExoSQL.Expr.simplify(expr, %{columns: data.columns}) 793 | 794 | Enum.reduce(data.rows, 0, fn row, acc -> 795 | case ExoSQL.Expr.run_expr(expr, %{row: row}) do 796 | nil -> acc 797 | _other -> 1 + acc 798 | end 799 | end) 800 | end 801 | 802 | def avg(data, expr) do 803 | # Logger.debug("Avg of #{inspect data} by #{inspect expr}") 804 | if data.columns == [] do 805 | nil 806 | else 807 | sum(data, expr) / count(data, {:lit, '*'}) 808 | end 809 | end 810 | 811 | def sum(data, expr) do 812 | # Logger.debug("Sum of #{inspect data} by #{inspect expr}") 813 | expr = ExoSQL.Expr.simplify(expr, %{columns: data.columns}) 814 | # Logger.debug("Simplified expression #{inspect expr}") 815 | Enum.reduce(data.rows, 0, fn row, acc -> 816 | n = ExoSQL.Expr.run_expr(expr, %{row: row}) 817 | 818 | n = 819 | case ExoSQL.Utils.to_number(n) do 820 | {:ok, n} -> n 821 | {:error, nil} -> 0 822 | end 823 | 824 | acc + n 825 | end) 826 | end 827 | 828 | def max_(data, expr) do 829 | expr = ExoSQL.Expr.simplify(expr, %{columns: data.columns}) 830 | 831 | Enum.reduce(data.rows, nil, fn row, acc -> 832 | n = ExoSQL.Expr.run_expr(expr, %{row: row}) 833 | {acc, n} = ExoSQL.Expr.match_types(acc, n) 834 | 835 | if ExoSQL.Expr.is_greater(acc, n) do 836 | acc 837 | else 838 | n 839 | end 840 | end) 841 | end 842 | 843 | def min_(data, expr) do 844 | expr = ExoSQL.Expr.simplify(expr, %{columns: data.columns}) 845 | 846 | Enum.reduce(data.rows, nil, fn row, acc -> 847 | n = ExoSQL.Expr.run_expr(expr, %{row: row}) 848 | {acc, n} = ExoSQL.Expr.match_types(acc, n) 849 | 850 | res = 851 | if acc != nil and ExoSQL.Expr.is_greater(n, acc) do 852 | acc 853 | else 854 | n 855 | end 856 | 857 | res 858 | end) 859 | end 860 | 861 | ## Simplications. 862 | 863 | # Precompile regex 864 | def simplify("format", [{:lit, format} | rest]) when is_binary(format) do 865 | compiled = ExoSQL.Format.compile_format(format) 866 | # Logger.debug("Simplify format: #{inspect compiled}") 867 | simplify("format", [{:lit, compiled} | rest]) 868 | end 869 | 870 | def simplify("regex", [str, {:lit, regexs}]) when is_binary(regexs) do 871 | regex = Regex.compile!(regexs) 872 | captures = String.contains?(regexs, "(?<") 873 | 874 | simplify("regex", [str, {:lit, regex}, {:lit, captures}]) 875 | end 876 | 877 | def simplify("regex", [str, {:lit, regexs}, {:lit, query}]) when is_binary(regexs) do 878 | regex = Regex.compile!(regexs) 879 | captures = String.contains?(regexs, "(?<") 880 | 881 | # this way jq can be simplified too 882 | params = [ 883 | simplify("regex", [str, {:lit, regex}, {:lit, captures}]), 884 | {:lit, query} 885 | ] 886 | 887 | simplify("jp", params) 888 | end 889 | 890 | def simplify("jp", [json, {:lit, path}]) when is_binary(path) do 891 | # Logger.debug("JP #{inspect json}") 892 | simplify("jp", [json, {:lit, String.split(path, "/")}]) 893 | end 894 | 895 | def simplify("random", []), do: {:fn, {{ExoSQL.Builtins, :random, "random"}, []}} 896 | def simplify("randint", params), do: {:fn, {{ExoSQL.Builtins, :randint, "randint"}, params}} 897 | 898 | # default: convert to {mod fun name} tuple 899 | def simplify(name, params) when is_binary(name) do 900 | # Logger.debug("Simplify #{inspect name} #{inspect params}") 901 | if not is_aggregate(name) and Enum.all?(params, &is_lit(&1)) do 902 | # Logger.debug("All is literal for #{inspect {name, params}}.. just do it once") 903 | params = Enum.map(params, fn {:lit, n} -> n end) 904 | ret = ExoSQL.Builtins.call_function(name, params) 905 | {:lit, ret} 906 | else 907 | case @functions[name] do 908 | nil -> 909 | throw({:unknown_function, name}) 910 | 911 | {mod, fun} -> 912 | {:fn, {{mod, fun, name}, params}} 913 | end 914 | end 915 | end 916 | 917 | def simplify(modfun, params), do: {:fn, {modfun, params}} 918 | 919 | def is_lit({:lit, '*'}), do: false 920 | def is_lit({:lit, _n}), do: true 921 | def is_lit(_), do: false 922 | end 923 | -------------------------------------------------------------------------------- /lib/datetime.ex: -------------------------------------------------------------------------------- 1 | require Logger 2 | 3 | defmodule ExoSQL.DateTime do 4 | @moduledoc """ 5 | Helpers for datetime 6 | """ 7 | 8 | def strftime(dt, format) do 9 | dt = to_datetime(dt) 10 | 11 | # Simplifications, %i from sqlite is supported 12 | format = 13 | format 14 | |> String.replace("%i", "%FT%T%z") 15 | 16 | Timex.format!(dt, format, :strftime) 17 | |> String.replace("+0000", "Z") 18 | end 19 | 20 | def to_datetime(n) when is_number(n) do 21 | {:ok, dt} = DateTime.from_unix(n) 22 | dt 23 | end 24 | 25 | def to_datetime(n) when is_binary(n) do 26 | if String.contains?(n, "-") do 27 | case String.length(n) do 28 | 10 -> 29 | n = "#{n} 00:00:00Z" 30 | {:ok, td, 0} = DateTime.from_iso8601(n) 31 | td 32 | 33 | 16 -> 34 | n = "#{n}:00Z" 35 | {:ok, td, 0} = DateTime.from_iso8601(n) 36 | td 37 | 38 | 19 -> 39 | {:ok, td, 0} = DateTime.from_iso8601(n <> "Z") 40 | td 41 | 42 | 20 -> 43 | {:ok, td, 0} = DateTime.from_iso8601(n) 44 | td 45 | 46 | 24 -> 47 | {:ok, td, _} = DateTime.from_iso8601(n) 48 | td 49 | 50 | 25 -> 51 | {:ok, td, _} = DateTime.from_iso8601(n) 52 | td 53 | end 54 | else 55 | {:ok, unixtime} = ExoSQL.Utils.to_number(n) 56 | # Logger.debug("To datetime #{inspect unixtime}") 57 | to_datetime(unixtime) 58 | end 59 | end 60 | 61 | def to_datetime(%DateTime{} = d), do: d 62 | 63 | def to_datetime(other) do 64 | raise ArgumentError, message: "cant convert #{inspect(other)} to date" 65 | end 66 | 67 | def to_datetime(dt, "-" <> _mod = orig) do 68 | dt = to_datetime(dt) 69 | ExoSQL.DateTime.Duration.datetime_add(dt, orig) 70 | end 71 | 72 | def to_datetime(dt, "+" <> _mod = orig) do 73 | dt = to_datetime(dt) 74 | ExoSQL.DateTime.Duration.datetime_add(dt, orig) 75 | end 76 | 77 | def to_datetime(dt, tz) do 78 | dt = to_datetime(dt) 79 | 80 | Timex.to_datetime(dt, tz) 81 | end 82 | 83 | @units %{ 84 | "minutes" => 60, 85 | "hours" => 60 * 60, 86 | "days" => 24 * 60 * 60, 87 | "weeks" => 7 * 24 * 60 * 60 88 | } 89 | 90 | def datediff({:range, {start, end_}}), do: datediff(start, end_, "days") 91 | def datediff({:range, {start, end_}}, units), do: datediff(start, end_, units) 92 | def datediff(start, end_), do: datediff(start, end_, "days") 93 | 94 | def datediff(%DateTime{} = start, %DateTime{} = end_, "years") do 95 | if DateTime.compare(start, end_) == :gt do 96 | -datediff(end_, start, "years") 97 | else 98 | years = end_.year - start.year 99 | 100 | if end_.month < start.month do 101 | years - 1 102 | else 103 | years 104 | end 105 | end 106 | end 107 | 108 | def datediff(%DateTime{} = start, %DateTime{} = end_, "months") do 109 | if DateTime.compare(start, end_) == :gt do 110 | -datediff(end_, start, "months") 111 | else 112 | years = datediff(start, end_, "years") 113 | 114 | months = 115 | if start.month <= end_.month do 116 | end_.month - start.month 117 | else 118 | end_.month + 12 - start.month 119 | end 120 | 121 | months = years * 12 + months 122 | 123 | # If start day is > end day means that it counted one month more. 124 | if start.day > end_.day do 125 | months - 1 126 | else 127 | months 128 | end 129 | end 130 | end 131 | 132 | def datediff(%DateTime{} = start, %DateTime{} = end_, "seconds") do 133 | DateTime.diff(end_, start, :second) 134 | end 135 | 136 | def datediff(%DateTime{} = start, %DateTime{} = end_, unit) do 137 | case Map.get(@units, unit) do 138 | nil -> 139 | raise "Unknown unit #{inspect unit} for datediff #{inspect @units}" 140 | divider -> 141 | div(DateTime.diff(end_, start, :second), divider) 142 | end 143 | end 144 | 145 | def datediff(start, end_, unit) do 146 | start = to_datetime(start) 147 | end_ = to_datetime(end_) 148 | 149 | datediff(start, end_, unit) 150 | end 151 | end 152 | 153 | defmodule ExoSQL.DateTime.Duration do 154 | @moduledoc """ 155 | Parses and manipulates 8601 durations 156 | 157 | https://en.wikipedia.org/wiki/ISO_8601#Durations 158 | """ 159 | defstruct seconds: 0, days: 0, months: 0, years: 0 160 | 161 | def parse(<>) do 162 | parse(rest) 163 | end 164 | 165 | def parse(<>) do 166 | parse(rest) 167 | end 168 | 169 | def parse(<>) do 170 | case parse(rest) do 171 | {:ok, res} -> 172 | {:ok, 173 | %ExoSQL.DateTime.Duration{ 174 | seconds: -res.seconds, 175 | days: -res.days, 176 | months: -res.months, 177 | years: -res.years 178 | }} 179 | 180 | other -> 181 | other 182 | end 183 | end 184 | 185 | # no need for P 186 | def parse(rest) do 187 | case parse_split(rest) do 188 | {:ok, parsed} -> 189 | {:ok, parse_date(parsed, %ExoSQL.DateTime.Duration{})} 190 | 191 | other -> 192 | other 193 | end 194 | end 195 | 196 | def parse!(str) do 197 | case parse(str) do 198 | {:ok, ret} -> ret 199 | other -> throw(other) 200 | end 201 | end 202 | 203 | defp parse_split(str) do 204 | case Regex.run(~r/^(\d+[YMWDHS]|T)+$/, str) do 205 | nil -> 206 | {:error, :invalid_duration} 207 | 208 | _other -> 209 | ret = 210 | Regex.scan(~r/(\d+)([YMWDHS])|(T)/, str) 211 | |> Enum.map(fn 212 | [_, a, b] -> 213 | {n, ""} = Integer.parse(a) 214 | {n, b} 215 | 216 | ["T" | _rest] -> 217 | :t 218 | end) 219 | 220 | {:ok, ret} 221 | end 222 | end 223 | 224 | defp parse_date([], acc), do: acc 225 | 226 | defp parse_date([{n, "Y"} | rest], acc), 227 | do: parse_date(rest, Map.update(acc, :years, n, &(&1 + n))) 228 | 229 | defp parse_date([{n, "M"} | rest], acc), 230 | do: parse_date(rest, Map.update(acc, :months, n, &(&1 + n))) 231 | 232 | defp parse_date([{n, "W"} | rest], acc), 233 | do: parse_date(rest, Map.update(acc, :days, n, &(&1 + n * 7))) 234 | 235 | defp parse_date([{n, "D"} | rest], acc), 236 | do: parse_date(rest, Map.update(acc, :days, n, &(&1 + n))) 237 | 238 | defp parse_date([:t | rest], acc), do: parse_time(rest, acc) 239 | 240 | defp parse_time([], acc), do: acc 241 | 242 | defp parse_time([{n, "H"} | rest], acc), 243 | do: parse_time(rest, Map.update(acc, :seconds, n * 60 * 60, &(&1 + n * 60 * 60))) 244 | 245 | defp parse_time([{n, "M"} | rest], acc), 246 | do: parse_time(rest, Map.update(acc, :seconds, n * 60, &(&1 + n * 60))) 247 | 248 | defp parse_time([{n, "S"} | rest], acc), 249 | do: parse_time(rest, Map.update(acc, :seconds, n, &(&1 + n))) 250 | 251 | def is_negative(%ExoSQL.DateTime.Duration{ 252 | seconds: seconds, 253 | days: days, 254 | months: months, 255 | years: years 256 | }) do 257 | seconds < 0 or days < 0 or months < 0 or years < 0 258 | end 259 | 260 | def datetime_add(date, duration) when is_binary(duration) do 261 | datetime_add(date, parse!(duration)) 262 | end 263 | 264 | def datetime_add(date, %ExoSQL.DateTime.Duration{} = duration) do 265 | date = 266 | case duration.seconds do 267 | 0 -> 268 | date 269 | 270 | seconds -> 271 | Timex.shift(date, seconds: seconds) 272 | end 273 | 274 | date = 275 | case duration.days do 276 | 0 -> 277 | date 278 | 279 | days -> 280 | Timex.shift(date, days: days) 281 | end 282 | 283 | date = 284 | case duration.months do 285 | 0 -> 286 | date 287 | 288 | months -> 289 | Timex.shift(date, months: months) 290 | end 291 | 292 | date = 293 | case duration.years do 294 | 0 -> 295 | date 296 | 297 | years -> 298 | Timex.shift(date, years: years) 299 | end 300 | 301 | date 302 | end 303 | end 304 | -------------------------------------------------------------------------------- /lib/exosql.ex: -------------------------------------------------------------------------------- 1 | require Logger 2 | 3 | defmodule ExoSQL do 4 | @moduledoc """ 5 | Creates a Generic universal parser that can access many tabular databases, 6 | and perform SQL queries. 7 | 8 | The databases can be heterogenic, so you can perform searches mixing 9 | data from postgres, mysql, csv or Google Analytics. 10 | 11 | For example: 12 | 13 | ``` 14 | iex> {:ok, result} = ExoSQL.query( 15 | ...> "SELECT urls.url, status_code FROM urls INNER JOIN request ON request.url = urls.url", 16 | ...> %{ 17 | ...> "A" => {ExoSQL.Csv, path: "test/data/csv/"}, 18 | ...> "B" => {ExoSQL.HTTP, []} 19 | ...> }) 20 | ...> ExoSQL.format_result(result) 21 | ''' 22 | A.urls.url | B.request.status_code 23 | ------------------------------------- 24 | https://serverboards.io/e404 | 404 25 | http://www.facebook.com | 302 26 | https://serverboards.io | 200 27 | http://www.serverboards.io | 301 28 | ''' |> to_string 29 | 30 | ``` 31 | 32 | It also contains functions for all the steps of the process: 33 | `parse` |> `plan` |> `execute`. They can be useful for debugging pourposes. 34 | 35 | Finally there are helper functions as `explain` that prints out an explanation 36 | of the plan, and `format_result` for pretty printing results. 37 | """ 38 | 39 | defmodule Query do 40 | defstruct select: [], 41 | distinct: nil, 42 | crosstab: false, 43 | from: [], 44 | where: nil, 45 | groupby: nil, 46 | join: nil, 47 | orderby: [], 48 | limit: nil, 49 | offset: nil, 50 | union: nil, 51 | with: [] 52 | end 53 | 54 | defmodule Result do 55 | defstruct columns: [], 56 | rows: [] 57 | end 58 | 59 | def parse(sql, context), do: ExoSQL.Parser.parse(sql, context) 60 | def plan(parsed, context), do: ExoSQL.Planner.plan(parsed, context) 61 | def execute(plan, context), do: ExoSQL.Executor.execute(plan, context) 62 | 63 | def query(sql, context) do 64 | # Logger.debug(inspect sql) 65 | try do 66 | with {:ok, parsed} <- ExoSQL.Parser.parse(sql, context), 67 | {:ok, plan} <- ExoSQL.Planner.plan(parsed, context), 68 | {:ok, result} <- ExoSQL.Executor.execute(plan, context) do 69 | {:ok, result} 70 | end 71 | rescue 72 | err in MatchError -> 73 | case err.term do 74 | {:error, error} -> 75 | {:error, error} 76 | 77 | other -> 78 | {:error, {:match, other}} 79 | end 80 | 81 | any -> 82 | {:error, any} 83 | catch 84 | any -> {:error, any} 85 | end 86 | 87 | # Logger.debug("parsed #{inspect parsed, pretty: true}") 88 | # Logger.debug("planned #{inspect plan, pretty: true}") 89 | end 90 | 91 | def explain(sql, context) do 92 | Logger.info("Explain #{inspect(sql)}") 93 | {:ok, parsed} = ExoSQL.Parser.parse(sql, context) 94 | {:ok, plan} = ExoSQL.Planner.plan(parsed, context) 95 | Logger.info(inspect(plan, pretty: true)) 96 | end 97 | 98 | def format_result(res), do: ExoSQL.Utils.format_result(res) 99 | 100 | def schema("self", _context) do 101 | {:ok, ["tables"]} 102 | end 103 | 104 | # Hack to allow internal non database varaibles at context 105 | def schema("__" <> _rest, _context), do: {:ok, []} 106 | 107 | def schema(db, context) do 108 | {db, opts} = context[db] 109 | 110 | apply(db, :schema, [opts]) 111 | end 112 | 113 | def schema("self", "tables", _context) do 114 | {:ok, 115 | %{ 116 | columns: ["db", "table", "column"] 117 | }} 118 | end 119 | 120 | def schema(db, table, context) do 121 | case context[db] do 122 | {db, opts} -> 123 | apply(db, :schema, [opts, table]) 124 | 125 | nil -> 126 | raise "#{inspect({db, table})} not found at extractors #{inspect(Map.keys(context))}" 127 | end 128 | end 129 | 130 | @default_context %{ 131 | "A" => {ExoSQL.Csv, path: "test/data/csv/"}, 132 | "B" => {ExoSQL.HTTP, []} 133 | } 134 | def repl(context \\ @default_context) do 135 | input = IO.gets("exosql> ") |> String.trim() 136 | 137 | case input do 138 | "\q" -> 139 | :eof 140 | 141 | "exit" -> 142 | :eof 143 | 144 | "quit" -> 145 | :eof 146 | 147 | "" -> 148 | repl(context) 149 | 150 | _other -> 151 | case query(input, context) do 152 | {:ok, result} -> 153 | IO.puts(format_result(result)) 154 | 155 | {:error, err} -> 156 | Logger.error(inspect(err)) 157 | end 158 | 159 | repl(context) 160 | end 161 | end 162 | 163 | def debug_mode(context) do 164 | get_in(context, ["__vars__", "debug"]) 165 | end 166 | end 167 | -------------------------------------------------------------------------------- /lib/expr.ex: -------------------------------------------------------------------------------- 1 | require Logger 2 | 3 | defmodule ExoSQL.Expr do 4 | @moduledoc """ 5 | Expression executor. 6 | 7 | Requires a simplified expression from `ExoSQL.Expr.simplify` that converts 8 | columns names to column positions, and then use as: 9 | 10 | ``` 11 | iex> context = %{ row: [1,2,3,4,5] } 12 | iex> expr = {:op, {"*", {:column, 1}, {:column, 2}}} 13 | iex> ExoSQL.Expr.run_expr(expr, context) 14 | 6 15 | ``` 16 | """ 17 | 18 | import ExoSQL.Utils, only: [to_number: 1] 19 | 20 | def run_expr({:op, {"AND", op1, op2}}, context) do 21 | r1 = run_expr(op1, context) 22 | r2 = run_expr(op2, context) 23 | r1 && r2 24 | end 25 | 26 | def run_expr({:op, {"OR", op1, op2}}, context) do 27 | r1 = run_expr(op1, context) 28 | r2 = run_expr(op2, context) 29 | r1 || r2 30 | end 31 | 32 | def run_expr({:op, {"=", op1, op2}}, context) do 33 | r1 = run_expr(op1, context) 34 | r2 = run_expr(op2, context) 35 | 36 | {r1, r2} = match_types(r1, r2) 37 | 38 | is_equal(r1, r2) 39 | end 40 | 41 | def run_expr({:op, {"IS", op1, op2}}, context) do 42 | r1 = run_expr(op1, context) 43 | r2 = run_expr(op2, context) 44 | 45 | r1 === r2 46 | end 47 | 48 | def run_expr({:op, {">", op1, op2}}, context) do 49 | r1 = run_expr(op1, context) 50 | r2 = run_expr(op2, context) 51 | {r1, r2} = match_types(r1, r2) 52 | 53 | is_greater(r1, r2) 54 | end 55 | 56 | def run_expr({:op, {">=", op1, op2}}, context) do 57 | r1 = run_expr(op1, context) 58 | r2 = run_expr(op2, context) 59 | 60 | {r1, r2} = match_types(r1, r2) 61 | is_greater_or_equal(r1, r2) 62 | end 63 | 64 | def run_expr({:op, {"==", op1, op2}}, context), do: run_expr({:op, {"=", op1, op2}}, context) 65 | 66 | def run_expr({:op, {"!=", op1, op2}}, context), 67 | do: not run_expr({:op, {"=", op1, op2}}, context) 68 | 69 | def run_expr({:op, {"<", op1, op2}}, context), 70 | do: not run_expr({:op, {">=", op1, op2}}, context) 71 | 72 | def run_expr({:op, {"<=", op1, op2}}, context), 73 | do: not run_expr({:op, {">", op1, op2}}, context) 74 | 75 | def run_expr({:op, {"*", op1, op2}}, context) do 76 | op1 = run_expr(op1, context) 77 | op2 = run_expr(op2, context) 78 | 79 | case {op1, op2} do 80 | {{:range, {starta, enda}}, {:range, {startb, endb}}} -> 81 | if enda < startb or endb < starta do 82 | nil 83 | else 84 | {:range, {ExoSQL.Builtins.greatest(starta, startb), ExoSQL.Builtins.least(enda, endb)}} 85 | end 86 | 87 | _ -> 88 | {:ok, n1} = to_number(op1) 89 | {:ok, n2} = to_number(op2) 90 | 91 | n1 * n2 92 | end 93 | end 94 | 95 | def run_expr({:op, {"/", op1, op2}}, context) do 96 | {:ok, n1} = to_number(run_expr(op1, context)) 97 | {:ok, n2} = to_number(run_expr(op2, context)) 98 | 99 | n1 / n2 100 | end 101 | 102 | def run_expr({:op, {"%", op1, op2}}, context) do 103 | {:ok, n1} = to_number(run_expr(op1, context)) 104 | {:ok, n2} = to_number(run_expr(op2, context)) 105 | 106 | rem(n1, n2) 107 | end 108 | 109 | def run_expr({:op, {"+", op1, op2}}, context) do 110 | {:ok, n1} = to_number(run_expr(op1, context)) 111 | {:ok, n2} = to_number(run_expr(op2, context)) 112 | 113 | n1 + n2 114 | end 115 | 116 | def run_expr({:op, {"-", op1, op2}}, context) do 117 | {:ok, n1} = to_number(run_expr(op1, context)) 118 | {:ok, n2} = to_number(run_expr(op2, context)) 119 | 120 | n1 - n2 121 | end 122 | 123 | def run_expr({:op, {"||", op1, op2}}, context) do 124 | s1 = to_string(run_expr(op1, context)) 125 | s2 = to_string(run_expr(op2, context)) 126 | 127 | s1 <> s2 128 | end 129 | 130 | def run_expr({:op, {:not, op}}, context), do: run_expr({:not, op}, context) 131 | 132 | def run_expr({:not, op}, context) do 133 | n = run_expr(op, context) 134 | 135 | cond do 136 | n == "" -> true 137 | n -> false 138 | true -> true 139 | end 140 | end 141 | 142 | def run_expr({:op, {"IN", op1, op2}}, context) do 143 | op1 = run_expr(op1, context) 144 | op2 = run_expr(op2, context) 145 | 146 | case op2 do 147 | op2 when is_list(op2) -> 148 | Enum.any?(op2, fn el2 -> 149 | {op1, el2} = match_types(op1, el2) 150 | op1 == el2 151 | end) 152 | 153 | {:range, {start, end_}} -> 154 | op1 >= start and op1 <= end_ 155 | 156 | other -> 157 | throw({:invalid_argument, {:in, other}}) 158 | end 159 | end 160 | 161 | def run_expr({:op, {"LIKE", op1, op2}}, context) do 162 | op1 = run_expr(op1, context) 163 | op2 = run_expr(op2, context) 164 | 165 | like(op1, op2) 166 | end 167 | 168 | def run_expr({:op, {"ILIKE", op1, op2}}, context) do 169 | op1 = run_expr(op1, context) 170 | op2 = run_expr(op2, context) 171 | 172 | like(String.downcase(op1), String.downcase(op2)) 173 | end 174 | 175 | def run_expr({:case, list}, context) do 176 | res = 177 | Enum.find_value(list, fn 178 | {condition, expr} -> 179 | case run_expr(condition, context) do 180 | "" -> 181 | nil 182 | 183 | val -> 184 | if val do 185 | {:ok, run_expr(expr, context)} 186 | else 187 | nil 188 | end 189 | end 190 | 191 | {expr} -> 192 | {:ok, run_expr(expr, context)} 193 | end) 194 | 195 | case res do 196 | {:ok, res} -> res 197 | nil -> nil 198 | end 199 | end 200 | 201 | def run_expr({:case, expr, list}, context) do 202 | val = run_expr(expr, context) 203 | 204 | res = 205 | Enum.find_value(list, fn 206 | {condition, expr} -> 207 | if run_expr(condition, context) == val do 208 | {:ok, run_expr(expr, context)} 209 | else 210 | nil 211 | end 212 | 213 | {expr} -> 214 | {:ok, run_expr(expr, context)} 215 | end) 216 | 217 | case res do 218 | {:ok, res} -> res 219 | nil -> nil 220 | end 221 | end 222 | 223 | def run_expr({:fn, {fun, exprs}}, context) do 224 | params = for e <- exprs, do: run_expr(e, context) 225 | ExoSQL.Builtins.call_function(fun, params) 226 | end 227 | 228 | def run_expr({:pass, val}, _context), do: val 229 | 230 | def run_expr({:lit, val}, _context), do: val 231 | 232 | def run_expr({:column, n}, %{row: row}) when is_number(n) do 233 | Enum.at(row, n) 234 | end 235 | 236 | def run_expr({:column, dtr}, %{parent_columns: parent, parent_row: row}) do 237 | idx = Enum.find_index(parent, &(&1 == dtr)) 238 | Enum.at(row, idx) 239 | end 240 | 241 | def run_expr({:select, query}, context) do 242 | context = Map.put(context, :parent_row, context[:row]) 243 | context = Map.put(context, :parent_columns, context[:columns]) 244 | {:ok, res} = ExoSQL.Executor.execute(query, context) 245 | 246 | data = 247 | case res.rows do 248 | [[data]] -> 249 | data 250 | 251 | [_something | _] -> 252 | throw({:error, {:nested_query_too_many_columns, Enum.count(res.rows)}}) 253 | 254 | [] -> 255 | nil 256 | end 257 | 258 | data 259 | end 260 | 261 | def run_expr({:list, data}, context) when is_list(data) do 262 | Enum.map(data, &run_expr(&1, context)) 263 | end 264 | 265 | def run_expr({:alias, {expr, _}}, context), do: run_expr(expr, context) 266 | 267 | def like(str, str), do: true 268 | def like(_str, ""), do: false 269 | def like(_str, "%"), do: true 270 | 271 | def like(str, "%" <> more) do 272 | # Logger.debug("Like #{inspect {str, "%", more}}") 273 | length = String.length(str) 274 | 275 | Enum.any?(0..length, fn n -> 276 | like(String.slice(str, n, length), more) 277 | end) 278 | end 279 | 280 | def like(<<_::size(8)>> <> str, "_" <> more), do: like(str, more) 281 | 282 | def like(<> <> str, <> <> more), do: like(str, more) 283 | 284 | def like(_str, _expr) do 285 | # Logger.debug("Like #{inspect {str, expr}} -> false") 286 | false 287 | end 288 | 289 | @doc """ 290 | Try to return matching types. 291 | 292 | * If any is datetime, return datetimes 293 | * If any is number, return numbers 294 | * Otherwise, as is 295 | """ 296 | def match_types(a, b) do 297 | case {a, b} do 298 | {t1, t2} when is_number(t1) and is_number(t2) -> 299 | {a, b} 300 | 301 | {nil, _} -> 302 | {a, b} 303 | 304 | {_, nil} -> 305 | {a, b} 306 | 307 | {%DateTime{}, _} -> 308 | {a, ExoSQL.Builtins.to_datetime(b)} 309 | 310 | {_, %DateTime{}} -> 311 | {ExoSQL.Builtins.to_datetime(a), b} 312 | 313 | {t1, _} when is_number(t1) -> 314 | {:ok, t2} = to_number(b) 315 | {t1, t2} 316 | 317 | {_, t2} when is_number(t2) -> 318 | {:ok, t1} = to_number(a) 319 | {t1, t2} 320 | 321 | _other -> 322 | {a, b} 323 | end 324 | end 325 | 326 | @doc ~S""" 327 | Unifies is greater comparison 328 | """ 329 | def is_greater(nil, _b), do: false 330 | def is_greater(_a, nil), do: true 331 | 332 | def is_greater(%DateTime{} = r1, %DateTime{} = r2) do 333 | DateTime.compare(r1, r2) == :gt 334 | end 335 | 336 | def is_greater(r1, r2) do 337 | with {:ok, n1} <- to_number(r1), 338 | {:ok, n2} <- to_number(r2) do 339 | n1 > n2 340 | else 341 | {:error, _} -> 342 | r1 > r2 343 | end 344 | end 345 | 346 | @doc ~S""" 347 | Unifies equal comparison 348 | """ 349 | def is_equal(%DateTime{} = r1, %DateTime{} = r2) do 350 | DateTime.compare(r1, r2) == :eq 351 | end 352 | 353 | def is_equal(r1, r2) when is_binary(r1) and is_binary(r2) do 354 | r1 == r2 355 | end 356 | 357 | def is_equal(r1, r2) do 358 | with {:ok, n1} <- to_number(r1), 359 | {:ok, n2} <- to_number(r2) do 360 | n1 == n2 361 | else 362 | {:error, _} -> 363 | r1 == r2 364 | end 365 | end 366 | 367 | @doc ~S""" 368 | Unifies greater or equal comparison 369 | """ 370 | def is_greater_or_equal(nil, _b), do: false 371 | def is_greater_or_equal(_a, nil), do: true 372 | 373 | def is_greater_or_equal(%DateTime{} = r1, %DateTime{} = r2) do 374 | res = DateTime.compare(r1, r2) 375 | res == :gt or res == :eq 376 | end 377 | 378 | def is_greater_or_equal(r1, r2) do 379 | with {:ok, n1} <- to_number(r1), 380 | {:ok, n2} <- to_number(r2) do 381 | n1 >= n2 382 | else 383 | {:error, _} -> 384 | r1 >= r2 385 | end 386 | end 387 | 388 | @doc ~S""" 389 | Try to simplify expressions. 390 | 391 | Will return always a valid expression. 392 | 393 | If any subexpression is of any of these types, the expression will be the 394 | maximum complxity. 395 | 396 | This makes for example to simplify: 397 | 398 | {:list, [lit: 1, lit: 2]} -> {:lit, [1,2]} 399 | """ 400 | def simplify({:lit, n}, _context), do: {:lit, n} 401 | 402 | def simplify({:op, {op, op1, op2}}, context) do 403 | op1 = simplify(op1, context) 404 | op2 = simplify(op2, context) 405 | 406 | case {op, op1, op2} do 407 | {"AND", {:lit, false}, _} -> 408 | {:lit, false} 409 | 410 | {"AND", _, {:lit, false}} -> 411 | {:lit, false} 412 | 413 | {_, {:lit, op1}, {:lit, op2}} -> 414 | {:lit, run_expr({:op, {op, {:lit, op1}, {:lit, op2}}}, [])} 415 | 416 | _other -> 417 | {:op, {op, op1, op2}} 418 | end 419 | end 420 | 421 | def simplify({:list, list}, context) do 422 | list = Enum.map(list, &simplify(&1, context)) 423 | 424 | all_literals = 425 | Enum.all?(list, fn 426 | {:lit, _n} -> true 427 | _other -> false 428 | end) 429 | 430 | if all_literals do 431 | list = Enum.map(list, fn {:lit, n} -> n end) 432 | {:lit, list} 433 | else 434 | {:list, list} 435 | end 436 | end 437 | 438 | def simplify(list, context) when is_list(list) do 439 | Enum.map(list, &simplify(&1, context)) 440 | end 441 | 442 | def simplify({:op, {:not, op}}, context) do 443 | case simplify(op, context) do 444 | {:lit, op} -> 445 | cond do 446 | op == "" -> 447 | {:lit, true} 448 | 449 | op -> 450 | {:lit, false} 451 | 452 | true -> 453 | {:lit, true} 454 | end 455 | 456 | other -> 457 | {:not, other} 458 | end 459 | end 460 | 461 | @doc """ 462 | Simplify the column ids to positions on the list of columns, to ease operations. 463 | 464 | This operation is required to change expressions from column names to column 465 | positions, so that `ExoSQL.Expr` can perform its operations on rows. 466 | """ 467 | def simplify({:column, cn}, _context) when is_number(cn) do 468 | {:column, cn} 469 | end 470 | 471 | def simplify({:alias, {expr, alias_}}, context) do 472 | {:alias, {simplify(expr, context), alias_}} 473 | end 474 | 475 | def simplify({:column, cn}, context) do 476 | names = Map.get(context, :columns, []) 477 | i = Enum.find_index(names, &(&1 == cn)) 478 | 479 | i = 480 | if i == nil do 481 | idx = Enum.find_index(Map.get(context, :parent_columns, []), &(&1 == cn)) 482 | 483 | if idx != nil do 484 | val = Enum.at(Map.get(context, :parent_row, []), idx) 485 | 486 | if val do 487 | {:lit, val} 488 | else 489 | {:column, cn} 490 | end 491 | else 492 | nil 493 | end 494 | else 495 | {:column, i} 496 | end 497 | 498 | # Logger.debug("Simplify #{inspect cn} -> #{inspect i} | #{inspect context}") 499 | 500 | case i do 501 | nil -> 502 | # Logger.debug("Unknown column #{inspect cn} | #{inspect context}") 503 | {:column, cn} 504 | 505 | _other -> 506 | i 507 | end 508 | end 509 | 510 | def simplify({:var, cn}, %{"__vars__" => vars}) do 511 | {:lit, vars[cn]} 512 | end 513 | 514 | def simplify({:op, {op, op1, op2}}, context) do 515 | op1 = simplify(op1, context) 516 | op2 = simplify(op2, context) 517 | {:op, {op, op1, op2}} 518 | end 519 | 520 | def simplify({:op, {op, op1}}, context) do 521 | op1 = simplify(op1, context) 522 | {:op, {op, op1}} 523 | end 524 | 525 | def simplify({:not, op1}, context) do 526 | op1 = simplify(op1, context) 527 | 528 | case op1 do 529 | {:lit, true} -> 530 | {:lit, false} 531 | 532 | {:lit, false} -> 533 | {:lit, true} 534 | 535 | _other -> 536 | {:not, op1} 537 | end 538 | end 539 | 540 | def simplify({:fn, {"regex", [str, {:lit, regexs}]}}, context) when is_binary(regexs) do 541 | str = simplify(str, context) 542 | regex = Regex.compile!(regexs) 543 | captures = String.contains?(regexs, "(?<") 544 | 545 | {:fn, {"regex", [str, {:lit, {regex, captures}}]}} 546 | end 547 | 548 | def simplify({:fn, {"regex", [str, {:lit, regexs}, query]}}, context) when is_binary(regexs) do 549 | str = simplify(str, context) 550 | query = simplify(query, context) 551 | regex = Regex.compile!(regexs) 552 | captures = String.contains?(regexs, "(?<") 553 | 554 | {:fn, {"regex", [str, {:lit, {regex, captures}}, query]}} 555 | end 556 | 557 | def simplify({:fn, {"regex_all", [str, {:lit, regexs}]}}, context) when is_binary(regexs) do 558 | str = simplify(str, context) 559 | regex = Regex.compile!(regexs) 560 | 561 | {:fn, {"regex_all", [str, {:lit, regex}]}} 562 | end 563 | 564 | def simplify({:fn, {"regex_all", [str, {:lit, regexs}, query]}}, context) 565 | when is_binary(regexs) do 566 | str = simplify(str, context) 567 | query = simplify(query, context) 568 | regex = Regex.compile!(regexs) 569 | 570 | {:fn, {"regex_all", [str, {:lit, regex}, query]}} 571 | end 572 | 573 | def simplify({:fn, {f, params}}, context) do 574 | params = Enum.map(params, &simplify(&1, context)) 575 | 576 | all_literals = 577 | Enum.all?(params, fn 578 | {:lit, _} -> true 579 | _ -> false 580 | end) 581 | 582 | if all_literals and not ExoSQL.Builtins.cant_simplify(f) do 583 | {:lit, run_expr({:fn, {f, params}}, context)} 584 | else 585 | {:fn, {f, params}} 586 | end 587 | end 588 | 589 | def simplify({:case, list}, context) do 590 | list = 591 | Enum.map(list, fn 592 | {e, v} -> 593 | {simplify(e, context), simplify(v, context)} 594 | 595 | {v} -> 596 | {simplify(v, context)} 597 | end) 598 | 599 | {:case, list} 600 | end 601 | 602 | def simplify({:case, expr, list}, context) do 603 | expr = simplify(expr, context) 604 | 605 | list = 606 | Enum.map(list, fn 607 | {e, v} -> 608 | {simplify(e, context), simplify(v, context)} 609 | 610 | {v} -> 611 | {simplify(v, context)} 612 | end) 613 | 614 | {:case, expr, list} 615 | end 616 | 617 | def simplify({:alias, expr, _alias_}, context) do 618 | simplify(expr, context) 619 | end 620 | 621 | def simplify(other, _context) do 622 | other 623 | end 624 | end 625 | -------------------------------------------------------------------------------- /lib/extractors/csv.ex: -------------------------------------------------------------------------------- 1 | require Logger 2 | 3 | defmodule ExoSQL.Csv do 4 | def schema(db) do 5 | {:ok, files} = File.ls(db[:path]) 6 | 7 | files = 8 | files 9 | |> Enum.filter(&String.ends_with?(&1, ".csv")) 10 | |> Enum.map(&String.slice(&1, 0, String.length(&1) - 4)) 11 | 12 | {:ok, files} 13 | end 14 | 15 | def schema(db, table) do 16 | filename = "#{Path.join(db[:path], table)}.csv" 17 | [{:ok, columns}] = File.stream!(filename) |> CSV.decode() |> Enum.take(1) 18 | 19 | {:ok, %{columns: columns}} 20 | end 21 | 22 | def execute(db, table, _quals, _columns) do 23 | # Logger.debug("Get #{inspect table}#{inspect columns} | #{inspect quals}") 24 | 25 | filename = "#{Path.join(db[:path], table)}.csv" 26 | # Logger.debug("filename #{inspect filename}") 27 | csv_data = File.stream!(filename) |> CSV.decode() 28 | 29 | data = 30 | for l <- csv_data do 31 | {:ok, l} = l 32 | l 33 | end 34 | 35 | [columns | rows] = data 36 | 37 | {:ok, %ExoSQL.Result{columns: columns, rows: rows}} 38 | end 39 | end 40 | -------------------------------------------------------------------------------- /lib/extractors/http.ex: -------------------------------------------------------------------------------- 1 | require Logger 2 | 3 | defmodule ExoSQL.HTTP do 4 | @moduledoc """ 5 | Example Extractor that performs HTTP requests 6 | 7 | This is a virtual extractor that requires an `url` to operate with. 8 | """ 9 | 10 | def schema(_db) do 11 | {:ok, ["request"]} 12 | end 13 | 14 | def schema(_db, "request"), do: {:ok, %{columns: ["url", "status_code", "body"]}} 15 | 16 | def execute(_db, "request", quals, _columns) do 17 | # Logger.debug("Get request #{inspect quals} #{inspect columns}") 18 | 19 | urls = 20 | Enum.find_value(quals, [], fn 21 | {"url", "IN", urls} -> urls 22 | _other -> false 23 | end) 24 | 25 | rows = 26 | Enum.map(urls, fn url -> 27 | res = HTTPoison.get(url) 28 | 29 | case res do 30 | {:ok, res} -> 31 | [url, res.status_code, res.body] 32 | 33 | {:error, error} -> 34 | [url, 0, IO.inspect(error.reason)] 35 | end 36 | end) 37 | 38 | {:ok, 39 | %{ 40 | columns: ["url", "status_code", "body"], 41 | rows: rows 42 | }} 43 | end 44 | end 45 | -------------------------------------------------------------------------------- /lib/extractors/node.ex: -------------------------------------------------------------------------------- 1 | defmodule ExoSQL.Node do 2 | @moduledoc """ 3 | Example extractor that gather information from the system 4 | 5 | Currently only supports the `passwd` table. 6 | """ 7 | 8 | def schema(_db), do: {:ok, ["passwd", "proc"]} 9 | 10 | def schema(_db, "passwd") do 11 | {:ok, 12 | %{ 13 | columns: [ 14 | "user", 15 | "x", 16 | "uid", 17 | "gid", 18 | "name", 19 | "home", 20 | "shell" 21 | ] 22 | }} 23 | end 24 | 25 | def schema(_db, "proc") do 26 | {:ok, 27 | %{ 28 | columns: [ 29 | "pid", 30 | "cmd", 31 | "args" 32 | ] 33 | }} 34 | end 35 | 36 | def execute(_db, "passwd", _quals, _columns) do 37 | csv_data = File.stream!("/etc/passwd") |> CSV.decode(separator: ?:) 38 | 39 | rows = 40 | for l <- csv_data do 41 | {:ok, l} = l 42 | l 43 | end 44 | 45 | {:ok, 46 | %{ 47 | columns: [ 48 | "user", 49 | "x", 50 | "uid", 51 | "gid", 52 | "name", 53 | "home", 54 | "shell" 55 | ], 56 | rows: rows 57 | }} 58 | end 59 | 60 | def execute(_config, "proc", _quals, columns) do 61 | known_columns = ["pid", "cmd", "args"] 62 | 63 | for c <- columns do 64 | if not (c in known_columns) do 65 | raise MatchError, {:unknown_column, c} 66 | end 67 | end 68 | 69 | {:ok, proc} = File.ls("/proc/") 70 | # Keep only numeric procs 71 | proc = 72 | Enum.flat_map(proc, fn n -> 73 | case Integer.parse(n) do 74 | {n, ""} -> [n] 75 | _ -> [] 76 | end 77 | end) 78 | 79 | rows = 80 | Enum.map(proc, &{File.open("/proc/#{&1}/cmdline"), &1}) 81 | |> Enum.flat_map(fn 82 | {{:ok, fd}, pid} -> 83 | case IO.read(fd, 1024) do 84 | :eof -> 85 | [] 86 | 87 | data -> 88 | [cmd | args] = String.split(data, "\0") 89 | [[pid, cmd, args]] 90 | end 91 | 92 | _ -> 93 | [] 94 | end) 95 | 96 | {:ok, 97 | %{ 98 | columns: known_columns, 99 | rows: rows 100 | }} 101 | end 102 | end 103 | -------------------------------------------------------------------------------- /lib/format.ex: -------------------------------------------------------------------------------- 1 | require Logger 2 | 3 | defmodule ExoSQL.Format do 4 | @format_re ~r/%[\d\.,\-+]*[%fsdk]/ 5 | @format_re_one ~r/([\d\.,\-+]*)([fsdk])/ 6 | 7 | def compile_format(str) do 8 | Regex.split(@format_re, str, include_captures: true) 9 | |> Enum.reduce([], fn 10 | "%%", acc -> 11 | ["%" | acc] 12 | 13 | "%" <> fs, acc -> 14 | [_all, mod, type] = Regex.run(@format_re_one, fs) 15 | [{mod, type} | acc] 16 | 17 | "", acc -> 18 | acc 19 | 20 | other, acc -> 21 | [other | acc] 22 | end) 23 | end 24 | 25 | def format(str, params) when is_binary(str) do 26 | # Logger.debug("Compile str #{inspect str}") 27 | precompiled = compile_format(str) 28 | # Logger.debug("Precompiled: #{inspect precompiled}") 29 | format(precompiled, params) 30 | end 31 | 32 | def format([], _params), do: "" 33 | 34 | def format(precompiled, params) when is_list(precompiled) do 35 | # Logger.debug("Format #{inspect {precompiled, params}}") 36 | res = 37 | Enum.reduce(precompiled, {[], Enum.reverse(params)}, fn 38 | {mod, type}, {acc, params} -> 39 | [head | rest] = params 40 | repl = format_one(mod, type, head) 41 | {[repl | acc], rest} 42 | 43 | str, {acc, params} -> 44 | {[str | acc], params} 45 | end) 46 | 47 | # Logger.debug("Result #{inspect {precompiled, params}} -> #{inspect res}") 48 | case res do 49 | {str, []} -> 50 | to_string(str) 51 | 52 | {_str, other} -> 53 | throw({:error, {:format, {:pending, Enum.count(other)}}}) 54 | end 55 | end 56 | 57 | def localized_number(number) do 58 | localized_number(number, 0) 59 | end 60 | 61 | def localized_number(number, 0) do 62 | int = :erlang.float_to_binary(Float.floor(number), decimals: 0) 63 | head = localized_number_str_comma(int) 64 | "#{head}" 65 | end 66 | 67 | def localized_number(number, decimals) do 68 | dec = :erlang.float_to_binary(number, decimals: decimals) 69 | {int, dec} = String.split_at(dec, String.length(dec) - decimals) 70 | head = localized_number_str_comma(String.slice(int, 0, String.length(int) - 1)) 71 | 72 | if dec == String.duplicate("0", decimals) do 73 | "#{head}" 74 | else 75 | "#{head},#{dec}" 76 | end 77 | end 78 | 79 | defp localized_number_str_comma(lit) do 80 | if String.length(lit) > 3 do 81 | {head, tail} = String.split_at(lit, String.length(lit) - 3) 82 | head = localized_number_str_comma(head) 83 | "#{head}.#{tail}" 84 | else 85 | lit 86 | end 87 | end 88 | 89 | def format_one(mod, type, data) do 90 | # to_string(data) 91 | case {mod, type} do 92 | {"", "s"} -> 93 | to_string(data) 94 | 95 | {"-" <> count, "s"} -> 96 | data = to_string(data) 97 | count = ExoSQL.Utils.to_number!(count) - String.length(data) 98 | 99 | if count > 0 do 100 | data <> String.duplicate(" ", count) 101 | else 102 | data 103 | end 104 | 105 | {count, "s"} -> 106 | data = to_string(data) 107 | count = ExoSQL.Utils.to_number!(count) - String.length(data) 108 | 109 | if count > 0 do 110 | String.duplicate(" ", count) <> data 111 | else 112 | data 113 | end 114 | 115 | {"", "f"} -> 116 | {:ok, data} = ExoSQL.Utils.to_float(data) 117 | :erlang.float_to_binary(data, decimals: 2) 118 | 119 | {"." <> decimals, "f"} -> 120 | {:ok, data} = ExoSQL.Utils.to_float(data) 121 | {:ok, decimals} = ExoSQL.Utils.to_number(decimals) 122 | :erlang.float_to_binary(data, decimals: decimals) 123 | 124 | {"+", "f"} -> 125 | {:ok, datan} = ExoSQL.Utils.to_float(data) 126 | data = :erlang.float_to_binary(datan, decimals: 2) 127 | 128 | if datan <= 0 do 129 | "#{data}" 130 | else 131 | "+#{data}" 132 | end 133 | 134 | {"", "d"} -> 135 | {:ok, data} = ExoSQL.Utils.to_number(data) 136 | data = Kernel.trunc(data) 137 | "#{data}" 138 | 139 | {"+", "d"} -> 140 | datan = ExoSQL.Utils.to_number!(data) 141 | datan = Kernel.trunc(datan) 142 | 143 | if datan <= 0 do 144 | "#{data}" 145 | else 146 | "+#{data}" 147 | end 148 | 149 | {<> <> count, "d"} -> 150 | {:ok, data} = ExoSQL.Utils.to_number(data) 151 | data = Kernel.trunc(data) 152 | data = "#{data}" 153 | count = ExoSQL.Utils.to_number!(count) - String.length(data) 154 | 155 | if count > 0 do 156 | String.duplicate(<>, count) <> data 157 | else 158 | data 159 | end 160 | 161 | {".", "k"} -> 162 | {:ok, data} = ExoSQL.Utils.to_float(data) 163 | 164 | {data, sufix} = 165 | cond do 166 | data >= 1_000_000 -> 167 | data = data / 1_000_000 168 | data = :erlang.float_to_binary(data, decimals: 1) 169 | {data, "M"} 170 | 171 | data >= 100_000 -> 172 | data = data / 1_000 173 | data = :erlang.float_to_binary(data, decimals: 1) 174 | {data, "K"} 175 | 176 | data >= 1_000 -> 177 | data = Kernel.trunc(data) 178 | {data, ""} 179 | 180 | true -> 181 | data = :erlang.float_to_binary(data, decimals: 2) 182 | {data, ""} 183 | end 184 | 185 | "#{data}#{sufix}" 186 | 187 | {",", "k"} -> 188 | {:ok, data} = ExoSQL.Utils.to_float(data) 189 | 190 | {data, sufix} = 191 | cond do 192 | data >= 1_000_000 -> 193 | data = data / 1_000_000 194 | data = localized_number(data, 1) 195 | {data, "MM"} 196 | 197 | data >= 100_000 -> 198 | data = data / 1_000 199 | data = localized_number(data, 1) 200 | {data, "K"} 201 | 202 | data >= 1_000 -> 203 | data = localized_number(data, 0) 204 | {data, ""} 205 | 206 | true -> 207 | data = localized_number(data, 2) 208 | {data, ""} 209 | end 210 | 211 | "#{data}#{sufix}" 212 | 213 | {"", "k"} -> 214 | {:ok, data} = ExoSQL.Utils.to_number(data) 215 | data = Kernel.trunc(data) 216 | 217 | {data, sufix} = 218 | cond do 219 | data >= 1_000_000 -> 220 | {div(data, 1_000_000), "M"} 221 | 222 | data >= 1_000 -> 223 | {div(data, 1_000), "k"} 224 | 225 | true -> 226 | {data, ""} 227 | end 228 | 229 | "#{data}#{sufix}" 230 | end 231 | end 232 | end 233 | -------------------------------------------------------------------------------- /lib/parser.ex: -------------------------------------------------------------------------------- 1 | require Logger 2 | 3 | defmodule ExoSQL.Parser do 4 | @moduledoc """ 5 | Parsed an SQL statement into a ExoSQL.Query. 6 | 7 | The Query then needs to be planned and executed. 8 | 9 | It also resolves partial column and table names using data from the context 10 | and its schema functions. 11 | 12 | Uses leex and yecc to perform a first phase parsing, and then 13 | convert an dprocess the structure using more context knowledge to return 14 | a proper Query struct. 15 | """ 16 | 17 | ~S""" 18 | Parses from the yeec provided parse tree to a more realistic and complete parse 19 | tree with all data resolved. 20 | """ 21 | 22 | defp real_parse(parsed, context) do 23 | %{ 24 | with: with_, 25 | select: select, 26 | from: from, 27 | where: where, 28 | groupby: groupby, 29 | join: join, 30 | orderby: orderby, 31 | limit: limit, 32 | offset: offset, 33 | union: union 34 | } = parsed 35 | 36 | {select, select_options} = select 37 | 38 | if ExoSQL.debug_mode(context) do 39 | Logger.debug("ExoSQL Parser #{inspect(parsed, pretty: true)} #{inspect(context)}") 40 | end 41 | 42 | {context, with_parsed} = 43 | if with_ != [] do 44 | # Logger.debug("Parsed #{inspect parsed, pretty: true}") 45 | context = Map.put(context, :with, %{}) 46 | 47 | # Context adds the known columns to be used later, and returns the 48 | # :with_parsed which are the queries to be executed once by the main query. 49 | Enum.reduce(with_, {context, []}, fn 50 | {name, select}, {context, with_parsed} -> 51 | {:ok, parsed} = real_parse(select, context) 52 | # Logger.debug("parse with #{inspect parsed}") 53 | columns = 54 | resolve_columns(parsed, context) 55 | |> Enum.map(fn {_, _, col} -> {:with, name, col} end) 56 | 57 | # Logger.debug("Columns for #{inspect(name)}: #{inspect(columns)}") 58 | 59 | context = put_in(context, [:with, name], columns) 60 | 61 | {context, with_parsed ++ [{name, parsed}]} 62 | end) 63 | else 64 | {context, []} 65 | end 66 | 67 | all_tables_at_context = resolve_all_tables(context) 68 | # Logger.debug("All tables #{inspect all_tables_at_context}") 69 | 70 | # Logger.debug("Resolve tables #{inspect(from, pretty: true)}") 71 | {from, cross_joins} = 72 | case from do 73 | [] -> 74 | {nil, []} 75 | 76 | [from | cross_joins] -> 77 | from = resolve_table(from, all_tables_at_context, context) 78 | 79 | cross_joins = 80 | Enum.map(cross_joins, fn 81 | # already a cross join lateral, not change 82 | {:cross_join_lateral, opts} -> 83 | {:cross_join_lateral, opts} 84 | 85 | # dificult for erlang parser, needs just redo here. 86 | {:alias, {{:cross_join_lateral, cjl}, alias_}} -> 87 | {:cross_join_lateral, {:alias, {cjl, alias_}}} 88 | 89 | # functions are always lateral 90 | {:fn, f} -> 91 | {:cross_join_lateral, {:fn, f}} 92 | 93 | {:alias, {{:fn, f}, al}} -> 94 | {:cross_join_lateral, {:alias, {{:fn, f}, al}}} 95 | 96 | # Was using , operator -> cross joins 97 | other -> 98 | {:cross_join, other} 99 | end) 100 | 101 | {from, cross_joins} 102 | end 103 | 104 | # Logger.debug("from #{inspect (cross_joins ++ join), pretty: true}") 105 | 106 | join = 107 | Enum.map(cross_joins ++ join, fn 108 | {:cross_join_lateral, table} -> 109 | context = 110 | Map.put( 111 | context, 112 | "__parent__", 113 | resolve_all_columns([from], context) ++ Map.get(context, "__parent__", []) 114 | ) 115 | 116 | # Logger.debug("Resolve table, may need my columns #{inspect table} #{inspect context}") 117 | resolved = resolve_table(table, all_tables_at_context, context) 118 | {:cross_join_lateral, resolved} 119 | 120 | {type, {:select, query}} -> 121 | {:ok, parsed} = real_parse(query, context) 122 | # Logger.debug("Resolved #{inspect parsed}") 123 | {type, parsed} 124 | 125 | {type, {{:select, query}, ops}} -> 126 | {:ok, parsed} = real_parse(query, context) 127 | # Logger.debug("Resolved #{inspect parsed}") 128 | {type, {parsed, ops}} 129 | 130 | {type, {:table, table}} -> 131 | resolved = resolve_table({:table, table}, all_tables_at_context, context) 132 | {type, resolved} 133 | 134 | {type, {{:table, table}, ops}} -> 135 | # Logger.debug("F is #{inspect {table, ops}}") 136 | resolved = resolve_table({:table, table}, all_tables_at_context, context) 137 | {type, {resolved, ops}} 138 | 139 | {_type, {{:alias, {{:fn, _}, _}}, _}} = orig -> 140 | orig 141 | 142 | {type, {{:alias, {orig, alias_}}, ops}} -> 143 | # Logger.debug("Table is #{inspect orig}") 144 | resolved = resolve_table(orig, all_tables_at_context, context) 145 | {type, {{:alias, {resolved, alias_}}, ops}} 146 | end) 147 | 148 | # Logger.debug("Prepare join tables #{inspect(join, pretty: true)}") 149 | all_tables = 150 | if join != [] do 151 | [from] ++ 152 | Enum.map(join, fn 153 | {_type, {:table, from}} -> 154 | {:table, from} 155 | 156 | {_type, {:alias, {from, alias_}}} -> 157 | {:alias, {from, alias_}} 158 | 159 | {_type, {:fn, args}} -> 160 | {:fn, args} 161 | 162 | {_type, {from, _on}} -> 163 | from 164 | 165 | {_type, from} -> 166 | from 167 | end) 168 | else 169 | if from == nil do 170 | [] 171 | else 172 | [from] 173 | end 174 | end 175 | 176 | # Logger.debug("All tables at all columns #{inspect(all_tables)}") 177 | select_columns = resolve_all_columns(all_tables, context) 178 | all_columns = Map.get(context, "__parent__", []) ++ select_columns 179 | # Logger.debug("Resolved columns at query: #{inspect(all_columns)}") 180 | 181 | # Now resolve references to tables, as in FROM xx, LATERAL nested(xx.json, "a") 182 | from = resolve_column(from, all_columns, context) 183 | 184 | groupby = 185 | if groupby do 186 | Enum.map(groupby, &resolve_column(&1, all_columns, context)) 187 | else 188 | nil 189 | end 190 | 191 | # Logger.debug("All tables #{inspect all_tables}") 192 | join = 193 | Enum.map(join, fn 194 | {type, {:fn, {func, params}}} -> 195 | # Logger.debug("params #{inspect params}") 196 | params = Enum.map(params, &resolve_column(&1, all_columns, context)) 197 | {type, {:fn, {func, params}}} 198 | 199 | {type, {:alias, {{:fn, {func, params}}, alias_}}} -> 200 | # Logger.debug("params #{inspect params}") 201 | params = Enum.map(params, &resolve_column(&1, all_columns, context)) 202 | {type, {:alias, {{:fn, {func, params}}, alias_}}} 203 | 204 | {type, {{:table, table}, expr}} -> 205 | {type, 206 | { 207 | {:table, table}, 208 | resolve_column(expr, all_columns, context) 209 | }} 210 | 211 | {type, {any, expr}} -> 212 | {type, 213 | { 214 | any, 215 | resolve_column(expr, all_columns, context) 216 | }} 217 | 218 | {type, %ExoSQL.Query{} = query} -> 219 | {type, query} 220 | end) 221 | 222 | # the resolve all expressions as we know which tables to use 223 | # Logger.debug("Get select resolved: #{inspect select}") 224 | select = 225 | case select do 226 | [{:all_columns}] -> 227 | # SELECT * do not include parent columns. 228 | select_columns |> Enum.map(&{:column, &1}) 229 | 230 | _other -> 231 | Enum.map(select, &resolve_column(&1, all_columns, context)) 232 | end 233 | 234 | # Logger.debug("Resolved: #{inspect select}") 235 | distinct = 236 | case Keyword.get(select_options, :distinct) do 237 | nil -> nil 238 | other -> resolve_column(other, all_columns, context) 239 | end 240 | 241 | crosstab = Keyword.get(select_options, :crosstab) 242 | 243 | where = 244 | if where do 245 | resolve_column(where, all_columns, context) 246 | else 247 | nil 248 | end 249 | 250 | # Resolve orderby 251 | orderby = 252 | Enum.map(orderby, fn {type, expr} -> 253 | {type, resolve_column(expr, all_columns, context)} 254 | end) 255 | 256 | # resolve union 257 | union = 258 | if union do 259 | {type, other} = union 260 | {:ok, other} = real_parse(other, context) 261 | {type, other} 262 | end 263 | 264 | with_ = with_parsed 265 | 266 | {:ok, 267 | %ExoSQL.Query{ 268 | select: select, 269 | distinct: distinct, 270 | crosstab: crosstab, 271 | # all the tables it gets data from, but use only the frist and the joins. 272 | from: from, 273 | where: where, 274 | groupby: groupby, 275 | join: join, 276 | orderby: orderby, 277 | limit: limit, 278 | offset: offset, 279 | union: union, 280 | with: with_ 281 | }} 282 | end 283 | 284 | @doc """ 285 | Parses an SQL statement and returns the parsed ExoSQL struct. 286 | """ 287 | def parse(sql, context) do 288 | try do 289 | sql = String.to_charlist(sql) 290 | 291 | lexed = 292 | case :sql_lexer.string(sql) do 293 | {:ok, lexed, _lines} -> lexed 294 | {:error, {other, _}} -> throw(other) 295 | end 296 | 297 | parsed = 298 | case :sql_parser.parse(lexed) do 299 | {:ok, parsed} -> parsed 300 | {:error, any} -> throw(any) 301 | end 302 | 303 | # Logger.debug("Yeec parsed: #{inspect parsed, pretty: true}") 304 | real_parse(parsed, context) 305 | catch 306 | {line_number, :sql_lexer, msg} -> 307 | {:error, {:syntax, {msg, line_number}}} 308 | 309 | {line_number, :sql_parser, msg} -> 310 | {:error, {:syntax, {to_string(msg), line_number}}} 311 | 312 | any -> 313 | Logger.debug("Generic error at SQL parse: #{inspect(any)}") 314 | {:error, any} 315 | end 316 | end 317 | 318 | @doc ~S""" 319 | Calculates the list of all FQcolumns. 320 | 321 | This simplifies later the gathering of which table has which column and so on, 322 | specially when aliases are taken into account 323 | """ 324 | def resolve_all_columns(tables, context) do 325 | # Logger.debug("Resolve all tables #{inspect tables}") 326 | Enum.flat_map(tables, &resolve_columns(&1, context)) 327 | end 328 | 329 | def resolve_columns({:alias, {any, alias_}}, context) do 330 | case resolve_columns(any, context) do 331 | # only one answer, same name as "table", alias it 332 | [{_, a, a}] -> 333 | [{:tmp, alias_, alias_}] 334 | 335 | other -> 336 | Enum.map(other, fn {_db, _table, column} -> {:tmp, alias_, column} end) 337 | end 338 | end 339 | 340 | def resolve_columns({:table, {:with, table}}, context) do 341 | # Logger.debug("Get :with columns: #{inspect table} #{inspect context, pretty: true}") 342 | context[:with][table] 343 | end 344 | 345 | def resolve_columns({:table, nil}, _context) do 346 | [] 347 | end 348 | 349 | def resolve_columns({:table, {db, table}}, context) do 350 | # Logger.debug("table #{inspect {db, table}}") 351 | {:ok, schema} = ExoSQL.schema(db, table, context) 352 | Enum.map(schema[:columns], &{db, table, &1}) 353 | end 354 | 355 | # no column names given, just unnest 356 | def resolve_columns({:fn, {"unnest", [_expr]}}, _context) do 357 | [{:tmp, "unnest", "unnest"}] 358 | end 359 | 360 | def resolve_columns({:fn, {"unnest", [_expr | columns]}}, _context) do 361 | columns |> Enum.map(fn {:lit, col} -> {:tmp, "unnest", col} end) 362 | end 363 | 364 | def resolve_columns({:fn, {function, _params}}, _context) do 365 | [{:tmp, function, function}] 366 | end 367 | 368 | def resolve_columns({:lateral, something}, context) do 369 | resolve_columns(something, context) 370 | end 371 | 372 | def resolve_columns({:select, query}, _context) do 373 | {columns, _} = 374 | Enum.reduce(query[:select], {[], 1}, fn column, {acc, count} -> 375 | # Logger.debug("Resolve column name for: #{inspect column}") 376 | column = 377 | case column do 378 | {:column, {_db, _table, column}} -> 379 | {:tmp, :tmp, column} 380 | 381 | {:alias, {_, alias_}} -> 382 | {:tmp, :tmp, alias_} 383 | 384 | _expr -> 385 | {:tmp, :tmp, "col_#{count}"} 386 | end 387 | 388 | # Logger.debug("Resolved: #{inspect column}") 389 | 390 | {acc ++ [column], count + 1} 391 | end) 392 | 393 | # Logger.debug("Get column from select #{inspect query[:select]}: #{inspect columns}") 394 | columns 395 | end 396 | 397 | def resolve_columns(%ExoSQL.Query{} = q, _context) do 398 | get_query_columns(q) 399 | end 400 | 401 | def resolve_columns({:columns, columns}, _context) do 402 | columns 403 | end 404 | 405 | def get_table_columns({db, table}, all_columns) do 406 | for {^db, ^table, column} <- all_columns, do: column 407 | end 408 | 409 | @doc ~S""" 410 | Resolves all known tables at this context. This helps to fully qualify tables. 411 | 412 | TODO Could be more efficient accessing as little as possible the schemas, but 413 | maybe not possible. 414 | """ 415 | def resolve_all_tables(context) do 416 | Enum.flat_map(context, fn 417 | {:with_parsed, with_} -> 418 | [] 419 | 420 | {:with, with_} -> 421 | Map.keys(with_) |> Enum.map(&{:with, &1}) 422 | 423 | {db, _config} -> 424 | {:ok, tables} = ExoSQL.schema(db, context) 425 | tables |> Enum.map(&{db, &1}) 426 | end) 427 | end 428 | 429 | @doc ~S""" 430 | Given a table-like tuple, returns the real table names 431 | 432 | The table-like can be a function, a lateral join, or a simple table. It 433 | resolves unknown parts, as for example {:table, {nil, "table"}}, will fill 434 | which db. 435 | 436 | It returns the same form, but with more data, and calling again will result 437 | in the same result. 438 | """ 439 | def resolve_table({:table, {nil, name}}, all_tables, _context) when is_binary(name) do 440 | # Logger.debug("Resolve #{inspect name} at #{inspect all_tables}") 441 | options = for {db, ^name} <- all_tables, do: {db, name} 442 | # Logger.debug("Options are #{inspect options}") 443 | 444 | case options do 445 | [table] -> {:table, table} 446 | l when l == [] -> raise "Cant find table #{inspect(name)}" 447 | _other -> raise "Ambiguous table name #{inspect(name)}" 448 | end 449 | end 450 | 451 | def resolve_table({:table, {_db, _name}} = orig, _all_tables, _context) do 452 | orig 453 | end 454 | 455 | def resolve_table({:select, query}, _all_tables, context) do 456 | {:ok, parsed} = real_parse(query, context) 457 | parsed 458 | end 459 | 460 | def resolve_table({:fn, _function} = orig, _all_tables, _context), do: orig 461 | 462 | def resolve_table({:alias, {table, alias_}}, all_tables, context) do 463 | {:alias, {resolve_table(table, all_tables, context), alias_}} 464 | end 465 | 466 | def resolve_table({:lateral, table}, all_tables, context) do 467 | resolved = resolve_table(table, all_tables, context) 468 | {:lateral, resolved} 469 | end 470 | 471 | def resolve_table(other, _all_tables, _context) do 472 | Logger.error("Cant resolve table #{inspect(other)}") 473 | # maybe it do not have the type tagged at other ({:table, other}). Typical fail here. 474 | raise "Cant resolve table #{inspect(other)}" 475 | end 476 | 477 | @doc ~S""" 478 | From the list of tables, and context, and an unknown column, return the 479 | FQN of the column. 480 | """ 481 | def resolve_column({:column, {nil, nil, column}}, all_columns, context) do 482 | found = 483 | Enum.filter(all_columns, fn 484 | {_db, _table, ^column} -> true 485 | _other -> false 486 | end) 487 | 488 | found = 489 | case found do 490 | [one] -> 491 | {:column, one} 492 | 493 | [] -> 494 | parent_schema = Map.get(context, "__parent__", false) 495 | 496 | if parent_schema do 497 | {:column, found} = 498 | resolve_column({:column, {nil, nil, column}}, parent_schema, context) 499 | 500 | # Logger.debug("Found column from parent #{inspect found}") 501 | {:column, found} 502 | else 503 | raise "Not found #{inspect(column)} in #{inspect(all_columns)}" 504 | end 505 | 506 | many -> 507 | raise "Ambiguous column #{inspect(column)} in #{inspect(many)}" 508 | end 509 | 510 | if found do 511 | found 512 | else 513 | raise "Not found #{inspect(column)} in #{inspect(all_columns)}" 514 | end 515 | end 516 | 517 | def resolve_column({:column, {nil, table, column}}, all_columns, context) do 518 | # Logger.debug("Find #{inspect {nil, table, column}} at #{inspect all_columns} + #{inspect Map.get(context, "__parent__", [])}") 519 | found = 520 | Enum.find(all_columns, fn 521 | {_db, ^table, ^column} -> true 522 | _other -> false 523 | end) 524 | 525 | if found do 526 | {:column, found} 527 | else 528 | parent_schema = Map.get(context, "__parent__", []) 529 | 530 | if parent_schema != [] do 531 | context = Map.drop(context, ["__parent__"]) 532 | 533 | case resolve_column({:column, {nil, table, column}}, parent_schema, context) do 534 | {:column, found} -> 535 | {:column, found} 536 | 537 | _ -> 538 | throw({:not_found, {table, column}, :in, all_columns}) 539 | end 540 | else 541 | throw({:not_found, {table, column}, :in, all_columns}) 542 | end 543 | end 544 | end 545 | 546 | def resolve_column({:column, _} = column, _schema, _context), do: column 547 | 548 | def resolve_column({:op, {op, ex1, ex2}}, all_columns, context) do 549 | {:op, 550 | {op, resolve_column(ex1, all_columns, context), resolve_column(ex2, all_columns, context)}} 551 | end 552 | 553 | def resolve_column({:fn, {f, params}}, all_columns, context) do 554 | params = Enum.map(params, &resolve_column(&1, all_columns, context)) 555 | {:fn, {f, params}} 556 | end 557 | 558 | def resolve_column({:distinct, expr}, all_columns, context) do 559 | {:distinct, resolve_column(expr, all_columns, context)} 560 | end 561 | 562 | def resolve_column({:case, list}, all_columns, context) do 563 | # Logger.debug("Resolve case #{inspect list, pretty: true}") 564 | list = 565 | Enum.map(list, fn 566 | {c, e} -> 567 | {resolve_column(c, all_columns, context), resolve_column(e, all_columns, context)} 568 | 569 | {e} -> 570 | {resolve_column(e, all_columns, context)} 571 | end) 572 | 573 | {:case, list} 574 | end 575 | 576 | def resolve_column({:case, expr, list}, all_columns, context) do 577 | expr = resolve_column(expr, all_columns, context) 578 | 579 | list = 580 | Enum.map(list, fn 581 | {c, e} -> 582 | {resolve_column(c, all_columns, context), resolve_column(e, all_columns, context)} 583 | 584 | {e} -> 585 | {resolve_column(e, all_columns, context)} 586 | end) 587 | 588 | {:case, expr, list} 589 | end 590 | 591 | def resolve_column({:alias, {expr, alias_}}, all_columns, context) do 592 | {:alias, {resolve_column(expr, all_columns, context), alias_}} 593 | end 594 | 595 | def resolve_column({:select, query}, all_columns, context) do 596 | all_columns = all_columns ++ Map.get(context, "__parent__", []) 597 | context = Map.put(context, "__parent__", all_columns) 598 | {:ok, parsed} = real_parse(query, context) 599 | # Logger.debug("Parsed query #{inspect query} -> #{inspect parsed, pretty: true}") 600 | {:select, parsed} 601 | end 602 | 603 | def resolve_column({:lateral, expr}, all_columns, context) do 604 | {:lateral, resolve_column(expr, all_columns, context)} 605 | end 606 | 607 | def resolve_column(other, _schema, _context) do 608 | other 609 | end 610 | 611 | defp get_query_columns(%ExoSQL.Query{select: select, crosstab: nil}) do 612 | get_column_names_or_alias(select, 1) 613 | end 614 | 615 | defp get_query_columns(%ExoSQL.Query{select: select, crosstab: :all_columns}) do 616 | # only the first is sure.. the rest too dynamic to know 617 | [hd(get_column_names_or_alias(select, 1))] 618 | end 619 | 620 | defp get_query_columns(%ExoSQL.Query{select: select, crosstab: crosstab}) 621 | when is_list(crosstab) do 622 | first = hd(get_column_names_or_alias(select, 1)) 623 | more = crosstab |> Enum.map(&{:tmp, :tmp, &1}) 624 | [first | more] 625 | end 626 | 627 | defp get_query_columns({:columns, columns}) do 628 | columns 629 | end 630 | 631 | defp get_column_names_or_alias([{:column, column} | rest], count) do 632 | [column | get_column_names_or_alias(rest, count + 1)] 633 | end 634 | 635 | defp get_column_names_or_alias([{:alias, {_column, alias_}} | rest], count) do 636 | [{:tmp, :tmp, alias_} | get_column_names_or_alias(rest, count + 1)] 637 | end 638 | 639 | defp get_column_names_or_alias([{:fn, {"unnest", [_from | columns]}} | rest], count) do 640 | Enum.map(columns, fn {:lit, name} -> {:tmp, :tmp, name} end) ++ 641 | get_column_names_or_alias(rest, count + 1) 642 | end 643 | 644 | defp get_column_names_or_alias([_head | rest], count) do 645 | [{:tmp, :tmp, "col_#{count}"} | get_column_names_or_alias(rest, count + 1)] 646 | end 647 | 648 | defp get_column_names_or_alias([], _count), do: [] 649 | end 650 | -------------------------------------------------------------------------------- /lib/planner.ex: -------------------------------------------------------------------------------- 1 | require Logger 2 | 3 | defmodule ExoSQL.Planner do 4 | @doc ~S""" 5 | Given a query, returns a tree of actions (AST) to perform to resolve the query. 6 | 7 | Each action is `{:plan, {step_function, step_data}}` to call. 8 | 9 | step_data may contain more tagged :plan to call recursively as required. 10 | 11 | They will be performed in reverse order and replaced where it is required. 12 | 13 | For example, it may return for a very simple: 14 | 15 | iex> {:ok, query} = ExoSQL.Parser.parse("SELECT name, price FROM products", %{"A" => {ExoSQL.Csv, path: "test/data/csv/"}}) 16 | iex> plan(query, %{}) 17 | {:ok, 18 | {:select, 19 | {:execute, {:table, {"A", "products"}}, [], [{"A", "products", "name"}, {"A", "products", "price"}]}, [ 20 | column: {"A", "products", "name"}, 21 | column: {"A", "products", "price"}] 22 | } 23 | } 24 | 25 | Or a more complex: 26 | 27 | iex> query = "SELECT users.name, products.name FROM users, purchases, products WHERE (users.id = purchases.user_id) AND (purchases.product_id = products.id)" 28 | iex> {:ok, query} = ExoSQL.Parser.parse(query, %{"A" => {ExoSQL.Csv, path: "test/data/csv/"}}) 29 | iex> plan(query, %{}) 30 | {:ok, 31 | {:select, 32 | {:filter, 33 | {:cross_join, 34 | {:cross_join, 35 | {:execute, {:table, {"A", "users"}}, [], [ 36 | {"A", "users", "id"}, 37 | {"A", "users", "name"} 38 | ]}, 39 | {:execute, {:table, {"A", "purchases"}}, [], [ 40 | {"A", "purchases", "user_id"}, 41 | {"A", "purchases", "product_id"} 42 | ]}}, 43 | {:execute, {:table, {"A", "products"}}, [], [ 44 | {"A", "products", "id"}, 45 | {"A", "products", "name"} 46 | ]} 47 | }, 48 | {:op, {"AND", 49 | {:op, {"=", 50 | {:column, {"A", "users", "id"}}, 51 | {:column, {"A", "purchases", "user_id"}}} 52 | }, 53 | {:op, {"=", 54 | {:column, {"A", "purchases", "product_id"}}, 55 | {:column, {"A", "products", "id"}}} 56 | }}}, 57 | }, 58 | [column: {"A", "users", "name"}, 59 | column: {"A", "products", "name"}] 60 | } } 61 | 62 | Which means that it will extract A.users, cross join with A.purchases, then cross 63 | join that with A.produtcs, apply a filter of the expession, and finally 64 | return only the users name and product name. 65 | 66 | TODO: explore different plans acording to some weights and return the optimal one. 67 | """ 68 | def plan(query, context) do 69 | where = ExoSQL.Expr.simplify(query.where, %{}) 70 | select = ExoSQL.Expr.simplify(query.select, %{}) 71 | 72 | all_expressions = [ 73 | where, 74 | select, 75 | query.groupby, 76 | query.from, 77 | query.join, 78 | Enum.map(query.orderby, fn {_type, expr} -> expr end), 79 | Enum.map(query.join, fn 80 | {:cross_join_lateral, q} -> q 81 | {_join, {_from, expr}} -> expr 82 | _other -> [] 83 | end) 84 | ] 85 | 86 | # Logger.debug("a All expressions: #{inspect query.from} | #{inspect all_expressions}") 87 | from = plan_execute(query.from, where, all_expressions, context) 88 | # Logger.debug("a Plan #{inspect from, pretty: true}") 89 | 90 | from_plan = 91 | if from == nil do 92 | # just one element 93 | %ExoSQL.Result{columns: ["?NONAME"], rows: [[1]]} 94 | else 95 | from 96 | end 97 | 98 | # Logger.debug("From plan #{inspect from} -> #{inspect from_plan}") 99 | 100 | join_plan = 101 | Enum.reduce(query.join, from_plan, fn 102 | {:cross_join, toplan}, acc -> 103 | # Logger.debug("b All expressions: #{inspect toplan} | #{inspect all_expressions}") 104 | from = plan_execute(toplan, all_expressions, context) 105 | # Logger.debug("b Plan #{inspect from, pretty: true}") 106 | {:cross_join, acc, from} 107 | 108 | {:cross_join_lateral, toplan}, acc -> 109 | from = plan_execute(toplan, all_expressions, context) 110 | {:cross_join_lateral, acc, from} 111 | 112 | {join_type, {toplan, expr}}, acc -> 113 | from = plan_execute(toplan, expr, all_expressions, context) 114 | {join_type, acc, from, expr} 115 | end) 116 | 117 | where_plan = 118 | if where do 119 | {:filter, join_plan, where} 120 | else 121 | join_plan 122 | end 123 | 124 | group_plan = 125 | if query.groupby do 126 | {:group_by, where_plan, query.groupby} 127 | else 128 | where_plan 129 | end 130 | 131 | # Order can be applied pre select or post select. This is the pre select. 132 | order_plan = 133 | query.orderby 134 | |> Enum.reverse() 135 | |> Enum.reduce(group_plan, fn 136 | {_type, {:lit, _n}}, acc -> 137 | acc 138 | 139 | {type, expr}, acc -> 140 | {:order_by, type, expr, acc} 141 | end) 142 | 143 | select = 144 | Enum.map(select, fn 145 | {:select, query} -> 146 | {:ok, plan} = plan(query, context) 147 | {:select, plan} 148 | 149 | other -> 150 | other 151 | end) 152 | 153 | select_plan = 154 | cond do 155 | # if grouping, special care on aggregate builtins 156 | query.groupby -> 157 | selectg = Enum.map(select, &fix_aggregates_select(&1, Enum.count(query.groupby))) 158 | {:select, order_plan, selectg} 159 | 160 | # groups full table, do a table to row conversion, and then the ops 161 | has_aggregates(select) -> 162 | table_in_a_row = {:table_to_row, order_plan} 163 | selecta = Enum.map(select, &fix_aggregates_select(&1, 0)) 164 | {:select, table_in_a_row, selecta} 165 | 166 | true -> 167 | {:select, order_plan, select} 168 | end 169 | 170 | select_plan = 171 | if Enum.any?(select, fn 172 | {:fn, {name, _args}} -> ExoSQL.Builtins.is_projectable(name) 173 | {:lit, %{columns: _, rows: _}} -> true 174 | {:alias, {{:lit, %{columns: _, rows: _}}, _}} -> true 175 | {:alias, {{:fn, {name, _args}}, _}} -> ExoSQL.Builtins.is_projectable(name) 176 | _other -> false 177 | end) do 178 | {:project, select_plan} 179 | else 180 | select_plan 181 | end 182 | 183 | distinct_plan = 184 | case query.distinct do 185 | nil -> 186 | select_plan 187 | 188 | other -> 189 | {:distinct, other, select_plan} 190 | end 191 | 192 | crosstab_plan = 193 | if query.crosstab do 194 | {:crosstab, query.crosstab, distinct_plan} 195 | else 196 | distinct_plan 197 | end 198 | 199 | # Order can be applied pre select or post select. This is the post select. 200 | order_plan = 201 | query.orderby 202 | |> Enum.reverse() 203 | |> Enum.reduce(crosstab_plan, fn 204 | {type, {:lit, n}}, acc -> 205 | {:order_by, type, {:column, n - 1}, acc} 206 | 207 | {_type, _expr}, acc -> 208 | acc 209 | end) 210 | 211 | limit_plan = 212 | case query.offset do 213 | nil -> 214 | order_plan 215 | 216 | number -> 217 | {:offset, number, order_plan} 218 | end 219 | 220 | limit_plan = 221 | case query.limit do 222 | nil -> 223 | limit_plan 224 | 225 | number -> 226 | {:limit, number, limit_plan} 227 | end 228 | 229 | union_plan = 230 | case query.union do 231 | nil -> 232 | limit_plan 233 | 234 | {:distinct, other} -> 235 | {:ok, other_plan} = plan(other, context) 236 | 237 | { 238 | :distinct, 239 | :all_columns, 240 | {:union, limit_plan, other_plan} 241 | } 242 | 243 | {:all, other} -> 244 | {:ok, other_plan} = plan(other, context) 245 | {:union, limit_plan, other_plan} 246 | end 247 | 248 | # On first with it will generate the :with plan, and for further just use 249 | # it. 250 | 251 | {with_plan, _} = 252 | query.with 253 | |> Enum.reverse() 254 | |> Enum.reduce({union_plan, %{}}, fn 255 | {name, cols}, {prev_plan, withs} when is_list(cols) -> 256 | # Logger.debug("Prepare plan cols: #{inspect(name)} #{inspect(cols)}") 257 | {prev_plan, withs} 258 | 259 | {name, query}, {prev_plan, withs} -> 260 | case withs[name] do 261 | nil -> 262 | # Logger.debug("Prepare plan query: #{inspect(name)} #{inspect(withs)}") 263 | {:ok, plan} = plan(query, context) 264 | next_plan = {:with, {name, plan}, prev_plan} 265 | {next_plan, Map.put(withs, name, [])} 266 | 267 | # no need to plan it 268 | _columns -> 269 | {prev_plan, withs} 270 | end 271 | end) 272 | 273 | plan = with_plan 274 | 275 | if ExoSQL.debug_mode(context) do 276 | Logger.debug("ExoSQL Plan: #{inspect(plan, pretty: true)}") 277 | end 278 | 279 | {:ok, plan} 280 | end 281 | 282 | defp plan_execute( 283 | {:alias, {{:fn, {function, params}}, alias_}}, 284 | _where, 285 | _all_expressions, 286 | _context 287 | ) do 288 | ex = {:fn, {function, params}} 289 | {:alias, ex, alias_} 290 | end 291 | 292 | defp plan_execute({:alias, {{:table, {db, table}}, alias_}}, where, all_expressions, _context) do 293 | columns = Enum.uniq(get_table_columns_at_expr(:tmp, alias_, all_expressions)) 294 | columns = Enum.map(columns, fn {:tmp, ^alias_, column} -> {db, table, column} end) 295 | quals = get_quals(:tmp, alias_, where) 296 | ex = {:execute, {:table, {db, table}}, quals, columns} 297 | {:alias, ex, alias_} 298 | end 299 | 300 | defp plan_execute({:alias, {%ExoSQL.Query{} = q, alias_}}, _where, _all_expressions, context) do 301 | {:ok, ex} = plan(q, context) 302 | {:alias, ex, alias_} 303 | end 304 | 305 | defp plan_execute({:table, {db, table}}, where, all_expressions, _context) do 306 | columns = Enum.uniq(get_table_columns_at_expr(db, table, all_expressions)) 307 | quals = get_quals(db, table, where) 308 | {:execute, {:table, {db, table}}, quals, columns} 309 | end 310 | 311 | defp plan_execute(nil, _where, _all_expressions, _context) do 312 | nil 313 | end 314 | 315 | defp plan_execute(%ExoSQL.Query{} = q, _where, _all_expressions, context) do 316 | {:ok, q} = plan(q, context) 317 | q 318 | end 319 | 320 | defp plan_execute({:fn, f}, _where, _all_expressions, _context) do 321 | {:fn, f} 322 | end 323 | 324 | # this are with no _where 325 | defp plan_execute({:alias, {{:fn, {function, params}}, alias_}}, _all_expressions, _context) do 326 | ex = {:fn, {function, params}} 327 | {:alias, ex, alias_} 328 | end 329 | 330 | defp plan_execute({:fn, _} = func, _all_expressions, _context), do: func 331 | 332 | defp plan_execute({:table, {db, table}}, all_expressions, _context) do 333 | columns = Enum.uniq(get_table_columns_at_expr(db, table, all_expressions)) 334 | {:execute, {:table, {db, table}}, [], columns} 335 | end 336 | 337 | defp plan_execute(%ExoSQL.Query{} = q, _all_expressions, context) do 338 | {:ok, q} = plan(q, context) 339 | q 340 | end 341 | 342 | ~S""" 343 | Gets all the vars referenced in an expression that refer to a given table 344 | 345 | Given a database and table, and an expression, return all columns from that 346 | {db, table} that are required by those expressions. 347 | 348 | This is used to know which columns to extract from the table. 349 | """ 350 | 351 | defp get_table_columns_at_expr(_db, _table, []) do 352 | [] 353 | end 354 | 355 | defp get_table_columns_at_expr(db, table, l) when is_list(l) do 356 | res = Enum.flat_map(l, &get_table_columns_at_expr(db, table, &1)) 357 | 358 | # Logger.debug("Get columns at table #{inspect {db, table}} at expr #{inspect l}: #{inspect res}") 359 | res 360 | end 361 | 362 | defp get_table_columns_at_expr(db, table, {:op, {_op, op1, op2}}) do 363 | get_table_columns_at_expr(db, table, op1) ++ get_table_columns_at_expr(db, table, op2) 364 | end 365 | 366 | defp get_table_columns_at_expr(db, table, {:column, {db, table, _var} = res}), do: [res] 367 | 368 | defp get_table_columns_at_expr(db, table, {:fn, {_f, params}}) do 369 | get_table_columns_at_expr(db, table, params) 370 | end 371 | 372 | defp get_table_columns_at_expr(db, table, {:alias, {expr, _alias}}) do 373 | get_table_columns_at_expr(db, table, expr) 374 | end 375 | 376 | defp get_table_columns_at_expr(db, table, {:select, query}) do 377 | res = get_table_columns_at_expr(db, table, [query.select, query.where, query.join]) 378 | 379 | # Logger.debug("Get parents #{inspect {db, table}} from #{inspect query, pretty: true}: #{inspect res}") 380 | res 381 | end 382 | 383 | defp get_table_columns_at_expr(db, table, {:case, list}) do 384 | Enum.flat_map(list, fn 385 | {e, v} -> 386 | Enum.flat_map([e, v], &get_table_columns_at_expr(db, table, &1)) 387 | 388 | {v} -> 389 | get_table_columns_at_expr(db, table, v) 390 | end) 391 | end 392 | 393 | defp get_table_columns_at_expr(db, table, {:distinct, expr}) do 394 | get_table_columns_at_expr(db, table, expr) 395 | end 396 | 397 | defp get_table_columns_at_expr(db, table, {:cross_join_lateral, expr}) do 398 | get_table_columns_at_expr(db, table, expr) 399 | end 400 | 401 | defp get_table_columns_at_expr(db, table, {:lateral, expr}) do 402 | get_table_columns_at_expr(db, table, expr) 403 | end 404 | 405 | defp get_table_columns_at_expr(_db, _table, _other) do 406 | [] 407 | end 408 | 409 | ~S""" 410 | If an aggregate function is found, rewrite it to be a real aggregate 411 | 412 | The way to do it is set as first argument the column with the aggregated table 413 | and the rest inside `{:pass, op}`, so its the real function that evaluates it 414 | over the first argument 415 | """ 416 | 417 | defp fix_aggregates_select({:op, {op, op1, op2}}, aggregate_column) do 418 | op1 = fix_aggregates_select(op1, aggregate_column) 419 | op2 = fix_aggregates_select(op2, aggregate_column) 420 | 421 | {:op, {op, op1, op2}} 422 | end 423 | 424 | defp fix_aggregates_select({:fn, {f, args}}, aggregate_column) do 425 | if ExoSQL.Builtins.is_aggregate(f) do 426 | args = for a <- args, do: {:pass, a} 427 | {:fn, {f, [{:column, aggregate_column} | args]}} 428 | else 429 | args = for a <- args, do: fix_aggregates_select(a, aggregate_column) 430 | {:fn, {f, args}} 431 | end 432 | end 433 | 434 | defp fix_aggregates_select({:alias, {expr, alias_}}, aggregate_column) do 435 | {:alias, {fix_aggregates_select(expr, aggregate_column), alias_}} 436 | end 437 | 438 | defp fix_aggregates_select(other, _) do 439 | other 440 | end 441 | 442 | defp has_aggregates({:op, {_op, op1, op2}}) do 443 | has_aggregates(op1) or has_aggregates(op2) 444 | end 445 | 446 | defp has_aggregates({:alias, {expr, _alias}}) do 447 | has_aggregates(expr) 448 | end 449 | 450 | defp has_aggregates({:fn, {f, args}}) do 451 | if not ExoSQL.Builtins.is_aggregate(f) do 452 | Enum.reduce(args, false, fn arg, acc -> 453 | acc or has_aggregates(arg) 454 | end) 455 | else 456 | true 457 | end 458 | end 459 | 460 | defp has_aggregates(l) when is_list(l), do: Enum.any?(l, &has_aggregates/1) 461 | defp has_aggregates(_other), do: false 462 | 463 | defp get_quals(db, table, expressions) when is_list(expressions) do 464 | Enum.flat_map(expressions, &get_quals(db, table, &1)) 465 | end 466 | 467 | defp get_quals(db, table, {:op, {op, {:column, {db, table, column}}, {:lit, value}}}) do 468 | [[column, op, value]] 469 | end 470 | 471 | defp get_quals(db, table, {:op, {op, {:lit, value}}, {:column, {db, table, column}}}) do 472 | [[column, op, value]] 473 | end 474 | 475 | defp get_quals(db, table, {:op, {op, {:column, {db, table, column}}, {:var, variable}}}) do 476 | [[column, op, {:var, variable}]] 477 | end 478 | 479 | defp get_quals(db, table, {:op, {op, {:var, variable}}, {:column, {db, table, column}}}) do 480 | [[column, op, {:var, variable}]] 481 | end 482 | 483 | defp get_quals(db, table, {:op, {"IN", {:column, {db, table, column}}, {:lit, list}}}) 484 | when is_list(list) do 485 | [[column, "IN", list]] 486 | end 487 | 488 | defp get_quals(db, table, {:op, {"AND", op1, op2}}) do 489 | Enum.flat_map([op1, op2], &get_quals(db, table, &1)) 490 | end 491 | 492 | defp get_quals(_db, _table, _expr), do: [] 493 | end 494 | -------------------------------------------------------------------------------- /lib/utils.ex: -------------------------------------------------------------------------------- 1 | require Logger 2 | 3 | defmodule ExoSQL.Utils do 4 | @moduledoc """ 5 | Various assorted utility functions. 6 | """ 7 | 8 | def to_number(nil), do: {:error, nil} 9 | def to_number(n) when is_number(n), do: {:ok, n} 10 | # Weak typing 11 | def to_number(n) when is_binary(n) do 12 | {n, rem} = 13 | if String.contains?(n, ".") do 14 | Float.parse(n) 15 | else 16 | Integer.parse(n) 17 | end 18 | 19 | if rem == "" do 20 | {:ok, n} 21 | else 22 | {:error, :bad_number} 23 | end 24 | end 25 | 26 | def to_number!(n) do 27 | {:ok, number} = to_number(n) 28 | number 29 | end 30 | 31 | # Maybe better way?? 32 | def to_float(n) when is_number(n), do: {:ok, n + 0.0} 33 | # Weak typing 34 | def to_float(n) when is_binary(n) do 35 | {n, rem} = Float.parse(n) 36 | 37 | if rem == "" do 38 | {:ok, n} 39 | else 40 | {:error, :bad_number} 41 | end 42 | end 43 | 44 | def to_float!(n) do 45 | {:ok, number} = to_float(n) 46 | number 47 | end 48 | 49 | def format_result(res) do 50 | s = 51 | for {h, n} <- Enum.with_index(res.columns) do 52 | case h do 53 | {db, table, column} -> 54 | "#{db}.#{table}.#{column}" 55 | 56 | str when is_binary(str) -> 57 | str 58 | 59 | _ -> 60 | "?COL#{n + 1}" 61 | end 62 | end 63 | 64 | widths = Enum.map(s, &String.length/1) 65 | s = [s |> Enum.join(" | ")] 66 | s = [s, "\n"] 67 | totalw = Enum.count(s) * 3 + Enum.reduce(widths, 0, &(&1 + &2)) 68 | # Logger.debug("#{inspect widths} #{inspect totalw}") 69 | s = [s, String.duplicate("-", totalw)] 70 | s = [s, "\n"] 71 | widths = Enum.drop(widths, -1) ++ [0] 72 | 73 | data = 74 | for r <- res.rows do 75 | c = 76 | Enum.join( 77 | Enum.map(Enum.zip(widths, r), fn {w, r} -> 78 | r = 79 | case r do 80 | r when is_list(r) -> "[#{Enum.join(r, ", ")}]" 81 | nil -> "NULL" 82 | other -> other 83 | end 84 | 85 | String.pad_trailing(to_string(r), w) 86 | end), 87 | " | " 88 | ) 89 | 90 | [c, "\n"] 91 | end 92 | 93 | s = [s, data] 94 | 95 | # Logger.debug(inspect s) 96 | to_string(s) 97 | end 98 | 99 | @doc ~S""" 100 | Fron an initial input value, can generate a list. 101 | 102 | The function receives a value and returns either: 103 | * :halt 104 | * {generated_value, next_input} 105 | 106 | All the generated values are returned in order to the caller when :halt is 107 | received. 108 | """ 109 | def generate(input, func) do 110 | case func.(input) do 111 | :halt -> [] 112 | {value, next} -> [value | generate(next, func)] 113 | end 114 | end 115 | end 116 | -------------------------------------------------------------------------------- /mix.exs: -------------------------------------------------------------------------------- 1 | defmodule ExoSQL.Mixfile do 2 | use Mix.Project 3 | 4 | def project do 5 | [ 6 | app: :exosql, 7 | version: "0.2.88", 8 | elixir: "~> 1.5", 9 | start_permanent: Mix.env() == :prod, 10 | deps: deps(), 11 | source_url: "https://github.com/serverboards/exosql/", 12 | homepage_url: "https://serverboards.io", 13 | description: description(), 14 | test_coverage: [tool: ExCoveralls], 15 | preferred_cli_env: [ 16 | coveralls: :test, 17 | "coveralls.detail": :test, 18 | "coveralls.post": :test, 19 | "coveralls.html": :test 20 | ], 21 | package: [ 22 | name: "exosql", 23 | licenses: ["Apache 2.0"], 24 | maintainers: ["David Moreno "], 25 | links: %{ 26 | "Serverboards" => "https://serverboards.io", 27 | "GitHub" => "https://github.com/serverboards/exosql/" 28 | } 29 | ] 30 | ] 31 | end 32 | 33 | # Run "mix help compile.app" to learn about applications. 34 | def application do 35 | [ 36 | extra_applications: [:logger, :httpoison] 37 | ] 38 | end 39 | 40 | defp description do 41 | "Universal SQL engine for Elixir. 42 | 43 | This library implements the SQL logic to perform queries on user provided 44 | databases using a simple interface based on Foreign Data Wrappers from 45 | PostgreSQL. 46 | " 47 | end 48 | 49 | # Run "mix help deps" to learn about dependencies. 50 | defp deps do 51 | [ 52 | {:ex_doc, "~> 0.19.0"}, 53 | {:timex, "~> 3.0"}, 54 | {:csv, "~> 2.1"}, 55 | {:httpoison, "~> 1.0"}, 56 | {:poison, "~> 3.1"}, 57 | {:excoveralls, "~> 0.10", only: :test} 58 | ] 59 | end 60 | end 61 | -------------------------------------------------------------------------------- /mix.lock: -------------------------------------------------------------------------------- 1 | %{ 2 | "certifi": {:hex, :certifi, "2.5.1", "867ce347f7c7d78563450a18a6a28a8090331e77fa02380b4a21962a65d36ee5", [:rebar3], [{:parse_trans, "~>3.3", [hex: :parse_trans, repo: "hexpm", optional: false]}], "hexpm"}, 3 | "combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm"}, 4 | "csv": {:hex, :csv, "2.3.1", "9ce11eff5a74a07baf3787b2b19dd798724d29a9c3a492a41df39f6af686da0e", [:mix], [{:parallel_stream, "~> 1.0.4", [hex: :parallel_stream, repo: "hexpm", optional: false]}], "hexpm"}, 5 | "earmark": {:hex, :earmark, "1.3.2", "b840562ea3d67795ffbb5bd88940b1bed0ed9fa32834915125ea7d02e35888a5", [:mix], [], "hexpm"}, 6 | "ex_doc": {:hex, :ex_doc, "0.19.3", "3c7b0f02851f5fc13b040e8e925051452e41248f685e40250d7e40b07b9f8c10", [:mix], [{:earmark, "~> 1.2", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.10", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm"}, 7 | "excoveralls": {:hex, :excoveralls, "0.11.1", "dd677fbdd49114fdbdbf445540ec735808250d56b011077798316505064edb2c", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"}, 8 | "gettext": {:hex, :gettext, "0.16.1", "e2130b25eebcbe02bb343b119a07ae2c7e28bd4b146c4a154da2ffb2b3507af2", [:mix], [], "hexpm"}, 9 | "hackney": {:hex, :hackney, "1.15.1", "9f8f471c844b8ce395f7b6d8398139e26ddca9ebc171a8b91342ee15a19963f4", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.4", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"}, 10 | "httpoison": {:hex, :httpoison, "1.5.1", "0f55b5b673b03c5c327dac7015a67cb571b99b631acc0bc1b0b98dcd6b9f2104", [:mix], [{:hackney, "~> 1.8", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, 11 | "idna": {:hex, :idna, "6.0.0", "689c46cbcdf3524c44d5f3dde8001f364cd7608a99556d8fbd8239a5798d4c10", [:rebar3], [{:unicode_util_compat, "0.4.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm"}, 12 | "jason": {:hex, :jason, "1.1.2", "b03dedea67a99223a2eaf9f1264ce37154564de899fd3d8b9a21b1a6fd64afe7", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm"}, 13 | "makeup": {:hex, :makeup, "0.5.5", "9e08dfc45280c5684d771ad58159f718a7b5788596099bdfb0284597d368a882", [:mix], [{:nimble_parsec, "~> 0.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm"}, 14 | "makeup_elixir": {:hex, :makeup_elixir, "0.10.0", "0f09c2ddf352887a956d84f8f7e702111122ca32fbbc84c2f0569b8b65cbf7fa", [:mix], [{:makeup, "~> 0.5.5", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm"}, 15 | "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm"}, 16 | "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm"}, 17 | "nimble_parsec": {:hex, :nimble_parsec, "0.4.0", "ee261bb53214943679422be70f1658fff573c5d0b0a1ecd0f18738944f818efe", [:mix], [], "hexpm"}, 18 | "parallel_stream": {:hex, :parallel_stream, "1.0.6", "b967be2b23f0f6787fab7ed681b4c45a215a81481fb62b01a5b750fa8f30f76c", [:mix], [], "hexpm"}, 19 | "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm"}, 20 | "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm"}, 21 | "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.4", "f0eafff810d2041e93f915ef59899c923f4568f4585904d010387ed74988e77b", [:make, :mix, :rebar3], [], "hexpm"}, 22 | "timex": {:hex, :timex, "3.5.0", "b0a23167da02d0fe4f1a4e104d1f929a00d348502b52432c05de875d0b9cffa5", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, 23 | "tzdata": {:hex, :tzdata, "0.5.20", "304b9e98a02840fb32a43ec111ffbe517863c8566eb04a061f1c4dbb90b4d84c", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, 24 | "unicode_util_compat": {:hex, :unicode_util_compat, "0.4.1", "d869e4c68901dd9531385bb0c8c40444ebf624e60b6962d95952775cac5e90cd", [:rebar3], [], "hexpm"}, 25 | } 26 | -------------------------------------------------------------------------------- /src/.gitignore: -------------------------------------------------------------------------------- 1 | sql_lexer.erl 2 | sql_parser.erl 3 | -------------------------------------------------------------------------------- /src/sql_lexer.xrl: -------------------------------------------------------------------------------- 1 | Definitions. 2 | 3 | INT = [0-9]+ 4 | COMMENT = --.* 5 | MINUS = - 6 | RESERVEDL = (select|where|from|as|inner|cross|left|right|outer|join|on|group|by|order|asc|desc|true|false|not|distinct|limit|offset|all|null|case|if|elif|when|then|else|end|union|with|lateral|crosstab) 7 | RESERVEDU = (SELECT|WHERE|FROM|AS|INNER|CROSS|LEFT|RIGHT|OUTER|JOIN|ON|GROUP|BY|ORDER|ASC|DESC|TRUE|FALSE|NOT|DISTINCT|LIMIT|OFFSET|ALL|NULL|CASE|IF|ELIF|WHEN|THEN|ELSE|END|UNION|WITH|LATERAL|CROSSTAB) 8 | ID = [_a-zA-Z][_a-zA-Z0-9]* 9 | COMMA = , 10 | DOT = \. 11 | OP1 = (and|AND) 12 | OP2 = (or|OR) 13 | OP3 = (<|>|<=|>=|==|!=|<>|<|>|=|\|\|) 14 | OP4 = (-|\+) 15 | OP5 = (\*|/|%) 16 | OP6 = (IN|IS|LIKE|ILIKE|CASE|in|is|like|ilike) 17 | SPACE = [\n\t\s]+ 18 | OPEN_PAR = \( 19 | CLOSE_PAR = \) 20 | OPEN_SQB = \[ 21 | CLOSE_SQB = \] 22 | OPEN_BR = \{ 23 | CLOSE_BR = \} 24 | QUOTED_STRING = ("([^"])*"|'([^'])*') 25 | %% " 26 | VAR = \$[_a-zA-Z][_a-zA-Z0-9]* 27 | 28 | Rules. 29 | 30 | {SPACE} : skip_token. 31 | {COMMENT} : skip_token. 32 | {RESERVEDL} : {token, {list_to_atom(string:to_upper(TokenChars)), TokenLine}}. 33 | {RESERVEDU} : {token, {list_to_atom(TokenChars), TokenLine}}. 34 | {OP1} : {token, {op1, TokenLine, TokenChars}}. 35 | {OP2} : {token, {op2, TokenLine, TokenChars}}. 36 | {OP3} : {token, {op3, TokenLine, TokenChars}}. 37 | {OP4} : {token, {op4, TokenLine, TokenChars}}. 38 | {OP5} : {token, {op5, TokenLine, TokenChars}}. 39 | {OP6} : {token, {op6, TokenLine, TokenChars}}. 40 | {ID} : {token, {id, TokenLine, TokenChars}}. 41 | {COMMA} : {token, {comma, TokenLine, TokenChars}}. 42 | {DOT} : {token, {dot, TokenLine, TokenChars}}. 43 | {OPEN_PAR} : {token, {open_par, TokenLine, TokenChars}}. 44 | {CLOSE_PAR} : {token, {close_par, TokenLine, TokenChars}}. 45 | {OPEN_SQB} : {token, {open_sqb, TokenLine, TokenChars}}. 46 | {CLOSE_SQB} : {token, {close_sqb, TokenLine, TokenChars}}. 47 | {OPEN_BR} : {token, {open_br, TokenLine, TokenChars}}. 48 | {CLOSE_BR} : {token, {close_br, TokenLine, TokenChars}}. 49 | {INT}{DOT}{INT} : {token, {litf, TokenLine, to_number(TokenChars)}}. 50 | {INT} : {token, {litn, TokenLine, to_number(TokenChars)}}. 51 | {MINUS}{INT}{DOT}{INT} : {token, {litf, TokenLine, to_number(TokenChars)}}. 52 | {MINUS}{INT} : {token, {litn, TokenLine, to_number(TokenChars)}}. 53 | {QUOTED_STRING} : {token, {lit, TokenLine, string:substr(TokenChars, 2, string:len(TokenChars)-2)}}. 54 | {VAR} : {token, {var, TokenLine, string:substr(TokenChars, 2, string:len(TokenChars))}}. 55 | 56 | Erlang code. 57 | 58 | to_number(S) -> 59 | S2 = list_to_binary(S), 60 | 'Elixir.ExoSQL.Utils':'to_number!'(S2). 61 | -------------------------------------------------------------------------------- /src/sql_parser.yrl: -------------------------------------------------------------------------------- 1 | Nonterminals 2 | query simple_query complex_query 3 | with_list with 4 | select select_expr select_expr_list 5 | from table_list 6 | where expr_list 7 | column table tableid groupby 8 | join join_type cross_join 9 | orderby order_expr_list order_expr asc_desc 10 | limit offset 11 | expr expr_l2 expr_l3 expr_l4 expr_l5 expr_l6 expr_l7 expr_atom 12 | case_expr_list case_expr if_expr_list id_list 13 | . 14 | 15 | Terminals 16 | id comma dot lit litn litf var 17 | open_par close_par open_br close_br open_sqb close_sqb 18 | op1 op2 op3 op4 op5 op6 19 | 'SELECT' 'FROM' 'AS' 'WITH' 20 | 'OUTER' 'LEFT' 'RIGHT' 'INNER' 'CROSS' 'JOIN' 'ON' 'LATERAL' 21 | 'WHERE' 'GROUP' 'BY' 'ORDER' 'ASC' 'DESC' 22 | 'TRUE' 'FALSE' 'NOT' 'NULL' 23 | 'DISTINCT' 'CROSSTAB' 'LIMIT' 'ALL' 'OFFSET' 24 | 'CASE' 'WHEN' 'THEN' 'ELSE' 'END' 25 | 'IF' 'ELIF' 26 | 'UNION' 27 | . 28 | 29 | Rootsymbol query. 30 | 31 | query -> 'WITH' with_list complex_query: maps:put(with, '$2', '$3'). 32 | query -> complex_query: '$1'. 33 | 34 | complex_query -> simple_query 'UNION' complex_query: maps:put(union, {all, '$3'}, '$1'). 35 | complex_query -> simple_query 'UNION' 'ALL' complex_query: maps:put(union, {distinct, '$4'}, '$1'). 36 | 37 | complex_query -> simple_query: '$1'. 38 | 39 | simple_query -> select from join where groupby orderby offset limit: 40 | #{select => '$1', from => '$2', join => '$3', where => '$4', 41 | groupby => '$5', orderby => '$6', offset => '$7', limit => '$8', union => nil, with => []}. 42 | simple_query -> select: 43 | #{select => '$1', from => [], join => [], where => nil, groupby => nil, 44 | orderby => [], limit => nil, offset => nil, union => nil, with => []}. 45 | 46 | with_list -> with: ['$1']. 47 | with_list -> with comma with_list: ['$1' | '$3']. 48 | 49 | with -> id 'AS' open_par complex_query close_par: {unwrap('$1'), '$4'}. 50 | 51 | select -> 'SELECT' 'DISTINCT' 'ON' open_par expr close_par select_expr_list : {'$7', [{distinct, '$5'}]}. 52 | select -> 'SELECT' 'DISTINCT' select_expr_list : {'$3', [{distinct, all_columns}]}. 53 | select -> 'SELECT' 'CROSSTAB' 'ON' open_par id_list close_par select_expr_list : {'$7', [{crosstab, '$5'}]}. 54 | select -> 'SELECT' 'CROSSTAB' select_expr_list : {'$3', [{crosstab, all_columns}]}. 55 | select -> 'SELECT' select_expr_list : {'$2', []}. 56 | 57 | id_list -> id : [unwrap('$1')]. 58 | id_list -> id comma id_list : [unwrap('$1')] ++ '$3'. 59 | 60 | select_expr_list -> select_expr : ['$1']. 61 | select_expr_list -> select_expr comma select_expr_list: ['$1'] ++ '$3'. 62 | select_expr -> expr: '$1'. 63 | select_expr -> expr 'AS' id: {alias, {'$1', unwrap('$3')}}. 64 | select_expr -> expr id: {alias, {'$1', unwrap('$2')}}. 65 | select_expr -> op5: tag('$1', "*"), {all_columns}. 66 | 67 | from -> 'FROM' table_list : '$2'. 68 | table_list -> 'LATERAL' expr: [{cross_join_lateral, '$2'}]. 69 | table_list -> 'LATERAL' expr 'AS' id: [{alias, {{cross_join_lateral, '$2'}, unwrap('$4')}}]. 70 | table_list -> 'LATERAL' expr comma table_list: [{cross_join_lateral, '$2'}] ++ '$4'. 71 | table_list -> 'LATERAL' expr 'AS' id comma table_list: [{alias, {{cross_join_lateral, '$2'}, unwrap('$4')}}] ++ '$4'. 72 | table_list -> table : ['$1']. 73 | table_list -> table comma table_list : ['$1'] ++ '$3'. 74 | 75 | join -> '$empty' : []. 76 | join -> cross_join table join : [{'$1', '$2'}] ++ '$3'. 77 | join -> join_type table 'ON' expr join : [{'$1', {'$2', '$4'}}] ++ '$5'. 78 | 79 | cross_join -> 'CROSS' 'JOIN' 'LATERAL': cross_join_lateral. 80 | cross_join -> 'CROSS' 'JOIN' : cross_join. 81 | 82 | join_type -> 'LEFT' 'JOIN' 'LATERAL': left_join_lateral. 83 | join_type -> 'LEFT' 'OUTER' 'JOIN' 'LATERAL': left_join_lateral. 84 | 85 | join_type -> 'JOIN' : inner_join. 86 | join_type -> 'INNER' 'JOIN' : inner_join. 87 | join_type -> 'LEFT' 'JOIN' : left_join. 88 | join_type -> 'LEFT' 'OUTER' 'JOIN' : left_join. 89 | join_type -> 'RIGHT' 'JOIN' : right_join. 90 | join_type -> 'RIGHT' 'OUTER' 'JOIN' : right_join. 91 | 92 | where -> '$empty' : nil. 93 | where -> 'WHERE' 'expr' : '$2'. 94 | 95 | groupby -> '$empty' : nil. 96 | groupby -> 'GROUP' 'BY' expr_list : '$3'. 97 | 98 | orderby -> '$empty' : []. 99 | orderby -> 'ORDER' 'BY' order_expr_list : '$3'. 100 | order_expr_list -> order_expr: ['$1']. 101 | order_expr_list -> order_expr comma order_expr_list: ['$1'] ++ '$3'. 102 | order_expr -> expr asc_desc: {'$2', '$1'}. 103 | asc_desc -> '$empty' : asc. 104 | asc_desc -> 'ASC' : asc. 105 | asc_desc -> 'DESC' : desc. 106 | 107 | limit -> '$empty': nil. 108 | limit -> 'LIMIT' litn: unwrap_raw('$2'). 109 | limit -> 'LIMIT' 'ALL': nil. 110 | 111 | offset -> '$empty': nil. 112 | offset -> 'OFFSET' litn: unwrap_raw('$2'). 113 | 114 | expr_list -> expr : ['$1']. 115 | expr_list -> expr comma expr_list: ['$1'] ++ '$3'. 116 | 117 | expr -> expr_l2 op1 expr: {op, {unwrap_u('$2'), '$1', '$3'}}. 118 | expr -> expr_l2: '$1'. 119 | 120 | expr_l2 -> expr_l3 op2 expr_l2: {op, {unwrap_u('$2'), '$1', '$3'}}. 121 | expr_l2 -> expr_l3: '$1'. 122 | 123 | expr_l3 -> expr_l4 op3 expr_l3: {op, {unwrap('$2'), '$1', '$3'}}. 124 | expr_l3 -> expr_l4: '$1'. 125 | 126 | expr_l4 -> expr_l5 op4 expr_l4: {op, {unwrap('$2'), '$1', '$3'}}. 127 | expr_l4 -> expr_l5: '$1'. 128 | 129 | expr_l5 -> expr_l6 op5 expr_l5: {op, {unwrap('$2'), '$1', '$3'}}. 130 | expr_l5 -> expr_l6: '$1'. 131 | 132 | expr_l6 -> expr_l7 op6 expr_l6: {op, {unwrap_u('$2'), '$1', '$3'}}. 133 | expr_l6 -> expr_l7: '$1'. 134 | 135 | expr_l7 -> 'NOT' expr_l7: {op, {'not', '$2'}}. 136 | expr_l7 -> expr_atom: '$1'. 137 | 138 | expr_atom -> column : {column, '$1'}. 139 | expr_atom -> lit : {lit, unwrap('$1')}. 140 | expr_atom -> litn : {lit, unwrap_raw('$1')}. 141 | expr_atom -> litf : {lit, unwrap_raw('$1')}. 142 | expr_atom -> 'TRUE' : {lit, true}. 143 | expr_atom -> 'FALSE' : {lit, false}. 144 | expr_atom -> 'NULL' : {lit, nil}. 145 | expr_atom -> var : {var, unwrap('$1')}. 146 | expr_atom -> open_par simple_query close_par : {select, '$2'}. 147 | expr_atom -> open_par expr close_par : '$2'. 148 | expr_atom -> id open_par close_par : {fn, {unwrap_d('$1'), []}}. 149 | expr_atom -> id open_par expr_list close_par : {fn, {unwrap_d('$1'), '$3'}}. 150 | expr_atom -> 'JOIN' open_par expr_list close_par : {fn, {'Elixir.List':to_string("join"), '$3'}}. 151 | expr_atom -> id open_par op5 close_par: tag('$3', "*"), {fn, {unwrap_d('$1'), [{lit, "*"}]}}. 152 | expr_atom -> id open_par 'DISTINCT' expr close_par: {fn, {unwrap_d('$1'), [{distinct, '$4'}]}}. 153 | expr_atom -> open_sqb expr_list close_sqb: {list, '$2'}. 154 | expr_atom -> 'CASE' expr case_expr_list: {'case', '$2', '$3'}. 155 | expr_atom -> 'CASE' case_expr_list: {'case', '$2'}. 156 | expr_atom -> 'IF' expr 'THEN' expr if_expr_list: {'case', [ {'$2', '$4'} | '$5' ]}. 157 | 158 | case_expr_list -> case_expr case_expr_list: ['$1' | '$2']. 159 | case_expr_list -> 'ELSE' expr 'END': [{'$2'}]. 160 | case_expr_list -> 'END': []. 161 | 162 | case_expr -> 'WHEN' expr 'THEN' expr: {'$2', '$4'}. 163 | 164 | if_expr_list -> 'ELIF' expr 'THEN' expr if_expr_list: [{'$2', '$4'} | '$5']. 165 | if_expr_list -> 'ELSE' expr 'END': [{{lit, true}, '$2'}]. 166 | if_expr_list -> 'END': []. 167 | 168 | 169 | column -> id dot id dot id : {unwrap('$1'), unwrap('$3'), unwrap('$5')}. 170 | column -> id dot id : {nil, unwrap('$1'), unwrap('$3')}. 171 | column -> id : {nil, nil, unwrap('$1')}. 172 | 173 | table -> tableid 'AS' id : {alias, {'$1', unwrap('$3')}}. 174 | table -> tableid id : {alias, {'$1', unwrap('$2')}}. 175 | table -> tableid : '$1'. 176 | tableid -> id dot id : {table, {unwrap('$1'), unwrap('$3')}}. 177 | tableid -> id : {table, {nil, unwrap('$1')}}. 178 | tableid -> open_par complex_query close_par : {select, '$2'}. 179 | tableid -> id open_par expr_list close_par : {fn, {unwrap_d('$1'), '$3'}}. 180 | 181 | select -> column comma select: [unwrap('$1')] ++ '$3'. 182 | 183 | Erlang code. 184 | 185 | unwrap_d({_,_,V}) -> 'Elixir.String':downcase('Elixir.List':to_string(V)). 186 | unwrap_u({_,_,V}) -> 'Elixir.String':upcase('Elixir.List':to_string(V)). 187 | unwrap({_,_,V}) -> 'Elixir.List':to_string(V). 188 | unwrap_raw({_,_,V}) -> V. 189 | tag(A, B) -> 190 | A1 = unwrap(A), 191 | %% io:format("DEBUG: ~p == ~p", [A1, B]), 192 | A2 = 'Elixir.String':upcase(A1), 193 | A2 = 'Elixir.List':to_string(B). 194 | -------------------------------------------------------------------------------- /test/builtins_test.exs: -------------------------------------------------------------------------------- 1 | require Logger 2 | 3 | defmodule ExoSQL.BuiltinsTest do 4 | use ExUnit.Case 5 | doctest ExoSQL.Builtins 6 | @moduletag :capture_log 7 | 8 | test "Datetime options" do 9 | # 01:02:03 AM 10 | orig_time = 60 * 60 * 1 + 60 * 2 + 3 11 | dt = ExoSQL.Builtins.to_datetime(orig_time) 12 | 13 | assert ExoSQL.Builtins.to_string_(dt) == ExoSQL.Builtins.strftime(dt) 14 | assert ExoSQL.Builtins.to_string_(dt) == ExoSQL.Builtins.strftime(dt, "%i") 15 | assert "1970-01-01" == ExoSQL.Builtins.strftime(dt, "%Y-%m-%d") 16 | assert "01:02:03" == ExoSQL.Builtins.strftime(dt, "%H:%M:%S") 17 | assert "#{orig_time}" == ExoSQL.Builtins.strftime(dt, "%s") 18 | assert "%s #{orig_time}" == ExoSQL.Builtins.strftime(dt, "%%s %s") 19 | assert "01" == ExoSQL.Builtins.strftime(dt, "%V") 20 | 21 | # all formats 22 | dt = ExoSQL.DateTime.to_datetime("2018-02-22") 23 | assert dt == ExoSQL.DateTime.to_datetime("2018-02-22 00:00") 24 | assert dt == ExoSQL.DateTime.to_datetime("2018-02-22T00:00") 25 | assert dt == ExoSQL.DateTime.to_datetime("2018-02-22T00:00:00") 26 | assert dt == ExoSQL.DateTime.to_datetime("2018-02-22 00:00:00") 27 | 28 | dt = ExoSQL.DateTime.to_datetime("2018-07-01T04:25:50Z") 29 | assert dt == ExoSQL.DateTime.to_datetime("2018-07-01T06:25:50+02:00") 30 | 31 | assert nil == ExoSQL.Builtins.to_datetime(nil) 32 | end 33 | 34 | test "String substr" do 35 | assert ExoSQL.Builtins.substr("test", 1) == "est" 36 | assert ExoSQL.Builtins.substr("test", 1, 2) == "es" 37 | assert ExoSQL.Builtins.substr("test", 1, -2) == "e" 38 | assert ExoSQL.Builtins.substr(nil, 1) == "" 39 | 40 | dt = ExoSQL.Builtins.to_datetime("2018-02-10T11:54:34") 41 | assert ExoSQL.Builtins.substr(dt, 0, 10) == "2018-02-10" 42 | end 43 | 44 | test "Split" do 45 | assert ExoSQL.Builtins.split(nil) == [] 46 | end 47 | 48 | test "jp test" do 49 | json = %{ 50 | "first_name" => "Anonymous", 51 | "last_name" => "--", 52 | "addresses" => [ 53 | %{ 54 | "street" => "Main Rd" 55 | }, 56 | %{ 57 | "street" => "Side Rd" 58 | } 59 | ], 60 | "email" => "admin@example.org" 61 | } 62 | 63 | assert ExoSQL.Builtins.jp(json, "") == json 64 | assert ExoSQL.Builtins.jp(json, "/first_name") == json["first_name"] 65 | assert ExoSQL.Builtins.jp(json, "none") == nil 66 | assert ExoSQL.Builtins.jp(json, "/addresses") == json["addresses"] 67 | assert ExoSQL.Builtins.jp(json, "/addresses/0/street") == "Main Rd" 68 | assert ExoSQL.Builtins.jp(json, "/addresses/10/street") == nil 69 | end 70 | 71 | test "urlparse" do 72 | url = "https://serverboards.io/download/" 73 | email = "connect@serverboards.io" 74 | email2 = "mailto://connect@serverboards.io" 75 | urlq = "https://serverboards.io/download/?q=test&utm_campaign=exosql" 76 | 77 | assert ExoSQL.Builtins.urlparse(url, "scheme") == "https" 78 | assert ExoSQL.Builtins.urlparse(url, "host") == "serverboards.io" 79 | 80 | assert ExoSQL.Builtins.urlparse(email, "scheme") == nil 81 | assert ExoSQL.Builtins.urlparse(email, "host") == nil 82 | assert ExoSQL.Builtins.urlparse(email, "path") == "connect@serverboards.io" 83 | 84 | assert ExoSQL.Builtins.urlparse(email2, "scheme") == "mailto" 85 | assert ExoSQL.Builtins.urlparse(email2, "host") == "serverboards.io" 86 | assert ExoSQL.Builtins.urlparse(email2, "user") == "connect" 87 | 88 | parsed = ExoSQL.Builtins.urlparse(urlq) 89 | # Logger.debug(inspect parsed) 90 | assert ExoSQL.Builtins.jp(parsed, "host") == "serverboards.io" 91 | assert ExoSQL.Builtins.jp(parsed, "query/q") == "test" 92 | assert ExoSQL.Builtins.urlparse(urlq, "query/utm_campaign") == "exosql" 93 | 94 | assert ExoSQL.Builtins.urlparse(nil, "query/utm_campaign") == nil 95 | 96 | assert ExoSQL.Builtins.urlparse("https://www.google.com", "domain") == "google" 97 | assert ExoSQL.Builtins.urlparse("https://linktr.ee", "domain") == "linktr" 98 | assert ExoSQL.Builtins.urlparse("https://www.google.co.uk", "domain") == "google" 99 | assert ExoSQL.Builtins.urlparse("https://beta.serverboards.io", "domain") == "serverboards" 100 | assert ExoSQL.Builtins.urlparse("https://www.csail.mit.edu/", "domain") == "mit" 101 | assert ExoSQL.Builtins.urlparse("https://en.wikipedia.org/", "domain") == "wikipedia" 102 | end 103 | 104 | test "format test" do 105 | assert ExoSQL.Builtins.format("%d €", 2.22) == "2 €" 106 | assert ExoSQL.Builtins.format("%.2f €", 2) == "2.00 €" 107 | 108 | assert ExoSQL.Builtins.format("%k €", 2) == "2 €" 109 | assert ExoSQL.Builtins.format("%k €", 2.33) == "2 €" 110 | assert ExoSQL.Builtins.format("%k €", 2000) == "2k €" 111 | assert ExoSQL.Builtins.format("%k €", 22_000) == "22k €" 112 | assert ExoSQL.Builtins.format("%k €", 222_000) == "222k €" 113 | assert ExoSQL.Builtins.format("%k €", 2_000_000) == "2M €" 114 | 115 | assert ExoSQL.Builtins.format("%.k €", 0.00) == "0.00 €" 116 | assert ExoSQL.Builtins.format("%.k €", 0.53) == "0.53 €" 117 | assert ExoSQL.Builtins.format("%.k €", 2.53) == "2.53 €" 118 | assert ExoSQL.Builtins.format("%.k €", 24.53) == "24.53 €" 119 | assert ExoSQL.Builtins.format("%.k €", 200.53) == "200.53 €" 120 | assert ExoSQL.Builtins.format("%.k €", 2_000.53) == "2000 €" 121 | assert ExoSQL.Builtins.format("%.k €", 20_200.53) == "20200 €" 122 | assert ExoSQL.Builtins.format("%.k €", 200_400.53) == "200.4K €" 123 | assert ExoSQL.Builtins.format("%.k €", 2_200_000.53) == "2.2M €" 124 | 125 | assert ExoSQL.Builtins.format("%,k €", 0.00) == "0 €" 126 | assert ExoSQL.Builtins.format("%,k €", 0.53) == "0,53 €" 127 | assert ExoSQL.Builtins.format("%,k €", 2.53) == "2,53 €" 128 | assert ExoSQL.Builtins.format("%,k €", 24.53) == "24,53 €" 129 | assert ExoSQL.Builtins.format("%,k €", 81.50) == "81,50 €" 130 | assert ExoSQL.Builtins.format("%,k €", 200.53) == "200,53 €" 131 | assert ExoSQL.Builtins.format("%,k €", 2_000.53) == "2.000 €" 132 | assert ExoSQL.Builtins.format("%,k €", 20_200.53) == "20.200 €" 133 | assert ExoSQL.Builtins.format("%,k €", 200_400.53) == "200,4K €" 134 | assert ExoSQL.Builtins.format("%,k €", 200_000.53) == "200K €" 135 | assert ExoSQL.Builtins.format("%,k €", 2_001_000.53) == "2MM €" 136 | assert ExoSQL.Builtins.format("%,k €", 2_200_000.53) == "2,2MM €" 137 | end 138 | 139 | test "datediff" do 140 | assert ExoSQL.DateTime.datediff("2018-01-01", "2018-02-01", "months") == 1 141 | assert ExoSQL.DateTime.datediff("2018-01-01", "2018-02-01", "years") == 0 142 | assert ExoSQL.DateTime.datediff("2018-01-01", "2018-02-01", "seconds") == 31 * 24 * 60 * 60 143 | assert ExoSQL.DateTime.datediff("2018-01-01", "2018-02-01", "days") == 31 144 | assert ExoSQL.DateTime.datediff("2018-01-01", "2018-02-01", "weeks") == 4 145 | 146 | assert ExoSQL.DateTime.datediff("2017-01-01", "2018-02-01", "months") == 13 147 | assert ExoSQL.DateTime.datediff("2017-01-01", "2018-02-01", "years") == 1 148 | 149 | assert ExoSQL.DateTime.datediff("2018-01-01", "2017-02-01", "months") == -11 150 | assert ExoSQL.DateTime.datediff("2018-01-01", "2017-02-01", "years") == 0 151 | 152 | assert ExoSQL.DateTime.datediff("2018-03-01", "2018-02-01", "seconds") == -2_419_200 153 | 154 | assert ExoSQL.DateTime.datediff(ExoSQL.Builtins.range("2018-01-01", "2018-01-02"), "seconds") == 155 | 24 * 60 * 60 156 | 157 | assert ExoSQL.DateTime.datediff(ExoSQL.Builtins.range("2018-01-01", "2018-01-02")) == 1 158 | assert ExoSQL.DateTime.datediff(ExoSQL.Builtins.range("2018-01-01", "2018-01-08")) == 7 159 | 160 | assert ExoSQL.DateTime.datediff(ExoSQL.Builtins.range("2018-01-01", "2018-01-08"), "days") == 161 | 7 162 | 163 | assert ExoSQL.DateTime.datediff(ExoSQL.Builtins.range("2018-01-01", "2018-01-08"), "seconds") == 164 | 7 * 24 * 60 * 60 165 | 166 | assert ExoSQL.DateTime.datediff( 167 | "2017-01-01T00:00:00+01:00", 168 | "2018-12-31T23:59:59+01:00", 169 | "months" 170 | ) == 24 171 | 172 | assert ExoSQL.DateTime.datediff( 173 | "2017-01-01T00:00:00+01:00", 174 | "2017-12-31T23:59:59+01:00", 175 | "months" 176 | ) == 12 177 | 178 | assert ExoSQL.DateTime.datediff( 179 | "2017-01-01T00:00:00+01:00", 180 | "2018-01-01T00:00:00+01:00", 181 | "months" 182 | ) == 12 183 | 184 | assert ExoSQL.DateTime.datediff( 185 | "2017-01-01T00:00:00+01:00", 186 | "2017-01-31T23:59:59+01:00", 187 | "months" 188 | ) == 1 189 | 190 | assert ExoSQL.DateTime.datediff( 191 | "2017-02-01T00:00:00+01:00", 192 | "2017-02-01T23:59:59+01:00", 193 | "months" 194 | ) == 0 195 | 196 | assert ExoSQL.DateTime.datediff( 197 | "2017-02-01T00:00:00+01:00", 198 | "2017-02-28T23:59:59+01:00", 199 | "months" 200 | ) == 0 201 | 202 | assert ExoSQL.DateTime.datediff( 203 | "2017-02-01T00:00:00+01:00", 204 | "2017-03-01T23:59:59+01:00", 205 | "months" 206 | ) == 1 207 | end 208 | 209 | test "format string" do 210 | assert ExoSQL.Format.format("%02d-%02d-%02d", [2018, 10, 1]) == "2018-10-01" 211 | assert ExoSQL.Format.format("%+d %+d %+d", [2018, "-10", "0"]) == "+2018 -10 0" 212 | 213 | assert ExoSQL.Format.format("%+f %+f %+f %+f", [2018, -0.43, 0.43, 0]) == 214 | "+2018.00 -0.43 +0.43 0.00" 215 | 216 | assert ExoSQL.Format.format("%10s|%-5s|%3s", ["spaces", "hash", "slash"]) == 217 | " spaces|hash |slash" 218 | end 219 | 220 | test "Builtin simplifications" do 221 | assert ExoSQL.Builtins.simplify("json", [{:lit, nil}]) == {:lit, nil} 222 | assert ExoSQL.Builtins.simplify("json", [{:lit, "1"}]) == {:lit, 1} 223 | 224 | assert ExoSQL.Builtins.simplify("regex", [{:column, {:tmp, :tmp, "a"}}, {:lit, ".*"}]) == 225 | {:fn, 226 | {{ExoSQL.Builtins, :regex, "regex"}, 227 | [column: {:tmp, :tmp, "a"}, lit: ~r/.*/, lit: false]}} 228 | 229 | assert ExoSQL.Builtins.simplify("jp", [{:column, {:tmp, :tmp, "a"}}, {:lit, "a/b/c"}]) == 230 | {:fn, 231 | {{ExoSQL.Builtins, :jp, "jp"}, [column: {:tmp, :tmp, "a"}, lit: ["a", "b", "c"]]}} 232 | 233 | assert ExoSQL.Builtins.simplify("format", [ 234 | {:lit, "W%02, %,k €"}, 235 | {:column, {:tmp, :tmp, "a"}}, 236 | {:column, {:tmp, :tmp, "b"}} 237 | ]) == 238 | {:fn, 239 | {{ExoSQL.Builtins, :format, "format"}, 240 | [ 241 | lit: [" €", {",", "k"}, "W%02, "], 242 | column: {:tmp, :tmp, "a"}, 243 | column: {:tmp, :tmp, "b"} 244 | ]}} 245 | end 246 | 247 | test "Duration parsing IS(8601)" do 248 | duration = ExoSQL.DateTime.Duration.parse!("P1Y2M3D4WT10H20M30S") 249 | 250 | assert duration.years == 1 251 | assert duration.months == 2 252 | # may be more than a month, but I dont know how much, so I will add days. 253 | assert duration.days == 7 * 4 + 3 254 | 255 | assert duration.seconds == 10 * 60 * 60 + 20 * 60 + 30 256 | 257 | duration = ExoSQL.DateTime.Duration.parse!("-T10M") 258 | assert duration.seconds == -10 * 60 259 | 260 | duration = ExoSQL.DateTime.Duration.parse!("1Y") 261 | assert duration == %ExoSQL.DateTime.Duration{years: 1} 262 | 263 | {:error, _} = ExoSQL.DateTime.Duration.parse("nonsense") 264 | 265 | duration = ExoSQL.DateTime.Duration.parse!("-30DT30M") 266 | assert duration == %ExoSQL.DateTime.Duration{days: -30, seconds: -30 * 60} 267 | end 268 | 269 | test "Add durations" do 270 | date = ExoSQL.DateTime.to_datetime("2016-02-01T10:30:00") 271 | 272 | duration = ExoSQL.DateTime.Duration.parse!("1D") 273 | ndate = ExoSQL.DateTime.Duration.datetime_add(date, duration) 274 | assert date.year == ndate.year 275 | assert date.month == ndate.month 276 | assert date.day + 1 == ndate.day 277 | assert date.hour == ndate.hour 278 | assert date.minute == ndate.minute 279 | assert date.second == ndate.second 280 | 281 | ndate = ExoSQL.DateTime.Duration.datetime_add(date, "1Y1M") 282 | assert date.year + 1 == ndate.year 283 | assert date.month + 1 == ndate.month 284 | assert date.day == ndate.day 285 | assert date.hour == ndate.hour 286 | assert date.minute == ndate.minute 287 | assert date.second == ndate.second 288 | 289 | ndate = ExoSQL.DateTime.Duration.datetime_add(date, "T29M10S") 290 | assert date.year == ndate.year 291 | assert date.month == ndate.month 292 | assert date.day == ndate.day 293 | assert date.hour == ndate.hour 294 | assert date.minute + 29 == ndate.minute 295 | assert date.second + 10 == ndate.second 296 | 297 | ndate = ExoSQL.DateTime.Duration.datetime_add(date, "30D") 298 | assert date.year == ndate.year 299 | assert date.month + 1 == ndate.month 300 | assert date.day + 1 == ndate.day 301 | assert date.hour == ndate.hour 302 | assert date.minute == ndate.minute 303 | assert date.second == ndate.second 304 | 305 | ndate = ExoSQL.DateTime.Duration.datetime_add(date, "-31DT2360M20S") 306 | assert 2015 == ndate.year 307 | assert 12 == ndate.month 308 | assert 30 == ndate.day 309 | assert 19 == ndate.hour 310 | assert 9 == ndate.minute 311 | assert 40 == ndate.second 312 | end 313 | 314 | test "Math test" do 315 | epsilon = 0.000005 316 | e = 2.71828 317 | 318 | assert -1 == ExoSQL.Builtins.sign(-32) 319 | assert -1 == ExoSQL.Builtins.sign("-32") 320 | assert 1 == ExoSQL.Builtins.sign("32") 321 | assert 1 == ExoSQL.Builtins.sign(32) 322 | 323 | assert 4 == ExoSQL.Builtins.power(-2, 2) 324 | assert 4 == ExoSQL.Builtins.power(2, "2") 325 | assert 4 == ExoSQL.Builtins.power("2", "2") 326 | assert abs(2 - ExoSQL.Builtins.power(4, 0.5)) < epsilon 327 | 328 | assert 2 == ExoSQL.Builtins.sqrt(4) 329 | assert 4 == ExoSQL.Builtins.sqrt(16) 330 | 331 | assert 1 == ExoSQL.Builtins.log(10) 332 | assert 2 == ExoSQL.Builtins.log(100) 333 | 334 | assert abs(1 - ExoSQL.Builtins.ln(e)) < epsilon 335 | 336 | assert 0 == ExoSQL.Builtins.mod(10, 2) 337 | assert 0 == ExoSQL.Builtins.mod(10, 2.5) 338 | assert 1 == ExoSQL.Builtins.mod(10, 3) 339 | end 340 | 341 | test "Regex" do 342 | context = %{row: ["test"]} 343 | 344 | assert ExoSQL.Builtins.regex("test", "t") == ["t"] 345 | 346 | regx = ExoSQL.Expr.simplify({:fn, {"regex", [{:column, 0}, {:lit, "t"}]}}, context) 347 | Logger.debug("simpl #{inspect(regx)}") 348 | {:fn, {"regex", [_arg1, arg2]}} = regx 349 | assert arg2 != {:lit, "t"} 350 | res = ExoSQL.Expr.run_expr(regx, context) 351 | assert res == ["t"] 352 | 353 | regx = 354 | ExoSQL.Expr.simplify({:fn, {"regex", [{:column, 0}, {:lit, "(t)"}, {:lit, 1}]}}, context) 355 | 356 | Logger.debug("simpl #{inspect(regx)}") 357 | {:fn, {"regex", [_arg1, arg2, {:lit, 1}]}} = regx 358 | assert arg2 != {:lit, "(t)"} 359 | res = ExoSQL.Expr.run_expr(regx, context) 360 | assert res == "t" 361 | end 362 | 363 | test "Regex All" do 364 | context = %{row: ["test"]} 365 | 366 | assert ExoSQL.Builtins.regex("test", "t") == ["t"] 367 | 368 | regx = ExoSQL.Expr.simplify({:fn, {"regex_all", [{:column, 0}, {:lit, "t"}]}}, context) 369 | Logger.debug("simpl #{inspect(regx)}") 370 | {:fn, {"regex_all", [_arg1, arg2]}} = regx 371 | assert arg2 != {:lit, "t"} 372 | res = ExoSQL.Expr.run_expr(regx, context) 373 | assert res == [["t"], ["t"]] 374 | 375 | regx = 376 | ExoSQL.Expr.simplify( 377 | {:fn, {"regex_all", [{:column, 0}, {:lit, "(t)"}, {:lit, 1}]}}, 378 | context 379 | ) 380 | 381 | Logger.debug("simpl #{inspect(regx)}") 382 | {:fn, {"regex_all", [_arg1, arg2, {:lit, 1}]}} = regx 383 | assert arg2 != {:lit, "(t)"} 384 | res = ExoSQL.Expr.run_expr(regx, context) 385 | assert res == ["t", "t"] 386 | end 387 | end 388 | -------------------------------------------------------------------------------- /test/data/csv/campaigns.csv: -------------------------------------------------------------------------------- 1 | id,name,datestart,dateend 2 | 1,2017q1,2017-01-01,2017-03-31 3 | 2,2017q2,2017-04-01,2017-05-31 4 | 3,2017h2,2017-06-01,2017-12-31 5 | 4,2018h1,2018-01-01,2018-05-31 6 | -------------------------------------------------------------------------------- /test/data/csv/family.csv: -------------------------------------------------------------------------------- 1 | id,name,parentA_id,parentB_id 2 | 0,LUCA,NULL,NULL 3 | 1,Mom,0,0 4 | 2,Dad,0,0 5 | 3,Son,1,2 6 | 4,Alice,0,0 7 | 5,Grandson,3,4 8 | -------------------------------------------------------------------------------- /test/data/csv/json.csv: -------------------------------------------------------------------------------- 1 | id,json 2 | 1,"[{""email"": ""one@example.com"", ""name"": ""uno""}, {""email"": ""two@example.com"", ""name"": ""dos""}]" 3 | 2,"[{""email"": ""three@example.com"", ""name"": ""tres""}, {""email"": ""four@example.com"", ""name"": ""cuatro""}]" 4 | -------------------------------------------------------------------------------- /test/data/csv/products.csv: -------------------------------------------------------------------------------- 1 | id,name,price,stock 2 | 1,sugus,3,1000 3 | 2,lollipop,11,20 4 | 3,donut,30,0 5 | 4,water,20,1 6 | -------------------------------------------------------------------------------- /test/data/csv/purchases.csv: -------------------------------------------------------------------------------- 1 | id,user_id,product_id,amount,date 2 | 1,1,1,10,2017-01-21 3 | 2,1,2,20,2017-03-10 4 | 3,1,3,10,2018-01-02 5 | 4,2,2,100,2015-08-10 6 | 5,2,4,10,2017-10-12 7 | 6,3,1,10,2017-12-31 8 | -------------------------------------------------------------------------------- /test/data/csv/unnestarray.csv: -------------------------------------------------------------------------------- 1 | name,ids 2 | dmoreno,1;2;3;4;5 3 | admin, 4 | -------------------------------------------------------------------------------- /test/data/csv/urls.csv: -------------------------------------------------------------------------------- 1 | id,url,name 2 | 2,http://www.serverboards.io,Serverboards 3 | 3,https://serverboards.io,Serverboards 4 | 4,http://www.facebook.com,Facebook 5 | 5,https://serverboards.io/e404,Serverboards 6 | -------------------------------------------------------------------------------- /test/data/csv/users.csv: -------------------------------------------------------------------------------- 1 | id,name,email 2 | 1,David,dmono@example.org 3 | 2,Javier,javier@example.org 4 | 3,Patricio,patricio@example.org 5 | -------------------------------------------------------------------------------- /test/debug_test.exs: -------------------------------------------------------------------------------- 1 | require Logger 2 | 3 | defmodule DebugTest do 4 | use ExUnit.Case 5 | import ExUnit.CaptureLog 6 | @moduletag :capture_log 7 | @moduletag timeout: 5_000 8 | 9 | @context %{ 10 | "A" => {ExoSQL.Csv, path: "test/data/csv/"}, 11 | "__vars__" => %{"debug" => true} 12 | } 13 | 14 | def analyze_query!(query, context \\ @context) do 15 | Logger.debug("Query is:\n\n#{query}") 16 | {:ok, parsed} = ExoSQL.parse(query, context) 17 | Logger.debug("Parsed is #{inspect(parsed, pretty: true)}") 18 | {:ok, plan} = ExoSQL.Planner.plan(parsed, context) 19 | Logger.debug("Plan is #{inspect(plan, pretty: true)}") 20 | {:ok, result} = ExoSQL.Executor.execute(plan, context) 21 | # Logger.debug("Raw result is #{inspect(result, pretty: true)}") 22 | Logger.debug("Result:\n#{ExoSQL.format_result(result)}") 23 | result 24 | end 25 | 26 | test "Simple parse SQL" do 27 | assert capture_log(fn -> 28 | analyze_query!("SELECT * FROM purchases") 29 | end) =~ "ExoSQL Executor" 30 | end 31 | end 32 | -------------------------------------------------------------------------------- /test/esql_test.exs: -------------------------------------------------------------------------------- 1 | require Logger 2 | 3 | defmodule ExoSQLTest do 4 | use ExUnit.Case 5 | doctest ExoSQL 6 | doctest ExoSQL.Expr 7 | @moduletag :capture_log 8 | 9 | test "Get schema data" do 10 | context = %{ 11 | "A" => {ExoSQL.Csv, path: "test/data/csv/"} 12 | } 13 | 14 | {:ok, tables} = ExoSQL.schema("A", context) 15 | Logger.debug("Schema data: #{inspect(tables)}") 16 | 17 | for t <- tables do 18 | {:ok, table} = ExoSQL.schema("A", t, context) 19 | Logger.debug("Table data A.#{inspect(table)}") 20 | end 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /test/executor_test.exs: -------------------------------------------------------------------------------- 1 | require Logger 2 | 3 | defmodule ExecutorTest do 4 | use ExUnit.Case 5 | doctest ExoSQL.Executor, import: true 6 | @moduletag :capture_log 7 | 8 | @context %{"A" => {ExoSQL.Csv, path: "test/data/csv/"}} 9 | 10 | test "Execute a simple manual plan" do 11 | plan = 12 | {:execute, {:table, {"A", "products"}}, [], 13 | [{"A", "products", "name"}, {"A", "products", "price"}]} 14 | 15 | {:ok, result} = ExoSQL.Executor.execute(plan, @context) 16 | 17 | assert result == %ExoSQL.Result{ 18 | columns: [{"A", "products", "name"}, {"A", "products", "price"}], 19 | rows: [ 20 | ["sugus", "3"], 21 | ["lollipop", "11"], 22 | ["donut", "30"], 23 | ["water", "20"] 24 | ] 25 | } 26 | end 27 | 28 | test "Execute a mix complex manual plan" do 29 | plan = { 30 | :select, 31 | {:filter, 32 | {:execute, {:table, {"A", "products"}}, [], 33 | [{"A", "products", "name"}, {"A", "products", "price"}]}, 34 | {:op, {">", {:column, {"A", "products", "price"}}, {:lit, "10"}}}}, 35 | [{:column, {"A", "products", "name"}}, {:column, {"A", "products", "price"}}] 36 | } 37 | 38 | {:ok, result} = ExoSQL.Executor.execute(plan, @context) 39 | 40 | assert result == %ExoSQL.Result{ 41 | columns: [{"A", "products", "name"}, {"A", "products", "price"}], 42 | rows: [ 43 | ["lollipop", "11"], 44 | ["donut", "30"], 45 | ["water", "20"] 46 | ] 47 | } 48 | end 49 | 50 | test "Execute a cross join" do 51 | plan = { 52 | :cross_join, 53 | {:execute, {:table, {"A", "purchases"}}, [], 54 | [{"A", "purchases", "user_id"}, {"A", "purchases", "product_id"}]}, 55 | {:execute, {:table, {"A", "products"}}, [], 56 | [{"A", "products", "id"}, {"A", "products", "name"}]} 57 | } 58 | 59 | {:ok, result} = ExoSQL.Executor.execute(plan, @context) 60 | 61 | assert result == %ExoSQL.Result{ 62 | columns: [ 63 | {"A", "purchases", "user_id"}, 64 | {"A", "purchases", "product_id"}, 65 | {"A", "products", "id"}, 66 | {"A", "products", "name"} 67 | ], 68 | rows: [ 69 | ["1", "1", "1", "sugus"], 70 | ["1", "1", "2", "lollipop"], 71 | ["1", "1", "3", "donut"], 72 | ["1", "1", "4", "water"], 73 | ["1", "2", "1", "sugus"], 74 | ["1", "2", "2", "lollipop"], 75 | ["1", "2", "3", "donut"], 76 | ["1", "2", "4", "water"], 77 | ["1", "3", "1", "sugus"], 78 | ["1", "3", "2", "lollipop"], 79 | ["1", "3", "3", "donut"], 80 | ["1", "3", "4", "water"], 81 | ["2", "2", "1", "sugus"], 82 | ["2", "2", "2", "lollipop"], 83 | ["2", "2", "3", "donut"], 84 | ["2", "2", "4", "water"], 85 | ["2", "4", "1", "sugus"], 86 | ["2", "4", "2", "lollipop"], 87 | ["2", "4", "3", "donut"], 88 | ["2", "4", "4", "water"], 89 | ["3", "1", "1", "sugus"], 90 | ["3", "1", "2", "lollipop"], 91 | ["3", "1", "3", "donut"], 92 | ["3", "1", "4", "water"] 93 | ] 94 | } 95 | end 96 | 97 | test "Execute a complex manual plan" do 98 | plan = 99 | {:select, 100 | {:filter, 101 | {:cross_join, 102 | {:execute, {:table, {"A", "users"}}, [], [{"A", "users", "id"}, {"A", "users", "name"}]}, 103 | {:cross_join, 104 | {:execute, {:table, {"A", "purchases"}}, [], 105 | [{"A", "purchases", "user_id"}, {"A", "purchases", "product_id"}]}, 106 | {:execute, {:table, {"A", "products"}}, [], 107 | [{"A", "products", "id"}, {"A", "products", "name"}]}}}, 108 | {:op, 109 | {"AND", 110 | {:op, {"=", {:column, {"A", "users", "id"}}, {:column, {"A", "purchases", "user_id"}}}}, 111 | {:op, 112 | {"=", {:column, {"A", "purchases", "product_id"}}, {:column, {"A", "products", "id"}}}}}}}, 113 | [{:column, {"A", "users", "name"}}, {:column, {"A", "products", "name"}}]} 114 | 115 | {:ok, result} = ExoSQL.Executor.execute(plan, @context) 116 | 117 | assert result == %ExoSQL.Result{ 118 | columns: [{"A", "users", "name"}, {"A", "products", "name"}], 119 | rows: [ 120 | ["David", "sugus"], 121 | ["David", "lollipop"], 122 | ["David", "donut"], 123 | ["Javier", "lollipop"], 124 | ["Javier", "water"], 125 | ["Patricio", "sugus"] 126 | ] 127 | } 128 | end 129 | 130 | test "Execute manual simple aggregation" do 131 | # SELECT COUNT(*) FROM products 132 | # converted to 133 | # SELECT COUNT(A.products.*) FROM products GROUP BY true ## all to one set, returns the table true, {"A","products","*"} 134 | plan = 135 | {:select, 136 | {:group_by, 137 | {:execute, {:table, {"A", "products"}}, [], 138 | [{"A", "products", "id"}, {"A", "products", "name"}]}, [{:lit, true}]}, 139 | [{:fn, {"count", [{:column, 1}, {:pass, {:lit, "*"}}]}}]} 140 | 141 | {:ok, result} = ExoSQL.Executor.execute(plan, @context) 142 | assert result == %ExoSQL.Result{columns: [{:tmp, :tmp, "col_1"}], rows: [[4]]} 143 | end 144 | 145 | test "Execute complex aggregation" do 146 | plan = 147 | {:select, 148 | {:group_by, 149 | {:execute, {:table, {"A", "purchases"}}, [], [{"A", "purchases", "product_id"}]}, 150 | [{:column, {"A", "purchases", "product_id"}}]}, 151 | [ 152 | {:column, {"A", "purchases", "product_id"}}, 153 | {:fn, {"count", [{:column, 1}, {:pass, {:lit, "*"}}]}} 154 | ]} 155 | 156 | {:ok, result} = ExoSQL.Executor.execute(plan, @context) 157 | 158 | assert result == %ExoSQL.Result{ 159 | columns: [{"A", "purchases", "product_id"}, {:tmp, :tmp, "col_2"}], 160 | rows: [["1", 2], ["2", 2], ["3", 1], ["4", 1]] 161 | } 162 | end 163 | 164 | test "Execute complex aggregation 2" do 165 | # SELECT users.name, SUM(price*amount) FROM users, purchases, products 166 | # WHERE users.id = purchases.user_id AND products.id = purchases.product_id 167 | # GROUP BY product.id 168 | plan = 169 | {:select, 170 | {:group_by, 171 | {:filter, 172 | {:cross_join, 173 | {:execute, {:table, {"A", "users"}}, [], 174 | [{"A", "users", "id"}, {"A", "users", "name"}]}, 175 | {:cross_join, 176 | {:execute, {:table, {"A", "purchases"}}, [], 177 | [ 178 | {"A", "purchases", "user_id"}, 179 | {"A", "purchases", "product_id"}, 180 | {"A", "purchases", "amount"} 181 | ]}, 182 | {:execute, {:table, {"A", "products"}}, [], 183 | [{"A", "products", "id"}, {"A", "products", "name"}, {"A", "products", "price"}]}}}, 184 | {:op, 185 | {"AND", 186 | {:op, 187 | {"=", {:column, {"A", "users", "id"}}, {:column, {"A", "purchases", "user_id"}}}}, 188 | {:op, 189 | {"=", {:column, {"A", "purchases", "product_id"}}, {:column, {"A", "products", "id"}}}}}}}, 190 | [{:column, {"A", "users", "name"}}]}, 191 | [ 192 | {:column, {"A", "users", "name"}}, 193 | {:fn, 194 | {"sum", 195 | [ 196 | {:column, 1}, 197 | {:pass, 198 | {:op, 199 | {"*", {:column, {"A", "products", "price"}}, 200 | {:column, {"A", "purchases", "amount"}}}}} 201 | ]}} 202 | ]} 203 | 204 | {:ok, result} = ExoSQL.Executor.execute(plan, @context) 205 | 206 | assert result == %ExoSQL.Result{ 207 | columns: [{"A", "users", "name"}, {:tmp, :tmp, "col_2"}], 208 | rows: [["David", 550], ["Javier", 1300], ["Patricio", 30]] 209 | } 210 | 211 | Logger.info(ExoSQL.format_result(result)) 212 | end 213 | 214 | test "extra quals on = and ==" do 215 | q = 216 | ExoSQL.Executor.get_extra_quals( 217 | %{columns: [{:tmp, :tmp, "A"}], rows: []}, 218 | {:op, {"=", {:column, {:tmp, :tmp, "A"}}, {:column, {:tmp, "B", "B"}}}}, 219 | @context 220 | ) 221 | 222 | assert q != [] 223 | 224 | q = 225 | ExoSQL.Executor.get_extra_quals( 226 | %{columns: [{:tmp, :tmp, "A"}], rows: []}, 227 | {:op, {"==", {:column, {:tmp, :tmp, "A"}}, {:column, {:tmp, "B", "B"}}}}, 228 | @context 229 | ) 230 | 231 | assert q != [] 232 | end 233 | end 234 | -------------------------------------------------------------------------------- /test/nested_select_test.exs: -------------------------------------------------------------------------------- 1 | require Logger 2 | 3 | defmodule NestedSelectTest do 4 | use ExUnit.Case 5 | doctest ExoSQL 6 | doctest ExoSQL.Expr 7 | @moduletag :capture_log 8 | 9 | @context %{ 10 | "A" => {ExoSQL.Csv, path: "test/data/csv/"} 11 | } 12 | 13 | def analyze_query!(query, context \\ @context) do 14 | Logger.debug("Query is:\n\n#{query}") 15 | {:ok, parsed} = ExoSQL.parse(query, context) 16 | Logger.debug("Parsed is #{inspect(parsed, pretty: true)}") 17 | {:ok, plan} = ExoSQL.Planner.plan(parsed, context) 18 | Logger.debug("Plan is #{inspect(plan, pretty: true)}") 19 | {:ok, result} = ExoSQL.Executor.execute(plan, context) 20 | Logger.debug(inspect(result, pretty: true)) 21 | Logger.debug("Result:\n#{ExoSQL.format_result(result)}") 22 | result 23 | end 24 | 25 | test "Nested SELECT" do 26 | {:ok, query} = 27 | ExoSQL.parse( 28 | """ 29 | SELECT * FROM ( 30 | SELECT user_id, SUM(amount) 31 | FROM purchases 32 | GROUP BY user_id 33 | ) ORDER BY 2 34 | """, 35 | @context 36 | ) 37 | 38 | Logger.debug("Query: #{inspect(query, pretty: true)}") 39 | {:ok, plan} = ExoSQL.plan(query, @context) 40 | Logger.debug("Plan: #{inspect(plan, pretty: true)}") 41 | {:ok, result} = ExoSQL.execute(plan, @context) 42 | Logger.debug("Result:\n#{ExoSQL.format_result(result)}") 43 | 44 | assert Enum.count(result.rows) == 3 45 | end 46 | 47 | test "Nested SELECT 2" do 48 | {:ok, query} = 49 | ExoSQL.parse( 50 | """ 51 | SELECT name, col_2 FROM ( 52 | SELECT user_id, SUM(amount) 53 | FROM purchases 54 | GROUP BY user_id 55 | ), (SELECT id, name FROM users) 56 | WHERE user_id = id 57 | ORDER BY 2 58 | """, 59 | @context 60 | ) 61 | 62 | Logger.debug("Query: #{inspect(query, pretty: true)}") 63 | {:ok, plan} = ExoSQL.plan(query, @context) 64 | Logger.debug("Plan: #{inspect(plan, pretty: true)}") 65 | {:ok, result} = ExoSQL.execute(plan, @context) 66 | Logger.debug("Result: #{inspect(result, pretty: true)}") 67 | Logger.debug("Result:\n#{ExoSQL.format_result(result)}") 68 | 69 | assert Enum.count(result.rows) == 3 70 | 71 | assert result == %ExoSQL.Result{ 72 | columns: [{"A", "users", "name"}, {:tmp, :tmp, "col_2"}], 73 | rows: [["Patricio", 10], ["David", 40], ["Javier", 110]] 74 | } 75 | end 76 | 77 | test "SELECT FROM with alias" do 78 | res = analyze_query!("SELECT * FROM (SELECT id, name FROM products LIMIT 3)") 79 | assert Enum.count(res.rows) == 3 80 | 81 | res = analyze_query!("SELECT * FROM (SELECT id, name FROM products LIMIT 3) AS prods") 82 | assert res.columns == [{:tmp, "prods", "id"}, {:tmp, "prods", "name"}] 83 | assert Enum.count(res.rows) == 3 84 | 85 | res = 86 | analyze_query!( 87 | "SELECT * FROM (SELECT id AS pid, name AS product_name FROM products LIMIT 3) AS prods ORDER BY prods.pid" 88 | ) 89 | 90 | assert Enum.count(res.rows) == 3 91 | assert res.columns == [{:tmp, "prods", "pid"}, {:tmp, "prods", "product_name"}] 92 | end 93 | 94 | test "Complex select with alias" do 95 | res = analyze_query!(" 96 | SELECT name, amount*price AS total, to_number(stock) AS stock 97 | FROM ( 98 | SELECT product_id AS pid, SUM(amount) AS amount 99 | FROM purchases 100 | GROUP BY product_id 101 | ORDER BY 2 DESC 102 | LIMIT 2 103 | ) AS purchss 104 | LEFT JOIN products 105 | ON products.id = pid 106 | ORDER BY 3 DESC") 107 | 108 | assert res.columns == [ 109 | {"A", "products", "name"}, 110 | {:tmp, :tmp, "total"}, 111 | {:tmp, :tmp, "stock"} 112 | ] 113 | 114 | assert Enum.count(res.rows) == 2 115 | end 116 | end 117 | -------------------------------------------------------------------------------- /test/parser_test.exs: -------------------------------------------------------------------------------- 1 | require Logger 2 | 3 | defmodule ParserTest do 4 | use ExUnit.Case 5 | doctest ExoSQL.Parser 6 | @moduletag :capture_log 7 | 8 | @context %{ 9 | "A" => {ExoSQL.Csv, path: "test/data/csv/"} 10 | } 11 | 12 | test "Lex and parse" do 13 | {:ok, res, 1} = 14 | :sql_lexer.string( 15 | 'SELECT A.products.name, A.products.stock FROM A.products WHERE (A.products.price > 0) and (a.products.stock >= 1)' 16 | ) 17 | 18 | Logger.debug("Lexed: #{inspect(res)}") 19 | 20 | {:ok, res} = :sql_parser.parse(res) 21 | 22 | Logger.debug("Parsed: #{inspect(res)}") 23 | end 24 | 25 | test "Elixir parsing to proper struct" do 26 | {:ok, res} = 27 | ExoSQL.Parser.parse( 28 | "SELECT A.products.name, A.products.stock FROM A.products WHERE (A.products.price > 0) and (A.products.stock >= 1)", 29 | @context 30 | ) 31 | 32 | Logger.debug("Parsed: #{inspect(res)}") 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /test/planner_test.exs: -------------------------------------------------------------------------------- 1 | require Logger 2 | 3 | defmodule PlannerTest do 4 | use ExUnit.Case 5 | doctest ExoSQL.Planner, import: true 6 | @moduletag :capture_log 7 | 8 | @context %{ 9 | "A" => {ExoSQL.Csv, path: "test/data/csv/"} 10 | } 11 | 12 | test "Plan something" do 13 | q = "SELECT name, stock FROM products WHERE (price > 0) and (stock >= 1)" 14 | 15 | {:ok, parsed} = ExoSQL.Parser.parse(q, @context) 16 | {:ok, plan} = ExoSQL.Planner.plan(parsed, @context) 17 | 18 | assert plan == 19 | {:select, 20 | {:filter, 21 | {:execute, {:table, {"A", "products"}}, [["price", ">", 0], ["stock", ">=", 1]], 22 | [ 23 | {"A", "products", "price"}, 24 | {"A", "products", "stock"}, 25 | {"A", "products", "name"} 26 | ]}, 27 | {:op, 28 | {"AND", {:op, {">", {:column, {"A", "products", "price"}}, {:lit, 0}}}, 29 | {:op, {">=", {:column, {"A", "products", "stock"}}, {:lit, 1}}}}}}, 30 | [ 31 | column: {"A", "products", "name"}, 32 | column: {"A", "products", "stock"} 33 | ]} 34 | 35 | ExoSQL.explain(q, @context) 36 | end 37 | 38 | test "Ask for quals" do 39 | {:ok, parsed} = 40 | ExoSQL.parse( 41 | "SELECT name, stock FROM products WHERE (stock > 0) AND (price <= 100)", 42 | @context 43 | ) 44 | 45 | Logger.debug("Parsed: #{inspect(parsed, pretty: true)}") 46 | {:ok, plan} = ExoSQL.plan(parsed, @context) 47 | Logger.debug("Planned: #{inspect(plan, pretty: true)}") 48 | 49 | {:execute, _from, quals, _columns} = plan |> elem(1) |> elem(1) 50 | Logger.debug("quals: #{inspect(quals)}, should be stock > 0, price <= 100") 51 | 52 | assert quals == [["stock", ">", 0], ["price", "<=", 100]] 53 | 54 | # Maybe OR 55 | {:ok, parsed} = 56 | ExoSQL.parse( 57 | "SELECT name, stock>0 FROM products WHERE (stock > 0) OR (price <= 100)", 58 | @context 59 | ) 60 | 61 | Logger.debug("Parsed: #{inspect(parsed, pretty: true)}") 62 | {:ok, plan} = ExoSQL.plan(parsed, @context) 63 | Logger.debug("Planned: #{inspect(plan, pretty: true)}") 64 | 65 | {:execute, _from, quals, _columns} = plan |> elem(1) |> elem(1) 66 | Logger.debug("quals: #{inspect(quals)}, should be []") 67 | 68 | assert quals == [] 69 | 70 | # 71 | {:ok, parsed} = 72 | ExoSQL.parse( 73 | "SELECT name, stock FROM products WHERE (stock > $test) AND (price <= 100)", 74 | @context 75 | ) 76 | 77 | Logger.debug("Parsed: #{inspect(parsed, pretty: true)}") 78 | {:ok, plan} = ExoSQL.plan(parsed, @context) 79 | Logger.debug("Planned: #{inspect(plan, pretty: true)}") 80 | 81 | {:execute, _from, quals, _columns} = plan |> elem(1) |> elem(1) 82 | Logger.debug("quals: #{inspect(quals)}, should be stock > $test, price <= 100") 83 | 84 | assert quals == [["stock", ">", {:var, "test"}], ["price", "<=", 100]] 85 | end 86 | 87 | test "quals when ALIAS" do 88 | {:ok, parsed} = ExoSQL.parse("SELECT * FROM products AS p WHERE id > 0", @context) 89 | Logger.debug("Parsed: #{inspect(parsed, pretty: true)}") 90 | {:ok, plan} = ExoSQL.plan(parsed, @context) 91 | Logger.debug("Planned: #{inspect(plan, pretty: true)}") 92 | 93 | {:select, {:filter, {:alias, {:execute, _table, quals, _columns}, _}, _}, _} = plan 94 | 95 | Logger.debug("quals: #{inspect(quals)}") 96 | assert quals == [["id", ">", 0]] 97 | end 98 | end 99 | -------------------------------------------------------------------------------- /test/test_helper.exs: -------------------------------------------------------------------------------- 1 | ExUnit.start() 2 | --------------------------------------------------------------------------------