├── .github └── workflows │ ├── checks.yaml │ ├── docs.yml │ ├── release.yaml │ └── test.yaml ├── .gitignore ├── CHANGES.rst ├── LICENSE ├── README.rst ├── beanquery ├── __init__.py ├── __main__.py ├── compiler.py ├── cursor.py ├── cursor_test.py ├── errors.py ├── hashable.py ├── hashable_test.py ├── numberify.py ├── numberify_test.py ├── parser │ ├── __init__.py │ ├── ast.py │ ├── bql.ebnf │ └── parser.py ├── parser_test.py ├── query.py ├── query_compile.py ├── query_compile_test.py ├── query_env.py ├── query_env_test.py ├── query_execute.py ├── query_execute_test.py ├── query_render.py ├── query_render_test.py ├── query_test.py ├── render │ ├── beancount.py │ ├── csv.py │ └── text.py ├── shell.py ├── shell_test.py ├── sources │ ├── beancount.py │ ├── csv.py │ ├── memory.py │ └── test.py ├── tables.py ├── types.py └── types_test.py ├── bin └── bean-query ├── docs ├── conf.py ├── index.rst ├── logo.inkscape └── logo.svg ├── etc └── env ├── pyproject.toml └── requirements.txt /.github/workflows/checks.yaml: -------------------------------------------------------------------------------- 1 | name: checks 2 | on: 3 | - push 4 | - pull_request 5 | jobs: 6 | ruff: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v4 10 | - uses: actions/setup-python@v5 11 | with: 12 | python-version: '3.11' 13 | - run: python -m pip install ruff 14 | - run: ruff check beanquery/ 15 | coverage: 16 | runs-on: ubuntu-latest 17 | steps: 18 | - uses: actions/checkout@v4 19 | - uses: actions/setup-python@v5 20 | with: 21 | python-version: '3.11' 22 | - run: pip install -r requirements.txt coverage 23 | - name: Check coverage by module 24 | # Check that each module is exaustively tested by the dedicated units tests. 25 | run: | 26 | set -x 27 | echo '{ 28 | 29 | "beanquery/parser/*": "beanquery/parser_test.py", 30 | "beanquery/query_render.py": "beanquery/query_render_test.py" 31 | 32 | }' | jq -rc 'to_entries | .[] | (.key + "=" + .value)' | while IFS='=' read src test 33 | do 34 | python -m coverage run --branch --include "$src" --omit beanquery/parser/parser.py -m unittest "$test" 35 | python -m coverage report --fail-under=100 -m 36 | python -m coverage erase 37 | done 38 | - name: Check overall coverage 39 | run: | 40 | python -m coverage run --branch -m unittest discover -t . -s beanquery/ -p \*_test.py 41 | python -m coverage report --sort cover 42 | build: 43 | runs-on: ubuntu-latest 44 | steps: 45 | - uses: actions/checkout@v4 46 | - uses: actions/setup-python@v5 47 | with: 48 | python-version: '3.11' 49 | - run: pip install -r requirements.txt wheel build 50 | - run: python -m build --no-isolation 51 | - run: python -m pip install dist/beanquery-*.whl 52 | -------------------------------------------------------------------------------- /.github/workflows/docs.yml: -------------------------------------------------------------------------------- 1 | name: docs 2 | on: 3 | push: 4 | pull_request: 5 | 6 | jobs: 7 | 8 | build: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/setup-python@v5 12 | with: 13 | python-version: '3.11' 14 | - uses: actions/checkout@v4 15 | - run: python -m pip install .[docs] 16 | - run: python -m sphinx -W -b html docs/ build/html/ 17 | - uses: actions/upload-pages-artifact@v3 18 | with: 19 | path: build/html 20 | 21 | deploy: 22 | needs: build 23 | permissions: 24 | pages: write 25 | id-token: write 26 | environment: 27 | name: github-pages 28 | runs-on: ubuntu-latest 29 | if: github.ref == 'refs/heads/master' 30 | steps: 31 | - uses: actions/deploy-pages@v4 32 | -------------------------------------------------------------------------------- /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | name: release 2 | on: 3 | push: 4 | tags: 5 | - 'v[0-9]*' 6 | jobs: 7 | build: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/checkout@v4 11 | - run: python -m pip install build 12 | - run: python -m build 13 | - uses: actions/upload-artifact@v4 14 | with: 15 | path: dist/* 16 | upload: 17 | needs: build 18 | runs-on: ubuntu-latest 19 | environment: upload 20 | permissions: 21 | id-token: write 22 | steps: 23 | - uses: actions/download-artifact@v4 24 | with: 25 | merge-multiple: true 26 | path: dist 27 | - uses: pypa/gh-action-pypi-publish@release/v1 28 | with: 29 | attestations: false 30 | -------------------------------------------------------------------------------- /.github/workflows/test.yaml: -------------------------------------------------------------------------------- 1 | name: test 2 | on: 3 | - push 4 | - pull_request 5 | jobs: 6 | test: 7 | runs-on: ubuntu-latest 8 | strategy: 9 | fail-fast: false 10 | matrix: 11 | python: 12 | - '3.8' 13 | - '3.9' 14 | - '3.10' 15 | - '3.11' 16 | - '3.12' 17 | - '3.13' 18 | beancount: 19 | - '~= 2.3.6' 20 | - '~= 3.0.0' 21 | - '@ git+https://github.com/beancount/beancount.git' 22 | exclude: 23 | - python: '3.8' 24 | beancount: '@ git+https://github.com/beancount/beancount.git' 25 | - python: '3.9' 26 | beancount: '@ git+https://github.com/beancount/beancount.git' 27 | steps: 28 | - uses: actions/checkout@v4 29 | - uses: actions/setup-python@v5 30 | with: 31 | python-version: ${{ matrix.python }} 32 | allow-prereleases: true 33 | - run: pip install 'beancount ${{ matrix.beancount }}' 34 | - run: pip install -r requirements.txt 35 | - run: python -m unittest discover -p '*_test.py' 36 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/ 2 | *.egg-info/ 3 | build/ -------------------------------------------------------------------------------- /CHANGES.rst: -------------------------------------------------------------------------------- 1 | Version 0.1 (unreleased) 2 | ------------------------ 3 | 4 | - The ``HAVING`` clause for aggregate queries is now supported. 5 | 6 | - The ``empty()`` BQL function to determine whether an Inventory 7 | object as returned by the ``sum()`` aggregate function is empty has 8 | been added. 9 | 10 | - Added the ``round()`` BQL function. 11 | 12 | - ``NULL`` values in ``SORT BY`` clause are now always considered to 13 | be smaller than any other values. This may results in rows to be 14 | returned in a slightly different order. 15 | 16 | - It is now possible to specify the direction of the ordering for each 17 | column in the ``SORT BY`` clause. This brings BQL closer to SQL 18 | specification but queries written with the old behaviour in mind 19 | will return rows in a different order. The query:: 20 | 21 | SELECT date, narration ORDER BY date, narration DESC 22 | 23 | used to return rows in descending order by both ``date`` and 24 | ``narration`` while now it would order the rows ascending by 25 | ``date`` and descending by ``narration``. To recover the old 26 | behavior, the query should be written:: 27 | 28 | SELECT date, narration ORDER BY date DESC, narration DESC 29 | 30 | - Type casting functions ``int()``, ``decimal()``, ``str()``, 31 | ``date()`` have been added. These are mostly useful to convert the 32 | generic ``object`` type returned by the metadata retrieval functions 33 | but can also be used to convert between types. If the conversion 34 | fails, ``NULL`` is returned. 35 | 36 | - The ``str()`` BQL function used to return a string representation of 37 | its argument using the Python :py:func:`repr()` function. This 38 | clashes with the use of ``str()`` as a type casting function. The 39 | function is renamed ``repr()``. 40 | 41 | - The ``date()`` BQL function used to extract a date from string 42 | arguments with a very relaxed parser. This clashes with the use of 43 | ``date()`` as a type casting function. The function is renamed 44 | ``parse_date()``. Another form of ``parse_date()`` that accepts the 45 | date format as second argument has been added. 46 | 47 | - The ``getitem()`` BQL function return type has been changed from a 48 | string to a generic ``object`` to match the return type of function 49 | retrieving entries from metadata dictionaries. The old behavior can 50 | be obtained with ``str(getitem(x, key))``. 51 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 2, June 1991 3 | 4 | Copyright (C) 1989, 1991 Free Software Foundation, Inc., 5 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 6 | Everyone is permitted to copy and distribute verbatim copies 7 | of this license document, but changing it is not allowed. 8 | 9 | Preamble 10 | 11 | The licenses for most software are designed to take away your 12 | freedom to share and change it. By contrast, the GNU General Public 13 | License is intended to guarantee your freedom to share and change free 14 | software--to make sure the software is free for all its users. This 15 | General Public License applies to most of the Free Software 16 | Foundation's software and to any other program whose authors commit to 17 | using it. (Some other Free Software Foundation software is covered by 18 | the GNU Lesser General Public License instead.) You can apply it to 19 | your programs, too. 20 | 21 | When we speak of free software, we are referring to freedom, not 22 | price. Our General Public Licenses are designed to make sure that you 23 | have the freedom to distribute copies of free software (and charge for 24 | this service if you wish), that you receive source code or can get it 25 | if you want it, that you can change the software or use pieces of it 26 | in new free programs; and that you know you can do these things. 27 | 28 | To protect your rights, we need to make restrictions that forbid 29 | anyone to deny you these rights or to ask you to surrender the rights. 30 | These restrictions translate to certain responsibilities for you if you 31 | distribute copies of the software, or if you modify it. 32 | 33 | For example, if you distribute copies of such a program, whether 34 | gratis or for a fee, you must give the recipients all the rights that 35 | you have. You must make sure that they, too, receive or can get the 36 | source code. And you must show them these terms so they know their 37 | rights. 38 | 39 | We protect your rights with two steps: (1) copyright the software, and 40 | (2) offer you this license which gives you legal permission to copy, 41 | distribute and/or modify the software. 42 | 43 | Also, for each author's protection and ours, we want to make certain 44 | that everyone understands that there is no warranty for this free 45 | software. If the software is modified by someone else and passed on, we 46 | want its recipients to know that what they have is not the original, so 47 | that any problems introduced by others will not reflect on the original 48 | authors' reputations. 49 | 50 | Finally, any free program is threatened constantly by software 51 | patents. We wish to avoid the danger that redistributors of a free 52 | program will individually obtain patent licenses, in effect making the 53 | program proprietary. To prevent this, we have made it clear that any 54 | patent must be licensed for everyone's free use or not licensed at all. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | GNU GENERAL PUBLIC LICENSE 60 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 61 | 62 | 0. This License applies to any program or other work which contains 63 | a notice placed by the copyright holder saying it may be distributed 64 | under the terms of this General Public License. The "Program", below, 65 | refers to any such program or work, and a "work based on the Program" 66 | means either the Program or any derivative work under copyright law: 67 | that is to say, a work containing the Program or a portion of it, 68 | either verbatim or with modifications and/or translated into another 69 | language. (Hereinafter, translation is included without limitation in 70 | the term "modification".) Each licensee is addressed as "you". 71 | 72 | Activities other than copying, distribution and modification are not 73 | covered by this License; they are outside its scope. The act of 74 | running the Program is not restricted, and the output from the Program 75 | is covered only if its contents constitute a work based on the 76 | Program (independent of having been made by running the Program). 77 | Whether that is true depends on what the Program does. 78 | 79 | 1. You may copy and distribute verbatim copies of the Program's 80 | source code as you receive it, in any medium, provided that you 81 | conspicuously and appropriately publish on each copy an appropriate 82 | copyright notice and disclaimer of warranty; keep intact all the 83 | notices that refer to this License and to the absence of any warranty; 84 | and give any other recipients of the Program a copy of this License 85 | along with the Program. 86 | 87 | You may charge a fee for the physical act of transferring a copy, and 88 | you may at your option offer warranty protection in exchange for a fee. 89 | 90 | 2. You may modify your copy or copies of the Program or any portion 91 | of it, thus forming a work based on the Program, and copy and 92 | distribute such modifications or work under the terms of Section 1 93 | above, provided that you also meet all of these conditions: 94 | 95 | a) You must cause the modified files to carry prominent notices 96 | stating that you changed the files and the date of any change. 97 | 98 | b) You must cause any work that you distribute or publish, that in 99 | whole or in part contains or is derived from the Program or any 100 | part thereof, to be licensed as a whole at no charge to all third 101 | parties under the terms of this License. 102 | 103 | c) If the modified program normally reads commands interactively 104 | when run, you must cause it, when started running for such 105 | interactive use in the most ordinary way, to print or display an 106 | announcement including an appropriate copyright notice and a 107 | notice that there is no warranty (or else, saying that you provide 108 | a warranty) and that users may redistribute the program under 109 | these conditions, and telling the user how to view a copy of this 110 | License. (Exception: if the Program itself is interactive but 111 | does not normally print such an announcement, your work based on 112 | the Program is not required to print an announcement.) 113 | 114 | These requirements apply to the modified work as a whole. If 115 | identifiable sections of that work are not derived from the Program, 116 | and can be reasonably considered independent and separate works in 117 | themselves, then this License, and its terms, do not apply to those 118 | sections when you distribute them as separate works. But when you 119 | distribute the same sections as part of a whole which is a work based 120 | on the Program, the distribution of the whole must be on the terms of 121 | this License, whose permissions for other licensees extend to the 122 | entire whole, and thus to each and every part regardless of who wrote it. 123 | 124 | Thus, it is not the intent of this section to claim rights or contest 125 | your rights to work written entirely by you; rather, the intent is to 126 | exercise the right to control the distribution of derivative or 127 | collective works based on the Program. 128 | 129 | In addition, mere aggregation of another work not based on the Program 130 | with the Program (or with a work based on the Program) on a volume of 131 | a storage or distribution medium does not bring the other work under 132 | the scope of this License. 133 | 134 | 3. You may copy and distribute the Program (or a work based on it, 135 | under Section 2) in object code or executable form under the terms of 136 | Sections 1 and 2 above provided that you also do one of the following: 137 | 138 | a) Accompany it with the complete corresponding machine-readable 139 | source code, which must be distributed under the terms of Sections 140 | 1 and 2 above on a medium customarily used for software interchange; or, 141 | 142 | b) Accompany it with a written offer, valid for at least three 143 | years, to give any third party, for a charge no more than your 144 | cost of physically performing source distribution, a complete 145 | machine-readable copy of the corresponding source code, to be 146 | distributed under the terms of Sections 1 and 2 above on a medium 147 | customarily used for software interchange; or, 148 | 149 | c) Accompany it with the information you received as to the offer 150 | to distribute corresponding source code. (This alternative is 151 | allowed only for noncommercial distribution and only if you 152 | received the program in object code or executable form with such 153 | an offer, in accord with Subsection b above.) 154 | 155 | The source code for a work means the preferred form of the work for 156 | making modifications to it. For an executable work, complete source 157 | code means all the source code for all modules it contains, plus any 158 | associated interface definition files, plus the scripts used to 159 | control compilation and installation of the executable. However, as a 160 | special exception, the source code distributed need not include 161 | anything that is normally distributed (in either source or binary 162 | form) with the major components (compiler, kernel, and so on) of the 163 | operating system on which the executable runs, unless that component 164 | itself accompanies the executable. 165 | 166 | If distribution of executable or object code is made by offering 167 | access to copy from a designated place, then offering equivalent 168 | access to copy the source code from the same place counts as 169 | distribution of the source code, even though third parties are not 170 | compelled to copy the source along with the object code. 171 | 172 | 4. You may not copy, modify, sublicense, or distribute the Program 173 | except as expressly provided under this License. Any attempt 174 | otherwise to copy, modify, sublicense or distribute the Program is 175 | void, and will automatically terminate your rights under this License. 176 | However, parties who have received copies, or rights, from you under 177 | this License will not have their licenses terminated so long as such 178 | parties remain in full compliance. 179 | 180 | 5. You are not required to accept this License, since you have not 181 | signed it. However, nothing else grants you permission to modify or 182 | distribute the Program or its derivative works. These actions are 183 | prohibited by law if you do not accept this License. Therefore, by 184 | modifying or distributing the Program (or any work based on the 185 | Program), you indicate your acceptance of this License to do so, and 186 | all its terms and conditions for copying, distributing or modifying 187 | the Program or works based on it. 188 | 189 | 6. Each time you redistribute the Program (or any work based on the 190 | Program), the recipient automatically receives a license from the 191 | original licensor to copy, distribute or modify the Program subject to 192 | these terms and conditions. You may not impose any further 193 | restrictions on the recipients' exercise of the rights granted herein. 194 | You are not responsible for enforcing compliance by third parties to 195 | this License. 196 | 197 | 7. If, as a consequence of a court judgment or allegation of patent 198 | infringement or for any other reason (not limited to patent issues), 199 | conditions are imposed on you (whether by court order, agreement or 200 | otherwise) that contradict the conditions of this License, they do not 201 | excuse you from the conditions of this License. If you cannot 202 | distribute so as to satisfy simultaneously your obligations under this 203 | License and any other pertinent obligations, then as a consequence you 204 | may not distribute the Program at all. For example, if a patent 205 | license would not permit royalty-free redistribution of the Program by 206 | all those who receive copies directly or indirectly through you, then 207 | the only way you could satisfy both it and this License would be to 208 | refrain entirely from distribution of the Program. 209 | 210 | If any portion of this section is held invalid or unenforceable under 211 | any particular circumstance, the balance of the section is intended to 212 | apply and the section as a whole is intended to apply in other 213 | circumstances. 214 | 215 | It is not the purpose of this section to induce you to infringe any 216 | patents or other property right claims or to contest validity of any 217 | such claims; this section has the sole purpose of protecting the 218 | integrity of the free software distribution system, which is 219 | implemented by public license practices. Many people have made 220 | generous contributions to the wide range of software distributed 221 | through that system in reliance on consistent application of that 222 | system; it is up to the author/donor to decide if he or she is willing 223 | to distribute software through any other system and a licensee cannot 224 | impose that choice. 225 | 226 | This section is intended to make thoroughly clear what is believed to 227 | be a consequence of the rest of this License. 228 | 229 | 8. If the distribution and/or use of the Program is restricted in 230 | certain countries either by patents or by copyrighted interfaces, the 231 | original copyright holder who places the Program under this License 232 | may add an explicit geographical distribution limitation excluding 233 | those countries, so that distribution is permitted only in or among 234 | countries not thus excluded. In such case, this License incorporates 235 | the limitation as if written in the body of this License. 236 | 237 | 9. The Free Software Foundation may publish revised and/or new versions 238 | of the General Public License from time to time. Such new versions will 239 | be similar in spirit to the present version, but may differ in detail to 240 | address new problems or concerns. 241 | 242 | Each version is given a distinguishing version number. If the Program 243 | specifies a version number of this License which applies to it and "any 244 | later version", you have the option of following the terms and conditions 245 | either of that version or of any later version published by the Free 246 | Software Foundation. If the Program does not specify a version number of 247 | this License, you may choose any version ever published by the Free Software 248 | Foundation. 249 | 250 | 10. If you wish to incorporate parts of the Program into other free 251 | programs whose distribution conditions are different, write to the author 252 | to ask for permission. For software which is copyrighted by the Free 253 | Software Foundation, write to the Free Software Foundation; we sometimes 254 | make exceptions for this. Our decision will be guided by the two goals 255 | of preserving the free status of all derivatives of our free software and 256 | of promoting the sharing and reuse of software generally. 257 | 258 | NO WARRANTY 259 | 260 | 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY 261 | FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN 262 | OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES 263 | PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED 264 | OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 265 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS 266 | TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE 267 | PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, 268 | REPAIR OR CORRECTION. 269 | 270 | 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 271 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR 272 | REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, 273 | INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING 274 | OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED 275 | TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY 276 | YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER 277 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE 278 | POSSIBILITY OF SUCH DAMAGES. 279 | 280 | END OF TERMS AND CONDITIONS 281 | 282 | How to Apply These Terms to Your New Programs 283 | 284 | If you develop a new program, and you want it to be of the greatest 285 | possible use to the public, the best way to achieve this is to make it 286 | free software which everyone can redistribute and change under these terms. 287 | 288 | To do so, attach the following notices to the program. It is safest 289 | to attach them to the start of each source file to most effectively 290 | convey the exclusion of warranty; and each file should have at least 291 | the "copyright" line and a pointer to where the full notice is found. 292 | 293 | 294 | Copyright (C) 295 | 296 | This program is free software; you can redistribute it and/or modify 297 | it under the terms of the GNU General Public License as published by 298 | the Free Software Foundation; either version 2 of the License, or 299 | (at your option) any later version. 300 | 301 | This program is distributed in the hope that it will be useful, 302 | but WITHOUT ANY WARRANTY; without even the implied warranty of 303 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 304 | GNU General Public License for more details. 305 | 306 | You should have received a copy of the GNU General Public License along 307 | with this program; if not, write to the Free Software Foundation, Inc., 308 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 309 | 310 | Also add information on how to contact you by electronic and paper mail. 311 | 312 | If the program is interactive, make it output a short notice like this 313 | when it starts in an interactive mode: 314 | 315 | Gnomovision version 69, Copyright (C) year name of author 316 | Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 317 | This is free software, and you are welcome to redistribute it 318 | under certain conditions; type `show c' for details. 319 | 320 | The hypothetical commands `show w' and `show c' should show the appropriate 321 | parts of the General Public License. Of course, the commands you use may 322 | be called something other than `show w' and `show c'; they could even be 323 | mouse-clicks or menu items--whatever suits your program. 324 | 325 | You should also get your employer (if you work as a programmer) or your 326 | school, if any, to sign a "copyright disclaimer" for the program, if 327 | necessary. Here is a sample; alter the names: 328 | 329 | Yoyodyne, Inc., hereby disclaims all copyright interest in the program 330 | `Gnomovision' (which makes passes at compilers) written by James Hacker. 331 | 332 | , 1 April 1989 333 | Ty Coon, President of Vice 334 | 335 | This General Public License does not permit incorporating your program into 336 | proprietary programs. If your program is a subroutine library, you may 337 | consider it more useful to permit linking proprietary applications with the 338 | library. If this is what you want to do, use the GNU Lesser General 339 | Public License instead of this License. 340 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | beanquery: Customizable lightweight SQL query tool 2 | ================================================== 3 | 4 | beanquery is a customizable and extensible lightweight SQL query tool 5 | that works on tabular data, including `Beancount`__ ledger data. 6 | 7 | __ https://beancount.github.io/ 8 | -------------------------------------------------------------------------------- /beanquery/__init__.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | 3 | from urllib.parse import urlparse 4 | 5 | from . import parser 6 | from . import compiler 7 | from . import tables 8 | 9 | from .compiler import CompilationError 10 | from .cursor import Cursor, Column 11 | from .errors import Warning, Error, InterfaceError, DatabaseError, DataError, OperationalError 12 | from .errors import IntegrityError, InternalError, ProgrammingError, NotSupportedError 13 | from .parser import ParseError 14 | 15 | 16 | __version__ = '0.3.0.dev0' 17 | 18 | 19 | # DB-API compliance 20 | apilevel = '2.0' 21 | threadsafety = 2 22 | paramstyle = 'pyformat' 23 | 24 | 25 | def connect(dsn, **kwargs): 26 | return Connection(dsn, **kwargs) 27 | 28 | 29 | class Connection: 30 | def __init__(self, dsn='', **kwargs): 31 | self.tables = {None: tables.NullTable()} 32 | 33 | # The ``None`` table is the default table. The ``''`` table is the 34 | # table that is explicitly selected with ``FROM #``. Having the 35 | # default table and the ``''`` table allows to select the empty table 36 | # when the ``beancount`` data source is initialized and it sets the 37 | # default table to the ``postings`` table. 38 | self.tables[''] = self.tables[None] 39 | 40 | # These are used only by the ``beancount`` data source. 41 | self.options = {} 42 | self.errors = [] 43 | 44 | if dsn: 45 | self.attach(dsn, **kwargs) 46 | 47 | def attach(self, dsn, **kwargs): 48 | scheme = urlparse(dsn).scheme 49 | source = importlib.import_module(f'beanquery.sources.{scheme}') 50 | source.attach(self, dsn, **kwargs) 51 | 52 | def close(self): 53 | # Required by the DB-API. 54 | pass 55 | 56 | def parse(self, query): 57 | return parser.parse(query) 58 | 59 | def compile(self, query): 60 | return compiler.compile(self, query) 61 | 62 | def execute(self, query, params=None): 63 | return self.cursor().execute(query, params) 64 | 65 | def cursor(self): 66 | return Cursor(self) 67 | 68 | 69 | __all__ = [ 70 | 'Column', 71 | 'CompilationError', 72 | 'Connection', 73 | 'Cursor', 74 | 'DataError', 75 | 'DatabaseError', 76 | 'Error', 77 | 'IntegrityError', 78 | 'InterfaceError', 79 | 'InternalError', 80 | 'NotSupportedError', 81 | 'OperationalError', 82 | 'ParseError', 83 | 'ProgrammingError', 84 | 'Warning', 85 | 'apilevel', 86 | 'connet', 87 | 'paramstyle', 88 | 'threadsafety', 89 | ] 90 | -------------------------------------------------------------------------------- /beanquery/__main__.py: -------------------------------------------------------------------------------- 1 | from beanquery import shell 2 | 3 | if __name__ == '__main__': 4 | shell.main() 5 | -------------------------------------------------------------------------------- /beanquery/cursor.py: -------------------------------------------------------------------------------- 1 | from operator import attrgetter 2 | from typing import Sequence 3 | 4 | from . import types 5 | from . import parser 6 | from . import compiler 7 | 8 | 9 | class Column(Sequence): 10 | __module__ = 'beanquery' 11 | 12 | def __init__(self, name, datatype): 13 | self._name = name 14 | self._type = datatype 15 | 16 | _vars = tuple(attrgetter(name) for name in 'name type_code display_size internal_size precision scale null_ok'.split()) 17 | 18 | def __eq__(self, other): 19 | if isinstance(other, type(self)): 20 | return tuple(self) == tuple(other) 21 | if isinstance(other, tuple): 22 | # Used in tests. 23 | return (self._name, self._type) == other 24 | return NotImplemented 25 | 26 | def __repr__(self): 27 | return f'{self.__module__}.{self.__class__.__name__}({self._name!r}, {types.name(self._type)})' 28 | 29 | def __len__(self): 30 | return 7 31 | 32 | def __getitem__(self, key): 33 | if isinstance(key, slice): 34 | return tuple(getter(self) for getter in self._vars(key)) 35 | return self._vars[key](self) 36 | 37 | @property 38 | def name(self): 39 | return self._name 40 | 41 | @property 42 | def datatype(self): 43 | # Extension to the DB-API. 44 | return self._type 45 | 46 | @property 47 | def type_code(self): 48 | # The DB-API specification is vague on this point, but other 49 | # database connection libraries expose this as an int. It does 50 | # not make much sense to keep a mapping between int type code 51 | # and actual types, thus just return the hash of the type 52 | # object. 53 | return hash(self._type) 54 | 55 | @property 56 | def display_size(self): 57 | return None 58 | 59 | @property 60 | def internal_size(self): 61 | return None 62 | 63 | @property 64 | def precision(self): 65 | return None 66 | 67 | @property 68 | def scale(self): 69 | return None 70 | 71 | @property 72 | def null_ok(self): 73 | return None 74 | 75 | 76 | class Cursor: 77 | def __init__(self, connection): 78 | self._context = connection 79 | self._description = None 80 | self._rows = None 81 | self._pos = 0 82 | self.arraysize = 1 83 | 84 | @property 85 | def connection(self): 86 | return self._context 87 | 88 | def execute(self, query, params=None): 89 | if not isinstance(query, parser.ast.Node): 90 | query = parser.parse(query) 91 | query = compiler.compile(self._context, query, params) 92 | description, rows = query() 93 | self._description = description 94 | self._rows = rows 95 | self._pos = 0 96 | return self 97 | 98 | def executemany(self, query, params=None): 99 | query = parser.parse(query) 100 | for p in params: 101 | self.execute(query, p) 102 | 103 | @property 104 | def description(self): 105 | return self._description 106 | 107 | @property 108 | def rowcount(self): 109 | return len(self._rows) if self._rows is not None else -1 110 | 111 | @property 112 | def rownumber(self): 113 | return self._pos 114 | 115 | def fetchone(self): 116 | # This implementation pops items from the front of the results 117 | # rows list and is thus not efficient, especially for large 118 | # results sets. 119 | if self._rows is None or not len(self._rows): 120 | return None 121 | self._pos += 1 122 | return self._rows.pop(0) 123 | 124 | def fetchmany(self, size=None): 125 | if self._rows is None: 126 | return [] 127 | n = size if size is not None else self.arraysize 128 | rows = self._rows[:n] 129 | self._rows = self._rows[n:] 130 | self._pos += len(rows) 131 | return rows 132 | 133 | def fetchall(self): 134 | if self._rows is None: 135 | return [] 136 | rows = self._rows 137 | self._rows = [] 138 | self._pos += len(rows) 139 | return rows 140 | 141 | def close(self): 142 | # Required by the DB-API. 143 | pass 144 | 145 | def setinputsizes(self, sizes): 146 | # Required by the DB-API. 147 | pass 148 | 149 | def setoutputsize(self, size, column=None): 150 | # Required by the DB-API. 151 | pass 152 | 153 | def __iter__(self): 154 | return iter(self._rows if self._rows is not None else []) 155 | -------------------------------------------------------------------------------- /beanquery/cursor_test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import sqlite3 3 | 4 | import beanquery 5 | from beanquery.sources import test 6 | 7 | 8 | class APITests: 9 | def test_description(self): 10 | curs = self.conn.cursor() 11 | self.assertIsNone(curs.description) 12 | curs.execute(f'SELECT x FROM {self.table} WHERE x = 0') 13 | self.assertEqual([c[0] for c in curs.description], ['x']) 14 | column = curs.description[0] 15 | self.assertEqual(len(column), 7) 16 | 17 | def test_cursor_not_initialized(self): 18 | curs = self.conn.cursor() 19 | self.assertIsNone(curs.fetchone()) 20 | self.assertEqual(curs.fetchmany(), []) 21 | self.assertEqual(curs.fetchall(), []) 22 | 23 | def test_cursor_fetchone(self): 24 | curs = self.conn.cursor() 25 | curs.execute(f'SELECT x FROM {self.table} WHERE x < 2') 26 | row = curs.fetchone() 27 | self.assertEqual(row, (0, )) 28 | row = curs.fetchone() 29 | self.assertEqual(row, (1, )) 30 | row = curs.fetchone() 31 | self.assertIsNone(row) 32 | 33 | def test_cursor_fetchall(self): 34 | curs = self.conn.cursor() 35 | curs.execute(f'SELECT x FROM {self.table} WHERE x < 2') 36 | rows = curs.fetchall() 37 | self.assertEqual(rows, [(0, ), (1, )]) 38 | rows = curs.fetchall() 39 | self.assertEqual(rows, []) 40 | 41 | def test_cursor_fethmany(self): 42 | curs = self.conn.cursor() 43 | curs.execute(f'SELECT x FROM {self.table} WHERE x < 2') 44 | rows = curs.fetchmany() 45 | self.assertEqual(rows, [(0, )]) 46 | rows = curs.fetchmany() 47 | self.assertEqual(rows, [(1, )]) 48 | rows = curs.fetchmany() 49 | self.assertEqual(rows, []) 50 | 51 | def test_cursor_iterator(self): 52 | curs = self.conn.cursor() 53 | o = object() 54 | row = next(iter(curs), o) 55 | self.assertIs(row, o) 56 | curs = self.conn.cursor() 57 | curs.execute(f'SELECT x FROM {self.table} WHERE x < 2') 58 | iterator = iter(curs) 59 | row = next(iterator) 60 | self.assertEqual(row, (0, )) 61 | row = next(iterator) 62 | self.assertEqual(row, (1, )) 63 | row = next(iterator, o) 64 | self.assertIs(row, o) 65 | 66 | 67 | class TestSQLite(APITests, unittest.TestCase): 68 | @classmethod 69 | def setUpClass(cls): 70 | cls.table = 'test' 71 | cls.conn = sqlite3.connect(':memory:') 72 | curs = cls.conn.cursor() 73 | curs.execute('CREATE TABLE test (x int)') 74 | curs.executemany('INSERT INTO test VALUES (?)', [(i, ) for i in range(16)]) 75 | 76 | @classmethod 77 | def tearDownClass(cls): 78 | cls.conn.close() 79 | 80 | 81 | class TestBeanquery(APITests, unittest.TestCase): 82 | @classmethod 83 | def setUpClass(cls): 84 | cls.table = '#test' 85 | cls.conn = beanquery.Connection() 86 | cls.conn.tables['test'] = test.Table(16) 87 | -------------------------------------------------------------------------------- /beanquery/errors.py: -------------------------------------------------------------------------------- 1 | """ 2 | Exceptions hierarchy defined by the DB-API: 3 | 4 | Exception 5 | Warning 6 | Error 7 | InterfaceError 8 | DatabaseError 9 | DataError 10 | OperationalError 11 | IntegrityError 12 | InternalError 13 | ProgrammingError 14 | NotSupportedError 15 | """ 16 | 17 | 18 | class Warning(Exception): 19 | """Exception raised for important warnings.""" 20 | 21 | __module__ = 'beanquery' 22 | 23 | 24 | class Error(Exception): 25 | """Base exception for all errors.""" 26 | 27 | __module__ = 'beanquery' 28 | 29 | 30 | class InterfaceError(Error): 31 | """An error related to the database interface rather than the database itself.""" 32 | 33 | __module__ = 'beanquery' 34 | 35 | 36 | class DatabaseError(Error): 37 | """Exception raised for errors that are related to the database.""" 38 | 39 | __module__ = 'beanquery' 40 | 41 | 42 | class DataError(DatabaseError): 43 | """An error caused by problems with the processed data.""" 44 | 45 | __module__ = 'beanquery' 46 | 47 | 48 | class OperationalError(DatabaseError): 49 | """An error related to the database's operation.""" 50 | 51 | __module__ = 'beanquery' 52 | 53 | 54 | class IntegrityError(DatabaseError): 55 | """An error caused when the relational integrity of the database is affected.""" 56 | 57 | __module__ = 'beanquery' 58 | 59 | 60 | class InternalError(DatabaseError): 61 | """An error generated when the database encounters an internal error.""" 62 | 63 | __module__ = 'beanquery' 64 | 65 | 66 | class ProgrammingError(DatabaseError): 67 | """Exception raised for programming errors.""" 68 | 69 | __module__ = 'beanquery' 70 | 71 | 72 | class NotSupportedError(DatabaseError): 73 | """A method or database API was used which is not supported by the database.""" 74 | 75 | __module__ = 'beanquery' 76 | -------------------------------------------------------------------------------- /beanquery/hashable.py: -------------------------------------------------------------------------------- 1 | import decimal 2 | import pickle 3 | import textwrap 4 | 5 | from . import types 6 | 7 | # Hashable types. Checking ``issubclass(T, types.Hashable)`` does not 8 | # work because named tuples pass that test and beancount uses many named 9 | # tuples that have dictionary members making them effectively not 10 | # hashable. 11 | FUNDAMENTAL = frozenset({ 12 | bool, 13 | bytes, 14 | complex, 15 | decimal.Decimal, 16 | float, 17 | int, 18 | str, 19 | # These are hashable only if the contained objects are hashable. 20 | frozenset, 21 | tuple, 22 | }) 23 | 24 | # Function reducing non-hashable types to something hashable. 25 | REDUCERS = {} 26 | 27 | 28 | def register(datatype, func): 29 | """Register reduce function for non-hashable type. 30 | 31 | The reduce function maps an non-hashable object into an hashable 32 | representation. This representation does not need to capture all the 33 | object facets, but it should retrurn someting unique enough to avoid 34 | too many hashing collisions. 35 | """ 36 | REDUCERS[datatype] = func 37 | 38 | 39 | def make(columns): 40 | """Build an hashable tuple subclass.""" 41 | 42 | # When all columns are hashable, pass the input tuple through as is. 43 | if all(column.datatype in FUNDAMENTAL for column in columns): 44 | return lambda x: x 45 | 46 | datatypes = ', '.join(types.name(column.datatype) for column in columns) 47 | 48 | # Code generation inspired by standard library ``dataclasses.py``. 49 | parts = [] 50 | locals = {} 51 | for i, column in enumerate(columns): 52 | if column.datatype in FUNDAMENTAL: 53 | parts.append(f'self[{i}]') 54 | elif column.datatype is dict: 55 | parts.append(f'*self[{i}].keys(), *self[{i}].values()') 56 | elif column.datatype is set: 57 | parts.append(f'*self[{i}]') 58 | else: 59 | func = REDUCERS.get(column.datatype, pickle.dumps) 60 | fname = f'func{i}' 61 | locals[fname] = func 62 | parts.append(f'{fname}(self[{i}])') 63 | 64 | objs = ', '.join(parts) 65 | names = ', '.join(locals.keys()) 66 | code = textwrap.dedent(f''' 67 | def create({names}): 68 | def __hash__(self): 69 | return hash(({objs})) 70 | return __hash__ 71 | ''') 72 | clsname = f'Hashable[{datatypes}]' 73 | 74 | ns = {} 75 | exec(code, globals(), ns) 76 | func = ns['create'](**locals) 77 | func.__qualname__ = f'{clsname}.{func.__name__}' 78 | 79 | members = dict(tuple.__dict__) 80 | members['__hash__'] = func 81 | 82 | return type(clsname, (tuple,), members) 83 | -------------------------------------------------------------------------------- /beanquery/hashable_test.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | import unittest 3 | 4 | from beanquery import hashable 5 | from beanquery.cursor import Column 6 | 7 | 8 | class TestHashable(unittest.TestCase): 9 | 10 | def test_fundamental(self): 11 | columns = (Column('b', bool), Column('i', int), Column('s', str)) 12 | wrap = hashable.make(columns) 13 | obj = (True, 42, 'universe') 14 | self.assertIs(wrap(obj), obj) 15 | hash(obj) 16 | 17 | def test_dict(self): 18 | columns = (Column('b', bool), Column('d', dict)) 19 | wrap = hashable.make(columns) 20 | obja = (True, {'answer': 42}) 21 | a = hash(wrap(obja)) 22 | objb = (True, {'answer': 42}) 23 | b = hash(wrap(objb)) 24 | self.assertIsNot(obja, objb) 25 | self.assertEqual(a, b) 26 | objc = (False, {'answer': 42}) 27 | c = hash(wrap(objc)) 28 | self.assertNotEqual(a, c) 29 | objd = (True, {'answer': 43}) 30 | d = hash(wrap(objd)) 31 | self.assertNotEqual(a, d) 32 | 33 | def test_registered(self): 34 | 35 | @dataclasses.dataclass 36 | class Foo: 37 | xid: int 38 | meta: dict 39 | 40 | hashable.register(Foo, lambda obj: obj.xid) 41 | 42 | columns = (Column('b', bool), Column('foo', Foo)) 43 | wrap = hashable.make(columns) 44 | obja = (True, Foo(1, {'test': 1})) 45 | a = hash(wrap(obja)) 46 | objb = (True, Foo(1, {'test': 2})) 47 | b = hash(wrap(objb)) 48 | self.assertEqual(a, b) 49 | objc = (True, Foo(2, {'test': 2})) 50 | c = hash(wrap(objc)) 51 | self.assertNotEqual(a, c) 52 | -------------------------------------------------------------------------------- /beanquery/numberify.py: -------------------------------------------------------------------------------- 1 | """Code to split table columns containing amounts and inventories into number columns. 2 | 3 | For example, given a column with this content: 4 | 5 | ----- amount ------ 6 | 101.23 USD 7 | 200 JPY 8 | 99.23 USD 9 | 38.34 USD, 100 JPY 10 | 11 | We can convert this into two columns and remove the currencies: 12 | 13 | -amount (USD)- -amount (JPY)- 14 | 101.23 15 | 200 16 | 99.23 17 | 38.34 100 18 | 19 | The point is that the columns should be typed as numbers to make this importable 20 | into a spreadsheet and able to be processed. 21 | 22 | Notes: 23 | 24 | * This handles the Amount, Position and Inventory datatypes. There is code to 25 | automatically recognize columns containing such types from a table of strings 26 | and convert such columns to their corresponding guessed data types. 27 | 28 | * The per-currency columns are ordered in decreasing order of the number of 29 | instances of numbers seen for each currency. So if the most numbers you have 30 | in a column are USD, then the USD column renders first. 31 | 32 | * Cost basis specifications should be unmodified and reported to a dedicated 33 | extra column, like this: 34 | 35 | ----- amount ------ 36 | 1 AAPL {21.23 USD} 37 | 38 | We can convert this into two columns and remove the currencies: 39 | 40 | -amount (AAPL)- -Cost basis- 41 | 1 {21.23 USD} 42 | 43 | (Eventually we might support the conversion of cost amounts as well, but they 44 | may contain other information, such as a label or a date, so for now we don't 45 | convert them. I'm not sure there's a good practical use case in doing that 46 | yet.) 47 | 48 | * We may provide some options to break out only some of the currencies into 49 | columns, in order to handle the case where an inventory contains a large 50 | number of currencies and we want to only operate on a restricted set of 51 | operating currencies. 52 | 53 | * If you provide a DisplayFormatter object to the numberification routine, they 54 | quantize each column according to their currency's precision. It is 55 | recommended that you do that. 56 | 57 | """ 58 | __copyright__ = "Copyright (C) 2015-2017 Martin Blais" 59 | __license__ = "GNU GPLv2" 60 | 61 | import collections 62 | from decimal import Decimal 63 | 64 | from beancount.core import amount 65 | from beancount.core import position 66 | from beancount.core import inventory 67 | 68 | from .cursor import Column 69 | 70 | 71 | def numberify_results(columns, drows, dformat=None): 72 | """Number rows containing Amount, Position or Inventory types. 73 | 74 | Args: 75 | result_types: A list of items describing the names and data types of the items in 76 | each column. 77 | result_rows: A list of ResultRow instances. 78 | dformat: An optional DisplayFormatter. If set, quantize the numbers by 79 | their currency-specific precision when converting the Amount's, 80 | Position's or Inventory'es.. 81 | Returns: 82 | A pair of modified (result_types, result_rows) with converted datatypes. 83 | """ 84 | # Build an array of converters. 85 | converters = [] 86 | for index, column in enumerate(columns): 87 | convert_col_fun = CONVERTING_TYPES.get(column.datatype) 88 | if convert_col_fun is None: 89 | converters.append(IdentityConverter(column.name, column.datatype, index)) 90 | else: 91 | col_converters = convert_col_fun(column.name, drows, index) 92 | converters.extend(col_converters) 93 | 94 | # Derive the output types from the expected outputs from the converters 95 | # themselves. 96 | otypes = tuple(Column(c.name, c.dtype) for c in converters) 97 | 98 | # Convert the input rows by processing them through the converters. 99 | orows = [] 100 | for drow in drows: 101 | orow = [] 102 | for converter in converters: 103 | orow.append(converter(drow, dformat)) 104 | orows.append(orow) 105 | 106 | return otypes, orows 107 | 108 | 109 | class IdentityConverter: 110 | """A converter that simply copies its column.""" 111 | 112 | def __init__(self, name, dtype, index): 113 | self.name = name 114 | self.dtype = dtype 115 | self.index = index 116 | 117 | def __call__(self, drow, _): 118 | return drow[self.index] 119 | 120 | 121 | class AmountConverter: 122 | """A converter that extracts the number of an amount for a specific currency.""" 123 | 124 | dtype = Decimal 125 | 126 | def __init__(self, name, index, currency): 127 | self.name = name 128 | self.index = index 129 | self.currency = currency 130 | 131 | def __call__(self, drow, dformat): 132 | vamount = drow[self.index] 133 | if vamount and vamount.currency == self.currency: 134 | number = vamount.number 135 | if dformat: 136 | number = dformat.quantize(number, self.currency) 137 | else: 138 | number = None 139 | return number 140 | 141 | 142 | def convert_col_Amount(name, drows, index): 143 | """Create converters for a column of type Amount. 144 | 145 | Args: 146 | name: A string, the column name. 147 | drows: The table of objects. 148 | index: The column number. 149 | Returns: 150 | A list of Converter instances, one for each of the currency types found. 151 | """ 152 | currency_map = collections.defaultdict(int) 153 | for drow in drows: 154 | vamount = drow[index] 155 | if vamount and vamount.currency: 156 | currency_map[vamount.currency] += 1 157 | return [AmountConverter('{} ({})'.format(name, currency), index, currency) 158 | for currency, _ in sorted(currency_map.items(), 159 | key=lambda item: (item[1], item[0]), 160 | reverse=True)] 161 | 162 | 163 | class PositionConverter: 164 | """A converter that extracts the number of a position for a specific currency.""" 165 | 166 | dtype = Decimal 167 | 168 | def __init__(self, name, index, currency): 169 | self.name = name 170 | self.index = index 171 | self.currency = currency 172 | 173 | def __call__(self, drow, dformat): 174 | pos = drow[self.index] 175 | if pos and pos.units.currency == self.currency: 176 | number = pos.units.number 177 | if dformat: 178 | number = dformat.quantize(pos.units.number, self.currency) 179 | else: 180 | number = None 181 | return number 182 | 183 | 184 | def convert_col_Position(name, drows, index): 185 | """Create converters for a column of type Position. 186 | 187 | Args: 188 | name: A string, the column name. 189 | drows: The table of objects. 190 | index: The column number. 191 | Returns: 192 | A list of Converter instances, one for each of the currency types found. 193 | """ 194 | currency_map = collections.defaultdict(int) 195 | for drow in drows: 196 | pos = drow[index] 197 | if pos and pos.units.currency: 198 | currency_map[pos.units.currency] += 1 199 | return [PositionConverter('{} ({})'.format(name, currency), index, currency) 200 | for currency, _ in sorted(currency_map.items(), 201 | key=lambda item: (item[1], item[0]), 202 | reverse=True)] 203 | 204 | 205 | class InventoryConverter: 206 | """A converter that extracts the number of a inventory for a specific currency. 207 | If there are multiple lots we aggregate by currency.""" 208 | 209 | dtype = Decimal 210 | 211 | def __init__(self, name, index, currency): 212 | self.name = name 213 | self.index = index 214 | self.currency = currency 215 | 216 | def __call__(self, drow, dformat): 217 | inv = drow[self.index] 218 | # FIXME:: get_currency_units() returns ZERO and not None when the value 219 | # isn't present. This should be fixed to distinguish between the two. 220 | number = inv.get_currency_units(self.currency).number 221 | if number and dformat: 222 | number = dformat.quantize(number, self.currency) 223 | return number or None 224 | 225 | 226 | def convert_col_Inventory(name, drows, index): 227 | """Create converters for a column of type Inventory. 228 | 229 | Args: 230 | name: A string, the column name. 231 | drows: The table of objects. 232 | index: The column number. 233 | Returns: 234 | A list of Converter instances, one for each of the currency types found. 235 | """ 236 | currency_map = collections.defaultdict(int) 237 | for drow in drows: 238 | inv = drow[index] 239 | for currency in inv.currencies(): 240 | currency_map[currency] += 1 241 | return [InventoryConverter('{} ({})'.format(name, currency), index, currency) 242 | for currency, _ in sorted(currency_map.items(), 243 | key=lambda item: (item[1], item[0]), 244 | reverse=True)] 245 | 246 | 247 | # A mapping of data types to their converter factory. 248 | CONVERTING_TYPES = { 249 | amount.Amount : convert_col_Amount, 250 | position.Position : convert_col_Position, 251 | inventory.Inventory : convert_col_Inventory, 252 | } 253 | -------------------------------------------------------------------------------- /beanquery/numberify_test.py: -------------------------------------------------------------------------------- 1 | __copyright__ = "Copyright (C) 2015-2017 Martin Blais" 2 | __license__ = "GNU GPLv2" 3 | 4 | import datetime 5 | import unittest 6 | from decimal import Decimal 7 | 8 | from beancount.core.number import D 9 | from beancount.core.amount import A 10 | from beancount.core import amount 11 | from beancount.core import position 12 | from beancount.core import inventory 13 | from beancount.core import display_context 14 | 15 | from beanquery import numberify 16 | from beanquery.cursor import Column 17 | 18 | 19 | class TestNumerifySimple(unittest.TestCase): 20 | 21 | input_amounts = ["24.17 CAD", 22 | "-77.02 CAD", 23 | "11.39 CAD", 24 | "800.00 USD", 25 | "41.17 CAD", 26 | "950.00 USD", 27 | "110 JPY", 28 | "-947.00 USD"] 29 | 30 | expected_types = (('pos (CAD)', Decimal), 31 | ('pos (USD)', Decimal), 32 | ('pos (JPY)', Decimal)) 33 | 34 | expected_rows = [[D('24.17'), None, None], 35 | [D('-77.02'), None, None], 36 | [D('11.39'), None, None], 37 | [None, D('800.00'), None], 38 | [D('41.17'), None, None], 39 | [None, D('950.00'), None], 40 | [None, None, D('110')], 41 | [None, D('-947.00'), None]] 42 | 43 | def test_amount(self): 44 | itypes = (Column('pos', amount.Amount), ) 45 | irows = [(A(string),) for string in self.input_amounts] 46 | atypes, arows = numberify.numberify_results(itypes, irows) 47 | self.assertEqual(self.expected_types, atypes) 48 | self.assertEqual(self.expected_rows, arows) 49 | 50 | def test_position(self): 51 | itypes = (Column('pos', position.Position), ) 52 | irows = [(position.from_string(string),) for string in self.input_amounts] 53 | atypes, arows = numberify.numberify_results(itypes, irows) 54 | self.assertEqual(self.expected_types, atypes) 55 | self.assertEqual(self.expected_rows, arows) 56 | 57 | def test_inventory(self): 58 | itypes = (Column('pos', inventory.Inventory), ) 59 | irows = [(inventory.from_string(string),) for string in self.input_amounts] 60 | atypes, arows = numberify.numberify_results(itypes, irows) 61 | self.assertEqual(self.expected_types, atypes) 62 | self.assertEqual(self.expected_rows, arows) 63 | 64 | 65 | class TestNumerifyIdentity(unittest.TestCase): 66 | 67 | def test_identity(self): 68 | itypes = (Column('date', datetime.date), Column('name', str), Column('count', int), ) 69 | irows = [[datetime.date(2015, 9, 8), 'Testing', 3]] 70 | atypes, arows = numberify.numberify_results(itypes, irows) 71 | self.assertEqual(itypes, atypes) 72 | self.assertEqual(irows, arows) 73 | 74 | 75 | class TestNumerifyInventory(unittest.TestCase): 76 | 77 | def test_inventory(self): 78 | itypes = (Column('balance', inventory.Inventory), ) 79 | irows = [[inventory.from_string('10 HOOL {23.00 USD}')], 80 | [inventory.from_string('2.11 USD, 3.44 CAD')], 81 | [inventory.from_string('-2 HOOL {24.00 USD}, 5.66 CAD')]] 82 | atypes, arows = numberify.numberify_results(itypes, irows) 83 | 84 | self.assertEqual((('balance (HOOL)', Decimal), 85 | ('balance (CAD)', Decimal), 86 | ('balance (USD)', Decimal), ), atypes) 87 | 88 | self.assertEqual([[D('10'), None, None], 89 | [None, D('3.44'), D('2.11')], 90 | [D('-2'), D('5.66'), None]], arows) 91 | 92 | 93 | class TestNumerifyPrecision(unittest.TestCase): 94 | 95 | def test_precision(self): 96 | # Some display context. 97 | dcontext = display_context.DisplayContext() 98 | dcontext.update(D('111'), 'JPY') 99 | dcontext.update(D('1.111'), 'RGAGX') 100 | dcontext.update(D('1.11'), 'USD') 101 | dformat = dcontext.build() 102 | 103 | # Input data. 104 | itypes = (Column('number', Decimal), 105 | Column('amount', amount.Amount), 106 | Column('position', position.Position), 107 | Column('inventory', inventory.Inventory)) 108 | irows = [[D(amt.split()[0]), 109 | A(amt), 110 | position.from_string(amt), 111 | inventory.from_string(amt)] 112 | for amt in ['123.45678909876 JPY', 113 | '1.67321232123 RGAGX', 114 | '5.67345434543 USD']] 115 | 116 | # First check with no explicit quantization. 117 | atypes, arows = numberify.numberify_results(itypes, irows) 118 | erows = [[D('123.45678909876'), 119 | None, None, D('123.45678909876'), 120 | None, None, D('123.45678909876'), 121 | None, None, D('123.45678909876')], 122 | [D('1.67321232123'), 123 | None, D('1.67321232123'), None, 124 | None, D('1.67321232123'), None, 125 | None, D('1.67321232123'), None], 126 | [D('5.67345434543'), 127 | D('5.67345434543'), None, None, 128 | D('5.67345434543'), None, None, 129 | D('5.67345434543'), None, None]] 130 | self.assertEqual(erows, arows) 131 | 132 | # Then compare with quantization. 133 | atypes, arows = numberify.numberify_results(itypes, irows, dformat) 134 | 135 | erows = [[D('123.45678909876'), 136 | None, None, D('123'), 137 | None, None, D('123'), 138 | None, None, D('123')], 139 | [D('1.67321232123'), 140 | None, D('1.673'), None, None, 141 | D('1.673'), None, None, 142 | D('1.673'), None], 143 | [D('5.67345434543'), 144 | D('5.67'), None, None, 145 | D('5.67'), None, None, 146 | D('5.67'), None, None]] 147 | self.assertEqual(erows, arows) 148 | 149 | 150 | if __name__ == '__main__': 151 | unittest.main() 152 | -------------------------------------------------------------------------------- /beanquery/parser/__init__.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import decimal 3 | 4 | import tatsu 5 | 6 | from ..errors import ProgrammingError 7 | from .parser import BQLParser 8 | from . import ast 9 | 10 | 11 | class BQLSemantics: 12 | 13 | def set_context(self, ctx): 14 | self._ctx = ctx 15 | 16 | def null(self, value): 17 | return None 18 | 19 | def integer(self, value): 20 | return int(value) 21 | 22 | def decimal(self, value): 23 | return decimal.Decimal(value) 24 | 25 | def date(self, value): 26 | return datetime.date.fromisoformat(value) 27 | 28 | def string(self, value): 29 | return value[1:-1] 30 | 31 | def boolean(self, value): 32 | return value == 'TRUE' 33 | 34 | def unquoted_identifier(self, value): 35 | return value.lower() 36 | 37 | def quoted_identifier(self, value): 38 | return value.replace('""', '"') 39 | 40 | def asterisk(self, value): 41 | return ast.Asterisk() 42 | 43 | def list(self, value): 44 | return list(value) 45 | 46 | def ordering(self, value): 47 | return ast.Ordering[value or 'ASC'] 48 | 49 | def _default(self, value, typename=None): 50 | if typename is not None: 51 | func = getattr(ast, typename) 52 | return func(**{name.rstrip('_'): value for name, value in value.items()}) 53 | return value 54 | 55 | 56 | class ParseError(ProgrammingError): 57 | def __init__(self, parseinfo): 58 | super().__init__('syntax error') 59 | self.parseinfo = parseinfo 60 | 61 | 62 | def parse(text): 63 | try: 64 | return BQLParser().parse(text, semantics=BQLSemantics()) 65 | except tatsu.exceptions.ParseError as exc: 66 | line = exc.tokenizer.line_info(exc.pos).line 67 | parseinfo = tatsu.infos.ParseInfo(exc.tokenizer, exc.item, exc.pos, exc.pos + 1, line, []) 68 | raise ParseError(parseinfo) from exc 69 | -------------------------------------------------------------------------------- /beanquery/parser/ast.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import dataclasses 4 | import datetime 5 | import enum 6 | import sys 7 | import textwrap 8 | import typing 9 | 10 | 11 | if typing.TYPE_CHECKING: 12 | from typing import Any, Optional, Union 13 | 14 | 15 | def _indent(text): 16 | return textwrap.indent(text, ' ') 17 | 18 | 19 | def _fields(node): 20 | for field in dataclasses.fields(node): 21 | if field.repr: 22 | yield field.name, getattr(node, field.name) 23 | 24 | 25 | def tosexp(node): 26 | if isinstance(node, Node): 27 | return f'({node.__class__.__name__.lower()}\n' + _indent( 28 | '\n'.join(f'{name.replace("_", "-")}: {tosexp(value)}' 29 | for name, value in _fields(node) if value is not None) + ')') 30 | if isinstance(node, list): 31 | return '(\n' + _indent('\n'.join(tosexp(i) for i in node)) + ')' 32 | if isinstance(node, enum.Enum): 33 | return node.name.lower() 34 | return repr(node) 35 | 36 | 37 | def walk(node): 38 | if isinstance(node, Node): 39 | for name, child in _fields(node): 40 | yield from walk(child) 41 | yield node 42 | if isinstance(node, list): 43 | for child in node: 44 | yield from walk(child) 45 | 46 | 47 | class Node: 48 | """Base class for BQL AST nodes.""" 49 | __slots__ = () 50 | parseinfo = None 51 | 52 | @property 53 | def text(self): 54 | if not self.parseinfo: 55 | return None 56 | text = self.parseinfo.tokenizer.text 57 | return text[self.parseinfo.pos:self.parseinfo.endpos] 58 | 59 | def tosexp(self): 60 | return tosexp(self) 61 | 62 | def walk(self): 63 | return walk(self) 64 | 65 | 66 | def node(name, fields): 67 | """Manufacture an AST node class.""" 68 | 69 | return dataclasses.make_dataclass( 70 | name, 71 | [*fields.split(), ('parseinfo', None, dataclasses.field(default=None, compare=False, repr=False))], 72 | bases=(Node,), 73 | **({'slots': True} if sys.version_info[:2] >= (3, 10) else {})) 74 | 75 | 76 | # A 'select' query action. 77 | # 78 | # Attributes: 79 | # targets: Either a single 'Asterisk' instance of a list of 'Target' 80 | # instances. 81 | # from_clause: An instance of 'From', or None if absent. 82 | # where_clause: A root expression node, or None if absent. 83 | # group_by: An instance of 'GroupBy', or None if absent. 84 | # order_by: An instance of 'OrderBy', or None if absent. 85 | # pivot_by: An instance of 'PivotBy', or None if absent. 86 | # limit: An integer, or None is absent. 87 | # distinct: A boolean value (True), or None if absent. 88 | Select = node('Select', 'targets from_clause where_clause group_by order_by pivot_by limit distinct') 89 | 90 | # A select query that produces final balances for accounts. 91 | # This is equivalent to 92 | # 93 | # SELECT account, sum(position) 94 | # FROM ... 95 | # WHERE ... 96 | # GROUP BY account 97 | # 98 | # Attributes: 99 | # summary_func: A method on an inventory to call on the position column. 100 | # May be to extract units, value at cost, etc. 101 | # from_clause: An instance of 'From', or None if absent. 102 | Balances = node('Balances', 'summary_func from_clause where_clause') 103 | 104 | # A select query that produces a journal of postings. 105 | # This is equivalent to 106 | # 107 | # SELECT date, flag, payee, narration, ... FROM 108 | # WHERE account = 109 | # 110 | # Attributes: 111 | # account: A string, the name of the account to restrict to. 112 | # summary_func: A method on an inventory to call on the position column. 113 | # May be to extract units, value at cost, etc. 114 | # from_clause: An instance of 'From', or None if absent. 115 | Journal = node('Journal', 'account summary_func from_clause') 116 | 117 | # A query that will simply print the selected entries in Beancount format. 118 | # 119 | # Attributes: 120 | # from_clause: An instance of 'From', or None if absent. 121 | Print = node('Print', 'from_clause') 122 | 123 | # A parsed SELECT column or target. 124 | # 125 | # Attributes: 126 | # expression: A tree of expression nodes from the parser. 127 | # name: A string, the given name of the target (given by "AS "). 128 | Target = node('Target', 'expression name') 129 | 130 | # A placeholder in SELECT * or COUNT(*) constructs. 131 | Asterisk = node('Asterisk', '') 132 | 133 | # A FROM clause. 134 | # 135 | # Attributes: 136 | # expression: A tree of expression nodes from the parser. 137 | # close: A CLOSE clause, either None if absent, a boolean if the clause 138 | # was present by no date was provided, or a datetime.date instance if 139 | # a date was provided. 140 | @dataclasses.dataclass(**({'slots': True} if sys.version_info[:2] >= (3, 10) else {})) 141 | class From(Node): 142 | expression: Optional[Node] = None 143 | open: Optional[datetime.date] = None 144 | close: Optional[Union[datetime.date, bool]] = None 145 | clear: Optional[bool] = None 146 | parseinfo: Any = dataclasses.field(default=None, compare=False, repr=False) 147 | 148 | # A GROUP BY clause. 149 | # 150 | # Attributes: 151 | # columns: A list of group-by expressions, simple Column() or otherwise. 152 | # having: An expression tree for the optional HAVING clause, or None. 153 | GroupBy = node('GroupBy', 'columns having') 154 | 155 | # An ORDER BY clause. 156 | # 157 | # Attributes: 158 | # column: order-by expression, simple Column() or otherwise. 159 | # ordering: The sort order as an Ordering enum value. 160 | OrderBy = node('OrderBy', 'column ordering') 161 | 162 | class Ordering(enum.IntEnum): 163 | # The enum values are chosen in this way to be able to use them 164 | # directly as the reverse parameter to the list sort() method. 165 | ASC = 0 166 | DESC = 1 167 | 168 | def __repr__(self): 169 | return f"{self.__class__.__name__}.{self.name}" 170 | 171 | # An PIVOT BY clause. 172 | # 173 | # Attributes: 174 | # columns: A list of group-by expressions, simple Column() or otherwise. 175 | PivotBy = node('PivotBy', 'columns') 176 | 177 | # A reference to a table. 178 | # 179 | # Attributes: 180 | # name: The table name. 181 | Table = node('Table', 'name') 182 | 183 | # A reference to a column. 184 | # 185 | # Attributes: 186 | # name: A string, the name of the column to access. 187 | Column = node('Column', 'name') 188 | 189 | # A function call. 190 | # 191 | # Attributes: 192 | # fname: A string, the name of the function. 193 | # operands: A list of other expressions, the arguments of the function to 194 | # evaluate. This is possibly an empty list. 195 | Function = node('Function', 'fname operands') 196 | 197 | Attribute = node('Attribute', 'operand name') 198 | 199 | Subscript = node('Subscript', 'operand key') 200 | 201 | # A constant node. 202 | # 203 | # Attributes: 204 | # value: The constant value this represents. 205 | Constant = node('Constant', 'value') 206 | 207 | # A query parameter placeholder. 208 | # 209 | # Attributes: 210 | # name: The placeholder name 211 | Placeholder = node('Placeholder', 'name') 212 | 213 | # Base class for unary operators. 214 | # 215 | # Attributes: 216 | # operand: An expression, the operand of the operator. 217 | UnaryOp = node('UnaryOp', 'operand') 218 | 219 | # Base class for binary operators. 220 | # 221 | # Attributes: 222 | # left: An expression, the left operand. 223 | # right: An expression, the right operand. 224 | BinaryOp = node('BinaryOp', 'left right') 225 | 226 | # Base class for boolean operators. 227 | BoolOp = node('BoolOp', 'args') 228 | 229 | # Between 230 | Between = node('Between', 'operand lower upper') 231 | 232 | # Negation operator. 233 | class Not(UnaryOp): 234 | __slots__ = () 235 | 236 | class IsNull(UnaryOp): 237 | __slots__ = () 238 | 239 | class IsNotNull(UnaryOp): 240 | __slots__ = () 241 | 242 | 243 | # Boolean operators. 244 | 245 | class And(BoolOp): 246 | __slots__ = () 247 | 248 | class Or(BoolOp): 249 | __slots__ = () 250 | 251 | 252 | # Equality and inequality comparison operators. 253 | 254 | class Equal(BinaryOp): 255 | __slots__ = () 256 | 257 | class NotEqual(BinaryOp): 258 | __slots__ = () 259 | 260 | class Greater(BinaryOp): 261 | __slots__ = () 262 | 263 | class GreaterEq(BinaryOp): 264 | __slots__ = () 265 | 266 | class Less(BinaryOp): 267 | __slots__ = () 268 | 269 | class LessEq(BinaryOp): 270 | __slots__ = () 271 | 272 | 273 | # Regular expression match operator. 274 | 275 | class Match(BinaryOp): 276 | __slots__ = () 277 | 278 | 279 | class NotMatch(BinaryOp): 280 | __slots__ = () 281 | 282 | 283 | class Matches(BinaryOp): 284 | __slots__ = () 285 | 286 | 287 | # Membership operators. 288 | 289 | class In(BinaryOp): 290 | __slots__ = () 291 | 292 | class NotIn(BinaryOp): 293 | __slots__ = () 294 | 295 | 296 | # Arithmetic operators. 297 | 298 | class Neg(UnaryOp): 299 | __slots__ = () 300 | 301 | class Mul(BinaryOp): 302 | __slots__ = () 303 | 304 | class Div(BinaryOp): 305 | __slots__ = () 306 | 307 | class Mod(BinaryOp): 308 | __slots__ = () 309 | 310 | class Add(BinaryOp): 311 | __slots__ = () 312 | 313 | class Sub(BinaryOp): 314 | __slots__ = () 315 | 316 | 317 | Any = node('Any', 'left op right') 318 | All = node('All', 'left op right') 319 | 320 | 321 | CreateTable = node('CreateTable', 'name columns using query') 322 | 323 | Insert = node('Insert', 'table columns values') 324 | -------------------------------------------------------------------------------- /beanquery/parser/bql.ebnf: -------------------------------------------------------------------------------- 1 | @@grammar :: BQL 2 | @@parseinfo :: True 3 | @@ignorecase :: True 4 | @@keyword :: 'AND' 'AS' 'ASC' 'BY' 'DESC' 'DISTINCT' 'FALSE' 'FROM' 5 | 'GROUP' 'HAVING' 'IN' 'IS' 'LIMIT' 'NOT' 'OR' 'ORDER' 'PIVOT' 6 | 'SELECT' 'TRUE' 'WHERE' 7 | @@keyword :: 'CREATE' 'TABLE' 'USING' 'INSERT' 'INTO' 8 | @@keyword :: 'BALANCES' 'JOURNAL' 'PRINT' 9 | @@comments :: /(\/\*([^*]|[\r\n]|(\*+([^*\/]|[\r\n])))*\*+\/)/ 10 | @@eol_comments :: /\;[^\n]*?$/ 11 | 12 | bql 13 | = @:statement [';'] $ 14 | ; 15 | 16 | statement 17 | = 18 | | select 19 | | balances 20 | | journal 21 | | print 22 | | create_table 23 | | insert 24 | ; 25 | 26 | select::Select 27 | = 'SELECT' ['DISTINCT' distinct:`True`] targets:(','.{ target }+ | asterisk) 28 | ['FROM' from_clause:(_table | subselect | from)] 29 | ['WHERE' where_clause:expression] 30 | ['GROUP' 'BY' group_by:groupby] 31 | ['ORDER' 'BY' order_by:','.{order}+] 32 | ['PIVOT' 'BY' pivot_by:pivotby] 33 | ['LIMIT' limit:integer] 34 | ; 35 | 36 | subselect 37 | = '(' @:select ')' 38 | ; 39 | 40 | from::From 41 | = 42 | | 'OPEN' ~ 'ON' open:date ['CLOSE' ('ON' close:date | {} close:`True`)] ['CLEAR' clear:`True`] 43 | | 'CLOSE' ~ ('ON' close:date | {} close:`True`) ['CLEAR' clear:`True`] 44 | | 'CLEAR' ~ clear:`True` 45 | | expression:expression ['OPEN' 'ON' open:date] ['CLOSE' ('ON' close:date | {} close:`True`)] ['CLEAR' clear:`True`] 46 | ; 47 | 48 | _table::Table 49 | = 50 | | name:/#([a-zA-Z_][a-zA-Z0-9_]*)?/ 51 | | name:quoted_identifier 52 | ; 53 | 54 | table::Table 55 | = name:identifier 56 | ; 57 | 58 | groupby::GroupBy 59 | = columns:','.{ (integer | expression) }+ ['HAVING' having:expression] 60 | ; 61 | 62 | order::OrderBy 63 | = column:(integer | expression) ordering:ordering 64 | ; 65 | 66 | ordering 67 | = ['DESC' | 'ASC'] 68 | ; 69 | 70 | pivotby::PivotBy 71 | = columns+:(integer | column) ',' columns+:(integer | column) 72 | ; 73 | 74 | target::Target 75 | = expression:expression ['AS' name:identifier] 76 | ; 77 | 78 | expression 79 | = 80 | | disjunction 81 | | conjunction 82 | ; 83 | 84 | disjunction 85 | = 86 | | or 87 | | conjunction 88 | ; 89 | 90 | or::Or::BoolOp 91 | = args+:conjunction { 'OR' args+:conjunction }+ 92 | ; 93 | 94 | conjunction 95 | = 96 | | and 97 | | inversion 98 | ; 99 | 100 | and::And::BoolOp 101 | = args+:inversion { 'AND' args+:inversion }+ 102 | ; 103 | 104 | inversion 105 | = 106 | | not 107 | | comparison 108 | ; 109 | 110 | not::Not::UnaryOp 111 | = 'NOT' operand:inversion 112 | ; 113 | 114 | comparison 115 | = 116 | | any 117 | | all 118 | | lt 119 | | lte 120 | | gt 121 | | gte 122 | | eq 123 | | neq 124 | | in 125 | | notin 126 | | match 127 | | notmatch 128 | | matches 129 | | isnull 130 | | isnotnull 131 | | between 132 | | sum 133 | ; 134 | 135 | any::Any 136 | = left:sum op:op 'any' '(' right:expression ')' 137 | ; 138 | 139 | all::All 140 | = left:sum op:op 'all' '(' right:expression ')' 141 | ; 142 | 143 | op 144 | = 145 | | '<' 146 | | '<=' 147 | | '>' 148 | | '>=' 149 | | '=' 150 | | '!=' 151 | | '~' 152 | | '!~' 153 | | '?~' 154 | ; 155 | 156 | lt::Less::BinaryOp 157 | = left:sum '<' right:sum 158 | ; 159 | 160 | lte::LessEq::BinaryOp 161 | = left:sum '<=' right:sum 162 | ; 163 | 164 | gt::Greater::BinaryOp 165 | = left:sum '>' right:sum 166 | ; 167 | 168 | gte::GreaterEq::BinaryOp 169 | = left:sum '>=' right:sum 170 | ; 171 | 172 | eq::Equal::BinaryOp 173 | = left:sum '=' right:sum 174 | ; 175 | 176 | neq::NotEqual::BinaryOp 177 | = left:sum '!=' right:sum 178 | ; 179 | 180 | in::In::BinaryOp 181 | = left:sum 'IN' right:sum 182 | ; 183 | 184 | notin::NotIn::BinaryOp 185 | = left:sum 'NOT' 'IN' right:sum 186 | ; 187 | 188 | match::Match::BinaryOp 189 | = left:sum '~' right:sum 190 | ; 191 | 192 | notmatch::NotMatch::BinaryOp 193 | = left:sum '!~' right:sum 194 | ; 195 | 196 | matches::Matches::BinaryOp 197 | = left:sum '?~' right:sum 198 | ; 199 | 200 | isnull::IsNull::UnaryOp 201 | = operand:sum 'IS' 'NULL' 202 | ; 203 | 204 | isnotnull::IsNotNull::UnaryOp 205 | = operand:sum 'IS' 'NOT' 'NULL' 206 | ; 207 | 208 | between::Between 209 | = operand:sum 'BETWEEN' lower:sum 'AND' upper:sum 210 | ; 211 | 212 | sum 213 | = 214 | | add 215 | | sub 216 | | term 217 | ; 218 | 219 | add::Add::BinaryOp 220 | = left:sum '+' ~ right:term 221 | ; 222 | 223 | sub::Sub::BinaryOp 224 | = left:sum '-' ~ right:term 225 | ; 226 | 227 | term 228 | = 229 | | mul 230 | | div 231 | | mod 232 | | factor 233 | ; 234 | 235 | mul::Mul::BinaryOp 236 | = left:term '*' ~ right:factor 237 | ; 238 | 239 | div::Div::BinaryOp 240 | = left:term '/' ~ right:factor 241 | ; 242 | 243 | mod::Mod::BinaryOp 244 | = left:term '%' ~ right:factor 245 | ; 246 | 247 | factor 248 | = 249 | | unary 250 | | '(' @:expression ')' 251 | ; 252 | 253 | unary 254 | = 255 | | uplus 256 | | uminus 257 | | primary 258 | ; 259 | 260 | uplus 261 | = '+' @:atom 262 | ; 263 | 264 | uminus::Neg::UnaryOp 265 | = '-' operand:factor 266 | ; 267 | 268 | primary 269 | = 270 | | attribute 271 | | subscript 272 | | atom 273 | ; 274 | 275 | attribute::Attribute 276 | = operand:primary '.' name:identifier 277 | ; 278 | 279 | subscript::Subscript 280 | = operand:primary '[' key:string ']' 281 | ; 282 | 283 | atom 284 | = 285 | | select 286 | | function 287 | | constant 288 | | column 289 | | placeholder 290 | ; 291 | 292 | placeholder::Placeholder 293 | = 294 | | '%s' name:`` 295 | | '%(' name:identifier ')s' 296 | ; 297 | 298 | function::Function 299 | = 300 | | fname:identifier '(' operands:','.{ expression } ')' 301 | | fname:identifier '(' operands+:asterisk ')' 302 | ; 303 | 304 | column::Column 305 | = name:identifier 306 | ; 307 | 308 | literal 309 | = 310 | | date 311 | | decimal 312 | | integer 313 | | string 314 | | null 315 | | boolean 316 | ; 317 | 318 | constant::Constant 319 | = value:(literal | list) 320 | ; 321 | 322 | list 323 | = '(' &( literal ',') @:','.{ (literal | ()) }+ ')' 324 | ; 325 | 326 | identifier 327 | = 328 | | unquoted_identifier 329 | | quoted_identifier 330 | ; 331 | 332 | @name 333 | unquoted_identifier 334 | = /[a-zA-Z_][a-zA-Z0-9_]*/ 335 | ; 336 | 337 | quoted_identifier 338 | = /\"((?:[^\"]|\"\")+)\"/ 339 | ; 340 | 341 | asterisk 342 | = '*' 343 | ; 344 | 345 | string 346 | = /\"[^\"]*\"|\'(?:[^\']|\'\')*\'/ 347 | ; 348 | 349 | boolean 350 | = 'TRUE' | 'FALSE' 351 | ; 352 | 353 | null 354 | = 'NULL' 355 | ; 356 | 357 | integer 358 | = /[0-9]+/ 359 | ; 360 | 361 | decimal 362 | = /[0-9]+\.[0-9]*|[0-9]*\.[0-9]+/ 363 | ; 364 | 365 | date 366 | = /[0-9]{4}-[0-9]{2}-[0-9]{2}/ 367 | ; 368 | 369 | balances::Balances 370 | = 'BALANCES' 371 | ['AT' summary_func:identifier] 372 | ['FROM' from_clause:from] 373 | ['WHERE' where_clause:expression] 374 | ; 375 | 376 | journal::Journal 377 | = 'JOURNAL' 378 | [account:string] 379 | ['AT' summary_func:identifier] 380 | ['FROM' from_clause:from] 381 | ; 382 | 383 | print::Print 384 | = 'PRINT' 385 | ['FROM' from_clause:from] 386 | ; 387 | 388 | create_table::CreateTable 389 | = 'CREATE' 'TABLE' ~ name:identifier 390 | ( 391 | | '(' columns:','.{( identifier identifier )} ')' ['USING' using:string] 392 | | 'USING' using:string 393 | | 'AS' query:select 394 | ) 395 | ; 396 | 397 | insert::Insert 398 | = 'INSERT' 'INTO' ~ table:table 399 | ['(' columns:','.{column} ')'] 400 | 'VALUES' '(' values:','.{expression} ')' 401 | ; 402 | -------------------------------------------------------------------------------- /beanquery/query.py: -------------------------------------------------------------------------------- 1 | """A library to run queries. This glues together all the parts of the query engine. 2 | """ 3 | __copyright__ = "Copyright (C) 2015-2017 Martin Blais" 4 | __license__ = "GNU GPLv2" 5 | 6 | import beanquery 7 | import beanquery.numberify 8 | 9 | 10 | def run_query(entries, options, query, *args, numberify=False): 11 | """Compile and execute a query, return the result types and rows. 12 | 13 | Args: 14 | entries: A list of entries, as produced by the loader. 15 | options: A dict of options, as produced by the loader. 16 | query: A string, a single BQL query, optionally containing some new-style 17 | (e.g., {}) formatting specifications. 18 | args: A tuple of arguments to be formatted in the query. This is 19 | just provided as a convenience. 20 | numberify: If true, numberify the results before returning them. 21 | Returns: 22 | A pair of result types and result rows. 23 | Raises: 24 | ParseError: If the statement cannot be parsed. 25 | CompilationError: If the statement cannot be compiled. 26 | """ 27 | 28 | # Execute the query. 29 | ctx = beanquery.connect('beancount:', entries=entries, errors=[], options=options) 30 | curs = ctx.execute(query.format(*args)) 31 | rrows = curs.fetchall() 32 | rtypes = curs.description 33 | 34 | # Numberify the results, if requested. 35 | if numberify: 36 | dformat = options['dcontext'].build() 37 | rtypes, rrows = beanquery.numberify.numberify_results(rtypes, rrows, dformat) 38 | 39 | return rtypes, rrows 40 | -------------------------------------------------------------------------------- /beanquery/query_compile.py: -------------------------------------------------------------------------------- 1 | """Interpreter for the query language's AST. 2 | 3 | This code accepts the abstract syntax tree produced by the query parser, 4 | resolves the column and function names, compiles and interpreter and prepares a 5 | query to be run against a list of entries. 6 | """ 7 | 8 | from __future__ import annotations 9 | 10 | __copyright__ = "Copyright (C) 2014-2016 Martin Blais" 11 | __license__ = "GNU GPLv2" 12 | 13 | import collections 14 | import dataclasses 15 | import datetime 16 | import itertools 17 | import re 18 | import operator 19 | 20 | from decimal import Decimal 21 | from typing import List 22 | 23 | from dateutil.relativedelta import relativedelta 24 | 25 | from beanquery import cursor 26 | from beanquery.parser import ast 27 | from beanquery import query_execute 28 | from beanquery import types 29 | from beanquery import tables 30 | 31 | 32 | MARKER = object() 33 | 34 | FUNCTIONS = collections.defaultdict(list) 35 | OPERATORS = collections.defaultdict(list) 36 | 37 | 38 | class EvalNode: 39 | __slots__ = ('dtype',) 40 | 41 | def __init__(self, dtype): 42 | # The output data type produce by this node. This is intended to be 43 | # inferred by the nodes on construction. 44 | assert dtype is not None, "Internal erro: Invalid dtype, must be deduced." 45 | self.dtype = dtype 46 | 47 | def __eq__(self, other): 48 | if not isinstance(other, type(self)): 49 | return NotImplemented 50 | return all(getattr(self, attr) == getattr(other, attr) for attr in self.__slots__) 51 | 52 | def __str__(self): 53 | return "{}({})".format(type(self).__name__, 54 | ', '.join(repr(getattr(self, child)) 55 | for child in self.__slots__)) 56 | __repr__ = __str__ 57 | 58 | def childnodes(self): 59 | """Returns the child nodes of this node. 60 | Yields: 61 | A list of EvalNode instances. 62 | """ 63 | for attr in self.__slots__: 64 | child = getattr(self, attr) 65 | if isinstance(child, EvalNode): 66 | yield child 67 | elif isinstance(child, list): 68 | for element in child: 69 | if isinstance(element, EvalNode): 70 | yield element 71 | 72 | def __call__(self, context): 73 | """Evaluate this node. This is designed to recurse on its children. 74 | All subclasses must override and implement this method. 75 | 76 | Args: 77 | context: The evaluation object to which the evaluation need to apply. 78 | This is either an entry, a Posting instance, or a particular result 79 | set row from a sub-select. This is the provider for the underlying 80 | data. 81 | Returns: 82 | The evaluated value for this sub-expression tree. 83 | """ 84 | raise NotImplementedError 85 | 86 | 87 | class EvalConstant(EvalNode): 88 | __slots__ = ('value',) 89 | 90 | def __init__(self, value, dtype=None): 91 | super().__init__(type(value) if dtype is None else dtype) 92 | self.value = value 93 | 94 | def __call__(self, _): 95 | return self.value 96 | 97 | 98 | class EvalUnaryOp(EvalNode): 99 | __slots__ = ('operand', 'operator') 100 | 101 | def __init__(self, operator, operand, dtype): 102 | super().__init__(dtype) 103 | self.operand = operand 104 | self.operator = operator 105 | 106 | def __call__(self, context): 107 | operand = self.operand(context) 108 | return self.operator(operand) 109 | 110 | def __repr__(self): 111 | return f'{self.__class__.__name__}({self.operator!r})' 112 | 113 | 114 | class EvalUnaryOpSafe(EvalUnaryOp): 115 | 116 | def __call__(self, context): 117 | operand = self.operand(context) 118 | if operand is None: 119 | return None 120 | return self.operator(operand) 121 | 122 | 123 | class EvalBinaryOp(EvalNode): 124 | __slots__ = ('left', 'right', 'operator') 125 | 126 | def __init__(self, operator, left, right, dtype): 127 | super().__init__(dtype) 128 | self.operator = operator 129 | self.left = left 130 | self.right = right 131 | 132 | def __call__(self, context): 133 | left = self.left(context) 134 | if left is None: 135 | return None 136 | right = self.right(context) 137 | if right is None: 138 | return None 139 | return self.operator(left, right) 140 | 141 | def __repr__(self): 142 | return f'{self.__class__.__name__}({self.left!r}, {self.right!r})' 143 | 144 | 145 | class EvalBetween(EvalNode): 146 | __slots__ = ('operand', 'lower', 'upper') 147 | 148 | def __init__(self, operand, lower, upper): 149 | super().__init__(bool) 150 | self.operand = operand 151 | self.lower = lower 152 | self.upper = upper 153 | 154 | def __call__(self, context): 155 | operand = self.operand(context) 156 | if operand is None: 157 | return None 158 | lower = self.lower(context) 159 | if lower is None: 160 | return None 161 | upper = self.upper(context) 162 | if upper is None: 163 | return None 164 | return lower <= operand <= upper 165 | 166 | 167 | def unaryop(op, intypes, outtype, nullsafe=False): 168 | def decorator(func): 169 | class Op(EvalUnaryOp if nullsafe else EvalUnaryOpSafe): 170 | __intypes__ = intypes 171 | def __init__(self, operand): 172 | super().__init__(func, operand, outtype) 173 | Op.__name__ = f'{op.__name__}[{intypes[0].__name__}]' 174 | OPERATORS[op].append(Op) 175 | return func 176 | return decorator 177 | 178 | 179 | def binaryop(op, intypes, outtype): 180 | def decorator(func): 181 | class Op(EvalBinaryOp): 182 | __intypes__ = intypes 183 | def __init__(self, left, right): 184 | super().__init__(func, left, right, outtype) 185 | Op.__name__ = f'{op.__name__}[{intypes[0].__name__},{intypes[1].__name__}]' 186 | OPERATORS[op].append(Op) 187 | return func 188 | return decorator 189 | 190 | 191 | def Operator(op, operands): 192 | op = types.function_lookup(OPERATORS, op, operands) 193 | if op is not None: 194 | return op(*operands) 195 | raise KeyError 196 | 197 | 198 | unaryop(ast.Not, [types.Any], bool, nullsafe=True)(operator.not_) 199 | 200 | @unaryop(ast.Neg, [int], int) 201 | @unaryop(ast.Neg, [Decimal], Decimal) 202 | def neg_(x): 203 | return -x 204 | 205 | 206 | @unaryop(ast.IsNull, [types.Any], bool, nullsafe=True) 207 | def null(x): 208 | return x is None 209 | 210 | 211 | @unaryop(ast.IsNotNull, [types.Any], bool, nullsafe=True) 212 | def not_null(x): 213 | return x is not None 214 | 215 | 216 | @binaryop(ast.Mul, [Decimal, Decimal], Decimal) 217 | @binaryop(ast.Mul, [Decimal, int], Decimal) 218 | @binaryop(ast.Mul, [int, Decimal], Decimal) 219 | @binaryop(ast.Mul, [int, int], int) 220 | def mul_(x, y): 221 | return x * y 222 | 223 | 224 | @binaryop(ast.Div, [Decimal, Decimal], Decimal) 225 | @binaryop(ast.Div, [Decimal, int], Decimal) 226 | @binaryop(ast.Div, [int, Decimal], Decimal) 227 | def div_(x, y): 228 | if y == 0: 229 | return None 230 | return x / y 231 | 232 | 233 | @binaryop(ast.Div, [int, int], Decimal) 234 | def div_int(x, y): 235 | if y == 0: 236 | return None 237 | return Decimal(x) / y 238 | 239 | 240 | @binaryop(ast.Mod, [int, int], int) 241 | @binaryop(ast.Mod, [Decimal, int], Decimal) 242 | @binaryop(ast.Mod, [int, Decimal], Decimal) 243 | @binaryop(ast.Mod, [Decimal, Decimal], Decimal) 244 | def mod_(x, y): 245 | if y == 0: 246 | return None 247 | return x % y 248 | 249 | 250 | @binaryop(ast.Add, [Decimal, Decimal], Decimal) 251 | @binaryop(ast.Add, [Decimal, int], Decimal) 252 | @binaryop(ast.Add, [int, Decimal], Decimal) 253 | @binaryop(ast.Add, [int, int], int) 254 | @binaryop(ast.Add, [str, str], str) 255 | @binaryop(ast.Add, [datetime.date, relativedelta], datetime.date) 256 | @binaryop(ast.Add, [relativedelta, datetime.date], datetime.date) 257 | @binaryop(ast.Add, [relativedelta, relativedelta], relativedelta) 258 | def add_(x, y): 259 | return x + y 260 | 261 | 262 | @binaryop(ast.Sub, [Decimal, Decimal], Decimal) 263 | @binaryop(ast.Sub, [Decimal, int], Decimal) 264 | @binaryop(ast.Sub, [int, Decimal], Decimal) 265 | @binaryop(ast.Sub, [int, int], int) 266 | @binaryop(ast.Sub, [datetime.date, relativedelta], datetime.date) 267 | @binaryop(ast.Sub, [relativedelta, datetime.date], datetime.date) 268 | @binaryop(ast.Sub, [relativedelta, relativedelta], datetime.date) 269 | def sub_(x, y): 270 | return x - y 271 | 272 | 273 | @binaryop(ast.Add, [datetime.date, int], datetime.date) 274 | def add_date_int(x, y): 275 | return x + datetime.timedelta(days=y) 276 | 277 | 278 | @binaryop(ast.Add, [int, datetime.date], datetime.date) 279 | def add_int_date(x, y): 280 | return y + datetime.timedelta(days=x) 281 | 282 | 283 | @binaryop(ast.Sub, [datetime.date, int], datetime.date) 284 | def sub_date_int(x, y): 285 | return x - datetime.timedelta(days=y) 286 | 287 | 288 | @binaryop(ast.Sub, [datetime.date, datetime.date], int) 289 | def sub_date_date(x, y): 290 | return (x - y).days 291 | 292 | 293 | @binaryop(ast.Match, [str, str], bool) 294 | def match_(x, y): 295 | return bool(re.search(y, x, re.IGNORECASE)) 296 | 297 | 298 | @binaryop(ast.NotMatch, [str, str], bool) 299 | def not_match_(x, y): 300 | return not bool(re.search(y, x, re.IGNORECASE)) 301 | 302 | 303 | @binaryop(ast.Matches, [str, str], bool) 304 | def matches_(x, y): 305 | return bool(re.search(x, y)) 306 | 307 | 308 | @binaryop(ast.In, [types.Any, set], bool) 309 | @binaryop(ast.In, [types.Any, list], bool) 310 | @binaryop(ast.In, [types.Any, dict], bool) 311 | def in_(x, y): 312 | return operator.contains(y, x) 313 | 314 | 315 | @binaryop(ast.NotIn, [types.Any, set], bool) 316 | @binaryop(ast.NotIn, [types.Any, list], bool) 317 | @binaryop(ast.NotIn, [types.Any, dict], bool) 318 | def not_in_(x, y): 319 | return not operator.contains(y, x) 320 | 321 | 322 | _comparisons = [ 323 | (ast.Equal, operator.eq), 324 | (ast.NotEqual, operator.ne), 325 | (ast.Greater, operator.gt), 326 | (ast.GreaterEq, operator.ge), 327 | (ast.Less, operator.lt), 328 | (ast.LessEq, operator.le), 329 | ] 330 | 331 | _intypes = [ 332 | [int, int], 333 | [Decimal, int], 334 | [int, Decimal], 335 | [Decimal, Decimal], 336 | [datetime.date, datetime.date], 337 | [str, str], 338 | ] 339 | 340 | for node, op in _comparisons: 341 | for intypes in _intypes: 342 | binaryop(node, intypes, bool)(op) 343 | 344 | _comparable = [ 345 | # lists of types that can be compared with each other 346 | [int, Decimal], 347 | [datetime.date], 348 | [str], 349 | ] 350 | 351 | for comparable in _comparable: 352 | for intypes in itertools.product(comparable, repeat=3): 353 | class Between(EvalBetween): 354 | __intypes__ = list(intypes) 355 | OPERATORS[ast.Between].append(Between) 356 | 357 | 358 | class EvalAnd(EvalNode): 359 | __slots__ = ('args',) 360 | 361 | def __init__(self, args): 362 | super().__init__(bool) 363 | self.args = args 364 | 365 | def __call__(self, context): 366 | for arg in self.args: 367 | value = arg(context) 368 | if value is None: 369 | return None 370 | if not value: 371 | return False 372 | return True 373 | 374 | 375 | class EvalOr(EvalNode): 376 | __slots__ = ('args',) 377 | 378 | def __init__(self, args): 379 | super().__init__(bool) 380 | self.args = args 381 | 382 | def __call__(self, context): 383 | r = False 384 | for arg in self.args: 385 | value = arg(context) 386 | if value is None: 387 | r = None 388 | if value: 389 | return True 390 | return r 391 | 392 | 393 | class EvalCoalesce(EvalNode): 394 | __slots__ = ('args',) 395 | 396 | def __init__(self, args): 397 | super().__init__(args[0].dtype) 398 | self.args = args 399 | 400 | def __call__(self, context): 401 | for arg in self.args: 402 | value = arg(context) 403 | if value is not None: 404 | return value 405 | return None 406 | 407 | 408 | class EvalFunction(EvalNode): 409 | __slots__ = ('operands',) 410 | 411 | # Type constraints on the input arguments. 412 | __intypes__ = [] 413 | 414 | def __init__(self, context, operands, dtype): 415 | super().__init__(dtype) 416 | self.context = context 417 | self.operands = operands 418 | 419 | 420 | class EvalGetItem(EvalNode): 421 | __slots__ = ('operand', 'key') 422 | 423 | def __init__(self, operand, key): 424 | super().__init__(object) 425 | self.operand = operand 426 | self.key = key 427 | 428 | def __call__(self, context): 429 | operand = self.operand(context) 430 | if operand is None: 431 | return None 432 | return operand.get(self.key) 433 | 434 | 435 | class EvalGetter(EvalNode): 436 | __slots__ = ('operand', 'getter') 437 | 438 | def __init__(self, operand, getter, dtype): 439 | super().__init__(dtype) 440 | self.operand = operand 441 | self.getter = getter 442 | 443 | def __call__(self, context): 444 | operand = self.operand(context) 445 | if operand is None: 446 | return None 447 | return self.getter(operand) 448 | 449 | 450 | class EvalAny(EvalNode): 451 | __slots__ = ('op', 'left', 'right') 452 | 453 | def __init__(self, op, left, right): 454 | super().__init__(bool) 455 | self.op = op 456 | self.left = left 457 | self.right = right 458 | 459 | def __call__(self, row): 460 | left = self.left(row) 461 | if left is None: 462 | return None 463 | right = self.right(row) 464 | if right is None: 465 | return None 466 | return any(self.op(left, x) for x in right) 467 | 468 | 469 | class EvalAll(EvalNode): 470 | __slots__ = ('op', 'left', 'right') 471 | 472 | def __init__(self, op, left, right): 473 | super().__init__(bool) 474 | self.op = op 475 | self.left = left 476 | self.right = right 477 | 478 | def __call__(self, row): 479 | left = self.left(row) 480 | if left is None: 481 | return None 482 | right = self.right(row) 483 | if right is None: 484 | return None 485 | return all(self.op(left, x) for x in right) 486 | 487 | 488 | class EvalRow(EvalNode): 489 | __slots__ = () 490 | 491 | def __init__(self): 492 | super().__init__(object) 493 | 494 | def __call__(self, context): 495 | return context 496 | 497 | 498 | class EvalColumn(EvalNode): 499 | pass 500 | 501 | 502 | class EvalAggregator(EvalFunction): 503 | pure = False 504 | 505 | def __init__(self, context, operands, dtype=None): 506 | super().__init__(context, operands, dtype or operands[0].dtype) 507 | self.value = None 508 | 509 | def allocate(self, allocator): 510 | """Allocate handles to store data for a node's aggregate storage. 511 | 512 | This is called once before beginning aggregations. If you need any 513 | kind of per-aggregate storage during the computation phase, get it 514 | in this method. 515 | 516 | Args: 517 | allocator: An instance of Allocator, on which you can call allocate() to 518 | obtain a handle for a slot to store data on store objects later on. 519 | """ 520 | self.handle = allocator.allocate() 521 | 522 | def initialize(self, store): 523 | """Initialize this node's aggregate data. 524 | 525 | Args: 526 | store: An object indexable by handles appropriated during allocate(). 527 | """ 528 | store[self.handle] = self.dtype() 529 | self.value = None 530 | 531 | def update(self, store, context): 532 | """Evaluate this node. This is designed to recurse on its children. 533 | 534 | Args: 535 | store: An object indexable by handles appropriated during allocate(). 536 | context: The object to which the evaluation need to apply (see __call__). 537 | """ 538 | # Do nothing by default. 539 | 540 | def finalize(self, store): 541 | """Finalize this node's aggregate data. 542 | 543 | Args: 544 | store: An object indexable by handles appropriated during allocate(). 545 | """ 546 | self.value = store[self.handle] 547 | 548 | def __call__(self, context): 549 | """Return the value on evaluation. 550 | 551 | Args: 552 | context: The evaluation object to which the evaluation need to apply. 553 | Returns: 554 | The final aggregated value. 555 | """ 556 | return self.value 557 | 558 | 559 | class SubqueryTable(tables.Table): 560 | def __init__(self, subquery): 561 | self.columns = {} 562 | self.subquery = subquery 563 | for i, target in enumerate(target for target in subquery.c_targets if target.name is not None): 564 | column = self.column(i, target.name, target.c_expr.dtype) 565 | self.columns[target.name] = column() 566 | 567 | @staticmethod 568 | def column(i, name, dtype): 569 | class Column(EvalColumn): 570 | def __init__(self): 571 | super().__init__(dtype) 572 | __call__ = staticmethod(operator.itemgetter(i)) 573 | return Column 574 | 575 | def __iter__(self): 576 | columns, rows = self.subquery() 577 | return iter(rows) 578 | 579 | 580 | class EvalConstantSubquery1D(EvalNode): 581 | def __init__(self, subquery): 582 | self.dtype = List[subquery.columns[0].c_expr.dtype] 583 | self.subquery = subquery 584 | self.value = MARKER 585 | 586 | def __call__(self, context): 587 | if self.value is MARKER: 588 | # Subqueries accessing the current row are not supported yet, 589 | # thus the subquery result can be simply cached. 590 | columns, rows = self.subquery() 591 | value = [row[0] for row in rows] 592 | # Subqueries not returning any row are threates as NULL. 593 | self.value = value if value else None 594 | return self.value 595 | 596 | 597 | # A compiled target. 598 | # 599 | # Attributes: 600 | # c_expr: A compiled expression tree (an EvalNode root node). 601 | # name: The name of the target. If None, this is an invisible 602 | # target that gets evaluated but not displayed. 603 | # is_aggregate: A boolean, true if 'c_expr' is an aggregate. 604 | EvalTarget = collections.namedtuple('EvalTarget', 'c_expr name is_aggregate') 605 | 606 | 607 | # A compiled query, ready for execution. 608 | # 609 | # Attributes: 610 | # c_targets: A list of compiled targets (instancef of EvalTarget). 611 | # c_where: An instance of EvalNode, a compiled expression tree, for postings. 612 | # group_indexes: A list of integers that describe which target indexes to 613 | # group by. All the targets referenced here should be non-aggregates. In fact, 614 | # this list of indexes should always cover all non-aggregates in 'c_targets'. 615 | # And this list may well include some invisible columns if only specified in 616 | # the GROUP BY clause. 617 | # order_spec: A list of (integer indexes, sort order) tuples. 618 | # This list may refer to either aggregates or non-aggregates. 619 | # limit: An optional integer used to cut off the number of result rows returned. 620 | # distinct: An optional boolean that requests we should uniquify the result rows. 621 | @dataclasses.dataclass 622 | class EvalQuery: 623 | table: tables.Table 624 | c_targets: list 625 | c_where: EvalNode 626 | group_indexes: list[int] 627 | having_index: int 628 | order_spec: list[tuple[int, ast.Ordering]] 629 | limit: int 630 | distinct: bool 631 | 632 | @property 633 | def columns(self): 634 | return [t for t in self.c_targets if t.name is not None] 635 | 636 | def __call__(self): 637 | return query_execute.execute_select(self) 638 | 639 | 640 | @dataclasses.dataclass 641 | class EvalPivot: 642 | """Implement PIVOT BY clause.""" 643 | 644 | query: EvalQuery 645 | pivots: list[int] 646 | 647 | def __call__(self): 648 | columns, rows = self.query() 649 | 650 | col1, col2 = self.pivots 651 | othercols = [i for i in range(len(columns)) if i not in self.pivots] 652 | nother = len(othercols) 653 | other = lambda x: tuple(x[i] for i in othercols) 654 | keys = sorted({row[col2] for row in rows}) 655 | 656 | # Compute the new column names and dtypes. 657 | if nother > 1: 658 | it = itertools.product(keys, other(columns)) 659 | names = [f'{columns[col1].name}/{columns[col2].name}'] + [f'{key}/{col.name}' for key, col in it] 660 | else: 661 | names = [f'{columns[col1].name}/{columns[col2].name}'] + [f'{key}' for key in keys] 662 | datatypes = [columns[col1].datatype] + [col.datatype for col in other(columns)] * len(keys) 663 | columns = tuple(cursor.Column(name, datatype) for name, datatype in zip(names, datatypes)) 664 | 665 | # Populate the pivoted table. 666 | pivoted = [] 667 | rows.sort(key=operator.itemgetter(col1)) 668 | for field1, group in itertools.groupby(rows, key=operator.itemgetter(col1)): 669 | outrow = [field1] + [None] * (len(columns) - 1) 670 | for row in group: 671 | index = keys.index(row[col2]) * nother + 1 672 | outrow[index:index+nother] = other(row) 673 | pivoted.append(tuple(outrow)) 674 | 675 | return columns, pivoted 676 | 677 | 678 | @dataclasses.dataclass 679 | class EvalCreateTable: 680 | context: object = dataclasses.field(repr=False) 681 | name: str 682 | columns: list[tuple[str, str]] 683 | using: str 684 | data: EvalQuery | None 685 | create: object = dataclasses.field(repr=False) 686 | 687 | def __call__(self): 688 | table = self.create(self.name, self.columns, self.using) 689 | if self.data is not None: 690 | columns, rows = self.data() 691 | for row in rows: 692 | table.insert(row) 693 | self.context.tables[self.name] = table 694 | return (), [] 695 | 696 | 697 | @dataclasses.dataclass 698 | class EvalInsert: 699 | table: tables.Table 700 | values: list[EvalNode] 701 | 702 | def __call__(self): 703 | values = tuple(value(None) for value in self.values) 704 | self.table.insert(values) 705 | return (), [] 706 | -------------------------------------------------------------------------------- /beanquery/query_env_test.py: -------------------------------------------------------------------------------- 1 | __copyright__ = "Copyright (C) 2014-2016 Martin Blais" 2 | __license__ = "GNU GPLv2" 3 | 4 | import datetime 5 | import unittest 6 | from decimal import Decimal 7 | 8 | from beancount.core.number import D 9 | from beancount.parser import parser 10 | from beancount import loader 11 | from beanquery import query_compile as qc 12 | from beanquery import query_env as qe 13 | from beanquery import query 14 | 15 | 16 | class TestCompileDataTypes(unittest.TestCase): 17 | 18 | def test_compile_Length(self): 19 | c_length = qe.Function('length', [qc.EvalConstant('testing')]) 20 | self.assertEqual(int, c_length.dtype) 21 | 22 | def test_compile_Sum(self): 23 | c_sum = qe.SumInt(None, [qc.EvalConstant(17)]) 24 | self.assertEqual(int, c_sum.dtype) 25 | c_sum = qe.SumDecimal(None, [qc.EvalConstant(D('17.'))]) 26 | self.assertEqual(Decimal, c_sum.dtype) 27 | 28 | def test_compile_Count(self): 29 | c_count = qe.Count(None, [qc.EvalConstant(17)]) 30 | self.assertEqual(int, c_count.dtype) 31 | 32 | def test_compile_First(self): 33 | c_first = qe.First(None, [qc.EvalConstant(17.)]) 34 | self.assertEqual(float, c_first.dtype) 35 | 36 | def test_compile_Last(self): 37 | c_last = qe.Last(None, [qc.EvalConstant(17.)]) 38 | self.assertEqual(float, c_last.dtype) 39 | 40 | 41 | class TestEnv(unittest.TestCase): 42 | 43 | @parser.parse_doc() 44 | def test_GrepN(self, entries, _, options_map): 45 | """ 46 | 2016-11-20 * "prev match in context next" 47 | Assets:Banking 1 USD 48 | """ 49 | rtypes, rrows = query.run_query(entries, options_map, ''' 50 | SELECT GREPN("in", narration, 0) as m 51 | ''') 52 | self.assertEqual([('in',)], rrows) 53 | 54 | rtypes, rrows = query.run_query(entries, options_map, ''' 55 | SELECT GREPN("match (.*) context", narration, 1) as m 56 | ''') 57 | self.assertEqual([('in',)], rrows) 58 | 59 | rtypes, rrows = query.run_query(entries, options_map, ''' 60 | SELECT GREPN("(.*) in (.*)", narration, 2) as m 61 | ''') 62 | self.assertEqual([('context next',)], rrows) 63 | 64 | rtypes, rrows = query.run_query(entries, options_map, ''' 65 | SELECT GREPN("ab(at)hing", "abathing", 1) as m 66 | ''') 67 | self.assertEqual([('at',)], rrows) 68 | 69 | @parser.parse_doc() 70 | def test_Subst(self, entries, _, options_map): 71 | """ 72 | 2016-11-20 * "I love candy" 73 | Assets:Banking -1 USD 74 | 75 | 2016-11-21 * "Buy thing thing" 76 | Assets:Cash -1 USD 77 | """ 78 | rtypes, rrows = query.run_query(entries, options_map, ''' 79 | SELECT SUBST("[Cc]andy", "carrots", narration) as m where date = 2016-11-20 80 | ''') 81 | self.assertEqual([('I love carrots',)], rrows) 82 | 83 | rtypes, rrows = query.run_query(entries, options_map, ''' 84 | SELECT SUBST("thing", "t", narration) as m where date = 2016-11-21 85 | ''') 86 | self.assertEqual([('Buy t t',)], rrows) 87 | 88 | rtypes, rrows = query.run_query(entries, options_map, ''' 89 | SELECT SUBST("random", "t", narration) as m where date = 2016-11-21 90 | ''') 91 | self.assertEqual([('Buy thing thing',)], rrows) 92 | 93 | rtypes, rrows = query.run_query(entries, options_map, ''' 94 | SELECT SUBST("(love)", "\\1 \\1", narration) as m where date = 2016-11-20 95 | ''') 96 | self.assertEqual([('I love love candy',)], rrows) 97 | 98 | rtypes, rrows = query.run_query(entries, options_map, ''' 99 | SELECT SUBST("Assets:.*", "Savings", account) as a, str(sum(position)) as p 100 | ''') 101 | self.assertEqual([('Savings', '(-2 USD)')], rrows) 102 | 103 | @parser.parse_doc() 104 | def test_Date(self, entries, _, options_map): 105 | """ 106 | 2016-11-20 * "ok" 107 | Assets:Banking 1 USD 108 | """ 109 | rtypes, rrows = query.run_query(entries, options_map, 110 | 'SELECT date(2020, 1, 2) as m') 111 | self.assertEqual([(datetime.date(2020, 1, 2),)], rrows) 112 | 113 | rtypes, rrows = query.run_query(entries, options_map, 114 | 'SELECT date(year, month, 1) as m') 115 | self.assertEqual([(datetime.date(2016, 11, 1),)], rrows) 116 | 117 | rtypes, rrows = query.run_query(entries, options_map, 118 | 'SELECT date(2020, 2, 32) as m') 119 | self.assertEqual([(None,)], rrows) 120 | 121 | rtypes, rrows = query.run_query(entries, options_map, 122 | 'SELECT date("2020-01-02") as m') 123 | self.assertEqual([(datetime.date(2020, 1, 2),)], rrows) 124 | 125 | 126 | @parser.parse_doc() 127 | def test_DateDiffAdjust(self, entries, _, options_map): 128 | """ 129 | 2016-11-20 * "ok" 130 | Assets:Banking -1 STOCK { 5 USD, 2016-10-30 } 131 | """ 132 | rtypes, rrows = query.run_query(entries, options_map, 133 | 'SELECT date_diff(date, cost_date) as m') 134 | self.assertEqual([(21,)], rrows) 135 | 136 | rtypes, rrows = query.run_query(entries, options_map, 137 | 'SELECT date_diff(cost_date, date) as m') 138 | self.assertEqual([(-21,)], rrows) 139 | 140 | rtypes, rrows = query.run_query(entries, options_map, 141 | 'SELECT date_add(date, 1) as m') 142 | self.assertEqual([(datetime.date(2016, 11, 21),)], rrows) 143 | 144 | rtypes, rrows = query.run_query(entries, options_map, 145 | 'SELECT date_add(date, -1) as m') 146 | self.assertEqual([(datetime.date(2016, 11, 19),)], rrows) 147 | 148 | def test_func_meta(self): 149 | # use the loader to have the pad transaction inserted 150 | entries, _, options = loader.load_string(''' 151 | 2019-01-01 open Assets:Main USD 152 | 2019-01-01 open Assets:Other USD 153 | 2019-01-14 * "Test" 154 | entry: 1 155 | both: 3 156 | Assets:Main 100.00 USD 157 | post: 2 158 | both: 3 159 | Assets:Other 160 | post: 4 161 | 2019-01-15 pad Assets:Main Assets:Other 162 | 2019-01-16 balance Assets:Main 1000.00 USD 163 | ''', dedent=True) 164 | rtypes, rrows = query.run_query(entries, options, ''' 165 | SELECT 166 | entry_meta('post'), 167 | meta('post'), 168 | any_meta('post'), 169 | entry_meta('entry'), 170 | meta('entry'), 171 | any_meta('entry'), 172 | any_meta('both') 173 | ''') 174 | self.assertEqual([ 175 | (None, D(2), D(2), D(1), None, D(1), D(3)), 176 | (None, D(4), D(4), D(1), None, D(1), D(3)), 177 | # postings from pad directive 178 | (None, None, None, None, None, None, None), 179 | (None, None, None, None, None, None, None), 180 | ], rrows) 181 | 182 | 183 | if __name__ == '__main__': 184 | unittest.main() 185 | -------------------------------------------------------------------------------- /beanquery/query_execute.py: -------------------------------------------------------------------------------- 1 | """Execution of interpreter on data rows. 2 | """ 3 | __copyright__ = "Copyright (C) 2014-2016 Martin Blais" 4 | __license__ = "GNU GPLv2" 5 | 6 | import collections 7 | import itertools 8 | import operator 9 | 10 | from . import compiler 11 | from . import hashable 12 | from . import cursor 13 | 14 | 15 | 16 | class Unique: 17 | def __init__(self, columns): 18 | self.wrap = hashable.make(columns) 19 | 20 | def __call__(self, iterable): 21 | wrap = self.wrap 22 | seen = set() 23 | add = seen.add 24 | for obj in iterable: 25 | h = wrap(obj) 26 | if h not in seen: 27 | add(h) 28 | yield obj 29 | 30 | 31 | class Allocator: 32 | """A helper class to count slot allocations and return unique handles to them. 33 | """ 34 | def __init__(self): 35 | self.size = 0 36 | 37 | def allocate(self): 38 | """Allocate a new slot to store row aggregation information. 39 | 40 | Returns: 41 | A unique handle used to index into an row-aggregation store (an integer). 42 | """ 43 | handle = self.size 44 | self.size += 1 45 | return handle 46 | 47 | def create_store(self): 48 | """Create a new row-aggregation store suitable to contain all the node allocations. 49 | 50 | Returns: 51 | A store that can accommodate and be indexed by all the allocated slot handles. 52 | """ 53 | return [None] * self.size 54 | 55 | 56 | class NullType: 57 | """An object that compares smaller than anything. 58 | 59 | An instance of this class is used to replace None in BQL query 60 | results in sort keys to obtain sorting semantics similar to SQL 61 | where NULL is sortet at the beginning. 62 | 63 | """ 64 | __slots__ = () 65 | 66 | def __repr__(self): 67 | return 'NULL' 68 | 69 | __str__ = __repr__ 70 | 71 | def __lt__(self, other): 72 | # Make sure that instances of this class compare equal. 73 | if isinstance(other, NullType): 74 | return False 75 | return True 76 | 77 | def __gt__(self, other): 78 | # Make sure that instances of this class compare equal. 79 | if isinstance(other, NullType): 80 | return True 81 | return False 82 | 83 | 84 | NULL = NullType() 85 | 86 | 87 | def nullitemgetter(item, *items): 88 | """An itemgetter() that replaces None values with NULL.""" 89 | if items: 90 | items = (item, *items) 91 | def func(obj): 92 | r = [] 93 | for i in items: 94 | value = obj[i] 95 | r.append(value if value is not None else NULL) 96 | return tuple(r) 97 | return func 98 | def func(obj): 99 | value = obj[item] 100 | return value if value is not None else NULL 101 | return func 102 | 103 | 104 | def execute_select(query): 105 | """Given a compiled select statement, execute the query. 106 | 107 | Args: 108 | query: An instance of a query_compile.Query 109 | entries: A list of directives. 110 | options_map: A parser's option_map. 111 | Returns: 112 | A pair of: 113 | result_types: A list of (name, data-type) item pairs. 114 | result_rows: A list of ResultRow tuples of length and types described by 115 | 'result_types'. 116 | """ 117 | # Figure out the result types that describe what we return. 118 | result_types = tuple(cursor.Column(target.name, target.c_expr.dtype) 119 | for target in query.c_targets 120 | if target.name is not None) 121 | 122 | # Pre-compute lists of the expressions to evaluate. 123 | group_indexes = (set(query.group_indexes) 124 | if query.group_indexes is not None 125 | else query.group_indexes) 126 | 127 | # Indexes of the columns for result rows and order rows. 128 | result_indexes = [index 129 | for index, c_target in enumerate(query.c_targets) 130 | if c_target.name] 131 | order_spec = query.order_spec 132 | 133 | # Dispatch between the non-aggregated queries and aggregated queries. 134 | c_where = query.c_where 135 | rows = [] 136 | 137 | # Precompute a list of expressions to be evaluated. 138 | c_target_exprs = [c_target.c_expr for c_target in query.c_targets] 139 | 140 | if query.group_indexes is None: 141 | # This is a non-aggregated query. 142 | 143 | # Iterate over all the postings once. 144 | for context in query.table: 145 | if c_where is None or c_where(context): 146 | values = [c_expr(context) for c_expr in c_target_exprs] 147 | rows.append(values) 148 | 149 | else: 150 | # This is an aggregated query. 151 | 152 | # Precompute lists of non-aggregate and aggregate expressions to 153 | # evaluate. For aggregate targets, we hunt down the aggregate 154 | # sub-expressions to evaluate, to avoid recursion during iteration. 155 | c_nonaggregate_exprs = [] 156 | c_aggregate_exprs = [] 157 | for index, c_expr in enumerate(c_target_exprs): 158 | if index in group_indexes: 159 | c_nonaggregate_exprs.append(c_expr) 160 | else: 161 | _, aggregate_exprs = compiler.get_columns_and_aggregates(c_expr) 162 | c_aggregate_exprs.extend(aggregate_exprs) 163 | # Note: it is possible that there are no aggregates to compute here. You could 164 | # have all columns be non-aggregates and group-by the entire list of columns. 165 | 166 | # Pre-allocate handles in aggregation nodes. 167 | allocator = Allocator() 168 | for c_expr in c_aggregate_exprs: 169 | c_expr.allocate(allocator) 170 | 171 | def create(): 172 | # Create a new row in the aggregates store. 173 | store = allocator.create_store() 174 | for c_expr in c_aggregate_exprs: 175 | c_expr.initialize(store) 176 | return store 177 | 178 | context = None 179 | aggregates = collections.defaultdict(create) 180 | 181 | # Iterate over all the postings to evaluate the aggregates. 182 | for context in query.table: 183 | if c_where is None or c_where(context): 184 | 185 | # Compute the non-aggregate expressions. 186 | key = tuple(c_expr(context) for c_expr in c_nonaggregate_exprs) 187 | 188 | # Get an appropriate store for the unique key of this row. 189 | store = aggregates[key] 190 | 191 | # Update the aggregate expressions. 192 | for c_expr in c_aggregate_exprs: 193 | c_expr.update(store, context) 194 | 195 | # Iterate over all the aggregations. 196 | for key, store in aggregates.items(): 197 | key_iter = iter(key) 198 | values = [] 199 | 200 | # Finalize the store. 201 | for c_expr in c_aggregate_exprs: 202 | c_expr.finalize(store) 203 | 204 | for index, c_expr in enumerate(c_target_exprs): 205 | if index in group_indexes: 206 | value = next(key_iter) 207 | else: 208 | value = c_expr(context) 209 | values.append(value) 210 | 211 | # Skip row if HAVING clause expression is false. 212 | if query.having_index is not None: 213 | if not values[query.having_index]: 214 | continue 215 | 216 | rows.append(values) 217 | 218 | # Apply ORDER BY. 219 | if order_spec is not None: 220 | # Process the order-by clauses grouped by their ordering direction. 221 | for reverse, spec in itertools.groupby(reversed(order_spec), key=operator.itemgetter(1)): 222 | indexes = reversed([i[0] for i in spec]) 223 | # The rows may contain None values: nullitemgetter() 224 | # replaces these with a special value that compares 225 | # smaller than anything else. 226 | rows.sort(key=nullitemgetter(*indexes), reverse=reverse) 227 | 228 | # Extract results set and convert into tuples. 229 | rows = (tuple(row[i] for i in result_indexes) for row in rows) 230 | 231 | # Apply DISTINCT. 232 | if query.distinct: 233 | unique = Unique(result_types) 234 | rows = unique(rows) 235 | 236 | # Apply LIMIT. 237 | if query.limit is not None: 238 | rows = itertools.islice(rows, query.limit) 239 | 240 | return result_types, list(rows) 241 | -------------------------------------------------------------------------------- /beanquery/query_render.py: -------------------------------------------------------------------------------- 1 | """Rendering of rows. 2 | """ 3 | __copyright__ = "Copyright (C) 2014-2016 Martin Blais" 4 | __license__ = "GNU GPLv2" 5 | 6 | import collections 7 | import csv 8 | import datetime 9 | import enum 10 | import typing 11 | 12 | from decimal import Decimal 13 | 14 | from beancount.core import amount 15 | from beancount.core import display_context 16 | from beancount.core import inventory 17 | from beancount.core import position 18 | 19 | 20 | class Align(enum.Enum): 21 | LEFT = 0 22 | RIGHT = 1 23 | 24 | 25 | class RenderContext: 26 | """Hold the query rendering configuration.""" 27 | 28 | def __init__(self, dcontext, expand=False, listsep=', ', spaced=False, null=' '): 29 | self.dcontext = dcontext 30 | self.expand = expand 31 | self.listsep = listsep 32 | self.spaced = spaced 33 | self.null = null 34 | 35 | 36 | # Map of data-types to renderer classes. This is populated by 37 | # subclassing ColumnRenderer via an __init_subclass__ hook. 38 | RENDERERS = {} 39 | 40 | 41 | class ColumnRenderer: 42 | """Base class for classes that render column values. 43 | 44 | The column renderers are responsible to render uniform type values 45 | in a way that will align nicely in a column whereby all the values 46 | render to the same width. 47 | 48 | The formatters are instantiated and are feed all the values in the 49 | column via the ``update()`` method to accumulate the dimensions it 50 | will need to format them later on. The ``prepare()`` method then 51 | computes internal status required to format these values in 52 | consistent fashion. The ``width()`` method can then be used to 53 | retrieve the computer maximum width of the column. Individual 54 | values are formatted with the ``format()`` method. Values are 55 | assumed to be of the expected type for the formatter. Formatting 56 | values outside the set of the values fed via the ``update()`` 57 | method is undefined behavior. 58 | 59 | """ 60 | dtype = None 61 | align = Align.LEFT 62 | 63 | def __init__(self, ctx): 64 | self.maxwidth = 0 65 | self.prepared = False 66 | 67 | def __init_subclass__(cls, **kwargs): 68 | super().__init_subclass__(**kwargs) 69 | RENDERERS[cls.dtype] = cls 70 | 71 | def update(self, value): 72 | """Update the rendered with the given value. 73 | 74 | Args: 75 | value: Any object of type ``dtype``. 76 | 77 | """ 78 | 79 | def prepare(self): 80 | """Prepare to render the column. 81 | 82 | Returns: 83 | Computed column width. 84 | 85 | """ 86 | self.prepared = True 87 | return self.maxwidth 88 | 89 | @property 90 | def width(self): 91 | if not self.prepared: 92 | raise RuntimeError('width property access before calling prepare()') 93 | return self.maxwidth 94 | 95 | def format(self, value): 96 | """Format the value. 97 | 98 | Args: 99 | value: Any object of type ``dtype``. 100 | 101 | Returns: 102 | A string or list of strings representing the rendered value. 103 | 104 | """ 105 | raise NotImplementedError 106 | 107 | 108 | class ObjectRenderer(ColumnRenderer): 109 | dtype = object 110 | 111 | def update(self, value): 112 | self.maxwidth = max(self.maxwidth, len(self.format(value))) 113 | 114 | def format(self, value): 115 | return str(value) 116 | 117 | 118 | class DictRenderer(ObjectRenderer): 119 | dtype = dict 120 | 121 | 122 | class BoolRenderer(ColumnRenderer): 123 | dtype = bool 124 | 125 | def update(self, value): 126 | self.maxwidth = max(self.maxwidth, 4 if value else 5) 127 | 128 | def format(self, value): 129 | return ('TRUE' if value else 'FALSE') 130 | 131 | 132 | class StringRenderer(ObjectRenderer): 133 | dtype = str 134 | 135 | 136 | class SetRenderer(ColumnRenderer): 137 | dtype = set 138 | 139 | def __init__(self, ctx): 140 | super().__init__(ctx) 141 | self.sep = ctx.listsep 142 | 143 | def update(self, value): 144 | self.maxwidth = max(self.maxwidth, sum(len(x) + len(self.sep) for x in value) - len(self.sep)) 145 | 146 | def format(self, value): 147 | return self.sep.join(str(x) for x in sorted(value)) 148 | 149 | 150 | class DateRenderer(ColumnRenderer): 151 | dtype = datetime.date 152 | 153 | def update(self, value): 154 | self.maxwidth = 10 155 | 156 | def format(self, value): 157 | return value.strftime('%Y-%m-%d') 158 | 159 | 160 | class IntRenderer(ObjectRenderer): 161 | dtype = int 162 | align = Align.RIGHT 163 | 164 | 165 | class EnumRenderer(ObjectRenderer): 166 | dtype = enum.Enum 167 | 168 | def format(self, value): 169 | return value.name 170 | 171 | 172 | class DecimalRenderer(ColumnRenderer): 173 | """Renderer for Decimal numbers. 174 | 175 | Numbers are left padded to align on the decimal point:: 176 | 177 | - 123.40 178 | - 5.000 179 | - -67 180 | 181 | """ 182 | dtype = Decimal 183 | 184 | def __init__(self, ctx): 185 | super().__init__(ctx) 186 | # Max number of digits before the decimal point including sign. 187 | self.nintegral = 0 188 | # Max number of digits after the decimal point. 189 | self.nfractional = 0 190 | 191 | def update(self, value): 192 | n = value.as_tuple() 193 | if n.exponent > 0: 194 | # Special case for decimal numbers with positive exponent 195 | # and thus represented in scientific notation. 196 | self.nintegral = max(self.nintegral, len(str(value))) 197 | else: 198 | self.nintegral = max(self.nintegral, max(1, len(n.digits) + n.exponent) + n.sign) 199 | self.nfractional = max(self.nfractional, -n.exponent) 200 | 201 | def prepare(self): 202 | self.maxwidth = self.nintegral + self.nfractional + (1 if self.nfractional > 0 else 0) 203 | return super().prepare() 204 | 205 | def format(self, value): 206 | n = value.as_tuple() 207 | if n.exponent > 0: 208 | # Special case for decimal numbers with positive exponent 209 | # and thus represented in scientific notation. 210 | return str(value).rjust(self.nintegral).ljust(self.maxwidth) 211 | # Compute the padding required to align the decimal point. 212 | left = self.nintegral - (max(1, len(n.digits) + n.exponent) + n.sign) 213 | return f'{"":>{left}}{value:<{self.maxwidth - left}}' 214 | 215 | 216 | class AmountRenderer(ColumnRenderer): 217 | """Renderer for Amount instances. 218 | 219 | The numerical part is formatted with the right quantization 220 | determined by ``dcontext`` in the rendering context and aligned on 221 | the decimal point across rows. Numbers are right padded with 222 | spaces to alignt the commodity symbols across rows:: 223 | 224 | - 1234.00 USD 225 | - 42 TEST 226 | - 0.0001 ETH 227 | - 567.00 USD 228 | 229 | """ 230 | dtype = amount.Amount 231 | 232 | def __init__(self, ctx): 233 | super().__init__(ctx) 234 | # Use the display context inferred from the input ledger to 235 | # determine the quantization of the column values. 236 | self.quantize = ctx.dcontext.quantize 237 | # Use column specific display context for formatting. 238 | self.dcontext = display_context.DisplayContext() 239 | # Maximum width of the commodity symbol. 240 | self.curwidth = 0 241 | 242 | def update(self, value): 243 | # Need to handle None to reuse this in PositionRenderer. 244 | if value is not None: 245 | number = self.quantize(value.number, value.currency) 246 | self.dcontext.update(number, value.currency) 247 | self.curwidth = max(self.curwidth, len(value.currency)) 248 | 249 | def prepare(self): 250 | self.func = self.dcontext.build(display_context.Align.DOT) 251 | zero = Decimal() 252 | for commodity in self.dcontext.ccontexts: 253 | if commodity != '__default__': 254 | self.maxwidth = max(self.maxwidth, len(self.func(zero, commodity)) + 1 + self.curwidth) 255 | return super().prepare() 256 | 257 | def format(self, value): 258 | return f'{self.func(value.number, value.currency)} {value.currency:<{self.curwidth}}' 259 | 260 | 261 | class CostRenderer(ObjectRenderer): 262 | dtype = position.Cost 263 | 264 | def __init__(self, ctx): 265 | super().__init__(ctx) 266 | self.amount_renderer = AmountRenderer(ctx) 267 | self.date_width = 0 268 | self.label_width = 0 269 | 270 | def update(self, value): 271 | self.amount_renderer.update(value) 272 | if value.date is not None: 273 | self.date_width = 10 + 2 274 | if value.label is not None: 275 | self.label_width = max(self.label_width, len(value.label) + 4) 276 | 277 | def prepare(self): 278 | cost_width = self.amount_renderer.prepare() 279 | self.maxwidth = cost_width + self.date_width + self.label_width 280 | return super().prepare() 281 | 282 | def format(self, value): 283 | parts = [self.amount_renderer.format(value)] 284 | if value.date is not None: 285 | parts.append(f'{value.date:%Y-%m-%d}') 286 | if value.label is not None: 287 | parts.append(f'"{value.label}"') 288 | return ', '.join(parts) 289 | 290 | 291 | class PositionRenderer(ColumnRenderer): 292 | """Renderer for Position instrnaces. 293 | 294 | Both the unit numbers and the cost numbers are aligned:: 295 | 296 | - 5.000 HOOL {500.23 USD } 297 | - 123 CA { 1.000 HOOL} 298 | - 3.00 USD 299 | - 42.000 HOOL 300 | - 3.00 AAPL 301 | - 3.0 XY 302 | 303 | """ 304 | dtype = position.Position 305 | 306 | def __init__(self, ctx): 307 | super().__init__(ctx) 308 | self.units_renderer = AmountRenderer(ctx) 309 | self.cost_renderer = AmountRenderer(ctx) 310 | 311 | def update(self, value): 312 | self.units_renderer.update(value.units) 313 | self.cost_renderer.update(value.cost) 314 | 315 | def prepare(self): 316 | units_width = self.units_renderer.prepare() 317 | cost_width = self.cost_renderer.prepare() 318 | self.maxwidth = units_width + cost_width + (3 if cost_width > 0 else 0) 319 | return super().prepare() 320 | 321 | def format(self, value): 322 | units = self.units_renderer.format(value.units) 323 | if value.cost is None: 324 | return units.ljust(self.maxwidth) 325 | cost = self.cost_renderer.format(value.cost) 326 | return f'{units} {{{cost}}}' 327 | 328 | 329 | class InventoryRenderer(ColumnRenderer): 330 | """Renderer for Inventory instances. 331 | 332 | Inventories renders as a list of position strings. The format used 333 | differs whether expansion of list-like values to multiple rows in 334 | enabled or not. 335 | 336 | When row expansion is enabled, positions in each inventory values 337 | are sorted alphabetically by commodity symbol and are formatted 338 | with the same position formatter, resulting in all commodity 339 | strings to be aligned:: 340 | 341 | - 1234.00 USD 342 | 42 TEST 343 | - 0.0001 ETH 344 | 567.00 USD 345 | 346 | When row expansion is disabled, the position formatters are unique 347 | for each commodity symbol and the values are rendered in a table 348 | like structure. The positions appear sorted by frequency of 349 | occurence in the column and alphabetically by commodity symbol:: 350 | 351 | - 1234.00 USD 0.0001 ETH 352 | - 567.00 USD 42 TEST 353 | 354 | The separator between positions is determined by ``listsep`` in 355 | the rendering context. 356 | 357 | """ 358 | dtype = inventory.Inventory 359 | 360 | def __init__(self, ctx): 361 | super().__init__(ctx) 362 | self.listsep = ctx.listsep 363 | # We look this up for each value, it makes sense to cache it 364 | # to avoid the attribute lookup in the context. 365 | self.expand = ctx.expand 366 | # How many times at most we have seen a commodity in an inventory. 367 | self.counts = collections.defaultdict(int) 368 | # Commodity specific renderers. 369 | self.renderers = collections.defaultdict(lambda: PositionRenderer(ctx)) 370 | # How many distinct commodity need to be rendered. 371 | self.distinct = 0 372 | 373 | def update(self, value): 374 | for pos in value.get_positions(): 375 | # We use the little indexing trick to do not have to 376 | # conditionalize this code on whether rows expansion is 377 | # enabled or not. 378 | self.renderers[self.expand or pos.units.currency].update(pos) 379 | counts = collections.Counter(pos.units.currency for pos in value.get_positions()) 380 | for key, val in counts.items(): 381 | self.counts[key] = max(self.counts[key], val) 382 | 383 | def prepare(self): 384 | if self.expand: 385 | self.maxwidth = self.renderers[self.expand].prepare() 386 | else: 387 | for commodity, renderer in self.renderers.items(): 388 | w = renderer.prepare() 389 | self.distinct += self.counts[commodity] 390 | self.maxwidth += self.counts[commodity] * (w + len(self.listsep)) 391 | self.maxwidth -= len(self.listsep) 392 | return super().prepare() 393 | 394 | @staticmethod 395 | def positionsortkey(position): 396 | # Sort positions combining fields in a more intuitive way than the default. 397 | return (position.units.currency, -position.units.number, 398 | (position.cost.currency, -position.cost.number, position.cost.date) if position.cost else ()) 399 | 400 | def format(self, value): 401 | # Expanded row format. 402 | if self.expand: 403 | strings = [] 404 | for pos in sorted(value.get_positions(), key=self.positionsortkey): 405 | strings.append(self.renderers[self.expand].format(pos)) 406 | return strings 407 | # Too many distinct commodities to present in tabular format. 408 | if self.distinct > 5: 409 | strings = [] 410 | for pos in sorted(value.get_positions(), key=self.positionsortkey): 411 | strings.append(self.renderers[pos.units.currency].format(pos)) 412 | return self.listsep.join(strings).ljust(self.maxwidth) 413 | # Tabular format with same commodity positions vertically aligned. 414 | positions = collections.defaultdict(list) 415 | for pos in sorted(value.get_positions(), key=self.positionsortkey): 416 | positions[pos.units.currency].append(pos) 417 | strings = [] 418 | for commodity, renderer in sorted(self.renderers.items()): 419 | strings += [renderer.format(pos) for pos in positions[commodity]] 420 | strings += [''.ljust(renderer.width)] * (self.counts[commodity] - len(positions[commodity])) 421 | return self.listsep.join(strings) 422 | 423 | 424 | def render_rows(rows, renderers, ctx): 425 | """Render results set row.""" 426 | 427 | # Filler for NULL values. 428 | null = ctx.null 429 | 430 | # Spacing row. 431 | spacerow = [''] * len(renderers) 432 | 433 | for row in rows: 434 | 435 | # Render the row cells. Do not pass missing values to the 436 | # renderers but substitute them with the appropriate 437 | # placeholder string. 438 | cells = [render.format(value) if value is not None else null for render, value in zip(renderers, row)] 439 | 440 | if not any(isinstance(cell, list) for cell in cells): 441 | # No multi line cells. Yield the row. 442 | yield cells 443 | 444 | else: 445 | # At least one multi line cell. Ensure that all cells are lists. 446 | cells = [cell if isinstance(cell, list) else [cell] for cell in cells] 447 | 448 | # Compute the maximum number of lines in any cell. 449 | nlines = max(len(cell) for cell in cells) 450 | 451 | # Add placeholder lines to short multi line cells. 452 | for cell in cells: 453 | if len(cell) < nlines: 454 | cell.extend([''] * (nlines - len(cell))) 455 | 456 | # Yield the obtained rows. 457 | yield from zip(*cells) 458 | 459 | # Add spacing row when needed. 460 | if ctx.spaced: 461 | yield spacerow 462 | 463 | 464 | def _get_renderer(datatype, ctx): 465 | datatype = typing.get_origin(datatype) or datatype 466 | for d in datatype.__mro__: # pragma: no branch 467 | renderer = RENDERERS.get(d) 468 | if renderer: 469 | return renderer(ctx) 470 | 471 | 472 | def render_text(columns, rows, dcontext, file, expand=False, boxed=False, 473 | spaced=False, listsep=' ', nullvalue='', narrow=True, unicode=False, **kwargs): 474 | """Render the result of executing a query in text format. 475 | 476 | Args: 477 | columns: A list of beanquery.Column descrining the table columns. 478 | rows: Data to render. 479 | dcontext: A DisplayContext object prepared for rendering numbers. 480 | file: A file object to render the results to. 481 | expand: When true expand columns that render to lists to multiple rows. 482 | boxed: When true draw an ascii-art table borders. 483 | spaced: When true insert an empty line between rows. 484 | listsep: String to use to separate values in list-like column values. 485 | nullvalue: String to use to represent NULL values. 486 | narrow: When true truncate headers to the maximum column values width. 487 | unicode: When true use unicode box drawing characters to draw tables. 488 | 489 | """ 490 | ctx = RenderContext(dcontext, expand=expand, spaced=spaced, listsep=listsep, null=nullvalue) 491 | renderers = [_get_renderer(column.datatype, ctx) for column in columns] 492 | headers = [column.name for column in columns] 493 | alignment = [renderer.align for renderer in renderers] 494 | 495 | # Prime the renderers. 496 | for row in rows: 497 | for value, renderer in zip(row, renderers): 498 | if value is not None: 499 | renderer.update(value) 500 | 501 | # Compute columns widths. 502 | widths = [max(1, narrow or len(header), len(nullvalue), render.prepare()) for header, render in zip(headers, renderers)] 503 | 504 | # Initialize table style. For unicode box drawing characters, 505 | # see https://www.unicode.org/charts/PDF/U2500.pdf 506 | if boxed: 507 | if unicode: 508 | frmt = '\u2502 {} \u2502\n' 509 | colsep = ' \u2502 ' 510 | lines = [''.rjust(width, '\u2500') for width in widths] 511 | top = '\u250C\u2500{}\u2500\u2510\n'.format('\u2500\u252C\u2500'.join(lines)) 512 | hline = '\u251C\u2500{}\u2500\u2524\n'.format('\u2500\u253C\u2500'.join(lines)) 513 | bottom = '\u2514\u2500{}\u2500\u2518\n'.format('\u2500\u2534\u2500'.join(lines)) 514 | else: 515 | frmt = '| {} |\n' 516 | colsep = ' | ' 517 | top = hline = bottom = '+-{}-+\n'.format('-+-'.join(''.rjust(width, '-') for width in widths)) 518 | else: 519 | frmt = '{}\n' 520 | colsep = ' ' 521 | top = bottom = '' 522 | hline = '{}\n'.format(colsep.join(''.rjust(width, '\u2500' if unicode else '-') for width in widths)) 523 | 524 | # Header. 525 | file.write(top) 526 | file.write(frmt.format(colsep.join(header[:width].center(width) for header, width in zip(headers, widths)))) 527 | file.write(hline) 528 | 529 | # Rows. 530 | for row in render_rows(rows, renderers, ctx): 531 | file.write(frmt.format(colsep.join(x.ljust(w) if a == Align.LEFT else x.rjust(w) 532 | for x, w, a in zip(row, widths, alignment)))) 533 | 534 | # Footer. 535 | file.write(bottom) 536 | 537 | 538 | def render_csv(columns, rows, dcontext, file, expand=False, nullvalue='', **kwargs): 539 | """Render the result of executing a query in text format. 540 | 541 | Args: 542 | columns: A list of beanquery.Column describing the table columns. 543 | rows: Data to render. 544 | dcontext: A DisplayContext object prepared for rendering numbers. 545 | file: A file object to render the results to. 546 | expand: A boolean, if true, expand columns that render to lists on multiple rows. 547 | nullvalue: String to use to represent NULL values. 548 | """ 549 | ctx = RenderContext(dcontext, expand=expand, spaced=False, listsep=',', null=nullvalue) 550 | renderers = [_get_renderer(column.datatype, ctx) for column in columns] 551 | headers = [column.name for column in columns] 552 | 553 | # Prime the renderers. 554 | for row in rows: 555 | for value, renderer in zip(row, renderers): 556 | if value is not None: 557 | renderer.update(value) 558 | 559 | # Prepare the renders. 560 | [render.prepare() for render in renderers] 561 | 562 | # Write the CSV file. 563 | writer = csv.writer(file) 564 | writer.writerow(headers) 565 | writer.writerows(render_rows(rows, renderers, ctx)) 566 | -------------------------------------------------------------------------------- /beanquery/query_render_test.py: -------------------------------------------------------------------------------- 1 | __copyright__ = "Copyright (C) 2014-2016 Martin Blais" 2 | __license__ = "GNU GPLv2" 3 | 4 | import datetime 5 | import enum 6 | import io 7 | import textwrap 8 | import unittest 9 | 10 | from decimal import Decimal 11 | 12 | from beancount.core import display_context 13 | from beancount.core.amount import Amount, A 14 | from beancount.core.inventory import Inventory, from_string as I 15 | from beancount.core.number import D 16 | from beancount.core.position import Cost, Position, from_string as P 17 | 18 | from beanquery import query_render 19 | from beanquery.cursor import Column 20 | 21 | 22 | class TestColumnRenderer(unittest.TestCase): 23 | 24 | def test_column_renderer(self): 25 | dcontext = display_context.DisplayContext() 26 | ctx = query_render.RenderContext(dcontext) 27 | renderer = query_render.ColumnRenderer(ctx) 28 | with self.assertRaises(RuntimeError): 29 | w = renderer.width 30 | w = renderer.prepare() 31 | self.assertEqual(w, 0) 32 | self.assertEqual(renderer.width, 0) 33 | with self.assertRaises(NotImplementedError): 34 | renderer.format(None) 35 | 36 | 37 | class RendererTestBase(unittest.TestCase): 38 | 39 | def setUp(self): 40 | self.dcontext = display_context.DisplayContext() 41 | 42 | def render(self, dtype, values, **kwargs): 43 | out = io.StringIO() 44 | rows = [(x, ) for x in values] 45 | query_render.render_text([Column('', dtype)], rows, self.dcontext, out, **kwargs) 46 | return out.getvalue().splitlines()[2:] 47 | 48 | 49 | class Foo(enum.Enum): 50 | SHORT = 1 51 | LONG = 2 52 | 53 | 54 | class TestRenderer(RendererTestBase): 55 | 56 | def test_object(self): 57 | self.assertEqual(self.render(object, ["foo", 1, D('1.23'), datetime.date(1970, 1, 1)]), [ 58 | 'foo ', 59 | '1 ', 60 | '1.23 ', 61 | '1970-01-01', 62 | ]) 63 | 64 | def test_bool(self): 65 | self.assertEqual(self.render(bool, [True, True]), [ 66 | 'TRUE', 67 | 'TRUE', 68 | ]) 69 | self.assertEqual(self.render(bool, [False, True]), [ 70 | 'FALSE', 71 | 'TRUE ', 72 | ]) 73 | self.assertEqual(self.render(bool, [False, False]), [ 74 | 'FALSE', 75 | 'FALSE', 76 | ]) 77 | 78 | def test_str(self): 79 | self.assertEqual(self.render(str, ['a', 'bb', 'ccc', '']), [ 80 | 'a ', 81 | 'bb ', 82 | 'ccc', 83 | ' ', 84 | ]) 85 | 86 | def test_str_null(self): 87 | self.assertEqual(self.render(str, ['', None]), [ 88 | ' ', 89 | ' ', 90 | ]) 91 | self.assertEqual(self.render(str, ['', None], nullvalue='NULL'), [ 92 | ' ', 93 | 'NULL', 94 | ]) 95 | 96 | def test_set_str(self): 97 | self.assertEqual(self.render(set, [{}, {'aaaa'}, {'bb', 'ccc'}]), [ 98 | ' ', 99 | 'aaaa ', 100 | 'bb ccc', 101 | ]) 102 | 103 | def test_date(self): 104 | self.assertEqual(self.render(datetime.date, [datetime.date(2014, 10, 3)]), [ 105 | '2014-10-03' 106 | ]) 107 | 108 | def test_int(self): 109 | self.assertEqual(self.render(int, [1, 22, 333]), [ 110 | ' 1', 111 | ' 22', 112 | '333', 113 | ]) 114 | self.assertEqual(self.render(int, [1, -22, 333]), [ 115 | ' 1', 116 | '-22', 117 | '333', 118 | ]) 119 | self.assertEqual(self.render(int, [1, 22, -333]), [ 120 | ' 1', 121 | ' 22', 122 | '-333', 123 | ]) 124 | 125 | def test_decimal_integral(self): 126 | self.assertEqual(self.render(Decimal, [D('1'), D('12'), D('123'), D('1e4')]), [ 127 | ' 1', 128 | ' 12', 129 | ' 123', 130 | '1E+4', 131 | ]) 132 | self.assertEqual(self.render(Decimal, [D('1'), D('-12'), D('123')]), [ 133 | ' 1', 134 | '-12', 135 | '123', 136 | ]) 137 | self.assertEqual(self.render(Decimal, [D('1'), D('12'), D('-123')]), [ 138 | ' 1', 139 | ' 12', 140 | '-123', 141 | ]) 142 | self.assertEqual(self.render(Decimal, [D('1'), D('12'), D('-1e3')]), [ 143 | ' 1', 144 | ' 12', 145 | '-1E+3', 146 | ]) 147 | 148 | def test_decimal_fractional(self): 149 | self.assertEqual(self.render(Decimal, [D('0.1'), D('1.2'), D('1.23'), D('1.234')]), [ 150 | '0.1 ', 151 | '1.2 ', 152 | '1.23 ', 153 | '1.234', 154 | ]) 155 | self.assertEqual(self.render(Decimal, [D('12'), D('1.2'), D('1.23'), D('12.345')]), [ 156 | '12 ', 157 | ' 1.2 ', 158 | ' 1.23 ', 159 | '12.345', 160 | ]) 161 | self.assertEqual(self.render(Decimal, [D('12'), D('1.2'), D('1.23'), D('-12.345')]), [ 162 | ' 12 ', 163 | ' 1.2 ', 164 | ' 1.23 ', 165 | '-12.345', 166 | ]) 167 | self.assertEqual(self.render(Decimal, [D('12'), D('1.2'), D('1.23'), D('-1.2e3')]), [ 168 | ' 12 ', 169 | ' 1.2 ', 170 | ' 1.23', 171 | '-1.2E+3 ', 172 | ]) 173 | 174 | def test_enum(self): 175 | self.assertEqual(self.render(Foo, [Foo.SHORT, Foo.LONG]), [ 176 | 'SHORT', 177 | 'LONG ', 178 | ]) 179 | 180 | 181 | class TestAmountRenderer(RendererTestBase): 182 | 183 | def render(self, values, **kwargs): 184 | return super().render(Amount, values, **kwargs) 185 | 186 | def test_amount(self): 187 | self.assertEqual( 188 | self.render([A('100.00 USD')]), ['100.00 USD']) 189 | 190 | def test_quantization_one(self): 191 | self.dcontext.update(Decimal('1.0000'), 'ETH') 192 | self.assertEqual(self.dcontext.quantize(Decimal('1.0'), 'ETH'), Decimal('1.0000')) 193 | self.assertEqual(self.render([A('1 ETH')]), ['1.0000 ETH']) 194 | self.assertEqual(self.render([A('0.00001 ETH')]), ['0.0000 ETH']) 195 | 196 | def test_quantization_many(self): 197 | self.dcontext.update(Decimal('1.0000'), 'ETH') 198 | self.dcontext.update(Decimal('1.00'), 'USD') 199 | self.dcontext.update(Decimal('1'), 'XYZ') 200 | self.assertEqual(self.render([A('1.0 ETH')]), ['1.0000 ETH']) 201 | self.assertEqual(self.render([A('1.0 USD')]), ['1.00 USD']) 202 | self.assertEqual(self.render([A('1.0 XYZ')]), ['1 XYZ']) 203 | 204 | def test_number_padding(self): 205 | # FIXME: The leading space seems like a bug in 206 | # DisplayContext. Either it should always be there or it 207 | # should be there to support minus signs not encountered in 208 | # training or it shoudl not be there at all. 209 | self.assertEqual(self.render([A('1 XY'), A('12 XY'), A('123 XY'), A('-1 XY')]), [ 210 | ' 1 XY', 211 | ' 12 XY', 212 | ' 123 XY', 213 | ' -1 XY', 214 | ]) 215 | self.assertEqual(self.render([A('1 XY'), A('12 XY'), A('-12 XY')]), [ 216 | ' 1 XY', 217 | ' 12 XY', 218 | '-12 XY', 219 | ]) 220 | 221 | def test_decimal_alignment(self): 222 | self.assertEqual(self.render([A('1.0 AA'), A('1.00 BB'), A('1.000 CC')]), [ 223 | '1.0 AA', 224 | '1.00 BB', 225 | '1.000 CC', 226 | ]) 227 | 228 | def test_currency_padding(self): 229 | self.assertEqual(self.render([A('1.00 XY'), A('1.00 XYZ'), A('1.00 XYZK')]), [ 230 | '1.00 XY ', 231 | '1.00 XYZ ', 232 | '1.00 XYZK', 233 | ]) 234 | 235 | def test_many(self): 236 | self.assertEqual(self.render([A('0.0001 USD'), A('20.002 HOOL'), A('33 CA'), A('1098.20 AAPL')]), [ 237 | ' 0.0001 USD ', 238 | ' 20.002 HOOL', 239 | ' 33 CA ', 240 | '1098.20 AAPL', 241 | ]) 242 | 243 | 244 | class TestPositionRenderer(RendererTestBase): 245 | 246 | def render(self, values, **kwargs): 247 | return super().render(Position, values, **kwargs) 248 | 249 | def setUp(self): 250 | super().setUp() 251 | # Prime the display context for some known commodities. 252 | self.dcontext = display_context.DisplayContext() 253 | self.dcontext.update(D('1.00'), 'USD') 254 | self.dcontext.update(D('1.00'), 'CAD') 255 | self.dcontext.update(D('1.000'), 'HOOL') 256 | self.dcontext.update(D('1'), 'CA') 257 | self.dcontext.update(D('1.00'), 'AAPL') 258 | 259 | def test_simple_poitions(self): 260 | self.assertEqual(self.render([P('3.0 USD'), P('3.0 CAD'), P('3.0 HOOL'), P('3.0 CA'), P('3.0 AAPL'), P('3.0 XY')]), [ 261 | '3.00 USD ', 262 | '3.00 CAD ', 263 | '3.000 HOOL', 264 | '3 CA ', 265 | '3.00 AAPL', 266 | '3.0 XY ', 267 | ]) 268 | 269 | def test_positions_with_price(self): 270 | self.assertEqual(self.render([P('5 HOOL {500.230000 USD}'), P('123.0 CA {1 HOOL}')]), [ 271 | ' 5.000 HOOL {500.23 USD }', 272 | '123 CA { 1.000 HOOL}', 273 | ]) 274 | 275 | 276 | class TestInventoryRenderer(RendererTestBase): 277 | 278 | def render(self, values, **kwargs): 279 | return super().render(Inventory, values, **kwargs) 280 | 281 | def setUp(self): 282 | super().setUp() 283 | # Prime the display context for some known commodities. 284 | self.dcontext = display_context.DisplayContext() 285 | self.dcontext.update(D('1.00'), 'USD') 286 | self.dcontext.update(D('1.00'), 'CAD') 287 | self.dcontext.update(D('1.000'), 'HOOL') 288 | self.dcontext.update(D('1'), 'CA') 289 | self.dcontext.update(D('1.00'), 'AAPL') 290 | 291 | def test_position_sortkey(self): 292 | inventory = I('1 AAAAA, 5 SHARE {100 USD}, 5 SHARE {200 USD}, 5 TESTS {666 USD}') 293 | self.assertEqual( 294 | sorted(inventory, key=query_render.InventoryRenderer.positionsortkey), [ 295 | P('1 AAAAA'), 296 | P('5 SHARE {200 USD}'), 297 | P('5 SHARE {100 USD}'), 298 | P('5 TESTS {666 USD}'), 299 | ]) 300 | 301 | def test_inventory(self): 302 | self.assertEqual(self.render([I('100 USD')], expand=True),[ 303 | '100.00 USD' 304 | ]) 305 | self.assertEqual(self.render([I('5 HOOL {500.23 USD}')], expand=True), [ 306 | '5.000 HOOL {500.23 USD}' 307 | ]) 308 | self.assertEqual(self.render([I('5 HOOL {500.23 USD}, 12.3456 CAAD')], expand=True), [ 309 | '12.3456 CAAD ', 310 | ' 5.000 HOOL {500.23 USD}', 311 | ]) 312 | 313 | def test_inventory_tabular(self): 314 | self.assertEqual(self.render([I('100 USD')], expand=False, listsep=' & '), [ 315 | '100.00 USD' 316 | ]) 317 | self.assertEqual(self.render([I('5 HOOL {500.23 USD}')], expand=False, listsep=' & '), [ 318 | '5.000 HOOL {500.23 USD}' 319 | ]) 320 | self.assertEqual(self.render([I('5 HOOL {500.23 USD}, 12.3456 CAAD')], expand=False, listsep=' & '), [ 321 | '12.3456 CAAD & 5.000 HOOL {500.23 USD}', 322 | ]) 323 | self.assertEqual(self.render([I('5 HOOL {500.23 USD}, 12.3456 CAAD'), 324 | I('55 HOOL {50.23 USD}, 2.3 CAAD')], expand=False, listsep=' & '), [ 325 | '12.3456 CAAD & 5.000 HOOL {500.23 USD}', 326 | ' 2.3000 CAAD & 55.000 HOOL { 50.23 USD}', 327 | ]) 328 | self.assertEqual(self.render([I('5 HOOL {500.23 USD}, 1 HOOL {567.89 USD}'), 329 | I('55 HOOL {50.23 USD}, 2.3 CAAD')], expand=False, listsep=' & '), [ 330 | ' & 5.000 HOOL {500.23 USD} & 1.000 HOOL {567.89 USD}', 331 | '2.3 CAAD & 55.000 HOOL { 50.23 USD} & ', 332 | ]) 333 | 334 | def test_inventory_too_many(self): 335 | self.assertEqual(self.render([I('10 AA, 2 BB, 3 CC, 4 DD'), 336 | I('5 AA, 6 EE, 7 FF')], expand=False, listsep=' & '), [ 337 | '10 AA & 2 BB & 3 CC & 4 DD ', 338 | ' 5 AA & 6 EE & 7 FF ', 339 | ]) 340 | 341 | 342 | class TestCostRenderer(RendererTestBase): 343 | 344 | def render(self, values, **kwargs): 345 | return super().render(Cost, values, **kwargs) 346 | 347 | def test_cost(self): 348 | self.dcontext.update(Decimal('1.0000'), 'ETH') 349 | self.assertEqual(self.render([Cost(D('1.0'), 'ETH', None, None), 350 | Cost(D('1.0'), 'ETH', datetime.date(2023, 8, 14), None), 351 | Cost(D('1.0'), 'ETH', datetime.date(2023, 8, 14), 'label')]), [ 352 | '1.0000 ETH ', 353 | '1.0000 ETH, 2023-08-14 ', 354 | '1.0000 ETH, 2023-08-14, "label"', 355 | ]) 356 | 357 | 358 | class TestQueryRenderText(unittest.TestCase): 359 | 360 | def setUp(self): 361 | self.dcontext = display_context.DisplayContext() 362 | 363 | def render(self, types, rows, **kwargs): 364 | types = [Column(*t) for t in types] 365 | oss = io.StringIO() 366 | query_render.render_text(types, rows, self.dcontext, oss, **kwargs) 367 | return oss.getvalue() 368 | 369 | def test_render_simple(self): 370 | self.assertEqual(self.render( 371 | [('x', int), ('y', int), ('z', int)], 372 | [(1, 2, 3), (4, 5, 6)]), textwrap.dedent( 373 | """\ 374 | x y z 375 | - - - 376 | 1 2 3 377 | 4 5 6 378 | """)) 379 | 380 | def test_render_simple_unicode(self): 381 | self.assertEqual(self.render( 382 | [('x', int), ('y', int), ('z', int)], 383 | [(1, 2, 3), (4, 5, 6)], unicode=True), textwrap.dedent( 384 | """\ 385 | x y z 386 | ─ ─ ─ 387 | 1 2 3 388 | 4 5 6 389 | """)) 390 | 391 | def test_render_boxed(self): 392 | self.assertEqual(self.render( 393 | [('x', int), ('y', int), ('z', int)], 394 | [(1, 2, 3), (4, 5, 6)], boxed=True), textwrap.dedent( 395 | """\ 396 | +---+---+---+ 397 | | x | y | z | 398 | +---+---+---+ 399 | | 1 | 2 | 3 | 400 | | 4 | 5 | 6 | 401 | +---+---+---+ 402 | """)) 403 | 404 | def test_render_boxed_unicode(self): 405 | self.assertEqual(self.render( 406 | [('x', int), ('y', int), ('z', int)], 407 | [(1, 2, 3), (4, 5, 6)], boxed=True, unicode=True), textwrap.dedent( 408 | """\ 409 | ┌───┬───┬───┐ 410 | │ x │ y │ z │ 411 | ├───┼───┼───┤ 412 | │ 1 │ 2 │ 3 │ 413 | │ 4 │ 5 │ 6 │ 414 | └───┴───┴───┘ 415 | """)) 416 | 417 | def test_render_header_centering(self): 418 | self.assertEqual(self.render( 419 | [('x', int), ('y', int), ('z', int)], 420 | [(1, 22222, 3), (4, 5, 6)]), textwrap.dedent( 421 | """\ 422 | x y z 423 | - ----- - 424 | 1 22222 3 425 | 4 5 6 426 | """)) 427 | 428 | def test_render_header_truncation(self): 429 | self.assertEqual(self.render( 430 | [('x', int), ('abcdefg', int), ('z', int)], 431 | [(1, 222, 3), (4, 5, 6)]), textwrap.dedent( 432 | """\ 433 | x abc z 434 | - --- - 435 | 1 222 3 436 | 4 5 6 437 | """)) 438 | 439 | def test_render_missing_values(self): 440 | self.assertEqual(self.render( 441 | [('xx', int), ('yy', int), ('zz', int)], 442 | [(12, None, 34), (None, 56, 78)]), textwrap.dedent( 443 | """\ 444 | xx yy zz 445 | -- -- -- 446 | 12 34 447 | 56 78 448 | """)) 449 | 450 | def test_render_missing_values_boxed(self): 451 | self.assertEqual(self.render( 452 | [('xx', int), ('yy', int), ('zz', int)], 453 | [(12, None, 34), (None, 56, 78)], boxed=True), textwrap.dedent( 454 | """\ 455 | +----+----+----+ 456 | | xx | yy | zz | 457 | +----+----+----+ 458 | | 12 | | 34 | 459 | | | 56 | 78 | 460 | +----+----+----+ 461 | """)) 462 | 463 | def test_render_expand(self): 464 | self.assertEqual(self.render( 465 | [('x', int), ('inv', Inventory), ('q', int)], 466 | [(11, I('1.00 USD, 1.00 EUR, 1 TESTS'), 2), 467 | (33, I('2.00 EUR, 42 TESTS'), 4)], expand=True, boxed=True), textwrap.dedent( 468 | """\ 469 | +----+-------------+---+ 470 | | x | inv | q | 471 | +----+-------------+---+ 472 | | 11 | 1.00 EUR | 2 | 473 | | | 1 TESTS | | 474 | | | 1.00 USD | | 475 | | 33 | 2.00 EUR | 4 | 476 | | | 42 TESTS | | 477 | +----+-------------+---+ 478 | """)) 479 | 480 | def test_render_spaced(self): 481 | self.assertEqual(self.render( 482 | [('x', int)], 483 | [(1, ), (22, ), (333, )], spaced=True, boxed=True), textwrap.dedent( 484 | """\ 485 | +-----+ 486 | | x | 487 | +-----+ 488 | | 1 | 489 | | | 490 | | 22 | 491 | | | 492 | | 333 | 493 | | | 494 | +-----+ 495 | """)) 496 | 497 | 498 | class TestQueryRenderCSV(unittest.TestCase): 499 | 500 | def setUp(self): 501 | self.dcontext = display_context.DisplayContext() 502 | 503 | def render(self, types, rows, **kwargs): 504 | oss = io.StringIO() 505 | query_render.render_csv(types, rows, self.dcontext, oss, **kwargs) 506 | # The csv modules emits DOS-style newlines. 507 | return oss.getvalue().replace('\r\n', '\n') 508 | 509 | def test_render_simple(self): 510 | self.assertEqual(self.render( 511 | [Column('x', int), Column('y', int), Column('z', int)], 512 | [(1, 2, 3), (4, 5, 6)]), textwrap.dedent( 513 | """\ 514 | x,y,z 515 | 1,2,3 516 | 4,5,6 517 | """)) 518 | 519 | def test_render_missing(self): 520 | self.assertEqual(self.render( 521 | [Column('x', int), Column('y', int), Column('z', int)], 522 | [(None, 2, 3), (4, None, 6)]), textwrap.dedent( 523 | """\ 524 | x,y,z 525 | ,2,3 526 | 4,,6 527 | """)) 528 | 529 | def test_render_expand(self): 530 | self.assertEqual(self.render( 531 | [Column('x', int), Column('inv', Inventory), Column('q', int)], 532 | [(11, I('1.00 USD, 1.00 EUR, 1 TESTS'), 2), 533 | (33, I('2.00 EUR, 42 TESTS'), 4)], expand=True), "\n".join([ 534 | "x,inv,q", 535 | "11, 1.00 EUR ,2", 536 | ", 1 TESTS,", 537 | ", 1.00 USD ,", 538 | "33, 2.00 EUR ,4", 539 | ",42 TESTS,", 540 | "", 541 | ])) 542 | -------------------------------------------------------------------------------- /beanquery/query_test.py: -------------------------------------------------------------------------------- 1 | __copyright__ = "Copyright (C) 2015-2017 Martin Blais" 2 | __license__ = "GNU GPLv2" 3 | 4 | import unittest 5 | 6 | from beancount import loader 7 | from beanquery import query 8 | 9 | 10 | class TestSimple(unittest.TestCase): 11 | 12 | @loader.load_doc() 13 | def test_run_query(self, entries, _, options): 14 | """ 15 | 2022-01-01 open Assets:Checking USD 16 | 2022-01-01 open Income:ACME USD 17 | 2022-01-01 open Expenses:Taxes:Federal USD 18 | 2022-01-01 open Assets:Federal:401k IRAUSD 19 | 2022-01-01 open Expenses:Taxes:401k IRAUSD 20 | 2022-01-01 open Assets:Vacation VACHR 21 | 2022-01-01 open Income:Vacation VACHR 22 | 2022-01-01 open Expenses:Vacation VACHR 23 | 2022-01-01 open Expenses:Tests USD 24 | 25 | 2022-01-01 * "ACME" "Salary" 26 | Assets:Checking 10.00 USD 27 | Income:ACME -11.00 USD 28 | Expenses:Taxes:Federal 1.00 USD 29 | Assets:Federal:401k -2.00 IRAUSD 30 | Expenses:Taxes:401k 2.00 IRAUSD 31 | Assets:Vacation 5 VACHR 32 | Income:Vacation -5 VACHR 33 | 34 | 2022-01-02 * "Holidays" 35 | Assets:Vacation -1 VACHR 36 | Expenses:Vacation 37 | 38 | 2022-01-03 * "Test" 39 | Assets:Checking 3.00 USD 40 | Expenses:Tests 41 | 42 | """ 43 | 44 | sql_query = r""" 45 | SELECT 46 | account, 47 | sum(position) AS amount 48 | WHERE root(account, 1) = '{0}' 49 | GROUP BY 1 50 | ORDER BY 2 DESC 51 | """ 52 | 53 | rtypes, rrows = query.run_query(entries, options, sql_query, 'Expenses', numberify=True) 54 | columns = [c.name for c in rtypes] 55 | self.assertEqual(columns, ['account', 'amount (USD)', 'amount (VACHR)', 'amount (IRAUSD)']) 56 | self.assertEqual(len(rrows[0]), 4) 57 | 58 | 59 | if __name__ == '__main__': 60 | unittest.main() 61 | -------------------------------------------------------------------------------- /beanquery/render/beancount.py: -------------------------------------------------------------------------------- 1 | from beancount.core import display_context 2 | from beancount.parser import printer 3 | 4 | 5 | def render(desc, rows, file, *, dcontext, **kwargs): 6 | # Create a display context that renders all numbers with their 7 | # natural precision, but honors the commas option in the ledger. 8 | commas = dcontext.commas 9 | dcontext = display_context.DisplayContext() 10 | dcontext.set_commas(commas) 11 | return printer.print_entries([entry for entry, in rows], dcontext, file=file) 12 | -------------------------------------------------------------------------------- /beanquery/render/csv.py: -------------------------------------------------------------------------------- 1 | from ..query_render import render_csv 2 | 3 | 4 | def render(desc, rows, file, *, dcontext, **kwargs): 5 | return render_csv(desc, rows, dcontext, file, **kwargs) 6 | -------------------------------------------------------------------------------- /beanquery/render/text.py: -------------------------------------------------------------------------------- 1 | from ..query_render import render_text 2 | 3 | 4 | def render(desc, rows, file, *, dcontext, **kwargs): 5 | if not rows: 6 | return 7 | return render_text(desc, rows, dcontext, file, **kwargs) 8 | -------------------------------------------------------------------------------- /beanquery/shell_test.py: -------------------------------------------------------------------------------- 1 | __copyright__ = "Copyright (C) 2014-2016 Martin Blais" 2 | __license__ = "GNU GPLv2" 3 | 4 | import functools 5 | import re 6 | import sys 7 | import textwrap 8 | import unittest 9 | 10 | import click.testing 11 | 12 | from beancount import loader 13 | from beancount.utils import test_utils 14 | 15 | from beanquery import shell 16 | 17 | 18 | @functools.lru_cache(None) 19 | def load(): 20 | entries, errors, options = loader.load_string(textwrap.dedent(""" 21 | 2022-01-01 open Assets:Checking USD 22 | 2022-01-01 open Assets:Federal:401k IRAUSD 23 | 2022-01-01 open Assets:Gold GLD 24 | 2022-01-01 open Assets:Vacation VACHR 25 | 2022-01-01 open Assets:Vanguard:RGAGX RGAGX 26 | 2022-01-01 open Expenses:Commissions USD 27 | 2022-01-01 open Expenses:Food USD 28 | 2022-01-01 open Expenses:Home:Rent USD 29 | 2022-01-01 open Expenses:Taxes:401k IRAUSD 30 | 2022-01-01 open Expenses:Taxes:Federal USD 31 | 2022-01-01 open Expenses:Tests USD 32 | 2022-01-01 open Expenses:Vacation VACHR 33 | 2022-01-01 open Income:ACME USD 34 | 2022-01-01 open Income:Gains USD 35 | 2022-01-01 open Income:Vacation VACHR 36 | 37 | 2022-01-01 * "ACME" "Salary" 38 | Assets:Checking 10.00 USD 39 | Income:ACME -11.00 USD 40 | Expenses:Taxes:Federal 1.00 USD 41 | Assets:Federal:401k -2.00 IRAUSD 42 | Expenses:Taxes:401k 2.00 IRAUSD 43 | Assets:Vacation 5 VACHR 44 | Income:Vacation -5 VACHR 45 | 46 | 2022-01-01 * "Rent" 47 | Assets:Checking 42.00 USD 48 | Expenses:Home:Rent 42.00 USD 49 | 50 | 2022-01-02 * "Holidays" 51 | Assets:Vacation -1 VACHR 52 | Expenses:Vacation 53 | 54 | 2022-01-03 * "Test 01" 55 | Assets:Checking 1.00 USD 56 | Expenses:Tests 57 | 58 | 2022-01-04 * "My Fovorite Plase" "Eating out alone" 59 | Assets:Checking 4.00 USD 60 | Expenses:Food 61 | 62 | 2022-01-05 * "Invest" 63 | Assets:Checking -359.94 USD 64 | Assets:Vanguard:RGAGX 2.086 RGAGX {172.55 USD} 65 | 66 | 2013-10-23 * "Buy Gold" 67 | Assets:Checking -1278.67 USD 68 | Assets:Gold 9 GLD {141.08 USD} 69 | Expenses:Commissions 8.95 USD 70 | 71 | 2022-01-07 * "Sell Gold" 72 | Assets:Gold -16 GLD {147.01 USD} @ 135.50 USD 73 | Assets:Checking 2159.05 USD 74 | Expenses:Commissions 8.95 USD 75 | Income:Gains 184.16 USD 76 | 77 | 2022-01-08 * "Sell Gold" 78 | Assets:Gold -16 GLD {147.01 USD} @ 135.50 USD 79 | Assets:Checking 2159.05 USD 80 | Expenses:Commissions 8.95 USD 81 | Income:Gains 184.16 USD 82 | 83 | 2022-02-01 * "ACME" "Salary" 84 | Assets:Checking 10.00 USD 85 | Income:ACME -11.00 USD 86 | Expenses:Taxes:Federal 1.00 USD 87 | Assets:Federal:401k -2.00 IRAUSD 88 | Expenses:Taxes:401k 2.00 IRAUSD 89 | Assets:Vacation 5 VACHR 90 | Income:Vacation -5 VACHR 91 | 92 | 2022-02-01 * "Rent" 93 | Assets:Checking 43.00 USD 94 | Expenses:Home:Rent 43.00 USD 95 | 96 | 2022-02-02 * "Test 02" 97 | Assets:Checking 2.00 USD 98 | Expenses:Tests 99 | 100 | 2030-01-01 query "taxes" " 101 | SELECT 102 | date, description, position, balance 103 | WHERE 104 | account ~ 'Taxes' 105 | ORDER BY date DESC 106 | LIMIT 20" 107 | 108 | 2015-01-01 query "home" " 109 | SELECT 110 | last(date) as latest, 111 | account, 112 | sum(position) as total 113 | WHERE 114 | account ~ ':Home:' 115 | GROUP BY account" 116 | 117 | """)) 118 | return entries, errors, options 119 | 120 | 121 | def run_shell_command(cmd): 122 | """Run a shell command and return its output.""" 123 | with test_utils.capture('stdout') as stdout, test_utils.capture('stderr') as stderr: 124 | shell_obj = shell.BQLShell('', sys.stdout) 125 | entries, errors, options = load() 126 | shell_obj.context.attach('beancount:', entries=entries, errors=errors, options=options) 127 | shell_obj._extract_queries(entries) # pylint: disable=protected-access 128 | shell_obj.onecmd(cmd) 129 | return stdout.getvalue(), stderr.getvalue() 130 | 131 | 132 | def runshell(function): 133 | """Decorate a function to run the shell and return the output.""" 134 | def wrapper(self): 135 | out, err = run_shell_command(function.__doc__) 136 | return function(self, out, err) 137 | return wrapper 138 | 139 | 140 | class TestUseCases(unittest.TestCase): 141 | """Testing all the use cases from the proposal here. 142 | I'm hoping to replace reports by these queries instead.""" 143 | 144 | @runshell 145 | def test_print_from(self, out, err): 146 | """ 147 | PRINT FROM narration ~ 'alone' 148 | """ 149 | self.assertRegex(out, 'Eating out alone') 150 | 151 | @runshell 152 | def test_accounts(self, out, err): 153 | """ 154 | SELECT DISTINCT account, open_date(account) 155 | ORDER BY account_sortkey(account); 156 | """ 157 | self.assertRegex(out, 'Assets:Checking *2022-01-01') 158 | self.assertRegex(out, 'Income:ACME *2022-01-01') 159 | 160 | @runshell 161 | def test_commodities(self, out, err): 162 | """ 163 | SELECT DISTINCT currency ORDER BY 1; 164 | """ 165 | self.assertRegex(out, 'USD') 166 | self.assertRegex(out, 'IRAUSD') 167 | self.assertRegex(out, 'VACHR') 168 | 169 | @runshell 170 | def test_commodities_cost(self, out, err): 171 | """ 172 | SELECT DISTINCT cost_currency ORDER BY 1; 173 | """ 174 | self.assertRegex(out, 'USD') 175 | 176 | @runshell 177 | def test_commodities_pairs(self, out, err): 178 | """ 179 | SELECT DISTINCT currency, cost_currency ORDER BY 1, 2; 180 | """ 181 | self.assertRegex(out, 'GLD *USD') 182 | 183 | @runshell 184 | def test_balances(self, out, err): 185 | """ 186 | BALANCES AT cost; 187 | """ 188 | self.assertRegex(out, r'Assets:Gold *\d+\.\d+ USD') 189 | 190 | @runshell 191 | def test_balances_with_where(self, out, err): 192 | """ 193 | JOURNAL 'Assets:Checking'; 194 | """ 195 | self.assertRegex(out, 'Salary') 196 | 197 | @runshell 198 | def test_balance_sheet(self, out, err): 199 | """ 200 | BALANCES AT cost 201 | FROM OPEN ON 2022-01-02 CLOSE ON 2022-02-01 CLEAR; 202 | """ 203 | self.assertRegex(out, r'Assets:Gold * \d+\.\d+ USD') 204 | 205 | @runshell 206 | def test_income_statement(self, out, err): 207 | """ 208 | SELECT account, cost(sum(position)) 209 | FROM OPEN ON 2022-01-01 CLOSE ON 2023-01-01 210 | WHERE account ~ '(Income|Expenses):*' 211 | GROUP BY account, account_sortkey(account) 212 | ORDER BY account_sortkey(account); 213 | """ 214 | self.assertRegex( 215 | out, 'Expenses:Taxes:401k *4.00 IRAUSD') 216 | 217 | @runshell 218 | def test_journal(self, out, err): 219 | """ 220 | JOURNAL 'Assets:Checking' 221 | FROM OPEN ON 2022-02-01 CLOSE ON 2022-03-01; 222 | """ 223 | self.assertRegex(out, "2022-01-31 +S +Opening balance for 'Assets:Checking'") 224 | self.assertRegex(out, "Test 02") 225 | 226 | @runshell 227 | def test_conversions(self, out, err): 228 | """ 229 | SELECT date, payee, narration, position, balance 230 | FROM OPEN ON 2022-01-01 CLOSE ON 2023-01-01 231 | WHERE flag = 'C' 232 | """ 233 | self.assertRegex(out, "2022-12-31 *Conversion for") 234 | 235 | @runshell 236 | def test_documents(self, out, err): 237 | """ 238 | SELECT date, account, narration 239 | WHERE type = 'Document'; 240 | """ 241 | ## FIXME: Make this possible, we need an example with document entries. 242 | 243 | @runshell 244 | def test_holdings(self, out, err): 245 | """ 246 | SELECT account, currency, cost_currency, sum(position) 247 | GROUP BY account, currency, cost_currency; 248 | """ 249 | ## FIXME: Here we need to finally support FLATTEN to make this happen properly. 250 | 251 | 252 | class TestRun(unittest.TestCase): 253 | 254 | @runshell 255 | def test_run_custom__list(self, out, err): 256 | """ 257 | .run 258 | """ 259 | self.assertEqual("home taxes", 260 | re.sub(r'[] \n\t]+', ' ', out).strip()) 261 | 262 | @runshell 263 | def test_run_custom__query_not_exists(self, out, err): 264 | """ 265 | .run something 266 | """ 267 | self.assertEqual('error: query "something" not found', err.strip()) 268 | 269 | @runshell 270 | def test_run_custom__query_id(self, out, err): 271 | """ 272 | .run taxes 273 | """ 274 | self.assertRegex(out, 'date +description +position +balance') 275 | self.assertRegex(out, r'ACME \| Salary') 276 | 277 | @runshell 278 | def test_run_custom__query_string(self, out, err): 279 | """ 280 | RUN "taxes" 281 | """ 282 | self.assertRegex(out, 'date +description +position +balance') 283 | self.assertRegex(out, r'ACME \| Salary') 284 | 285 | @runshell 286 | def test_run_custom__all(self, out, err): 287 | """ 288 | RUN * 289 | """ 290 | self.assertRegex(out, 'date +description +position +balance') 291 | self.assertRegex(out, r'ACME \| Salary') 292 | self.assertRegex(out, 'account +total') 293 | self.assertRegex(out, 'Expenses:Home:Rent') 294 | 295 | 296 | class TestCommands(unittest.TestCase): 297 | 298 | @runshell 299 | def test_tables(self, out, err): 300 | """ 301 | .tables 302 | """ 303 | self.assertEqual(out, textwrap.dedent('''\ 304 | accounts 305 | balances 306 | commodities 307 | documents 308 | entries 309 | events 310 | notes 311 | postings 312 | prices 313 | transactions 314 | ''')) 315 | 316 | 317 | class TestHelp(unittest.TestCase): 318 | 319 | def test_help_functions(self): 320 | for name in dir(shell.BQLShell): 321 | if name.startswith('help_'): 322 | run_shell_command('help ' + name[5:]) 323 | 324 | 325 | class ClickTestCase(unittest.TestCase): 326 | """Base class for command-line program test cases.""" 327 | 328 | def main(self, *args): 329 | init_filename = shell.INIT_FILENAME 330 | history_filename = shell.HISTORY_FILENAME 331 | try: 332 | shell.INIT_FILENAME = '' 333 | shell.HISTORY_FILENAME = '' 334 | runner = click.testing.CliRunner() 335 | result = runner.invoke(shell.main, args, catch_exceptions=False) 336 | self.assertEqual(result.exit_code, 0) 337 | return result 338 | finally: 339 | shell.INIT_FILENAME = init_filename 340 | shell.HISTORY_FILENAME = history_filename 341 | 342 | 343 | class TestShell(ClickTestCase): 344 | 345 | @test_utils.docfile 346 | def test_success(self, filename): 347 | """ 348 | 2013-01-01 open Assets:Account1 349 | 2013-01-01 open Assets:Account2 350 | 2013-01-01 open Assets:Account3 351 | 2013-01-01 open Equity:Unknown 352 | 353 | 2013-04-05 * 354 | Equity:Unknown 355 | Assets:Account1 5000 USD 356 | 357 | 2013-04-05 * 358 | Assets:Account1 -3000 USD 359 | Assets:Account2 30 BOOG {100 USD} 360 | 361 | 2013-04-05 * 362 | Assets:Account1 -1000 USD 363 | Assets:Account3 800 EUR @ 1.25 USD 364 | """ 365 | result = self.main(filename, "SELECT 1;") 366 | self.assertTrue(result.stdout) 367 | 368 | @test_utils.docfile 369 | def test_format_csv(self, filename): 370 | """ 371 | """ 372 | r = self.main(filename, '--format=csv', "SELECT 111 AS one, 222 AS two FROM #") 373 | self.assertEqual(r.stdout, textwrap.dedent('''\ 374 | one,two 375 | 111,222 376 | ''')) 377 | 378 | @test_utils.docfile 379 | def test_format_text(self, filename): 380 | """ 381 | """ 382 | r = self.main(filename, '--format=text', "SELECT 111 AS one, 222 AS two FROM #") 383 | self.assertEqual(r.stdout, textwrap.dedent('''\ 384 | one two 385 | --- --- 386 | 111 222 387 | ''')) 388 | 389 | 390 | if __name__ == '__main__': 391 | unittest.main() 392 | -------------------------------------------------------------------------------- /beanquery/sources/beancount.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import datetime 3 | import sys 4 | import types as _types 5 | import typing 6 | 7 | from decimal import Decimal 8 | from functools import lru_cache as cache 9 | from urllib.parse import urlparse 10 | 11 | from beancount import loader 12 | from beancount import parser 13 | from beancount.core import amount 14 | from beancount.core import convert 15 | from beancount.core import data 16 | from beancount.core import getters 17 | from beancount.core import inventory 18 | from beancount.core import position 19 | from beancount.core import prices 20 | from beancount.core.compare import hash_entry 21 | from beancount.core.getters import get_account_open_close, get_commodity_directives 22 | from beancount.ops import summarize 23 | 24 | from beanquery import tables 25 | from beanquery import types 26 | from beanquery import query_compile 27 | from beanquery import query_render 28 | from beanquery import hashable 29 | from beanquery.query_env import ColumnsRegistry 30 | 31 | 32 | if sys.version_info >= (3, 10): 33 | _UNIONTYPES = {typing.Union, _types.UnionType} 34 | else: 35 | _UNIONTYPES = {typing.Union} 36 | 37 | 38 | hashable.register(data.Transaction, lambda t: (t.date, t.flag, t.narration)) 39 | 40 | 41 | _TABLES = [] 42 | 43 | 44 | def attach(context, dsn, *, entries=None, errors=None, options=None): 45 | filename = urlparse(dsn).path 46 | if filename: 47 | entries, errors, options = loader.load_file(filename) 48 | for table in _TABLES: 49 | context.tables[table.name] = table(entries, options) 50 | context.options.update(options) 51 | context.errors.extend(errors) 52 | # Set the default table. This eventually will have to be removed. 53 | context.tables[None] = context.tables['postings'] 54 | 55 | 56 | class Metadata(dict): 57 | pass 58 | 59 | 60 | class MetadataRenderer(query_render.ObjectRenderer): 61 | dtype = Metadata 62 | 63 | def format(self, value): 64 | return str({k: v for k, v in value.items() if k not in {'filename', 'lineno'} and not k.startswith('__')}) 65 | 66 | 67 | class GetAttrColumn(query_compile.EvalColumn): 68 | def __init__(self, name, dtype): 69 | super().__init__(dtype) 70 | self.name = name 71 | 72 | def __call__(self, context): 73 | return getattr(context, self.name) 74 | 75 | 76 | def _simplify_typing_annotation(dtype): 77 | if typing.get_origin(dtype) in _UNIONTYPES: 78 | args = typing.get_args(dtype) 79 | if len(args) == 2: 80 | if args[0] is type(None): 81 | return args[1], True 82 | if args[1] is type(None): 83 | return args[0], True 84 | raise NotImplementedError 85 | return typing.get_origin(dtype) or dtype, False 86 | 87 | 88 | def _typed_namedtuple_to_columns(cls, renames=None): 89 | columns = {} 90 | for name, dtype in typing.get_type_hints(cls).items(): 91 | dtype, nullable = _simplify_typing_annotation(dtype) 92 | if name == 'meta' and dtype is dict: 93 | dtype = Metadata 94 | if dtype is frozenset: 95 | dtype = set 96 | colname = renames.get(name, name) if renames is not None else name 97 | columns[colname] = GetAttrColumn(name, dtype) 98 | return columns 99 | 100 | 101 | class Position(types.Structure): 102 | name = 'position' 103 | columns = _typed_namedtuple_to_columns(position.Position) 104 | 105 | 106 | class Cost(types.Structure): 107 | name = 'cost' 108 | columns = _typed_namedtuple_to_columns(data.Cost) 109 | 110 | 111 | class Amount(types.Structure): 112 | name = 'amount' 113 | columns = _typed_namedtuple_to_columns(data.Amount) 114 | 115 | 116 | class Transaction(types.Structure): 117 | name = 'transaction' 118 | columns = ColumnsRegistry(_typed_namedtuple_to_columns(data.Transaction)) 119 | del columns['postings'] 120 | 121 | @columns.register(typing.Set[str]) 122 | def accounts(row): 123 | return {p.account for p in row.postings} 124 | 125 | 126 | types.ALIASES[position.Position] = Position 127 | types.ALIASES[data.Cost] = Cost 128 | types.ALIASES[data.Amount] = Amount 129 | types.ALIASES[data.Transaction] = Transaction 130 | 131 | 132 | class Open(types.Structure): 133 | name = 'open' 134 | columns = _typed_namedtuple_to_columns(data.Open) 135 | 136 | 137 | class Close(types.Structure): 138 | name = 'close' 139 | columns = _typed_namedtuple_to_columns(data.Close) 140 | 141 | 142 | class Table(tables.Table): 143 | datatype = None 144 | 145 | def __init__(self, entries, options): 146 | self.entries = entries 147 | self.options = options 148 | 149 | def __init_subclass__(cls): 150 | _TABLES.append(cls) 151 | 152 | def __iter__(self): 153 | datatype = self.datatype 154 | for entry in self.entries: 155 | if isinstance(entry, datatype): 156 | yield entry 157 | 158 | @property 159 | def wildcard_columns(self): 160 | return tuple(col for col in self.columns.keys() if col != 'meta') 161 | 162 | 163 | class TransactionsTable(Table): 164 | name = 'transactions' 165 | datatype = data.Transaction 166 | # There is not a way to inherit the __init_subclass__() and 167 | # __iter__() methods while inheriting the columns attribute 168 | # definition from the Transaction class. 169 | columns = Transaction.columns 170 | 171 | 172 | class PricesTable(Table): 173 | name = 'prices' 174 | datatype = data.Price 175 | columns = _typed_namedtuple_to_columns(datatype) 176 | 177 | def __init__(self, entries, options): 178 | super().__init__(entries, options) 179 | self.price_map = prices.build_price_map(entries) 180 | 181 | 182 | class BalancesTable(Table): 183 | name = 'balances' 184 | datatype = data.Balance 185 | columns = _typed_namedtuple_to_columns(datatype, {'diff_amount': 'discrepancy'}) 186 | 187 | 188 | class NotesTable(Table): 189 | name = 'notes' 190 | datatype = data.Note 191 | columns = _typed_namedtuple_to_columns(datatype) 192 | 193 | 194 | class EventsTable(Table): 195 | name = 'events' 196 | datatype = data.Event 197 | columns = _typed_namedtuple_to_columns(datatype) 198 | 199 | 200 | class DocumentsTable(Table): 201 | name = 'documents' 202 | datatype = data.Document 203 | columns = _typed_namedtuple_to_columns(datatype) 204 | 205 | 206 | class GetItemColumn(query_compile.EvalColumn): 207 | def __init__(self, key, dtype): 208 | super().__init__(dtype) 209 | self.key = key 210 | 211 | def __call__(self, row): 212 | return row[self.key] 213 | 214 | 215 | class AccountsTable(tables.Table): 216 | name = 'accounts' 217 | columns = { 218 | 'account': GetItemColumn(0, str), 219 | 'open': GetItemColumn(1, Open), 220 | 'close': GetItemColumn(2, Close), 221 | } 222 | 223 | def __init__(self, entries, options): 224 | self.accounts = get_account_open_close(entries) 225 | self.types = parser.options.get_account_types(options) 226 | 227 | def __iter__(self): 228 | return ((name, value[0], value[1]) for name, value in self.accounts.items()) 229 | 230 | _TABLES.append(AccountsTable) 231 | 232 | 233 | class CommoditiesTable(tables.Table): 234 | name = 'commodities' 235 | columns = _typed_namedtuple_to_columns(data.Commodity, {'currency': 'name'}) 236 | 237 | def __init__(self, entries, options): 238 | self.commodities = get_commodity_directives(entries) 239 | 240 | def __iter__(self): 241 | return iter(self.commodities.values()) 242 | 243 | _TABLES.append(CommoditiesTable) 244 | 245 | 246 | class _BeancountTable(tables.Table): 247 | def __init__(self, entries, options, open=None, close=None, clear=None): 248 | super().__init__() 249 | self.entries = entries 250 | self.options = options 251 | self.open = open 252 | self.close = close 253 | self.clear = clear 254 | 255 | def evolve(self, **kwargs): 256 | table = copy.copy(self) 257 | for name, value in kwargs.items(): 258 | setattr(table, name, value) 259 | return table 260 | 261 | def prepare(self): 262 | """Filter the entries applying the FROM clause qualifiers OPEN, CLOSE, CLEAR.""" 263 | entries = self.entries 264 | options = self.options 265 | 266 | # Process the OPEN clause. 267 | if self.open is not None: 268 | entries, index = summarize.open_opt(entries, self.open, options) 269 | 270 | # Process the CLOSE clause. 271 | if self.close is not None: 272 | if isinstance(self.close, datetime.date): 273 | entries, index = summarize.close_opt(entries, self.close, options) 274 | elif self.close is True: 275 | entries, index = summarize.close_opt(entries, None, options) 276 | 277 | # Process the CLEAR clause. 278 | if self.clear is not None: 279 | entries, index = summarize.clear_opt(entries, None, options) 280 | 281 | return entries 282 | 283 | 284 | class EntriesTable(_BeancountTable): 285 | name = 'entries' 286 | columns = ColumnsRegistry() 287 | 288 | def __iter__(self): 289 | entries = self.prepare() 290 | yield from iter(entries) 291 | 292 | @columns.register(str) 293 | def id(entry): 294 | """Unique id of a directive.""" 295 | return hash_entry(entry) 296 | 297 | @columns.register(str) 298 | def type(entry): 299 | """The data type of the directive.""" 300 | return type(entry).__name__.lower() 301 | 302 | @columns.register(str) 303 | def filename(entry): 304 | """The filename where the directive was parsed from or created.""" 305 | return entry.meta["filename"] 306 | 307 | @columns.register(int) 308 | def lineno(entry): 309 | """The line number from the file the directive was parsed from.""" 310 | return entry.meta["lineno"] 311 | 312 | @columns.register(datetime.date) 313 | def date(entry): 314 | """The date of the directive.""" 315 | return entry.date 316 | 317 | @columns.register(int) 318 | def year(entry): 319 | """The year of the date year of the directive.""" 320 | return entry.date.year 321 | 322 | @columns.register(int) 323 | def month(entry): 324 | """The year of the date month of the directive.""" 325 | return entry.date.month 326 | 327 | @columns.register(int) 328 | def day(entry): 329 | """The year of the date day of the directive.""" 330 | return entry.date.day 331 | 332 | @columns.register(str) 333 | def flag(entry): 334 | """The flag the transaction.""" 335 | if not isinstance(entry, data.Transaction): 336 | return None 337 | return entry.flag 338 | 339 | @columns.register(str) 340 | def payee(entry): 341 | """The payee of the transaction.""" 342 | if not isinstance(entry, data.Transaction): 343 | return None 344 | return entry.payee 345 | 346 | @columns.register(str) 347 | def narration(entry): 348 | """The narration of the transaction.""" 349 | if not isinstance(entry, data.Transaction): 350 | return None 351 | return entry.narration 352 | 353 | @columns.register(str) 354 | def description(entry): 355 | """A combination of the payee + narration of the transaction, if present.""" 356 | if not isinstance(entry, data.Transaction): 357 | return None 358 | return ' | '.join(filter(None, [entry.payee, entry.narration])) 359 | 360 | @columns.register(set) 361 | def tags(entry): 362 | """The set of tags of the transaction.""" 363 | return getattr(entry, 'tags', None) 364 | 365 | @columns.register(set) 366 | def links(entry): 367 | """The set of links of the transaction.""" 368 | return getattr(entry, 'links', None) 369 | 370 | @columns.register(dict) 371 | def meta(entry): 372 | return entry.meta 373 | 374 | @columns.register(typing.Set[str]) 375 | def accounts(entry): 376 | return getters.get_entry_accounts(entry) 377 | 378 | _TABLES.append(EntriesTable) 379 | 380 | 381 | class _PostingsTableRow: 382 | """A dumb container for information used by a row expression.""" 383 | 384 | def __init__(self): 385 | self.rowid = 0 386 | self.balance = inventory.Inventory() 387 | 388 | # The current transaction of the posting being evaluated. 389 | self.entry = None 390 | 391 | # The current posting being evaluated. 392 | self.posting = None 393 | 394 | def __hash__(self): 395 | # The context hash is used in caching column accessor functions. 396 | # Instead than hashing the row context content, use the rowid as 397 | # hash. 398 | return self.rowid 399 | 400 | 401 | class PostingsTable(_BeancountTable): 402 | name = 'postings' 403 | columns = ColumnsRegistry() 404 | wildcard_columns = 'date flag payee narration position'.split() 405 | 406 | def __iter__(self): 407 | entries = self.prepare() 408 | context = _PostingsTableRow() 409 | for entry in entries: 410 | if isinstance(entry, data.Transaction): 411 | context.entry = entry 412 | for posting in entry.postings: 413 | context.rowid += 1 414 | context.posting = posting 415 | yield context 416 | 417 | @columns.register(str) 418 | def type(context): 419 | return 'transaction' 420 | 421 | @columns.register(str) 422 | def id(context): 423 | """Unique id of a directive.""" 424 | return hash_entry(context.entry) 425 | 426 | @columns.register(datetime.date) 427 | def date(context): 428 | """The date of the directive.""" 429 | return context.entry.date 430 | 431 | @columns.register(int) 432 | def year(context): 433 | """The year of the date year of the directive.""" 434 | return context.entry.date.year 435 | 436 | @columns.register(int) 437 | def month(context): 438 | """The year of the date month of the directive.""" 439 | return context.entry.date.month 440 | 441 | @columns.register(int) 442 | def day(context): 443 | """The year of the date day of the directive.""" 444 | return context.entry.date.day 445 | 446 | @columns.register(str) 447 | def filename(context): 448 | """The ledger where the posting is defined.""" 449 | meta = context.posting.meta 450 | # Postings for pad transactions have their meta fields set to 451 | # None. See https://github.com/beancount/beancount/issues/767 452 | if meta is None: 453 | return None 454 | return meta["filename"] 455 | 456 | @columns.register(int) 457 | def lineno(context): 458 | """The line number in the ledger file where the posting is defined.""" 459 | meta = context.posting.meta 460 | # Postings for pad transactions have their meta fields set to 461 | # None. See https://github.com/beancount/beancount/issues/767 462 | if meta is None: 463 | return None 464 | return meta["lineno"] 465 | 466 | @columns.register(str) 467 | def location(context): 468 | """The filename:lineno location where the posting is defined.""" 469 | meta = context.posting.meta 470 | # Postings for pad transactions have their meta fields set to 471 | # None. See https://github.com/beancount/beancount/issues/767 472 | if meta is None: 473 | return None 474 | return '{:s}:{:d}:'.format(meta['filename'], meta['lineno']) 475 | 476 | @columns.register(str) 477 | def flag(context): 478 | """The flag of the parent transaction for this posting.""" 479 | return context.entry.flag 480 | 481 | @columns.register(str) 482 | def payee(context): 483 | """The payee of the parent transaction for this posting.""" 484 | return context.entry.payee 485 | 486 | @columns.register(str) 487 | def narration(context): 488 | """The narration of the parent transaction for this posting.""" 489 | return context.entry.narration 490 | 491 | @columns.register(str) 492 | def description(context): 493 | """A combination of the payee + narration for the transaction of this posting.""" 494 | return ' | '.join(filter(None, [context.entry.payee, context.entry.narration])) 495 | 496 | @columns.register(set) 497 | def tags(context): 498 | """The set of tags of the parent transaction for this posting.""" 499 | return context.entry.tags 500 | 501 | @columns.register(set) 502 | def links(context): 503 | """The set of links of the parent transaction for this posting.""" 504 | return context.entry.links 505 | 506 | @columns.register(str) 507 | def posting_flag(context): 508 | """The flag of the posting itself.""" 509 | return context.posting.flag 510 | 511 | @columns.register(str) 512 | def account(context): 513 | """The account of the posting.""" 514 | return context.posting.account 515 | 516 | @columns.register(set) 517 | def other_accounts(context): 518 | """The list of other accounts in the transaction, excluding that of this posting.""" 519 | return sorted({posting.account for posting in context.entry.postings if posting is not context.posting}) 520 | 521 | @columns.register(Decimal) 522 | def number(context): 523 | """The number of units of the posting.""" 524 | return context.posting.units.number 525 | 526 | @columns.register(str) 527 | def currency(context): 528 | """The currency of the posting.""" 529 | return context.posting.units.currency 530 | 531 | @columns.register(Decimal) 532 | def cost_number(context): 533 | """The number of cost units of the posting.""" 534 | cost = context.posting.cost 535 | return cost.number if cost else None 536 | 537 | @columns.register(str) 538 | def cost_currency(context): 539 | """The cost currency of the posting.""" 540 | cost = context.posting.cost 541 | return cost.currency if cost else None 542 | 543 | @columns.register(datetime.date) 544 | def cost_date(context): 545 | """The cost currency of the posting.""" 546 | cost = context.posting.cost 547 | return cost.date if cost else None 548 | 549 | @columns.register(str) 550 | def cost_label(context): 551 | """The cost currency of the posting.""" 552 | cost = context.posting.cost 553 | return cost.label if cost else '' 554 | 555 | @columns.register(position.Position) 556 | def position(context): 557 | """The position for the posting. These can be summed into inventories.""" 558 | posting = context.posting 559 | return position.Position(posting.units, posting.cost) 560 | 561 | @columns.register(amount.Amount) 562 | def price(context): 563 | """The price attached to the posting.""" 564 | return context.posting.price 565 | 566 | @columns.register(amount.Amount) 567 | def weight(context): 568 | """The computed weight used for this posting.""" 569 | return convert.get_weight(context.posting) 570 | 571 | @columns.register(inventory.Inventory) 572 | @cache(maxsize=1) # noqa: B019 573 | def balance(context): 574 | """The balance for the posting. These can be summed into inventories.""" 575 | # Caching protects against multiple balance updates per row when 576 | # the columns appears more than once in the execurted query. The 577 | # rowid in the row context guarantees that otherwise identical 578 | # rows do not hit the cache and thus that the balance is correctly 579 | # updated. 580 | context.balance.add_position(context.posting) 581 | return copy.copy(context.balance) 582 | 583 | @columns.register(dict) 584 | def meta(context): 585 | return context.posting.meta 586 | 587 | @columns.register(data.Transaction) 588 | def entry(context): 589 | return context.entry 590 | 591 | @columns.register(typing.Set[str]) 592 | def accounts(context): 593 | return {p.account for p in context.entry.postings} 594 | 595 | _TABLES.append(PostingsTable) 596 | -------------------------------------------------------------------------------- /beanquery/sources/csv.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import datetime 3 | 4 | from os import path 5 | from urllib.parse import urlparse, parse_qsl 6 | 7 | from beanquery import tables 8 | from beanquery import query_compile 9 | from beanquery.parser import BQLParser, BQLSemantics 10 | 11 | # Support conversions from all fundamental types supported by the BQL parser: 12 | # 13 | # literal 14 | # = 15 | # | date 16 | # | decimal 17 | # | integer 18 | # | string 19 | # | null 20 | # | boolean 21 | # ; 22 | # 23 | 24 | def _guess_type(value): 25 | try: 26 | r = BQLParser().parse(value, start='literal', semantics=BQLSemantics()) 27 | except Exception: 28 | # Everything that is not recognized as something else is a string. 29 | return str 30 | return type(r) 31 | 32 | 33 | def _parse_bool(value): 34 | x = value.strip().lower() 35 | if x == '1' or x == 'true': 36 | return True 37 | if x == '0' or x == 'false': 38 | return False 39 | raise ValueError(value) 40 | 41 | 42 | _TYPES_TO_PARSERS = { 43 | bool: _parse_bool, 44 | datetime.date: datetime.date.fromisoformat, 45 | } 46 | 47 | 48 | class Column(query_compile.EvalColumn): 49 | def __init__(self, key, datatype, func): 50 | super().__init__(datatype) 51 | self.key = key 52 | self.func = func 53 | 54 | def __call__(self, row): 55 | return self.func(row[self.key]) 56 | 57 | 58 | class Table(tables.Table): 59 | def __init__(self, name, columns, data, header=False, **fmtparams): 60 | self.name = name 61 | self.data = data 62 | self.header = header 63 | # Skip white space after field separator by default to make parsing 64 | # columns accordingly to their type easier, unless the setting is 65 | # overridden by the user. 66 | fmtparams.setdefault('skipinitialspace', True) 67 | self.reader = csv.reader(data, **fmtparams) 68 | self.columns = {} 69 | if columns is None: 70 | names = next(self.reader, []) 71 | values = next(self.reader, []) 72 | datatypes = (_guess_type(value) for value in values) 73 | columns = zip(names, datatypes) 74 | for cname, ctype in columns: 75 | converter = _TYPES_TO_PARSERS.get(ctype, ctype) 76 | self.columns[cname] = Column(len(self.columns), ctype, converter) 77 | 78 | def __del__(self): 79 | self.data.close() 80 | 81 | def __iter__(self): 82 | self.data.seek(0) 83 | it = iter(self.reader) 84 | if self.header: 85 | next(it) 86 | return it 87 | 88 | 89 | def create(name, columns, using): 90 | parts = urlparse(using) 91 | filename = parts.path 92 | params = dict(parse_qsl(parts.query)) 93 | encoding = params.pop('encoding', None) 94 | header = params.pop('header', columns is None) 95 | if filename: 96 | data = open(filename, encoding=encoding) 97 | return Table(name, columns, data, header=header, **params) 98 | 99 | 100 | def attach(context, dsn, *, data=None): 101 | parts = urlparse(dsn) 102 | filename = parts.path 103 | params = dict(parse_qsl(parts.query)) 104 | encoding = params.pop('encoding', None) 105 | if filename: 106 | data = open(filename, encoding=encoding) 107 | name = params.pop('name', None) or path.splitext(path.basename(filename))[0] or 'csv' 108 | context.tables[name] = Table(name, None, data, header=True, **params) 109 | -------------------------------------------------------------------------------- /beanquery/sources/memory.py: -------------------------------------------------------------------------------- 1 | from beanquery import tables 2 | from beanquery.sources.beancount import GetItemColumn 3 | 4 | 5 | class Table(tables.Table): 6 | def __init__(self, name, columns): 7 | self.name = name 8 | self.data = [] 9 | self.columns = {} 10 | for cname, ctype in columns: 11 | self.columns[cname] = GetItemColumn(len(self.columns), ctype) 12 | 13 | def __iter__(self): 14 | return iter(self.data) 15 | 16 | def insert(self, row): 17 | assert len(row) == len(self.columns) 18 | self.data.append(row) 19 | 20 | 21 | def create(name, columns, *args, **kwargs): 22 | return Table(name, columns) 23 | -------------------------------------------------------------------------------- /beanquery/sources/test.py: -------------------------------------------------------------------------------- 1 | import re 2 | from urllib.parse import urlparse, parse_qsl 3 | 4 | from beanquery import tables 5 | from beanquery.query_compile import EvalColumn 6 | 7 | 8 | class Column(EvalColumn): 9 | def __init__(self, datatype, func): 10 | super().__init__(datatype) 11 | self.func = func 12 | 13 | def __call__(self, row): 14 | return self.func(row) 15 | 16 | 17 | class TableMeta(type): 18 | def __new__(mcs, name, bases, dct): 19 | columns = {} 20 | members = {} 21 | for key, value in dct.items(): 22 | if isinstance(value, EvalColumn): 23 | value.name = key 24 | columns[key] = value 25 | continue 26 | members[key] = value 27 | assert 'columns' not in members 28 | members['columns'] = columns 29 | return super().__new__(mcs, name, bases, members) 30 | 31 | 32 | class Table(tables.Table, metaclass=TableMeta): 33 | x = Column(int, lambda row: row) 34 | 35 | def __init__(self, *args): 36 | self.rows = range(*args) 37 | 38 | def __iter__(self): 39 | return iter(self.rows) 40 | 41 | 42 | class MagicColumnsRegistry(dict): 43 | def get(self, key, default=None): 44 | if re.fullmatch(r'x+', key): 45 | return Column(int, lambda x: x * len(key)) 46 | return default 47 | 48 | 49 | class MagicTable(tables.Table): 50 | columns = MagicColumnsRegistry() 51 | 52 | def __init__(self, *args): 53 | self.rows = range(*args) 54 | 55 | def __iter__(self): 56 | return iter(self.rows) 57 | 58 | 59 | def create(name, columns, using): 60 | assert columns is None 61 | parts = urlparse(using) 62 | params = dict(parse_qsl(parts.query, strict_parsing=True)) 63 | cls = MagicTable if parts.path == 'magic' else Table 64 | return cls(int(params.get('start', 0)), int(params.get('stop', 11)), int(params.get('step', 1))) 65 | 66 | 67 | def attach(context, dsn): 68 | parts = urlparse(dsn) 69 | params = dict(parse_qsl(parts.query, strict_parsing=True)) 70 | name = params.get('name', 'test') 71 | context.tables[name] = create(name, None, dsn) 72 | -------------------------------------------------------------------------------- /beanquery/tables.py: -------------------------------------------------------------------------------- 1 | class Table: 2 | columns = {} 3 | name = '' 4 | 5 | def __getitem__(self, name): 6 | return self.columns[name] 7 | 8 | @property 9 | def wildcard_columns(self): 10 | # For backward compatibility. Remove once the postings table 11 | # is updated to return all columns upon ``SELECT *`` and the 12 | # query compiler is updated not to rely on this property 13 | return self.columns.keys() 14 | 15 | 16 | class NullTable(Table): 17 | def __iter__(self): 18 | return iter([None]) 19 | -------------------------------------------------------------------------------- /beanquery/types.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import decimal 3 | import itertools 4 | import typing 5 | 6 | 7 | # Only Python >= 3.10 exposes NoneType in the types module. 8 | NoneType = type(None) 9 | 10 | 11 | class AnyType: 12 | """As written on the tin.""" 13 | __slots__ = () 14 | __name__ = 'any' 15 | 16 | def __eq__(self, other): 17 | """Compares equal to any other type.""" 18 | return isinstance(other, type) 19 | 20 | 21 | # Used in BQL functions signatures for arguments that can have any type. 22 | Any = AnyType() 23 | 24 | 25 | # Used for COUNT(*) 26 | Asterisk = typing.NewType('*', object) # noqa: PLC0132 27 | Asterisk.__mro__ = Asterisk, 28 | 29 | 30 | # Keep track of the defined structured types to allow introspection. 31 | TYPES = {} 32 | 33 | 34 | class Structure: 35 | """Base class for structured data types.""" 36 | name = None 37 | columns = {} 38 | 39 | def __init_subclass__(cls): 40 | if cls.name: 41 | TYPES[cls.name] = cls 42 | 43 | 44 | def _bases(t): 45 | if t is NoneType: 46 | return (object,) 47 | bases = t.__mro__ 48 | if len(bases) > 1 and bases[-1] is object: 49 | # All types that are not ``object`` have more than one class 50 | # in their ``__mro__``. BQL uses ``object`` for untypes 51 | # values. Do not return ``object`` as base for strict types, 52 | # to avoid functions taking untyped onjects to accept all 53 | # values. 54 | return bases[:-1] 55 | return bases 56 | 57 | 58 | def function_lookup(functions, name, operands): 59 | """Lookup a BQL function implementation. 60 | 61 | Args: 62 | functions: The functions registry to interrogate. 63 | name: The function name. 64 | operands: Function operands. 65 | 66 | Returns: 67 | A EvalNode (or subclass) instance or None if the function was not found. 68 | """ 69 | for signature in itertools.product(*(_bases(operand.dtype) for operand in operands)): 70 | for func in functions[name]: 71 | if func.__intypes__ == list(signature): 72 | return func 73 | return None 74 | 75 | 76 | # Map types to their BQL name. Used to find the name of the type cast funtion. 77 | MAP = { 78 | bool: 'bool', 79 | datetime.date: 'date', 80 | decimal.Decimal: 'decimal', 81 | int: 'int', 82 | str: 'str', 83 | } 84 | 85 | 86 | # Map between Python types and BQL structured types. Functions and 87 | # columns definitions can use Python types. The corresponding BQL 88 | # structured type is looked up when compiling the subscrip operator. 89 | ALIASES = {} 90 | 91 | 92 | def name(datatype): 93 | if datatype is NoneType: 94 | return 'NULL' 95 | if isinstance(datatype, typing._GenericAlias): 96 | return str(datatype).rsplit('.', 1)[-1].lower() 97 | return getattr(datatype, 'name', datatype.__name__.lower()) 98 | 99 | 100 | _STRING_TO_DATATYPE = { 101 | 'bool': bool, 102 | 'date': datetime.date, 103 | 'decimal': decimal.Decimal, 104 | 'int': int, 105 | 'object': object, 106 | 'str': str, 107 | 'text': str, 108 | 'varchar': str, 109 | } 110 | 111 | 112 | def parse(name): 113 | """Parse the string representation of a type into a type object. 114 | 115 | This does not (yet) work for all supprted data types. 116 | """ 117 | return _STRING_TO_DATATYPE.get(name, None) 118 | -------------------------------------------------------------------------------- /beanquery/types_test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from beanquery import types 4 | from beanquery.sources import beancount 5 | 6 | 7 | class TestName(unittest.TestCase): 8 | def test_transactions_names(self): 9 | table = beancount.TransactionsTable 10 | columns = {name: types.name(column.dtype) for name, column in table.columns.items()} 11 | self.assertEqual(columns, { 12 | 'meta': 'metadata', 13 | 'date': 'date', 14 | 'flag': 'str', 15 | 'payee': 'str', 16 | 'narration': 'str', 17 | 'tags': 'set', 18 | 'links': 'set', 19 | 'accounts': 'set[str]', 20 | }) 21 | -------------------------------------------------------------------------------- /bin/bean-query: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | __copyright__ = "Copyright (C) 2013-2017 Martin Blais" 3 | __license__ = "GNU GPLv2" 4 | from beanquery.shell import main; main() 5 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | project = 'beanquery' 2 | copyright = '2014-2022, beanquery Contributors' 3 | author = 'beanquery Contributors' 4 | version = '0.1' 5 | language = 'en' 6 | html_theme = 'furo' 7 | html_title = f'{project} {version}' 8 | html_logo = 'logo.svg' 9 | extensions = [ 10 | 'sphinx.ext.autodoc', 11 | 'sphinx.ext.napoleon', 12 | 'sphinx.ext.intersphinx', 13 | 'sphinx.ext.extlinks', 14 | 'sphinx.ext.githubpages', 15 | ] 16 | extlinks = { 17 | 'issue': ('https://github.com/beancount/beanquery/issues/%s', '#'), 18 | 'pull': ('https://github.com/beancount/beanquery/pull/%s', '#'), 19 | } 20 | intersphinx_mapping = { 21 | 'python': ('https://docs.python.org/3', None), 22 | 'beancount': ('https://beancount.github.io/docs/', None), 23 | } 24 | napoleon_google_docstring = True 25 | napoleon_use_param = False 26 | autodoc_typehints = 'none' 27 | autodoc_member_order = 'bysource' 28 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | beanquery: Customizable lightweight SQL query tool 2 | ================================================== 3 | 4 | beanquery is a customizable and extensible lightweight SQL query tool 5 | that works on tabular data, including `Beancount`__ ledger data. 6 | 7 | __ https://beancount.github.io/ 8 | -------------------------------------------------------------------------------- /docs/logo.inkscape: -------------------------------------------------------------------------------- 1 | 2 | 13 | 15 | 34 | 43 | 48 | 53 | 57 | 58 | 59 | -------------------------------------------------------------------------------- /docs/logo.svg: -------------------------------------------------------------------------------- 1 | 2 | 13 | 15 | 34 | 44 | 49 | 54 | 58 | 59 | 60 | -------------------------------------------------------------------------------- /etc/env: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | USERPATH=$USERPATH:$PROJDIR/bin 3 | PYTHONPATH=$PYTHONPATH:$PROJDIR 4 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ['setuptools'] 3 | build-backend = 'setuptools.build_meta' 4 | 5 | [project] 6 | name = 'beanquery' 7 | version = '0.3.0.dev0' 8 | description = 'Customizable lightweight SQL query tool' 9 | license = { file = 'LICENSE' } 10 | readme = 'README.rst' 11 | authors = [ 12 | { name = 'Martin Blais', email = 'blais@furius.ca' }, 13 | { name = 'Daniele Nicolodi', email = 'daniele@grinta.net' }, 14 | ] 15 | maintainers = [ 16 | { name = 'Daniele Nicolodi', email = 'daniele@grinta.net' }, 17 | ] 18 | keywords = [ 19 | 'accounting', 'ledger', 'beancount', 'SQL', 'BQL' 20 | ] 21 | classifiers = [ 22 | 'License :: OSI Approved :: GNU General Public License v2 (GPLv2)', 23 | 'Programming Language :: Python :: 3 :: Only', 24 | 'Programming Language :: Python :: 3.8', 25 | 'Programming Language :: Python :: 3.9', 26 | 'Programming Language :: Python :: 3.10', 27 | 'Programming Language :: Python :: 3.11', 28 | 'Programming Language :: Python :: 3.12', 29 | 'Programming Language :: Python :: 3.13', 30 | 'Programming Language :: SQL', 31 | 'Topic :: Office/Business :: Financial :: Accounting', 32 | ] 33 | requires-python = '>= 3.8' 34 | dependencies = [ 35 | 'beancount >= 2.3.4', 36 | 'click > 7.0', 37 | 'python-dateutil >= 2.6.0', 38 | 'tatsu-lts', 39 | ] 40 | 41 | [project.optional-dependencies] 42 | docs = [ 43 | 'furo >= 2024.08.06', 44 | 'sphinx ~= 8.1.0', 45 | ] 46 | 47 | [project.scripts] 48 | bean-query = 'beanquery.shell:main' 49 | 50 | [project.urls] 51 | homepage = 'https://github.com/beancount/beanquery' 52 | issues = 'https://github.com/beancount/beanquery/issues' 53 | 54 | [tool.setuptools.packages] 55 | find = {} 56 | 57 | [tool.coverage.run] 58 | branch = true 59 | 60 | [tool.coverage.report] 61 | exclude_also = [ 62 | 'if typing.TYPE_CHECKING:', 63 | ] 64 | 65 | [tool.ruff] 66 | line-length = 128 67 | target-version = 'py38' 68 | 69 | [tool.ruff.lint] 70 | select = ['E', 'F', 'W', 'UP', 'B', 'C4', 'PL', 'RUF'] 71 | ignore = [ 72 | 'B007', 73 | 'B905', 74 | 'C408', 75 | 'E731', 76 | 'PLR0911', 77 | 'PLR0912', 78 | 'PLR0913', 79 | 'PLR0915', 80 | 'PLR1714', 81 | 'PLR2004', 82 | 'PLW2901', 83 | 'RUF012', 84 | 'RUF023', # unsorted-dunder-slots 85 | 'UP007', 86 | 'UP032', 87 | ] 88 | exclude = [ 89 | 'beanquery/parser/parser.py' 90 | ] 91 | 92 | [tool.ruff.lint.per-file-ignores] 93 | 'beanquery/query_env.py' = ['F811'] 94 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | beancount >=2.3.4 2 | click >7.0 3 | python-dateutil >=2.6.0 4 | tatsu >=5.7.4, <5.8.0 5 | --------------------------------------------------------------------------------