├── .gitignore ├── LICENSE ├── README.org ├── bin └── plaid2text ├── img ├── netflix_account.png ├── netflix_payee.png └── netflix_tags.png ├── requirements.txt ├── setup.py └── src └── python └── plaid2text ├── __init__.py ├── config_manager.py ├── interact.py ├── online_accounts.py ├── plaid2text.py ├── renderers.py └── storage_manager.py /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Created by https://www.gitignore.io/api/python 3 | 4 | ### Emacs ### 5 | # -*- mode: gitignore; -*- 6 | *~ 7 | \#*\# 8 | /.emacs.desktop 9 | /.emacs.desktop.lock 10 | *.elc 11 | auto-save-list 12 | tramp 13 | .\#* 14 | 15 | # Org-mode 16 | .org-id-locations 17 | *_archive 18 | 19 | # flymake-mode 20 | *_flymake.* 21 | 22 | # eshell files 23 | /eshell/history 24 | /eshell/lastdir 25 | 26 | # elpa packages 27 | /elpa/ 28 | 29 | # reftex files 30 | *.rel 31 | 32 | # AUCTeX auto folder 33 | /auto/ 34 | 35 | # cask packages 36 | .cask/ 37 | dist/ 38 | 39 | # Flycheck 40 | flycheck_*.el 41 | 42 | # server auth directory 43 | /server/ 44 | 45 | # projectiles files 46 | .projectile 47 | ### Python ### 48 | # Byte-compiled / optimized / DLL files 49 | __pycache__/ 50 | *.py[cod] 51 | *$py.class 52 | 53 | # C extensions 54 | *.so 55 | 56 | # Distribution / packaging 57 | .Python 58 | env/ 59 | build/ 60 | develop-eggs/ 61 | dist/ 62 | downloads/ 63 | eggs/ 64 | .eggs/ 65 | lib/ 66 | lib64/ 67 | parts/ 68 | sdist/ 69 | var/ 70 | *.egg-info/ 71 | .installed.cfg 72 | *.egg 73 | 74 | # PyInstaller 75 | # Usually these files are written by a python script from a template 76 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 77 | *.manifest 78 | *.spec 79 | 80 | # Installer logs 81 | pip-log.txt 82 | pip-delete-this-directory.txt 83 | 84 | # Unit test / coverage reports 85 | htmlcov/ 86 | .tox/ 87 | .coverage 88 | .coverage.* 89 | .cache 90 | nosetests.xml 91 | coverage.xml 92 | *,cover 93 | .hypothesis/ 94 | 95 | # Translations 96 | *.mo 97 | *.pot 98 | 99 | # Django stuff: 100 | *.log 101 | local_settings.py 102 | 103 | # Flask stuff: 104 | instance/ 105 | .webassets-cache 106 | 107 | # Scrapy stuff: 108 | .scrapy 109 | 110 | # Sphinx documentation 111 | docs/_build/ 112 | 113 | # PyBuilder 114 | target/ 115 | 116 | # IPython Notebook 117 | .ipynb_checkpoints 118 | 119 | # pyenv 120 | .python-version 121 | 122 | # celery beat schedule file 123 | celerybeat-schedule 124 | 125 | # dotenv 126 | .env 127 | 128 | # virtualenv 129 | venv/ 130 | ENV/ 131 | 132 | # Spyder project settings 133 | .spyderproject 134 | 135 | # Rope project settings 136 | .ropeproject 137 | 138 | Pipfile* -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | The GNU General Public License is a free, copyleft license for 11 | software and other kinds of works. 12 | 13 | The licenses for most software and other practical works are designed 14 | to take away your freedom to share and change the works. By contrast, 15 | the GNU General Public License is intended to guarantee your freedom to 16 | share and change all versions of a program--to make sure it remains free 17 | software for all its users. We, the Free Software Foundation, use the 18 | GNU General Public License for most of our software; it applies also to 19 | any other work released this way by its authors. You can apply it to 20 | your programs, too. 21 | 22 | When we speak of free software, we are referring to freedom, not 23 | price. Our General Public Licenses are designed to make sure that you 24 | have the freedom to distribute copies of free software (and charge for 25 | them if you wish), that you receive source code or can get it if you 26 | want it, that you can change the software or use pieces of it in new 27 | free programs, and that you know you can do these things. 28 | 29 | To protect your rights, we need to prevent others from denying you 30 | these rights or asking you to surrender the rights. Therefore, you have 31 | certain responsibilities if you distribute copies of the software, or if 32 | you modify it: responsibilities to respect the freedom of others. 33 | 34 | For example, if you distribute copies of such a program, whether 35 | gratis or for a fee, you must pass on to the recipients the same 36 | freedoms that you received. You must make sure that they, too, receive 37 | or can get the source code. And you must show them these terms so they 38 | know their rights. 39 | 40 | Developers that use the GNU GPL protect your rights with two steps: 41 | (1) assert copyright on the software, and (2) offer you this License 42 | giving you legal permission to copy, distribute and/or modify it. 43 | 44 | For the developers' and authors' protection, the GPL clearly explains 45 | that there is no warranty for this free software. For both users' and 46 | authors' sake, the GPL requires that modified versions be marked as 47 | changed, so that their problems will not be attributed erroneously to 48 | authors of previous versions. 49 | 50 | Some devices are designed to deny users access to install or run 51 | modified versions of the software inside them, although the manufacturer 52 | can do so. This is fundamentally incompatible with the aim of 53 | protecting users' freedom to change the software. The systematic 54 | pattern of such abuse occurs in the area of products for individuals to 55 | use, which is precisely where it is most unacceptable. Therefore, we 56 | have designed this version of the GPL to prohibit the practice for those 57 | products. If such problems arise substantially in other domains, we 58 | stand ready to extend this provision to those domains in future versions 59 | of the GPL, as needed to protect the freedom of users. 60 | 61 | Finally, every program is threatened constantly by software patents. 62 | States should not allow patents to restrict development and use of 63 | software on general-purpose computers, but in those that do, we wish to 64 | avoid the special danger that patents applied to a free program could 65 | make it effectively proprietary. To prevent this, the GPL assures that 66 | patents cannot be used to render the program non-free. 67 | 68 | The precise terms and conditions for copying, distribution and 69 | modification follow. 70 | 71 | TERMS AND CONDITIONS 72 | 73 | 0. Definitions. 74 | 75 | "This License" refers to version 3 of the GNU General Public License. 76 | 77 | "Copyright" also means copyright-like laws that apply to other kinds of 78 | works, such as semiconductor masks. 79 | 80 | "The Program" refers to any copyrightable work licensed under this 81 | License. Each licensee is addressed as "you". "Licensees" and 82 | "recipients" may be individuals or organizations. 83 | 84 | To "modify" a work means to copy from or adapt all or part of the work 85 | in a fashion requiring copyright permission, other than the making of an 86 | exact copy. The resulting work is called a "modified version" of the 87 | earlier work or a work "based on" the earlier work. 88 | 89 | A "covered work" means either the unmodified Program or a work based 90 | on the Program. 91 | 92 | To "propagate" a work means to do anything with it that, without 93 | permission, would make you directly or secondarily liable for 94 | infringement under applicable copyright law, except executing it on a 95 | computer or modifying a private copy. Propagation includes copying, 96 | distribution (with or without modification), making available to the 97 | public, and in some countries other activities as well. 98 | 99 | To "convey" a work means any kind of propagation that enables other 100 | parties to make or receive copies. Mere interaction with a user through 101 | a computer network, with no transfer of a copy, is not conveying. 102 | 103 | An interactive user interface displays "Appropriate Legal Notices" 104 | to the extent that it includes a convenient and prominently visible 105 | feature that (1) displays an appropriate copyright notice, and (2) 106 | tells the user that there is no warranty for the work (except to the 107 | extent that warranties are provided), that licensees may convey the 108 | work under this License, and how to view a copy of this License. If 109 | the interface presents a list of user commands or options, such as a 110 | menu, a prominent item in the list meets this criterion. 111 | 112 | 1. Source Code. 113 | 114 | The "source code" for a work means the preferred form of the work 115 | for making modifications to it. "Object code" means any non-source 116 | form of a work. 117 | 118 | A "Standard Interface" means an interface that either is an official 119 | standard defined by a recognized standards body, or, in the case of 120 | interfaces specified for a particular programming language, one that 121 | is widely used among developers working in that language. 122 | 123 | The "System Libraries" of an executable work include anything, other 124 | than the work as a whole, that (a) is included in the normal form of 125 | packaging a Major Component, but which is not part of that Major 126 | Component, and (b) serves only to enable use of the work with that 127 | Major Component, or to implement a Standard Interface for which an 128 | implementation is available to the public in source code form. A 129 | "Major Component", in this context, means a major essential component 130 | (kernel, window system, and so on) of the specific operating system 131 | (if any) on which the executable work runs, or a compiler used to 132 | produce the work, or an object code interpreter used to run it. 133 | 134 | The "Corresponding Source" for a work in object code form means all 135 | the source code needed to generate, install, and (for an executable 136 | work) run the object code and to modify the work, including scripts to 137 | control those activities. However, it does not include the work's 138 | System Libraries, or general-purpose tools or generally available free 139 | programs which are used unmodified in performing those activities but 140 | which are not part of the work. For example, Corresponding Source 141 | includes interface definition files associated with source files for 142 | the work, and the source code for shared libraries and dynamically 143 | linked subprograms that the work is specifically designed to require, 144 | such as by intimate data communication or control flow between those 145 | subprograms and other parts of the work. 146 | 147 | The Corresponding Source need not include anything that users 148 | can regenerate automatically from other parts of the Corresponding 149 | Source. 150 | 151 | The Corresponding Source for a work in source code form is that 152 | same work. 153 | 154 | 2. Basic Permissions. 155 | 156 | All rights granted under this License are granted for the term of 157 | copyright on the Program, and are irrevocable provided the stated 158 | conditions are met. This License explicitly affirms your unlimited 159 | permission to run the unmodified Program. The output from running a 160 | covered work is covered by this License only if the output, given its 161 | content, constitutes a covered work. This License acknowledges your 162 | rights of fair use or other equivalent, as provided by copyright law. 163 | 164 | You may make, run and propagate covered works that you do not 165 | convey, without conditions so long as your license otherwise remains 166 | in force. You may convey covered works to others for the sole purpose 167 | of having them make modifications exclusively for you, or provide you 168 | with facilities for running those works, provided that you comply with 169 | the terms of this License in conveying all material for which you do 170 | not control copyright. Those thus making or running the covered works 171 | for you must do so exclusively on your behalf, under your direction 172 | and control, on terms that prohibit them from making any copies of 173 | your copyrighted material outside their relationship with you. 174 | 175 | Conveying under any other circumstances is permitted solely under 176 | the conditions stated below. Sublicensing is not allowed; section 10 177 | makes it unnecessary. 178 | 179 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 180 | 181 | No covered work shall be deemed part of an effective technological 182 | measure under any applicable law fulfilling obligations under article 183 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 184 | similar laws prohibiting or restricting circumvention of such 185 | measures. 186 | 187 | When you convey a covered work, you waive any legal power to forbid 188 | circumvention of technological measures to the extent such circumvention 189 | is effected by exercising rights under this License with respect to 190 | the covered work, and you disclaim any intention to limit operation or 191 | modification of the work as a means of enforcing, against the work's 192 | users, your or third parties' legal rights to forbid circumvention of 193 | technological measures. 194 | 195 | 4. Conveying Verbatim Copies. 196 | 197 | You may convey verbatim copies of the Program's source code as you 198 | receive it, in any medium, provided that you conspicuously and 199 | appropriately publish on each copy an appropriate copyright notice; 200 | keep intact all notices stating that this License and any 201 | non-permissive terms added in accord with section 7 apply to the code; 202 | keep intact all notices of the absence of any warranty; and give all 203 | recipients a copy of this License along with the Program. 204 | 205 | You may charge any price or no price for each copy that you convey, 206 | and you may offer support or warranty protection for a fee. 207 | 208 | 5. Conveying Modified Source Versions. 209 | 210 | You may convey a work based on the Program, or the modifications to 211 | produce it from the Program, in the form of source code under the 212 | terms of section 4, provided that you also meet all of these conditions: 213 | 214 | a) The work must carry prominent notices stating that you modified 215 | it, and giving a relevant date. 216 | 217 | b) The work must carry prominent notices stating that it is 218 | released under this License and any conditions added under section 219 | 7. This requirement modifies the requirement in section 4 to 220 | "keep intact all notices". 221 | 222 | c) You must license the entire work, as a whole, under this 223 | License to anyone who comes into possession of a copy. This 224 | License will therefore apply, along with any applicable section 7 225 | additional terms, to the whole of the work, and all its parts, 226 | regardless of how they are packaged. This License gives no 227 | permission to license the work in any other way, but it does not 228 | invalidate such permission if you have separately received it. 229 | 230 | d) If the work has interactive user interfaces, each must display 231 | Appropriate Legal Notices; however, if the Program has interactive 232 | interfaces that do not display Appropriate Legal Notices, your 233 | work need not make them do so. 234 | 235 | A compilation of a covered work with other separate and independent 236 | works, which are not by their nature extensions of the covered work, 237 | and which are not combined with it such as to form a larger program, 238 | in or on a volume of a storage or distribution medium, is called an 239 | "aggregate" if the compilation and its resulting copyright are not 240 | used to limit the access or legal rights of the compilation's users 241 | beyond what the individual works permit. Inclusion of a covered work 242 | in an aggregate does not cause this License to apply to the other 243 | parts of the aggregate. 244 | 245 | 6. Conveying Non-Source Forms. 246 | 247 | You may convey a covered work in object code form under the terms 248 | of sections 4 and 5, provided that you also convey the 249 | machine-readable Corresponding Source under the terms of this License, 250 | in one of these ways: 251 | 252 | a) Convey the object code in, or embodied in, a physical product 253 | (including a physical distribution medium), accompanied by the 254 | Corresponding Source fixed on a durable physical medium 255 | customarily used for software interchange. 256 | 257 | b) Convey the object code in, or embodied in, a physical product 258 | (including a physical distribution medium), accompanied by a 259 | written offer, valid for at least three years and valid for as 260 | long as you offer spare parts or customer support for that product 261 | model, to give anyone who possesses the object code either (1) a 262 | copy of the Corresponding Source for all the software in the 263 | product that is covered by this License, on a durable physical 264 | medium customarily used for software interchange, for a price no 265 | more than your reasonable cost of physically performing this 266 | conveying of source, or (2) access to copy the 267 | Corresponding Source from a network server at no charge. 268 | 269 | c) Convey individual copies of the object code with a copy of the 270 | written offer to provide the Corresponding Source. This 271 | alternative is allowed only occasionally and noncommercially, and 272 | only if you received the object code with such an offer, in accord 273 | with subsection 6b. 274 | 275 | d) Convey the object code by offering access from a designated 276 | place (gratis or for a charge), and offer equivalent access to the 277 | Corresponding Source in the same way through the same place at no 278 | further charge. You need not require recipients to copy the 279 | Corresponding Source along with the object code. If the place to 280 | copy the object code is a network server, the Corresponding Source 281 | may be on a different server (operated by you or a third party) 282 | that supports equivalent copying facilities, provided you maintain 283 | clear directions next to the object code saying where to find the 284 | Corresponding Source. Regardless of what server hosts the 285 | Corresponding Source, you remain obligated to ensure that it is 286 | available for as long as needed to satisfy these requirements. 287 | 288 | e) Convey the object code using peer-to-peer transmission, provided 289 | you inform other peers where the object code and Corresponding 290 | Source of the work are being offered to the general public at no 291 | charge under subsection 6d. 292 | 293 | A separable portion of the object code, whose source code is excluded 294 | from the Corresponding Source as a System Library, need not be 295 | included in conveying the object code work. 296 | 297 | A "User Product" is either (1) a "consumer product", which means any 298 | tangible personal property which is normally used for personal, family, 299 | or household purposes, or (2) anything designed or sold for incorporation 300 | into a dwelling. In determining whether a product is a consumer product, 301 | doubtful cases shall be resolved in favor of coverage. For a particular 302 | product received by a particular user, "normally used" refers to a 303 | typical or common use of that class of product, regardless of the status 304 | of the particular user or of the way in which the particular user 305 | actually uses, or expects or is expected to use, the product. A product 306 | is a consumer product regardless of whether the product has substantial 307 | commercial, industrial or non-consumer uses, unless such uses represent 308 | the only significant mode of use of the product. 309 | 310 | "Installation Information" for a User Product means any methods, 311 | procedures, authorization keys, or other information required to install 312 | and execute modified versions of a covered work in that User Product from 313 | a modified version of its Corresponding Source. The information must 314 | suffice to ensure that the continued functioning of the modified object 315 | code is in no case prevented or interfered with solely because 316 | modification has been made. 317 | 318 | If you convey an object code work under this section in, or with, or 319 | specifically for use in, a User Product, and the conveying occurs as 320 | part of a transaction in which the right of possession and use of the 321 | User Product is transferred to the recipient in perpetuity or for a 322 | fixed term (regardless of how the transaction is characterized), the 323 | Corresponding Source conveyed under this section must be accompanied 324 | by the Installation Information. But this requirement does not apply 325 | if neither you nor any third party retains the ability to install 326 | modified object code on the User Product (for example, the work has 327 | been installed in ROM). 328 | 329 | The requirement to provide Installation Information does not include a 330 | requirement to continue to provide support service, warranty, or updates 331 | for a work that has been modified or installed by the recipient, or for 332 | the User Product in which it has been modified or installed. Access to a 333 | network may be denied when the modification itself materially and 334 | adversely affects the operation of the network or violates the rules and 335 | protocols for communication across the network. 336 | 337 | Corresponding Source conveyed, and Installation Information provided, 338 | in accord with this section must be in a format that is publicly 339 | documented (and with an implementation available to the public in 340 | source code form), and must require no special password or key for 341 | unpacking, reading or copying. 342 | 343 | 7. Additional Terms. 344 | 345 | "Additional permissions" are terms that supplement the terms of this 346 | License by making exceptions from one or more of its conditions. 347 | Additional permissions that are applicable to the entire Program shall 348 | be treated as though they were included in this License, to the extent 349 | that they are valid under applicable law. If additional permissions 350 | apply only to part of the Program, that part may be used separately 351 | under those permissions, but the entire Program remains governed by 352 | this License without regard to the additional permissions. 353 | 354 | When you convey a copy of a covered work, you may at your option 355 | remove any additional permissions from that copy, or from any part of 356 | it. (Additional permissions may be written to require their own 357 | removal in certain cases when you modify the work.) You may place 358 | additional permissions on material, added by you to a covered work, 359 | for which you have or can give appropriate copyright permission. 360 | 361 | Notwithstanding any other provision of this License, for material you 362 | add to a covered work, you may (if authorized by the copyright holders of 363 | that material) supplement the terms of this License with terms: 364 | 365 | a) Disclaiming warranty or limiting liability differently from the 366 | terms of sections 15 and 16 of this License; or 367 | 368 | b) Requiring preservation of specified reasonable legal notices or 369 | author attributions in that material or in the Appropriate Legal 370 | Notices displayed by works containing it; or 371 | 372 | c) Prohibiting misrepresentation of the origin of that material, or 373 | requiring that modified versions of such material be marked in 374 | reasonable ways as different from the original version; or 375 | 376 | d) Limiting the use for publicity purposes of names of licensors or 377 | authors of the material; or 378 | 379 | e) Declining to grant rights under trademark law for use of some 380 | trade names, trademarks, or service marks; or 381 | 382 | f) Requiring indemnification of licensors and authors of that 383 | material by anyone who conveys the material (or modified versions of 384 | it) with contractual assumptions of liability to the recipient, for 385 | any liability that these contractual assumptions directly impose on 386 | those licensors and authors. 387 | 388 | All other non-permissive additional terms are considered "further 389 | restrictions" within the meaning of section 10. If the Program as you 390 | received it, or any part of it, contains a notice stating that it is 391 | governed by this License along with a term that is a further 392 | restriction, you may remove that term. If a license document contains 393 | a further restriction but permits relicensing or conveying under this 394 | License, you may add to a covered work material governed by the terms 395 | of that license document, provided that the further restriction does 396 | not survive such relicensing or conveying. 397 | 398 | If you add terms to a covered work in accord with this section, you 399 | must place, in the relevant source files, a statement of the 400 | additional terms that apply to those files, or a notice indicating 401 | where to find the applicable terms. 402 | 403 | Additional terms, permissive or non-permissive, may be stated in the 404 | form of a separately written license, or stated as exceptions; 405 | the above requirements apply either way. 406 | 407 | 8. Termination. 408 | 409 | You may not propagate or modify a covered work except as expressly 410 | provided under this License. Any attempt otherwise to propagate or 411 | modify it is void, and will automatically terminate your rights under 412 | this License (including any patent licenses granted under the third 413 | paragraph of section 11). 414 | 415 | However, if you cease all violation of this License, then your 416 | license from a particular copyright holder is reinstated (a) 417 | provisionally, unless and until the copyright holder explicitly and 418 | finally terminates your license, and (b) permanently, if the copyright 419 | holder fails to notify you of the violation by some reasonable means 420 | prior to 60 days after the cessation. 421 | 422 | Moreover, your license from a particular copyright holder is 423 | reinstated permanently if the copyright holder notifies you of the 424 | violation by some reasonable means, this is the first time you have 425 | received notice of violation of this License (for any work) from that 426 | copyright holder, and you cure the violation prior to 30 days after 427 | your receipt of the notice. 428 | 429 | Termination of your rights under this section does not terminate the 430 | licenses of parties who have received copies or rights from you under 431 | this License. If your rights have been terminated and not permanently 432 | reinstated, you do not qualify to receive new licenses for the same 433 | material under section 10. 434 | 435 | 9. Acceptance Not Required for Having Copies. 436 | 437 | You are not required to accept this License in order to receive or 438 | run a copy of the Program. Ancillary propagation of a covered work 439 | occurring solely as a consequence of using peer-to-peer transmission 440 | to receive a copy likewise does not require acceptance. However, 441 | nothing other than this License grants you permission to propagate or 442 | modify any covered work. These actions infringe copyright if you do 443 | not accept this License. Therefore, by modifying or propagating a 444 | covered work, you indicate your acceptance of this License to do so. 445 | 446 | 10. Automatic Licensing of Downstream Recipients. 447 | 448 | Each time you convey a covered work, the recipient automatically 449 | receives a license from the original licensors, to run, modify and 450 | propagate that work, subject to this License. You are not responsible 451 | for enforcing compliance by third parties with this License. 452 | 453 | An "entity transaction" is a transaction transferring control of an 454 | organization, or substantially all assets of one, or subdividing an 455 | organization, or merging organizations. If propagation of a covered 456 | work results from an entity transaction, each party to that 457 | transaction who receives a copy of the work also receives whatever 458 | licenses to the work the party's predecessor in interest had or could 459 | give under the previous paragraph, plus a right to possession of the 460 | Corresponding Source of the work from the predecessor in interest, if 461 | the predecessor has it or can get it with reasonable efforts. 462 | 463 | You may not impose any further restrictions on the exercise of the 464 | rights granted or affirmed under this License. For example, you may 465 | not impose a license fee, royalty, or other charge for exercise of 466 | rights granted under this License, and you may not initiate litigation 467 | (including a cross-claim or counterclaim in a lawsuit) alleging that 468 | any patent claim is infringed by making, using, selling, offering for 469 | sale, or importing the Program or any portion of it. 470 | 471 | 11. Patents. 472 | 473 | A "contributor" is a copyright holder who authorizes use under this 474 | License of the Program or a work on which the Program is based. The 475 | work thus licensed is called the contributor's "contributor version". 476 | 477 | A contributor's "essential patent claims" are all patent claims 478 | owned or controlled by the contributor, whether already acquired or 479 | hereafter acquired, that would be infringed by some manner, permitted 480 | by this License, of making, using, or selling its contributor version, 481 | but do not include claims that would be infringed only as a 482 | consequence of further modification of the contributor version. For 483 | purposes of this definition, "control" includes the right to grant 484 | patent sublicenses in a manner consistent with the requirements of 485 | this License. 486 | 487 | Each contributor grants you a non-exclusive, worldwide, royalty-free 488 | patent license under the contributor's essential patent claims, to 489 | make, use, sell, offer for sale, import and otherwise run, modify and 490 | propagate the contents of its contributor version. 491 | 492 | In the following three paragraphs, a "patent license" is any express 493 | agreement or commitment, however denominated, not to enforce a patent 494 | (such as an express permission to practice a patent or covenant not to 495 | sue for patent infringement). To "grant" such a patent license to a 496 | party means to make such an agreement or commitment not to enforce a 497 | patent against the party. 498 | 499 | If you convey a covered work, knowingly relying on a patent license, 500 | and the Corresponding Source of the work is not available for anyone 501 | to copy, free of charge and under the terms of this License, through a 502 | publicly available network server or other readily accessible means, 503 | then you must either (1) cause the Corresponding Source to be so 504 | available, or (2) arrange to deprive yourself of the benefit of the 505 | patent license for this particular work, or (3) arrange, in a manner 506 | consistent with the requirements of this License, to extend the patent 507 | license to downstream recipients. "Knowingly relying" means you have 508 | actual knowledge that, but for the patent license, your conveying the 509 | covered work in a country, or your recipient's use of the covered work 510 | in a country, would infringe one or more identifiable patents in that 511 | country that you have reason to believe are valid. 512 | 513 | If, pursuant to or in connection with a single transaction or 514 | arrangement, you convey, or propagate by procuring conveyance of, a 515 | covered work, and grant a patent license to some of the parties 516 | receiving the covered work authorizing them to use, propagate, modify 517 | or convey a specific copy of the covered work, then the patent license 518 | you grant is automatically extended to all recipients of the covered 519 | work and works based on it. 520 | 521 | A patent license is "discriminatory" if it does not include within 522 | the scope of its coverage, prohibits the exercise of, or is 523 | conditioned on the non-exercise of one or more of the rights that are 524 | specifically granted under this License. You may not convey a covered 525 | work if you are a party to an arrangement with a third party that is 526 | in the business of distributing software, under which you make payment 527 | to the third party based on the extent of your activity of conveying 528 | the work, and under which the third party grants, to any of the 529 | parties who would receive the covered work from you, a discriminatory 530 | patent license (a) in connection with copies of the covered work 531 | conveyed by you (or copies made from those copies), or (b) primarily 532 | for and in connection with specific products or compilations that 533 | contain the covered work, unless you entered into that arrangement, 534 | or that patent license was granted, prior to 28 March 2007. 535 | 536 | Nothing in this License shall be construed as excluding or limiting 537 | any implied license or other defenses to infringement that may 538 | otherwise be available to you under applicable patent law. 539 | 540 | 12. No Surrender of Others' Freedom. 541 | 542 | If conditions are imposed on you (whether by court order, agreement or 543 | otherwise) that contradict the conditions of this License, they do not 544 | excuse you from the conditions of this License. If you cannot convey a 545 | covered work so as to satisfy simultaneously your obligations under this 546 | License and any other pertinent obligations, then as a consequence you may 547 | not convey it at all. For example, if you agree to terms that obligate you 548 | to collect a royalty for further conveying from those to whom you convey 549 | the Program, the only way you could satisfy both those terms and this 550 | License would be to refrain entirely from conveying the Program. 551 | 552 | 13. Use with the GNU Affero General Public License. 553 | 554 | Notwithstanding any other provision of this License, you have 555 | permission to link or combine any covered work with a work licensed 556 | under version 3 of the GNU Affero General Public License into a single 557 | combined work, and to convey the resulting work. The terms of this 558 | License will continue to apply to the part which is the covered work, 559 | but the special requirements of the GNU Affero General Public License, 560 | section 13, concerning interaction through a network will apply to the 561 | combination as such. 562 | 563 | 14. Revised Versions of this License. 564 | 565 | The Free Software Foundation may publish revised and/or new versions of 566 | the GNU General Public License from time to time. Such new versions will 567 | be similar in spirit to the present version, but may differ in detail to 568 | address new problems or concerns. 569 | 570 | Each version is given a distinguishing version number. If the 571 | Program specifies that a certain numbered version of the GNU General 572 | Public License "or any later version" applies to it, you have the 573 | option of following the terms and conditions either of that numbered 574 | version or of any later version published by the Free Software 575 | Foundation. If the Program does not specify a version number of the 576 | GNU General Public License, you may choose any version ever published 577 | by the Free Software Foundation. 578 | 579 | If the Program specifies that a proxy can decide which future 580 | versions of the GNU General Public License can be used, that proxy's 581 | public statement of acceptance of a version permanently authorizes you 582 | to choose that version for the Program. 583 | 584 | Later license versions may give you additional or different 585 | permissions. However, no additional obligations are imposed on any 586 | author or copyright holder as a result of your choosing to follow a 587 | later version. 588 | 589 | 15. Disclaimer of Warranty. 590 | 591 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 592 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 593 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 594 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 595 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 596 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 597 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 598 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 599 | 600 | 16. Limitation of Liability. 601 | 602 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 603 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 604 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 605 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 606 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 607 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 608 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 609 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 610 | SUCH DAMAGES. 611 | 612 | 17. Interpretation of Sections 15 and 16. 613 | 614 | If the disclaimer of warranty and limitation of liability provided 615 | above cannot be given local legal effect according to their terms, 616 | reviewing courts shall apply local law that most closely approximates 617 | an absolute waiver of all civil liability in connection with the 618 | Program, unless a warranty or assumption of liability accompanies a 619 | copy of the Program in return for a fee. 620 | 621 | END OF TERMS AND CONDITIONS 622 | 623 | How to Apply These Terms to Your New Programs 624 | 625 | If you develop a new program, and you want it to be of the greatest 626 | possible use to the public, the best way to achieve this is to make it 627 | free software which everyone can redistribute and change under these terms. 628 | 629 | To do so, attach the following notices to the program. It is safest 630 | to attach them to the start of each source file to most effectively 631 | state the exclusion of warranty; and each file should have at least 632 | the "copyright" line and a pointer to where the full notice is found. 633 | 634 | {one line to give the program's name and a brief idea of what it does.} 635 | Copyright (C) {year} {name of author} 636 | 637 | This program is free software: you can redistribute it and/or modify 638 | it under the terms of the GNU General Public License as published by 639 | the Free Software Foundation, either version 3 of the License, or 640 | (at your option) any later version. 641 | 642 | This program is distributed in the hope that it will be useful, 643 | but WITHOUT ANY WARRANTY; without even the implied warranty of 644 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 645 | GNU General Public License for more details. 646 | 647 | You should have received a copy of the GNU General Public License 648 | along with this program. If not, see . 649 | 650 | Also add information on how to contact you by electronic and paper mail. 651 | 652 | If the program does terminal interaction, make it output a short 653 | notice like this when it starts in an interactive mode: 654 | 655 | {project} Copyright (C) {year} {fullname} 656 | This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 657 | This is free software, and you are welcome to redistribute it 658 | under certain conditions; type `show c' for details. 659 | 660 | The hypothetical commands `show w' and `show c' should show the appropriate 661 | parts of the General Public License. Of course, your program's commands 662 | might be different; for a GUI interface, you would use an "about box". 663 | 664 | You should also get your employer (if you work as a programmer) or school, 665 | if any, to sign a "copyright disclaimer" for the program, if necessary. 666 | For more information on this, and how to apply and follow the GNU GPL, see 667 | . 668 | 669 | The GNU General Public License does not permit incorporating your program 670 | into proprietary programs. If your program is a subroutine library, you 671 | may consider it more useful to permit linking proprietary applications with 672 | the library. If this is what you want to do, use the GNU Lesser General 673 | Public License instead of this License. But first, please read 674 | . 675 | -------------------------------------------------------------------------------- /README.org: -------------------------------------------------------------------------------- 1 | #+TITLE: Plaid2Text Documentation 2 | #+HTML_HEAD_EXTRA: 3 | 4 | * Table of Contents :TOC: 5 | - [[#synopsis][Synopsis]] 6 | - [[#contributions][Contributions]] 7 | - [[#requirements][Requirements]] 8 | - [[#installation][Installation]] 9 | - [[#plaid][Plaid]] 10 | - [[#plaid2text][Plaid2Text]] 11 | - [[#creating-a-plaid-account][Creating a Plaid Account]] 12 | - [[#arguments-summary][Arguments Summary]] 13 | - [[#options-summary][Options Summary]] 14 | - [[#options][Options]] 15 | - [[#example-uses][Example Uses]] 16 | - [[#processing-a-transaction][Processing a Transaction]] 17 | - [[#configuration-files][Configuration Files]] 18 | - [[#main-configuration-file][Main Configuration File]] 19 | - [[#template-file][Template File]] 20 | - [[#brief-field-description-only-of-main-use-fields][Brief Field Description (only of main use fields)]] 21 | - [[#defaults][Defaults]] 22 | - [[#beancount][beancount]] 23 | - [[#ledger][ledger]] 24 | - [[#headers-file][Headers File]] 25 | - [[#mappings-file][Mappings File]] 26 | - [[#fields][Fields]] 27 | - [[#important-point][Important Point]] 28 | - [[#sample-mappings-file][Sample Mappings File]] 29 | - [[#workflow][Workflow]] 30 | - [[#download-new-transactions][Download New Transactions]] 31 | - [[#export-new-transactions][Export New Transactions]] 32 | - [[#copy-transactions][Copy Transactions]] 33 | - [[#disclaimer][DISCLAIMER]] 34 | - [[#license][License]] 35 | 36 | * Synopsis 37 | The purpose of this python script is to bring those of us who chose to use 38 | =command line accounting=, some of the benefits of more conventional accounting 39 | programs, namely the ability to pull our transaction information from supported 40 | institutions via automated means and format them into our preferred text syntax. 41 | Currently, this program supports [[http://ledger-cli.org/][Ledger]] and [[http://furius.ca/beancount/][Beancount]] syntax exports. 42 | 43 | To download transactions, we use [[http://www.plaid.com][Plaid]]. This program will help you setup your 44 | accounts and download transactions from the Plaid API. I have tried to make this 45 | as simple as possible to setup. 46 | 47 | Also, once downloaded, your transactions will be stored in a Mongo database. It 48 | is actually the transactions pulled from the database that we run though our 49 | syntax renderers. This is required to help keep track of which transaction we 50 | have already processed (as well as have records to reconstruct our files should 51 | the need arise). 52 | 53 | The main inspiration for the workings of the export part of the script came from 54 | the excellent [[https://github.com/quentinsf/icsv2ledger][icsv2ledger]]. I borrowed heavily from Quentin's excellent script in 55 | making this tool, and in some places I have shamelessly stolen his code altogether. 56 | 57 | * Contributions 58 | Feedback and contributions are encouraged. I hope that this will pick up some 59 | traction in the community, and in the end we can all have a rock-solid program 60 | to help us with our accounting. 61 | 62 | * Requirements 63 | - Python => 3.5 64 | * PyMango => 0.1.1 65 | * prompt_toolkit => 0.57 66 | * plaid-python-legacy => 1.3.0 67 | - ledger-cli => 3 (if using ledger syntax) 68 | - Beancount => 2.0 (if using beancount syntax) 69 | - MongoDB => 3.2.3 70 | 71 | I have only tested this on Linux. I have no desire to run this on Windows, but 72 | feel free to give it a shot, it may work. The same goes for Mac, although 73 | sometime in the future I may test on OSX. 74 | 75 | *Note*: python dependencies must be installed prior to running the script. All 76 | can be installed from =pip= or of course whatever means you wish to use. 77 | 78 | * Installation 79 | This program is setup as a python package, and can be installed using your 80 | preferred tools, but this document will only cover using =pip=. 81 | 82 | First, clone the git repo: 83 | =git clone git@github.com:madhat2r/plaid2text.git= 84 | 85 | or if preferring HTTPS: 86 | =git clone https://github.com/madhat2r/plaid2text.git= 87 | 88 | Then change into folder: 89 | =cd plaid2text= 90 | 91 | Then use =pip= to install: 92 | =pip install .= 93 | 94 | If you plan on making modifications to the program, then you may want to install 95 | it as editable. (this is my preferred way) 96 | =pip install -e .= 97 | 98 | * Plaid 99 | Plaid is an API used in building web-based financial applications. It provides 100 | access to our transactions at a number of institutions. To get started with this 101 | program, you must sign up for a plaid account. Once you have done that Plaid 102 | will issue you some developer keys to use with their API. 103 | 104 | The keys we are interested in are: 105 | - client_id :: this is your developer ID 106 | - secret :: this is your authentication token 107 | 108 | Once you have obtained your keys then use =plaid2text= to create your 109 | configuration file and save your keys into it. You can do this by simply 110 | invoking plaid2text without arguments. =plaid2text= will prompt you for your 111 | keys and store them in your config file. 112 | 113 | A note about Plaid. Plaid is a paid service, but developers have access to the 114 | developer API without paying. The developer API has all the features of the 115 | production API. I have been using this for a few months now on my 6 accounts and 116 | everything is still working just fine. I did contact them about what the cost 117 | would be (and told them my use case), and was informed that a paid version comes 118 | to 0.25 USD per account, per month. That is still a heck-of-a-better deal than 119 | Quickbooks online in my opinion. I can get my 6 accounts for 1.50 USD per month, 120 | but like I mentioned, I have yet run into any caps on my developer account, so 121 | that may be all I ever use. 122 | 123 | * Plaid2Text 124 | 125 | In order to use =plaid2text= you must have already followed the instructions in 126 | the Plaid section. Once you have your initial config in place, then let's get 127 | started in creating your first account. 128 | 129 | ** Creating a Plaid Account 130 | In order to get transactions from Plaid, you must create an account. In order to 131 | create an account, you must authenticate yourself to your institution via your 132 | username and password, and also most institutions require some form of multi 133 | factor authentication, usually in the form of security questions, or codes sent 134 | to registered phone/email for the account. 135 | 136 | The =--create-account= flag will create a new account using the plaid-account argument as the new nickname. This semi-automate the process of creating and authenticating an account with your instituion for your Plaid account. Once you run =plaid2text accountName --create-account= you will be promted to open an html file located in your configuration folder (default is =\~/.config/plaid2text/auth.html=). Click on the button labeled =Open Link - Institution Select= and you will be prompted to authenticate with the instituion. Once you do, you will see your public_token displayed on the page. Enter that back into =plaid2text=. 137 | 138 | =plaid2text= will then display a list of accounts associated with that institution and their corresponding account_id's. Paste the desired account_id and your account is ready to be used with =plaid2text=. 139 | 140 | Note: wait at least 15 minutes before the first download of your transactions, 141 | this give Plaid time to collect the information from your institution. Plaid 142 | says it will have them within 240 seconds, but I think it's better to give it 143 | time. 144 | 145 | Also, different institutions keep your historical data for different 146 | lengths of time. 147 | 148 | * Arguments Summary 149 | 150 | #+BEGIN_SRC 151 | plaid_account: (mandantory) this is the nickname you assigned when creating account 152 | outfile: output filename or stdout in your chosen snytax (ledger,beancount) 153 | #+END_SRC 154 | 155 | *Note*: the outfile will be _overwritten_ each time this is run so be careful 156 | that you do not erase your current journal file, or any other file of importance. 157 | 158 | * Options Summary 159 | A lot of these options also have an equivalent setting in the config file 160 | (=~/.config/plaid2text/config=). Where this happens, the config file settings 161 | will be underscored versions of the command line long options: =--mappings-file= 162 | would become =mappings_file=. 163 | 164 | Also, note that when there are both config setting and command line options, the 165 | command line options take precedence over config file settings. 166 | 167 | #+BEGIN_SRC 168 | --accounts-file FILE file which holds a list of account names (LEDGER ONLY) 169 | (default : ~/.config/plaid2text/accounts) 170 | --all-transactions pull all transactions even those who have been 171 | previously marked as processed (default: False) 172 | --clear-screen, -C clear screen for every transaction (default: False) 173 | --cleared-character {*,!} 174 | character to clear a transaction (default: *) 175 | --create-account Create a new Plaid account using the plaid-account 176 | argument as the new nickname (Example: chase_savings) 177 | --currency STR the currency of amounts (default: USD ) 178 | --default-expense STR 179 | expense account used as default destination (default: 180 | Expenses:Unknown) 181 | --download-transactions, -d 182 | download transactions into Mongo for given plaid 183 | account 184 | --from-date STR specify a the starting date for transactions to be 185 | pulled; use in conjunction with --to-date to specify 186 | rangeDate format: YYYY-MM-DD 187 | --headers-file FILE file which contains contents to be written to the top 188 | of the output file (default: ~/.config/plaid2text/headers) 189 | --journal-file FILE, -j FILE 190 | journal file where to read payees/accounts Tip: you 191 | can use includes to pull in your other journal files 192 | (default journal file: ~/.config/plaid2text/journal) 193 | --mapping-file FILE file which holds the mappings (default: ~/.config/plaid2text/mapping) 194 | --dbtype {mongodb,sqlite} 195 | The database type to use for storing transactions. 196 | --mongo-db STR The name of the Mongo database (default: plaid2text) 197 | --mongo-db-uri STR The URI for your MongoDB in the MongoDB URI format 198 | (default: mongodb://localhost:27017) 199 | --sqlite-db FILE The path to the SQLite DB to use, if --dbtype is sqlite 200 | --no-mark-processed, -n 201 | Do not mark pulled transactions. When given, the 202 | pulled transactions will still be listed as new 203 | transactions upon the next run. (default: False) 204 | --output-date-format STR 205 | date format for output file (default: YYYY/MM/DD) 206 | --output-format {beancount,ledger}, -o {beancount,ledger} 207 | what format to use for the output file. (default 208 | format: beancount) 209 | --posting-account STR, -a STR 210 | posting account used as source (default: Assets:Bank:Checking) 211 | --quiet, -q do not prompt if account can be deduced from mappings 212 | (default: False) 213 | --tags, -t prompt for transaction tags (default: False) 214 | --template-file FILE file which holds the template (default: ~/.config/plaid2text/template) 215 | --to-date STR specify the ending date for transactions to be pulled; 216 | use in conjunction with --from-date to specify 217 | rangeDate format: YYYY-MM-DD 218 | -h, --help show this help message and exit 219 | #+END_SRC 220 | 221 | ** Options 222 | 223 | ~--accounts-file~ 224 | is a file that you can store predefined account definitions for Ledger in 225 | the form of =account Expenses:Unknown=. This file is parsed for the account 226 | names and all lines that do not start with *account* will be ignored. 227 | 228 | This is *LEDGER* specific setting. 229 | 230 | ~--all-transactions~ 231 | will pull all transactions regardless if they are marked as already pulled. 232 | By default only transactions that have not been pulled to text are returned. 233 | 234 | ~--clear-screen, -C~ 235 | clears the screen before every transaction prompt. Default is ~False~. 236 | 237 | ~--cleared-character {*,!}~ 238 | is the character mark a transactions as cleared or not. Default is =*= 239 | 240 | ~--create-account~ 241 | is used to create a new account. See creating account section above for more. 242 | 243 | ~--currency STR~ 244 | is the currency used for transactions. Default is =USD=. 245 | 246 | ~--default-expense STR~ 247 | is the default account for which to post transactions to. Default 248 | =Expenses:Unknown= 249 | 250 | ~--download-transactions, -d~ 251 | fetches new transactions from Plaid into Mongo for given account. 252 | 253 | Use: =plaid2text acct_nickname -d= 254 | 255 | ~--from-date STR~ 256 | specify a the starting date for transactions to be pulled. 257 | 258 | Use in conjunction with ~--to-date~ to specify range 259 | 260 | Date format: =YYYY-MM-DD= or =YYYY/MM/DD= 261 | 262 | ~--headers-file FILE~ 263 | file which contains contents to be written to the top of the output file. For 264 | example, I store my beancount files as OrgMode files, so I have my headers file 265 | setup to insert instructions at the top for =Emacs=, to help ease my editing of 266 | them once they are exported to text. And also I include my main beancount file 267 | which has all my accounts listed, this also allows for easy running of 268 | =bean-check= to verify the newly exported file. 269 | 270 | #+BEGIN_SRC 271 | ;; -*- mode: org; mode: beancount; -*- 272 | include "/path/to/somewhere/main.beancount" 273 | #+END_SRC 274 | 275 | Default: =~/.config/plaid2text/headers= 276 | 277 | ~--journal-file FILE, -j FILE~ 278 | journal file where to read payees/accounts. This could be your main ledger file 279 | or your main beancount file. 280 | 281 | Tip: you can use includes to pull in your other journal files 282 | 283 | Default journal file: =~/.config/plaid2text/journal= 284 | 285 | ~--mapping-file FILE~ 286 | file which holds the mappings for matching transactions to accounts/payees as 287 | well as some default tags, if you want. 288 | 289 | You can have a separate mappings file per account. 290 | 291 | default: =~/.config/plaid2text/mapping= 292 | 293 | ~--mongo-db STR~ 294 | name of the Mongo database that stores downloaded transactions. 295 | 296 | Default: ~plaid2text~ 297 | 298 | ~--mongo-db-uri STR~ 299 | The URI for your MongoDB in the MongoDB URI format 300 | 301 | Default: ~mongodb://localhost:27017~ 302 | 303 | ~--no-mark-processed, -n~ 304 | will not mark pulled transactions as pulled. When passed, the pulled transactions will still be listed as new 305 | transactions upon the next run. 306 | 307 | Default: ~False~ 308 | 309 | ~--output-date-format STR~ 310 | date format for output file 311 | 312 | Default: ~YYYY/MM/DD~ 313 | 314 | ~--output-format {beancount,ledger}, -o {beancount,ledger}~ 315 | what syntax to use for the output file. 316 | 317 | Default output format: beancount 318 | 319 | ~--posting-account STR, -a STR~ 320 | posting account used as source 321 | 322 | Default: ~Assets:Bank:Checking~ 323 | 324 | ~--quiet, -q~ 325 | do not prompt if account can be deduced from mappings 326 | 327 | Default: ~False~ 328 | 329 | ~--tags, -t~ 330 | causes the program to prompt for transaction tags 331 | 332 | Default: ~False~ 333 | 334 | ~--template-file FILE~ 335 | file which holds the text template used in the output file for formatting transactions. 336 | 337 | Default: =~/.config/plaid2text/template= 338 | 339 | ~--to-date STR~ 340 | specify the ending date for transactions to be pulled. 341 | 342 | use in conjunction with ~--from-date~ to specify range 343 | 344 | Date format: ~YYYY-MM-DD~ or ~YYYY/MM/DD~ 345 | 346 | * Example Uses 347 | 348 | The following will set up a new account with nickname =chase_checking= 349 | 350 | ~plaid2text chase_checking --create-account~ 351 | 352 | The following will download all new transactions for the account 353 | =chase_checking=. 354 | 355 | *NOTE*: when downloading for the first time, be sure to wait at least 15min 356 | after setting up the account. This gives Plaid time to pull your 357 | transactions from the institution. 358 | 359 | ~plaid2text chase_checking --downlad-transactions~ 360 | 361 | The following will pull all new transactions for account 362 | =chase_checking= and output them to =/tmp/onetime.ldg= Ledger syntax 363 | after prompting you for the correct information for every transaction and 364 | marking all pulled transaction in the database as pulled. 365 | 366 | ~plaid2text chase_checking /tmp/onetime.ldg --output-format ledger~ 367 | 368 | The following will pull *all* transactions starting from the given date for the 369 | =chase_checking= account and will not mark them as pulled in the database, and 370 | will output beancount syntax to stdout. 371 | 372 | ~plaid2text chase_checking --all-transactions --from-date 2015/04/15 --no-mark-processed~ 373 | 374 | * Processing a Transaction 375 | When you start processing transactions, you will be presented with 376 | several prompts related to the current transaction. These prompts will be to 377 | get the associated account, the payee, and optionally tags. If you have a 378 | mappings file, provided a journal file, or have already processed a few 379 | transactions, then ~TAB~ completion is available at all these prompts. 380 | 381 | During your first run, when your mappings file has not yet been established, you 382 | will have to manually (via prompts) establish the correct accounts and payees. 383 | But once you have done so, your mappings file will have the correct information 384 | for transactions in the future, and given that most of us are creatures of habit 385 | and make purchases from the same places, you will only occasionally have to 386 | account for new entries. 387 | 388 | Now let's walk through a transaction for you can get an idea of what to expect. 389 | Keeping with out sample account =chase_checking=, we will pull the latest 390 | transactions, and also prompt for tags (=--tags=) and suppress prompting for 391 | known transactions via our mappings (=--quiet=), we will also be using 392 | =beancount= output syntax (=--output-format=). 393 | 394 | ~plaid2text chase_checking /tmp/onetime.bnc --quiet --tags --output-format beancount~ 395 | 396 | When the above command is run, you will be presented with a prompt for the first 397 | non-matched transaction. The first prompt is for the payee. You will notice that 398 | the default answers are in =[]=, so if you just hit enter, that will be the 399 | value. When looking at the transaction prompt, you will see it starts with a 400 | date followed by the name that Plaid assigns this transaction, in this case 401 | Plaid got it correct, this will not always be the case. The next area shows the 402 | amount of the transaction. 403 | 404 | [[file:img/netflix_payee.png]] 405 | 406 | Following the payee prompt is the "Account" prompt. Enter the correct associated 407 | account, then hit enter. 408 | 409 | [[file:img/netflix_account.png]] 410 | 411 | Then we are prompted for tags (because we passed ~--tags~). Tags work a little 412 | differently, you will be prompted over and over for tags until you just hit 413 | enter without typing another value. If you make a mistake in entering your tag, 414 | you may prefix the tag with =-= (minus) to remove it. For instance say you 415 | accidentally typed =mvoie= and hit enter, when the prompt comes back you see 416 | your mistake in the default area and want to correct it. So now you type 417 | =-mvoie= and hit enter, and you will notice that the tag has been removed. 418 | 419 | [[file:img/netflix_tags.png]] 420 | 421 | When you hit enter the final time on tags, the program will move on to the next 422 | transaction needing your input. 423 | 424 | Again, all of these prompts use =TAB= completion, and the more information you 425 | give the program, via your config files, the better the completion becomes. 426 | 427 | 428 | * Configuration Files 429 | ** Main Configuration File 430 | This is an example config file that has an account setup that is nicknamed 431 | =chase_checking=. You will notice some settings that are obfuscated with xxx, 432 | these are created when setting up accounts, and are not entered manually. 433 | 434 | #+BEGIN_SRC 435 | [DEFAULT] 436 | posting_account = Assets:Bank:Checking 437 | default_expense = Expenses:Unknown 438 | encoding = utf-8 439 | currency = USD 440 | mongo_db = plaid2text 441 | mongo_db_uri: mongodb://localhost:27017 442 | quiet = False 443 | tags = False 444 | output_date_format = %%Y/%%m/%%d 445 | clear_screen = False 446 | cleared_character = * 447 | output_format = beancount 448 | 449 | [PLAID] 450 | client_id = xxxxxxxa66710877xxxxxxxx 451 | secret = xxxxxxxxx8c9a0cd27xxxxxxxxxxxx 452 | 453 | [chase_checking] 454 | access_token = access-development-xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx 455 | account = xxxxxxxxxxxxxPzJ3nAkFxxxxxxxxxxxxxxxx 456 | item_id = xxxxxxxxxxxxxxxcc4f53xxxxxxxxxxxxxxxxx 457 | currency = USD 458 | posting_account = Assets:Bank:Chase:Checking 459 | mapping_file = ~/.config/plaid2text/chase_checking/mapping_bc 460 | headers_file = ~/.config/plaid2text/chase_checking/headers_bc 461 | accounts_file = ~/somewhere/main.beancount 462 | journal_file = ~/somewhere/beancount/main.beancount 463 | template_file = ~/.config/plaid2text/chase_checking/template_bc 464 | #+END_SRC 465 | 466 | ** Template File 467 | The template file is what transforms your transactions into the desired text 468 | based accounting syntax. You have access to all the fields that plaid returns to 469 | use in your templates. But be aware that not all fields are returned with every 470 | transaction, and you might have to modify the source to handle this, should you 471 | choose to use them in your template. Below is a list of all fields available. 472 | The =A= column indicates if field is always available. 473 | 474 | | Field | Types | A | 475 | |-------------------------------+---------+---| 476 | | _account | String | y | 477 | | _id | String | y | 478 | | amount | Number | y | 479 | | name | String | y | 480 | | date | Date | y | 481 | | meta | Object | y | 482 | | meta.location | Object | y | 483 | | pending | Boolean | y | 484 | | score | Object | y | 485 | | score.location | Object | y | 486 | | score.name | Number | y | 487 | | type | Object | y | 488 | | type.primary | String | y | 489 | | meta.location.state | String | n | 490 | | score.location.state | Number | n | 491 | | category | Array | n | 492 | | category_id | String | n | 493 | | meta.location.city | String | n | 494 | | score.location.city | Number | n | 495 | | meta.location.coordinates | Object | n | 496 | | meta.location.coordinates.lat | Number | n | 497 | | meta.location.coordinates.lon | Number | n | 498 | | score.location.address | Number | n | 499 | | score.location.zip | Number | n | 500 | | meta.location.address | String | n | 501 | | meta.location.zip | String | n | 502 | | meta.location.store_number | String | n | 503 | | meta.payment_processor | String | n | 504 | | meta.ppd_id | String | n | 505 | | _pendingTransaction | String | n | 506 | | meta.reference_number | String | n | 507 | | meta.payee | String | n | 508 | | meta.payment_method | String | n | 509 | |-------------------------------+---------+---| 510 | 511 | In addition to the above fields =plaid2text= also provides the following: 512 | 513 | | Field | type | 514 | |---------------------+--------| 515 | | posting_account | String | 516 | | associated_accounts | String | 517 | | payee | String | 518 | | tags | String | 519 | |---------------------+--------| 520 | 521 | *** Brief Field Description (only of main use fields) 522 | - _account :: the Plaid account ID 523 | - _id :: the Plaid transaction ID, Also the MongoDB ~_id~ 524 | - name :: the Plaid name for the transaction. (i.e. Best Buy) 525 | - amount :: the amount of debit/credit. This is a *signed* number. 526 | - date :: the date the transaction occurred 527 | - posting_account :: the account transaction are posted to 528 | - associated_account :: the expense or other account attributed to the transaction 529 | - payee :: the payee for the transaction 530 | - tags :: the given tags for the transaction in a string 531 | - beancount :: format: '#tag1 #tag2 #etc' 532 | - ledger :: format: ':tag1:tag2:etc:' 533 | 534 | *NOTE:* The ~tags~ field is prefixed with a space, when tags are present, this 535 | allows us to loose the trailing space that would otherwise exist in situations 536 | where there were no tags, and the configured template supports them. 537 | 538 | Example of trailing space template: 539 | #+BEGIN_SRC 540 | {transaction_date} {cleared_character} "{payee}" "" {tags} 541 | #+END_SRC 542 | Using the above template would result in a trailing space when no tags are present. 543 | 544 | Example proper template: 545 | #+BEGIN_SRC 546 | {transaction_date} {cleared_character} "{payee}" ""{tags} 547 | #+END_SRC 548 | This template will add prefix the ~tags~ with a space only if they are present, 549 | otherwise it returns an empty string. This line will not have a trailing space. 550 | 551 | *** Defaults 552 | **** beancount 553 | #+BEGIN_SRC 554 | {transaction_date} {cleared_character} "{payee}" ""{tags} 555 | plaid_name: "{name}" 556 | plaid_id: "{_id}" 557 | {associated_account:<60} {amount} {currency} 558 | {posting_account} 559 | #+END_SRC 560 | 561 | **** ledger 562 | 563 | #+BEGIN_SRC 564 | {transaction_date} {cleared_character} {payee}{tags} 565 | ; plaid_name: {name} 566 | ; _id: {_id} 567 | {associated_account:<60} {currency} {amount} 568 | {posting_account:<60} 569 | #+END_SRC 570 | 571 | ** Headers File 572 | 573 | The headers file is used to add some text to the top of the output file. This 574 | can be anything you like. I use mine for adding some header info for =Emacs= to 575 | read for it sets the correct mode for me when I edit the file. 576 | 577 | I also use the ~include~ directive to pull in my main file, to aide in running =bean-check=. 578 | 579 | ** Mappings File 580 | The mappings file is simply a =CSV= formatted file, that contains four fields. When 581 | exporting transactions, this file will try to establish the proper accounts and 582 | payees for each transaction based on the fields in the file. It also handles 583 | adding some default tags. 584 | 585 | This file is created for you, if you do not have one defined in the settings. 586 | Also, it is appended to every time you are exporting transactions with the new 587 | matches, that way next time you export you will not have to enter the 588 | information again if you use =--quiet=. 589 | 590 | *** Fields 591 | 1. text to match against the Plaid =name= field. This can be either plain text 592 | or a regex. If the field starts and ends with =/= it is assumed to be a 593 | regex. Note: all the regexes will be matched /case insensitive/. 594 | 2. the name you wish to use for the =payee= 595 | 3. the associated expense or other account (i.e. ~Expenses:Unknown~) 596 | 4. tags to be used for this transaction. This should be in the form of a string. 597 | For ledger the format would be: ~:tag1:tag2:etc:~ and for beancount: ~#tag1 #tag2 #etc~ 598 | 599 | *** Important Point 600 | The matching algorithm will always use the latest match when processing entries. 601 | So if for example you have a regex setup that matches //best buy// at the top of 602 | the mappings file and another that has //buy// later in the file, the last match 603 | wins. 604 | 605 | *** Sample Mappings File 606 | Some of the listings will contain ledger formatted tags while other will be 607 | beancount, you of course will only have the type that you need, do not mix them. 608 | 609 | #+BEGIN_SRC 610 | /Amazon/,"Amazon",Expenses:Unknown:Amazon, #sort-out 611 | /PAYPAL INST XFER/,"PayPal",Expenses:Unknown:PayPal, :sort-out: 612 | /.*NETFLIX.*/,"Netflix",Expenses:Bills:Subscriptions:Netflix 613 | /.*DROPBOX.*/,"Dropbox",Expenses:Bills:Subscriptions:Dropbox 614 | /Amazon Video/,"Amazon Video",Expenses:Entertainment:Movies 615 | The Doughnut Palace,"The Doughnut Palace",Expenses:Food:FastFood 616 | 54th Street,"54th Street",Expenses:Food:Restaurant 617 | BJ'S RESTAURANTS,"BJ's Restaurant",Expenses:Food:Restaurant 618 | #+END_SRC 619 | 620 | Also notice the sorting of the entries so that =Amazon Video= gets categorized 621 | properly. If it were above the =Amazon= entry, it would use the setting from 622 | there instead, as the last entry always wins. 623 | 624 | 625 | * Workflow 626 | In this section I will just describe my basic workflow to demonstrate how I use 627 | this tool. Going forwards assumes you have already established your plaid setup 628 | as well as at least one account. I will continue to demonstrate with the example 629 | account =chase_checking= to keep things consistent. 630 | 631 | ** Download New Transactions 632 | When I get ready to work on my books, I start by downloading the newest 633 | transactions for the account I am working on. 634 | 635 | ~plaid2text chase_checking -d~ 636 | 637 | This will download all the newest transactions from my accounts into the MongoDB. 638 | 639 | You can of course setup a cron job to do this nightly, but I find it fits 640 | into my workflow just doing it manually. 641 | 642 | ** Export New Transactions 643 | I export new transaction (all the ones that haven't previously been pulled), into 644 | a temporary file, where I can do some manual checking and editing. 645 | 646 | ~plaid2text chase_checking /tmp/onetime.beancount --quiet~ 647 | 648 | Using the =--quiet= switch, the program will only prompt me for information on 649 | the transactions that it cannot deduce based on the mappings file. You can of 650 | course leave that switch off if you want to be able to change the defaults from 651 | the mapping file. 652 | 653 | Also, if you want to do a test run, without marking the transactions as pulled 654 | use the =--no-mark-pulled= switch. 655 | 656 | *IMPORTANT* I want to stress that the outfile is *OVERWRITTEN* or created 657 | every time this command is run. So be careful. :) 658 | 659 | ** Copy Transactions 660 | When I am satisfied that all is well with my temp file. I copy the new entries 661 | into my actual journal file. 662 | 663 | * DISCLAIMER 664 | This should be considered /*beta*/ version code. I have released it hoping that it 665 | will be of benefit to others in a similar situation as me. This version of the 666 | code is really hacked together and in need of serious refactoring, and will most 667 | likely contain bugs. I have had this working for myself for a few weeks, and 668 | have found it stable and usable. But I do caution you, to use at your own risk. 669 | 670 | ** License 671 | This program is free software; you can redistribute it and/or modify 672 | it under the terms of the GNU General Public License as published by 673 | the Free Software Foundation, either version 3 of the License, or 674 | (at your option) any later version. 675 | 676 | This program is distributed in the hope that it will be useful, 677 | but *WITHOUT ANY WARRANTY*; without even the implied warranty of 678 | *MERCHANTABILITY* or *FITNESS FOR A PARTICULAR PURPOSE*. See the 679 | GNU General Public License for more details. 680 | 681 | You can obtain a copy of the license here: [[http://www.gnu.org/licenses/][GNU General Public License]] 682 | -------------------------------------------------------------------------------- /bin/plaid2text: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | from plaid2text.plaid2text import main;main() 3 | -------------------------------------------------------------------------------- /img/netflix_account.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/madhat2r/plaid2text/49f5746466b6240f7520543bd656975585de1b8b/img/netflix_account.png -------------------------------------------------------------------------------- /img/netflix_payee.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/madhat2r/plaid2text/49f5746466b6240f7520543bd656975585de1b8b/img/netflix_payee.png -------------------------------------------------------------------------------- /img/netflix_tags.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/madhat2r/plaid2text/49f5746466b6240f7520543bd656975585de1b8b/img/netflix_tags.png -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | attrs==19.3.0 2 | beancount==2.2.1 3 | beautifulsoup4==4.9.1 4 | bottle==0.12.18 5 | cachetools==4.1.1 6 | certifi==2020.6.20 7 | chardet==3.0.4 8 | google-api-core==1.21.0 9 | google-api-python-client==1.9.3 10 | google-auth==1.18.0 11 | google-auth-httplib2==0.0.3 12 | googleapis-common-protos==1.52.0 13 | httplib2==0.18.1 14 | idna==2.10 15 | importlib-metadata==1.7.0 16 | lxml==4.5.1 17 | more-itertools==8.4.0 18 | packaging==20.4 19 | plaid-python==7.1.0 20 | pluggy==0.13.1 21 | ply==3.11 22 | prompt-toolkit==3.0.5 23 | protobuf==3.12.2 24 | py==1.9.0 25 | pyasn1==0.4.8 26 | pyasn1-modules==0.2.8 27 | pymongo==3.10.1 28 | pyparsing==2.4.7 29 | pytest==5.4.3 30 | python-dateutil==2.8.1 31 | python-magic==0.4.18 32 | pytz==2020.1 33 | requests==2.24.0 34 | rsa==4.6 35 | six==1.15.0 36 | soupsieve==2.0.1 37 | uritemplate==3.0.1 38 | urllib3==1.25.9 39 | wcwidth==0.2.5 40 | zipp==3.1.0 41 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | Install script for plaid2text. 4 | """ 5 | __author__ = "Micah Duke " 6 | 7 | import os 8 | from os import path 9 | import runpy 10 | import sys 11 | import warnings 12 | 13 | 14 | # Check if the version is sufficient. 15 | if sys.version_info[:2] < (3,5): 16 | raise SystemExit("ERROR: Insufficient Python version; you need v3.5 or higher.") 17 | 18 | 19 | # Import setup(). 20 | setup_extra_kwargs = {} 21 | try: 22 | from setuptools import setup, Extension 23 | setup_extra_kwargs.update(install_requires = [ 24 | # used for working with MongoDB 25 | 'pymongo==3.10.1', 26 | 27 | # used in console prompts/autocompletion 28 | 'prompt-toolkit==3.0.5', 29 | 30 | # the heart of the program 31 | 'plaid-python==7.1.0', 32 | 'beancount==2.2.1', 33 | ]) 34 | 35 | except ImportError: 36 | warnings.warn("Setuptools not installed; falling back on distutils. " 37 | "You will have to install dependencies explicitly.") 38 | from distutils.core import setup, Extension 39 | 40 | 41 | # Explicitly list the scripts to install. 42 | install_scripts = [path.join('bin', x) for x in """ 43 | plaid2text 44 | """.split() if x and not x.startswith('#')] 45 | 46 | 47 | # Create a setup. 48 | setup( 49 | name="plaid2text", 50 | version='0.1.2', 51 | description="Plaid API to ledger/beancount download/conversion", 52 | 53 | long_description= 54 | """ 55 | A program to setup Plaid accounts, and download account transactions then 56 | export them to a plain text account format. Currently this programs 57 | provides exports in beancount and ledger syntax formats. 58 | """, 59 | 60 | license="GPL", 61 | author="Micah Duke", 62 | author_email="MaDhAt2r@dukefoo.com", 63 | url="https://github.com/madhat2r/plaid2text", 64 | 65 | package_dir = {'': 'src/python',}, 66 | packages = ['plaid2text'], 67 | 68 | scripts=install_scripts, 69 | # Add optional arguments that only work with some variants of setup(). 70 | **setup_extra_kwargs 71 | ) 72 | -------------------------------------------------------------------------------- /src/python/plaid2text/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | """ 4 | A program to setup Plaid accounts, and download account transactions then 5 | export them to a plain text account format. Currently this programs 6 | provides exports in beancount and ledger syntax formats. 7 | """ 8 | 9 | __author__ = "Micah Duke " 10 | # Check the version requirements. 11 | import sys 12 | if (sys.version_info.major, sys.version_info.minor) < (3, 5): 13 | raise ImportError("Python 3.5 or above is required") 14 | -------------------------------------------------------------------------------- /src/python/plaid2text/config_manager.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | 3 | from collections import OrderedDict 4 | import configparser 5 | import os 6 | import sys 7 | 8 | from plaid2text.interact import prompt, NullValidator, YesNoValidator 9 | from plaid import Client 10 | from plaid import errors as plaid_errors 11 | 12 | import json 13 | 14 | 15 | class dotdict(dict): 16 | """ 17 | Enables dict.item syntax (instead of dict['item']) 18 | See http://stackoverflow.com/questions/224026 19 | """ 20 | __getattr__ = dict.__getitem__ 21 | __setattr__ = dict.__setitem__ 22 | __delattr__ = dict.__delitem__ 23 | 24 | 25 | def get_locale_currency_symbol(): 26 | """ 27 | Get currency symbol from locale 28 | """ 29 | import locale 30 | locale.setlocale(locale.LC_ALL, '') 31 | conv = locale.localeconv() 32 | return conv['int_curr_symbol'] 33 | 34 | DEFAULT_CONFIG_DIR = os.path.expanduser('~/.config/plaid2text') 35 | 36 | CONFIG_DEFAULTS = dotdict({ 37 | # For configparser, int must be converted to str 38 | # For configparser, boolean must be set to False 39 | 'create_account': False, 40 | 'posting_account': 'Assets:Bank:Checking', 41 | 'output_format': 'beancount', 42 | 'clear_screen': False, 43 | 'cleared_character': '*', 44 | 'currency': get_locale_currency_symbol(), 45 | 'default_expense': 'Expenses:Unknown', 46 | 'encoding': 'utf-8', 47 | 'output_date_format': '%Y/%m/%d', 48 | 'quiet': False, 49 | 'tags': False, 50 | 'dbtype': 'mongodb', 51 | 'mongo_db': 'plaid2text', 52 | 'mongo_db_uri': 'mongodb://localhost:27017', 53 | 'sqlite_db': os.path.join(DEFAULT_CONFIG_DIR, 'transactions.db') 54 | }) 55 | 56 | FILE_DEFAULTS = dotdict({ 57 | 'config_file': os.path.join(DEFAULT_CONFIG_DIR, 'config'), 58 | 'accounts_file': os.path.join(DEFAULT_CONFIG_DIR, 'accounts'), 59 | 'journal_file': os.path.join(DEFAULT_CONFIG_DIR, 'journal'), 60 | 'mapping_file': os.path.join(DEFAULT_CONFIG_DIR, 'mapping'), 61 | 'headers_file': os.path.join(DEFAULT_CONFIG_DIR, 'headers'), 62 | 'template_file': os.path.join(DEFAULT_CONFIG_DIR, 'template'), 63 | 'auth_file': os.path.join(DEFAULT_CONFIG_DIR, 'auth.html')}) 64 | 65 | DEFAULT_LEDGER_TEMPLATE = """\ 66 | {transaction_date} {cleared_character} {payee} {tags} 67 | ; plaid_name: {name} 68 | ; _id: {transaction_id} 69 | {associated_account:<60} {currency} {amount} 70 | {posting_account:<60} 71 | """ 72 | 73 | DEFAULT_BEANCOUNT_TEMPLATE = """\ 74 | {transaction_date} {cleared_character} "{payee}" ""{tags} 75 | plaid_name: "{name}" 76 | plaid_id: "{transaction_id}" 77 | {associated_account:<60} {amount} {currency} 78 | {posting_account} 79 | """ 80 | 81 | 82 | def touch(fname, mode=0o666, dir_fd=None, **kwargs): 83 | """ 84 | Implementation of coreutils touch 85 | http://stackoverflow.com/a/1160227 86 | """ 87 | flags = os.O_CREAT | os.O_APPEND 88 | with os.fdopen(os.open(fname, flags=flags, mode=mode, dir_fd=dir_fd)) as f: 89 | os.utime(f.fileno() if os.utime in os.supports_fd else fname, 90 | dir_fd=None if os.supports_fd else dir_fd, **kwargs) 91 | 92 | 93 | def get_custom_file_path(nickname, file_type, create_file=False): 94 | f = os.path.join(DEFAULT_CONFIG_DIR, nickname, file_type) 95 | if create_file: 96 | if not os.path.exists(f): 97 | _create_directory_tree(f) 98 | touch(f) 99 | if file_type == 'template': 100 | with open(f, mode='w') as temp: 101 | temp.write(DEFAULT_BEANCOUNT_TEMPLATE) 102 | return f 103 | 104 | 105 | def config_exists(): 106 | if not os.path.isfile(FILE_DEFAULTS.config_file): 107 | print('No configuration file found.') 108 | create = prompt( 109 | 'Do you want to create one now [Y/n]: ', 110 | validator=YesNoValidator() 111 | ).lower() 112 | if not bool(create) or create.startswith('y'): 113 | return init_config() 114 | elif create.startswith('n'): 115 | raise Exception('No configuration file found') 116 | else: 117 | return True 118 | 119 | 120 | def _get_config_parser(): 121 | config = configparser.ConfigParser(CONFIG_DEFAULTS, interpolation=None) 122 | config.read(FILE_DEFAULTS.config_file) 123 | return config 124 | 125 | 126 | def get_config(account): 127 | config = _get_config_parser() 128 | if not config.has_section(account): 129 | print( 130 | 'Config file {0} does not contain section for account: {1}\n\n' 131 | 'To create this account: run plaid2text {1} --create-account'.format( 132 | FILE_DEFAULTS.config_file, 133 | account 134 | ), 135 | file=sys.stderr 136 | ) 137 | sys.exit(1) 138 | defaults = OrderedDict(config.items(account)) 139 | defaults['plaid_account'] = account 140 | defaults['config_file'] = FILE_DEFAULTS.config_file 141 | defaults['addons'] = OrderedDict() 142 | for f in ['template_file', 'mapping_file', 'headers_file', 'journal_file', 'accounts_file']: 143 | if f in defaults: 144 | defaults[f] = os.path.expanduser(defaults[f]) 145 | if config.has_section(account + '_addons'): 146 | for item in config.items(account + '_addons'): 147 | if item not in config.defaults().items(): 148 | defaults['addons']['addon_' + item[0]] = int(item[1]) 149 | return defaults 150 | 151 | 152 | def get_configured_accounts(): 153 | config = _get_config_parser() 154 | accts = config.sections() 155 | accts.remove('PLAID') # Remove Plaid specific 156 | return accts 157 | 158 | 159 | def account_exists(account): 160 | config = _get_config_parser() 161 | if not config.has_section(account): 162 | return False 163 | return True 164 | 165 | 166 | def get_plaid_config(): 167 | config = _get_config_parser() 168 | plaid_section = config['PLAID'] 169 | return plaid_section['client_id'], plaid_section['secret'] 170 | 171 | 172 | def write_section(section_dict): 173 | config = _get_config_parser() 174 | try: 175 | config.read_dict(section_dict) 176 | except Exception as e: 177 | raise 178 | else: 179 | with open(FILE_DEFAULTS.config_file, mode='w') as f: 180 | config.write(f) 181 | 182 | 183 | def init_config(): 184 | try: 185 | _create_directory_tree(FILE_DEFAULTS.config_file) 186 | config = configparser.ConfigParser(interpolation=None) 187 | config['PLAID'] = OrderedDict() 188 | plaid = config['PLAID'] 189 | client_id = prompt('Enter your Plaid client_id: ', validator=NullValidator()) 190 | plaid['client_id'] = client_id 191 | secret = prompt('Enter your Plaid secret: ', validator=NullValidator()) 192 | plaid['secret'] = secret 193 | except Exception as e: 194 | return False 195 | else: 196 | with open(FILE_DEFAULTS.config_file, mode='w') as f: 197 | config.write(f) 198 | return True 199 | 200 | 201 | def _create_directory_tree(filename): 202 | """ 203 | This will create the entire directory path for the config file 204 | """ 205 | os.makedirs(os.path.dirname(filename), exist_ok=True) 206 | 207 | 208 | def find_first_file(arg_file, alternatives): 209 | """Because of http://stackoverflow.com/questions/12397681, 210 | parser.add_argument(type= or action=) on a file can not be used 211 | """ 212 | found = None 213 | file_locs = [arg_file] + [alternatives] 214 | for loc in file_locs: 215 | if loc is not None and os.access(loc, os.F_OK | os.R_OK): 216 | found = loc # existing and readable 217 | break 218 | return found 219 | 220 | 221 | def create_account(account): 222 | try: 223 | _create_directory_tree(FILE_DEFAULTS.config_file) 224 | config = configparser.ConfigParser(interpolation=None) 225 | config[account] = OrderedDict() 226 | plaid = config[account] 227 | client_id, secret = get_plaid_config() 228 | # client_id = prompt('Enter your Plaid client_id: ', validator=NullValidator()) 229 | # plaid['client_id'] = client_id 230 | # secret = prompt('Enter your Plaid secret: ', validator=NullValidator()) 231 | # plaid['secret'] = secret 232 | 233 | configs = { 234 | 'user': { 235 | 'client_user_id': '123-test-user-id', 236 | }, 237 | 'products': ['transactions'], 238 | 'client_name': "Plaid Test App", 239 | 'country_codes': ['US'], 240 | 'language': 'en', 241 | } 242 | 243 | # create link token 244 | client = Client(client_id, secret, "development", suppress_warnings=True) 245 | response = client.LinkToken.create(configs) 246 | link_token = response['link_token'] 247 | 248 | generate_auth_page(link_token) 249 | print("\n\nPlease open " + FILE_DEFAULTS.auth_file + " to authenticate your account with Plaid") 250 | public_token = prompt('Enter your public_token from the auth page: ', validator=NullValidator()) 251 | # plaid['public_token'] = public_token 252 | 253 | response = client.Item.public_token.exchange(public_token) 254 | access_token = response['access_token'] 255 | plaid['access_token'] = access_token 256 | item_id = response['item_id'] 257 | plaid['item_id'] = item_id 258 | 259 | response = client.Accounts.get(access_token) 260 | 261 | accounts = response['accounts'] 262 | 263 | print("\n\nAccounts:\n") 264 | for item in accounts: 265 | print(item['name'] + ":") 266 | print(item['account_id']) 267 | account_id = prompt('\nEnter account_id of desired account: ', validator=NullValidator()) 268 | plaid['account'] = account_id 269 | 270 | except plaid_errors.ItemError as ex: 271 | print(" %s" % ex, file=sys.stderr ) 272 | sys.exit(1) 273 | else: 274 | with open(FILE_DEFAULTS.config_file, mode='a') as f: 275 | config.write(f) 276 | return True 277 | 278 | def generate_auth_page(link_token): 279 | page = """ 280 | 281 | 282 |

283 | 284 | 313 | 314 | 315 | """ 316 | 317 | f = open(FILE_DEFAULTS.auth_file, mode='w') 318 | f.write(page) 319 | f.close() 320 | 321 | 322 | if __name__ == '__main__': 323 | get_locale_currency_symbol() 324 | -------------------------------------------------------------------------------- /src/python/plaid2text/interact.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | 3 | from prompt_toolkit import prompt # NOQA: F401 4 | from prompt_toolkit.validation import ValidationError, Validator 5 | from prompt_toolkit.completion.filesystem import PathCompleter 6 | from prompt_toolkit.completion.base import Completer, Completion 7 | from six import string_types 8 | 9 | 10 | PATH_COMPLETER = PathCompleter(expanduser=True) 11 | 12 | 13 | def separator_completer(words, sep=' '): 14 | return SeparatorCompleter(words, sep=sep) 15 | 16 | 17 | class SeparatorCompleter(Completer): 18 | """ 19 | Simple autocompletion on a list of accounts. i.e. "Expenses:Unknown" 20 | 21 | :param words: List of words. 22 | :param sep: The separator to use 23 | :param ignore_case: If True, case-insensitive completion. 24 | """ 25 | def __init__(self, words, ignore_case=True, sep=" "): 26 | self.words = list(words) 27 | self.ignore_case = ignore_case 28 | assert all(isinstance(w, string_types) for w in self.words) 29 | 30 | def get_completions(self, document, complete_event): 31 | # Get word/text before cursor. 32 | text_before_cursor = document.text_before_cursor 33 | if self.ignore_case: 34 | text_before_cursor = text_before_cursor.lower() 35 | 36 | text_len = len(text_before_cursor) 37 | if text_len < 1: 38 | return 39 | 40 | if self.ignore_case: 41 | text_before_cursor = text_before_cursor.lower() 42 | 43 | add_hyphen = False 44 | if text_before_cursor[0] == '-': 45 | text_before_cursor = text_before_cursor[1:] 46 | add_hyphen = True 47 | 48 | def word_matches(word): 49 | """ True when the word before the cursor matches. """ 50 | if self.ignore_case: 51 | word = word.lower() 52 | 53 | return word.startswith(text_before_cursor) 54 | 55 | word_parts = set() 56 | for w in self.words: 57 | if word_matches(w): 58 | last_colon = text_before_cursor.rfind(':') + 1 # Pos of last colon in text 59 | last_pos = last_colon if last_colon > 0 else 0 60 | next_colon = w.find(':', last_pos) 61 | next_pos = next_colon 62 | if next_colon < 0: 63 | next_pos = len(w) - 1 64 | next_colon = w.find(':', text_len) 65 | if text_len == next_colon: # Next char is colon 66 | next_colon = w.find(':', next_colon + 1) 67 | if next_colon < 0: 68 | next_colon = len(w) 69 | ret = (w[0:next_colon], w[text_len:next_colon]) 70 | elif next_colon < 0: # Next char is not colon 71 | last_word = text_before_cursor[last_colon:] 72 | display_word = w[last_colon:] 73 | if last_word == display_word.lower(): 74 | continue 75 | ret = (w, display_word) 76 | else: 77 | ret = (w[0:next_pos], w[last_pos:next_pos]) 78 | word_parts.add(ret) 79 | 80 | word_parts = sorted(list(word_parts), key=lambda x: x[1]) 81 | for c, d in list(word_parts): 82 | comp = '-' + c if add_hyphen else c 83 | yield Completion(comp, -text_len, display=d) 84 | 85 | 86 | class YesNoValidator(Validator): 87 | def validate(self, document): 88 | text = document.text.lower() 89 | # Assumes that there is a default for empty 90 | if not bool(text): 91 | return 92 | if not (text.startswith('y') or text.startswith('n')): 93 | raise ValidationError(message='Please enter y[es] or n[o]') 94 | 95 | 96 | class NullValidator(Validator): 97 | def __init__(self, message='You must enter a value', allow_quit=False): 98 | Validator.__init__(self) 99 | self.message = message if not allow_quit else message + ' or q to quit' 100 | self.allow_quit = allow_quit 101 | 102 | def validate(self, document): 103 | text = document.text 104 | if not text: 105 | raise ValidationError(message=self.message) 106 | elif self.allow_quit and text.lower() == 'q': 107 | return 108 | 109 | 110 | class NumberValidator(NullValidator): 111 | def __init__(self, 112 | message='You must enter a number', 113 | allow_quit=False, 114 | max_number=None): 115 | NullValidator.__init__(self, allow_quit=allow_quit) 116 | self.message = message if not allow_quit else message + ' or q to quit' 117 | self.max_number = max_number 118 | 119 | def validate(self, document): 120 | NullValidator.validate(self, document) 121 | text = document.text 122 | if self.allow_quit and text.lower() == 'q': 123 | return 124 | if not text.isdigit(): 125 | i = 0 126 | for i, c in enumerate(text): 127 | if not c.isdigit(): 128 | break 129 | raise ValidationError(message=self.message, cursor_position=i) 130 | 131 | if not bool(self.max_number): 132 | return 133 | valid = int(text) <= int(self.max_number) and not int(text) == 0 134 | if not valid: 135 | range_message = 'You must enter a number between 1 and {}'.format(self.max_number) 136 | raise ValidationError(message=range_message) 137 | 138 | 139 | class NumLengthValidator(NumberValidator): 140 | def __init__(self, 141 | message='You must enter at least {} characters', 142 | allow_quit=False, 143 | min_number=4): 144 | NumberValidator.__init__(self, allow_quit=allow_quit) 145 | message = message.format(min_number) 146 | self.message = message if not allow_quit else message + ' or q to quit' 147 | self.min_number = min_number 148 | 149 | def validate(self, document): 150 | NumberValidator.validate(self, document) 151 | text = document.text 152 | if self.allow_quit and text.lower() == 'q': 153 | return 154 | text_length = len(text) 155 | if not text_length >= self.min_number: 156 | raise ValidationError(message=self.message, cursor_position=text_length) 157 | 158 | 159 | def clear_screen(): 160 | print('\033[2J\033[;H') 161 | -------------------------------------------------------------------------------- /src/python/plaid2text/online_accounts.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | 3 | from collections import OrderedDict 4 | import datetime 5 | import os 6 | import sys 7 | import textwrap 8 | 9 | from plaid import Client 10 | from plaid import errors as plaid_errors 11 | 12 | import plaid2text.config_manager as cm 13 | from plaid2text.interact import prompt, clear_screen, NullValidator 14 | from plaid2text.interact import NumberValidator, NumLengthValidator, YesNoValidator, PATH_COMPLETER 15 | 16 | 17 | class PlaidAccess(): 18 | def __init__(self, client_id=None, secret=None): 19 | if client_id and secret: 20 | self.client_id = client_id 21 | self.secret = secret 22 | else: 23 | self.client_id, self.secret = cm.get_plaid_config() 24 | 25 | self.client = Client(self.client_id, self.secret, "development", suppress_warnings=True) 26 | 27 | def get_transactions(self, 28 | access_token, 29 | start_date, 30 | end_date, 31 | account_ids): 32 | """Get transaction for a given account for the given dates""" 33 | 34 | ret = [] 35 | total_transactions = None 36 | page = 0 37 | account_array = [] 38 | account_array.append(account_ids) 39 | while True: 40 | page += 1 41 | if total_transactions: 42 | print("Fetching page %d, already fetched %d/%d transactions" % ( page, len(ret), total_transactions)) 43 | else: 44 | print("Fetching page 1") 45 | 46 | try: 47 | response = self.client.Transactions.get( 48 | access_token, 49 | start_date.strftime("%Y-%m-%d"), 50 | end_date.strftime("%Y-%m-%d"), 51 | account_ids=account_array, 52 | offset=len(ret)) 53 | except plaid_errors.ItemError as ex: 54 | print("Unable to update plaid account [%s] due to: " % account_ids, file=sys.stderr) 55 | print(" %s" % ex, file=sys.stderr ) 56 | sys.exit(1) 57 | 58 | total_transactions = response['total_transactions'] 59 | 60 | ret.extend(response['transactions']) 61 | 62 | if len(ret) >= total_transactions: break 63 | 64 | print("Downloaded %d transactions for %s - %s" % ( len(ret), start_date.strftime("%Y-%m-%d"), end_date.strftime("%Y-%m-%d"))) 65 | 66 | return ret 67 | -------------------------------------------------------------------------------- /src/python/plaid2text/plaid2text.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | 3 | """ 4 | Access account information from Plaid.com accounts 5 | and generate ledger/beancount formatted file. 6 | 7 | Requires Python >=3.2 MongoDB >= 3.2.3 and (Ledger >=3.0 OR beancount >= 2.0) 8 | 9 | Ideas and Code heavily borrowed (read: shamelessly stolen) from the awesome: icsv2ledger 10 | https://github.com/quentinsf/icsv2ledger 11 | """ 12 | 13 | import argparse 14 | from datetime import datetime 15 | from operator import attrgetter 16 | import re 17 | import sys 18 | 19 | from plaid2text.renderers import LedgerRenderer, BeancountRenderer 20 | import plaid2text.config_manager as cm 21 | import plaid2text.storage_manager as storage_manager 22 | from plaid2text.online_accounts import PlaidAccess 23 | 24 | 25 | class FileType(object): 26 | """Based on `argparse.FileType` from python3.4.2, but with additional 27 | support for the `newline` parameter to `open`. 28 | """ 29 | def __init__(self, 30 | mode='r', 31 | bufsize=-1, 32 | encoding=None, 33 | errors=None, 34 | newline=None): 35 | self._mode = mode 36 | self._bufsize = bufsize 37 | self._encoding = encoding 38 | self._errors = errors 39 | self._newline = newline 40 | 41 | def __call__(self, string): 42 | # the special argument "-" means sys.std{in,out} 43 | if string == '-': 44 | if 'r' in self._mode: 45 | return sys.stdin 46 | elif 'w' in self._mode: 47 | return sys.stdout 48 | else: 49 | msg = 'argument "-" with mode %r' % self._mode 50 | raise ValueError(msg) 51 | 52 | # all other arguments are used as file names 53 | try: 54 | return open(string, 55 | self._mode, 56 | self._bufsize, 57 | self._encoding, 58 | self._errors, 59 | newline=self._newline) 60 | except OSError as e: 61 | message = "can't open '%s': %s" 62 | raise argparse.ArgumentTypeError(message % (string, e)) 63 | 64 | def __repr__(self): 65 | args = self._mode, self._bufsize 66 | kwargs = [('encoding', self._encoding), ('errors', self._errors), 67 | ('newline', self._newline)] 68 | args_str = ', '.join([repr(arg) for arg in args if arg != -1] + 69 | ['%s=%r' % (kw, arg) 70 | for kw, arg in kwargs if arg is not None]) 71 | return '%s(%s)' % (type(self).__name__, args_str) 72 | 73 | 74 | class SortingHelpFormatter(argparse.HelpFormatter): 75 | """Sort options alphabetically when -h prints usage 76 | See http://stackoverflow.com/questions/12268602 77 | """ 78 | 79 | def add_arguments(self, actions): 80 | actions = sorted(actions, key=attrgetter('option_strings')) 81 | super(SortingHelpFormatter, self).add_arguments(actions) 82 | 83 | 84 | def _parse_args_and_config_file(): 85 | """ Read options from config file and CLI args 86 | 1. Reads hard coded cm.CONFIG_DEFAULTS 87 | 2. Supersedes by values in config file 88 | 3. Supersedes by values from CLI args 89 | """ 90 | 91 | # Build preparser with only plaid account 92 | preparser = argparse.ArgumentParser(prog='Plaid2Text', add_help=False) 93 | preparser.add_argument( 94 | 'plaid_account', 95 | nargs='?', 96 | help=( 97 | 'Nickname of Plaid account to use' 98 | ' (Example: {0})'.format('boa_checking') 99 | ) 100 | ) 101 | 102 | preparser.add_argument( 103 | 'outfile', 104 | nargs='?', 105 | metavar='FILE', 106 | type=FileType('w', encoding='utf-8'), 107 | default=sys.stdout, 108 | help=( 109 | 'output filename or stdout in Ledger/Beancount syntax' 110 | ' (default: {0})'.format('stdout') 111 | ) 112 | ) 113 | 114 | # Parse args with preparser, and find config file 115 | args, remaining_argv = preparser.parse_known_args() 116 | 117 | if "--create-account" in remaining_argv: 118 | cm.create_account(args.plaid_account) 119 | 120 | defaults = cm.get_config(args.plaid_account) if args.plaid_account else {} 121 | # defaults = cm.CONFIG_DEFAULTS 122 | 123 | # Build parser for args on command line 124 | parser = argparse.ArgumentParser( 125 | prog='Plaid2Text', 126 | # Don't surpress add_help here so it will handle -h 127 | # print script description with -h/--help 128 | description=__doc__, 129 | parents=[preparser], 130 | # sort options alphabetically 131 | formatter_class=SortingHelpFormatter 132 | ) 133 | 134 | parser.set_defaults(**defaults) 135 | parser.add_argument( 136 | '--accounts-file', 137 | metavar='FILE', 138 | help=( 139 | 'file which holds a list of account names (LEDGER ONLY)' 140 | ' (default : {0})'.format(cm.FILE_DEFAULTS.accounts_file) 141 | ) 142 | ) 143 | parser.add_argument( 144 | '--headers-file', 145 | metavar='FILE', 146 | help=( 147 | 'file which contains contents to be written to the top of the output file' 148 | ' (default : {0})'.format(cm.FILE_DEFAULTS.headers_file) 149 | ) 150 | ) 151 | parser.add_argument( 152 | '--create-account', 153 | action='store_true', 154 | help=( 155 | 'create a new account' 156 | ' (default : {0})'.format(cm.CONFIG_DEFAULTS.create_account) 157 | ) 158 | ) 159 | 160 | parser.add_argument( 161 | '--output-format', 162 | '-o', 163 | choices=['beancount', 'ledger'], 164 | help=( 165 | 'what format to use for the output file.' 166 | ' (default format: {})'.format(cm.CONFIG_DEFAULTS.output_format) 167 | ) 168 | ) 169 | parser.add_argument( 170 | '--posting-account', 171 | '-a', 172 | metavar='STR', 173 | help=( 174 | 'posting account used as source' 175 | ' (default: {0})'.format(cm.CONFIG_DEFAULTS.posting_account) 176 | ) 177 | ) 178 | 179 | parser.add_argument( 180 | '--journal-file', 181 | '-j', 182 | metavar='FILE', 183 | help=( 184 | 'journal file where to read payees/accounts\n' 185 | 'Tip: you can use includes to pull in your other journal files' 186 | ' (default journal file: {0})'.format(cm.FILE_DEFAULTS.journal_file) 187 | ) 188 | ) 189 | parser.add_argument( 190 | '--quiet', 191 | '-q', 192 | action='store_true', 193 | help=( 194 | 'do not prompt if account can be deduced from mappings' 195 | ' (default: {0})'.format(cm.CONFIG_DEFAULTS.quiet) 196 | ) 197 | ) 198 | parser.add_argument( 199 | '--download-transactions', 200 | '-d', 201 | action='store_true', 202 | help=( 203 | 'download transactions into Mongo for given plaid account' 204 | ) 205 | ) 206 | 207 | parser.add_argument( 208 | '--dbtype', 209 | choices=['mongodb', 'sqlite'], 210 | help=( 211 | 'The type of database to use for storing transactions [mongodb | sqlite]' 212 | ' (default: {0})'.format(cm.CONFIG_DEFAULTS.dbtype) 213 | ) 214 | ) 215 | 216 | parser.add_argument( 217 | '--mongo-db', 218 | metavar='STR', 219 | help=( 220 | 'The name of the Mongo database' 221 | ' (default: {0})'.format(cm.CONFIG_DEFAULTS.mongo_db) 222 | ) 223 | ) 224 | 225 | parser.add_argument( 226 | '--mongo-db-uri', 227 | metavar='STR', 228 | help=( 229 | 'The URI for your MongoDB in the MongoDB URI format' 230 | ' (default: {0})'.format(cm.CONFIG_DEFAULTS.mongo_db_uri) 231 | ) 232 | ) 233 | 234 | parser.add_argument( 235 | '--sqlite-db', 236 | metavar='STR', 237 | help=( 238 | 'The path to the SQLite database for storing transactions' 239 | ' (default: {0})'.format(cm.CONFIG_DEFAULTS.sqlite_db) 240 | ) 241 | ) 242 | parser.add_argument( 243 | '--default-expense', 244 | metavar='STR', 245 | help=( 246 | 'expense account used as default destination' 247 | ' (default: {0})'.format(cm.CONFIG_DEFAULTS.default_expense) 248 | ) 249 | ) 250 | parser.add_argument( 251 | '--cleared-character', 252 | choices='*!', 253 | help=( 254 | 'character to clear a transaction' 255 | ' (default: {0})'.format(cm.CONFIG_DEFAULTS.cleared_character) 256 | ) 257 | ) 258 | 259 | parser.add_argument( 260 | '--output-date-format', 261 | metavar='STR', 262 | help=( 263 | 'date format for output file' 264 | ' (default: YYYY/MM/DD)' 265 | ) 266 | ) 267 | 268 | parser.add_argument( 269 | '--currency', 270 | metavar='STR', 271 | help=( 272 | 'the currency of amounts' 273 | ' (default: {0})'.format(cm.CONFIG_DEFAULTS.currency) 274 | ) 275 | ) 276 | 277 | parser.add_argument( 278 | '--mapping-file', 279 | metavar='FILE', 280 | help=( 281 | 'file which holds the mappings' 282 | ' (default: {0})' 283 | .format(cm.FILE_DEFAULTS.mapping_file) 284 | ) 285 | ) 286 | parser.add_argument( 287 | '--template-file', 288 | metavar='FILE', 289 | help=( 290 | 'file which holds the template' 291 | ' (default: {0})' 292 | .format(cm.FILE_DEFAULTS.template_file) 293 | ) 294 | ) 295 | parser.add_argument( 296 | '--tags', 297 | '-t', 298 | action='store_true', 299 | help=( 300 | 'prompt for transaction tags' 301 | ' (default: {0})'.format(cm.CONFIG_DEFAULTS.tags) 302 | ) 303 | ) 304 | parser.add_argument( 305 | '--clear-screen', 306 | '-C', 307 | action='store_true', 308 | help=( 309 | 'clear screen for every transaction' 310 | ' (default: {0})'.format(cm.CONFIG_DEFAULTS.clear_screen) 311 | ) 312 | ) 313 | parser.add_argument( 314 | '--no-mark-pulled', 315 | '-n', 316 | action='store_false', 317 | help=( 318 | 'Do not mark pulled transactions. ' 319 | 'When given, the pulled transactions will still be listed ' 320 | 'as new transactions upon the next run.' 321 | ' (default: False)' 322 | ) 323 | ) 324 | 325 | parser.add_argument( 326 | '--all-transactions', 327 | action='store_true', 328 | help=( 329 | 'pull all transactions even those who have been previously marked as processed' 330 | ' (default: False' 331 | ) 332 | ) 333 | 334 | parser.add_argument( 335 | '--to-date', 336 | metavar='STR', 337 | help=( 338 | 'specify the ending date for transactions to be pulled; ' 339 | 'use in conjunction with --from-date to specify range' 340 | 'Date format: YYYY-MM-DD' 341 | ) 342 | ) 343 | 344 | parser.add_argument( 345 | '--from-date', 346 | metavar='STR', 347 | help=( 348 | 'specify a the starting date for transactions to be pulled; ' 349 | 'use in conjunction with --to-date to specify range' 350 | 'Date format: YYYY-MM-DD' 351 | ) 352 | ) 353 | 354 | # TODO NEED TO FIX - USING PARENTS causes file to be opened twice 355 | args = parser.parse_args() 356 | 357 | args.journal_file = cm.find_first_file( 358 | args.journal_file, 359 | cm.FILE_DEFAULTS.journal_file 360 | ) 361 | args.mapping_file = cm.find_first_file( 362 | args.mapping_file, 363 | cm.FILE_DEFAULTS.mapping_file 364 | ) 365 | args.accounts_file = cm.find_first_file( 366 | args.accounts_file, 367 | cm.FILE_DEFAULTS.accounts_file 368 | ) 369 | args.template_file = cm.find_first_file( 370 | args.template_file, 371 | cm.FILE_DEFAULTS.template_file 372 | ) 373 | args.headers_file = cm.find_first_file( 374 | args.headers_file, 375 | cm.FILE_DEFAULTS.headers_file 376 | ) 377 | # Make sure we have a plaid account and we are not calling --help 378 | if not args.plaid_account and 'help' not in args: 379 | print('You must provide the Plaid account as the first argument', 380 | file=sys.stderr) 381 | sys.exit(1) 382 | 383 | if args.from_date: 384 | y, m, d = [int(i) for i in re.split(r'[/-]', args.from_date)] 385 | args.from_date = datetime(y, m, d) 386 | 387 | if args.to_date: 388 | y, m, d = [int(i) for i in re.split(r'[/-]', args.to_date)] 389 | args.to_date = datetime(y, m, d) 390 | 391 | return args 392 | 393 | 394 | def main(): 395 | # Make sure we have config file 396 | if not cm.config_exists(): 397 | return 398 | 399 | options = _parse_args_and_config_file() 400 | truthy = ['true', 'yes', '1', 't'] 401 | # Convert config values to Boolean if pulled from file 402 | if not isinstance(options.quiet, bool): 403 | options.quiet = options.quiet.lower() in truthy 404 | if not isinstance(options.tags, bool): 405 | options.tags = options.tags.lower() in truthy 406 | if not isinstance(options.clear_screen, bool): 407 | options.clear_screen = options.clear_screen.lower() in truthy 408 | 409 | if options.dbtype == 'mongodb': 410 | sm = storage_manager.MongoDBStorage( 411 | options.mongo_db, 412 | options.mongo_db_uri, 413 | options.plaid_account, 414 | options.posting_account 415 | ) 416 | else: 417 | sm = storage_manager.SQLiteStorage( 418 | options.sqlite_db, 419 | options.plaid_account, 420 | options.posting_account 421 | ) 422 | 423 | if options.download_transactions: 424 | if 'to_date' not in options or 'from_date' not in options: 425 | print('When downloading, both start and end date are required', file=sys.stderr) 426 | sys.exit(1) 427 | 428 | trans = PlaidAccess().get_transactions(options.access_token, start_date=options.from_date, end_date=options.to_date,account_ids=options.account) 429 | sm.save_transactions(trans) 430 | print('Transactions successfully downloaded and saved into %s' % options.dbtype, file=sys.stdout) 431 | sys.exit(0) 432 | 433 | if not options.config_file: 434 | print('Configuration file is required.', file=sys.stderr) 435 | sys.exit(1) 436 | 437 | to_date = None if 'to_date' not in options else options.to_date 438 | from_date = None if 'from_date' not in options else options.from_date 439 | only_new = not options.all_transactions 440 | 441 | trxs = sm.get_transactions(to_date=to_date, 442 | from_date=from_date, 443 | only_new=only_new) 444 | 445 | if options.output_format == 'beancount': 446 | out = BeancountRenderer(trxs, options) 447 | else: 448 | out = LedgerRenderer(trxs, options) 449 | 450 | callback = None 451 | if options.no_mark_pulled: 452 | callback = lambda dict: sm.update_transaction(dict, mark_pulled=False) 453 | 454 | try: 455 | update_dict = out.process_transactions(callback=callback) 456 | except (KeyboardInterrupt, EOFError): 457 | print("\nProcess interrupted by keyboard interrupt."); 458 | 459 | if __name__ == '__main__': 460 | main() 461 | -------------------------------------------------------------------------------- /src/python/plaid2text/renderers.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | 3 | from abc import ABCMeta, abstractmethod 4 | import csv 5 | import os 6 | import re 7 | import subprocess 8 | import sys 9 | 10 | import plaid2text.config_manager as cm 11 | from plaid2text.interact import separator_completer, prompt 12 | 13 | 14 | class Entry: 15 | """ 16 | This represents one entry (transaction) from Plaid. 17 | """ 18 | 19 | def __init__(self, transaction, options={}): 20 | """Parameters: 21 | transaction: a plaid transaction 22 | 23 | options: from CLI args and config file 24 | """ 25 | self.options = options 26 | 27 | self.transaction = transaction 28 | # TODO: document this 29 | if 'addons' in options: 30 | self.transaction['addons'] = dict( 31 | (k, fields[v - 1]) for k, v in options.addons.items() # NOQA 32 | ) 33 | else: 34 | self.transaction['addons'] = {} 35 | 36 | # The id for the transaction 37 | self.transaction['transaction_id'] = self.transaction['transaction_id'] 38 | 39 | # Get the date and convert it into a ledger/beancount formatted date. 40 | d8 = self.transaction['date'] 41 | d8_format = options.output_date_format if options and 'output_date_format' in options else '%Y-%m-%d' 42 | self.transaction['transaction_date'] = d8.date().strftime(d8_format) 43 | 44 | self.desc = self.transaction['name'] 45 | 46 | # amnt = self.transaction['amount'] 47 | self.transaction['currency'] = options.currency 48 | # self.transaction['debit_amount'] = amnt 49 | # self.transaction['debit_currency'] = currency 50 | # self.transaction['credit_amount'] = '' 51 | # self.transaction['credit_currency'] = '' 52 | 53 | self.transaction['posting_account'] = options.posting_account 54 | self.transaction['cleared_character'] = options.cleared_character 55 | 56 | if options.template_file: 57 | with open(options.template_file, 'r', encoding='utf-8') as f: 58 | self.transaction['transaction_template'] = f.read() 59 | else: 60 | self.transaction['transaction_template'] = '' 61 | 62 | def query(self): 63 | """ 64 | We print a summary of the record on the screen, and allow you to 65 | choose the destination account. 66 | """ 67 | return '{0} {1:<40} {2}'.format( 68 | self.transaction['date'], 69 | self.desc, 70 | self.transaction['amount'] 71 | ) 72 | 73 | def journal_entry(self, payee, account, tags): 74 | """ 75 | Return a formatted journal entry recording this Entry against 76 | the specified posting account 77 | """ 78 | if self.options.output_format == 'ledger': 79 | def_template = cm.DEFAULT_LEDGER_TEMPLATE 80 | else: 81 | def_template = cm.DEFAULT_BEANCOUNT_TEMPLATE 82 | if self.transaction['transaction_template']: 83 | template = (self.transaction['transaction_template']) 84 | else: 85 | template = (def_template) 86 | if self.options.output_format == 'beancount': 87 | ret_tags = ' {}'.format(tags) if tags else '' 88 | else: 89 | ret_tags = ' ; {}'.format(tags) if tags else '' 90 | 91 | format_data = { 92 | 'associated_account': account, 93 | 'payee': payee, 94 | 'tags': ret_tags 95 | } 96 | format_data.update(self.transaction['addons']) 97 | format_data.update(self.transaction) 98 | return template.format(**format_data) 99 | 100 | 101 | class OutputRenderer(metaclass=ABCMeta): 102 | """ 103 | Base class for output rendering. 104 | """ 105 | def __init__(self, transactions, options): 106 | self.transactions = transactions 107 | self.possible_accounts = set([]) 108 | self.possible_payees = set([]) 109 | self.possible_tags = set([]) 110 | self.mappings = [] 111 | self.map_file = options.mapping_file 112 | self.read_mapping_file() 113 | self.journal_file = options.journal_file 114 | self.journal_lines = [] 115 | self.options = options 116 | self.get_possible_accounts_and_payees() 117 | # Add payees/accounts/tags from mappings 118 | for m in self.mappings: 119 | self.possible_payees.add(m[1]) 120 | self.possible_accounts.add(m[2]) 121 | if m[3]: 122 | if options.output_format == 'ledger': 123 | self.possible_tags.update(set(m[3][0].split(':'))) 124 | else: 125 | self.possible_tags.update([t.replace('#', '') for t in m[3][0].split(' ')]) 126 | 127 | def read_mapping_file(self): 128 | """ 129 | Mappings are simply a CSV file with three columns. 130 | The first is a string to be matched against an entry description. 131 | The second is the payee against which such entries should be posted. 132 | The third is the account against which such entries should be posted. 133 | 134 | If the match string begins and ends with '/' it is taken to be a 135 | regular expression. 136 | """ 137 | if not self.map_file: 138 | return 139 | 140 | with open(self.map_file, 'r', encoding='utf-8', newline='') as f: 141 | map_reader = csv.reader(f) 142 | for row in map_reader: 143 | if len(row) > 1: 144 | pattern = row[0].strip() 145 | payee = row[1].strip() 146 | account = row[2].strip() 147 | tags = row[3:] 148 | if pattern.startswith('/') and pattern.endswith('/'): 149 | try: 150 | pattern = re.compile(pattern[1:-1], re.I) 151 | except re.error as e: 152 | print( 153 | "Invalid regex '{0}' in '{1}': {2}" 154 | .format(pattern, self.map_file, e), 155 | file=sys.stderr) 156 | sys.exit(1) 157 | self.mappings.append((pattern, payee, account, tags)) 158 | 159 | def append_mapping_file(self, desc, payee, account, tags): 160 | if self.map_file: 161 | with open(self.map_file, 'a', encoding='utf-8', newline='') as f: 162 | writer = csv.writer(f) 163 | ret_tags = tags if len(tags) > 0 else '' 164 | writer.writerow([desc, payee, account, ret_tags]) 165 | 166 | def process_transactions(self, callback=None): 167 | """ 168 | Read transactions from Mongo (Plaid) and 169 | process them. Writes Ledger/Beancount formatted 170 | lines either to out_file or stdout. 171 | 172 | Parameters: 173 | callback: A function taking a single transaction update object to store 174 | in the DB immediately after collecting the information from the user. 175 | """ 176 | out = self._process_plaid_transactions(callback=callback) 177 | 178 | if self.options.headers_file: 179 | headers = ''.join(open(self.options.headers_file, mode='r').readlines()) 180 | print(headers, file=self.options.outfile) 181 | print(*self.journal_lines, sep='\n', file=self.options.outfile) 182 | return out 183 | 184 | def _process_plaid_transactions(self, callback=None): 185 | """Process plaid transaction and return beancount/ledger formatted 186 | lines. 187 | """ 188 | out = [] 189 | for t in self.transactions: 190 | entry = Entry(t, self.options) 191 | payee, account, tags = self.get_payee_and_account(entry) 192 | dic = {} 193 | dic['transaction_id'] = t['transaction_id'] 194 | dic['tags'] = tags 195 | dic['associated_account'] = account 196 | dic['payee'] = payee 197 | dic['posting_account'] = self.options.posting_account 198 | out.append(dic) 199 | 200 | # save the transactions into the database as they are processed 201 | if callback: callback(dic) 202 | 203 | self.journal_lines.append(entry.journal_entry(payee, account, tags)) 204 | return out 205 | 206 | def prompt_for_value(self, text_prompt, values, default): 207 | sep = ':' if text_prompt == 'Payee' else ' ' 208 | a = prompt( 209 | '{} [{}]: '.format(text_prompt, default), 210 | completer=separator_completer(values, sep=sep) 211 | ) 212 | # Handle tag returning none if accepting 213 | return a if (a or text_prompt == 'Tag') else default 214 | 215 | def get_payee_and_account(self, entry): 216 | payee = entry.desc 217 | account = self.options.default_expense 218 | tags = '' 219 | found = False 220 | # Try to match entry desc with mappings patterns 221 | for m in self.mappings: 222 | pattern = m[0] 223 | if isinstance(pattern, str): 224 | if entry.desc == pattern: 225 | payee, account, tags = m[1], m[2], m[3] 226 | found = True # do not break here, later mapping must win 227 | else: 228 | # If the pattern isn't a string it's a regex 229 | if m[0].match(entry.desc): 230 | payee, account, tags = m[1], m[2], m[3] 231 | found = True 232 | # Tags gets read in as a list, but just contains one string 233 | if tags: 234 | tags = tags[0] 235 | 236 | modified = False 237 | if self.options.quiet and found: 238 | pass 239 | else: 240 | if self.options.clear_screen: 241 | print('\033[2J\033[;H') 242 | print('\n' + entry.query()) 243 | 244 | value = self.prompt_for_value('Payee', self.possible_payees, payee) 245 | if value: 246 | modified = modified if modified else value != payee 247 | payee = value 248 | 249 | value = self.prompt_for_value('Account', self.possible_accounts, account) 250 | if value: 251 | modified = modified if modified else value != account 252 | account = value 253 | 254 | if self.options.tags: 255 | value = self.prompt_for_tags('Tag', self.possible_tags, tags) 256 | if value: 257 | modified = modified if modified else value != tags 258 | tags = value 259 | 260 | if not found or (found and modified): 261 | # Add new or changed mapping to mappings and append to file 262 | self.mappings.append((entry.desc, payee, account, tags)) 263 | self.append_mapping_file(entry.desc, payee, account, tags) 264 | 265 | # Add new possible_values to possible values lists 266 | self.possible_payees.add(payee) 267 | self.possible_accounts.add(account) 268 | 269 | return (payee, account, tags) 270 | 271 | @abstractmethod 272 | def tagify(self, value): 273 | pass 274 | 275 | @abstractmethod 276 | def get_possible_accounts_and_payees(self): 277 | pass 278 | 279 | @abstractmethod 280 | def prompt_for_tags(self, prompt, values, default): 281 | pass 282 | 283 | 284 | class LedgerRenderer(OutputRenderer): 285 | def tagify(self, value): 286 | if value.find(':') < 0 and value[0] != '[' and value[-1] != ']': 287 | value = ':{0}:'.format(value.replace(' ', '-').replace(',', '')) 288 | return value 289 | 290 | def get_possible_accounts_and_payees(self): 291 | if self.journal_file: 292 | self.possible_payees = self._payees_from_ledger() 293 | self.possible_accounts = self._accounts_from_ledger() 294 | self.read_accounts_file() 295 | 296 | def prompt_for_tags(self, prompt, values, default): 297 | # tags = list(default[0].split(':')) 298 | tags = [':{}:'.format(t) for t in default.split(':') if t] if default else [] 299 | value = self.prompt_for_value(prompt, values, ''.join(tags).replace('::', ':')) 300 | while value: 301 | if value[0] == '-': 302 | value = self.tagify(value[1:]) 303 | if value in tags: 304 | tags.remove(value) 305 | else: 306 | value = self.tagify(value) 307 | if value not in tags: 308 | tags.append(value) 309 | value = self.prompt_for_value(prompt, values, ''.join(tags).replace('::', ':')) 310 | return ''.join(tags).replace('::', ':') 311 | 312 | def _payees_from_ledger(self): 313 | return self._from_ledger('payees') 314 | 315 | def _accounts_from_ledger(self): 316 | return self._from_ledger('accounts') 317 | 318 | def _from_ledger(self, command): 319 | ledger = 'ledger' 320 | for f in ['/usr/bin/ledger', '/usr/local/bin/ledger']: 321 | if os.path.exists(f): 322 | ledger = f 323 | break 324 | 325 | cmd = [ledger, '-f', self.journal_file, command] 326 | p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 327 | (stdout_data, stderr_data) = p.communicate() 328 | items = set() 329 | for item in stdout_data.decode('utf-8').splitlines(): 330 | items.add(item) 331 | return items 332 | 333 | def read_accounts_file(self): 334 | """ Process each line in the specified account file looking for account 335 | definitions. An account definition is a line containing the word 336 | 'account' followed by a valid account name, e.g: 337 | 338 | account Expenses 339 | account Expenses:Utilities 340 | 341 | All other lines are ignored. 342 | """ 343 | if not self.options.accounts_file: 344 | return 345 | accounts = [] 346 | pattern = re.compile('^\s*account\s+([:A-Za-z0-9-_ ]+)$') 347 | with open(self.options.accounts_file, 'r', encoding='utf-8') as f: 348 | for line in f.readlines(): 349 | mo = pattern.match(line) 350 | if mo: 351 | accounts.append(mo.group(1)) 352 | 353 | self.possible_accounts.update(accounts) 354 | 355 | 356 | class BeancountRenderer(OutputRenderer): 357 | import beancount 358 | 359 | def tagify(self, value): 360 | # No spaces or commas allowed 361 | return value.replace(' ', '-').replace(',', '') 362 | 363 | def get_possible_accounts_and_payees(self): 364 | if self.journal_file: 365 | self._payees_and_accounts_from_beancount() 366 | 367 | def _payees_and_accounts_from_beancount(self): 368 | try: 369 | payees = set() 370 | accounts = set() 371 | tags = set() 372 | from beancount import loader 373 | from beancount.core.data import Transaction, Open 374 | import sys 375 | entries, errors, options = loader.load_file(self.journal_file) 376 | 377 | except Exception as e: 378 | print(e.message, file=sys.stderr) 379 | sys.exit(1) 380 | else: 381 | for e in entries: 382 | if type(e) is Transaction: 383 | if e.payee: 384 | payees.add(e.payee) 385 | if e.tags: 386 | for t in e.tags: 387 | tags.add(t) 388 | if e.postings: 389 | for p in e.postings: 390 | accounts.add(p.account) 391 | elif type(e) is Open: 392 | accounts.add(e.account) 393 | 394 | self.possible_accounts.update(accounts) 395 | self.possible_tags.update(tags) 396 | self.possible_payees.update(payees) 397 | 398 | def prompt_for_tags(self, prompt, values, default): 399 | tags = ' '.join(['#{}'.format(t) for t in default.split() if t]) if default else [] 400 | value = self.prompt_for_value(prompt, values, ' '.join(['#{}'.format(t) for t in tags])) 401 | while value: 402 | if value[0] == '-': 403 | value = self.tagify(value[1:]) 404 | if value in tags: 405 | tags.remove(value) 406 | else: 407 | value = self.tagify(value) 408 | if value not in tags: 409 | tags.append(value) 410 | value = self.prompt_for_value( 411 | prompt, 412 | values, 413 | ' '.join(['#{}'.format(t) for t in tags]) 414 | ) 415 | return ' '.join(['#{}'.format(t) for t in tags]) 416 | -------------------------------------------------------------------------------- /src/python/plaid2text/storage_manager.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | 3 | import datetime 4 | from dateutil import parser as date_parser 5 | import sqlite3 6 | import json 7 | 8 | from abc import ABCMeta, abstractmethod 9 | from pymongo import MongoClient, ASCENDING 10 | 11 | from .renderers import Entry 12 | 13 | TEXT_DOC = { 14 | 'plaid2text': { 15 | 'tags': [], 16 | 'payee': '', 17 | 'posting_account': '', 18 | 'associated_account': '', 19 | 'date_downloaded': datetime.datetime.today(), 20 | 'date_last_pulled': datetime.datetime.today(), 21 | 'pulled_to_file': False 22 | } 23 | } 24 | 25 | class StorageManager(metaclass=ABCMeta): 26 | @abstractmethod 27 | def save_transactions(self, transactions): 28 | """ 29 | Saves the given transactions to the configured db. 30 | 31 | Occurs when using the --download-transactions option. 32 | """ 33 | pass 34 | 35 | @abstractmethod 36 | def get_transactions(self, from_date=None, to_date=None, only_new=True): 37 | """ 38 | Retrieve transactions for producing text file. 39 | """ 40 | pass 41 | 42 | @abstractmethod 43 | def update_transaction(self, update): 44 | pass 45 | 46 | class MongoDBStorage(StorageManager): 47 | """ 48 | Handles all Mongo related tasks 49 | """ 50 | def __init__(self, db, uri, account, posting_account): 51 | self.mc = MongoClient(uri) 52 | self.db_name = db 53 | self.db = self.mc[db] 54 | self.account = self.db[account] 55 | 56 | def save_transactions(self, transactions): 57 | for t in transactions: 58 | id = t['transaction_id'] 59 | # t.update(TEXT_DOC) 60 | # Convert datetime 61 | y, m, d = [int(i) for i in t['date'].split('-')] 62 | t['date'] = datetime.datetime(y, m, d) 63 | doc = {'$set': t} 64 | # Add default plaid2text to new inserts 65 | doc['$setOnInsert'] = TEXT_DOC 66 | self.account.update_many({'_id': id}, doc, True) 67 | 68 | def get_transactions(self, from_date=None, to_date=None, only_new=True): 69 | query = {} 70 | if only_new: 71 | query['plaid2text.pulled_to_file'] = {"$ne": True} 72 | 73 | if from_date and to_date and (from_date <= to_date): 74 | query['date'] = {'$gte': from_date, '$lte': to_date} 75 | elif from_date and not to_date: 76 | query['date'] = {'$gte': from_date} 77 | elif not from_date and to_date: 78 | query['date'] = {'$lte': to_date} 79 | 80 | transactions = self.account.find(query).sort('date', ASCENDING) 81 | return list(transactions) 82 | 83 | def update_transaction(self, update, mark_pulled=None): 84 | id = update.pop('transaction_id') 85 | 86 | if mark_pulled: 87 | update['pulled_to_file' ] = mark_pulled 88 | update['date_last_pulled'] = datetime.datetime.today() 89 | 90 | self.account.update( 91 | {'_id': id}, 92 | {'$set': {"plaid2text": update}} 93 | ) 94 | 95 | 96 | class SQLiteStorage(): 97 | def __init__(self, dbpath, account, posting_account): 98 | self.conn = sqlite3.connect(dbpath) 99 | 100 | c = self.conn.cursor() 101 | c.execute(""" 102 | create table if not exists transactions 103 | (account_id, transaction_id, created, updated, plaid_json, metadata) 104 | """) 105 | c.execute(""" 106 | create unique index if not exists transactions_idx 107 | ON transactions(account_id, transaction_id) 108 | """) 109 | self.conn.commit() 110 | 111 | # This might be needed if there's not consistent support for json_extract in sqlite3 installations 112 | # this will need to be modified to support the "$.prop" syntax 113 | #def json_extract(json_str, prop): 114 | # ret = json.loads(json_str).get(prop, None) 115 | # return ret 116 | #self.conn.create_function("json_extract", 2, json_extract) 117 | 118 | def save_transactions(self, transactions): 119 | """ 120 | Saves the given transactions to the configured db. 121 | 122 | Occurs when using the --download-transactions option. 123 | """ 124 | for t in transactions: 125 | trans_id = t['transaction_id'] 126 | act_id = t['account_id'] 127 | 128 | metadata = t.get('plaid2text', None) 129 | if metadata is not None: 130 | metadata = json.dumps(metadata) 131 | 132 | c = self.conn.cursor() 133 | c.execute(""" 134 | insert into 135 | transactions(account_id, transaction_id, created, updated, plaid_json, metadata) 136 | values(?,?,strftime('%Y-%m-%dT%H:%M:%SZ', 'now'),strftime('%Y-%m-%dT%H:%M:%SZ', 'now'),?,?) 137 | on conflict(account_id, transaction_id) DO UPDATE 138 | set updated = strftime('%Y-%m-%dT%H:%M:%SZ', 'now'), 139 | plaid_json = excluded.plaid_json, 140 | metadata = excluded.metadata 141 | """, [act_id, trans_id, json.dumps(t), metadata]) 142 | self.conn.commit() 143 | 144 | def get_transactions(self, from_date=None, to_date=None, only_new=True): 145 | query = "select plaid_json, metadata from transactions"; 146 | 147 | conditions = [] 148 | if only_new: 149 | conditions.append("coalesce(json_extract(plaid_json, '$.pulled_to_file'), false) = false") 150 | 151 | params = [] 152 | if from_date and to_date and (from_date <= to_date): 153 | conditions.append("json_extract(plaid_json, '$.date') between ? and ?") 154 | params += [from_date.strftime("%Y-%m-%d"), to_date.strftime("%Y-%m-%d")] 155 | elif from_date and not to_date: 156 | conditions.append("json_extract(plaid_json, '$.date') >= ?") 157 | params += [from_date] 158 | elif not from_date and to_date: 159 | conditions.append("json_extract(plaid_json, '$.date') <= ?") 160 | params += [to_date] 161 | 162 | if len(conditions) > 0: 163 | query = "%s where %s" % ( query, " AND ".join( conditions ) ) 164 | 165 | transactions = self.conn.cursor().execute(query, params).fetchall() 166 | 167 | ret = [] 168 | for row in transactions: 169 | t = json.loads(row[0]) 170 | if row[1]: 171 | t['plaid2text'] = json.loads(row[1]) 172 | else: 173 | t['plaid2text'] = {} 174 | 175 | if ( len(t['plaid2text']) == 0 ): 176 | # set empty objects ({}) to None to account for assumptions that None means not processed 177 | t['plaid2text'] = None 178 | 179 | t['date'] = date_parser.parse( t['date'] ) 180 | 181 | ret.append(t) 182 | 183 | return ret 184 | 185 | def update_transaction(self, update, mark_pulled=None): 186 | trans_id = update.pop('transaction_id') 187 | if mark_pulled: 188 | update['pulled_to_file' ] = mark_pulled 189 | update['date_last_pulled'] = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%SZ") 190 | 191 | update['archived'] = null 192 | 193 | c = self.conn.cursor() 194 | c.execute(""" 195 | update transactions set metadata = json_patch(coalesce(metadata, '{}'), ?) 196 | where transaction_id = ? 197 | """, [json.dumps(update), trans_id] ) 198 | self.conn.commit() 199 | --------------------------------------------------------------------------------