├── .emacs └── 29.3 │ └── straight │ └── versions │ └── default.el ├── .github └── workflows │ └── test.yml ├── .gitignore ├── CHANGELOG.md ├── Cask ├── LICENSE ├── Makefile ├── README.md ├── doc ├── erd │ ├── erd-mysql.png │ ├── erd-postgres.png │ ├── erd-sqlite.png │ └── erd-sqlserver.png ├── org-sql-doc.el └── table-descriptions.md ├── docker-compose.yml ├── env-29.3.yml ├── init.el ├── org-sql.el └── test ├── docker ├── mariadb │ ├── Dockerfile │ └── init │ │ └── org_sql.sql ├── postgres │ ├── Dockerfile │ └── init │ │ └── org_sql.sql └── sql-server │ ├── Dockerfile │ ├── entrypoint.sh │ ├── init-db.sh │ └── setup.sql ├── files ├── fancy.org ├── foo1.org ├── foo2.org └── foo3.org ├── org-sql-test-stateful.el ├── org-sql-test-stateless.el └── scripts ├── clear_hook.sql ├── init_hook.sql └── update_hook.sql /.emacs/29.3/straight/versions/default.el: -------------------------------------------------------------------------------- 1 | (("dash.el" . "39d067b9fbb2db65fc7a6938bfb21489ad990cb4") 2 | ("el-get" . "c0713e8d8e8ad987fe1283d76b9c637a10f048ef") 3 | ("emacs-buttercup" . "a1a86b027ffe030e1c78a9f43c50cd20a6fed19a") 4 | ("emacsmirror-mirror" . "8cbbdaa750c897d05ee71980834699a7d7c2d208") 5 | ("f.el" . "de6d4d40ddc844eee643e92d47b9d6a63fbebb48") 6 | ("gnu-elpa-mirror" . "3d0759ef4792b6461f2979a4e70e1c819df7283a") 7 | ("melpa" . "3126dac37def07fcaa667a325ee79349fb80d285") 8 | ("nongnu-elpa" . "a9a649210a8d8b9295b5a1d0c7b60a77db03c14c") 9 | ("org" . "071c6e986c424d2e496be7d0815d6e9cd83ae4e6") 10 | ("org-ml" . "7a7b1e918e8440f3f6ddb37db9bd1471d0dad37d") 11 | ("s.el" . "4d7d83122850cf70dc60662a73124f0be41ad186") 12 | ("straight.el" . "88e574ae75344e39b436f863ef0344135c7b6517")) 13 | :gamma 14 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | on: [push] 3 | 4 | jobs: 5 | build: 6 | runs-on: ubuntu-latest 7 | strategy: 8 | matrix: 9 | emacs_version: 10 | - '29.3' 11 | steps: 12 | - uses: actions/checkout@v2 13 | 14 | - uses: mamba-org/setup-micromamba@v1.9.0 15 | with: 16 | micromamba-version: '1.5.6-0' 17 | environment-file: env-${{ matrix.emacs_version }}.yml 18 | cache-environment: true 19 | post-cleanup: 'all' 20 | 21 | - uses: hoverkraft-tech/compose-action@v2.0.1 22 | with: 23 | up-flags: "--build" 24 | down-flags: "--volumes" 25 | 26 | - name: Run tests 27 | shell: micromamba-shell {0} 28 | run: | 29 | export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libc_malloc_debug.so 30 | make install 31 | make 32 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.elc 2 | .emacs/*/straight/build 3 | .emacs/*/straight/repos 4 | .emacs/*/straight/build-cache.el -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## 4.0.0 4 | 5 | breaking changes 6 | - drop support for lots of unsupported database versions (technically a breaking 7 | change, hence major bump) 8 | 9 | new features 10 | - timestamps in headlines are now pulled into database (thanks aviav) 11 | 12 | bugfixes 13 | - fix compilation and documentation errors (thanks aviav) 14 | - fix whitespace issues with go-sqlcmd 15 | 16 | ## 3.0.4 17 | 18 | - fix compiler errors and warnings 19 | - fix typo in cask file preventing build from properly testing 20 | 21 | ## 3.0.3 22 | 23 | - fix error when using pull for files with no preamble 24 | 25 | ## 3.0.2 26 | 27 | - fix typos in `org-sql-user-push`/`org-sql-user-pull` 28 | 29 | ## 3.0.1 30 | 31 | - fix bug timestamp bug when using org-mode 9.4 32 | 33 | ## 3.0.0 34 | 35 | - new features 36 | - add preamble (the text before headlines) to `outlines` table 37 | - add `org-sql-pull-from-db` 38 | - breaking changes 39 | - drop MySQL 5.7 support due to need for recursive queries 40 | - `org-sql-dump-table` now returns deserialized output (eg output is 41 | converted to real numbers, symbols, and strings) 42 | - bug fixes 43 | - properly handle strings with newlines, tabs, and other control characters 44 | - property handle strings that literally say "NULL" 45 | 46 | ## 2.1.0 47 | 48 | - add habits to the `timestamp_repeater` table 49 | - add headline_index column to `headlines` table to denote order 50 | - add level column to `headlines` table 51 | - fix constraints to agree with intended cardinality 52 | - fix compile errors 53 | 54 | ## 2.0.0 55 | 56 | - new features 57 | - added support for SQL-Server, MySQL/MariaDB, and added testing for all 58 | current versions of listed databases 59 | - added entity relationship diagrams to documentation 60 | - added hooks (eg `org-sql-post-push-hook` and related) to allow custom SQL 61 | commands to be run 62 | - `:args` and `:env` options to extend the client program call as needed 63 | - comprehensive public API for manipulating the database (in addition to the 64 | 'interactive' functions) 65 | - statistics cookies are now tracked in the `headlines` table 66 | - performance improvements 67 | - option to spawn asynchronous client process (`org-sql-async`) 68 | - optional `:unlogged` tables for Postgres 69 | - now use one-line bulk INSERT syntax 70 | - changes (all of which are **breaking changes**) 71 | - switched role of `file_hash` and `file_path`; the former now has a 72 | one-to-many relationship with the latter which allows identical org-files to 73 | exist in the database where they couldn't before 74 | - all tables now use surrogate keys, which is much faster (possibly two orders 75 | of magnitude) and doesn't require buffer-specific information to make a 76 | primary key which would be unknown for new rows 77 | - `files` table renamed to `outlines` and `file_metadata` table 78 | added 79 | - `md5` column has been renamed to `outline_hash` 80 | - `file_properties` was removed (it was redundant) 81 | - `headline_id` removed from `planning_entries` table 82 | - warning and repeater information in the `timestamps` table has been split 83 | off into separate tables 84 | - removed `ON UPDATE CASCADE` from all tables (no longer needed) 85 | - split `org-sql-user-reset` into `-reset` and `-init` functions (which are 86 | functionally inverses of each other) 87 | - `org-sql-user-update` was renamed to `org-sql-user-push` ...to make room 88 | for a `-pull` ;) 89 | 90 | ## 1.1.1 91 | 92 | - recognize all effort formats (not just `MM` and `HH:MM`) 93 | - fix f.el dependency 94 | 95 | ## 1.1.0 96 | 97 | - use latest logbook/contents code from `org-ml` 98 | 99 | ## 1.0.3 100 | 101 | - fixed execution paths and temp file path 102 | (![jarifuri](https://github.com/jarifuri)) 103 | 104 | ## 1.0.2 105 | 106 | - added `org-sql-log-note-headings-overrides` 107 | 108 | ## 1.0.1 109 | 110 | - various bugfixes 111 | 112 | ## 1.0.0 (relative to previous unversioned release) 113 | 114 | - use `org-ml` to simplify code 115 | - add support for postgres 116 | - use subprocesses for SQL interaction instead of built-in Emacs SQL comint mode 117 | - various performance improvements 118 | - add tests (stateless and stateful) 119 | - total rewrite of the schema: 120 | - moved/added file tags, file properties, headline closures, headline tags, 121 | headline properties, and planning entries as seperate tables 122 | - no longer defer foreign keys 123 | - use enum where fixed data types are expected (not sqlite) 124 | -------------------------------------------------------------------------------- /Cask: -------------------------------------------------------------------------------- 1 | (source gnu) 2 | (source melpa) 3 | 4 | (package-file "org-sql.el") 5 | (files "*.el") 6 | 7 | (development 8 | (depends-on "org-ml") 9 | (depends-on "dash") 10 | (depends-on "s") 11 | (depends-on "f") 12 | (depends-on "buttercup")) 13 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | The GNU General Public License is a free, copyleft license for 11 | software and other kinds of works. 12 | 13 | The licenses for most software and other practical works are designed 14 | to take away your freedom to share and change the works. By contrast, 15 | the GNU General Public License is intended to guarantee your freedom to 16 | share and change all versions of a program--to make sure it remains free 17 | software for all its users. We, the Free Software Foundation, use the 18 | GNU General Public License for most of our software; it applies also to 19 | any other work released this way by its authors. You can apply it to 20 | your programs, too. 21 | 22 | When we speak of free software, we are referring to freedom, not 23 | price. Our General Public Licenses are designed to make sure that you 24 | have the freedom to distribute copies of free software (and charge for 25 | them if you wish), that you receive source code or can get it if you 26 | want it, that you can change the software or use pieces of it in new 27 | free programs, and that you know you can do these things. 28 | 29 | To protect your rights, we need to prevent others from denying you 30 | these rights or asking you to surrender the rights. Therefore, you have 31 | certain responsibilities if you distribute copies of the software, or if 32 | you modify it: responsibilities to respect the freedom of others. 33 | 34 | For example, if you distribute copies of such a program, whether 35 | gratis or for a fee, you must pass on to the recipients the same 36 | freedoms that you received. You must make sure that they, too, receive 37 | or can get the source code. And you must show them these terms so they 38 | know their rights. 39 | 40 | Developers that use the GNU GPL protect your rights with two steps: 41 | (1) assert copyright on the software, and (2) offer you this License 42 | giving you legal permission to copy, distribute and/or modify it. 43 | 44 | For the developers' and authors' protection, the GPL clearly explains 45 | that there is no warranty for this free software. For both users' and 46 | authors' sake, the GPL requires that modified versions be marked as 47 | changed, so that their problems will not be attributed erroneously to 48 | authors of previous versions. 49 | 50 | Some devices are designed to deny users access to install or run 51 | modified versions of the software inside them, although the manufacturer 52 | can do so. This is fundamentally incompatible with the aim of 53 | protecting users' freedom to change the software. The systematic 54 | pattern of such abuse occurs in the area of products for individuals to 55 | use, which is precisely where it is most unacceptable. Therefore, we 56 | have designed this version of the GPL to prohibit the practice for those 57 | products. If such problems arise substantially in other domains, we 58 | stand ready to extend this provision to those domains in future versions 59 | of the GPL, as needed to protect the freedom of users. 60 | 61 | Finally, every program is threatened constantly by software patents. 62 | States should not allow patents to restrict development and use of 63 | software on general-purpose computers, but in those that do, we wish to 64 | avoid the special danger that patents applied to a free program could 65 | make it effectively proprietary. To prevent this, the GPL assures that 66 | patents cannot be used to render the program non-free. 67 | 68 | The precise terms and conditions for copying, distribution and 69 | modification follow. 70 | 71 | TERMS AND CONDITIONS 72 | 73 | 0. Definitions. 74 | 75 | "This License" refers to version 3 of the GNU General Public License. 76 | 77 | "Copyright" also means copyright-like laws that apply to other kinds of 78 | works, such as semiconductor masks. 79 | 80 | "The Program" refers to any copyrightable work licensed under this 81 | License. Each licensee is addressed as "you". "Licensees" and 82 | "recipients" may be individuals or organizations. 83 | 84 | To "modify" a work means to copy from or adapt all or part of the work 85 | in a fashion requiring copyright permission, other than the making of an 86 | exact copy. The resulting work is called a "modified version" of the 87 | earlier work or a work "based on" the earlier work. 88 | 89 | A "covered work" means either the unmodified Program or a work based 90 | on the Program. 91 | 92 | To "propagate" a work means to do anything with it that, without 93 | permission, would make you directly or secondarily liable for 94 | infringement under applicable copyright law, except executing it on a 95 | computer or modifying a private copy. Propagation includes copying, 96 | distribution (with or without modification), making available to the 97 | public, and in some countries other activities as well. 98 | 99 | To "convey" a work means any kind of propagation that enables other 100 | parties to make or receive copies. Mere interaction with a user through 101 | a computer network, with no transfer of a copy, is not conveying. 102 | 103 | An interactive user interface displays "Appropriate Legal Notices" 104 | to the extent that it includes a convenient and prominently visible 105 | feature that (1) displays an appropriate copyright notice, and (2) 106 | tells the user that there is no warranty for the work (except to the 107 | extent that warranties are provided), that licensees may convey the 108 | work under this License, and how to view a copy of this License. If 109 | the interface presents a list of user commands or options, such as a 110 | menu, a prominent item in the list meets this criterion. 111 | 112 | 1. Source Code. 113 | 114 | The "source code" for a work means the preferred form of the work 115 | for making modifications to it. "Object code" means any non-source 116 | form of a work. 117 | 118 | A "Standard Interface" means an interface that either is an official 119 | standard defined by a recognized standards body, or, in the case of 120 | interfaces specified for a particular programming language, one that 121 | is widely used among developers working in that language. 122 | 123 | The "System Libraries" of an executable work include anything, other 124 | than the work as a whole, that (a) is included in the normal form of 125 | packaging a Major Component, but which is not part of that Major 126 | Component, and (b) serves only to enable use of the work with that 127 | Major Component, or to implement a Standard Interface for which an 128 | implementation is available to the public in source code form. A 129 | "Major Component", in this context, means a major essential component 130 | (kernel, window system, and so on) of the specific operating system 131 | (if any) on which the executable work runs, or a compiler used to 132 | produce the work, or an object code interpreter used to run it. 133 | 134 | The "Corresponding Source" for a work in object code form means all 135 | the source code needed to generate, install, and (for an executable 136 | work) run the object code and to modify the work, including scripts to 137 | control those activities. However, it does not include the work's 138 | System Libraries, or general-purpose tools or generally available free 139 | programs which are used unmodified in performing those activities but 140 | which are not part of the work. For example, Corresponding Source 141 | includes interface definition files associated with source files for 142 | the work, and the source code for shared libraries and dynamically 143 | linked subprograms that the work is specifically designed to require, 144 | such as by intimate data communication or control flow between those 145 | subprograms and other parts of the work. 146 | 147 | The Corresponding Source need not include anything that users 148 | can regenerate automatically from other parts of the Corresponding 149 | Source. 150 | 151 | The Corresponding Source for a work in source code form is that 152 | same work. 153 | 154 | 2. Basic Permissions. 155 | 156 | All rights granted under this License are granted for the term of 157 | copyright on the Program, and are irrevocable provided the stated 158 | conditions are met. This License explicitly affirms your unlimited 159 | permission to run the unmodified Program. The output from running a 160 | covered work is covered by this License only if the output, given its 161 | content, constitutes a covered work. This License acknowledges your 162 | rights of fair use or other equivalent, as provided by copyright law. 163 | 164 | You may make, run and propagate covered works that you do not 165 | convey, without conditions so long as your license otherwise remains 166 | in force. You may convey covered works to others for the sole purpose 167 | of having them make modifications exclusively for you, or provide you 168 | with facilities for running those works, provided that you comply with 169 | the terms of this License in conveying all material for which you do 170 | not control copyright. Those thus making or running the covered works 171 | for you must do so exclusively on your behalf, under your direction 172 | and control, on terms that prohibit them from making any copies of 173 | your copyrighted material outside their relationship with you. 174 | 175 | Conveying under any other circumstances is permitted solely under 176 | the conditions stated below. Sublicensing is not allowed; section 10 177 | makes it unnecessary. 178 | 179 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 180 | 181 | No covered work shall be deemed part of an effective technological 182 | measure under any applicable law fulfilling obligations under article 183 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 184 | similar laws prohibiting or restricting circumvention of such 185 | measures. 186 | 187 | When you convey a covered work, you waive any legal power to forbid 188 | circumvention of technological measures to the extent such circumvention 189 | is effected by exercising rights under this License with respect to 190 | the covered work, and you disclaim any intention to limit operation or 191 | modification of the work as a means of enforcing, against the work's 192 | users, your or third parties' legal rights to forbid circumvention of 193 | technological measures. 194 | 195 | 4. Conveying Verbatim Copies. 196 | 197 | You may convey verbatim copies of the Program's source code as you 198 | receive it, in any medium, provided that you conspicuously and 199 | appropriately publish on each copy an appropriate copyright notice; 200 | keep intact all notices stating that this License and any 201 | non-permissive terms added in accord with section 7 apply to the code; 202 | keep intact all notices of the absence of any warranty; and give all 203 | recipients a copy of this License along with the Program. 204 | 205 | You may charge any price or no price for each copy that you convey, 206 | and you may offer support or warranty protection for a fee. 207 | 208 | 5. Conveying Modified Source Versions. 209 | 210 | You may convey a work based on the Program, or the modifications to 211 | produce it from the Program, in the form of source code under the 212 | terms of section 4, provided that you also meet all of these conditions: 213 | 214 | a) The work must carry prominent notices stating that you modified 215 | it, and giving a relevant date. 216 | 217 | b) The work must carry prominent notices stating that it is 218 | released under this License and any conditions added under section 219 | 7. This requirement modifies the requirement in section 4 to 220 | "keep intact all notices". 221 | 222 | c) You must license the entire work, as a whole, under this 223 | License to anyone who comes into possession of a copy. This 224 | License will therefore apply, along with any applicable section 7 225 | additional terms, to the whole of the work, and all its parts, 226 | regardless of how they are packaged. This License gives no 227 | permission to license the work in any other way, but it does not 228 | invalidate such permission if you have separately received it. 229 | 230 | d) If the work has interactive user interfaces, each must display 231 | Appropriate Legal Notices; however, if the Program has interactive 232 | interfaces that do not display Appropriate Legal Notices, your 233 | work need not make them do so. 234 | 235 | A compilation of a covered work with other separate and independent 236 | works, which are not by their nature extensions of the covered work, 237 | and which are not combined with it such as to form a larger program, 238 | in or on a volume of a storage or distribution medium, is called an 239 | "aggregate" if the compilation and its resulting copyright are not 240 | used to limit the access or legal rights of the compilation's users 241 | beyond what the individual works permit. Inclusion of a covered work 242 | in an aggregate does not cause this License to apply to the other 243 | parts of the aggregate. 244 | 245 | 6. Conveying Non-Source Forms. 246 | 247 | You may convey a covered work in object code form under the terms 248 | of sections 4 and 5, provided that you also convey the 249 | machine-readable Corresponding Source under the terms of this License, 250 | in one of these ways: 251 | 252 | a) Convey the object code in, or embodied in, a physical product 253 | (including a physical distribution medium), accompanied by the 254 | Corresponding Source fixed on a durable physical medium 255 | customarily used for software interchange. 256 | 257 | b) Convey the object code in, or embodied in, a physical product 258 | (including a physical distribution medium), accompanied by a 259 | written offer, valid for at least three years and valid for as 260 | long as you offer spare parts or customer support for that product 261 | model, to give anyone who possesses the object code either (1) a 262 | copy of the Corresponding Source for all the software in the 263 | product that is covered by this License, on a durable physical 264 | medium customarily used for software interchange, for a price no 265 | more than your reasonable cost of physically performing this 266 | conveying of source, or (2) access to copy the 267 | Corresponding Source from a network server at no charge. 268 | 269 | c) Convey individual copies of the object code with a copy of the 270 | written offer to provide the Corresponding Source. This 271 | alternative is allowed only occasionally and noncommercially, and 272 | only if you received the object code with such an offer, in accord 273 | with subsection 6b. 274 | 275 | d) Convey the object code by offering access from a designated 276 | place (gratis or for a charge), and offer equivalent access to the 277 | Corresponding Source in the same way through the same place at no 278 | further charge. You need not require recipients to copy the 279 | Corresponding Source along with the object code. If the place to 280 | copy the object code is a network server, the Corresponding Source 281 | may be on a different server (operated by you or a third party) 282 | that supports equivalent copying facilities, provided you maintain 283 | clear directions next to the object code saying where to find the 284 | Corresponding Source. Regardless of what server hosts the 285 | Corresponding Source, you remain obligated to ensure that it is 286 | available for as long as needed to satisfy these requirements. 287 | 288 | e) Convey the object code using peer-to-peer transmission, provided 289 | you inform other peers where the object code and Corresponding 290 | Source of the work are being offered to the general public at no 291 | charge under subsection 6d. 292 | 293 | A separable portion of the object code, whose source code is excluded 294 | from the Corresponding Source as a System Library, need not be 295 | included in conveying the object code work. 296 | 297 | A "User Product" is either (1) a "consumer product", which means any 298 | tangible personal property which is normally used for personal, family, 299 | or household purposes, or (2) anything designed or sold for incorporation 300 | into a dwelling. In determining whether a product is a consumer product, 301 | doubtful cases shall be resolved in favor of coverage. For a particular 302 | product received by a particular user, "normally used" refers to a 303 | typical or common use of that class of product, regardless of the status 304 | of the particular user or of the way in which the particular user 305 | actually uses, or expects or is expected to use, the product. A product 306 | is a consumer product regardless of whether the product has substantial 307 | commercial, industrial or non-consumer uses, unless such uses represent 308 | the only significant mode of use of the product. 309 | 310 | "Installation Information" for a User Product means any methods, 311 | procedures, authorization keys, or other information required to install 312 | and execute modified versions of a covered work in that User Product from 313 | a modified version of its Corresponding Source. The information must 314 | suffice to ensure that the continued functioning of the modified object 315 | code is in no case prevented or interfered with solely because 316 | modification has been made. 317 | 318 | If you convey an object code work under this section in, or with, or 319 | specifically for use in, a User Product, and the conveying occurs as 320 | part of a transaction in which the right of possession and use of the 321 | User Product is transferred to the recipient in perpetuity or for a 322 | fixed term (regardless of how the transaction is characterized), the 323 | Corresponding Source conveyed under this section must be accompanied 324 | by the Installation Information. But this requirement does not apply 325 | if neither you nor any third party retains the ability to install 326 | modified object code on the User Product (for example, the work has 327 | been installed in ROM). 328 | 329 | The requirement to provide Installation Information does not include a 330 | requirement to continue to provide support service, warranty, or updates 331 | for a work that has been modified or installed by the recipient, or for 332 | the User Product in which it has been modified or installed. Access to a 333 | network may be denied when the modification itself materially and 334 | adversely affects the operation of the network or violates the rules and 335 | protocols for communication across the network. 336 | 337 | Corresponding Source conveyed, and Installation Information provided, 338 | in accord with this section must be in a format that is publicly 339 | documented (and with an implementation available to the public in 340 | source code form), and must require no special password or key for 341 | unpacking, reading or copying. 342 | 343 | 7. Additional Terms. 344 | 345 | "Additional permissions" are terms that supplement the terms of this 346 | License by making exceptions from one or more of its conditions. 347 | Additional permissions that are applicable to the entire Program shall 348 | be treated as though they were included in this License, to the extent 349 | that they are valid under applicable law. If additional permissions 350 | apply only to part of the Program, that part may be used separately 351 | under those permissions, but the entire Program remains governed by 352 | this License without regard to the additional permissions. 353 | 354 | When you convey a copy of a covered work, you may at your option 355 | remove any additional permissions from that copy, or from any part of 356 | it. (Additional permissions may be written to require their own 357 | removal in certain cases when you modify the work.) You may place 358 | additional permissions on material, added by you to a covered work, 359 | for which you have or can give appropriate copyright permission. 360 | 361 | Notwithstanding any other provision of this License, for material you 362 | add to a covered work, you may (if authorized by the copyright holders of 363 | that material) supplement the terms of this License with terms: 364 | 365 | a) Disclaiming warranty or limiting liability differently from the 366 | terms of sections 15 and 16 of this License; or 367 | 368 | b) Requiring preservation of specified reasonable legal notices or 369 | author attributions in that material or in the Appropriate Legal 370 | Notices displayed by works containing it; or 371 | 372 | c) Prohibiting misrepresentation of the origin of that material, or 373 | requiring that modified versions of such material be marked in 374 | reasonable ways as different from the original version; or 375 | 376 | d) Limiting the use for publicity purposes of names of licensors or 377 | authors of the material; or 378 | 379 | e) Declining to grant rights under trademark law for use of some 380 | trade names, trademarks, or service marks; or 381 | 382 | f) Requiring indemnification of licensors and authors of that 383 | material by anyone who conveys the material (or modified versions of 384 | it) with contractual assumptions of liability to the recipient, for 385 | any liability that these contractual assumptions directly impose on 386 | those licensors and authors. 387 | 388 | All other non-permissive additional terms are considered "further 389 | restrictions" within the meaning of section 10. If the Program as you 390 | received it, or any part of it, contains a notice stating that it is 391 | governed by this License along with a term that is a further 392 | restriction, you may remove that term. If a license document contains 393 | a further restriction but permits relicensing or conveying under this 394 | License, you may add to a covered work material governed by the terms 395 | of that license document, provided that the further restriction does 396 | not survive such relicensing or conveying. 397 | 398 | If you add terms to a covered work in accord with this section, you 399 | must place, in the relevant source files, a statement of the 400 | additional terms that apply to those files, or a notice indicating 401 | where to find the applicable terms. 402 | 403 | Additional terms, permissive or non-permissive, may be stated in the 404 | form of a separately written license, or stated as exceptions; 405 | the above requirements apply either way. 406 | 407 | 8. Termination. 408 | 409 | You may not propagate or modify a covered work except as expressly 410 | provided under this License. Any attempt otherwise to propagate or 411 | modify it is void, and will automatically terminate your rights under 412 | this License (including any patent licenses granted under the third 413 | paragraph of section 11). 414 | 415 | However, if you cease all violation of this License, then your 416 | license from a particular copyright holder is reinstated (a) 417 | provisionally, unless and until the copyright holder explicitly and 418 | finally terminates your license, and (b) permanently, if the copyright 419 | holder fails to notify you of the violation by some reasonable means 420 | prior to 60 days after the cessation. 421 | 422 | Moreover, your license from a particular copyright holder is 423 | reinstated permanently if the copyright holder notifies you of the 424 | violation by some reasonable means, this is the first time you have 425 | received notice of violation of this License (for any work) from that 426 | copyright holder, and you cure the violation prior to 30 days after 427 | your receipt of the notice. 428 | 429 | Termination of your rights under this section does not terminate the 430 | licenses of parties who have received copies or rights from you under 431 | this License. If your rights have been terminated and not permanently 432 | reinstated, you do not qualify to receive new licenses for the same 433 | material under section 10. 434 | 435 | 9. Acceptance Not Required for Having Copies. 436 | 437 | You are not required to accept this License in order to receive or 438 | run a copy of the Program. Ancillary propagation of a covered work 439 | occurring solely as a consequence of using peer-to-peer transmission 440 | to receive a copy likewise does not require acceptance. However, 441 | nothing other than this License grants you permission to propagate or 442 | modify any covered work. These actions infringe copyright if you do 443 | not accept this License. Therefore, by modifying or propagating a 444 | covered work, you indicate your acceptance of this License to do so. 445 | 446 | 10. Automatic Licensing of Downstream Recipients. 447 | 448 | Each time you convey a covered work, the recipient automatically 449 | receives a license from the original licensors, to run, modify and 450 | propagate that work, subject to this License. You are not responsible 451 | for enforcing compliance by third parties with this License. 452 | 453 | An "entity transaction" is a transaction transferring control of an 454 | organization, or substantially all assets of one, or subdividing an 455 | organization, or merging organizations. If propagation of a covered 456 | work results from an entity transaction, each party to that 457 | transaction who receives a copy of the work also receives whatever 458 | licenses to the work the party's predecessor in interest had or could 459 | give under the previous paragraph, plus a right to possession of the 460 | Corresponding Source of the work from the predecessor in interest, if 461 | the predecessor has it or can get it with reasonable efforts. 462 | 463 | You may not impose any further restrictions on the exercise of the 464 | rights granted or affirmed under this License. For example, you may 465 | not impose a license fee, royalty, or other charge for exercise of 466 | rights granted under this License, and you may not initiate litigation 467 | (including a cross-claim or counterclaim in a lawsuit) alleging that 468 | any patent claim is infringed by making, using, selling, offering for 469 | sale, or importing the Program or any portion of it. 470 | 471 | 11. Patents. 472 | 473 | A "contributor" is a copyright holder who authorizes use under this 474 | License of the Program or a work on which the Program is based. The 475 | work thus licensed is called the contributor's "contributor version". 476 | 477 | A contributor's "essential patent claims" are all patent claims 478 | owned or controlled by the contributor, whether already acquired or 479 | hereafter acquired, that would be infringed by some manner, permitted 480 | by this License, of making, using, or selling its contributor version, 481 | but do not include claims that would be infringed only as a 482 | consequence of further modification of the contributor version. For 483 | purposes of this definition, "control" includes the right to grant 484 | patent sublicenses in a manner consistent with the requirements of 485 | this License. 486 | 487 | Each contributor grants you a non-exclusive, worldwide, royalty-free 488 | patent license under the contributor's essential patent claims, to 489 | make, use, sell, offer for sale, import and otherwise run, modify and 490 | propagate the contents of its contributor version. 491 | 492 | In the following three paragraphs, a "patent license" is any express 493 | agreement or commitment, however denominated, not to enforce a patent 494 | (such as an express permission to practice a patent or covenant not to 495 | sue for patent infringement). To "grant" such a patent license to a 496 | party means to make such an agreement or commitment not to enforce a 497 | patent against the party. 498 | 499 | If you convey a covered work, knowingly relying on a patent license, 500 | and the Corresponding Source of the work is not available for anyone 501 | to copy, free of charge and under the terms of this License, through a 502 | publicly available network server or other readily accessible means, 503 | then you must either (1) cause the Corresponding Source to be so 504 | available, or (2) arrange to deprive yourself of the benefit of the 505 | patent license for this particular work, or (3) arrange, in a manner 506 | consistent with the requirements of this License, to extend the patent 507 | license to downstream recipients. "Knowingly relying" means you have 508 | actual knowledge that, but for the patent license, your conveying the 509 | covered work in a country, or your recipient's use of the covered work 510 | in a country, would infringe one or more identifiable patents in that 511 | country that you have reason to believe are valid. 512 | 513 | If, pursuant to or in connection with a single transaction or 514 | arrangement, you convey, or propagate by procuring conveyance of, a 515 | covered work, and grant a patent license to some of the parties 516 | receiving the covered work authorizing them to use, propagate, modify 517 | or convey a specific copy of the covered work, then the patent license 518 | you grant is automatically extended to all recipients of the covered 519 | work and works based on it. 520 | 521 | A patent license is "discriminatory" if it does not include within 522 | the scope of its coverage, prohibits the exercise of, or is 523 | conditioned on the non-exercise of one or more of the rights that are 524 | specifically granted under this License. You may not convey a covered 525 | work if you are a party to an arrangement with a third party that is 526 | in the business of distributing software, under which you make payment 527 | to the third party based on the extent of your activity of conveying 528 | the work, and under which the third party grants, to any of the 529 | parties who would receive the covered work from you, a discriminatory 530 | patent license (a) in connection with copies of the covered work 531 | conveyed by you (or copies made from those copies), or (b) primarily 532 | for and in connection with specific products or compilations that 533 | contain the covered work, unless you entered into that arrangement, 534 | or that patent license was granted, prior to 28 March 2007. 535 | 536 | Nothing in this License shall be construed as excluding or limiting 537 | any implied license or other defenses to infringement that may 538 | otherwise be available to you under applicable patent law. 539 | 540 | 12. No Surrender of Others' Freedom. 541 | 542 | If conditions are imposed on you (whether by court order, agreement or 543 | otherwise) that contradict the conditions of this License, they do not 544 | excuse you from the conditions of this License. If you cannot convey a 545 | covered work so as to satisfy simultaneously your obligations under this 546 | License and any other pertinent obligations, then as a consequence you may 547 | not convey it at all. For example, if you agree to terms that obligate you 548 | to collect a royalty for further conveying from those to whom you convey 549 | the Program, the only way you could satisfy both those terms and this 550 | License would be to refrain entirely from conveying the Program. 551 | 552 | 13. Use with the GNU Affero General Public License. 553 | 554 | Notwithstanding any other provision of this License, you have 555 | permission to link or combine any covered work with a work licensed 556 | under version 3 of the GNU Affero General Public License into a single 557 | combined work, and to convey the resulting work. The terms of this 558 | License will continue to apply to the part which is the covered work, 559 | but the special requirements of the GNU Affero General Public License, 560 | section 13, concerning interaction through a network will apply to the 561 | combination as such. 562 | 563 | 14. Revised Versions of this License. 564 | 565 | The Free Software Foundation may publish revised and/or new versions of 566 | the GNU General Public License from time to time. Such new versions will 567 | be similar in spirit to the present version, but may differ in detail to 568 | address new problems or concerns. 569 | 570 | Each version is given a distinguishing version number. If the 571 | Program specifies that a certain numbered version of the GNU General 572 | Public License "or any later version" applies to it, you have the 573 | option of following the terms and conditions either of that numbered 574 | version or of any later version published by the Free Software 575 | Foundation. If the Program does not specify a version number of the 576 | GNU General Public License, you may choose any version ever published 577 | by the Free Software Foundation. 578 | 579 | If the Program specifies that a proxy can decide which future 580 | versions of the GNU General Public License can be used, that proxy's 581 | public statement of acceptance of a version permanently authorizes you 582 | to choose that version for the Program. 583 | 584 | Later license versions may give you additional or different 585 | permissions. However, no additional obligations are imposed on any 586 | author or copyright holder as a result of your choosing to follow a 587 | later version. 588 | 589 | 15. Disclaimer of Warranty. 590 | 591 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 592 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 593 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 594 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 595 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 596 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 597 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 598 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 599 | 600 | 16. Limitation of Liability. 601 | 602 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 603 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 604 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 605 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 606 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 607 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 608 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 609 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 610 | SUCH DAMAGES. 611 | 612 | 17. Interpretation of Sections 15 and 16. 613 | 614 | If the disclaimer of warranty and limitation of liability provided 615 | above cannot be given local legal effect according to their terms, 616 | reviewing courts shall apply local law that most closely approximates 617 | an absolute waiver of all civil liability in connection with the 618 | Program, unless a warranty or assumption of liability accompanies a 619 | copy of the Program in return for a fee. 620 | 621 | END OF TERMS AND CONDITIONS 622 | 623 | How to Apply These Terms to Your New Programs 624 | 625 | If you develop a new program, and you want it to be of the greatest 626 | possible use to the public, the best way to achieve this is to make it 627 | free software which everyone can redistribute and change under these terms. 628 | 629 | To do so, attach the following notices to the program. It is safest 630 | to attach them to the start of each source file to most effectively 631 | state the exclusion of warranty; and each file should have at least 632 | the "copyright" line and a pointer to where the full notice is found. 633 | 634 | 635 | Copyright (C) 636 | 637 | This program is free software: you can redistribute it and/or modify 638 | it under the terms of the GNU General Public License as published by 639 | the Free Software Foundation, either version 3 of the License, or 640 | (at your option) any later version. 641 | 642 | This program is distributed in the hope that it will be useful, 643 | but WITHOUT ANY WARRANTY; without even the implied warranty of 644 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 645 | GNU General Public License for more details. 646 | 647 | You should have received a copy of the GNU General Public License 648 | along with this program. If not, see . 649 | 650 | Also add information on how to contact you by electronic and paper mail. 651 | 652 | If the program does terminal interaction, make it output a short 653 | notice like this when it starts in an interactive mode: 654 | 655 | Copyright (C) 656 | This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 657 | This is free software, and you are welcome to redistribute it 658 | under certain conditions; type `show c' for details. 659 | 660 | The hypothetical commands `show w' and `show c' should show the appropriate 661 | parts of the General Public License. Of course, your program's commands 662 | might be different; for a GUI interface, you would use an "about box". 663 | 664 | You should also get your employer (if you work as a programmer) or school, 665 | if any, to sign a "copyright disclaimer" for the program, if necessary. 666 | For more information on this, and how to apply and follow the GNU GPL, see 667 | . 668 | 669 | The GNU General Public License does not permit incorporating your program 670 | into proprietary programs. If your program is a subroutine library, you 671 | may consider it more useful to permit linking proprietary applications with 672 | the library. If this is what you want to do, use the GNU Lesser General 673 | Public License instead of this License. But first, please read 674 | . -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | EMACS ?= emacs -Q --batch --load init.el -l org-sql.el 2 | 3 | all: test 4 | 5 | test: 6 | ${MAKE} stateless 7 | ${MAKE} stateful 8 | ${MAKE} compile 9 | 10 | docs: 11 | ${EMACS} \ 12 | -l doc/org-sql-doc.el \ 13 | -f create-docs-file \ 14 | -f org-sql-create-all-erds 15 | 16 | stateless: 17 | ${EMACS} -l test/org-sql-test-stateless.el -f buttercup-run-discover 18 | 19 | stateful: 20 | ${EMACS} -l test/org-sql-test-stateful.el -f buttercup-run-discover 21 | 22 | compile: 23 | ${EMACS} build 24 | ${MAKE} stateless 25 | ${MAKE} stateful 26 | ${MAKE} clean-elc 27 | 28 | clean-elc: 29 | ${EMACS} clean-elc 30 | 31 | # install all development packages for the current version 32 | install: 33 | ${EMACS} --eval '(print "Install finished")' 34 | 35 | # write lockfile for current emacs version given each repo dependency 36 | freeze: 37 | ${EMACS} -f straight-freeze-versions 38 | 39 | thaw: 40 | ${EMACS} -f straight-thaw-versions 41 | 42 | .PHONY: all test unit 43 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Org-SQL ![Github Workflow Status](https://github.com/ndwarshuis/org-sql/actions/workflows/test.yml/badge.svg) ![MELPA VERSION](https://melpa.org/packages/org-sql-badge.svg) 2 | 3 | This package converts org-mode files to Structured Query Language (SQL) and 4 | stores them in a database, which can then be used for comprehensive data 5 | analysis and visualization. Supports SQLite, PostgreSQL, MySQL/MariaDB, and 6 | Microsoft SQL-Server. 7 | 8 | # Installation 9 | 10 | Download the package from MELPA 11 | 12 | ``` 13 | M-x package-install RET org-sql RET 14 | ``` 15 | 16 | Alternatively, clone this repository into your config directory 17 | 18 | ``` sh 19 | git clone git@github.com:ndwarshuis/org-sql.git ~/config/path/org-sql/ 20 | ``` 21 | 22 | Once obtained, add the package to `load-path` and require it 23 | 24 | ``` emacs-lisp 25 | (add-to-list 'load-path "~/config/path/org-sql/") 26 | (require 'org-sql) 27 | ``` 28 | 29 | One can also use `use-package` to automate this entire process 30 | 31 | ``` emacs-lisp 32 | (use-package org-sql 33 | :ensure t 34 | :config 35 | ;; add config options here... 36 | ) 37 | ``` 38 | 39 | ## Dependencies 40 | 41 | Only Emacs 29.3 has been tested. It will probably work for others. 42 | 43 | ### Emacs packages 44 | 45 | - org-ml.el (5.8.8) 46 | - dash.el (2.19.1) 47 | - s.el (1.13) 48 | - f.el (0.20.0) 49 | 50 | Versions indicated those that have been tested. Others may work but are not 51 | guaranteed. 52 | 53 | As of version 5.8.8, org-ml.el requires org 9.6.x to work. 9.7.x and later 54 | *will* break. 55 | 56 | ### Database Clients 57 | 58 | Only the client binary for your desired implementation are required (ensure they 59 | are in your PATH): 60 | 61 | - sqlite3 62 | - psql (PostgreSQL) 63 | - mysql (MariaDB/MySQL) 64 | - sqlcmd (SQL-Server) 65 | 66 | See the conda environment file at `env-XX.Y.yml` (where XX.Y corresponds to the 67 | emacs version) for the exact versions of the each client used for testing. These 68 | were provided with the following packages: 69 | 70 | - sqlite (for sqlite) 71 | - postgresql (for PostgreSQL) 72 | - mysqlclient (for MariaDB and MySQL) 73 | - go-sqlcmd (for SQL-Server) 74 | 75 | 76 | ### Database Servers 77 | 78 | The following databases servers/versions are supported and tested: 79 | 80 | - PostgreSQL (16, 15, 14, 13) 81 | - MariaDB (11.4, 10.6, 10.5) 82 | - MySQL (8.4, 8.0) 83 | - SQL-Server (2022, 2019, 2017) 84 | 85 | Many versions besides these will likely work; these are simply those that are in 86 | the testing suite. 87 | 88 | # Configuration 89 | 90 | ## General Behavior 91 | 92 | - `org-sql-files`: list of org files to sync with the database 93 | - `org-sql-async`: turn on to spawn the database client process asynchronously 94 | (and hence not block Emacs while the client is updating the database) 95 | - `org-sql-debug`: turn on SQL transaction debug output in the message buffer 96 | 97 | ## Database Storage 98 | 99 | Options following the pattern `org-sql-exclude-X` or `org-sql-excluded-X` 100 | dictate what not to store in the database. By default all these variables are 101 | nil (include everything). See the help page for each of these for further 102 | details. 103 | 104 | ## Database Connection 105 | 106 | The database connection is controlled by `org-sql-db-config`. This is where one 107 | would choose the database client (and server connection if applicable) as well 108 | as database-specific behavior. The format for this variable is like `(DB-TYPE 109 | [KEY VAL] [[KEY VAL] ...])` where `DB-TYPE` is the type of database to use and 110 | the `KEY-VAL` pairs are options for that database. 111 | 112 | `DB-TYPE` is one of `sqlite`, `pgsql`, `mysql`, or `sqlserver` (a symbol). 113 | 114 | An explanation of the `KEY-VAL` pairs is below. 115 | 116 | ### General Guidelines 117 | 118 | While the docstring of `org-sql-db-config` is a good reference, the following 119 | are some sane guidelines for each database configuration. 120 | 121 | #### SQLite 122 | 123 | This is by far the simplest and only requires the `:path` key (the path to the 124 | file where the database will be stored). 125 | 126 | #### All Databases Except SQLite 127 | 128 | The only required key for these is `:database` which is the database name to use. 129 | 130 | Most other needs should be satisfied by the database-specific keys in each 131 | subsection below. If your configuration requires more than this, the `:args` 132 | and `:env` key exists as catchall keys. The former is a list of additional 133 | arguments to send to the client command, and the latter is a list of 2-membered 134 | lists like `(VAR VAL)` which sets the environmental values with which the client 135 | command will run. Consult the documentation for the client command (eg `psql`, 136 | `mysql`, or `sqlcmd`) for which arguments and environemtal variables make sense. 137 | 138 | #### Postgres 139 | 140 | Likely one would set the `:hostname` key unless using the localhost. 141 | 142 | From here many other options are possible. A simple setup (eg one using a 143 | straightforward docker deployment) might define a username, password, and port 144 | (denoted by the `:username`, `:password`, and `:port` keys respectively). If the 145 | database stores other data alongside that from `org-sql`, one can create a 146 | schema specifically for `org-sql` and set the `:schema` key with the name of 147 | this schema. 148 | 149 | To prevent leaking a password in plain text, one can use a `.pgpass` file as 150 | normally used with the `psql` command, or set the `:pass-file` key to the path 151 | of the password file. More advanced setups can utilize the `.pg_service` file as 152 | normal, or set the `:service-file` key to the desired path to the service file. 153 | 154 | As an additional performance optimization, set the `:unlogged` key as t to use 155 | unlogged tables. This may significantly boost performance, particularly for 156 | functions in `org-sql` that do bulk inserts (eg `org-sql-user-push` and 157 | `org-sql-push-to-db`). The tradeoff is data loss if the database crashes during 158 | a transaction, which may be acceptable if the org-files denoted by 159 | `org-sql-files` are more permanent than the database itself. NOTE: this only 160 | sets the unlogged property on the tables that `org-sql` uses; no other tables 161 | will be changed. 162 | 163 | #### MySQL/MariaDB 164 | 165 | Likely one would set the `:hostname` key unless using the localhost. 166 | 167 | Similar to Postgres, a simple setup might define a username, password, and port 168 | (denoted by the `:username`, `:password`, and `:port` keys respectively). Unlike 169 | Postgres, one might also need to set the `:args` key with `"--protocol=TCP"` if 170 | using a TCP connection (see above for explanation of `:args`). 171 | 172 | To prevent leaking a password in plain text, one can use an options file as 173 | normally used with the `mysql` command (eg `.my.cnf`), as well as any other 174 | connection parameters in the place of keys. If the options file is in a 175 | non-default location, set it with the `:defaults-file` key. A similar key exists 176 | for the defaults-extra file (`:defaults-extra-file`). 177 | 178 | #### SQL Server 179 | 180 | Likely one would set the `:server` key to denote the instance of the server 181 | to use (eg `"tcp:example.com,1443"`). Note that this takes the place of the 182 | `:hostname`/`:port` keys for MySQL and Postgres. 183 | 184 | Specify the username and password using the keys `:username` and `password` 185 | respectively. If the database stores other data alongside that from `org-sql`, 186 | one can create a schema specifically for `org-sql` and set the `:schema` key 187 | with the name of this schema. 188 | 189 | To prevent hardcoding the password in Emacs code, one can set the `"SQLCMDINI"` 190 | environmental variable in the `:env` key (see above) to the path of a startup 191 | file which sets the password using the `"SQLCMDPASSWORD"` environmental 192 | variable. 193 | 194 | ## Database Preparation 195 | 196 | Since `org-sql` cannot assume it has superuser access to your database and/or 197 | filesystem, external configuration will be necessary in many cases before 198 | running any commands with this package. 199 | 200 | ### SQLite 201 | 202 | The only configuration necessary is to ensure that the path denoted by `:path` 203 | is writable to the same user running emacs. 204 | 205 | ### Postgres and SQL-Server 206 | 207 | The database server as well as the database itself (eg the database defined by 208 | the `:database` key) must already exist. Additionally, there must be a role 209 | defined that `org-sql` can use for the connection. If `:schema` is non-nil, the 210 | schema defined by this key must already exist. If it is undefined, `org-sql` 211 | will use the default schema (`public` for Postgres and usually `dbo` for 212 | SQL-Server). In any case, the role to be used by `org-sql` must have 213 | authorization to create tables and insert/delete rows from those tables on the 214 | configured schema. 215 | 216 | See init files for [Postgres](test/docker/postgres/init/org_sql.sql) and 217 | [SQL-Server](test/docker/sql-server/setup.sql) for bare-bones examples. 218 | 219 | ### MySQL/MariaDB 220 | 221 | The database server must already exist and the database defined by the 222 | `:database` key must also already exist. The user used by `org-sql` to connect 223 | must have permissions to create tables and insert/delete data from said tables. 224 | 225 | See the init file for [MariaDB](test/docker/mariadb/init/org_sql.sql) for 226 | bare-bones example. 227 | 228 | ## Database Customization 229 | 230 | `org-sql` by default will only create tables (with pimary and foreign keys) and 231 | insert/delete data in these tables. If you want to do anything beyond this such 232 | as creating additional indexes, adding triggers, defining and calling 233 | procedures, etc, one can do so through 'hooks'. These are variables that hold 234 | additional SQL statements that will be run along with the functions in 235 | `org-sql`. 236 | 237 | These variables are: 238 | - `org-sql-post-init-hooks`: run after `org-sql-init-db` 239 | - `org-sql-post-push-hooks`: run after `org-sql-push-to-db` 240 | - `org-sql-post-clear-hooks`: run after `org-sql-clear-db` 241 | - `org-sql-pre-reset-hooks`: run before `org-sql-reset-db` 242 | 243 | See the docstrings of these variables for how to define the custom SQL 244 | statements and how to control their execution. 245 | 246 | # Usage 247 | 248 | ## Interactive Functions 249 | 250 | The following functions can be invoked using `M-x` and should cover the simple 251 | use case of creating a database and syncing org files to it. 252 | 253 | ### Initializing 254 | 255 | Run `org-sql-user-init`. In the case of SQLite, this will create a new database 256 | file. In all cases this will create the tables associated with `org-sql`. 257 | 258 | ### Inserting data 259 | 260 | Run `org-sql-user-push`. This will synchronize the database with all files as 261 | indicated in `org-sql-files` by first checking if the file is in the database 262 | and inserting it if not. Any renamed files will also be updated. If the contents 263 | of a file are changed, the entire file is deleted from the database and 264 | reinserted. Files with identical contents are only stored once (with the 265 | exception of the file paths and attributes that point to the identical files). 266 | 267 | This may take several seconds/minutes if inserting many files depending on the 268 | speed of your device (particularly IO) and the size/number of files. This 269 | operation will also block Emacs until complete. Even if `org-sql-async` is t, 270 | Emacs will still block for all computation internal to Emacs (getting the 271 | org-element trees and converting them to SQL statements that will sync their 272 | contents). 273 | 274 | If performance/blocking is a concern, the best way to improve update speeds is 275 | to use many small org files rather than a few big ones. Because the only 276 | efficient way to 'update' a file is to delete and reinsert it into the database, 277 | changing one character in a large file will cause that entire file to be 278 | inserted. 279 | 280 | ### Removing all data 281 | 282 | Run `org-sql-user-clear-all`. This will clear all data but leave the schema. 283 | 284 | ### Resetting 285 | 286 | Run `org-sql-user-reset`. This will drop all tables associated with `org-sql`. 287 | In the case of SQLite, this will also delete the database file. 288 | 289 | ### Pulling data out 290 | 291 | If you make changes in the database, run `org-sql-user-pull` to obtain the 292 | current database state. This will return a list where each member has the file 293 | path and its corresponding org-tree. Each org-tree can then be converted to a 294 | string using `org-ml-to-string` from the `org-ml` library. 295 | 296 | For now this will pull all the contents of the database. Fine-grained query 297 | control is planned for a future release. 298 | 299 | ### Debugging 300 | 301 | The interactive functions above will print a "success" message if the client 302 | command returns an exit code of 0. While a non-zero exit code almost certainly 303 | means something went wrong, **the transaction may still have failed even if the 304 | client returned 0.** If running a command seems to have no effect on the 305 | database, set `org-sql-debug` to t and run the command again. This will print 306 | any additional output given by the client (which are configured when called by 307 | `org-sql` to print errors to stdout/stderr) and will likely explain what went 308 | wrong. 309 | 310 | Additionally, the command `org-sql-dump-push-transaction` will print the 311 | transaction used by the `org-sql-push-to-db` and `org-sql-user-push` commands. 312 | 313 | ## Public API 314 | 315 | `org-sql` exposes the following public functions for interacting with the 316 | database beyond the use cases covered by the above interactive functions: 317 | 318 | - Table-level Operations 319 | - `org-sql-create-tables` 320 | - `org-sql-drop-tables` 321 | - `org-sql-list-tables` 322 | - Database-level Operations 323 | - `org-sql-create-db` 324 | - `org-sql-drop-db` 325 | - `org-sql-db-exists` 326 | - Init/Teardown Operations 327 | - `org-sql-init-db` 328 | - `org-sql-reset-db` 329 | - Data-level Operations 330 | - `org-sql-dump-table` 331 | - `org-sql-push-to-db` 332 | - `org-sql-pull-from-db` 333 | - `org-sql-clear-db` 334 | - Other SQL Commands 335 | - `org-sql-send-sql` 336 | 337 | # Limitations 338 | 339 | ## OS support 340 | 341 | This has currently only been tested on Linux and will likely break on Windows 342 | (it may work on MacOS). Support for other operating systems is planned for 343 | future releases. 344 | 345 | ## Logbook variables 346 | 347 | The structure of the logbook (eg the the thing that holds clocks, notes, state 348 | changes, etc under a given headline) is determined by several variables. 349 | `org-sql` understands these three: 350 | - `org-log-into-drawer` 351 | - `org-clock-into-drawer` 352 | - `org-log-note-clock-out` 353 | 354 | These variables (may not be exhaustive) are **not** understood: 355 | - `org-log-state-notes-insert-after-drawers` 356 | - `org-log-note-headings`. 357 | 358 | The reason for this scope of support is due to `org-ml`, the library on which 359 | `org-sql` depends to parse org-mode syntax. 360 | 361 | ### Inserting data 362 | 363 | When inserting data (`org-sql-push-to-db`), the three supported variables above 364 | will be used to determine what a valid logbook *should* look like. File-level 365 | (eg defined with the `#+PROPERTY` keyword) and headline-level (eg defined in a 366 | `PROPERTIES` drawer) values of these variables are also understood. 367 | 368 | This has two consequences: 369 | 1) modifications to any unsupported variable that change the logbook may result 370 | in an unrecognizeable logbook that will not be inserted into the proper 371 | tables (most likely it will end up in the `headlines` table under the 372 | `contents` column) 373 | 2) manual edits to the logbook that are out of sync with how it would normally 374 | be produced using the given variables will result in a similar situation as 375 | (1) 376 | 377 | ### Pulling data 378 | 379 | When pulling data (`org-sql-pull-from-db`), the three supported variables are 380 | used to reassemble the logbook data from the database into org syntax. Any 381 | unsupported variables will simply be ignored. 382 | 383 | Unlike `org-sql-push-to-db`, the pull mechanism corrently only considers the 384 | global value of the three supported variables. Support for file- and 385 | headline-level values is planned for a future release. 386 | 387 | # Database Layout 388 | 389 | ## General design features 390 | 391 | - All foreign keys are set with `DELETE CASCADE` 392 | - All time values are store as unixtime (integers in seconds) 393 | - No triggers or indexes (outside of the primary keys) are created by `org-sql` 394 | 395 | ## Entity Relationship Diagrams 396 | 397 | The table layouts for each implementation are more or less identical; the only 398 | differences are the types. 399 | 400 | [MySQL/MariaDB](doc/erd/erd-mysql.png) 401 | 402 | [PostgreSQL](doc/erd/erd-postgres.png) 403 | 404 | [SQLite](doc/erd/erd-sqlite.png) 405 | 406 | [SQL Server](doc/erd/erd-sql-server.png) 407 | 408 | ## Table Descriptions 409 | 410 | See [here](doc/table-descriptions.md) for 411 | a description of each table and its columns. 412 | 413 | # Contributing 414 | 415 | Contributions welcome! But please take advantage of the testing environment 416 | (especially when contributing to code which directly interacts with the database 417 | servers). 418 | 419 | ## Reproducible Environment 420 | 421 | The entire development environment is designed to be self-contained and 422 | reproducible. All binaries (including emacs itself) are specified in a conda 423 | environment, the databases are specified in a docker-compose file, and the 424 | dependencies for emacs are specified in a `straight.el` profile in 425 | `.emacs/XX.Y/straight/versions/default.el`. 426 | 427 | The only prerequisites for running this are a working conda and docker-compose 428 | installation. 429 | 430 | It isn't stricly necessary to use this, but doing so ensures that all tests run 431 | in a standardized manner across all machines and therefore minimizes 'bit rot 432 | bugs'. 433 | 434 | ### Conda dependencies 435 | 436 | Assuming a working mamba installation, install all binary dependencies and 437 | activate: 438 | 439 | ``` sh 440 | mamba env create -f env-XX.Y.yml 441 | conda activate env-XX.Y.yml 442 | ``` 443 | 444 | ### Emacs dependencies 445 | 446 | Run the following 447 | 448 | ``` sh 449 | export LD_PRELOAD=/usr/lib/libc_malloc_debug.so 450 | make install 451 | ``` 452 | 453 | Note, the LD_PRELOAD setting is necessary if one gets and error about undefined 454 | symbols for malloc_set_state. This is necessary for all `make ...` commands. 455 | 456 | ### Database setup 457 | 458 | Except for SQLite, the each database for testing is encoded and set up using 459 | `docker-compose` (see the included `docker-compose.yml` file). These are 460 | necessary to run the stateful tests (see below). 461 | 462 | To set up the environment, start the docker-daemon (may require sudo). 463 | 464 | ``` sh 465 | docker-compose up -d -V 466 | ``` 467 | 468 | Add `--build` to rebuild images if altered (see below). 469 | 470 | To shut down the environment: 471 | 472 | ``` sh 473 | docker-compose down 474 | ``` 475 | 476 | #### Dockerfile/Docker-compose Layout 477 | 478 | Customization of the `docker-compose` files should not be necessary except when 479 | adding a new database for testing (or a new version). The 'base' docker images 480 | are defined using [Dockerfiles](test/docker), which in turn are built with SQL 481 | initialization scripts (which are necessary to test the containers with minimal 482 | privileges). Each Dockerfile has an overridable `IMAGE` argument whose default 483 | is set to the latest version of the container to pull. Note that MariaDB and 484 | MySQL are assumed to share the exact same container configuration, and thus they 485 | share the same Dockerfile. 486 | 487 | ### Running tests 488 | 489 | Tests are divided into stateless (pure functions, don't rely on external 490 | database implementations) and stateful (impure, interacts with the database 491 | and/or files on disk). 492 | 493 | Run all stateless tests: 494 | 495 | ``` 496 | make stateless 497 | ``` 498 | 499 | Run all stateful tests: 500 | 501 | ``` 502 | make stateful 503 | ``` 504 | 505 | Compile code and run stateful and stateless tests: 506 | 507 | ``` 508 | make compile 509 | ``` 510 | 511 | Run all tests using both interpreted and compiled code: 512 | 513 | ``` 514 | make test 515 | ``` 516 | 517 | ### Building documentation 518 | 519 | To generate documentation: 520 | 521 | ``` 522 | make docs 523 | ``` 524 | 525 | requires [erd](https://github.com/BurntSushi/erd) (which is unfortunately not in 526 | conda). 527 | 528 | ## Interactive development 529 | 530 | To use Emacs to edit the code of `org-sql`, one has several options, from most 531 | to least reliable. 532 | 533 | ### Emacs from conda 534 | 535 | Simply activate the conda environment set up from above and run emacs from the 536 | shell. This will have all the required dependencies, but also won't have your 537 | personal setup. 538 | 539 | ### Personal emacs with straight dependencies 540 | 541 | More clunkily, if one wants/needs the exact versions of each emacs package, one 542 | can copy the hashes from `emacs.d/XX.Y/straight/versions/default.el` into their 543 | own straight config (or make a new profile). 544 | 545 | ### Personal emacs config with cask 546 | 547 | Install [cask](https://github.com/cask/cask) and run the following (without the 548 | conda env activated): 549 | 550 | ``` sh 551 | cask install 552 | ``` 553 | 554 | Run emacs as normal, and activate the conda environment to run the tests. 555 | 556 | # Acknowledgements 557 | 558 | The idea for this is based on [John 559 | Kitchin's](http://kitchingroup.cheme.cmu.edu/blog/2017/01/03/Find-stuff-in-org-mode-anywhere/) 560 | implementation, which uses `emacsql` as the SQL backend. 561 | -------------------------------------------------------------------------------- /doc/erd/erd-mysql.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ndwarshuis/org-sql/3dbf11d692cf0d5e64235ad4041ed0b5a6775064/doc/erd/erd-mysql.png -------------------------------------------------------------------------------- /doc/erd/erd-postgres.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ndwarshuis/org-sql/3dbf11d692cf0d5e64235ad4041ed0b5a6775064/doc/erd/erd-postgres.png -------------------------------------------------------------------------------- /doc/erd/erd-sqlite.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ndwarshuis/org-sql/3dbf11d692cf0d5e64235ad4041ed0b5a6775064/doc/erd/erd-sqlite.png -------------------------------------------------------------------------------- /doc/erd/erd-sqlserver.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ndwarshuis/org-sql/3dbf11d692cf0d5e64235ad4041ed0b5a6775064/doc/erd/erd-sqlserver.png -------------------------------------------------------------------------------- /doc/org-sql-doc.el: -------------------------------------------------------------------------------- 1 | ;;; org-sql-doc.el --- Build documentation for org-sql -*- lexical-binding: t; -*- 2 | 3 | ;; Copyright (C) 2020 Nathan Dwarshuis 4 | 5 | ;; This program is free software; you can redistribute it and/or modify 6 | ;; it under the terms of the GNU General Public License as published by 7 | ;; the Free Software Foundation, either version 3 of the License, or 8 | ;; (at your option) any later version. 9 | 10 | ;; This program is distributed in the hope that it will be useful, 11 | ;; but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | ;; GNU General Public License for more details. 14 | 15 | ;; You should have received a copy of the GNU General Public License 16 | ;; along with this program. If not, see . 17 | 18 | ;;; Commentary: 19 | 20 | ;;; These are functions to build the ERDs and table descriptions 21 | 22 | ;;; Code: 23 | 24 | (require 'package) 25 | (require 'dash) 26 | (require 's) 27 | (require 'f) 28 | (require 'org-sql) 29 | 30 | (defconst org-sql-doc-dir "doc" 31 | "The location of the doc files") 32 | 33 | ;;; entity relationship diagrams 34 | 35 | (defun org-sql-er-format-column (pks fks config column) 36 | (-let* (((name . (&plist :type :constraints)) column) 37 | (type* (if (eq type 'enum) 38 | (org-sql--case-mode config 39 | ((mysql postgres) "ENUM") 40 | ((sqlite sqlserver) 41 | (org-sql--format-create-tables-type config name column))) 42 | (org-sql--format-create-tables-type config name column))) 43 | (constraints* (-some->> constraints 44 | (org-sql--format-column-constraints))) 45 | (name* (--> (org-sql--format-column-name name) 46 | (if (memq name pks) (format "*%s" it) it) 47 | (if (memq name fks) (format "+%s" it) it)))) 48 | (format "%s {label: \"%s\"}" name* (s-join ", " (-non-nil (list type* constraints*)))))) 49 | 50 | (defun org-sql-er-format-table (config table) 51 | (-let* (((name . (&alist 'columns 'constraints)) table) 52 | (pks (plist-get (alist-get 'primary constraints) :keys)) 53 | (fks (->> (--filter (eq (car it) 'foreign) constraints) 54 | (--mapcat (plist-get (cdr it) :keys)) 55 | (-uniq))) 56 | (columns* (->> columns 57 | (--map (org-sql-er-format-column pks fks config it)) 58 | (s-join "\n")))) 59 | (format "[%s]\n%s" name columns*))) 60 | 61 | (defun org-sql-er-get-tables (config) 62 | (->> org-sql--table-alist 63 | (--map (org-sql-er-format-table config it)) 64 | (s-join "\n\n"))) 65 | 66 | (defun org-sql-er-format-table-relationships (table) 67 | (-let* (((name . (&alist 'constraints)) table) 68 | (fks (--filter (eq 'foreign (car it)) constraints))) 69 | (--map (-let* (((&plist :ref :cardinality :keys) (cdr it)) 70 | (rel (pcase cardinality 71 | (`nil (error "Need cardinality for %s" name)) 72 | (`one-to-one "1--1") 73 | (`one-or-none-to-one "?--1") 74 | (`many-to-one "+--1") 75 | (`many-or-none-to-one "*--1") 76 | (e (error "Unknown cardinality: %s" e)))) 77 | (parent (plist-get (cdr it) :ref))) 78 | (if (< 1 (length fks)) 79 | (->> (-map #'org-sql--format-column-name keys) 80 | (s-join ", ") 81 | (format "%s %s %s {label: \"%s\"}" name rel parent)) 82 | (format "%s %s %s" name rel parent))) 83 | fks))) 84 | 85 | (defun org-sql-er-get-relationships () 86 | (->> org-sql--table-alist 87 | (-mapcat #'org-sql-er-format-table-relationships) 88 | (-non-nil) 89 | (s-join "\n"))) 90 | 91 | (defun org-sql-format-er-file (config) 92 | (let* ((db-name (org-sql--case-mode config 93 | (mysql "MySQL/MariaDB") 94 | (postgres "PostgreSQL") 95 | (sqlite "SQLite") 96 | (sqlserver "SQL Server"))) 97 | (title (format "title {label: \"Org-SQL ERD (%s)\"}" db-name)) 98 | (tables (org-sql-er-get-tables config)) 99 | (relationships (org-sql-er-get-relationships))) 100 | (s-join "\n\n" (list title tables relationships)))) 101 | 102 | (defun org-sql-write-erd (config) 103 | (if (not (executable-find "erd")) 104 | (error "Install erd to complete documentation") 105 | (let ((er (org-sql-format-er-file config)) 106 | (inpath (f-join (temporary-file-directory) "org-sql-erd.er")) 107 | (outpath (->> (org-sql--case-mode config 108 | (mysql "mysql") 109 | (postgres "postgres") 110 | (sqlite "sqlite") 111 | (sqlserver "sqlserver")) 112 | (format "erd-%s.png") 113 | (f-join org-sql-doc-dir "erd")))) 114 | (f-write-text er 'utf-8 inpath) 115 | (call-process "erd" nil nil nil "-i" inpath "-o" outpath) 116 | (f-delete inpath t)))) 117 | 118 | (defun org-sql-create-all-erds () 119 | (-each '((mysql) (postgres) (sqlite) (sqlserver)) #'org-sql-write-erd)) 120 | 121 | ;;; table descriptions 122 | 123 | (defun org-sql-get-package-version () 124 | "Get version of om package." 125 | (with-current-buffer (find-file-noselect "org-sql.el") 126 | (->> (package-buffer-info) 127 | (package-desc-version) 128 | (-map 'number-to-string) 129 | (s-join version-separator)))) 130 | 131 | (defun org-sql-doc-format-quotes (s) 132 | (s-replace-regexp "`\\([^[:space:]]+\\)'" "`\\1`" s)) 133 | 134 | (defun org-sql-doc-format-table-row (members) 135 | (format "| %s |" (s-join " | " members))) 136 | 137 | (defun org-sql-doc-format-allowed-values (members) 138 | (when members 139 | (-let* ((members* (--map (format "`%s`" it) members)) 140 | ((rest last) (-split-at (1- (length members*)) members*)) 141 | (rest* (s-join ", " rest))) 142 | (format "%s, or %s" rest* (car last))))) 143 | 144 | ;; TODO this is a good idea but could be clearer, perhaps a separate column? 145 | ;; (defun org-sql-doc-format-org-element-props (desc props) 146 | ;; (cl-flet 147 | ;; ((join-with-and 148 | ;; (xs) 149 | ;; (format "%s and %s" (s-join ", " (-drop-last 1 xs)) (-last-item xs)))) 150 | ;; (cond 151 | ;; ((= 0 (length props)) 152 | ;; desc) 153 | ;; ((= 1 (length props)) 154 | ;; (->> (car props) 155 | ;; (format "%s (corresponds to the org-element `%s` property)" desc ))) 156 | ;; ((< 1 (length props)) 157 | ;; (->> (--map (format "`%s`" it) props) 158 | ;; (join-with-and) 159 | ;; (format "%s (corresponds to the org-element %s properties)" desc)))))) 160 | 161 | (defun org-sql-doc-format-column (column-meta constraints-meta) 162 | (-let* (((column-name . (&plist :desc :type :properties :constraints :allowed)) 163 | column-meta) 164 | ((&alist 'primary) constraints-meta) 165 | (primary-keys (plist-get primary :keys)) 166 | (column-name* (org-sql--format-column-name column-name)) 167 | (allowed* (org-sql-doc-format-allowed-values allowed)) 168 | (desc* (-> (if (consp desc) (s-join " " desc) desc))) 169 | ;; (org-sql-doc-format-org-element-props properties))) 170 | ;; (org-sql-doc-format-quotes it) 171 | ;; (if property 172 | ;; (format "%s (corresponds to the org-element `%s` property)" 173 | ;; it property) 174 | ;; it))) 175 | (desc-full (if allowed* (format "%s (%s)" desc* allowed*) desc*))) 176 | (->> (list column-name* desc-full) 177 | (org-sql-doc-format-table-row)))) 178 | 179 | (defun org-sql-doc-format-schema (table-meta) 180 | (-let* (((table-name . meta) table-meta) 181 | ((&alist 'desc 'columns 'constraints) meta) 182 | (header (->> (symbol-name table-name) 183 | (format "## %s"))) 184 | (table-headers (list "Column" "Description")) 185 | (table-line (->> (-repeat (length table-headers) " - ") 186 | (org-sql-doc-format-table-row))) 187 | (table-headers* (org-sql-doc-format-table-row table-headers)) 188 | (table-rows (->> columns 189 | (--map (org-sql-doc-format-column it constraints)) 190 | (s-join "\n")))) 191 | (->> (list header 192 | "" 193 | (s-join " " desc) 194 | "" 195 | table-headers* 196 | table-line 197 | table-rows) 198 | (s-join "\n")))) 199 | 200 | (defun create-docs-file () 201 | (with-temp-file (f-join org-sql-doc-dir "table-descriptions.md") 202 | (insert "# Table Descriptions\n\n") 203 | 204 | (insert (->> org-sql--table-alist 205 | (-map #'org-sql-doc-format-schema) 206 | (s-join "\n\n"))) 207 | 208 | (insert "\n\n") 209 | 210 | (insert (format "Version: %s" (org-sql-get-package-version))))) 211 | 212 | ;;; org-sql-doc.el ends here 213 | -------------------------------------------------------------------------------- /doc/table-descriptions.md: -------------------------------------------------------------------------------- 1 | # Table Descriptions 2 | 3 | ## outlines 4 | 5 | Each row stores the hash, size, and toplevel section for an org file (here called an `outline`). Note that if there are identical org files, only one `outline` will be stored in the database (as determined by the unique hash) and the paths shared the outline will be reflected in the `file_metadata` table. 6 | 7 | | Column | Description | 8 | | - | - | 9 | | outline_hash | hash (MD5) of this org outline | 10 | | outline_size | number of characters of the org outline | 11 | | outline_lines | number of lines in the org file | 12 | | outline_preamble | the content before the first headline | 13 | 14 | ## file_metadata 15 | 16 | Each row stores filesystem metadata for one tracked org file. 17 | 18 | | Column | Description | 19 | | - | - | 20 | | file_path | path to org file | 21 | | outline_hash | hash (MD5) of the org outline with this path | 22 | | file_uid | UID of the file | 23 | | file_gid | GID of the file | 24 | | file_modification_time | time of the file's last modification | 25 | | file_attr_change_time | time of the file's last attribute change | 26 | | file_modes | permission mode bits for the file | 27 | 28 | ## headlines 29 | 30 | Each row stores one headline in a given org outline. 31 | 32 | | Column | Description | 33 | | - | - | 34 | | headline_id | id of this headline | 35 | | outline_hash | hash (MD5) of the org outline with this headline | 36 | | headline_text | raw text of the headline without leading stars or tags | 37 | | level | the level of this headline | 38 | | headline_index | the order of this headline relative to its neighbors | 39 | | keyword | the TODO state keyword | 40 | | effort | the value of the `Effort` property in minutes | 41 | | priority | character value of the priority | 42 | | stats_cookie_type | type of the statistics cookie (the `[n/d]` or `[p%]` at the end of some headlines) (`fraction`, or `percent`) | 43 | | stats_cookie_value | value of the statistics cookie (between 0 and 1) | 44 | | is_archived | TRUE if the headline has an ARCHIVE tag | 45 | | is_commented | TRUE if the headline has a COMMENT keyword | 46 | | content | the headline contents (everything after the planning entries, property-drawer, and/or logbook) | 47 | 48 | ## headline_closures 49 | 50 | Each row stores the ancestor and depth of a headline relationship. All headlines will have a 0-depth entry in which `parent_id` and `headline_id` are equal. 51 | 52 | | Column | Description | 53 | | - | - | 54 | | headline_id | id of this headline | 55 | | parent_id | id of this headline's parent | 56 | | depth | levels between this headline and the referred parent | 57 | 58 | ## timestamps 59 | 60 | Each row stores one timestamp. Any timestamps in this table that are not referenced in other tables are part of the headlines's contents (the part after the logbook) or title. 61 | 62 | | Column | Description | 63 | | - | - | 64 | | timestamp_id | id of this timestamp | 65 | | headline_id | id of the headline for this timestamp | 66 | | raw_value | text representation of this timestamp | 67 | | is_active | true if the timestamp is active | 68 | | time_start | the start time (or only time) of this timestamp | 69 | | time_end | the end time of this timestamp | 70 | | start_is_long | true if the start time is in long format (eg `[YYYY-MM-DD DOW HH:MM]` vs `[YYYY-MM-DD DOW]`) | 71 | | end_is_long | true if the end time is in long format (see `start_is_long`) | 72 | 73 | ## timestamp_warnings 74 | 75 | Each row stores the warning component for a timestamp. 76 | 77 | | Column | Description | 78 | | - | - | 79 | | timestamp_id | id of the timestamp for this warning | 80 | | warning_value | shift of this warning | 81 | | warning_unit | unit of this warning (`hour`, `day`, `week`, `month`, or `year`) | 82 | | warning_type | type of this warning (`all`, or `first`) | 83 | 84 | ## timestamp_repeaters 85 | 86 | Each row stores the repeater component for a timestamp. If the repeater also has a habit appended to it, this will be stored as well. 87 | 88 | | Column | Description | 89 | | - | - | 90 | | timestamp_id | id of the timestamp for this repeater | 91 | | repeater_value | shift of this repeater | 92 | | repeater_unit | unit of this repeater (`hour`, `day`, `week`, `month`, or `year`) | 93 | | repeater_type | type of this repeater (`catch-up`, `restart`, or `cumulate`) | 94 | | habit_value | shift of this repeater's habit | 95 | | habit_unit | unit of this repeaters habit (`hour`, `day`, `week`, `month`, or `year`) | 96 | 97 | ## planning_entries 98 | 99 | Each row denotes a timestamp which is a planning entry (eg `DEADLINE`, `SCHEDULED`, or `CLOSED`). 100 | 101 | | Column | Description | 102 | | - | - | 103 | | timestamp_id | id of the timestamp for this planning entry | 104 | | planning_type | the type of this planning entry (`closed`, `scheduled`, or `deadline`) | 105 | 106 | ## file_tags 107 | 108 | Each row stores one tag denoted by the `#+FILETAGS` keyword 109 | 110 | | Column | Description | 111 | | - | - | 112 | | outline_hash | hash (MD5) of the org outline with this tag | 113 | | tag | the text value of this tag | 114 | 115 | ## headline_tags 116 | 117 | Each row stores one tag attached to a headline. This includes tags actively attached to a headlines as well as those in the `ARCHIVE_ITAGS` property within archive files. The `is_inherited` field will only be TRUE for the latter. 118 | 119 | | Column | Description | 120 | | - | - | 121 | | headline_id | id of the headline for this tag | 122 | | tag | the text value of this tag | 123 | | is_inherited | TRUE if this tag is from the `ARCHIVE_ITAGS` property | 124 | 125 | ## properties 126 | 127 | Each row stores one property. Note this includes properties under headlines as well as properties defined at the file-level using `#+PROPERTY`. 128 | 129 | | Column | Description | 130 | | - | - | 131 | | outline_hash | hash (MD5) of the org outline with this property | 132 | | property_id | id of this property | 133 | | key_text | this property's key | 134 | | val_text | this property's value | 135 | 136 | ## headline_properties 137 | 138 | Each row stores a property under a headline. 139 | 140 | | Column | Description | 141 | | - | - | 142 | | headline_id | id of the headline for this property | 143 | | property_id | id of this property | 144 | 145 | ## clocks 146 | 147 | Each row stores one clock entry. 148 | 149 | | Column | Description | 150 | | - | - | 151 | | clock_id | id of this clock | 152 | | headline_id | id of the headline for this clock | 153 | | time_start | timestamp for the start of this clock | 154 | | time_end | timestamp for the end of this clock | 155 | | clock_note | the note entry beneath this clock | 156 | 157 | ## logbook_entries 158 | 159 | Each row stores one logbook entry (except for clocks). Note that the possible values of `entry_type` depend on `org-log-note-headlines`. By default, the possible types are: `reschedule`, `delschedule`, `redeadline`, `deldeadline`, `state`, `done`, `note`, and `refile`. Note that while `clock-out` is also a default type in `org-log-note-headings` but this is already covered by the `clock_note` column in the `clocks` table and thus won't be stored in this table. 160 | 161 | | Column | Description | 162 | | - | - | 163 | | entry_id | id of this entry | 164 | | headline_id | id of the headline for this logbook entry | 165 | | entry_type | type of this entry | 166 | | time_logged | timestamp for when this entry was taken | 167 | | header | the first line of this entry (usually standardized) | 168 | | note | the text underneath the header of this entry | 169 | 170 | ## state_changes 171 | 172 | Each row stores the new and old states for logbook entries of type `state`. 173 | 174 | | Column | Description | 175 | | - | - | 176 | | entry_id | id of the entry for this state change | 177 | | state_old | former todo state keyword | 178 | | state_new | updated todo state keyword | 179 | 180 | ## planning_changes 181 | 182 | Each row stores the former timestamp for logbook entries with type `reschedule`, `delschedule`, `redeadline`, and `deldeadline`. 183 | 184 | | Column | Description | 185 | | - | - | 186 | | entry_id | id of the entry for this planning change | 187 | | timestamp_id | id of the former timestamp | 188 | 189 | ## links 190 | 191 | Each row stores one link. 192 | 193 | | Column | Description | 194 | | - | - | 195 | | link_id | id of this link | 196 | | headline_id | id of the headline for this link | 197 | | link_path | target of this link (eg url, file path, etc) | 198 | | link_text | text of this link that isn't part of the path | 199 | | link_type | type of this link (eg http, mu4e, file, etc) | 200 | 201 | Version: 4.0.0 -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.7' 2 | 3 | services: 4 | # postgres 5 | 6 | psql-latest: &psql 7 | build: ./test/docker/postgres 8 | container_name: org-sql-postgres-latest 9 | restart: 'no' 10 | environment: 11 | - POSTGRES_PASSWORD=toor 12 | - POSTGRES_USER=root 13 | ports: 14 | - 60016:5432 15 | 16 | psql-15: 17 | <<: *psql 18 | build: 19 | context: ./test/docker/postgres 20 | args: 21 | IMAGE: postgres:15.8-alpine 22 | container_name: org-sql-postgres-15 23 | ports: 24 | - 60015:5432 25 | 26 | psql-14: 27 | <<: *psql 28 | build: 29 | context: ./test/docker/postgres 30 | args: 31 | IMAGE: postgres:14.13-alpine 32 | container_name: org-sql-postgres-14 33 | ports: 34 | - 60014:5432 35 | 36 | psql-13: 37 | <<: *psql 38 | build: 39 | context: ./test/docker/postgres 40 | args: 41 | IMAGE: postgres:13.16-alpine 42 | container_name: org-sql-postgres-13 43 | ports: 44 | - 60013:5432 45 | 46 | ## mariadb 47 | 48 | ## ASSUME the mariadb and mysql docker images use the same service config 49 | mariadb-latest: &mariadb 50 | build: ./test/docker/mariadb 51 | container_name: org-sql-mariadb-latest 52 | restart: 'no' 53 | environment: 54 | - MYSQL_ROOT_PASSWORD=toor 55 | ports: 56 | - 60114:3306 57 | 58 | mariadb-1011: 59 | <<: *mariadb 60 | build: 61 | context: ./test/docker/mariadb 62 | args: 63 | IMAGE: mariadb:10.11.8 64 | container_name: org-sql-mariadb-1011 65 | ports: 66 | - 60111:3306 67 | 68 | mariadb-106: 69 | <<: *mariadb 70 | build: 71 | context: ./test/docker/mariadb 72 | args: 73 | IMAGE: mariadb:10.6.18 74 | container_name: org-sql-mariadb-106 75 | ports: 76 | - 60106:3306 77 | 78 | mariadb-105: 79 | <<: *mariadb 80 | build: 81 | context: ./test/docker/mariadb 82 | args: 83 | IMAGE: mariadb:10.5.25 84 | container_name: org-sql-mariadb-105 85 | ports: 86 | - 60105:3306 87 | 88 | ## mysql 89 | 90 | mysql-latest: 91 | <<: *mariadb 92 | build: 93 | context: ./test/docker/mariadb 94 | args: 95 | IMAGE: mysql:8.4.2 96 | container_name: org-sql-mysql-latest 97 | ports: 98 | - 60284:3306 99 | 100 | mysql80: 101 | <<: *mariadb 102 | build: 103 | context: ./test/docker/mariadb 104 | args: 105 | IMAGE: mysql:8.0.39 106 | container_name: org-sql-mysql-80 107 | ports: 108 | - 60280:3306 109 | 110 | ## sqlserver 111 | 112 | sqlserver-latest: &sqlserver 113 | build: ./test/docker/sql-server 114 | container_name: org-sql-sqlserver-latest 115 | restart: 'no' 116 | environment: 117 | - ACCEPT_EULA=Y 118 | - SA_PASSWORD=SFDwIcGvZdx&9g1f4Uy 119 | ports: 120 | - 60322:1433 121 | 122 | sqlserver-2019: 123 | <<: *sqlserver 124 | container_name: org-sql-sqlserver-2019 125 | build: 126 | context: ./test/docker/sql-server 127 | args: 128 | IMAGE: mcr.microsoft.com/mssql/server:2019-CU27-ubuntu-20.04 129 | ports: 130 | - 60319:1433 131 | 132 | sqlserver-2017: 133 | <<: *sqlserver 134 | container_name: org-sql-sqlserver-2017 135 | build: 136 | context: ./test/docker/sql-server 137 | args: 138 | IMAGE: mcr.microsoft.com/mssql/server:2017-CU29-ubuntu-16.04 139 | ports: 140 | - 60317:1433 141 | -------------------------------------------------------------------------------- /env-29.3.yml: -------------------------------------------------------------------------------- 1 | name: org-sql-29.3 2 | channels: 3 | - conda-forge 4 | dependencies: 5 | - _libgcc_mutex=0.1=conda_forge 6 | - _openmp_mutex=4.5=2_gnu 7 | - adwaita-icon-theme=46.2=unix_0 8 | - at-spi2-atk=2.38.0=h0630a04_3 9 | - at-spi2-core=2.40.3=h0630a04_0 10 | - atk-1.0=2.38.0=h04ea711_2 11 | - bzip2=1.0.8=h4bc722e_7 12 | - ca-certificates=2024.7.4=hbcca054_0 13 | - cairo=1.18.0=hbb29018_2 14 | - dbus=1.13.6=h5008d03_3 15 | - emacs=29.3=hc93ec10_0 16 | - epoxy=1.5.10=h166bdaf_1 17 | - expat=2.6.2=h59595ed_0 18 | - font-ttf-dejavu-sans-mono=2.37=hab24e00_0 19 | - font-ttf-inconsolata=3.000=h77eed37_0 20 | - font-ttf-source-code-pro=2.038=h77eed37_0 21 | - font-ttf-ubuntu=0.83=h77eed37_2 22 | - fontconfig=2.14.2=h14ed4e7_0 23 | - fonts-conda-ecosystem=1=0 24 | - fonts-conda-forge=1=0 25 | - freetype=2.12.1=h267a509_2 26 | - fribidi=1.0.10=h36c2ea0_0 27 | - gdk-pixbuf=2.42.12=hb9ae30d_0 28 | - gettext=0.22.5=he02047a_3 29 | - gettext-tools=0.22.5=he02047a_3 30 | - giflib=5.2.2=hd590300_0 31 | - glib=2.80.3=h315aac3_2 32 | - glib-tools=2.80.3=h8fdd7da_2 33 | - gmp=6.3.0=hac33072_2 34 | - gnutls=3.7.9=hb077bed_0 35 | - go-sqlcmd=1.8.0=ha8f183a_0 36 | - graphite2=1.3.13=h59595ed_1003 37 | - gtk3=3.24.43=h0359ba6_0 38 | - harfbuzz=8.5.0=hfac3d4d_0 39 | - hicolor-icon-theme=0.17=ha770c72_2 40 | - icu=73.2=h59595ed_0 41 | - jansson=2.14=h0b41bf4_1 42 | - keyutils=1.6.1=h166bdaf_0 43 | - krb5=1.21.3=h659f571_0 44 | - ld_impl_linux-64=2.40=hf3520f5_7 45 | - lerc=4.0.0=h27087fc_0 46 | - libasprintf=0.22.5=he8f35ee_3 47 | - libasprintf-devel=0.22.5=he8f35ee_3 48 | - libcups=2.3.3=h4637d8d_4 49 | - libdeflate=1.21=h4bc722e_0 50 | - libedit=3.1.20191231=he28a2e2_2 51 | - libexpat=2.6.2=h59595ed_0 52 | - libffi=3.4.2=h7f98852_5 53 | - libgcc-ng=14.1.0=h77fa898_0 54 | - libgettextpo=0.22.5=he02047a_3 55 | - libgettextpo-devel=0.22.5=he02047a_3 56 | - libglib=2.80.3=h315aac3_2 57 | - libgomp=14.1.0=h77fa898_0 58 | - libiconv=1.17=hd590300_2 59 | - libidn2=2.3.7=hd590300_0 60 | - libjpeg-turbo=3.0.0=hd590300_1 61 | - libnsl=2.0.1=hd590300_0 62 | - libpng=1.6.43=h2797004_0 63 | - libpq=16.4=h482b261_0 64 | - librsvg=2.58.2=hf0cb8fb_0 65 | - libsqlite=3.46.0=hde9e2c9_0 66 | - libstdcxx-ng=14.1.0=hc0a3c3a_0 67 | - libtasn1=4.19.0=h166bdaf_0 68 | - libtiff=4.6.0=h46a8edc_4 69 | - libtree-sitter=0.22.6=h4ab18f5_0 70 | - libunistring=0.9.10=h7f98852_0 71 | - libuuid=2.38.1=h0b41bf4_0 72 | - libwebp-base=1.4.0=hd590300_0 73 | - libxcb=1.16=hd590300_0 74 | - libxcrypt=4.4.36=hd590300_1 75 | - libxkbcommon=1.7.0=h2c5496b_1 76 | - libxml2=2.12.7=h4c95cb1_3 77 | - libzlib=1.3.1=h4ab18f5_1 78 | - mysql-common=8.3.0=h70512c7_5 79 | - mysql-libs=8.3.0=ha479ceb_5 80 | - mysqlclient=2.2.4=py312h30efb56_1 81 | - ncurses=6.5=h59595ed_0 82 | - nettle=3.9.1=h7ab15ed_0 83 | - openssl=3.3.1=h4bc722e_2 84 | - p11-kit=0.24.1=hc5aa10d_0 85 | - packaging=24.1=pyhd8ed1ab_0 86 | - pango=1.54.0=h84a9a3c_0 87 | - pcre2=10.44=hba22ea6_2 88 | - pip=24.2=pyhd8ed1ab_0 89 | - pixman=0.43.2=h59595ed_0 90 | - postgresql=16.4=ha8faf9a_0 91 | - pthread-stubs=0.4=h36c2ea0_1001 92 | - python=3.12.5=h2ad013b_0_cpython 93 | - python_abi=3.12=5_cp312 94 | - readline=8.2=h8228510_1 95 | - setuptools=72.1.0=pyhd8ed1ab_0 96 | - sqlite=3.46.0=h6d4b2fc_0 97 | - tk=8.6.13=noxft_h4845f30_101 98 | - tzcode=2024a=h3f72095_0 99 | - tzdata=2024a=h0c530f3_0 100 | - wayland=1.23.0=h5291e77_0 101 | - wheel=0.44.0=pyhd8ed1ab_0 102 | - xkeyboard-config=2.42=h4ab18f5_0 103 | - xorg-compositeproto=0.4.2=h7f98852_1001 104 | - xorg-damageproto=1.2.1=h7f98852_1002 105 | - xorg-fixesproto=5.0=h7f98852_1002 106 | - xorg-inputproto=2.3.2=h7f98852_1002 107 | - xorg-kbproto=1.0.7=h7f98852_1002 108 | - xorg-libice=1.1.1=hd590300_0 109 | - xorg-libsm=1.2.4=h7391055_0 110 | - xorg-libx11=1.8.9=hb711507_1 111 | - xorg-libxau=1.0.11=hd590300_0 112 | - xorg-libxaw=1.0.14=h7f98852_1 113 | - xorg-libxcomposite=0.4.6=h0b41bf4_1 114 | - xorg-libxcursor=1.2.0=h0b41bf4_1 115 | - xorg-libxdamage=1.1.5=h7f98852_1 116 | - xorg-libxdmcp=1.1.3=h7f98852_0 117 | - xorg-libxext=1.3.4=h0b41bf4_2 118 | - xorg-libxfixes=5.0.3=h7f98852_1004 119 | - xorg-libxft=2.3.8=hf69aa0a_0 120 | - xorg-libxi=1.7.10=h4bc722e_1 121 | - xorg-libxinerama=1.1.5=h27087fc_0 122 | - xorg-libxmu=1.1.3=h4ab18f5_1 123 | - xorg-libxpm=3.5.17=hd590300_0 124 | - xorg-libxrandr=1.5.2=h7f98852_1 125 | - xorg-libxrender=0.9.11=hd590300_0 126 | - xorg-libxt=1.3.0=hd590300_1 127 | - xorg-libxtst=1.2.5=h4bc722e_0 128 | - xorg-randrproto=1.5.0=h7f98852_1001 129 | - xorg-recordproto=1.14.2=h7f98852_1002 130 | - xorg-renderproto=0.11.1=h7f98852_1002 131 | - xorg-util-macros=1.19.3=h7f98852_0 132 | - xorg-xextproto=7.3.0=h0b41bf4_1003 133 | - xorg-xineramaproto=1.2.1=h7f98852_1001 134 | - xorg-xproto=7.0.31=h7f98852_1007 135 | - xz=5.2.6=h166bdaf_0 136 | - zlib=1.3.1=h4ab18f5_1 137 | - zstd=1.5.6=ha6fb4c9_0 138 | -------------------------------------------------------------------------------- /init.el: -------------------------------------------------------------------------------- 1 | (defun fix-null-term (s) 2 | "Fix string S with extra wonky null terminators. 3 | 4 | For whatever reason this affects certain strings in the conda 5 | package for Emacs. These look like `blabla\0\0\0\0\0\0\0`." 6 | (declare (pure t) (side-effect-free t)) 7 | (save-match-data 8 | (if (string-match "\0+" s) 9 | (replace-match "" t t s) 10 | s))) 11 | 12 | ;; HACK stuff won't install unless this string is fixed 13 | (if (< (round (string-to-number emacs-version)) 29) 14 | (setq Info-default-directory-list 15 | (cons 16 | (fix-null-term (car Info-default-directory-list)) 17 | (cdr Info-default-directory-list))) 18 | (setq configure-info-directory (fix-null-term configure-info-directory))) 19 | 20 | (setq package-enable-at-startup nil) 21 | 22 | (setq user-emacs-directory 23 | (file-name-concat (expand-file-name ".emacs") emacs-version)) 24 | 25 | (defvar bootstrap-version) 26 | 27 | (let ((bootstrap-file 28 | (file-name-concat 29 | user-emacs-directory 30 | "straight/repos/straight.el/bootstrap.el")) 31 | (bootstrap-version 7)) 32 | (unless (file-exists-p bootstrap-file) 33 | (with-current-buffer 34 | (url-retrieve-synchronously 35 | "https://raw.githubusercontent.com/radian-software/straight.el/develop/install.el" 36 | 'silent 'inhibit-cookies) 37 | (goto-char (point-max)) 38 | (eval-print-last-sexp))) 39 | (load bootstrap-file nil 'nomessage)) 40 | 41 | (straight-use-package 's) 42 | (straight-use-package 'f) 43 | (straight-use-package 'dash) 44 | (straight-use-package 'buttercup) 45 | (straight-use-package 'org) 46 | (straight-use-package 'org-ml) 47 | 48 | (defun compile-target () 49 | "Compile org-ml." 50 | (byte-compile-file "org-ml-macs.el") 51 | (byte-compile-file "org-ml.el")) 52 | -------------------------------------------------------------------------------- /test/docker/mariadb/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG IMAGE=mariadb:10.11.4 2 | FROM $IMAGE 3 | 4 | COPY ./init/* /docker-entrypoint-initdb.d/ 5 | -------------------------------------------------------------------------------- /test/docker/mariadb/init/org_sql.sql: -------------------------------------------------------------------------------- 1 | CREATE DATABASE org_sql; 2 | USE org_sql; 3 | 4 | CREATE USER 'org_sql' IDENTIFIED BY 'org_sql'; 5 | -- MySQL needs to REFERENCES permission for foreign keys, MariaDB apparently 6 | -- ignores it 7 | GRANT CREATE, DROP, DELETE, INSERT, REFERENCES, SELECT ON org_sql.* TO 'org_sql'; 8 | -------------------------------------------------------------------------------- /test/docker/postgres/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG IMAGE=postgres:16.4-alpine 2 | FROM $IMAGE 3 | 4 | COPY ./init/* /docker-entrypoint-initdb.d/ 5 | -------------------------------------------------------------------------------- /test/docker/postgres/init/org_sql.sql: -------------------------------------------------------------------------------- 1 | CREATE DATABASE org_sql; 2 | \connect org_sql; 3 | 4 | -- org_sql db "admin" (not root but has the power to create objects in this db) 5 | CREATE ROLE org_sql WITH LOGIN PASSWORD 'org_sql'; 6 | 7 | -- create alt schema to test 8 | CREATE SCHEMA nonpublic AUTHORIZATION org_sql; 9 | GRANT ALL ON SCHEMA nonpublic TO org_sql; 10 | GRANT ALL ON SCHEMA public TO org_sql; 11 | -------------------------------------------------------------------------------- /test/docker/sql-server/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG IMAGE=mcr.microsoft.com/mssql/server:2022-CU11-ubuntu-22.04 2 | FROM $IMAGE 3 | 4 | ## Start the sql-server docker image with an init script to set up the test db 5 | 6 | USER root 7 | COPY . /usr/src/app 8 | RUN chmod +x /usr/src/app/init-db.sh 9 | 10 | # for some reason the 2017 image doesn't have the mssql user 11 | # USER mssql 12 | 13 | ENTRYPOINT /bin/bash /usr/src/app/entrypoint.sh 14 | -------------------------------------------------------------------------------- /test/docker/sql-server/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #start SQL Server, start the script to create the DB and import the data, start the app 2 | set -m 3 | /opt/mssql/bin/sqlservr & /usr/src/app/init-db.sh 4 | fg 5 | -------------------------------------------------------------------------------- /test/docker/sql-server/init-db.sh: -------------------------------------------------------------------------------- 1 | #run the setup script to create the testing db 2 | #do this in a loop because the timing for when the SQL instance is ready is indeterminate 3 | for i in {1..50}; 4 | do 5 | /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P "SFDwIcGvZdx&9g1f4Uy" \ 6 | -d master -i /usr/src/app/setup.sql 7 | if [ $? -eq 0 ] 8 | then 9 | echo "setup.sql completed" 10 | break 11 | else 12 | echo "not ready yet..." 13 | sleep 1 14 | fi 15 | done 16 | -------------------------------------------------------------------------------- /test/docker/sql-server/setup.sql: -------------------------------------------------------------------------------- 1 | CREATE DATABASE org_sql; 2 | GO 3 | CREATE LOGIN org_sql WITH PASSWORD = 'o%4XlS14tPO!J@q@16v'; 4 | GO 5 | USE org_sql; 6 | GO 7 | CREATE USER org_sql; 8 | GO 9 | CREATE SCHEMA nondbo AUTHORIZATION org_sql; 10 | GO 11 | GRANT CREATE TABLE TO org_sql; 12 | GO 13 | -------------------------------------------------------------------------------- /test/files/fancy.org: -------------------------------------------------------------------------------- 1 | #+filetags: one two three 2 | #+property: p1 v1 v2 v3 3 | 4 | * plain 5 | * archived :ARCHIVE: 6 | * TODO parent 7 | ** DONE COMMENT child :sometag: 8 | CLOSED: [2020-09-15 Tue 17:59] 9 | :PROPERTIES: 10 | :Effort: 1:00 11 | :thing: thingy 12 | :END: 13 | :LOGBOOK: 14 | - State "DONE" from "TODO" [2020-09-15 Tue 18:05] 15 | CLOCK: [2020-09-15 Tue 17:57]--[2020-09-15 Tue 18:00] => 0:03 16 | - this is a clock note 17 | :END: 18 | [[file:/dev/null][NULL]] 19 | ** TODO [#B] other child 20 | DEADLINE: <2020-09-22 Tue +2d -1m> SCHEDULED: <2020-09-18 Fri> 21 | :LOGBOOK: 22 | - Rescheduled from "[2020-09-17 Thu]" on [2020-09-15 Tue 18:00] 23 | - Not scheduled, was "[2020-09-19 Sat]" on [2020-09-15 Tue 17:55] 24 | - Removed deadline, was "[2020-09-22 Tue]" on [2020-09-15 Tue 17:50] 25 | - New deadline from "[2020-09-17 Thu]" on [2020-09-15 Tue 17:45] 26 | :END: 27 | https://downloadmoreram.gov 28 | <2020-09-15 Tue> 29 | hopefully this hits all the relevant code paths :) 30 | 31 | make this line super loooooooooooooooooooooooooooooooooooooooooooooooooog so that sqlcmd will be confused (since it's default settings don't allow columns more than 255 chars or something absurd like that) 32 | 33 | here's "some|" 34 | weird character\\s, {for} 35 | good \n\t measure. 36 | -------------------------------------------------------------------------------- /test/files/foo1.org: -------------------------------------------------------------------------------- 1 | * foo 2 | -------------------------------------------------------------------------------- /test/files/foo2.org: -------------------------------------------------------------------------------- 1 | * foo 2 | -------------------------------------------------------------------------------- /test/files/foo3.org: -------------------------------------------------------------------------------- 1 | * moo 2 | -------------------------------------------------------------------------------- /test/org-sql-test-stateful.el: -------------------------------------------------------------------------------- 1 | ;;; org-sql-test-stateful.el --- IO tests for org-sql -*- lexical-binding: t; -*- 2 | 3 | ;; Copyright (C) 2020 Nathan Dwarshuis 4 | 5 | ;; This program is free software; you can redistribute it and/or modify 6 | ;; it under the terms of the GNU General Public License as published by 7 | ;; the Free Software Foundation, either version 3 of the License, or 8 | ;; (at your option) any later version. 9 | 10 | ;; This program is distributed in the hope that it will be useful, 11 | ;; but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | ;; GNU General Public License for more details. 14 | 15 | ;; You should have received a copy of the GNU General Public License 16 | ;; along with this program. If not, see . 17 | 18 | ;;; Commentary: 19 | 20 | ;; These specs test IO functions on org-sql, including reading/writing 21 | ;; to databases and reading the status of org files on disk. These are at 22 | ;; a higher level of complexity than the stateless tests and thus assume 23 | ;; that specification is fully met. 24 | 25 | ;;; Code: 26 | 27 | (require 'org-sql) 28 | (require 's) 29 | (require 'f) 30 | (require 'dash) 31 | (require 'buttercup) 32 | 33 | (defconst test-dir (f-dirname (f-this-file))) 34 | 35 | (defconst test-files (f-join test-dir "files")) 36 | 37 | (defconst test-scripts (f-join test-dir "scripts")) 38 | 39 | (defun expect-exit-success (res) 40 | (if org-sql-async 41 | (let ((out (or (while (accept-process-output res)) "")) 42 | (status (process-status res))) 43 | (if (and (eq 'exit status) (equal out "")) (expect t) 44 | (error out))) 45 | (-let (((rc . out) res)) 46 | (if (and (= 0 rc) (equal out "")) (expect t) 47 | (error out))))) 48 | 49 | (defun org-sql--count-rows (config tbl-name) 50 | ;; hacky AF... 51 | (let* ((tbl-name* (org-sql--format-table-name config tbl-name)) 52 | (select (format "SELECT Count(*) FROM %s" tbl-name*))) 53 | (org-sql-send-sql select))) 54 | 55 | (defmacro expect-db-has-tables (config header &rest table-specs) 56 | (declare (indent 1)) 57 | (let ((it-forms 58 | (--map `(it ,(format "table %s has %s row(s)" (car it) (cdr it)) 59 | (let ((n (--> (org-sql--count-rows ',config ',(car it)) 60 | (cdr it) 61 | (s-trim it) 62 | (if (equal it "") 0 (string-to-number it))))) 63 | (expect n :to-be ,(cdr it)))) 64 | table-specs))) 65 | `(describe "test that correct tables are populated" ,@it-forms))) 66 | 67 | (defun expect-db-has-table-contents (tbl-name &rest rows) 68 | (declare (indent 1)) 69 | (expect (org-sql-dump-table tbl-name) :to-equal rows)) 70 | 71 | (defun expect-db-has-table-contents* (tbl-name &rest rows) 72 | (declare (indent 1)) 73 | (let ((out (->> (org-sql-dump-table tbl-name)))) 74 | (->> (--zip-with (->> (--zip-with (if (functionp it) 75 | (funcall it other) 76 | (equal it other)) 77 | it other) 78 | (--all? (eq t it))) 79 | rows out) 80 | (--all? (eq t it)) 81 | (expect)))) 82 | 83 | ;; this will only work with one line and one field 84 | (defun expect-db-has-table-contents-raw (config tbl-name &rest rows) 85 | (declare (indent 1)) 86 | (let* ((tbl-name* (org-sql--format-table-name config tbl-name)) 87 | (select (format "SELECT * FROM %s" tbl-name*))) 88 | (org-sql--on-success* (org-sql-send-sql select) 89 | (let ((out (list (substring it-out 0 -1)))) 90 | (expect out :to-equal rows))))) 91 | 92 | (defmacro describe-reset-db (header &rest body) 93 | (declare (indent 1)) 94 | `(describe ,header 95 | (after-all 96 | (org-sql-reset-db) 97 | (org-sql-init-db)) 98 | ,@body)) 99 | 100 | (defmacro describe-sql-database-spec (config) 101 | (let ((it-forms 102 | (org-sql--case-mode config 103 | ((mysql postgres sqlserver) 104 | '((it "create database should error" 105 | (should-error (org-sql-create-db))) 106 | (it "drop database should error" 107 | (should-error (org-sql-drop-db))) 108 | ;; TODO add a condition where this will return nil? 109 | (it "database should exist regardless" 110 | (expect (org-sql-db-exists))))) 111 | (sqlite 112 | '((it "create database" 113 | (expect-exit-success (org-sql-create-db))) 114 | (it "database should exist" 115 | (expect (org-sql-db-exists))) 116 | (it "drop database" 117 | (expect-exit-success (org-sql-drop-db))) 118 | (it "database should not exist" 119 | (expect (not (org-sql-db-exists))))))))) 120 | `(describe "Database Admin Spec" 121 | (before-all 122 | (setq org-sql-db-config ',config)) 123 | (after-all 124 | (ignore-errors 125 | (org-sql-drop-db))) 126 | ,@it-forms))) 127 | 128 | (defmacro describe-sql-table-spec (config) 129 | `(describe "Table Admin Spec" 130 | (before-all 131 | (setq org-sql-db-config ',config) 132 | (ignore-errors 133 | (org-sql-create-db))) 134 | (after-all 135 | (ignore-errors 136 | (org-sql-drop-tables)) 137 | (ignore-errors 138 | (org-sql-drop-db))) 139 | (it "create tables" 140 | (expect-exit-success (org-sql-create-tables))) 141 | (it "tables should exist" 142 | (expect (org-sql--sets-equal org-sql-table-names (org-sql-list-tables) 143 | :test #'equal))) 144 | (it "drop tables" 145 | (expect-exit-success (org-sql-drop-tables))) 146 | (it "tables should not exist" 147 | (expect (length (org-sql-list-tables)) :to-be 0)))) 148 | 149 | (defmacro describe-sql-init-spec (config) 150 | (let ((should-exist (org-sql--case-mode config 151 | (sqlite 152 | '(it "database should not exist" 153 | (expect (not (org-sql-db-exists))))) 154 | ((mysql postgres sqlserver) 155 | '(it "database should still exist" 156 | (expect (org-sql-db-exists))))))) 157 | (cl-flet 158 | ((mk-form 159 | (async) 160 | (let ((title (concat "Initialization/Reset Spec" 161 | (when async " (async)")))) 162 | `(describe ,title 163 | (before-all 164 | (setq org-sql-db-config ',config)) 165 | (after-all 166 | (ignore-errors 167 | (org-sql-drop-tables)) 168 | (ignore-errors 169 | (org-sql-drop-db))) 170 | (it "initialize database" 171 | (let ((org-sql-async ,async)) 172 | (expect-exit-success (org-sql-init-db)))) 173 | (it "database should exist" 174 | (expect (org-sql-db-exists))) 175 | (it "tables should exist" 176 | (expect (org-sql--sets-equal org-sql-table-names 177 | (org-sql-list-tables) 178 | :test #'equal))) 179 | (it "reset database" 180 | (let ((org-sql-async ,async)) 181 | (expect-exit-success (org-sql-reset-db)))) 182 | ,should-exist 183 | (it "tables should not exist" 184 | (expect (not (org-sql--sets-equal org-sql-table-names 185 | (org-sql-list-tables) 186 | :test #'equal)))))))) 187 | `(progn 188 | ,(mk-form nil) 189 | ,(mk-form t))))) 190 | 191 | (defun org-sql-is-number (x) 192 | (or (null x) (and (stringp x) (s-matches? "[0-9]+" x)))) 193 | 194 | (defmacro describe-sql-update-spec (config) 195 | ;; ASSUME init/reset work 196 | (cl-flet 197 | ((mk-single 198 | (async) 199 | (let ((title (concat "single file" (when async " (async)")))) 200 | `(describe-reset-db ,title 201 | (it "update database" 202 | (let ((org-sql-files (list (f-join test-files "foo1.org"))) 203 | (org-sql-async ,async)) 204 | (expect-exit-success (org-sql-push-to-db)))) 205 | (expect-db-has-tables ,config 206 | (outlines . 1) 207 | (file_metadata . 1) 208 | (headlines . 1) 209 | (headline_closures . 1) 210 | (planning_entries . 0) 211 | (timestamps . 0) 212 | (timestamp_warnings . 0) 213 | (timestamp_repeaters . 0) 214 | (links . 0) 215 | (headline_tags . 0) 216 | (headline_properties . 0) 217 | (properties . 0) 218 | (logbook_entries . 0) 219 | (state_changes . 0) 220 | (planning_changes . 0) 221 | (clocks . 0)))))) 222 | `(describe "Update DB Spec" 223 | (before-all 224 | (setq org-sql-db-config ',config) 225 | (org-sql-init-db)) 226 | 227 | (after-all 228 | (ignore-errors 229 | (org-sql-reset-db))) 230 | 231 | ,(mk-single nil) 232 | ,(mk-single t) 233 | 234 | (describe-reset-db "two different files" 235 | (it "update database" 236 | (let ((org-sql-files (list (f-join test-files "foo1.org") 237 | (f-join test-files "foo3.org")))) 238 | (expect-exit-success (org-sql-push-to-db)))) 239 | (expect-db-has-tables ,config 240 | (outlines . 2) 241 | (file_metadata . 2) 242 | (headlines . 2) 243 | (headline_closures . 2) 244 | (planning_entries . 0) 245 | (timestamps . 0) 246 | (timestamp_warnings . 0) 247 | (timestamp_repeaters . 0) 248 | (links . 0) 249 | (headline_tags . 0) 250 | (headline_properties . 0) 251 | (properties . 0) 252 | (logbook_entries . 0) 253 | (state_changes . 0) 254 | (planning_changes . 0) 255 | (clocks . 0))) 256 | 257 | (describe-reset-db "two identical files" 258 | (it "update database" 259 | (let ((org-sql-files (list (f-join test-files "foo1.org") 260 | (f-join test-files "foo2.org")))) 261 | (expect-exit-success (org-sql-push-to-db)))) 262 | (expect-db-has-tables ,config 263 | (outlines . 1) 264 | (file_metadata . 2) 265 | (headlines . 1) 266 | (headline_closures . 1) 267 | (planning_entries . 0) 268 | (timestamps . 0) 269 | (timestamp_warnings . 0) 270 | (timestamp_repeaters . 0) 271 | (links . 0) 272 | (headline_tags . 0) 273 | (headline_properties . 0) 274 | (properties . 0) 275 | (logbook_entries . 0) 276 | (state_changes . 0) 277 | (planning_changes . 0) 278 | (clocks . 0))) 279 | 280 | (describe-reset-db "fancy file" 281 | (before-all 282 | (setq test-path (f-join test-files "fancy.org") 283 | outline-hash "e1a1b70663f70d3aff08be43f46b5ab3" 284 | preamble "#+filetags: one two three\n#+property: p1 v1 v2 v3\n\n")) 285 | (it "update database" 286 | (let ((org-sql-files (list test-path)) 287 | (org-log-into-drawer "LOGBOOK") 288 | (org-log-note-clock-out t)) 289 | (expect-exit-success (org-sql-push-to-db)))) 290 | (it "pull database" 291 | (let ((org-log-into-drawer t) 292 | (org-clock-into-drawer t) 293 | (org-log-note-clock-out t)) 294 | (-let (((file . tree) (car (org-sql-pull-from-db))) 295 | (orig-string (f-read-text test-path 'utf-8))) 296 | (expect file :to-equal test-path) 297 | (expect (org-ml-to-string tree) :to-equal orig-string)))) 298 | (it "check metadata table" 299 | (expect-db-has-table-contents* 'file_metadata 300 | `(,test-path ,outline-hash integerp integerp integerp integerp "-rw-r--r--"))) 301 | (it "check outlines table" 302 | (expect-db-has-table-contents 'outlines 303 | `(,outline-hash 1235 36 ,preamble))) 304 | (it "check headlines table" 305 | (expect-db-has-table-contents 'headlines 306 | `(1 ,outline-hash "plain" 1 0 nil nil nil nil nil 0 0 nil) 307 | `(2 ,outline-hash "archived" 1 1 nil nil nil nil nil 1 0 nil) 308 | `(3 ,outline-hash "parent" 1 2 "TODO" nil nil nil nil 0 0 nil) 309 | `(4 ,outline-hash "child" 2 0 "DONE" 60 nil nil nil 0 1 "[[file:/dev/null][NULL]]\n") 310 | `(5 ,outline-hash "other child" 2 1 "TODO" nil "B" nil nil 0 0 311 | ,(s-join "\n" (list "https://downloadmoreram.gov" 312 | "<2020-09-15 Tue>" 313 | "hopefully this hits all the relevant code paths :)" 314 | "" 315 | (concat "make this line super " 316 | "loooooooooooooooooooooooooooooooooooooooooooooooooog " 317 | "so that sqlcmd will be confused " 318 | "(since it's default settings don't " 319 | "allow columns more than 255 chars or " 320 | "something absurd like that)") 321 | "" 322 | "here's \"some|\"" 323 | "weird character\\\\s, {for}" 324 | " good \\n\\t measure." 325 | ""))))) 326 | (it "check headline closures table" 327 | (expect-db-has-table-contents 'headline_closures 328 | '(1 1 0) 329 | '(2 2 0) 330 | '(3 3 0) 331 | '(4 3 1) 332 | '(4 4 0) 333 | '(5 3 1) 334 | '(5 5 0))) 335 | (it "check timestamps table" 336 | (expect-db-has-table-contents* 'timestamps 337 | '(1 4 "[2020-09-15 Tue 17:59]" 0 integerp nil 1 nil) 338 | '(2 5 "<2020-09-22 Tue +2d -1m>" 1 integerp nil 0 nil) 339 | '(3 5 "<2020-09-18 Fri>" 1 integerp nil 0 nil) 340 | '(4 5 "<2020-09-15 Tue>" 1 integerp nil 0 nil) 341 | '(5 5 "[2020-09-17 Thu]" 0 integerp nil 0 nil) 342 | '(6 5 "[2020-09-19 Sat]" 0 integerp nil 0 nil) 343 | '(7 5 "[2020-09-22 Tue]" 0 integerp nil 0 nil) 344 | '(8 5 "[2020-09-17 Thu]" 0 integerp nil 0 nil))) 345 | (it "check timestamp warnings table" 346 | (expect-db-has-table-contents* 'timestamp_warnings 347 | '(2 1 348 | (lambda (it) (equal "month" (symbol-name it))) 349 | (lambda (it) (equal "all" (symbol-name it)))))) 350 | (it "check timestamp repeaters table" 351 | (expect-db-has-table-contents* 'timestamp_repeaters 352 | '(2 2 353 | (lambda (it) (equal "day" (symbol-name it))) 354 | (lambda (it) (equal "cumulate" (symbol-name it))) 355 | nil nil))) 356 | (it "check logbook entries table" 357 | (expect-db-has-table-contents* 'logbook_entries 358 | '(1 4 "state" integerp 359 | "State \"DONE\" from \"TODO\" [2020-09-15 Tue 18:05]" 360 | nil) 361 | '(2 5 "reschedule" integerp 362 | "Rescheduled from \"[2020-09-17 Thu]\" on [2020-09-15 Tue 18:00]") 363 | '(3 5 "delschedule" integerp 364 | "Not scheduled, was \"[2020-09-19 Sat]\" on [2020-09-15 Tue 17:55]") 365 | '(4 5 "deldeadline" integerp 366 | "Removed deadline, was \"[2020-09-22 Tue]\" on [2020-09-15 Tue 17:50]") 367 | '(5 5 "redeadline" integerp 368 | "New deadline from \"[2020-09-17 Thu]\" on [2020-09-15 Tue 17:45]"))) 369 | (it "check planning changes table" 370 | (expect-db-has-table-contents 'planning_changes 371 | '(2 5) 372 | '(3 6) 373 | '(4 7) 374 | '(5 8))) 375 | (it "check state changes table" 376 | (expect-db-has-table-contents 'state_changes 377 | '(1 "TODO" "DONE"))) 378 | (it "check planning entries" 379 | (expect-db-has-table-contents* 'planning_entries 380 | '(1 (lambda (it) (equal "closed" (symbol-name it)))) 381 | '(2 (lambda (it) (equal "deadline" (symbol-name it)))) 382 | '(3 (lambda (it) (equal "scheduled" (symbol-name it)))))) 383 | (it "check properties table" 384 | (expect-db-has-table-contents 'properties 385 | `(,outline-hash 1 "p1" "v1 v2 v3") 386 | `(,outline-hash 2 "thing" "thingy"))) 387 | (it "check headline properties table" 388 | (expect-db-has-table-contents 'headline_properties 389 | '(4 2))) 390 | (it "check headline tags table" 391 | (expect-db-has-table-contents 'headline_tags 392 | '(4 "sometag" 0))) 393 | (it "check file tags table" 394 | (expect-db-has-table-contents 'file_tags 395 | `(,outline-hash, "one") 396 | `(,outline-hash, "three") 397 | `(,outline-hash, "two"))) 398 | (it "check clocks table" 399 | (expect-db-has-table-contents* 'clocks 400 | '(1 4 integerp integerp "this is a clock note"))) 401 | (it "check links table" 402 | (expect-db-has-table-contents 'links 403 | '(1 4 "/dev/null" "NULL" "file") 404 | '(2 5 "//downloadmoreram.gov" nil "https")))) 405 | 406 | 407 | ;; (expect-db-has-table-contents 'file_metadata 408 | ;; `(,test-path "4a374dde85114a7838950003337bf869" org-sql-is-number 409 | ;; org-sql-is-number org-sql-is-number org-sql-is-number 410 | ;; "-rw-r--r--"))) 411 | ;; (expect-db-has-tables ,config 412 | ;; (outlines . 1) 413 | ;; (file_metadata . 1) 414 | ;; (file_tags . 3) 415 | ;; (headlines . 5) 416 | ;; (headline_closures . 7) 417 | ;; (planning_entries . 3) 418 | ;; (timestamps . 8) 419 | ;; (timestamp_warnings . 1) 420 | ;; (timestamp_repeaters . 1) 421 | ;; (links . 1) 422 | ;; (headline_tags . 1) 423 | ;; (headline_properties . 1) 424 | ;; (properties . 2) 425 | ;; (logbook_entries . 5) 426 | ;; (state_changes . 1) 427 | ;; (planning_changes . 4) 428 | ;; (clocks . 1))) 429 | 430 | (describe-reset-db "renamed file" 431 | (describe "insert file" 432 | (before-all 433 | (setq test-path (f-join test-files "foo1.org"))) 434 | (it "update database" 435 | (let ((org-sql-files (list test-path))) 436 | (expect-exit-success (org-sql-push-to-db)))) 437 | (it "test for file in tables" 438 | (expect-db-has-table-contents* 'file_metadata 439 | `(,test-path "106e9f12c9e4ff3333425115d148fbd4" integerp integerp 440 | integerp integerp "-rw-r--r--")))) 441 | (describe "rename inserted file" 442 | ;; "rename" here means to point `org-sql-files' to an identical file 443 | ;; with a different name 444 | (before-all 445 | (setq test-path (f-join test-files "foo2.org"))) 446 | (it "update database" 447 | (let ((org-sql-files (list test-path))) 448 | (expect-exit-success (org-sql-push-to-db)))) 449 | (it "test for file in tables" 450 | (expect-db-has-table-contents* 'file_metadata 451 | `(,test-path "106e9f12c9e4ff3333425115d148fbd4" integerp integerp 452 | integerp integerp "-rw-r--r--"))))) 453 | 454 | (describe-reset-db "deleted file" 455 | (it "update database" 456 | (let ((org-sql-files (list (f-join test-files "foo1.org")))) 457 | (expect-exit-success (org-sql-push-to-db)))) 458 | (it "update database (untrack the original file)" 459 | (let ((org-sql-files nil)) 460 | (expect-exit-success (org-sql-push-to-db)))) 461 | (expect-db-has-tables ,config 462 | (file_metadata . 0) 463 | (outlines . 0))) 464 | 465 | (describe-reset-db "altered file" 466 | ;; in order to make this test work, make a file in /tmp and alter 467 | ;; its contents 468 | (describe "insert file" 469 | (before-all 470 | (setq test-path (f-join (temporary-file-directory) 471 | "org-sql-test-file.org"))) 472 | (it "update database" 473 | (let ((contents1 "* foo1") 474 | (org-sql-files (list test-path))) 475 | ;; write file and update db 476 | (f-write-text contents1 'utf-8 test-path) 477 | (expect-exit-success (org-sql-push-to-db)))) 478 | (it "test file hash" 479 | (expect-db-has-table-contents 'outlines 480 | `("ece424e0090cff9b6f1ac50722c336c0" 6 1 nil)))) 481 | (describe "alter the file" 482 | (before-all 483 | (setq test-path (f-join (temporary-file-directory) 484 | "org-sql-test-file.org"))) 485 | (it "update with new contents" 486 | (let ((contents2 "* foo2") 487 | (org-sql-files (list test-path))) 488 | ;; close buffer, alter the file, and update again 489 | (kill-buffer (find-file-noselect test-path t)) 490 | (f-write-text contents2 'utf-8 test-path) 491 | (expect-exit-success (org-sql-push-to-db)))) 492 | (it "test for new file hash" 493 | (expect-db-has-table-contents 'outlines 494 | `("399bc042f23ea976a04b9102c18e9cb5" 6 1 nil))) 495 | (it "clean up" 496 | ;; yes killing the buffer is necessary 497 | (kill-buffer (find-file-noselect test-path t)) 498 | (f-delete test-path t))))))) 499 | 500 | (defmacro describe-sql-clear-spec (config) 501 | ;; ASSUME init/reset work 502 | (cl-flet 503 | ((mk-clear 504 | (async) 505 | (let ((title (concat "loading a file and clearing" (when async " (async)")))) 506 | `(describe-reset-db ,title 507 | (it "update database" 508 | (let ((org-sql-files (list (f-join test-files "foo1.org"))) 509 | (org-sql-async ,async)) 510 | (expect-exit-success (org-sql-push-to-db)))) 511 | (it "clear database" 512 | (let ((org-sql-async ,async)) 513 | (expect-exit-success (org-sql-clear-db)))) 514 | (it "tables should still exist" 515 | (expect (org-sql--sets-equal org-sql-table-names (org-sql-list-tables) 516 | :test #'equal))) 517 | (expect-db-has-tables ,config 518 | (outlines . 0) 519 | (file_metadata . 0) 520 | (headlines . 0) 521 | (timestamps . 0) 522 | (timestamp_warnings . 0) 523 | (timestamp_repeaters . 0) 524 | (properties . 0) 525 | (headline_properties . 0) 526 | (file_tags . 0) 527 | (headline_tags . 0) 528 | (logbook_entries . 0) 529 | (planning_changes . 0) 530 | (state_changes . 0) 531 | (planning_entries . 0) 532 | (clocks . 0)))))) 533 | 534 | `(describe "Clear DB Spec" 535 | (before-all 536 | (setq org-sql-db-config ',config) 537 | (org-sql-init-db)) 538 | 539 | (after-all 540 | (ignore-errors 541 | (org-sql-reset-db))) 542 | 543 | (describe-reset-db "clearing an empty db" 544 | (it "clear database" 545 | (expect-exit-success (org-sql-clear-db))) 546 | (it "tables should still exist" 547 | (expect (org-sql--sets-equal org-sql-table-names (org-sql-list-tables) 548 | :test #'equal)))) 549 | 550 | ,(mk-clear nil) 551 | ,(mk-clear t)))) 552 | 553 | (defmacro describe-sql-hook-spec (config) 554 | `(describe "DB Hook Spec" 555 | (before-all 556 | (setq org-sql-db-config ',config) 557 | (ignore-errors 558 | (org-sql-drop-tables)) 559 | (ignore-errors 560 | (org-sql-drop-db))) 561 | 562 | (after-all 563 | (org-sql-send-sql "DROP TABLE IF EXISTS fake_init_table;") 564 | (org-sql-send-sql "DROP TABLE IF EXISTS fake_update_table;") 565 | (org-sql-send-sql "DROP TABLE IF EXISTS save_something;") 566 | (ignore-errors 567 | (org-sql-reset-db))) 568 | 569 | (it "init database" 570 | (let ((org-sql-post-init-hooks 571 | `((file ,(f-join test-scripts "init_hook.sql")) 572 | (sql "INSERT INTO fake_init_table VALUES (1);")))) 573 | (expect-exit-success (org-sql-init-db)))) 574 | (it "fake init table should exist" 575 | (expect-db-has-table-contents-raw ',config 'fake_init_table "1")) 576 | (it "update database" 577 | (let ((org-sql-post-push-hooks 578 | `((file+ ,(f-join test-scripts "update_hook.sql")) 579 | (sql+ "INSERT INTO fake_update_table VALUES (1);"))) 580 | (org-sql-files (list (f-join test-files "foo1.org")))) 581 | (expect-exit-success (org-sql-push-to-db)))) 582 | (it "fake update table should exist" 583 | (expect-db-has-table-contents-raw ',config 'fake_update_table "1")) 584 | (it "clear database" 585 | (let ((org-sql-post-clear-hooks 586 | `((file ,(f-join test-scripts "clear_hook.sql")) 587 | (sql "DROP TABLE fake_update_table;")))) 588 | (expect-exit-success (org-sql-clear-db)))) 589 | (it "fake init table should not exist" 590 | (expect (not (member "fake_init_table" (org-sql-list-tables))))) 591 | (it "fake update table should not exist" 592 | (expect (not (member "fake_update_table" (org-sql-list-tables))))) 593 | (it "reset database" 594 | (let ((org-sql-pre-reset-hooks 595 | `((sql "CREATE TABLE save_something (x INTEGER);")))) 596 | (expect-exit-success (org-sql-reset-db)))) 597 | (it "reset table should exist" 598 | (expect (member "save_something" (org-sql-list-tables)))))) 599 | 600 | (defmacro describe-io-spec (unique-name config) 601 | (declare (indent 1)) 602 | ;; this spec only works with the default schema; I suppose if I was less lazy 603 | ;; I could make it work for non-default schema's but it's nice when the same 604 | ;; SQL statements work for all tests and configs ;) 605 | (let ((hook-spec (org-sql--with-config-keys (:schema) config 606 | (unless schema `((describe-sql-hook-spec ,config)))))) 607 | 608 | `(describe ,unique-name 609 | (after-all 610 | (ignore-errors 611 | (org-sql-drop-tables)) 612 | (ignore-errors 613 | (org-sql-drop-db))) 614 | (describe-sql-database-spec ,config) 615 | (describe-sql-table-spec ,config) 616 | (describe-sql-init-spec ,config) 617 | (describe-sql-update-spec ,config) 618 | (describe-sql-clear-spec ,config) 619 | ,@hook-spec))) 620 | 621 | (defmacro describe-io-specs (&rest specs) 622 | (declare (indent 0)) 623 | (let ((forms (->> (-partition 2 specs) 624 | (--map `(describe-io-spec ,(car it) ,(cadr it)))))) 625 | `(describe "SQL IO Spec" 626 | ,@forms))) 627 | 628 | (cl-flet* 629 | ((mk-io-spec 630 | (db-name db-sym version alt-title key-vals) 631 | `(,(if alt-title (format "%s (v%s - %s)" db-name version alt-title) 632 | (format "%s (v%s)" db-name version)) 633 | (,db-sym ,@key-vals))) 634 | (mk-postgres 635 | (version port &optional alt-title key-vals) 636 | (->> (list :database "org_sql" 637 | :port port 638 | :hostname "localhost" 639 | :username "org_sql" 640 | :password "org_sql") 641 | (append key-vals) 642 | (mk-io-spec "Postgres" 'postgres version alt-title))) 643 | (mk-mysql 644 | (title version port &optional alt-title key-vals) 645 | (->> (list :database "org_sql" 646 | :port port 647 | :hostname "127.0.0.1" 648 | :username "org_sql" 649 | :password "org_sql") 650 | (append key-vals) 651 | (mk-io-spec title 'mysql version alt-title))) 652 | (mk-sqlserver 653 | (version port &optional alt-title key-vals) 654 | (->> (list :database "org_sql" 655 | :server (format "tcp:localhost,%s" port) 656 | :args '("-C") ;; trust server cert 657 | :username "org_sql" 658 | :password "o%4XlS14tPO!J@q@16v") 659 | (append key-vals) 660 | (mk-io-spec "SQL-Server" 'sqlserver version alt-title)))) 661 | (let* ((sqlite (list "SQLite" 662 | `(sqlite :path ,(f-join (temporary-file-directory) 663 | "org-sql-test.db")))) 664 | (postgres 665 | (append 666 | (mk-postgres 16 60016) 667 | (mk-postgres 16 60016 "Non-Default Schema" '(:schema "nonpublic")) 668 | (mk-postgres 16 60016 "Unlogged tables" '(:unlogged t)) 669 | (mk-postgres 15 60015) 670 | (mk-postgres 14 60014) 671 | (mk-postgres 13 60013) 672 | )) 673 | (mariadb 674 | (append 675 | (mk-mysql "MariaDB" 11.4 60114) 676 | (mk-mysql "MariaDB" 10.11 60111) 677 | (mk-mysql "MariaDB" 10.6 60106) 678 | (mk-mysql "MariaDB" 10.5 60105))) 679 | (mysql 680 | (append 681 | (mk-mysql "MySQL" 8.4 60284) 682 | (mk-mysql "MySQL" 8.0 60280))) 683 | (sqlserver 684 | (append 685 | (mk-sqlserver 2022 60322 nil '(:schema "nondbo")) 686 | (mk-sqlserver 2019 60319 nil '(:schema "nondbo")) 687 | (mk-sqlserver 2017 60317 nil '(:schema "nondbo")) 688 | ))) 689 | (eval 690 | `(describe-io-specs 691 | ,@sqlite 692 | ,@postgres 693 | ,@mariadb 694 | ,@mysql 695 | ,@sqlserver 696 | ) 697 | t))) 698 | 699 | 700 | ;;; org-sql-test-stateful ends here 701 | -------------------------------------------------------------------------------- /test/org-sql-test-stateless.el: -------------------------------------------------------------------------------- 1 | ;;; org-sql-test-stateless.el --- Stateless tests for org-sql -*- lexical-binding: t; -*- 2 | 3 | ;; Copyright (C) 2020 Nathan Dwarshuis 4 | 5 | ;; This program is free software; you can redistribute it and/or modify 6 | ;; it under the terms of the GNU General Public License as published by 7 | ;; the Free Software Foundation, either version 3 of the License, or 8 | ;; (at your option) any later version. 9 | 10 | ;; This program is distributed in the hope that it will be useful, 11 | ;; but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | ;; GNU General Public License for more details. 14 | 15 | ;; You should have received a copy of the GNU General Public License 16 | ;; along with this program. If not, see . 17 | 18 | ;;; Commentary: 19 | 20 | ;; This spec tests the stateless functions in org-sql 21 | 22 | ;;; Code: 23 | 24 | (require 'org-sql) 25 | (require 's) 26 | (require 'buttercup) 27 | 28 | (defmacro list-to-lines (in) 29 | "Convert IN to string. 30 | If IN is a string, return IN. If IN is a list starting with 31 | list then join the cdr of IN with newlines." 32 | (cond 33 | ((stringp in) in) 34 | ((consp in) `(s-join "\n" ,in)) 35 | (t (error "String or list of strings expected")))) 36 | 37 | (defun org-ts-to-unixtime (timestamp-string) 38 | "Convert TIMESTAMP-STRING to unixtime." 39 | (let ((decoded (org-parse-time-string timestamp-string))) 40 | (->> (-snoc decoded (current-time-zone)) 41 | (apply #'encode-time) 42 | (float-time) 43 | (round)))) 44 | 45 | (defconst testing-filepath "/tmp/dummy") 46 | 47 | ;; this is just the output from some random file (doesn't matter which one) 48 | (defconst testing-attributes (list nil 1 1000 1000 49 | '(21670 15994 677140 892000) 50 | '(21670 15994 677140 892000) 51 | '(22909 25484 42946 839000) 52 | 0 "-rw-r--r--" t 435278 65025)) 53 | 54 | (defconst testing-hash "123456") 55 | 56 | (defconst testing-size 2112) 57 | 58 | (defconst testing-lines 666) 59 | 60 | (defconst testing-file_metadata 61 | `(file_metadata (,testing-filepath 62 | ,testing-hash 63 | ,(file-attribute-user-id testing-attributes) 64 | ,(file-attribute-group-id testing-attributes) 65 | ,(->> (file-attribute-modification-time testing-attributes) 66 | (float-time) 67 | (round)) 68 | ,(->> (file-attribute-status-change-time testing-attributes) 69 | (float-time) 70 | (round)) 71 | ,(file-attribute-modes testing-attributes)))) 72 | 73 | (defconst testing-outlines* 74 | `(,testing-hash ,testing-size ,testing-lines)) 75 | 76 | (defconst init-ids 77 | (list :headline-id 1 78 | :timestamp-id 1 79 | :entry-id 1 80 | :link-id 1 81 | :property-id 1 82 | :clock-id 1)) 83 | 84 | (defmacro expect-sql* (in tbl res-form) 85 | `(progn 86 | (insert (list-to-lines ,in)) 87 | (let ((res ,res-form)) 88 | (expect res :to-equal ,tbl)))) 89 | 90 | (defun buffer-get-sml () 91 | (let ((lb-config (list :log-into-drawer org-log-into-drawer 92 | :clock-into-drawer org-clock-into-drawer 93 | :clock-out-notes org-log-note-clock-out)) 94 | (paths-with-attributes (list (cons testing-filepath testing-attributes))) 95 | (acc (org-sql--init-acc (-clone init-ids)))) 96 | (--> (org-ml-parse-this-buffer) 97 | (org-sql--to-outline-config testing-hash paths-with-attributes 98 | org-log-note-headings '("TODO" "DONE") 99 | lb-config testing-size testing-lines it) 100 | (org-sql--outline-config-to-insert-alist acc it) 101 | (plist-get it :inserts) 102 | (-filter #'cdr it)))) 103 | 104 | (defmacro expect-sql (in tbl) 105 | (declare (indent 1)) 106 | `(expect-sql* ,in ,tbl (buffer-get-sml))) 107 | 108 | (defmacro expect-sql-tbls (names in tbl) 109 | (declare (indent 2)) 110 | `(expect-sql* ,in ,tbl (->> (buffer-get-sml) 111 | (--filter (member (car it) ',names))))) 112 | 113 | (defmacro expect-sql-tbls-multi (names in &rest forms) 114 | (declare (indent 2)) 115 | (unless (= 0 (mod (length forms) 3)) 116 | (error "Missing form")) 117 | (let ((specs 118 | (->> (-partition 3 forms) 119 | (--map (-let (((title let-forms tbl) it)) 120 | `(it ,title 121 | (let (,@let-forms) 122 | (expect-sql-tbls ,names ,in ,tbl)))))))) 123 | `(progn ,@specs))) 124 | 125 | (defmacro expect-sql-logbook-item (in log-note-headings entry) 126 | (declare (indent 2)) 127 | `(progn 128 | (insert (list-to-lines ,in)) 129 | (cl-flet 130 | ((props-to-string 131 | (props e) 132 | (->> (cdr e) 133 | (-partition 2) 134 | (--map-when (memq (car it) props) 135 | (list (car it) 136 | (-some-> (cadr it) 137 | (org-ml-to-trimmed-string)))) 138 | (apply #'append) 139 | (cons (car e))))) 140 | (let* ((item (org-ml-parse-item-at 1)) 141 | (headline (->> (org-ml-build-headline! :title-text "dummy") 142 | ;; TODO shouldn't this be settable? 143 | (org-ml--set-property-nocheck :begin 1))) 144 | (lb-config (list :log-into-drawer org-log-into-drawer 145 | :clock-into-drawer org-clock-into-drawer 146 | :clock-out-notes org-log-note-clock-out)) 147 | (paths-with-attributes 148 | (list (cons testing-filepath testing-attributes))) 149 | (outline-config (org-sql--to-outline-config testing-hash paths-with-attributes 150 | ,log-note-headings '("TODO" "DONE") 151 | lb-config testing-size testing-lines 152 | nil)) 153 | (hstate (org-sql--to-hstate 1 outline-config headline)) 154 | (entry (->> (org-sql--item-to-entry hstate item) 155 | (props-to-string (list :ts 156 | :ts-active 157 | :short-ts 158 | :short-ts-active 159 | :old-ts 160 | :new-ts))))) 161 | (should (equal entry ,entry)))))) 162 | 163 | (describe "logbook entry spec" 164 | ;; don't test clocks here since they are way simpler 165 | (before-all 166 | (org-mode)) 167 | 168 | (before-each 169 | (erase-buffer)) 170 | 171 | (it "default - none" 172 | (expect-sql-logbook-item (list "- logbook item \\\\" 173 | " fancy note") 174 | org-log-note-headings 175 | `(none :outline-hash ,testing-hash 176 | :header-text "logbook item" 177 | :note-text "fancy note" 178 | :user nil 179 | :user-full nil 180 | :ts nil 181 | :ts-active nil 182 | :short-ts nil 183 | :short-ts-active nil 184 | :old-ts nil 185 | :new-ts nil 186 | :old-state nil 187 | :new-state nil))) 188 | 189 | (it "default - state" 190 | (let* ((ts "[2112-01-03 Sun]") 191 | (h (format "State \"DONE\" from \"TODO\" %s" ts))) 192 | (expect-sql-logbook-item (list (format "- %s \\\\" h) " fancy note") 193 | org-log-note-headings 194 | `(state :outline-hash ,testing-hash 195 | :header-text ,h 196 | :note-text "fancy note" 197 | :user nil 198 | :user-full nil 199 | :ts ,ts 200 | :ts-active nil 201 | :short-ts nil 202 | :short-ts-active nil 203 | :old-ts nil 204 | :new-ts nil 205 | :old-state "TODO" 206 | :new-state "DONE")))) 207 | 208 | (it "default - refile" 209 | (let* ((ts "[2112-01-03 Sun]") 210 | (h (format "Refiled on %s" ts))) 211 | (expect-sql-logbook-item (list (format "- %s \\\\" h) " fancy note") 212 | org-log-note-headings 213 | `(refile :outline-hash ,testing-hash 214 | :header-text ,h 215 | :note-text "fancy note" 216 | :user nil 217 | :user-full nil 218 | :ts ,ts 219 | :ts-active nil 220 | :short-ts nil 221 | :short-ts-active nil 222 | :old-ts nil 223 | :new-ts nil 224 | :old-state nil 225 | :new-state nil)))) 226 | 227 | (it "default - note" 228 | (let* ((ts "[2112-01-03 Sun]") 229 | (h (format "Note taken on %s" ts))) 230 | (expect-sql-logbook-item (list (format "- %s \\\\" h) " fancy note") 231 | org-log-note-headings 232 | `(note :outline-hash ,testing-hash 233 | :header-text ,h 234 | :note-text "fancy note" 235 | :user nil 236 | :user-full nil 237 | :ts ,ts 238 | :ts-active nil 239 | :short-ts nil 240 | :short-ts-active nil 241 | :old-ts nil 242 | :new-ts nil 243 | :old-state nil 244 | :new-state nil)))) 245 | 246 | (it "default - done" 247 | (let* ((ts "[2112-01-03 Sun]") 248 | (h (format "CLOSING NOTE %s" ts))) 249 | (expect-sql-logbook-item (list (format "- %s \\\\" h) " fancy note") 250 | org-log-note-headings 251 | `(done :outline-hash ,testing-hash 252 | :header-text ,h 253 | :note-text "fancy note" 254 | :user nil 255 | :user-full nil 256 | :ts ,ts 257 | :ts-active nil 258 | :short-ts nil 259 | :short-ts-active nil 260 | :old-ts nil 261 | :new-ts nil 262 | :old-state nil 263 | :new-state nil)))) 264 | 265 | (it "default - reschedule" 266 | (let* ((ts "[2112-01-03 Sun]") 267 | (ts0 "[2112-01-04 Mon]") 268 | (h (format "Rescheduled from \"%s\" on %s" ts0 ts))) 269 | (expect-sql-logbook-item (list (format "- %s \\\\" h) " fancy note") 270 | org-log-note-headings 271 | `(reschedule :outline-hash ,testing-hash 272 | :header-text ,h 273 | :note-text "fancy note" 274 | :user nil 275 | :user-full nil 276 | :ts ,ts 277 | :ts-active nil 278 | :short-ts nil 279 | :short-ts-active nil 280 | :old-ts ,ts0 281 | :new-ts nil 282 | :old-state nil 283 | :new-state nil)))) 284 | 285 | (it "default - delschedule" 286 | (let* ((ts "[2112-01-03 Sun]") 287 | (ts0 "[2112-01-04 Mon]") 288 | (h (format "Not scheduled, was \"%s\" on %s" ts0 ts))) 289 | (expect-sql-logbook-item (list (format "- %s \\\\" h) " fancy note") 290 | org-log-note-headings 291 | `(delschedule :outline-hash ,testing-hash 292 | :header-text ,h 293 | :note-text "fancy note" 294 | :user nil 295 | :user-full nil 296 | :ts ,ts 297 | :ts-active nil 298 | :short-ts nil 299 | :short-ts-active nil 300 | :old-ts ,ts0 301 | :new-ts nil 302 | :old-state nil 303 | :new-state nil)))) 304 | 305 | (it "default - redeadline" 306 | (let* ((ts "[2112-01-03 Sun]") 307 | (ts0 "[2112-01-04 Mon]") 308 | (h (format "New deadline from \"%s\" on %s" ts0 ts))) 309 | (expect-sql-logbook-item (list (format "- %s \\\\" h) " fancy note") 310 | org-log-note-headings 311 | `(redeadline :outline-hash ,testing-hash 312 | :header-text ,h 313 | :note-text "fancy note" 314 | :user nil 315 | :user-full nil 316 | :ts ,ts 317 | :ts-active nil 318 | :short-ts nil 319 | :short-ts-active nil 320 | :old-ts ,ts0 321 | :new-ts nil 322 | :old-state nil 323 | :new-state nil)))) 324 | 325 | (it "default - deldeadline" 326 | (let* ((ts "[2112-01-03 Sun]") 327 | (ts0 "[2112-01-04 Mon]") 328 | (h (format "Removed deadline, was \"%s\" on %s" ts0 ts))) 329 | (expect-sql-logbook-item (list (format "- %s \\\\" h) " fancy note") 330 | org-log-note-headings 331 | `(deldeadline :outline-hash ,testing-hash 332 | :header-text ,h 333 | :note-text "fancy note" 334 | :user nil 335 | :user-full nil 336 | :ts ,ts 337 | :ts-active nil 338 | :short-ts nil 339 | :short-ts-active nil 340 | :old-ts ,ts0 341 | :new-ts nil 342 | :old-state nil 343 | :new-state nil)))) 344 | 345 | (it "custom - user" 346 | (let* ((user "eddie666") 347 | (h (format "User %s is the best user" user))) 348 | (expect-sql-logbook-item (list (format "- %s \\\\" h) " fancy note") 349 | '((user . "User %u is the best user")) 350 | `(user :outline-hash ,testing-hash 351 | :header-text ,h 352 | :note-text "fancy note" 353 | :user ,user 354 | :user-full nil 355 | :ts nil 356 | :ts-active nil 357 | :short-ts nil 358 | :short-ts-active nil 359 | :old-ts nil 360 | :new-ts nil 361 | :old-state nil 362 | :new-state nil)))) 363 | 364 | (it "custom - user full" 365 | ;; TODO this variable can have spaces and such, which will currently not 366 | ;; be matched 367 | (let* ((userfull "FullName") 368 | (h (format "User %s is the best user" userfull))) 369 | (expect-sql-logbook-item (list (format "- %s \\\\" h) " fancy note") 370 | '((userfull . "User %u is the best user")) 371 | `(userfull :outline-hash ,testing-hash 372 | :header-text ,h 373 | :note-text "fancy note" 374 | :user ,userfull 375 | :user-full nil 376 | :ts nil 377 | :ts-active nil 378 | :short-ts nil 379 | :short-ts-active nil 380 | :old-ts nil 381 | :new-ts nil 382 | :old-state nil 383 | :new-state nil)))) 384 | 385 | (it "custom - active timestamp" 386 | (let* ((ts "<2112-01-01 Fri 00:00>") 387 | (h (format "I'm active now: %s" ts))) 388 | (expect-sql-logbook-item (list (format "- %s \\\\" h) " fancy note") 389 | '((activets . "I'm active now: %T")) 390 | `(activets :outline-hash ,testing-hash 391 | :header-text ,h 392 | :note-text "fancy note" 393 | :user nil 394 | :user-full nil 395 | :ts nil 396 | :ts-active ,ts 397 | :short-ts nil 398 | :short-ts-active nil 399 | :old-ts nil 400 | :new-ts nil 401 | :old-state nil 402 | :new-state nil)))) 403 | 404 | (it "custom - short timestamp" 405 | (let* ((ts "[2112-01-01 Fri]") 406 | (h (format "Life feels short now: %s" ts))) 407 | (expect-sql-logbook-item (list (format "- %s \\\\" h) " fancy note") 408 | '((shortts . "Life feels short now: %d")) 409 | `(shortts :outline-hash ,testing-hash 410 | :header-text ,h 411 | :note-text "fancy note" 412 | :user nil 413 | :user-full nil 414 | :ts nil 415 | :ts-active nil 416 | :short-ts ,ts 417 | :short-ts-active nil 418 | :old-ts nil 419 | :new-ts nil 420 | :old-state nil 421 | :new-state nil)))) 422 | 423 | (it "custom - short active timestamp" 424 | (let* ((ts "<2112-01-01 Fri>") 425 | (h (format "Life feels short now: %s" ts))) 426 | (expect-sql-logbook-item (list (format "- %s \\\\" h) " fancy note") 427 | '((shortts . "Life feels short now: %D")) 428 | `(shortts :outline-hash ,testing-hash 429 | :header-text ,h 430 | :note-text "fancy note" 431 | :user nil 432 | :user-full nil 433 | :ts nil 434 | :ts-active nil 435 | :short-ts nil 436 | :short-ts-active ,ts 437 | :old-ts nil 438 | :new-ts nil 439 | :old-state nil 440 | :new-state nil)))) 441 | 442 | (it "custom - old/new timestamps" 443 | (let* ((ts0 "[2112-01-01 Fri]") 444 | (ts1 "[2112-01-02 Sat]") 445 | (h (format "Fake clock: \"%s\"--\"%s\"" ts0 ts1))) 446 | (expect-sql-logbook-item (list (format "- %s \\\\" h) " fancy note") 447 | '((fakeclock . "Fake clock: %S--%s")) 448 | `(fakeclock :outline-hash ,testing-hash 449 | :header-text ,h 450 | :note-text "fancy note" 451 | :user nil 452 | :user-full nil 453 | :ts nil 454 | :ts-active nil 455 | :short-ts nil 456 | :short-ts-active nil 457 | :old-ts ,ts0 458 | :new-ts ,ts1 459 | :old-state nil 460 | :new-state nil))))) 461 | 462 | (describe "bulk insert spec" 463 | (before-all 464 | (org-mode)) 465 | 466 | (before-each 467 | (erase-buffer)) 468 | 469 | (describe "headlines" 470 | (it "single" 471 | (expect-sql "* headline" 472 | `((outlines (,@testing-outlines* nil)) 473 | ,testing-file_metadata 474 | (headlines (1 ,testing-hash "headline" 1 0 nil nil nil nil nil 0 0 nil)) 475 | (headline_closures (1 1 0))))) 476 | 477 | (it "two" 478 | (expect-sql (list "* headline" 479 | "* another headline") 480 | `((outlines (,@testing-outlines* nil)) 481 | ,testing-file_metadata 482 | (headlines (2 ,testing-hash "another headline" 1 1 nil nil nil nil nil 0 0 nil) 483 | (1 ,testing-hash "headline" 1 0 nil nil nil nil nil 0 0 nil)) 484 | (headline_closures (2 2 0) 485 | (1 1 0))))) 486 | 487 | (it "fancy" 488 | (expect-sql (list "stuff at the top" 489 | "* TODO [#A] COMMENT another headline <2024-08-12 Mon> [1/2]" 490 | ":PROPERTIES:" 491 | ":Effort: 0:30" 492 | ":END:" 493 | "this /should/ appear") 494 | `((outlines (,@testing-outlines* "stuff at the top\n")) 495 | ,testing-file_metadata 496 | (headlines 497 | (1 ,testing-hash "another headline <2024-08-12 Mon> [1/2]" 1 0 "TODO" 30 "A" fraction 0.5 498 | 0 1 "this /should/ appear\n")) 499 | (headline_closures 500 | (1 1 0)) 501 | (timestamps (1 1 "<2024-08-12 Mon>" 1 502 | ,(org-ts-to-unixtime "<2024-08-12 Mon>") nil 0 nil))))) 503 | 504 | (expect-sql-tbls-multi (outlines file_metadata headlines headline_closures) 505 | (list "* headline" 506 | "** nested headline") 507 | "nested (predicate applied to parent)" 508 | ((org-sql-exclude-headline-predicate 509 | (lambda (h) 510 | (= 1 (org-ml-get-property :level h))))) 511 | `((outlines (,@testing-outlines* nil)) 512 | ,testing-file_metadata) 513 | 514 | "nested (predicate applied to child)" 515 | ((org-sql-exclude-headline-predicate 516 | (lambda (h) 517 | (= 2 (org-ml-get-property :level h))))) 518 | `((outlines (,@testing-outlines* nil)) 519 | ,testing-file_metadata 520 | (headlines (1 ,testing-hash "headline" 1 0 nil nil nil nil nil 0 0 nil)) 521 | (headline_closures (1 1 0))) 522 | 523 | "nested (no predicate)" 524 | nil 525 | `((outlines (,@testing-outlines* nil)) 526 | ,testing-file_metadata 527 | (headlines (2 ,testing-hash "nested headline" 2 0 nil nil nil nil nil 0 0 nil) 528 | (1 ,testing-hash "headline" 1 0 nil nil nil nil nil 0 0 nil)) 529 | (headline_closures (2 2 0) 530 | (2 1 1) 531 | (1 1 0)))) 532 | 533 | (it "two (many nested)" 534 | (expect-sql (list "* a" 535 | "* b" 536 | "** c" 537 | "** d" 538 | "* e" 539 | "** f") 540 | `((outlines (,@testing-outlines* nil)) 541 | ,testing-file_metadata 542 | (headlines (6 ,testing-hash "f" 2 0 nil nil nil nil nil 0 0 nil) 543 | (5 ,testing-hash "e" 1 2 nil nil nil nil nil 0 0 nil) 544 | (4 ,testing-hash "d" 2 1 nil nil nil nil nil 0 0 nil) 545 | (3 ,testing-hash "c" 2 0 nil nil nil nil nil 0 0 nil) 546 | (2 ,testing-hash "b" 1 1 nil nil nil nil nil 0 0 nil) 547 | (1 ,testing-hash "a" 1 0 nil nil nil nil nil 0 0 nil)) 548 | (headline_closures (6 6 0) 549 | (6 5 1) 550 | (5 5 0) 551 | (4 4 0) 552 | (4 2 1) 553 | (3 3 0) 554 | (3 2 1) 555 | (2 2 0) 556 | (1 1 0))))) 557 | 558 | (it "archived" 559 | (expect-sql "* headline :ARCHIVE:" 560 | `((outlines (,@testing-outlines* nil)) 561 | ,testing-file_metadata 562 | (headlines (1 ,testing-hash "headline" 1 0 nil nil nil nil nil 1 0 nil)) 563 | (headline_closures (1 1 0)))))) 564 | 565 | (describe "planning entries" 566 | (let ((ts0 "<2112-01-01 Thu>") 567 | (ts1 "<2112-01-02 Fri>") 568 | (ts2 "[2112-01-03 Sat]")) 569 | (expect-sql-tbls-multi (planning_entries timestamps) 570 | (list "* headline" 571 | (format "SCHEDULED: %s DEADLINE: %s CLOSED: %s" ts0 ts1 ts2)) 572 | "multiple (included)" 573 | nil 574 | `((timestamps (3 1 ,ts0 1 ,(org-ts-to-unixtime ts0) nil 0 nil) 575 | (2 1 ,ts1 1 ,(org-ts-to-unixtime ts1) nil 0 nil) 576 | (1 1 ,ts2 0 ,(org-ts-to-unixtime ts2) nil 0 nil)) 577 | (planning_entries (3 scheduled) 578 | (2 deadline) 579 | (1 closed))) 580 | 581 | "multiple (exclude some)" 582 | ((org-sql-excluded-headline-planning-types '(:closed))) 583 | `((timestamps (2 1 ,ts0 1 ,(org-ts-to-unixtime ts0) nil 0 nil) 584 | (1 1 ,ts1 1 ,(org-ts-to-unixtime ts1) nil 0 nil)) 585 | (planning_entries (2 scheduled) 586 | (1 deadline))) 587 | 588 | "multiple (exclude all)" 589 | ((org-sql-excluded-headline-planning-types '(:closed :scheduled :deadline))) 590 | nil))) 591 | 592 | (describe "tags" 593 | (expect-sql-tbls-multi (headline_tags) (list "* headline :onetag:" 594 | "* headline :twotag:") 595 | "multiple (included)" 596 | nil 597 | `((headline_tags (2 "twotag" 0) 598 | (1 "onetag" 0))) 599 | 600 | "multiple (exclude one)" 601 | ((org-sql-excluded-tags '("onetag"))) 602 | `((headline_tags (2 "twotag" 0))) 603 | 604 | "multiple (exclude all)" 605 | ((org-sql-excluded-tags 'all)) 606 | nil) 607 | 608 | (it "single (child headline)" 609 | (setq org-sql-use-tag-inheritance t) 610 | (expect-sql-tbls (headline_tags) (list "* parent :onetag:" 611 | "** nested") 612 | `((headline_tags (1 "onetag" 0))))) 613 | 614 | (expect-sql-tbls-multi (headline_tags) (list "* parent" 615 | ":PROPERTIES:" 616 | ":ARCHIVE_ITAGS: sometag" 617 | ":END:") 618 | "inherited (included)" 619 | nil 620 | `((headline_tags (1 "sometag" 1))) 621 | 622 | "inherited (excluded)" 623 | ((org-sql-exclude-inherited-tags t)) 624 | nil)) 625 | 626 | (describe "file tags" 627 | (it "single" 628 | (expect-sql-tbls (file_tags) (list "#+FILETAGS: foo" 629 | "* headline") 630 | `((file_tags (,testing-hash "foo"))))) 631 | 632 | (it "multiple" 633 | (expect-sql-tbls (file_tags) (list "#+FILETAGS: foo bar" 634 | "#+FILETAGS: bang" 635 | "#+FILETAGS: bar" 636 | "* headline") 637 | `((file_tags (,testing-hash "bang") 638 | (,testing-hash "bar") 639 | (,testing-hash "foo")))))) 640 | 641 | (describe "timestamp" 642 | (it "closed" 643 | (let* ((ts "<2112-01-01 Thu>") 644 | (planning (format "CLOSED: %s" ts))) 645 | (expect-sql-tbls (timestamps) (list "* parent" 646 | planning) 647 | `((timestamps (1 1 ,ts 1 ,(org-ts-to-unixtime ts) nil 0 nil)))))) 648 | 649 | (it "closed (long)" 650 | (let* ((ts "<2112-01-01 Thu 00:00>") 651 | (planning (format "CLOSED: %s" ts))) 652 | (expect-sql-tbls (timestamps) (list "* parent" 653 | planning) 654 | `((timestamps (1 1 ,ts 1 ,(org-ts-to-unixtime ts) nil 1 nil)))))) 655 | 656 | (it "deadline (repeater)" 657 | (let* ((ts "<2112-01-01 Thu +2d>") 658 | (planning (format "DEADLINE: %s" ts))) 659 | (expect-sql-tbls (timestamps timestamp_modifiers timestamp_repeaters) 660 | (list "* parent" 661 | planning) 662 | `((timestamps (1 1 ,ts 1 ,(org-ts-to-unixtime ts) nil 0 nil)) 663 | (timestamp_repeaters (1 2 day cumulate nil nil)))))) 664 | 665 | (it "deadline (repeater + habit)" 666 | (let* ((ts "<2112-01-01 Thu +2d/3d>") 667 | (planning (format "DEADLINE: %s" ts))) 668 | (expect-sql-tbls (timestamps timestamp_modifiers timestamp_repeaters) 669 | (list "* parent" 670 | planning) 671 | `((timestamps (1 1 ,ts 1 ,(org-ts-to-unixtime ts) nil 0 nil)) 672 | (timestamp_repeaters (1 2 day cumulate 3 day)))))) 673 | 674 | (it "deadline (warning)" 675 | (let* ((ts "<2112-01-01 Thu -2d>") 676 | (planning (format "DEADLINE: %s" ts))) 677 | (expect-sql-tbls (timestamps timestamp_modifiers timestamp_warnings) 678 | (list "* parent" 679 | planning) 680 | `((timestamps (1 1 ,ts 1 ,(org-ts-to-unixtime ts) nil 0 nil)) 681 | (timestamp_warnings (1 2 day all)))))) 682 | 683 | (let* ((ts1 "<2112-01-01 Thu>") 684 | (ts2 "[2112-01-02 Fri]")) 685 | (expect-sql-tbls-multi (timestamps) (list "* parent" 686 | ts1 687 | ts2) 688 | "multiple content (included)" 689 | nil 690 | `((timestamps (2 1 ,ts2 0 ,(org-ts-to-unixtime ts2) nil 0 nil) 691 | (1 1 ,ts1 1 ,(org-ts-to-unixtime ts1) nil 0 nil))) 692 | 693 | "multiple content (exclude some)" 694 | ((org-sql-excluded-contents-timestamp-types '(inactive))) 695 | `((timestamps (1 1 ,ts1 1 ,(org-ts-to-unixtime ts1) nil 0 nil))) 696 | 697 | "multiple content (exclude all)" 698 | ((org-sql-excluded-contents-timestamp-types 'all)) 699 | nil)) 700 | 701 | (it "content (nested)" 702 | (let* ((ts "<2112-01-01 Thu>")) 703 | (expect-sql-tbls (timestamps) (list "* parent" 704 | "** child" 705 | ts) 706 | `((timestamps (1 2 ,ts 1 ,(org-ts-to-unixtime ts) nil 0 nil)))))) 707 | 708 | (it "content (ranged)" 709 | (let* ((ts0 "<2112-01-01 Thu>") 710 | (ts1 "<2112-01-02 Fri>") 711 | (ts (format "%s--%s" ts0 ts1))) 712 | (expect-sql-tbls (timestamps) (list "* parent" 713 | ts) 714 | `((timestamps (1 1 ,ts 1 ,(org-ts-to-unixtime ts0) 715 | ,(org-ts-to-unixtime ts1) 0 0))))))) 716 | 717 | (describe "links" 718 | (expect-sql-tbls-multi (links) (list "* parent" 719 | "https://example.org" 720 | "file:///the/glass/prison") 721 | "multiple (included)" 722 | nil 723 | `((links (2 1 "/the/glass/prison" nil "file") 724 | (1 1 "//example.org" nil "https"))) 725 | 726 | "multiple (exclude some)" 727 | ((org-sql-excluded-link-types '("file"))) 728 | `((links (1 1 "//example.org" nil "https"))) 729 | 730 | "multiple (exclude all)" 731 | ((org-sql-excluded-link-types 'all)) 732 | nil) 733 | 734 | (it "single (nested)" 735 | (expect-sql-tbls (links) (list "* parent" 736 | "** child" 737 | "https://example.com") 738 | `((links (1 2 "//example.com" nil "https"))))) 739 | 740 | (it "with description" 741 | (expect-sql-tbls (links) (list "* parent" 742 | "[[https://example.org][relevant]]") 743 | `((links (1 1 "//example.org" "relevant" "https")))))) 744 | 745 | (describe "properties" 746 | (it "single" 747 | (expect-sql-tbls (properties headline_properties) 748 | (list "* parent" 749 | ":PROPERTIES:" 750 | ":key: val" 751 | ":END:") 752 | `((properties (,testing-hash 1 "key" "val")) 753 | (headline_properties (1 1))))) 754 | 755 | (it "multiple" 756 | (expect-sql-tbls (properties headline_properties) 757 | (list "* parent" 758 | ":PROPERTIES:" 759 | ":p1: ragtime dandies" 760 | ":p2: this time its personal" 761 | ":END:") 762 | `((properties (,testing-hash 2 "p2" "this time its personal") 763 | (,testing-hash 1 "p1" "ragtime dandies")) 764 | (headline_properties (1 2) 765 | (1 1))))) 766 | 767 | ;; TODO add inherited properties once they exist 768 | 769 | (it "single file" 770 | (expect-sql-tbls (properties file_properties) 771 | (list "#+PROPERTY: FOO bar" 772 | "* parent") 773 | `((properties (,testing-hash 1 "FOO" "bar")))))) 774 | 775 | (describe "logbook" 776 | (describe "clocks" 777 | (it "single (closed)" 778 | (let* ((ts0 "[2112-01-01 Fri 00:00]") 779 | (ts1 "[2112-01-02 Sat 01:00]") 780 | (clock (format "CLOCK: %s--%s => 1:00" ts0 ts1))) 781 | (expect-sql-tbls (clocks) (list "* parent" 782 | ":LOGBOOK:" 783 | clock 784 | ":END:") 785 | `((clocks (1 1 ,(org-ts-to-unixtime ts0) 786 | ,(org-ts-to-unixtime ts1) nil)))))) 787 | 788 | (it "single (open)" 789 | (let* ((ts "[2112-01-01 Fri 00:00]") 790 | (clock (format "CLOCK: %s" ts))) 791 | (expect-sql-tbls (clocks) (list "* parent" 792 | ":LOGBOOK:" 793 | clock 794 | ":END:") 795 | `((clocks (1 1 ,(org-ts-to-unixtime ts) nil nil)))))) 796 | 797 | (let* ((ts "[2112-01-01 Fri 00:00]") 798 | (clock (format "CLOCK: %s" ts))) 799 | (expect-sql-tbls-multi (clocks) (list "* parent" 800 | ":LOGBOOK:" 801 | clock 802 | "- random" 803 | ":END:") 804 | "single (note - included)" 805 | ((org-log-note-clock-out t)) 806 | `((clocks (1 1 ,(org-ts-to-unixtime ts) nil "random"))) 807 | 808 | "single (note - excluded)" 809 | ((org-log-note-clock-out t) 810 | (org-sql-exclude-clock-notes t)) 811 | `((clocks (1 1 ,(org-ts-to-unixtime ts) nil nil))))) 812 | 813 | (it "multiple" 814 | (let* ((ts0 "[2112-01-01 Fri 00:00]") 815 | (ts1 "[2112-01-01 Fri 01:00]") 816 | (clock0 (format "CLOCK: %s" ts0)) 817 | (clock1 (format "CLOCK: %s" ts1))) 818 | (expect-sql-tbls (clocks) (list "* parent" 819 | ":LOGBOOK:" 820 | clock0 821 | clock1 822 | ":END:") 823 | `((clocks (2 1 ,(org-ts-to-unixtime ts1) nil nil) 824 | (1 1 ,(org-ts-to-unixtime ts0) nil nil))))))) 825 | 826 | (describe "items" 827 | (it "multiple" 828 | (let* ((ts0 "[2112-01-01 Fri 00:00]") 829 | (ts1 "[2112-01-01 Fri 01:00]") 830 | (note0 (format "Note taken on %s" ts0)) 831 | (note1 (format "Note taken on %s" ts1))) 832 | (expect-sql-tbls (logbook_entries) (list "* parent" 833 | (format "- %s" note0) 834 | (format "- %s" note1)) 835 | `((logbook_entries (2 1 "note" ,(org-ts-to-unixtime ts1) ,note1 nil) 836 | (1 1 "note" ,(org-ts-to-unixtime ts0) ,note0 nil)))))) 837 | 838 | (let* ((ts "[2112-01-01 Fri 00:00]") 839 | (header (format "Note taken on %s" ts)) 840 | (note "fancy note")) 841 | (expect-sql-tbls-multi (logbook_entries) (list "* parent" 842 | (format "- %s \\\\" header) 843 | (format " %s" note)) 844 | "note (included)" 845 | nil 846 | `((logbook_entries (1 1 "note" ,(org-ts-to-unixtime ts) ,header ,note))) 847 | 848 | "note (exclude)" 849 | ((org-sql-excluded-logbook-types '(note))) 850 | nil)) 851 | 852 | (let* ((ts "[2112-01-01 Fri 00:00]") 853 | (header (format "State \"DONE\" from \"TODO\" %s" ts))) 854 | (expect-sql-tbls-multi (logbook_entries state_changes) 855 | (list "* parent" 856 | (format "- %s" header)) 857 | "state change (included)" 858 | nil 859 | `((logbook_entries (1 1 "state" ,(org-ts-to-unixtime ts) ,header nil)) 860 | (state_changes (1 "TODO" "DONE"))) 861 | 862 | "state change (excluded)" 863 | ((org-sql-excluded-logbook-types '(state))) 864 | nil)) 865 | 866 | (let* ((ts0 "[2112-01-01 Fri 00:00]") 867 | (ts1 "[2112-01-01 Fri 01:00]") 868 | (header (format "Rescheduled from \"%s\" on %s" ts0 ts1))) 869 | (expect-sql-tbls-multi (logbook_entries timestamps planning_changes) 870 | (list "* parent" 871 | (format "- %s" header)) 872 | "rescheduled (included)" 873 | nil 874 | `((timestamps (1 1 ,ts0 0 ,(org-ts-to-unixtime ts0) nil 1 nil)) 875 | (logbook_entries (1 1 "reschedule" ,(org-ts-to-unixtime ts1) ,header nil)) 876 | (planning_changes (1 1))) 877 | 878 | "rescheduled (excluded)" 879 | ((org-sql-excluded-logbook-types '(reschedule))) 880 | nil)) 881 | 882 | (let* ((ts0 "[2112-01-01 Fri 00:00]") 883 | (ts1 "[2112-01-01 Fri 01:00]") 884 | (header (format "New deadline from \"%s\" on %s" ts0 ts1))) 885 | (expect-sql-tbls-multi (logbook_entries timestamps planning_changes) 886 | (list "* parent" 887 | (format "- %s" header)) 888 | "redeadline (included)" 889 | nil 890 | `((timestamps (1 1 ,ts0 0 ,(org-ts-to-unixtime ts0) nil 1 nil)) 891 | (logbook_entries (1 1 "redeadline" ,(org-ts-to-unixtime ts1) 892 | ,header nil)) 893 | (planning_changes (1 1))) 894 | 895 | "redeadline (excluded)" 896 | ((org-sql-excluded-logbook-types '(redeadline))) 897 | nil)) 898 | 899 | (let* ((ts0 "[2112-01-01 Fri 00:00]") 900 | (ts1 "[2112-01-01 Fri 01:00]") 901 | (header (format "Not scheduled, was \"%s\" on %s" ts0 ts1))) 902 | (expect-sql-tbls-multi (logbook_entries timestamps planning_changes) 903 | (list "* parent" 904 | (format "- %s" header)) 905 | "delschedule (included)" 906 | nil 907 | `((timestamps (1 1 ,ts0 0 ,(org-ts-to-unixtime ts0) nil 1 nil)) 908 | (logbook_entries (1 1 "delschedule" ,(org-ts-to-unixtime ts1) 909 | ,header nil)) 910 | (planning_changes (1 1))) 911 | 912 | "delschedule (excluded)" 913 | ((org-sql-excluded-logbook-types '(delschedule))) 914 | nil)) 915 | 916 | (let* ((ts0 "[2112-01-01 Fri 00:00]") 917 | (ts1 "[2112-01-01 Fri 01:00]") 918 | (header (format "Removed deadline, was \"%s\" on %s" ts0 ts1))) 919 | (expect-sql-tbls-multi (logbook_entries timestamps planning_changes) 920 | (list "* parent" 921 | (format "- %s" header)) 922 | "deldeadline (included)" 923 | nil 924 | `((timestamps (1 1 ,ts0 0 ,(org-ts-to-unixtime ts0) nil 1 nil)) 925 | (logbook_entries (1 1 "deldeadline" ,(org-ts-to-unixtime ts1) 926 | ,header nil)) 927 | (planning_changes (1 1))) 928 | 929 | "deldeadline (excluded)" 930 | ((org-sql-excluded-logbook-types '(deldeadline))) 931 | nil)) 932 | 933 | (let* ((ts "[2112-01-01 Fri 00:00]") 934 | (header (format "Refiled on %s" ts))) 935 | (expect-sql-tbls-multi (logbook_entries) (list "* parent" 936 | (format "- %s" header)) 937 | "refile (included)" 938 | nil 939 | `((logbook_entries (1 1 "refile" ,(org-ts-to-unixtime ts) ,header nil))) 940 | 941 | "refile (excluded)" 942 | ((org-sql-excluded-logbook-types '(refile))) 943 | nil)) 944 | 945 | (let* ((ts "[2112-01-01 Fri 00:00]") 946 | (header (format "CLOSING NOTE %s" ts))) 947 | (expect-sql-tbls-multi (logbook_entries) (list "* parent" 948 | (format "- %s" header)) 949 | "done (included)" 950 | nil 951 | `((logbook_entries (1 1 "done" ,(org-ts-to-unixtime ts) ,header nil))) 952 | 953 | "done (excluded)" 954 | ((org-sql-excluded-logbook-types '(done))) 955 | nil))) 956 | 957 | (describe "mixture" 958 | (it "clock + non-note" 959 | (let* ((ts "[2112-01-01 Fri 00:00]") 960 | (header (format "CLOSING NOTE %s" ts)) 961 | (ts0 "[2112-01-01 Fri 00:00]") 962 | (ts1 "[2112-01-02 Sat 01:00]") 963 | (clock (format "CLOCK: %s--%s => 1:00" ts0 ts1))) 964 | (expect-sql-tbls (clocks logbook_entries) (list "* parent" 965 | ":LOGBOOK:" 966 | clock 967 | ":END:" 968 | (format "- %s" header)) 969 | `((clocks (1 1 ,(org-ts-to-unixtime ts0) 970 | ,(org-ts-to-unixtime ts1) nil)) 971 | (logbook_entries (1 1 "done" ,(org-ts-to-unixtime ts) ,header nil)))))) 972 | 973 | (it "clock + note + non-note" 974 | (let* ((org-log-note-clock-out t) 975 | (ts "[2112-01-01 Fri 00:00]") 976 | (header (format "CLOSING NOTE %s" ts)) 977 | (ts0 "[2112-01-01 Fri 00:00]") 978 | (ts1 "[2112-01-02 Sat 01:00]") 979 | (clock (format "CLOCK: %s--%s => 1:00" ts0 ts1))) 980 | (expect-sql-tbls (clocks logbook_entries) (list "* parent" 981 | ":LOGBOOK:" 982 | clock 983 | " - this is a clock note" 984 | ":END:" 985 | (format "- %s" header)) 986 | `((clocks (1 1 ,(org-ts-to-unixtime ts0) 987 | ,(org-ts-to-unixtime ts1) "this is a clock note")) 988 | (logbook_entries (1 1 "done" ,(org-ts-to-unixtime ts) ,header nil)))))) 989 | 990 | (it "non-note + clock" 991 | (let* ((ts "[2112-01-01 Fri 00:00]") 992 | (header (format "CLOSING NOTE %s" ts)) 993 | (ts0 "[2112-01-01 Fri 00:00]") 994 | (ts1 "[2112-01-02 Sat 01:00]") 995 | (clock (format "CLOCK: %s--%s => 1:00" ts0 ts1))) 996 | (expect-sql-tbls (clocks logbook_entries) (list "* parent" 997 | (format "- %s" header) 998 | ":LOGBOOK:" 999 | clock 1000 | ":END:") 1001 | `((clocks (1 1 ,(org-ts-to-unixtime ts0) 1002 | ,(org-ts-to-unixtime ts1) nil)) 1003 | (logbook_entries (1 1 "done" ,(org-ts-to-unixtime ts) ,header nil)))))) 1004 | 1005 | (it "non-note + clock + clock note" 1006 | (let* ((org-log-note-clock-out t) 1007 | (ts "[2112-01-01 Fri 00:00]") 1008 | (header (format "CLOSING NOTE %s" ts)) 1009 | (ts0 "[2112-01-01 Fri 00:00]") 1010 | (ts1 "[2112-01-02 Sat 01:00]") 1011 | (clock (format "CLOCK: %s--%s => 1:00" ts0 ts1))) 1012 | (expect-sql-tbls (clocks logbook_entries) (list "* parent" 1013 | (format "- %s" header) 1014 | ":LOGBOOK:" 1015 | clock 1016 | "- this is a clock note" 1017 | ":END:") 1018 | `((clocks (1 1 ,(org-ts-to-unixtime ts0) 1019 | ,(org-ts-to-unixtime ts1) "this is a clock note")) 1020 | (logbook_entries (1 1 "done" ,(org-ts-to-unixtime ts) ,header nil))))))) 1021 | 1022 | (describe "non-default drawer configs" 1023 | (it "log drawer (global)" 1024 | (let* ((org-log-into-drawer "LOGGING") 1025 | (ts "[2112-01-01 Fri 00:00]") 1026 | (header (format "CLOSING NOTE %s" ts))) 1027 | (expect-sql-tbls (logbook_entries) (list "* parent" 1028 | ":LOGGING:" 1029 | (format "- %s" header) 1030 | ":END:") 1031 | `((logbook_entries (1 1 "done" ,(org-ts-to-unixtime ts) ,header nil)))))) 1032 | 1033 | (it "log drawer (file)" 1034 | (let* ((ts "[2112-01-01 Fri 00:00]") 1035 | (header (format "CLOSING NOTE %s" ts))) 1036 | (expect-sql-tbls (logbook_entries) (list "#+STARTUP: logdrawer" 1037 | "* parent" 1038 | ":LOGBOOK:" 1039 | (format "- %s" header) 1040 | ":END:") 1041 | `((logbook_entries (1 1 "done" ,(org-ts-to-unixtime ts) ,header nil)))))) 1042 | (it "log drawer (property)" 1043 | (let* ((ts "[2112-01-01 Fri 00:00]") 1044 | (header (format "CLOSING NOTE %s" ts))) 1045 | (expect-sql-tbls (logbook_entries) (list "* parent" 1046 | ":PROPERTIES:" 1047 | ":LOG_INTO_DRAWER: LOGGING" 1048 | ":END:" 1049 | ":LOGGING:" 1050 | (format "- %s" header) 1051 | ":END:") 1052 | `((logbook_entries (1 1 "done" ,(org-ts-to-unixtime ts) ,header nil)))))) 1053 | 1054 | (it "clock drawer (global)" 1055 | (let* ((org-clock-into-drawer "CLOCKING") 1056 | (ts0 "[2112-01-01 Fri 00:00]") 1057 | (ts1 "[2112-01-02 Sat 01:00]") 1058 | (clock (format "CLOCK: %s--%s => 1:00" ts0 ts1))) 1059 | (expect-sql-tbls (clocks) (list "* parent" 1060 | ":CLOCKING:" 1061 | clock 1062 | ":END:") 1063 | `((clocks (1 1 ,(org-ts-to-unixtime ts0) 1064 | ,(org-ts-to-unixtime ts1) nil)))))) 1065 | 1066 | (it "clock drawer (property)" 1067 | (let* ((ts0 "[2112-01-01 Fri 00:00]") 1068 | (ts1 "[2112-01-02 Sat 01:00]") 1069 | (clock (format "CLOCK: %s--%s => 1:00" ts0 ts1))) 1070 | (expect-sql-tbls (clocks) (list "* parent" 1071 | ":PROPERTIES:" 1072 | ":CLOCK_INTO_DRAWER: CLOCKING" 1073 | ":END:" 1074 | ":CLOCKING:" 1075 | clock 1076 | ":END:") 1077 | `((clocks (1 1 ,(org-ts-to-unixtime ts0) 1078 | ,(org-ts-to-unixtime ts1) nil)))))) 1079 | 1080 | (it "clock note (global)" 1081 | (let* ((org-log-note-clock-out t) 1082 | (ts0 "[2112-01-01 Fri 00:00]") 1083 | (ts1 "[2112-01-02 Sat 01:00]") 1084 | (clock (format "CLOCK: %s--%s => 1:00" ts0 ts1))) 1085 | (expect-sql-tbls (clocks) (list "* parent" 1086 | ":LOGBOOK:" 1087 | clock 1088 | "- clock out note" 1089 | ":END:") 1090 | `((clocks (1 1 ,(org-ts-to-unixtime ts0) 1091 | ,(org-ts-to-unixtime ts1) "clock out note")))))) 1092 | 1093 | (it "clock note (file)" 1094 | (let* ((ts0 "[2112-01-01 Fri 00:00]") 1095 | (ts1 "[2112-01-02 Sat 01:00]") 1096 | (clock (format "CLOCK: %s--%s => 1:00" ts0 ts1))) 1097 | (expect-sql-tbls (clocks) (list "#+STARTUP: lognoteclock-out" 1098 | "* parent" 1099 | ":LOGBOOK:" 1100 | clock 1101 | "- clock out note" 1102 | ":END:") 1103 | `((clocks (1 1 ,(org-ts-to-unixtime ts0) 1104 | ,(org-ts-to-unixtime ts1) "clock out note"))))))))) 1105 | 1106 | (defun format-with (config type value) 1107 | (funcall (org-sql--compile-serializer config type) value)) 1108 | 1109 | (defun format-with-sqlite (type value) 1110 | (format-with '(sqlite) type value)) 1111 | 1112 | (defun expect-formatter (type input &rest value-plist) 1113 | (declare (indent 2)) 1114 | (-let (((&plist :sqlite :postgres :mysql :sqlserver) value-plist)) 1115 | (expect (format-with '(mysql) type input) :to-equal mysql) 1116 | (expect (format-with '(postgres) type input) :to-equal postgres) 1117 | (expect (format-with '(sqlite) type input) :to-equal sqlite) 1118 | (expect (format-with '(sqlserver) type input) :to-equal sqlserver))) 1119 | 1120 | (describe "type formatting spec" 1121 | (describe "boolean" 1122 | (it "NULL" 1123 | (expect-formatter 'boolean nil 1124 | :mysql "NULL" 1125 | :postgres "NULL" 1126 | :sqlite "NULL" 1127 | :sqlserver "NULL")) 1128 | 1129 | (it "TRUE" 1130 | (expect-formatter 'boolean 1 1131 | :mysql "TRUE" 1132 | :postgres "TRUE" 1133 | :sqlite "1" 1134 | :sqlserver "1")) 1135 | 1136 | (it "FALSE" 1137 | (expect-formatter 'boolean 0 1138 | :mysql "FALSE" 1139 | :postgres "FALSE" 1140 | :sqlite "0" 1141 | :sqlserver "0"))) 1142 | 1143 | (describe "enum" 1144 | (it "NULL" 1145 | (expect-formatter 'enum nil 1146 | :mysql "NULL" 1147 | :postgres "NULL" 1148 | :sqlite "NULL" 1149 | :sqlserver "NULL")) 1150 | 1151 | (it "defined" 1152 | (expect-formatter 'enum 'foo 1153 | :mysql "'foo'" 1154 | :postgres "'foo'" 1155 | :sqlite "'foo'" 1156 | :sqlserver "'foo'"))) 1157 | 1158 | ;; ASSUME this is the same as REAL 1159 | (describe "integer" 1160 | (it "NULL" 1161 | (expect-formatter 'integer nil 1162 | :mysql "NULL" 1163 | :postgres "NULL" 1164 | :sqlite "NULL" 1165 | :sqlserver "NULL")) 1166 | 1167 | (it "defined" 1168 | (expect-formatter 'integer 123456 1169 | :mysql "123456" 1170 | :postgres "123456" 1171 | :sqlite "123456" 1172 | :sqlserver "123456"))) 1173 | 1174 | ;; ASSUME this is the same as VARCHAR and CHAR 1175 | (describe "text" 1176 | (it "NULL" 1177 | (expect-formatter 'text nil 1178 | :mysql "NULL" 1179 | :postgres "NULL" 1180 | :sqlite "NULL" 1181 | :sqlserver "NULL")) 1182 | 1183 | (it "plain" 1184 | (expect-formatter 'text "foo" 1185 | :mysql "'foo'" 1186 | :postgres "'foo'" 1187 | :sqlite "'foo'" 1188 | :sqlserver "'foo'")) 1189 | 1190 | (it "newlines" 1191 | (expect-formatter 'text "foo\nbar" 1192 | :mysql "'foo\nbar'" 1193 | :postgres "'foo\nbar'" 1194 | :sqlite "'foo\nbar'" 1195 | :sqlserver "'foo\nbar'")) 1196 | 1197 | (it "quotes" 1198 | (expect-formatter 'text "'foo'" 1199 | :mysql "'''foo'''" 1200 | :postgres "'''foo'''" 1201 | :sqlite "'''foo'''" 1202 | :sqlserver "'''foo'''")))) 1203 | 1204 | (describe "meta-query language statement formatting spec" 1205 | (before-all 1206 | (setq org-sql--table-alist 1207 | '((table-foo 1208 | (columns 1209 | (:bool :type boolean) 1210 | (:enum :type enum :allowed (bim bam boo)) 1211 | (:int :type integer) 1212 | (:text :type text)) 1213 | (constraints 1214 | (primary :keys (:int)))) 1215 | (table-bar 1216 | (columns 1217 | (:intone :type integer) 1218 | (:inttwo :type integer)) 1219 | (constraints 1220 | (primary :keys (:intone)) 1221 | (foreign :ref table-foo 1222 | :keys (:inttwo) 1223 | :parent-keys (:int) 1224 | ;; :on_update cascade 1225 | :on-delete cascade)))) 1226 | test-insert-alist 1227 | '((table-foo (0 bim 0 "xxx") (1 bam 1 "yyy")) 1228 | (table-bar (0 1) (2 3))))) 1229 | 1230 | (describe "bulk insert" 1231 | (it "SQLite" 1232 | (let ((config '(sqlite))) 1233 | (expect 1234 | (org-sql--format-bulk-inserts config test-insert-alist) 1235 | :to-equal 1236 | (concat "INSERT INTO table-foo (bool,enum,int,text) VALUES (0,'bim',0,'xxx'),(1,'bam',1,'yyy');" 1237 | "INSERT INTO table-bar (intone,inttwo) VALUES (0,1),(2,3);")))) 1238 | 1239 | (it "Postgres" 1240 | (let ((config '(postgres))) 1241 | (expect 1242 | (org-sql--format-bulk-inserts config test-insert-alist) 1243 | :to-equal 1244 | (concat "INSERT INTO table-foo (bool,enum,int,text) VALUES (FALSE,'bim',0,'xxx'),(TRUE,'bam',1,'yyy');" 1245 | "INSERT INTO table-bar (intone,inttwo) VALUES (0,1),(2,3);")))) 1246 | 1247 | (it "MySQL" 1248 | (let ((config '(mysql))) 1249 | (expect 1250 | (org-sql--format-bulk-inserts config test-insert-alist) 1251 | :to-equal 1252 | (concat "INSERT INTO table-foo (bool,enum,int,text) VALUES (FALSE,'bim',0,'xxx'),(TRUE,'bam',1,'yyy');" 1253 | "INSERT INTO table-bar (intone,inttwo) VALUES (0,1),(2,3);")))) 1254 | 1255 | (it "SQL-Server" 1256 | (let ((config '(sqlserver))) 1257 | (expect 1258 | (org-sql--format-bulk-inserts config test-insert-alist) 1259 | :to-equal 1260 | (concat "INSERT INTO table-foo (bool,enum,int,text) SELECT * FROM (VALUES (0,'bim',0,'xxx'),(1,'bam',1,'yyy')) d (bool,enum,int,text);" 1261 | "INSERT INTO table-bar (intone,inttwo) SELECT * FROM (VALUES (0,1),(2,3)) d (intone,inttwo);"))))) 1262 | 1263 | ;; TODO add bulk deletes...eventually 1264 | 1265 | (describe "create table" 1266 | (it "SQLite" 1267 | (let ((config '(sqlite))) 1268 | (expect 1269 | (org-sql--format-create-tables config org-sql--table-alist) 1270 | :to-equal 1271 | (list 1272 | "CREATE TABLE IF NOT EXISTS table-foo (bool INTEGER,enum TEXT,int INTEGER,text TEXT,PRIMARY KEY (int));" 1273 | "CREATE TABLE IF NOT EXISTS table-bar (intone INTEGER,inttwo INTEGER,PRIMARY KEY (intone),FOREIGN KEY (inttwo) REFERENCES table-foo (int) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED);")))) 1274 | 1275 | (it "postgres" 1276 | (let ((config '(postgres))) 1277 | (expect 1278 | (org-sql--format-create-tables config org-sql--table-alist) 1279 | :to-equal 1280 | (list 1281 | "CREATE TYPE enum_table-foo_enum AS ENUM ('bim','bam','boo');" 1282 | "CREATE TABLE IF NOT EXISTS table-foo (bool BOOLEAN,enum enum_table-foo_enum,int INTEGER,text TEXT,PRIMARY KEY (int));" 1283 | "CREATE TABLE IF NOT EXISTS table-bar (intone INTEGER,inttwo INTEGER,PRIMARY KEY (intone),FOREIGN KEY (inttwo) REFERENCES table-foo (int) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED);")))) 1284 | 1285 | (it "postgres - unlogged" 1286 | (let ((config '(postgres :unlogged t))) 1287 | (expect 1288 | (org-sql--format-create-tables config org-sql--table-alist) 1289 | :to-equal 1290 | (list 1291 | "CREATE TYPE enum_table-foo_enum AS ENUM ('bim','bam','boo');" 1292 | "CREATE UNLOGGED TABLE IF NOT EXISTS table-foo (bool BOOLEAN,enum enum_table-foo_enum,int INTEGER,text TEXT,PRIMARY KEY (int));" 1293 | "CREATE UNLOGGED TABLE IF NOT EXISTS table-bar (intone INTEGER,inttwo INTEGER,PRIMARY KEY (intone),FOREIGN KEY (inttwo) REFERENCES table-foo (int) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED);")))) 1294 | 1295 | (it "postgres - nonpublic" 1296 | (let ((config '(postgres :schema "nonpublic"))) 1297 | (expect 1298 | (org-sql--format-create-tables config org-sql--table-alist) 1299 | :to-equal 1300 | (list 1301 | "CREATE TYPE nonpublic.enum_table-foo_enum AS ENUM ('bim','bam','boo');" 1302 | "CREATE TABLE IF NOT EXISTS nonpublic.table-foo (bool BOOLEAN,enum nonpublic.enum_table-foo_enum,int INTEGER,text TEXT,PRIMARY KEY (int));" 1303 | "CREATE TABLE IF NOT EXISTS nonpublic.table-bar (intone INTEGER,inttwo INTEGER,PRIMARY KEY (intone),FOREIGN KEY (inttwo) REFERENCES nonpublic.table-foo (int) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED);")))) 1304 | 1305 | (it "mysql" 1306 | (let ((config '(mysql))) 1307 | (expect 1308 | (org-sql--format-create-tables config org-sql--table-alist) 1309 | :to-equal 1310 | (list 1311 | "CREATE TABLE IF NOT EXISTS table-foo (bool BOOLEAN,enum ENUM('bim','bam','boo'),int INTEGER,text TEXT,PRIMARY KEY (int));" 1312 | "CREATE TABLE IF NOT EXISTS table-bar (intone INTEGER,inttwo INTEGER,PRIMARY KEY (intone),FOREIGN KEY (inttwo) REFERENCES table-foo (int) ON DELETE CASCADE);")))) 1313 | 1314 | (it "sqlserver" 1315 | (let ((config '(sqlserver))) 1316 | (expect 1317 | (org-sql--format-create-tables config org-sql--table-alist) 1318 | :to-equal 1319 | (list 1320 | "IF NOT EXISTS (SELECT * FROM sys.tables where name = 'table-foo') CREATE TABLE table-foo (bool BIT,enum NVARCHAR(MAX),int INTEGER,text NVARCHAR(MAX),PRIMARY KEY (int));" 1321 | "IF NOT EXISTS (SELECT * FROM sys.tables where name = 'table-bar') CREATE TABLE table-bar (intone INTEGER,inttwo INTEGER,PRIMARY KEY (intone),FOREIGN KEY (inttwo) REFERENCES table-foo (int) ON DELETE CASCADE);")))) 1322 | 1323 | (it "sqlserver - nonpublic" 1324 | (let ((config '(sqlserver :schema "nonpublic"))) 1325 | (expect 1326 | (org-sql--format-create-tables config org-sql--table-alist) 1327 | :to-equal 1328 | (list 1329 | "IF NOT EXISTS (SELECT * FROM sys.tables where name = 'nonpublic.table-foo') CREATE TABLE nonpublic.table-foo (bool BIT,enum NVARCHAR(MAX),int INTEGER,text NVARCHAR(MAX),PRIMARY KEY (int));" 1330 | "IF NOT EXISTS (SELECT * FROM sys.tables where name = 'nonpublic.table-bar') CREATE TABLE nonpublic.table-bar (intone INTEGER,inttwo INTEGER,PRIMARY KEY (intone),FOREIGN KEY (inttwo) REFERENCES nonpublic.table-foo (int) ON DELETE CASCADE);"))))) 1331 | 1332 | (describe "transaction" 1333 | (it "sqlite" 1334 | (let ((config '(sqlite)) 1335 | (statements (list "INSERT INTO foo (bar) values (1);"))) 1336 | (expect 1337 | (org-sql--format-sql-transaction config statements) 1338 | :to-equal 1339 | "PRAGMA foreign_keys = ON;BEGIN;INSERT INTO foo (bar) values (1);COMMIT;"))) 1340 | 1341 | (it "postgres" 1342 | (let ((config '(postgres)) 1343 | (statements (list "INSERT INTO foo (bar) values (1);"))) 1344 | (expect 1345 | (org-sql--format-sql-transaction config statements) 1346 | :to-equal 1347 | "BEGIN;INSERT INTO foo (bar) values (1);COMMIT;"))) 1348 | 1349 | (it "mysql" 1350 | (let ((config '(mysql)) 1351 | (statements (list "INSERT INTO foo (bar) values (1);"))) 1352 | (expect 1353 | (org-sql--format-sql-transaction config statements) 1354 | :to-equal 1355 | "SET sql_mode = NO_BACKSLASH_ESCAPES;BEGIN;INSERT INTO foo (bar) values (1);COMMIT;"))) 1356 | 1357 | (it "sqlserver" 1358 | (let ((config '(sqlserver)) 1359 | (statements (list "INSERT INTO foo (bar) values (1);"))) 1360 | (expect 1361 | (org-sql--format-sql-transaction config statements) 1362 | :to-equal 1363 | "BEGIN TRANSACTION;INSERT INTO foo (bar) values (1);COMMIT;"))))) 1364 | 1365 | (describe "file metadata spec" 1366 | (it "classify file metadata" 1367 | (let ((on-disk '(("123" . "/bar.org") 1368 | ("654" . "/bam.org") 1369 | ("456" . "/foo.org"))) 1370 | (in-db '(("123" . "/bar.org") 1371 | ("654" . "/bam0.org") 1372 | ("789" . "/foo0.org")))) 1373 | (expect (org-sql--partition-hashpathpairs on-disk in-db) 1374 | :to-equal 1375 | '((files-to-insert ("456" . "/foo.org")) 1376 | (paths-to-insert ("654" . "/bam.org")) 1377 | (paths-to-delete ("654" . "/bam0.org")) 1378 | (files-to-delete ("789" . "/foo0.org"))))))) 1379 | 1380 | ;;; org-sql-test-stateless.el ends here 1381 | -------------------------------------------------------------------------------- /test/scripts/clear_hook.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE fake_init_table; 2 | -------------------------------------------------------------------------------- /test/scripts/init_hook.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE fake_init_table ( silly INTEGER PRIMARY KEY ); 2 | -------------------------------------------------------------------------------- /test/scripts/update_hook.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE fake_update_table ( silly INTEGER PRIMARY KEY ); 2 | --------------------------------------------------------------------------------