├── .github └── workflows │ └── test.yml ├── .gitignore ├── COPYING ├── README ├── README.md ├── TODO.org ├── journal.asd ├── src ├── doc.lisp ├── file-position-test-data ├── interrupt.lisp ├── journal.lisp ├── mgl-jrn.el └── package.lisp └── test ├── package.lisp ├── profile.lisp ├── registration └── 00000000.jrn ├── test-journal.lisp └── test.sh /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: 4 | push: 5 | pull_request: 6 | schedule: 7 | # Every day at 18:23 8 | - cron: "23 18 * * *" 9 | 10 | jobs: 11 | test: 12 | name: ${{ matrix.lisp }} on ${{ matrix.os }} 13 | runs-on: ${{ matrix.os }} 14 | timeout-minutes: 30 15 | strategy: 16 | matrix: 17 | lisp: [sbcl-bin, ccl-bin, cmu-bin, ecl, abcl-bin] 18 | os: [ubuntu-latest] 19 | fail-fast: false 20 | 21 | steps: 22 | - name: Checkout 23 | uses: actions/checkout@v4 24 | 25 | - name: Let ASDF find the project 26 | run: mkdir ~/common-lisp; ln -s `pwd` ~/common-lisp/journal 27 | 28 | - name: Install Roswell 29 | env: 30 | LISP: ${{ matrix.lisp }} 31 | run: | 32 | curl -L https://raw.githubusercontent.com/roswell/roswell/master/scripts/install-for-ci.sh | sh 33 | 34 | - name: Install latest MGL-PAX from GitHub 35 | run: ros install melisgl/mgl-pax 36 | 37 | - name: Install latest Try from GitHub 38 | run: ros install melisgl/try 39 | 40 | - name: Run tests 41 | env: 42 | LISP: ${{ matrix.lisp }} 43 | run: | 44 | ros run -e '(ql:quickload :journal)' -q 45 | ./test/test.sh $LISP 46 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | *.abcl-tmp 3 | *.fasl 4 | *.*fsl 5 | *.fas 6 | *.lib 7 | *.sse2f 8 | doc/ 9 | -------------------------------------------------------------------------------- /COPYING: -------------------------------------------------------------------------------- 1 | Copyright (c) 2020 Gábor Melis 2 | 3 | Permission is hereby granted, free of charge, to any person 4 | obtaining a copy of this software and associated documentation 5 | files (the "Software"), to deal in the Software without 6 | restriction, including without limitation the rights to use, 7 | copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | copies of the Software, and to permit persons to whom the 9 | Software is furnished to do so, subject to the following 10 | conditions: 11 | 12 | The above copyright notice and this permission notice shall be 13 | included in all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 17 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 19 | HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 20 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 21 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | # Journal manual 2 | 3 | ###### \[in package JOURNAL with nicknames JRN\] 4 | - [system] "journal" 5 | - _Version:_ 0.1.0 6 | - _Description:_ A library built around explicit execution traces for 7 | logging, tracing, testing and persistence. 8 | - _Licence:_ MIT, see COPYING. 9 | - _Author:_ Gábor Melis 10 | - _Homepage:_ [http://github.com/melisgl/journal](http://github.com/melisgl/journal) 11 | - _Bug tracker:_ [http://github.com/melisgl/journal/issues](http://github.com/melisgl/journal/issues) 12 | - _Source control:_ [GIT](https://github.com/melisgl/journal.git) 13 | - *Depends on:* alexandria, bordeaux-threads, local-time, mgl-pax, osicat(?), sb-posix(?), trivial-features, trivial-garbage 14 | 15 | ## Links 16 | 17 | Here is the [official repository](https://github.com/melisgl/journal) 18 | and the [HTML 19 | documentation](http://melisgl.github.io/mgl-pax-world/journal-manual.html) 20 | for the latest version. 21 | 22 | ## Portability 23 | 24 | Tested and supported on on ABCL, CCL, CMUCL, ECL, and SBCL. 25 | AllegroCL Express edition runs out of heap while running the tests. 26 | On Lisps that seem to lack support for disabling and enabling of 27 | interrupts, such as ABCL, durability is compromised, and any attempt 28 | to SYNC-JOURNAL (see @SYNCHRONIZATION-STRATEGIES and @SAFETY) will 29 | be a runtime error. 30 | 31 | Journal depends on BORDEAUX-THREADS. Consequently, it does not load 32 | on implementations without real thread such as CLISP. 33 | 34 | ## Background 35 | 36 | Logging, tracing, testing, and persistence are about what happened 37 | during code execution. Recording machine-readable logs and traces 38 | can be repurposed for white-box testing. More, when the code is 39 | rerun, selected frames may return their recorded values without 40 | executing the code, which could serve as a @MOCK-OBJECT framework 41 | for writing tests. This ability to isolate external interactions and 42 | to reexecute traces is sufficient to reconstruct the state of a 43 | program, achieving simple persistence not unlike a @JOURNALING-FS or 44 | @EVENT-SOURCING. 45 | 46 | Journal is the library to log, trace, test and persist. It has a 47 | single macro at its heart: JOURNALED, which does pretty much what 48 | was described. It can be thought of as generating two events around 49 | its body: one that records the name and an argument list (as in a 50 | function call), and another that records the return values. In 51 | Lisp-like pseudocode: 52 | 53 | ``` 54 | (defmacro journaled (name args &body body) 55 | `(progn 56 | (record-event `(:in ,name :args ,args)) 57 | (let ((,return-values (multiple-value-list (progn ,@body)))) 58 | (record-event `(:out ,name :values ,return-values)) 59 | (values-list ,return-values)))) 60 | ``` 61 | 62 | This is basically how recording works. When replaying events from a 63 | previous run, the return values of BODY can be checked against the 64 | recorded ones, or we may return the recorded values without even 65 | running BODY. 66 | 67 | In summary, we can produce selective execution traces by wrapping 68 | code in JOURNALED and use those traces for various purposes. The 69 | Journal library is this idea taken to its logical conclusion. 70 | 71 | ## Distinguishing features 72 | 73 | ##### As a logging facility 74 | 75 | - Nested contexts and single messages 76 | 77 | - Customizable content and format 78 | 79 | - Human- or machine-readable output 80 | 81 | ``` 82 | #68200.234: ("some-context") 83 | #68200.234: Informative log message 84 | #68200.250: => NIL 85 | ``` 86 | 87 | See @LOGGING for a complete example. 88 | 89 | ##### Compared to CL:TRACE 90 | 91 | - Ability to handle non-local exits 92 | 93 | - Customizable content and format 94 | 95 | - Optional timestamps, internal real- and run-time 96 | 97 | ``` 98 | (FOO 2.1) 99 | (1+ 2.1) 100 | => 3.1 101 | =E "SIMPLE-ERROR" "The assertion (INTEGERP 3.1) failed." 102 | ``` 103 | 104 | See @TRACING for a complete example. 105 | 106 | ##### As a test framework 107 | 108 | - White-box testing based on execution traces 109 | 110 | - Isolation of external dependencies 111 | 112 | - Record-and-replay testing 113 | 114 | ``` 115 | (define-file-bundle-test (test-user-registration :directory "registration") 116 | (let ((username (replayed ("ask-username") 117 | (format t "Please type your username: ") 118 | (read-line)))) 119 | (add-user username) 120 | (assert (user-exists-p username)))) 121 | ``` 122 | 123 | See @TESTING for a complete example. 124 | 125 | ##### As a solution for persistence 126 | 127 | - Event Sourcing: replay interactions with the external world 128 | 129 | - Unchanged control flow 130 | 131 | - Easy to implement history, undo 132 | 133 | ``` 134 | (defun my-resumable-autosaving-game-with-history () 135 | (with-bundle (bundle) 136 | (play-guess-my-number))) 137 | ``` 138 | 139 | See @PERSISTENCE for a complete example. 140 | 141 | ## Basics 142 | 143 | The JOURNALED macro does both recording and replaying of events, 144 | possibly at the same time. Recording is easy: events generated by 145 | JOURNALED are simply written to a journal, which is a sequence of 146 | events much like a file. What events are generated is described in 147 | JOURNALED. @REPLAY is much more involved, thus it gets its own 148 | section. The journals used for recording and replaying are specified 149 | by WITH-JOURNALING or by WITH-BUNDLE. 150 | 151 | The @JOURNALS-REFERENCE is presented later, but for most purposes, 152 | creating them (e.g. with MAKE-IN-MEMORY-JOURNAL, MAKE-FILE-JOURNAL) 153 | and maybe querying their contents with LIST-EVENTS will suffice. 154 | Some common cases of journal creation are handled by the convenience 155 | function TO-JOURNAL. 156 | 157 | Built on top of journals, @BUNDLES juggle repeated replay-and-record 158 | cycles focussing on persistence. 159 | 160 | - [generic-function] TO-JOURNAL DESIGNATOR 161 | 162 | Return the journal designated by DESIGNATOR or 163 | signal an error. The default implementation 164 | 165 | - returns DESIGNATOR itself if it is of type JOURNAL, 166 | 167 | - returns a new IN-MEMORY-JOURNAL if DESIGNATOR is T, 168 | 169 | - returns a new FILE-JOURNAL if DESIGNATOR is a PATHNAME. 170 | 171 | - [macro] WITH-JOURNALING (&KEY RECORD REPLAY REPLAY-EOJ-ERROR-P) &BODY BODY 172 | 173 | Turn recording and/or replaying of events on or off for the 174 | duration of BODY. Both RECORD and REPLAY should be a JOURNAL 175 | designator (in the sense of TO-JOURNAL) or NIL. 176 | 177 | If RECORD designates a JOURNAL, then events generated by enclosed 178 | JOURNALED @BLOCKs are written to that journal (with exceptions, see 179 | the LOG-RECORD argument of JOURNALED). If REPLAY designates a 180 | JOURNAL, then the generated events are matched against events from 181 | that journal according to the rules of @REPLAY. 182 | 183 | A JOURNAL-ERROR is signalled if RECORD is a JOURNAL that has been 184 | previously recorded to by another WITH-JOURNALING (that is, if its 185 | JOURNAL-STATE is not :NEW) or if REPLAY is a JOURNAL that is not a 186 | complete recording of successful replay (i.e. its JOURNAL-STATE is 187 | not :COMPLETED). These checks are intended to catch mistakes that 188 | would render the new or existing records unusable for replay. When 189 | WITH-JOURNALING finishes, the RECORD journal is marked :COMPLETED or 190 | :FAILED in its JOURNAL-STATE. 191 | 192 | REPLAY-EOJ-ERROR-P controls whether END-OF-JOURNAL is signalled when 193 | a new event is being matched to the replay journal from which there 194 | are no more events to read. If there was a JOURNALING-FAILURE or a 195 | REPLAY-FAILURE during execution, then END-OF-JOURNAL is not 196 | signalled. 197 | 198 | If BODY completes successfully, but REPLAY has unprocessed events, 199 | then REPLAY-INCOMPLETE is signalled. 200 | 201 | WITH-JOURNALING for different RECORD journals can be nested and run 202 | independently. 203 | 204 | - [glossary-term] block 205 | 206 | A journaled block, or simply block, is a number of forms wrapped in 207 | JOURNALED. When a block is executed, a @FRAME is created. 208 | 209 | - [glossary-term] frame 210 | 211 | A frame is an IN-EVENT, OUT-EVENT pair, which are created when a 212 | @BLOCK is entered and left, respectively. 213 | 214 | - [function] RECORD-JOURNAL 215 | 216 | Return the JOURNAL in which events are currently being 217 | recorded (see WITH-JOURNALING and WITH-BUNDLE) or NIL. 218 | 219 | - [function] REPLAY-JOURNAL 220 | 221 | Return the JOURNAL from which events are currently being 222 | replayed (see WITH-JOURNALING and WITH-BUNDLE) or NIL. 223 | 224 | - [macro] JOURNALED (NAME &KEY (LOG-RECORD :RECORD) VERSION ARGS VALUES CONDITION INSERTABLE REPLAY-VALUES REPLAY-CONDITION) &BODY BODY 225 | 226 | JOURNALED generates events upon entering and leaving the dynamic 227 | extent of BODY (also known as the journaled @BLOCK), which we call 228 | the @IN-EVENTS and @OUT-EVENTS. Between generating the two events, 229 | BODY is typically executed normally (except for 230 | @REPLAYING-THE-OUTCOME). 231 | 232 | Where the generated events are written is determined by the :RECORD 233 | argument of the enclosing WITH-JOURNALING. If there is no enclosing 234 | WITH-JOURNALING and LOG-RECORD is NIL, then event recording is 235 | turned off and JOURNALED imposes minimal overhead. 236 | 237 | - NAME can be of any type except NULL, not evaluated. For 238 | names, and for anything that gets written to a journal, a 239 | non-keyword symbol is a reasonable choice as it can be easily made 240 | unique. However, it also exposes the package structure, which 241 | might make reading stuff back more difficult. Keywords and strings 242 | do not have this problem. 243 | 244 | - ARGS can be of any type, but is typically a list. 245 | 246 | Also see @LOG-RECORD in the @LOGGING section. For a description of 247 | VERSION, INSERTABLE, REPLAY-VALUES and REPLAY-CONDITION, see 248 | @JOURNALED-FOR-REPLAY. 249 | 250 | ### In-events 251 | 252 | Upon entering a @BLOCK, JOURNALED generates an IN-EVENT, 253 | which conceptually opens a new @FRAME. These in-events are created 254 | from the NAME, VERSION and ARGS arguments of JOURNALED. For example, 255 | 256 | ``` 257 | (journaled (name :version version :args args) ...) 258 | ``` 259 | 260 | creates an event like this: 261 | 262 | ``` 263 | `(:in ,name :version ,version :args ,args) 264 | ``` 265 | 266 | where :VERSION and :ARGS may be omitted if they are NIL. Versions 267 | are used for @REPLAY. 268 | 269 | ### Out-events 270 | 271 | Upon leaving a @BLOCK, JOURNALED generates an OUT-EVENT, closing 272 | the @FRAME opened by the corresponding IN-EVENT. These out-events 273 | are property lists like this: 274 | 275 | ``` 276 | (:out foo :version 1 :values (42)) 277 | ``` 278 | 279 | Their NAME and VERSION (`FOO` and `1` in the example) are the same 280 | as in the in-event: they come from the corresponding arguments of 281 | JOURNALED. EXIT and OUTCOME are filled in differently depending on 282 | how the block finished its execution. 283 | 284 | - [type] EVENT-EXIT 285 | 286 | One of :VALUES, :CONDITION, :ERROR and :NLX. Indicates whether a 287 | journaled @BLOCK 288 | 289 | - returned normally (:VALUES, see @VALUES-OUTCOME), 290 | 291 | - unwound on an expected condition (:CONDITION, see @CONDITION-OUTCOME), 292 | 293 | - unwound on an unexpected condition (:ERROR, see @ERROR-OUTCOME), 294 | 295 | - unwound by performing a non-local exit of some other kind 296 | such as a throw (:NLX, see @NLX-OUTCOME). 297 | 298 | The first two are @EXPECTED-OUTCOMEs, while the latter two are 299 | @UNEXPECTED-OUTCOMEs. 300 | 301 | - [glossary-term] values outcome 302 | 303 | If the JOURNALED @BLOCK returns normally, EVENT-EXIT is 304 | :VALUES, and the outcome is the list of values returned: 305 | 306 | ``` 307 | (journaled (foo) (values 7 t)) 308 | ;; generates the out-event 309 | (:out foo :values (7 t)) 310 | ``` 311 | 312 | The list of return values of the block is transformed by the VALUES 313 | argument of JOURNALED, whose default is `#'IDENTITY`. Also see 314 | @WORKING-WITH-UNREADABLE-VALUES). 315 | 316 | - [glossary-term] condition outcome 317 | 318 | If the @BLOCK unwound due to a condition, and JOURNALED's 319 | CONDITION argument (a function whose default is `(CONSTANTLY NIL)`) 320 | returns non-NIL when invoked on it, then EVENT-EXIT is 321 | :CONDITION, and the outcome is this return value: 322 | 323 | ``` 324 | (journaled (foo :condition (lambda (c) (prin1-to-string c))) 325 | (error "xxx")) 326 | ;; generates the out-event 327 | (:out foo :condition "xxx") 328 | ``` 329 | 330 | Conditions thus recognized are those that can be considered part of 331 | normal execution. Just like return values, these expected conditions 332 | may be required to match what's in the replay journal. Furthermore, 333 | given a suitable REPLAY-CONDITION in JOURNALED, they may be replayed 334 | without running the @BLOCK. 335 | 336 | - [glossary-term] error outcome 337 | 338 | If the JOURNALED @BLOCK unwound due to a condition, but 339 | JOURNALED's CONDITION argument returns NIL when invoked on it, then 340 | EVENT-EXIT is :ERROR and the outcome the string 341 | representations of the type of the condition and the condition 342 | itself. 343 | 344 | ``` 345 | (journaled (foo) 346 | (error "xxx")) 347 | ;; generates this out-event: 348 | ;; (:out foo :error ("simple-error" "xxx")) 349 | ``` 350 | 351 | The conversion to string is performed with PRINC in 352 | WITH-STANDARD-IO-SYNTAX. This scheme is intended to avoid leaking 353 | random implementation details into the journal, which would make 354 | `READ`ing it back difficult. 355 | 356 | In contrast with @CONDITION-OUTCOMEs, error outcomes are what the 357 | code is not prepared to handle or replay in a meaningful way. 358 | 359 | - [glossary-term] nlx outcome 360 | 361 | If the JOURNALED @BLOCK performed a non-local exit that 362 | was not due to a condition, then EVENT-EXIT is :NLX and the 363 | outcome is NIL. 364 | 365 | ``` 366 | (catch 'xxx 367 | (journaled (foo) 368 | (throw 'xxx nil))) 369 | ;; generates the out-event 370 | (:out foo :nlx nil) 371 | ``` 372 | 373 | Note that @CONDITION-OUTCOMEs and @ERROR-OUTCOMEs are also due to 374 | non-local exits but are distinct from nlx outcomes. 375 | 376 | Currently, nlx outcomes are detected rather heuristically as there 377 | is no portable way to detect what really caused the unwinding of the 378 | stack. 379 | 380 | There is a further grouping of outcomes into expected and unexpected. 381 | 382 | - [glossary-term] expected outcome 383 | 384 | An OUT-EVENT is said to have an expected outcome if it had a 385 | @VALUES-OUTCOME or a @CONDITION-OUTCOME, or equivalently, when its 386 | EVENT-EXIT is :VALUES or :CONDITION. 387 | 388 | - [glossary-term] unexpected outcome 389 | 390 | An OUT-EVENT is said to have an unexpected outcome if it had an 391 | @ERROR-OUTCOME or an @NLX-OUTCOME, or equivalently, when its 392 | EVENT-EXIT is :ERROR or :NLX. 393 | 394 | ### Working with unreadable values 395 | 396 | The events recorded often need to be @READABLE. This is always 397 | required with FILE-JOURNALs, often with IN-MEMORY-JOURNALs, but 398 | never with PPRINT-JOURNALs. By choosing an appropriate identifier or 399 | string representation of the unreadable object to journal, this is 400 | not a problem in practice. JOURNALED provides the VALUES 401 | hook for this purpose. 402 | 403 | With EXTERNAL-EVENTs, whose outcome is replayed (see 404 | @REPLAYING-THE-OUTCOME), we also need to be able to reverse the 405 | transformation of VALUES, and this is what the 406 | REPLAY-VALUES argument of JOURNALED is for. 407 | 408 | Let's see a complete example. 409 | 410 | ``` 411 | (defclass user () 412 | ((id :initarg :id :reader user-id))) 413 | 414 | (defmethod print-object ((user user) stream) 415 | (print-unreadable-object (user stream :type t) 416 | (format stream "~S" (slot-value user 'id)))) 417 | 418 | (defvar *users* (make-hash-table)) 419 | 420 | (defun find-user (id) 421 | (gethash id *users*)) 422 | 423 | (defun add-user (id) 424 | (setf (gethash id *users*) (make-instance 'user :id id))) 425 | 426 | (defvar *user7* (add-user 7)) 427 | 428 | (defun get-message () 429 | (replayed (listen :values (values-> #'user-id) 430 | :replay-values (values<- #'find-user)) 431 | (values *user7* "hello"))) 432 | 433 | (jtrace user-id find-user get-message) 434 | 435 | (let ((bundle (make-file-bundle "/tmp/user-example/"))) 436 | (format t "Recording") 437 | (with-bundle (bundle) 438 | (get-message)) 439 | (format t "~%Replaying") 440 | (with-bundle (bundle) 441 | (get-message))) 442 | .. Recording 443 | .. (GET-MESSAGE) 444 | .. (USER-ID #) 445 | .. => 7 446 | .. => #, "hello" 447 | .. Replaying 448 | .. (GET-MESSAGE) 449 | .. (FIND-USER 7) 450 | .. => #, T 451 | .. => #, "hello" 452 | ==> # 453 | => "hello" 454 | ``` 455 | 456 | To be able to journal the return values of `GET-MESSAGE`, the `USER` 457 | object must be transformed to something @READABLE. On the 458 | `Recording` run, `(VALUES-> #'USER-ID)` replaces the user object 459 | with its id in the EVENT-OUTCOME recorded, but the original user 460 | object is returned. 461 | 462 | When `Replaying`, the journaled OUT-EVENT is replayed (see 463 | @REPLAYING-THE-OUTCOME): 464 | 465 | ``` 466 | (:OUT GET-MESSAGE :VERSION :INFINITY :VALUES (7 "hello")) 467 | ``` 468 | 469 | The user object is looked up according to :REPLAY-VALUES and is 470 | returned along with `"hello"`. 471 | 472 | - [function] VALUES-> &REST FNS 473 | 474 | A utility to create a function suitable as the VALUES 475 | argument of JOURNALED. The VALUES function is called with the list 476 | of values returned by the @BLOCK and returns a transformed set of 477 | values that may be recorded in a journal. While arbitrary 478 | transformations are allowed, `VALUES->` handles the common case of 479 | transforming individual elements of the list independently by 480 | calling the functions in FN with the values of the list of the same 481 | position. 482 | 483 | ``` 484 | (funcall (values-> #'1+) '(7 :something)) 485 | => (8 :SOMETHING) 486 | ``` 487 | 488 | Note how `#'1+` is applied only to the first element of the values 489 | list. The list of functions is shorter than the values list, so 490 | `:SOMETHING` is not transformed. A value can be left explicitly 491 | untransformed by specifying #'IDENTITY or NIL as the function: 492 | 493 | ``` 494 | (funcall (values-> #'1+ nil #'symbol-name) 495 | '(7 :something :another)) 496 | => (8 :SOMETHING "ANOTHER") 497 | ``` 498 | 499 | - [function] VALUES<- &REST FNS 500 | 501 | The inverse of `VALUES->`, this returns a function suitable as 502 | the REPLAY-VALUES argument of JOURNALED. It does pretty much what 503 | `VALUES->` does, but the function returned returns the transformed 504 | list as multiple values instead of as a list. 505 | 506 | ``` 507 | (funcall (values<- #'1-) '(8 :something)) 508 | => 7 509 | => :SOMETHING 510 | ``` 511 | 512 | ### Utilities 513 | 514 | - [function] LIST-EVENTS &OPTIONAL (JOURNAL (RECORD-JOURNAL)) 515 | 516 | Return a list of all the events in the journal designated by 517 | JOURNAL. Calls SYNC-JOURNAL first to make sure that all writes are 518 | taken into account. 519 | 520 | - [function] EVENTS-TO-FRAMES EVENTS 521 | 522 | Convert a flat list of events, such as those returned by LIST-EVENTS, 523 | to a nested list representing the @FRAMEs. Each frame is a list of 524 | the form `( * ?)`. Like in 525 | PRINT-EVENTS, EVENTS may be a JOURNAL. 526 | 527 | ``` 528 | (events-to-frames '((:in foo :args (1 2)) 529 | (:in bar :args (7)) 530 | (:leaf "leaf") 531 | (:out bar :values (8)) 532 | (:out foo :values (2)) 533 | (:in foo :args (3 4)) 534 | (:in bar :args (8)))) 535 | => (((:IN FOO :ARGS (1 2)) 536 | ((:IN BAR :ARGS (7)) 537 | (:LEAF "leaf") 538 | (:OUT BAR :VALUES (8))) 539 | (:OUT FOO :VALUES (2))) 540 | ((:IN FOO :ARGS (3 4)) ((:IN BAR :ARGS (8))))) 541 | ``` 542 | 543 | Note that, as in the above example, incomplete frames (those without 544 | an OUT-EVENT) are included in the output. 545 | 546 | - [function] EXPECTED-TYPE TYPE 547 | 548 | Return a function suitable as the CONDITION argument of JOURNALED, 549 | which returns the type of its single argument as a string if it is 550 | of TYPE, else NIL. 551 | 552 | ### Pretty-printing 553 | 554 | - [function] PRINT-EVENTS EVENTS &KEY STREAM 555 | 556 | Print EVENTS to STREAM as lists, starting a new line for each 557 | event and indenting them according to their nesting structure. 558 | EVENTS may be a sequence or a JOURNAL, in which case LIST-EVENTS is 559 | called on it first. 560 | 561 | ``` 562 | (print-events '((:in log :args ("first arg" 2)) 563 | (:in versioned :version 1 :args (3)) 564 | (:out versioned :version 1 :values (42 t)) 565 | (:out log :condition "a :CONDITION outcome") 566 | (:in log-2) 567 | (:out log-2 :nlx nil) 568 | (:in external :version :infinity) 569 | (:out external :version :infinity 570 | :error ("ERROR" "an :ERROR outcome")))) 571 | .. 572 | .. (:IN LOG :ARGS ("first arg" 2)) 573 | .. (:IN VERSIONED :VERSION 1 :ARGS (3)) 574 | .. (:OUT VERSIONED :VERSION 1 :VALUES (42 T)) 575 | .. (:OUT LOG :CONDITION "a :CONDITION outcome") 576 | .. (:IN LOG-2) 577 | .. (:OUT LOG-2 :NLX NIL) 578 | .. (:IN EXTERNAL :VERSION :INFINITY) 579 | .. (:OUT EXTERNAL :VERSION :INFINITY :ERROR ("ERROR" "an :ERROR outcome")) 580 | => ; No value 581 | ``` 582 | 583 | - [function] PPRINT-EVENTS EVENTS &KEY STREAM (PRETTIFIER 'PRETTIFY-EVENT) 584 | 585 | Like PRINT-EVENTS, but produces terser, more human readable 586 | output. 587 | 588 | ``` 589 | (pprint-events '((:in log :args ("first arg" 2)) 590 | (:in versioned :version 1 :args (3)) 591 | (:leaf "This is a leaf, not a frame.") 592 | (:out versioned :version 1 :values (42 t)) 593 | (:out log :condition "a :CONDITION outcome") 594 | (:in log-2) 595 | (:out log-2 :nlx nil) 596 | (:in external :version :infinity) 597 | (:out external :version :infinity 598 | :error ("ERROR" "an :ERROR outcome")))) 599 | .. 600 | .. (LOG "first arg" 2) 601 | .. (VERSIONED 3) v1 602 | .. This is a leaf, not a frame. 603 | .. => 42, T 604 | .. =C "a :CONDITION outcome" 605 | .. (LOG-2) 606 | .. =X 607 | .. (EXTERNAL) ext 608 | .. =E "ERROR" "an :ERROR outcome" 609 | => ; No value 610 | ``` 611 | 612 | The function given as the PRETTIFIER argument formats individual 613 | events. The above output was produced with PRETTIFY-EVENT. For a 614 | description of PRETTIFIER's arguments see PRETTIFY-EVENT. 615 | 616 | - [function] PRETTIFY-EVENT EVENT DEPTH STREAM 617 | 618 | Write EVENT to STREAM in a somewhat human-friendly format. 619 | This is the function PPRINT-JOURNAL, PPRINT-EVENTS, and @TRACING use 620 | by default. In addition to the basic example in PPRINT-EVENTS, 621 | @DECORATION on events is printed before normal, indented output like 622 | this: 623 | 624 | ``` 625 | (pprint-events '((:leaf "About to sleep" :time "19:57:00" :function "FOO"))) 626 | .. 627 | .. 19:57:00 FOO: About to sleep 628 | ``` 629 | 630 | DEPTH is the nesting level of the EVENT. Top-level events have depth 631 | 0. PRETTIFY-EVENT prints indents the output after printing the 632 | decorations by 2 spaces per depth. 633 | 634 | Instead of collecting events and then printing them, events can 635 | be pretty-printed to a stream as they generated. This is 636 | accomplished with @PPRINT-JOURNALS, discussed in detail later, in 637 | the following way: 638 | 639 | ``` 640 | (let ((journal (make-pprint-journal))) 641 | (with-journaling (:record journal) 642 | (journaled (foo) "Hello"))) 643 | .. 644 | .. (FOO) 645 | .. => "Hello" 646 | ``` 647 | 648 | Note that @PPRINT-JOURNALS are not tied to WITH-JOURNALING and are 649 | most often used for @LOGGING and @TRACING. 650 | 651 | ### Error handling 652 | 653 | - [condition] JOURNALING-FAILURE SERIOUS-CONDITION 654 | 655 | Signalled during the dynamic extent of 656 | WITH-JOURNALING when an error threatens to leave the journaling 657 | mechanism in an inconsistent state. These include I/O errors 658 | encountered reading or writing journals by WITH-JOURNALING, 659 | JOURNALED, LOGGED, WITH-REPLAY-FILTER, SYNC-JOURNAL, and also 660 | STORAGE-CONDITIONs, assertion failures, errors calling JOURNALED's 661 | VALUES and CONDITION function arguments. 662 | Crucially, this does not apply to non-local exits from other 663 | code, such as JOURNALED @BLOCKs, whose error handling is largely 664 | unaltered (see @OUT-EVENTS and @REPLAY-FAILURES). 665 | 666 | In general, any non-local exit from critical parts of the 667 | code is turned into a JOURNALING-FAILURE to protect the integrity of 668 | the RECORD-JOURNAL. The condition that caused the unwinding is in 669 | JOURNALING-FAILURE-EMBEDDED-CONDITION, or NIL if it was a pure 670 | non-local exit like THROW. This is a SERIOUS-CONDITION, not 671 | to be handled within WITH-JOURNALING. 672 | 673 | After a JOURNALING-FAILURE, the journaling mechanism cannot be 674 | trusted anymore. The REPLAY-JOURNAL might have failed a read and be 675 | out-of-sync. The RECORD-JOURNAL may have missing events (or even 676 | half-written events with FILE-JOURNALs without SYNC, see 677 | @SYNCHRONIZATION-STRATEGIES), and further writes to it would risk 678 | replayability, which is equivalent to database corruption. Thus, 679 | upon signalling JOURNALING-FAILURE, JOURNAL-STATE is set to 680 | 681 | - :COMPLETED if the journal is in state :RECORDING or :LOGGING and 682 | the transition to :RECORDING was reflected in storage, 683 | 684 | - else it is set to :FAILED. 685 | 686 | After a JOURNALING-FAILURE, any further attempt within the affected 687 | WITH-JOURNALING to use the critical machinery mentioned 688 | above (JOURNALED, LOGGED, etc) resignals the same journal failure 689 | condition. As a consequence, the record journal cannot be changed, 690 | and the only way to recover is to leave WITH-JOURNALING. This does 691 | not affect processing in other threads, which by design cannot write 692 | to the record journal. 693 | 694 | Note that in contrast with JOURNALING-FAILURE and REPLAY-FAILURE, 695 | which necessitate leaving WITH-JOURNALING to recover from, the other 696 | conditions – JOURNAL-ERROR, and STREAMLET-ERROR – are subclasses of 697 | ERROR as the their handling need not be so 698 | heavy-handed. 699 | 700 | - [reader] JOURNALING-FAILURE-EMBEDDED-CONDITION [JOURNALING-FAILURE][3956] (:EMBEDDED-CONDITION) 701 | 702 | - [condition] RECORD-UNEXPECTED-OUTCOME 703 | 704 | Signalled (with SIGNAL: this is not an 705 | ERROR) by JOURNALED when a VERSIONED-EVENT or an 706 | EXTERNAL-EVENT had an UNEXPECTED-OUTCOME while in JOURNAL-STATE 707 | :RECORDING. Upon signalling this condition, JOURNAL-STATE is set to 708 | :LOGGING, thus no more events can be recorded that will affect 709 | replay of the journal being recorded. The event that triggered this 710 | condition is recorded in state :LOGGING, with its version 711 | downgraded. Since @REPLAY (except @INVOKED) is built on the 712 | assumption that control flow is deterministic, an unexpected outcome 713 | is significant because it makes this assumption to hold unlikely. 714 | 715 | Also see REPLAY-UNEXPECTED-OUTCOME. 716 | 717 | - [condition] DATA-EVENT-LOSSAGE JOURNALING-FAILURE 718 | 719 | Signalled when a @DATA-EVENT is about to be recorded 720 | in JOURNAL-STATE :MISMATCHED or :LOGGING. Since the data event will 721 | not be replayed that constitutes data loss. 722 | 723 | - [condition] JOURNAL-ERROR ERROR 724 | 725 | Signalled by WITH-JOURNALING, WITH-BUNDLE and by 726 | @LOG-RECORD. It is also signalled by the low-level streamlet 727 | interface (see @STREAMLETS-REFERENCE). 728 | 729 | - [condition] END-OF-JOURNAL JOURNAL-ERROR 730 | 731 | This might be signalled by the replay mechanism if 732 | WITH-JOURNALING's REPLAY-EOJ-ERROR-P is true. Unlike 733 | REPLAY-FAILUREs, this does not affect JOURNAL-STATE of 734 | RECORD-JOURNAL. At a lower level, it is signalled by READ-EVENT upon 735 | reading past the end of the JOURNAL if EOJ-ERROR-P. 736 | 737 | ## Logging 738 | 739 | Before we get into the details, here is a self-contained example 740 | that demonstrates typical use. 741 | 742 | ``` 743 | (defvar *communication-log* nil) 744 | (defvar *logic-log* nil) 745 | (defvar *logic-log-level* 0) 746 | 747 | (defun call-with-connection (port fn) 748 | (framed (call-with-connection :log-record *communication-log* 749 | :args `(,port)) 750 | (funcall fn))) 751 | 752 | (defun fetch-data (key) 753 | (let ((value 42)) 754 | (logged ((and (<= 1 *logic-log-level*) *logic-log*)) 755 | "The value of ~S is ~S." key value) 756 | value)) 757 | 758 | (defun init-logging (&key (logic-log-level 1)) 759 | (let* ((stream (open "/tmp/xxx.log" 760 | :direction :output 761 | :if-does-not-exist :create 762 | :if-exists :append)) 763 | (journal (make-pprint-journal 764 | :stream (make-broadcast-stream 765 | (make-synonym-stream '*standard-output*) 766 | stream)))) 767 | (setq *communication-log* journal) 768 | (setq *logic-log* journal) 769 | (setq *logic-log-level* logic-log-level))) 770 | 771 | (init-logging) 772 | 773 | (call-with-connection 8080 (lambda () (fetch-data :foo))) 774 | .. 775 | .. (CALL-WITH-CONNECTION 8080) 776 | .. The value of :FOO is 42. 777 | .. => 42 778 | => 42 779 | 780 | (setq *logic-log-level* 0) 781 | (call-with-connection 8080 (lambda () (fetch-data :foo))) 782 | .. 783 | .. (CALL-WITH-CONNECTION 8080) 784 | .. => 42 785 | => 42 786 | 787 | (ignore-errors 788 | (call-with-connection 8080 (lambda () (error "Something unexpected.")))) 789 | .. 790 | .. (CALL-WITH-CONNECTION 8080) 791 | .. =E "SIMPLE-ERROR" "Something unexpected." 792 | ``` 793 | 794 | ##### Default to muffling 795 | 796 | Imagine a utility library called glib. 797 | 798 | ``` 799 | (defvar *glib-log* nil) 800 | (defvar *patience* 1) 801 | 802 | (defun sl33p (seconds) 803 | (logged (*glib-log*) "Sleeping for ~As." seconds) 804 | (sleep (* *patience* seconds))) 805 | ``` 806 | 807 | Glib follows the recommendation to have a special variable globally 808 | bound to NIL by default. The value of `*GLIB-LOG*` is the journal to 809 | which glib log messages will be routed. Since it's NIL, the log 810 | messages are muffled, and to record any log message, we need to 811 | change its value. 812 | 813 | ##### Routing logs to a journal 814 | 815 | Let's send the logs to a PPRINT-JOURNAL: 816 | 817 | ``` 818 | (setq *glib-log* (make-pprint-journal 819 | :log-decorator (make-log-decorator :time t))) 820 | (sl33p 0.01) 821 | .. 822 | .. 2020-08-31T12:45:23.827172+02:00: Sleeping for 0.01s. 823 | ``` 824 | 825 | That's a bit too wordy. For this tutorial, let's stick to less 826 | verbose output: 827 | 828 | ``` 829 | (setq *glib-log* (make-pprint-journal)) 830 | (sl33p 0.01) 831 | .. 832 | .. Sleeping for 0.01s. 833 | ``` 834 | 835 | To log to a file: 836 | 837 | ``` 838 | (setq *glib-log* (make-pprint-journal 839 | :stream (open "/tmp/glib.log" 840 | :direction :output 841 | :if-does-not-exist :create 842 | :if-exists :append))) 843 | ``` 844 | 845 | ##### Capturing logs in WITH-JOURNALING's RECORD-JOURNAL 846 | 847 | If we were recording a journal for replay and wanted to include glib 848 | logs in the journal, we would do something like this: 849 | 850 | ``` 851 | (with-journaling (:record t) 852 | (let ((*glib-log* :record)) 853 | (sl33p 0.01) 854 | (journaled (non-glib-stuff :version 1))) 855 | (list-events)) 856 | => ((:LEAF "Sleeping for 0.01s.") 857 | (:IN NON-GLIB-STUFF :VERSION 1) 858 | (:OUT NON-GLIB-STUFF :VERSION 1 :VALUES (NIL))) 859 | ``` 860 | 861 | We could even `(SETQ *GLIB-LOG* :RECORD)` to make it so that glib 862 | messages are included by default in the RECORD-JOURNAL. In this 863 | example, the special `*GLIB-LOG*` acts like a log category for all 864 | the log messages of the glib library (currently one). 865 | 866 | ##### Rerouting a category 867 | 868 | Next, we route `*GLIB-LOG*` to wherever `*APP-LOG*` is pointing by 869 | binding `*GLIB-LOG*` *to the symbol* `*APP-LOG*` (see @LOG-RECORD). 870 | 871 | ``` 872 | (defvar *app-log* nil) 873 | 874 | (let ((*glib-log* '*app-log*)) 875 | (setq *app-log* nil) 876 | (logged (*glib-log*) "This is not written anywhere.") 877 | (setq *app-log* (make-pprint-journal :pretty nil)) 878 | (sl33p 0.01)) 879 | .. 880 | .. (:LEAF "Sleeping for 0.01s.") 881 | ``` 882 | 883 | Note how pretty-printing was turned off, and we see the LEAF-EVENT 884 | generated by LOGGED in its raw plist form. 885 | 886 | ##### Conditional routing 887 | 888 | Finally, to make routing decisions conditional we need to change 889 | `SL33P`: 890 | 891 | ``` 892 | (defvar *glib-log-level* 1) 893 | 894 | (defun sl33p (seconds) 895 | (logged ((and (<= 2 *glib-log-level*) *glib-log*)) 896 | "Sleeping for ~As." (* *patience* seconds)) 897 | (sleep seconds)) 898 | 899 | ;;; Check that it all works: 900 | (let ((*glib-log-level* 1) 901 | (*glib-log* (make-pprint-journal))) 902 | (format t "~%With log-level ~A" *glib-log-level*) 903 | (sl33p 0.01) 904 | (setq *glib-log-level* 2) 905 | (format t "~%With log-level ~A" *glib-log-level*) 906 | (sl33p 0.01)) 907 | .. 908 | .. With log-level 1 909 | .. With log-level 2 910 | .. Sleeping for 0.01s. 911 | ``` 912 | 913 | ##### Nested log contexts 914 | 915 | LOGGED is for single messages. JOURNALED, or in this example FRAMED, 916 | can provide nested context: 917 | 918 | ``` 919 | (defun callv (var value symbol &rest args) 920 | "Call SYMBOL-FUNCTION of SYMBOL with VAR dynamically bound to VALUE." 921 | (framed ("glib:callv" :log-record *glib-log* 922 | :args `(,var ,value ,symbol ,@args)) 923 | (progv (list var) (list value) 924 | (apply (symbol-function symbol) args)))) 925 | 926 | (callv '*print-base* 2 'print 10) 927 | .. 928 | .. ("glib:callv" *PRINT-BASE* 2 PRINT 10) 929 | .. 1010 930 | .. => 10 931 | => 10 932 | 933 | (let ((*glib-log-level* 2)) 934 | (callv '*patience* 7 'sl33p 0.01)) 935 | .. 936 | .. ("glib:callv" *PATIENCE* 7 SL33P 0.01) 937 | .. Sleeping for 0.07s. 938 | .. => NIL 939 | ``` 940 | 941 | 942 | ### Customizing logs 943 | 944 | Customizing the output format is possible if we don't necessarily 945 | expect to be able to read the logs back programmatically. There is 946 | an example in @TRACING, which is built on @PPRINT-JOURNALS. 947 | 948 | Here, we discuss how to make logs more informative. 949 | 950 | - [glossary-term] decoration 951 | 952 | JOURNAL-LOG-DECORATOR adds additional data to LOG-EVENTs as they 953 | are written to the journal. This data is called decoration, and it is 954 | to capture the context in which the event was triggered. See 955 | MAKE-LOG-DECORATOR for a typical example. Decorations, since they 956 | can be on LOG-EVENTs only, do not affect @REPLAY. Decorations are 957 | most often used with @PRETTY-PRINTING. 958 | 959 | - [accessor] JOURNAL-LOG-DECORATOR [JOURNAL][5082] (:LOG-DECORATOR = NIL) 960 | 961 | If non-NIL, this is a function to add @DECORATION 962 | to LOG-EVENTs before they are written to a journal. The only 963 | allowed transformation is to *append* a plist to the event, which 964 | is a plist itself. The keys can be anything. 965 | 966 | - [function] MAKE-LOG-DECORATOR &KEY TIME REAL-TIME RUN-TIME THREAD DEPTH OUT-NAME 967 | 968 | Return a function suitable as JOURNAL-LOG-DECORATOR that may add 969 | a string timestamp, the internal real-time or run-time (both in 970 | seconds), the name of the thread, to events, which will be handled 971 | by PRETTIFY-EVENT. If DEPTH, then PRETTIFY-EVENT will the nesting 972 | level of the event being printed. If OUT-NAME, the PRETTIFY-EVENT 973 | will print the name of @OUT-EVENTS. 974 | 975 | All arguments are @BOOLEAN-VALUED-SYMBOLs. 976 | 977 | ``` 978 | (funcall (make-log-decorator :depth t :out-name t :thread t 979 | :time t :real-time t :run-time t) 980 | (make-leaf-event :foo)) 981 | => (:LEAF :FOO :DEPTH T :OUT-NAME T :THREAD "worker" 982 | :TIME "2023-05-26T12:27:44.172614+01:00" 983 | :REAL-TIME 2531.3254 :RUN-TIME 28.972797) 984 | ``` 985 | 986 | ### :LOG-RECORD 987 | 988 | WITH-JOURNALING and WITH-BUNDLE control replaying and recording 989 | within their dynamic extent, which is rather a necessity because 990 | @REPLAY needs to read the events in the same order as the JOURNALED 991 | @BLOCKs are being executed. However, LOG-EVENTs do not affect 992 | replay, so we can allow more flexibility in routing them. 993 | 994 | The LOG-RECORD argument of JOURNALED and LOGGED controls where 995 | LOG-EVENTs are written both within WITH-JOURNALING and without. The 996 | algorithm to determine the target journal is this: 997 | 998 | 1. If LOG-RECORD is :RECORD, then the RECORD-JOURNAL is returned. 999 | 1000 | 2. If LOG-RECORD is NIL, then it is returned. 1001 | 1002 | 3. If LOG-RECORD is a JOURNAL, then it is returned. 1003 | 1004 | 4. If LOG-RECORD is a symbol (other than NIL), then the SYMBOL-VALUE 1005 | of that symbol is assigned to LOG-RECORD, and we go to step 1. 1006 | 1007 | If the return value is NIL, then the event will not be written 1008 | anywhere, else it is written to the journal returned. 1009 | 1010 | This is reminiscent of SYNONYM-STREAMs, also in that it is possible 1011 | end up in cycles in the resolution. For this reason, the algorithm 1012 | stop with a JOURNAL-ERROR after 100 iterations. 1013 | 1014 | ##### Interactions 1015 | 1016 | Events may be written to LOG-RECORD even without an enclosing 1017 | WITH-JOURNALING, and it does not affect the JOURNAL-STATE. However, 1018 | it is a JOURNAL-ERROR to write to a :COMPLETED journal (see 1019 | JOURNAL-STATE). 1020 | 1021 | When multiple threads log to the same journal, it is guaranteed that 1022 | individual events are written atomically, but frames from different 1023 | threads do not necessarily nest. To keep the log informative, the 1024 | name of thread may be added to the events as @DECORATION. 1025 | 1026 | Also, see notes on thread @SAFETY. 1027 | 1028 | ### Logging with LEAF-EVENTs 1029 | 1030 | - [macro] LOGGED (&OPTIONAL (LOG-RECORD :RECORD)) FORMAT-CONTROL &REST FORMAT-ARGS 1031 | 1032 | LOGGED creates a single LEAF-EVENT, whose name is the string 1033 | constructed by FORMAT. For example: 1034 | 1035 | ``` 1036 | (with-journaling (:record t) 1037 | (logged () "Hello, ~A." "world") 1038 | (list-events)) 1039 | => ((:LEAF "Hello, world.")) 1040 | ``` 1041 | 1042 | LEAF-EVENTs are LOG-EVENTs with no separate in- and out-events. They 1043 | have an EVENT-NAME and no other properties. Use LOGGED for 1044 | point-in-time textual log messages, and JOURNALED with VERSION 1045 | NIL (i.e. FRAMED) to provide context. 1046 | 1047 | Also, see @LOG-RECORD. 1048 | 1049 | ## Tracing 1050 | 1051 | JTRACE behaves similarly to CL:TRACE but deals with 1052 | non-local exits gracefully. 1053 | 1054 | ##### Basic tracing 1055 | 1056 | ```common-lisp 1057 | (defun foo (x) 1058 | (sleep 0.12) 1059 | (1+ x)) 1060 | 1061 | (defun bar (x) 1062 | (foo (+ x 2)) 1063 | (error "xxx")) 1064 | 1065 | (jtrace foo bar) 1066 | 1067 | (ignore-errors (bar 1)) 1068 | .. 1069 | .. 0: (BAR 1) 1070 | .. 1: (FOO 3) 1071 | .. 1: FOO => 4 1072 | .. 0: BAR =E "SIMPLE-ERROR" "xxx" 1073 | ``` 1074 | 1075 | ##### Log-like output 1076 | 1077 | It can also include the name of the originating thread and 1078 | timestamps in the output: 1079 | 1080 | ``` 1081 | (let ((*trace-thread* t) 1082 | (*trace-time* t) 1083 | (*trace-depth* nil) 1084 | (*trace-out-name* nil)) 1085 | (ignore-errors (bar 1))) 1086 | .. 1087 | .. 2020-09-02T19:58:19.415204+02:00 worker: (BAR 1) 1088 | .. 2020-09-02T19:58:19.415547+02:00 worker: (FOO 3) 1089 | .. 2020-09-02T19:58:19.535766+02:00 worker: => 4 1090 | .. 2020-09-02T19:58:19.535908+02:00 worker: =E "SIMPLE-ERROR" "xxx" 1091 | ``` 1092 | 1093 | ##### Profiler-like output 1094 | 1095 | ``` 1096 | (let ((*trace-real-time* t) 1097 | (*trace-run-time* t) 1098 | (*trace-depth* nil) 1099 | (*trace-out-name* nil)) 1100 | (ignore-errors (bar 1))) 1101 | .. 1102 | .. #16735.736 !68.368: (BAR 1) 1103 | .. #16735.736 !68.369: (FOO 3) 1104 | .. #16735.857 !68.369: => 4 1105 | .. #16735.857 !68.369: =E "SIMPLE-ERROR" "xxx" 1106 | ``` 1107 | 1108 | ##### Customizing the content and the format 1109 | 1110 | If these options are insufficient, the content and the format of the 1111 | trace can be customized: 1112 | 1113 | ``` 1114 | (let ((*trace-journal* 1115 | (make-pprint-journal :pretty '*trace-pretty* 1116 | :prettifier (lambda (event depth stream) 1117 | (format stream "~%Depth: ~A, event: ~S" 1118 | depth event)) 1119 | :stream (make-synonym-stream '*error-output*) 1120 | :log-decorator (lambda (event) 1121 | (append event '(:custom 7)))))) 1122 | (ignore-errors (bar 1))) 1123 | .. 1124 | .. Depth: 0, event: (:IN BAR :ARGS (1) :CUSTOM 7) 1125 | .. Depth: 1, event: (:IN FOO :ARGS (3) :CUSTOM 7) 1126 | .. Depth: 1, event: (:OUT FOO :VALUES (4) :CUSTOM 7) 1127 | .. Depth: 0, event: (:OUT BAR :ERROR ("SIMPLE-ERROR" "xxx") :CUSTOM 7) 1128 | ``` 1129 | 1130 | In the above, *TRACE-JOURNAL* was bound locally to keep the example 1131 | from wrecking the global default, but the same effect could be 1132 | achieved by `SETF`ing PPRINT-JOURNAL-PRETTIFIER, 1133 | PPRINT-JOURNAL-STREAM and JOURNAL-LOG-DECORATOR. 1134 | 1135 | - [macro] JTRACE &REST NAMES 1136 | 1137 | Like CL:TRACE, JTRACE takes a list of symbols. When functions 1138 | denoted by those NAMES are invoked, their names, arguments and 1139 | outcomes are printed in human readable form to *TRACE-OUTPUT*. These 1140 | values may not be @READABLE, JTRACE does not care. 1141 | 1142 | The format of the output is the same as that of PPRINT-EVENTS. 1143 | Behind the scenes, JTRACE encapsulates the global functions with 1144 | NAMES in wrapper that behaves as if `FOO` in the example above was 1145 | defined like this: 1146 | 1147 | ``` 1148 | (defun foo (x) 1149 | (framed (foo :args `(,x) :log-record *trace-journal*) 1150 | (1+ x))) 1151 | ``` 1152 | 1153 | If JTRACE is invoked with no arguments, it returns the list of 1154 | symbols currently traced. 1155 | 1156 | On Lisps other than SBCL, where a function encapsulation facility is 1157 | not available or it is not used by Journal, JTRACE simply sets 1158 | SYMBOL-FUNCTION. This solution loses the tracing encapsulation when 1159 | the function is recompiled. On these platforms, `(JTRACE)` also 1160 | retraces all functions that should be traced but aren't. 1161 | 1162 | The main advantage of JTRACE over CL:TRACE is the ability to trace 1163 | errors, not just normal return values. As it is built on JOURNALED, 1164 | it can also detect – somewhat heuristically – THROWs and similar. 1165 | 1166 | - [macro] JUNTRACE &REST NAMES 1167 | 1168 | Like CL:UNTRACE, JUNTRACE makes it so that the global functions 1169 | denoted by the symbols NAMES are no longer traced by JTRACE. When 1170 | invoked with no arguments, it untraces all traced functions. 1171 | 1172 | - [variable] *TRACE-PRETTY* T 1173 | 1174 | If *TRACE-PRETTY* is true, then JTRACE produces output like 1175 | PPRINT-EVENTS, else it's like PRINT-EVENTS. 1176 | 1177 | - [variable] *TRACE-DEPTH* T 1178 | 1179 | Controls whether to decorate the trace with the depth of event. 1180 | See MAKE-LOG-DECORATOR. 1181 | 1182 | - [variable] *TRACE-OUT-NAME* T 1183 | 1184 | Controls whether trace should print the EVENT-NAME of @OUT-EVENTS, 1185 | which is redundant with the EVENT-NAME of the corresponding 1186 | @IN-EVENTS. See MAKE-LOG-DECORATOR. 1187 | 1188 | - [variable] *TRACE-THREAD* NIL 1189 | 1190 | Controls whether to decorate the trace with the name of the 1191 | originating thread. See MAKE-LOG-DECORATOR. 1192 | 1193 | - [variable] *TRACE-TIME* NIL 1194 | 1195 | Controls whether to decorate the trace with a timestamp. See 1196 | MAKE-LOG-DECORATOR. 1197 | 1198 | - [variable] *TRACE-REAL-TIME* NIL 1199 | 1200 | Controls whether to decorate the trace with the internal real-time. 1201 | See MAKE-LOG-DECORATOR. 1202 | 1203 | - [variable] *TRACE-RUN-TIME* NIL 1204 | 1205 | Controls whether to decorate the trace with the internal run-time. 1206 | See MAKE-LOG-DECORATOR. 1207 | 1208 | - [variable] *TRACE-JOURNAL* \#\ 1209 | 1210 | The JOURNAL where JTRACE writes LOG-EVENTs. By default, it is a 1211 | PPRINT-JOURNAL that sets up a SYNONYM-STREAM to *TRACE-OUTPUT* and 1212 | sends its output there. It pays attention to *TRACE-PRETTY*, and its 1213 | log decorator is affected by *TRACE-TIME* and *TRACE-THREAD*. 1214 | However, by changing JOURNAL-LOG-DECORATOR and 1215 | PPRINT-JOURNAL-PRETTIFIER, content and output can be customized. 1216 | 1217 | ### Slime integration 1218 | 1219 | [Slime](https://common-lisp.net/project/slime/), by default, 1220 | binds `C-c C-t` to toggling CL:TRACE. To integrate JTRACE into 1221 | Slime, load `src/mgl-jrn.el` into Emacs. 1222 | 1223 | - If you installed Journal with Quicklisp, the location of 1224 | `mgl-jrn.el` may change with updates, and you may want to copy the 1225 | current version to a stable location: 1226 | 1227 | (journal:install-journal-elisp "~/quicklisp/") 1228 | 1229 | Then, assuming the Elisp file is in the quicklisp directory, add 1230 | this to your `.emacs`: 1231 | 1232 | ```elisp 1233 | (load "~/quicklisp/mgl-jrn.el") 1234 | ``` 1235 | 1236 | Since JTRACE lacks some features of CL:TRACE, most notably that of 1237 | tracing non-global functions, it is assigned a separate binding, 1238 | `C-c C-j`. 1239 | 1240 | - [function] INSTALL-JOURNAL-ELISP TARGET-DIR 1241 | 1242 | Copy `mgl-jrn.el` distributed with this package to TARGET-DIR. 1243 | 1244 | ## Replay 1245 | 1246 | During replay, code is executed normally with special rules for 1247 | @BLOCKs. There are two modes for dealing with blocks: replaying the 1248 | code and replaying the outcome. When code is replayed, upon entering 1249 | and leaving a block, the events generated are matched to events read 1250 | from the journal being replayed. If the events don't match, 1251 | REPLAY-FAILURE is signalled, which marks the record journal as having 1252 | failed the replay. This is intended to make sure that the state of 1253 | the program during the replay matches the state at the time of 1254 | recording. In the other mode, when the outcome is replayed, a block 1255 | may not be executed at all, but its recorded outcome is 1256 | reproduced (i.e. the recorded return values are simply returned). 1257 | 1258 | Replay can be only be initiated with WITH-JOURNALING (or its close 1259 | kin WITH-BUNDLE). After the per-event processing described below, 1260 | when WITH-JOURNALING finishes, it might signal REPLAY-INCOMPLETE if 1261 | there are unprocessed non-log events left in the replay journal. 1262 | 1263 | Replay is deemed successful or failed depending on whether all 1264 | events are replayed from the replay journal without a 1265 | REPLAY-FAILURE. A journal that records events from a successful 1266 | replay can be used in place of the journal that was replayed, and so 1267 | on. The logic of replacing journals with their successful replays is 1268 | automated by @BUNDLES. WITH-JOURNALING does not allow replay from 1269 | journals that were failed replays themselves. The mechanism, in 1270 | terms of which tracking success and failure of replays is 1271 | implemented, revolves around JOURNAL-STATE and 1272 | EVENT-VERSIONs, which we discuss next. 1273 | 1274 | - [type] JOURNAL-STATE 1275 | 1276 | JOURNAL's state with respect to replay is updated during 1277 | WITH-JOURNALING. The possible states are: 1278 | 1279 | - **:NEW**: This journal was just created but never recorded to. 1280 | 1281 | - **:REPLAYING**: Replaying events has started, some events may have 1282 | been replayed successfully, but there are more non-log events to 1283 | replay. 1284 | 1285 | - **:MISMATCHED**: There was a REPLAY-FAILURE. In this state, 1286 | VERSIONED-EVENTs generated are downgraded to LOG-EVENTs, 1287 | EXTERNAL-EVENTs and @INVOKED trigger DATA-EVENT-LOSSAGE. 1288 | 1289 | - **:RECORDING**: All events from the replay journal were 1290 | successfully replayed, and now new events are being recorded 1291 | without being matched to the replay journal. 1292 | 1293 | - **:LOGGING**: There was a RECORD-UNEXPECTED-OUTCOME. In this 1294 | state, VERSIONED-EVENTs generated are downgraded to LOG-EVENTs, 1295 | EXTERNAL-EVENTs and @INVOKED trigger DATA-EVENT-LOSSAGE. 1296 | 1297 | - **:FAILED**: The journal is to be discarded. It encountered a 1298 | JOURNALING-FAILURE or a REPLAY-FAILURE without completing the 1299 | replay and reaching :RECORDING. 1300 | 1301 | - **:COMPLETED**: All events were successfully replayed and 1302 | WITH-JOURNALING finished or a JOURNALING-FAILURE occurred while 1303 | :RECORDING or :LOGGING. 1304 | 1305 | The state transitions are: 1306 | 1307 | :NEW -> :REPLAYING (on entering WITH-JOURNALING) 1308 | :REPLAYING -> :MISMATCHED (on REPLAY-FAILURE) 1309 | :REPLAYING -> :FAILED (on REPLAY-INCOMPLETE) 1310 | :REPLAYING -> :FAILED (on JOURNALING-FAILURE) 1311 | :REPLAYING -> :RECORDING (on successfully replaying all events) 1312 | :MISMATCHED -> :FAILED (on leaving WITH-JOURNALING) 1313 | :RECORDING -> :LOGGING (on RECORD-UNEXPECTED-OUTCOME) 1314 | :RECORDING/:LOGGING -> :COMPLETED (on leaving WITH-JOURNALING) 1315 | :RECORDING/:LOGGING -> :COMPLETED (on JOURNALING-FAILURE) 1316 | 1317 | :NEW is the starting state. It is a JOURNAL-ERROR to attempt to 1318 | write to journals in :COMPLETED. Note that once in :RECORDING, the 1319 | only possible terminal state is :COMPLETED. 1320 | 1321 | ### Journaled for replay 1322 | 1323 | The following arguments of JOURNALED control behaviour under replay. 1324 | 1325 | - VERSION: see EVENT-VERSION below. 1326 | 1327 | - INSERTABLE controls whether VERSIONED-EVENTs and EXTERNAL-EVENTs 1328 | may be replayed with the *insert* replay strategy (see 1329 | @THE-REPLAY-STRATEGY). Does not affect LOG-EVENTs, which are 1330 | always \_insert\_ed. Note that inserting EXTERNAL-EVENTs while 1331 | :REPLAYING is often not meaningful (e.g. asking the user for input 1332 | may lead to a REPLAY-FAILURE). See PEEK-REPLAY-EVENT for an 1333 | example on how to properly insert these kinds of EXTERNAL-EVENTs. 1334 | 1335 | - REPLAY-VALUES, a function or NIL, may be called with EVENT-OUTCOME 1336 | when replaying and :VERSION :INFINITY. NIL is equivalent to 1337 | VALUES-LIST. See `VALUES<-` for an example. 1338 | 1339 | - REPLAY-CONDITION, a function or NIL, may be called with 1340 | EVENT-OUTCOME (the return value of the function provided as 1341 | :CONDITION) when replaying and :VERSION is :INFINITY. NIL is 1342 | equivalent to the ERROR function. Replaying conditions is 1343 | cumbersome and best avoided. 1344 | 1345 | 1346 | - [variable] *FORCE-INSERTABLE* NIL 1347 | 1348 | The default value of the INSERTABLE argument of JOURNALED for 1349 | VERSIONED-EVENTs. Binding this to T allows en-masse structural 1350 | upgrades in combination with WITH-REPLAY-FILTER. Does not affect 1351 | EXTERNAL-EVENTs. See @UPGRADES-AND-REPLAY. 1352 | 1353 | - [type] EVENT-VERSION 1354 | 1355 | An event's version is either NIL, a positive FIXNUM, or :INFINITY, 1356 | which correspond to LOG-EVENTs, VERSIONED-EVENTs, and 1357 | EXTERNAL-EVENTs, respectively, and have an increasingly strict 1358 | behaviour with regards to @REPLAY. All EVENTs have versions. The 1359 | versions of the in- and out-events belonging to the same @FRAME are 1360 | the same. 1361 | 1362 | - [type] LOG-EVENT 1363 | 1364 | Events with EVENT-VERSION NIL called log events. During @REPLAY, 1365 | they are never matched to events from the replay journal, and log 1366 | events in the replay do not affect events being recorded either. 1367 | These properties allow log events to be recorded in arbitrary 1368 | journals with JOURNALED's LOG-RECORD argument. The convenience macro 1369 | FRAMED is creating frames of log-events, while the LOGGED generates 1370 | a log-event that's a LEAF-EVENT. 1371 | 1372 | - [type] VERSIONED-EVENT 1373 | 1374 | Events with a positive integer EVENT-VERSION are called 1375 | versioned events. In @REPLAY, they undergo consistency checks unlike 1376 | LOG-EVENTs, but the rules for them are less strict than for 1377 | EXTERNAL-EVENTs. In particular, higher versions are always 1378 | considered compatible with lower versions, they become an *upgrade* 1379 | in terms of the @THE-REPLAY-STRATEGY, and versioned events can be 1380 | inserted into the record without a corresponding @REPLAY-EVENT with 1381 | JOURNALED's INSERTABLE. 1382 | 1383 | If a VERSIONED-EVENT has an @UNEXPECTED-OUTCOME, 1384 | RECORD-UNEXPECTED-OUTCOME is signalled. 1385 | 1386 | - [type] EXTERNAL-EVENT 1387 | 1388 | Events with EVENT-VERSION :INFINITY are called external events. 1389 | They are like VERSIONED-EVENTs whose version was bumped all the way 1390 | to infinity, which rules out easy, non-matching upgrades. Also, they 1391 | are never inserted to the record without a matching replay 1392 | event (see @THE-REPLAY-STRATEGY). 1393 | 1394 | In return for these restrictions, external events can be replayed 1395 | without running the corresponding @BLOCK (see 1396 | @REPLAYING-THE-OUTCOME). This allows their out-event variety, called 1397 | @DATA-EVENTs, to be non-deterministic. Data events play a crucial 1398 | role in @PERSISTENCE. 1399 | 1400 | If an EXTERNAL-EVENT has an @UNEXPECTED-OUTCOME, 1401 | RECORD-UNEXPECTED-OUTCOME is signalled. 1402 | 1403 | Built on top of JOURNALED, the macros below record a pair of 1404 | @IN-EVENTS and @OUT-EVENTS but differ in how they are replayed and 1405 | the requirements on their @BLOCKs. The following table names the 1406 | type of EVENT produced (`Event`), how @IN-EVENTS are 1407 | replayed (`In-e.`), whether the block is always run (`Run`), how 1408 | @OUT-EVENTS are replayed (`Out-e.`), whether the block must be 1409 | deterministic (`Det`) or side-effect free (`SEF`). 1410 | 1411 | | | Event | In-e. | Run | Out-e. | Det | SEF | 1412 | |----------+-----------+--------+-----+--------+-----+-----| 1413 | | FRAMED | log | skip | y | skip | n | n | 1414 | | CHECKED | versioned | match | y | match | y | n | 1415 | | REPLAYED | external | match | n | replay | n | y | 1416 | | INVOKED | versioned | replay | y | match | y | n | 1417 | 1418 | Note that the replay-replay combination is not implemented because 1419 | there is nowhere to return values from replay-triggered functions. 1420 | 1421 | - [macro] FRAMED (NAME &KEY LOG-RECORD ARGS VALUES CONDITION) &BODY BODY 1422 | 1423 | A wrapper around JOURNALED to produce @FRAMEs of LOG-EVENTs. That 1424 | is, VERSION is always NIL, and some irrelevant arguments are 1425 | omitted. The related LOGGED creates a single LEAF-EVENT. 1426 | 1427 | With FRAMED, BODY is always run and no REPLAY-FAILUREs are 1428 | triggered. BODY is not required to be deterministic, and it may have 1429 | side-effects. 1430 | 1431 | - [macro] CHECKED (NAME &KEY (VERSION 1) ARGS VALUES CONDITION INSERTABLE) &BODY BODY 1432 | 1433 | A wrapper around JOURNALED to produce @FRAMEs of VERSIONED-EVENTs. 1434 | VERSION defaults to 1. CHECKED is for ensuring that supposedly 1435 | deterministic processing does not veer off the replay. 1436 | 1437 | With CHECKED, BODY – which must be deterministic – is always run and 1438 | REPLAY-FAILUREs are triggered when the events generated do not match 1439 | the events in the replay journal. BODY may have side-effects. 1440 | 1441 | For further discussion of determinism, see REPLAYED. 1442 | 1443 | - [macro] REPLAYED (NAME &KEY ARGS VALUES CONDITION INSERTABLE REPLAY-VALUES REPLAY-CONDITION) &BODY BODY 1444 | 1445 | A wrapper around JOURNALED to produce @FRAMEs of EXTERNAL-EVENTs. 1446 | VERSION is :INFINITY. REPLAYED is for primarily for marking and 1447 | isolating non-deterministic processing. 1448 | 1449 | With REPLAYED, the IN-EVENT is checked for consistency with the 1450 | replay (as with CHECKED), but BODY is not run (assuming it has a 1451 | recorded @EXPECTED-OUTCOME), and the outcome in the OUT-EVENT is 1452 | reproduced (see @REPLAYING-THE-OUTCOME). For this scheme to work, 1453 | REPLAYED requires its BODY to be side-effect free, but it may be 1454 | non-deterministic. 1455 | 1456 | - [glossary-term] invoked 1457 | 1458 | Invoked refers to functions and blocks defined by DEFINE-INVOKED or 1459 | FLET-INVOKED. Invoked frames may be recorded in response to 1460 | asynchronous events, and at replay the presence of its in-event 1461 | triggers the execution of the function associated with the name of 1462 | the event. 1463 | 1464 | On the one hand, FRAMED, CHECKED, REPLAYED or plain JOURNALED have 1465 | @IN-EVENTS that are always predictable from the code and the 1466 | preceding events. The control flow – on the level of recorded frames 1467 | – is deterministic in this sense. On the other hand, Invoked encodes 1468 | in its IN-EVENT what function to call next, introducing 1469 | non-deterministic control flow. 1470 | 1471 | By letting events choose the code to run, Invoked resembles typical 1472 | @EVENT-SOURCING frameworks. When Invoked is used exclusively, the 1473 | journal becomes a sequence of events. In contrast, JOURNALED and its 1474 | wrappers put code first, and the journal will be a projection of the 1475 | call tree. 1476 | 1477 | - [macro] DEFINE-INVOKED FUNCTION-NAME ARGS (NAME &KEY (VERSION 1) INSERTABLE) &BODY BODY 1478 | 1479 | DEFINE-INVOKED is intended for recording asynchronous function 1480 | invocations like event or signal handlers. It defines a function 1481 | that records VERSIONED-EVENTs with ARGS set to the actual arguments. 1482 | At replay, it is invoked whenever the recorded IN-EVENT becomes the 1483 | @REPLAY-EVENT. 1484 | 1485 | DEFUN and CHECKED rolled into one, DEFINE-INVOKED defines a 1486 | top-level function with FUNCTION-NAME and ARGS (only simple 1487 | positional arguments are allowed) and wraps CHECKED with NAME, the 1488 | same ARGS and INSERTABLE around BODY. Whenever an IN-EVENT becomes 1489 | the @REPLAY-EVENT, and it has a DEFINE-INVOKED defined with the name 1490 | of the event, FUNCTION-NAME is invoked with EVENT-ARGS. 1491 | 1492 | While BODY's return values are recorded as usual, the defined 1493 | function returns no values to make it less likely to affect control 1494 | flow in a way that's not possible to reproduce when the function is 1495 | called by the replay mechanism. 1496 | 1497 | ``` 1498 | (defvar *state*) 1499 | 1500 | (define-invoked foo (x) ("foo") 1501 | (setq *state* (1+ x))) 1502 | 1503 | (define-invoked bar (x) ("bar") 1504 | (setq *state* (+ 2 x))) 1505 | 1506 | (if (zerop (random 2)) 1507 | (foo 0) 1508 | (bar 1)) 1509 | ``` 1510 | 1511 | The above can be alternatively implemented with REPLAYED explicitly 1512 | encapsulating the non-determinism: 1513 | 1514 | ``` 1515 | (let ((x (replayed (choose) (random 2)))) 1516 | (if (zerop x) 1517 | (checked (foo :args `(,x)) 1518 | (setq *state* (1+ x))) 1519 | (checked (bar :args `(,x)) 1520 | (setq *state* (+ 2 x))))) 1521 | ``` 1522 | 1523 | - [macro] FLET-INVOKED DEFINITIONS &BODY BODY 1524 | 1525 | Like DEFINE-INVOKED, but with FLET instead of DEFUN. The event 1526 | name and the function are associated in the dynamic extent of BODY. 1527 | WITH-JOURNALING does not change the bindings. The example in 1528 | DEFINE-INVOKED can be rewritten as: 1529 | 1530 | ``` 1531 | (let ((state nil)) 1532 | (flet-invoked ((foo (x) ("foo") 1533 | (setq state (1+ x))) 1534 | (bar (x) ("bar") 1535 | (setq state (+ 2 x)))) 1536 | (if (zerop (random 2)) 1537 | (foo 0) 1538 | (bar 1)))) 1539 | ``` 1540 | 1541 | ### Bundles 1542 | 1543 | Consider replaying the same code repeatedly, hoping to make 1544 | progress in the processing. Maybe based on the availability of 1545 | external input, the code may error out. After each run, one has to 1546 | decide whether to keep the journal just recorded or stick with the 1547 | replay journal. A typical solution to this would look like this: 1548 | 1549 | ``` 1550 | (let ((record nil)) 1551 | (loop 1552 | (setq record (make-in-memory-journal)) 1553 | (with-journaling (:record record :replay replay) 1554 | ...) 1555 | (when (and 1556 | ;; RECORD is a valid replay of REPLAY ... 1557 | (eq (journal-state record) :completed) 1558 | ;; ... and is also significantly different from it ... 1559 | (journal-diverged-p record)) 1560 | ;; so use it for future replays. 1561 | (setq replay record)))) 1562 | ``` 1563 | 1564 | This is pretty much what bundles automate. The above becomes: 1565 | 1566 | ``` 1567 | (let ((bundle (make-in-memory-bundle))) 1568 | (loop 1569 | (with-bundle (bundle) 1570 | ...))) 1571 | ``` 1572 | 1573 | With FILE-JOURNALs, the motivating example above would be even more 1574 | complicated, but FILE-BUNDLEs work the same way as 1575 | IN-MEMORY-BUNDLEs. 1576 | 1577 | - [macro] WITH-BUNDLE (BUNDLE) &BODY BODY 1578 | 1579 | This is like WITH-JOURNALING where the REPLAY-JOURNAL is the last 1580 | successfully completed one in BUNDLE, and the RECORD-JOURNAL is a 1581 | new one created in BUNDLE. When WITH-BUNDLE finishes, the record 1582 | journal is in JOURNAL-STATE :FAILED or :COMPLETED. 1583 | 1584 | To avoid accumulating useless data, the new record is immediately 1585 | deleted when WITH-BUNDLE finishes if it has not diverged from the 1586 | replay journal (see JOURNAL-DIVERGENT-P). Because :FAILED journals 1587 | are always divergent in this sense, they are deleted instead based 1588 | on whether there is already a previous failed journal in the bundle 1589 | and the new record is identical to that journal (see 1590 | IDENTICAL-JOURNALS-P). 1591 | 1592 | It is a JOURNAL-ERROR to have concurrent or nested WITH-BUNDLEs on 1593 | the same bundle. 1594 | 1595 | ### The replay strategy 1596 | 1597 | The replay process for both @IN-EVENTS and @OUT-EVENTS starts by 1598 | determining how the generated event (the *new* event from now on) 1599 | shall be replayed. Roughly, the decision is based on the NAME and 1600 | VERSION of the new event and the @REPLAY-EVENT (the next event to be 1601 | read from the replay). There are four possible strategies: 1602 | 1603 | - **match**: A new in-event must match the replay event in its ARGS. 1604 | See @MATCHING-IN-EVENTS for details. A new out-event must match 1605 | the replay event's EXIT and OUTCOME, see @MATCHING-OUT-EVENTS. 1606 | 1607 | - **upgrade**: The new event is not matched to any replay event, but 1608 | an event is consumed from the replay journal. This happens if the 1609 | next new event has the same name as the replay event, but its 1610 | version is higher. 1611 | 1612 | - **insert**: The new event is not matched to any replay event, and 1613 | no events are consumed from the replay journal, which may be 1614 | empty. This is always the case for new LOG-EVENTs and when there 1615 | are no more events to read from the replay journal (unless 1616 | REPLAY-EOJ-ERROR-P). For VERSIONED-EVENTs, it is affected by 1617 | setting JOURNALED's INSERTABLE to true (see 1618 | @JOURNALED-FOR-REPLAY). 1619 | 1620 | The out-event's strategy is always *insert* if the strategy for 1621 | the corresponding in-event was *insert*. 1622 | 1623 | - Also, END-OF-JOURNAL, REPLAY-NAME-MISMATCH and 1624 | REPLAY-VERSION-DOWNGRADE may be signalled. See the algorithm below 1625 | details. 1626 | 1627 | The strategy is determined by the following algorithm, invoked 1628 | whenever an event is generated by a journaled @BLOCK: 1629 | 1630 | 1. Log events are not matched to the replay. If the new event is a 1631 | log event or a REPLAY-FAILURE has been signalled before (i.e. the 1632 | record journal's JOURNAL-STATE is :MISMATCHED), then **insert** 1633 | is returned. 1634 | 1635 | 2. Else, log events to be read in the replay journal are skipped, 1636 | and the next unread, non-log event is peeked at (without 1637 | advancing the replay journal). 1638 | 1639 | - **end of replay**: If there are no replay events left, then: 1640 | 1641 | - If REPLAY-EOJ-ERROR-P is NIL in WITH-JOURNALING (the 1642 | default), **insert** is returned. 1643 | 1644 | - If REPLAY-EOJ-ERROR-P is true, then **`END-OF-JOURNAL`** 1645 | is signalled. 1646 | 1647 | - **mismatched name**: Else, if the next unread replay event's 1648 | name is not EQUAL to the name of the new event, then: 1649 | 1650 | - For VERSIONED-EVENTs, **REPLAY-NAME-MISMATCH** is 1651 | signalled if INSERTABLE is NIL, else **insert** is 1652 | returned. 1653 | 1654 | - For EXTERNAL-EVENTs, **REPLAY-NAME-MISMATCH** is 1655 | signalled. 1656 | 1657 | - **matching name**: Else, if the name of the next unread event 1658 | in the replay journal is EQUAL to the name of new event, then 1659 | it is chosen as the *replay* event. 1660 | 1661 | - If the replay event's version is higher than the new 1662 | event's version, then **REPLAY-VERSION-DOWNGRADE** is 1663 | signalled. 1664 | 1665 | - If the two versions are equal, then **match** is returned. 1666 | 1667 | - If the new event's version is higher, then **upgrade** is 1668 | returned. 1669 | 1670 | Where :INFINITY is considered higher than any integer and 1671 | equal to itself. 1672 | 1673 | In summary: 1674 | 1675 | | new event | end-of-replay | mismatched name | matching name | 1676 | |-----------+-------------------+-------------------+---------------| 1677 | | Log | insert | insert | insert | 1678 | | Versioned | insert/eoj-error | insert/name-error | match-version | 1679 | | External | insert/eoj-error | insert/name-error | match-version | 1680 | 1681 | Version matching (`match-version` above) is based on which event has 1682 | a higher version: 1683 | 1684 | | replay event | = | new event | 1685 | |-----------------+-------+-----------| 1686 | | downgrade-error | match | upgrade | 1687 | 1688 | 1689 | - [glossary-term] replay event 1690 | 1691 | The replay event is the next event to be read from REPLAY-JOURNAL 1692 | which is not to be skipped. There may be no replay event if there 1693 | are no more unread events in the replay journal. 1694 | 1695 | An event in the replay journal is skipped if it is a LOG-EVENT or 1696 | there is a WITH-REPLAY-FILTER with a matching :SKIP. If :SKIP is in 1697 | effect, the replay event may be indeterminate. 1698 | 1699 | Events from the replay journal are read when they are `:MATCH`ed or 1700 | `:UPGRADE`d (see @THE-REPLAY-STRATEGY), when nested events are 1701 | echoed while @REPLAYING-THE-OUTCOME, or when there is an @INVOKED 1702 | defined with the same name as the replay event. 1703 | 1704 | The replay event is available via PEEK-REPLAY-EVENT. 1705 | 1706 | ### Matching in-events 1707 | 1708 | If the replay strategy is *match*, then, for in-events, the 1709 | matching process continues like this: 1710 | 1711 | - If the EVENT-ARGS are not EQUAL, then **`REPLAY-ARGS-MISMATCH`** 1712 | signalled. 1713 | 1714 | - At this point, two things might happen: 1715 | 1716 | - For VERSIONED-EVENTs, the @BLOCK will be executed as normal 1717 | and its outcome will be matched to the @REPLAY-EVENT (see 1718 | @MATCHING-OUT-EVENTS). 1719 | 1720 | - For EXTERNAL-EVENTs, the corresponding replay OUT-EVENT is 1721 | looked at. If there is one, meaning that the frame finished 1722 | with an @EXPECTED-OUTCOME, then its outcome will be 1723 | replayed (see @REPLAYING-THE-OUTCOME). If the OUT-EVENT is 1724 | missing, then EXTERNAL-EVENTs behave like VERSIONED-EVENTs, 1725 | and the @BLOCK is executed. 1726 | 1727 | 1728 | #### Replaying the outcome 1729 | 1730 | So, if an in-event is triggered that matches the replay, 1731 | EVENT-VERSION is :INFINITY, then normal execution is altered in the 1732 | following manner: 1733 | 1734 | - The journaled @BLOCK is not executed. 1735 | 1736 | - To keep execution and the replay journal in sync, events of frames 1737 | nested in the current one are skipped over in the replay journal. 1738 | 1739 | - All events (including LOG-EVENTs) skipped over are echoed to the 1740 | record journal. This serves to keep a trail of what happened 1741 | during the original recording. Note that functions corresponding 1742 | to @INVOKED frames are called when their IN-EVENT is skipped over. 1743 | 1744 | - The out-event corresponding to the in-event being processed is 1745 | then read from the replay journal and is recorded again (to allow 1746 | recording to function properly). 1747 | 1748 | To be able to reproduce the outcome in the replay journal, some 1749 | assistance may be required from REPLAY-VALUES and REPLAY-CONDITION: 1750 | 1751 | - If the @REPLAY-EVENT has a normal return (i.e. EVENT-EXIT :VALUES), 1752 | then the recorded return values (in EVENT-OUTCOME) are returned 1753 | immediately as in `(VALUES-LIST (EVENT-OUTCOME REPLAY-EVENT))`. If 1754 | REPLAY-VALUES is specified, it is called instead of VALUES-LIST. 1755 | See @WORKING-WITH-UNREADABLE-VALUES for an example. 1756 | 1757 | - Similarly, if the replay event has unwound with an expected 1758 | condition (has EVENT-EXIT :CONDITION), then the recorded 1759 | condition (in EVENT-OUTCOME) is signalled as 1760 | IN `(ERROR (EVENT-OUTCOME REPLAY-EVENT))`. If REPLAY-CONDITION is 1761 | specified, it is called instead of ERROR. REPLAY-CONDITION must 1762 | not return normally, and it's a JOURNAL-ERROR if it does. 1763 | 1764 | WITH-REPLAY-FILTER's NO-REPLAY-OUTCOME can selectively turn off 1765 | replaying the outcome. See @TESTING-ON-MULTIPLE-LEVELS, for an 1766 | example. 1767 | 1768 | ### Matching out-events 1769 | 1770 | If there were no @REPLAY-FAILURES during the matching of the 1771 | IN-EVENT, and the conditions for @REPLAYING-THE-OUTCOME were not 1772 | met, then the @BLOCK is executed. When the outcome of the block is 1773 | determined, an OUT-EVENT is triggered and is matched to the replay 1774 | journal. The matching of out-events starts out as in 1775 | @THE-REPLAY-STRATEGY with checks for EVENT-NAME and 1776 | EVENT-VERSION. 1777 | 1778 | If the replay strategy is *insert* or *upgrade*, then the out-event 1779 | is written to RECORD-JOURNAL, consuming an event with a matching 1780 | name from the REPLAY-JOURNAL in the latter case. If the strategy is 1781 | *match*, then: 1782 | 1783 | - If the new event has an @UNEXPECTED-OUTCOME, then 1784 | **REPLAY-UNEXPECTED-OUTCOME** is signalled. Note that the replay 1785 | event always has an @EXPECTED-OUTCOME due to the handling of 1786 | RECORD-UNEXPECTED-OUTCOME. 1787 | 1788 | - If the new event has an @EXPECTED-OUTCOME, then unless the new and 1789 | @REPLAY-EVENT's EVENT-EXITs are `EQ` and their EVENT-OUTCOMEs are 1790 | EQUAL, **REPLAY-OUTCOME-MISMATCH** is signalled. 1791 | 1792 | - Else, the replay event is consumed and the new event is written 1793 | the RECORD-JOURNAL. 1794 | 1795 | Note that @THE-REPLAY-STRATEGY for the in-event and the out-event of 1796 | the same @FRAME may differ if the corresponding out-event is not 1797 | present in REPLAY-JOURNAL, which may be the case when the recording 1798 | process failed hard without unwinding properly, or when an 1799 | @UNEXPECTED-OUTCOME triggered the transition to JOURNAL-STATE 1800 | :LOGGING. 1801 | 1802 | ### Replay failures 1803 | 1804 | - [condition] REPLAY-FAILURE SERIOUS-CONDITION 1805 | 1806 | A abstract superclass (never itself signalled) for 1807 | all kinds of mismatches between the events produced and the replay 1808 | journal. Signalled only in JOURNAL-STATE :REPLAYING and only once 1809 | per WITH-JOURNALING. If a REPLAY-FAILURE is signalled for an EVENT, 1810 | then the event will be recorded, but RECORD-JOURNAL will transition 1811 | to JOURNAL-STATE :MISMATCHED. Like JOURNALING-FAILURE, this is a 1812 | serious condition because it is to be handled outside the enclosing 1813 | WITH-JOURNALING. If a REPLAY-FAILURE were to be handled inside the 1814 | WITH-JOURNALING, keep in mind that in :MISMATCHED, replay always 1815 | uses the *insert* replay strategy (see @THE-REPLAY-STRATEGY). 1816 | 1817 | - [reader] REPLAY-FAILURE-NEW-EVENT [REPLAY-FAILURE][2e9b] (:NEW-EVENT) 1818 | 1819 | - [reader] REPLAY-FAILURE-REPLAY-EVENT [REPLAY-FAILURE][2e9b] (:REPLAY-EVENT) 1820 | 1821 | - [reader] REPLAY-FAILURE-REPLAY-JOURNAL [REPLAY-FAILURE][2e9b] (= '(REPLAY-JOURNAL)) 1822 | 1823 | - [condition] REPLAY-NAME-MISMATCH REPLAY-FAILURE 1824 | 1825 | Signalled when the new event's and @REPLAY-EVENT's 1826 | EVENT-NAME are not EQUAL. The REPLAY-FORCE-INSERT, 1827 | REPLAY-FORCE-UPGRADE restarts are provided. 1828 | 1829 | - [condition] REPLAY-VERSION-DOWNGRADE REPLAY-FAILURE 1830 | 1831 | Signalled when the new event and the @REPLAY-EVENT 1832 | have the same EVENT-NAME, but the new event has a lower version. The 1833 | REPLAY-FORCE-UPGRADE restart is provided. 1834 | 1835 | - [condition] REPLAY-ARGS-MISMATCH REPLAY-FAILURE 1836 | 1837 | Signalled when the new event's and @REPLAY-EVENT's 1838 | EVENT-ARGS are not EQUAL. The REPLAY-FORCE-UPGRADE restart is 1839 | provided. 1840 | 1841 | - [condition] REPLAY-OUTCOME-MISMATCH REPLAY-FAILURE 1842 | 1843 | Signalled when the new event's and @REPLAY-EVENT's 1844 | EVENT-EXIT and/or EVENT-OUTCOME are not EQUAL. The 1845 | REPLAY-FORCE-UPGRADE restart is provided. 1846 | 1847 | - [condition] REPLAY-UNEXPECTED-OUTCOME REPLAY-FAILURE 1848 | 1849 | Signalled when the new event has an 1850 | @UNEXPECTED-OUTCOME. Note that the @REPLAY-EVENT always has an 1851 | @EXPECTED-OUTCOME due to the logic of RECORD-UNEXPECTED-OUTCOME. No 1852 | restarts are provided. 1853 | 1854 | - [condition] REPLAY-INCOMPLETE REPLAY-FAILURE 1855 | 1856 | Signalled if there are unprocessed non-log events in 1857 | REPLAY-JOURNAL when WITH-JOURNALING finishes and the body of 1858 | WITH-JOURNALING returned normally, which is to prevent this 1859 | condition to cancel an ongoing unwinding. No restarts are provided. 1860 | 1861 | - [restart] REPLAY-FORCE-INSERT 1862 | 1863 | This restart forces @THE-REPLAY-STRATEGY to be :INSERT, overriding 1864 | REPLAY-NAME-MISMATCH. This is intended for upgrades, and extreme 1865 | care must be taken not to lose data. 1866 | 1867 | - [restart] REPLAY-FORCE-UPGRADE 1868 | 1869 | This restart forces @THE-REPLAY-STRATEGY to be :UPGRADE, overriding 1870 | REPLAY-NAME-MISMATCH, REPLAY-VERSION-DOWNGRADE, 1871 | REPLAY-ARGS-MISMATCH, REPLAY-OUTCOME-MISMATCH. This is intended for 1872 | upgrades, and extreme care must be taken not to lose data. 1873 | 1874 | ### Upgrades and replay 1875 | 1876 | The replay mechanism is built on the assumption that the tree of 1877 | @FRAMEs is the same when the code is replayed as it was when the 1878 | replay journal was originally recorded. Thus, non-deterministic 1879 | control flow poses a challenge, but non-determinism can be isolated 1880 | with EXTERNAL-EVENTs. However, when the code changes, we might find 1881 | the structure of frames in previous recordings hard to accommodate. 1882 | In this case, we might decide to alter the structure, giving up some 1883 | of the safety provided by the replay mechanism. There are various 1884 | tools at our disposal to control this tradeoff between safety and 1885 | flexibility: 1886 | 1887 | - We can insert individual frames with JOURNALED's INSERTABLE, 1888 | upgrade frames by bumping JOURNALED's VERSION, and filter frames 1889 | with WITH-REPLAY-FILTER. This option allows for the most 1890 | consistency checks. 1891 | 1892 | - The REPLAY-FORCE-UPGRADE and REPLAY-FORCE-INSERT restarts allow 1893 | overriding @THE-REPLAY-STRATEGY, but their use requires great care 1894 | to be taken. 1895 | 1896 | - Or we may decide to keep the bare minimum of the replay journal 1897 | around and discard everything except for EXTERNAL-EVENTs. This 1898 | option is equivalent to 1899 | 1900 | (let ((*force-insertable* t)) 1901 | (with-replay-filter (:skip '((:name nil))) 1902 | 42)) 1903 | 1904 | - Rerecording the journal without replay might be another option if 1905 | there are no EXTERNAL-EVENTs to worry about. 1906 | 1907 | - Finally, we can rewrite the replay journal using the low-level 1908 | interface (see @STREAMLETS-REFERENCE). In this case, extreme care 1909 | must be taken not to corrupt the journal (and lose data) as there 1910 | are no consistency checks to save us. 1911 | 1912 | With that, let's see how WITH-REPLAY-FILTER works. 1913 | 1914 | - [macro] WITH-REPLAY-STREAMLET (VAR) &BODY BODY 1915 | 1916 | Open REPLAY-JOURNAL for reading with WITH-OPEN-JOURNAL, set the 1917 | READ-POSITION on it to the event next read by the @REPLAY 1918 | mechanism (which is never a LOG-EVENT). The low-level 1919 | @READING-FROM-STREAMLETS api is then available to inspect the 1920 | contents of the replay. It is an error if REPLAY-JOURNAL is NIL. 1921 | 1922 | - [function] PEEK-REPLAY-EVENT 1923 | 1924 | Return the @REPLAY-EVENT to be read from REPLAY-JOURNAL. This is 1925 | roughly equivalent to 1926 | 1927 | ``` 1928 | (when (replay-journal) 1929 | (with-replay-streamlet (streamlet) 1930 | (peek-event streamlet)) 1931 | ``` 1932 | 1933 | except PEEK-REPLAY-EVENT takes into account WITH-REPLAY-FILTER 1934 | :MAP, and it may return `(:INDETERMINATE)` if WITH-REPLAY-FILTER 1935 | :SKIP is in effect and what events are to be skipped cannot be 1936 | decided until the next in-event generated by the code. 1937 | 1938 | Imagine a business process for paying an invoice. In the first 1939 | version of this process, we just pay the invoice: 1940 | 1941 | ``` 1942 | (replayed (pay)) 1943 | ``` 1944 | 1945 | We have left the implementation of PAY blank. In the second version, 1946 | we need to get an approval first: 1947 | 1948 | ``` 1949 | (when (replayed (get-approval) 1950 | (= (random 2) 0)) 1951 | (replayed (pay))) 1952 | ``` 1953 | 1954 | Replaying a journal produced by the first version of the code with 1955 | the second version would run into difficulties because inserting 1956 | EXTERNAL-EVENTs is tricky. 1957 | 1958 | We have to first decide how to handle the lack of approval in the 1959 | first version. Here, we just assume the processes started by the 1960 | first version get approval automatically. The implementation is 1961 | based on a dummy `PROCESS` block whose version is bumped when the 1962 | payment process changes and is inspected at the start of journaling. 1963 | 1964 | When v1 is replayed with v2, we introduce an INSERTABLE, versioned 1965 | `GET-APPROVAL` block that just returns T. When replaying the code 1966 | again, still with v2, the `GET-APPROVAL` block will be upgraded to 1967 | :INFINITY. 1968 | 1969 | ``` 1970 | (let ((bundle (make-in-memory-bundle))) 1971 | ;; First version of the payment process. Just pay. 1972 | (with-bundle (bundle) 1973 | (checked (process :version 1)) 1974 | (replayed (pay))) 1975 | ;; Second version of the payment process. Only pay if approved. 1976 | (loop repeat 2 do 1977 | (with-bundle (bundle) 1978 | (let ((replay-process-event (peek-replay-event))) 1979 | (checked (process :version 2)) 1980 | (when (if (and replay-process-event 1981 | (< (event-version replay-process-event) 2)) 1982 | ;; This will be upgraded to :INFINITY the second 1983 | ;; time around the LOOP. 1984 | (checked (get-approval :insertable t) 1985 | t) 1986 | (replayed (get-approval) 1987 | (= (random 2) 0))) 1988 | (replayed (pay))))))) 1989 | ``` 1990 | 1991 | - [macro] WITH-REPLAY-FILTER (&KEY MAP SKIP NO-REPLAY-OUTCOME) &BODY BODY 1992 | 1993 | WITH-REPLAY-FILTER performs journal upgrade during replay by 1994 | allowing events to be transformed as they are read from the replay 1995 | journal or skipped if they match some patterns. For how to add new 1996 | blocks in a code upgrade, see JOURNALED's :INSERTABLE argument. In 1997 | addition, it also allows some control over @REPLAYING-THE-OUTCOME. 1998 | 1999 | - MAP: A function called with an event read from the replay journal 2000 | which returns a transformed event. See @EVENTS-REFERENCE. MAP 2001 | takes effect before before SKIP. 2002 | 2003 | - SKIP: In addition to filtering out LOG-EVENTs (which always 2004 | happens during replay), filter out all events that belong to 2005 | frames that match any of its SKIP patterns. Filtered out events 2006 | are never seen by JOURNALED as it replays events. SKIP patterns 2007 | are of the format `(&KEY NAME VERSION<)`, where VERSION\< is a 2008 | valid EVENT-VERSION, and NAME may be NIL, which acts as a 2009 | wildcard. 2010 | 2011 | SKIP is for when JOURNALED @BLOCKs are removed from the code, 2012 | which would render replaying previously recorded journals 2013 | impossible. Note that, for reasons of safety, it is not possible 2014 | to filter EXTERNAL-EVENTs. 2015 | 2016 | - NO-REPLAY-OUTCOME is a list of EVENT-NAMEs. @REPLAYING-THE-OUTCOME 2017 | is prevented for frames with EQUAL names. See 2018 | @TESTING-ON-MULTIPLE-LEVELS for an example. 2019 | 2020 | WITH-REPLAY-FILTER affects only the immediately enclosing 2021 | WITH-JOURNALING. A WITH-REPLAY-FILTER nested within another in the 2022 | same WITH-JOURNALING inherits the SKIP patterns of its parent, to 2023 | which it adds its own. The MAP function is applied to before the 2024 | parent's MAP. 2025 | 2026 | Examples of SKIP patterns: 2027 | 2028 | ``` 2029 | ;; Match events with name FOO and version 1, 2, 3 or 4 2030 | (:name foo :version< 5) 2031 | ;; Match events with name BAR and any version 2032 | (:name bar :version< :infinity) 2033 | ;; Same as the previous 2034 | (:name bar) 2035 | ;; Match all names 2036 | (:name nil) 2037 | ;; Same as the previous 2038 | () 2039 | ``` 2040 | 2041 | Skipping can be thought of as removing nodes of the tree of frames, 2042 | connecting its children to its parent. The following example removes 2043 | frames `J1` and `J2` from around `J3`, the `J1` frame from within 2044 | `J3`, and the third `J1` frame. 2045 | 2046 | ``` 2047 | (let ((journal (make-in-memory-journal))) 2048 | ;; Record trees J1 -> J2 -> J3 -> J1, and J1. 2049 | (with-journaling (:record journal) 2050 | (checked (j1) 2051 | (checked (j2) 2052 | (checked (j3) 2053 | (checked (j1) 2054 | 42)))) 2055 | (checked (j1) 2056 | 7)) 2057 | ;; Filter out all occurrences of VERSIONED-EVENTs named J1 and 2058 | ;; J2 from the replay, leaving only J3 to match. 2059 | (with-journaling (:replay journal :record t :replay-eoj-error-p t) 2060 | (with-replay-filter (:skip '((:name j1) (:name j2))) 2061 | (checked (j3) 2062 | 42)))) 2063 | ``` 2064 | 2065 | ## Testing 2066 | 2067 | Having discussed the @REPLAY mechanism, next are @TESTING and 2068 | @PERSISTENCE, which rely heavily on replay. Suppose we want to unit 2069 | test user registration. Unfortunately, the code communicates with a 2070 | database service and also takes input from the user. A natural 2071 | solution is to create @MOCK-OBJECTs for these external systems to 2072 | unshackle the test from the cumbersome database dependency and to 2073 | allow it to run without user interaction. 2074 | 2075 | We do this below by wrapping external interaction in JOURNALED with 2076 | :VERSION :INFINITY (see @REPLAYING-THE-OUTCOME). 2077 | 2078 | ``` 2079 | (defparameter *db* (make-hash-table)) 2080 | 2081 | (defun set-key (key value) 2082 | (replayed ("set-key" :args `(,key ,value)) 2083 | (format t "Updating db~%") 2084 | (setf (gethash key *db*) value) 2085 | nil)) 2086 | 2087 | (defun get-key (key) 2088 | (replayed ("get-key" :args `(,key)) 2089 | (format t "Query db~%") 2090 | (gethash key *db*))) 2091 | 2092 | (defun ask-username () 2093 | (replayed ("ask-username") 2094 | (format t "Please type your username: ") 2095 | (read-line))) 2096 | 2097 | (defun maybe-win-the-grand-prize () 2098 | (checked ("maybe-win-the-grand-prize") 2099 | (when (= 1000000 (hash-table-count *db*)) 2100 | (format t "You are the lucky one!")))) 2101 | 2102 | (defun register-user (username) 2103 | (unless (get-key username) 2104 | (set-key username `(:user-object :username ,username)) 2105 | (maybe-win-the-grand-prize))) 2106 | ``` 2107 | 2108 | Now, we write a test that records these interactions in a file when 2109 | it's run for the first time. 2110 | 2111 | ``` 2112 | (define-file-bundle-test (test-user-registration 2113 | :directory (asdf:system-relative-pathname 2114 | :journal "test/registration/")) 2115 | (let ((username (ask-username))) 2116 | (register-user username) 2117 | (assert (get-key username)) 2118 | (register-user username) 2119 | (assert (get-key username)))) 2120 | 2121 | ;; Original recording: everything is executed 2122 | JRN> (test-user-registration) 2123 | Please type your username: joe 2124 | Query db 2125 | Updating db 2126 | Query db 2127 | Query db 2128 | Query db 2129 | => NIL 2130 | ``` 2131 | 2132 | On reruns, none of the external stuff is executed. The return values 2133 | of the external JOURNALED blocks are replayed from the journal: 2134 | 2135 | ``` 2136 | ;; Replay: all external interactions are mocked. 2137 | JRN> (test-user-registration) 2138 | => NIL 2139 | ``` 2140 | 2141 | Should the code change, we might want to upgrade carefully (see 2142 | @UPGRADES-AND-REPLAY) or just rerecord from scratch: 2143 | 2144 | ``` 2145 | JRN> (test-user-registration :rerecord t) 2146 | Please type your username: joe 2147 | Query db 2148 | Updating db 2149 | Query db 2150 | Query db 2151 | Query db 2152 | => NIL 2153 | ``` 2154 | 2155 | Thus satisfied that our test runs, we can commit the journal file in 2156 | the bundle into version control. Its contents are: 2157 | 2158 | ``` 2159 | 2160 | (:IN "ask-username" :VERSION :INFINITY) 2161 | (:OUT "ask-username" :VERSION :INFINITY :VALUES ("joe" NIL)) 2162 | (:IN "get-key" :VERSION :INFINITY :ARGS ("joe")) 2163 | (:OUT "get-key" :VERSION :INFINITY :VALUES (NIL NIL)) 2164 | (:IN "set-key" :VERSION :INFINITY :ARGS ("joe" (:USER-OBJECT :USERNAME "joe"))) 2165 | (:OUT "set-key" :VERSION :INFINITY :VALUES (NIL)) 2166 | (:IN "maybe-win-the-grand-prize" :VERSION 1) 2167 | (:OUT "maybe-win-the-grand-prize" :VERSION 1 :VALUES (NIL)) 2168 | (:IN "get-key" :VERSION :INFINITY :ARGS ("joe")) 2169 | (:OUT "get-key" :VERSION :INFINITY :VALUES ((:USER-OBJECT :USERNAME "joe") T)) 2170 | (:IN "get-key" :VERSION :INFINITY :ARGS ("joe")) 2171 | (:OUT "get-key" :VERSION :INFINITY :VALUES ((:USER-OBJECT :USERNAME "joe") T)) 2172 | (:IN "get-key" :VERSION :INFINITY :ARGS ("joe")) 2173 | (:OUT "get-key" :VERSION :INFINITY :VALUES ((:USER-OBJECT :USERNAME "joe") T)) 2174 | ``` 2175 | 2176 | Note that when this journal is replayed, new VERSIONED-EVENTs are 2177 | required to match the replay. So, after the original recording, we 2178 | can check by eyeballing that the record represents a correct 2179 | execution. Then on subsequent replays, even though 2180 | `MAYBE-WIN-THE-GRAND-PRIZE` sits behind `REGISTER-USER` and is hard 2181 | to test with ASSERTs, the replay mechanism verifies that it is 2182 | called only for new users. 2183 | 2184 | This record-and-replay style of testing is not the only possibility: 2185 | direct inspection of a journal with the low-level events api (see 2186 | @EVENTS-REFERENCE) can facilitate checking non-local invariants. 2187 | 2188 | - [macro] DEFINE-FILE-BUNDLE-TEST (NAME &KEY DIRECTORY (EQUIVALENTP T)) &BODY BODY 2189 | 2190 | Define a function with NAME for record-and-replay testing. The 2191 | function's BODY is executed in a WITH-BUNDLE to guarantee 2192 | replayability. The bundle in question is a FILE-BUNDLE created in 2193 | DIRECTORY. The function has a single keyword argument, RERECORD. If 2194 | RERECORD is true, the bundle is deleted with DELETE-FILE-BUNDLE to 2195 | start afresh. 2196 | 2197 | Furthermore, if BODY returns normally, and it is a replay of a 2198 | previous run, and EQUIVALENTP, then it is ASSERTed that the record 2199 | and replay journals are EQUIVALENT-REPLAY-JOURNALS-P. If this check 2200 | fails, RECORD-JOURNAL is discarded when the function returns. In 2201 | addition to the replay consistency, this checks that no inserts or 2202 | upgrades were performed (see @THE-REPLAY-STRATEGY). 2203 | 2204 | ### Testing on multiple levels 2205 | 2206 | Nesting REPLAYEDs (that is, @FRAMEs of EXTERNAL-EVENTs) is not 2207 | obviously useful since the outer REPLAYED will be replayed by 2208 | outcome, and the inner one will be just echoed to the record 2209 | journal. However, if we turn off @REPLAYING-THE-OUTCOME for the 2210 | outer, the inner will be replayed. 2211 | 2212 | This is useful for testing layered communication. For example, we 2213 | might have written code that takes input from an external 2214 | system (READ-LINE) and does some complicated 2215 | processing (READ-FROM-STRING) before returning the input in a form 2216 | suitable for further processing. Suppose we wrap REPLAYED around 2217 | READ-FROM-STRING for @PERSISTENCE because putting it around 2218 | READ-LINE would expose low-level protocol details in the journal, 2219 | making protocol changes difficult. 2220 | 2221 | However, upon realizing that READ-FROM-STRING was not the best tool 2222 | for the job and switching to PARSE-INTEGER, we want to test by 2223 | replaying all previously recorded journals. For this, we prevent the 2224 | outer REPLAYED from being replayed by outcome with 2225 | WITH-REPLAY-FILTER: 2226 | 2227 | ``` 2228 | (let ((bundle (make-in-memory-bundle))) 2229 | ;; Original with READ-FROM-STRING 2230 | (with-bundle (bundle) 2231 | (replayed ("accept-number") 2232 | (values (read-from-string (replayed ("input-number") 2233 | (read-line)))))) 2234 | ;; Switch to PARSE-INTEGER and test by replay. 2235 | (with-bundle (bundle) 2236 | (with-replay-filter (:no-replay-outcome '("accept-number")) 2237 | (replayed ("accept-number") 2238 | ;; 1+ is our bug. 2239 | (values (1+ (parse-integer (replayed ("input-number") 2240 | (read-line))))))))) 2241 | ``` 2242 | 2243 | The inner `input-number` block is replayed by outcome, and 2244 | PARSE-INTEGER is called with the string READ-LINE returned in the 2245 | original invocation. The outcome of the outer `accept-number` block 2246 | checked as if it was a VERSIONED-EVENT and we get a 2247 | REPLAY-OUTCOME-MISMATCH due to the bug. 2248 | 2249 | ## Persistence 2250 | 2251 | ### Persistence tutorial 2252 | 2253 | Let's write a simple game. 2254 | 2255 | ``` 2256 | (defun play-guess-my-number () 2257 | (let ((my-number (replayed (think-of-a-number) 2258 | (random 10)))) 2259 | (format t "~%I thought of a number.~%") 2260 | (loop for i upfrom 0 do 2261 | (write-line "Guess my number:") 2262 | (let ((guess (replayed (read-guess) 2263 | (values (parse-integer (read-line)))))) 2264 | (format t "You guessed ~D.~%" guess) 2265 | (when (= guess my-number) 2266 | (checked (game-won :args `(,(1+ i)))) 2267 | (format t "You guessed it in ~D tries!" (1+ i)) 2268 | (return)))))) 2269 | 2270 | (defparameter *the-evergreen-game* (make-in-memory-bundle)) 2271 | ``` 2272 | 2273 | ##### Original recording 2274 | 2275 | Unfortunately, the implementation is lacking in the input validation 2276 | department. In the transcript below, PARSE-INTEGER fails with `junk 2277 | in string` when the user enters `not a number`: 2278 | 2279 | ``` 2280 | CL-USER> (handler-case 2281 | (with-bundle (*the-evergreen-game*) 2282 | (play-guess-my-number)) 2283 | (error (e) 2284 | (format t "Oops. ~A~%" e))) 2285 | I thought of a number. 2286 | Guess my number: 2287 | 7 ; real user input 2288 | You guessed 7. 2289 | Guess my number: 2290 | not a number ; real user input 2291 | Oops. junk in string "not a number" 2292 | ``` 2293 | 2294 | ##### Replay and extension 2295 | 2296 | Instead of fixing this bug, we just restart the game from the 2297 | beginning, @REPLAYING-THE-OUTCOME of external interactions marked 2298 | with REPLAYED: 2299 | 2300 | ``` 2301 | CL-USER> (with-bundle (*the-evergreen-game*) 2302 | (play-guess-my-number)) 2303 | I thought of a number. 2304 | Guess my number: 2305 | You guessed 7. 2306 | Guess my number: ; New recording starts here 2307 | 5 ; real user input 2308 | You guessed 5. 2309 | Guess my number: 2310 | 4 ; real user input 2311 | You guessed 4. 2312 | Guess my number: 2313 | 2 ; real user input 2314 | You guessed 2. 2315 | You guessed it in 4 tries! 2316 | ``` 2317 | 2318 | ##### It's evergreen 2319 | 2320 | We can now replay this game many times without any user interaction: 2321 | 2322 | ``` 2323 | CL-USER> (with-bundle (*the-evergreen-game*) 2324 | (play-guess-my-number)) 2325 | I thought of a number. 2326 | Guess my number: 2327 | You guessed 7. 2328 | Guess my number: 2329 | You guessed 5. 2330 | Guess my number: 2331 | You guessed 4. 2332 | Guess my number: 2333 | You guessed 2. 2334 | You guessed it in 4 tries! 2335 | ``` 2336 | 2337 | ##### The generated events 2338 | 2339 | This simple mechanism allows us to isolate external interactions and 2340 | write tests in record-and-replay style based on the events produced: 2341 | 2342 | ``` 2343 | CL-USER> (list-events *the-evergreen-game*) 2344 | ((:IN THINK-OF-A-NUMBER :VERSION :INFINITY) 2345 | (:OUT THINK-OF-A-NUMBER :VERSION :INFINITY :VALUES (2)) 2346 | (:IN READ-GUESS :VERSION :INFINITY) 2347 | (:OUT READ-GUESS :VERSION :INFINITY :VALUES (7)) 2348 | (:IN READ-GUESS :VERSION :INFINITY :ARGS NIL) 2349 | (:OUT READ-GUESS :VERSION :INFINITY :VALUES (5)) 2350 | (:IN READ-GUESS :VERSION :INFINITY :ARGS NIL) 2351 | (:OUT READ-GUESS :VERSION :INFINITY :VALUES (4)) 2352 | (:IN READ-GUESS :VERSION :INFINITY :ARGS NIL) 2353 | (:OUT READ-GUESS :VERSION :INFINITY :VALUES (2)) 2354 | (:IN GAME-WON :VERSION 1 :ARGS (4)) 2355 | (:OUT GAME-WON :VERSION 1 :VALUES (NIL))) 2356 | ``` 2357 | 2358 | In fact, being able to replay this game at all already checks it 2359 | through the `GAME-WON` event that the number of tries calculation is 2360 | correct. 2361 | 2362 | In addition, thus being able to reconstruct the internal state of 2363 | the program gives us persistence by replay. If instead of a 2364 | IN-MEMORY-BUNDLE, we used a FILE-BUNDLE, the game would have been 2365 | saved on disk without having to write any code for saving and 2366 | loading the game state. 2367 | 2368 | ##### Discussion 2369 | 2370 | Persistence by replay, also known as @EVENT-SOURCING, is appropriate 2371 | when the external interactions are well-defined and stable. Storing 2372 | events shines in comparison to persisting state when the control 2373 | flow is too complicated to be interrupted and resumed easily. 2374 | Resuming execution in deeply nested function calls is fraught with 2375 | such peril that it is often easier to flatten the program into a 2376 | state machine, which is as pleasant as manually managing 2377 | @CONTINUATIONs. 2378 | 2379 | In contrast, the Journal library does not favour certain styles of 2380 | control flow and only requires that non-determinism is packaged up 2381 | in REPLAYED, which allows it to reconstruct the state of the program 2382 | from the recorded events at any point during its execution and 2383 | resume from there. 2384 | 2385 | ### Synchronization to storage 2386 | 2387 | In the following, we explore how journals can serve as a 2388 | persistence mechanism and the guarantees they offer. The high-level 2389 | summary is that journals with SYNC can serve as a durable and 2390 | consistent storage medium. The other two 2391 | [ACID](https://en.wikipedia.org/wiki/ACID) properties, atomicity and 2392 | isolation, do not apply because Journal is single-client and does 2393 | not need transactions. 2394 | 2395 | - [glossary-term] aborted execution 2396 | 2397 | Aborted execution is when the operating system or the application 2398 | crashes, calls `abort()`, is killed by a `SIGKILL` signal or there 2399 | is a power outage. Synchronization guarantees are defined in the 2400 | face of aborted execution and do not apply to hardware errors, Lisp 2401 | or OS bugs. 2402 | 2403 | - [glossary-term] data event 2404 | 2405 | Data events are the only events that may be non-deterministic. They 2406 | record information that could change if the same code were run 2407 | multiple times. Data events typically correspond to interactions 2408 | with the user, servers or even the random number generator. Due to 2409 | their non-determinism, they are the only parts of the journal not 2410 | reproducible by rerunning the code. In this sense, only the data 2411 | events are not redundant with the code, and whether other events are 2412 | persisted does not affect durability. There are two kinds of data 2413 | events: 2414 | 2415 | - An EXTERNAL-EVENT that is also an OUT-EVENT. 2416 | 2417 | - The IN-EVENT of an @INVOKED function, which lies outside the 2418 | normal, deterministic control flow. 2419 | 2420 | #### Synchronization strategies 2421 | 2422 | When a journal or bundle is created (see MAKE-IN-MEMORY-JOURNAL, 2423 | MAKE-FILE-JOURNAL, MAKE-IN-MEMORY-BUNDLE, MAKE-FILE-BUNDLE), the 2424 | SYNC option determines when – as a RECORD-JOURNAL – the recorded 2425 | events and JOURNAL-STATE changes are persisted durably. For 2426 | FILE-JOURNALs, persisting means calling something like `fsync`, 2427 | while for IN-MEMORY-JOURNALs, a user defined function is called to 2428 | persist the data. 2429 | 2430 | - NIL: Never synchronize. A FILE-JOURNAL's file may be corrupted on 2431 | @ABORTED-EXECUTION. In IN-MEMORY-JOURNALs, SYNC-FN is never 2432 | called. 2433 | 2434 | - T: This is the *no data loss* setting with minimal 2435 | synchronization. It guarantees *consistency* (i.e. no corruption) 2436 | and *durability* up to the most recent @DATA-EVENT written in 2437 | JOURNAL-STATE :RECORDING or for the entire record journal in 2438 | states :FAILED and :COMPLETED. :FAILED or :COMPLETED is guaranteed 2439 | when leaving WITH-JOURNALING at the latest. 2440 | 2441 | - Values other than NIL and T are reserved for future extensions. 2442 | Using them triggers a JOURNAL-ERROR. 2443 | 2444 | 2445 | #### Synchronization with in-memory journals 2446 | 2447 | Unlike FILE-JOURNALs, IN-MEMORY-JOURNALs do not have any built-in 2448 | persistent storage backing them, but with SYNC-FN, persistence can 2449 | be tacked on. If non-NIL, SYNC-FN must be a function of a single 2450 | argument, an IN-MEMORY-JOURNAL. SYNC-FN is called according to 2451 | @SYNCHRONIZATION-STRATEGIES, and upon normal return the journal must 2452 | be stored durably. 2453 | 2454 | The following example saves the entire journal history when a new 2455 | @DATA-EVENT is recorded. Note how `SYNC-TO-DB` is careful to 2456 | overwrite `*DB*` only if it is called with a journal that has not 2457 | failed the replay (as in @REPLAY-FAILURES) and is sufficiently 2458 | different from the replay journal as determined by 2459 | JOURNAL-DIVERGENT-P. 2460 | 2461 | ``` 2462 | (defparameter *db* ()) 2463 | 2464 | (defun sync-to-db (journal) 2465 | (when (and (member (journal-state journal) 2466 | '(:recording :logging :completed)) 2467 | (journal-divergent-p journal)) 2468 | (setq *db* (journal-events journal)) 2469 | (format t "Saved ~S~%New events from position ~S~%" *db* 2470 | (journal-previous-sync-position journal)))) 2471 | 2472 | (defun make-db-backed-record-journal () 2473 | (make-in-memory-journal :sync-fn 'sync-to-db)) 2474 | 2475 | (defun make-db-backed-replay-journal () 2476 | (make-in-memory-journal :events *db*)) 2477 | 2478 | (with-journaling (:record (make-db-backed-record-journal) 2479 | :replay (make-db-backed-replay-journal)) 2480 | (replayed (a) 2481 | 2) 2482 | (ignore-errors 2483 | (replayed (b) 2484 | (error "Whoops")))) 2485 | .. Saved #((:IN A :VERSION :INFINITY) 2486 | .. (:OUT A :VERSION :INFINITY :VALUES (2))) 2487 | .. New events from position 0 2488 | .. Saved #((:IN A :VERSION :INFINITY) 2489 | .. (:OUT A :VERSION :INFINITY :VALUES (2)) 2490 | .. (:IN B :VERSION :INFINITY) 2491 | .. (:OUT B :ERROR ("SIMPLE-ERROR" "Whoops"))) 2492 | .. New events from position 2 2493 | .. 2494 | ``` 2495 | 2496 | In a real application, external events often involve unreliable or 2497 | high-latency communication. In the above example, block `B` signals 2498 | an error, say, to simulate some kind of network condition. Now, a 2499 | new journal *for replay* is created and initialized with the saved 2500 | events, and the whole process is restarted. 2501 | 2502 | ``` 2503 | (defun run-with-db () 2504 | (with-journaling (:record (make-db-backed-record-journal) 2505 | :replay (make-db-backed-replay-journal)) 2506 | (replayed (a) 2507 | (format t "A~%") 2508 | 2) 2509 | (replayed (b) 2510 | (format t "B~%") 2511 | 3))) 2512 | 2513 | (run-with-db) 2514 | .. B 2515 | .. Saved #((:IN A :VERSION :INFINITY) 2516 | .. (:OUT A :VERSION :INFINITY :VALUES (2)) 2517 | .. (:IN B :VERSION :INFINITY) 2518 | .. (:OUT B :VERSION :INFINITY :VALUES (3))) 2519 | .. New events from position 0 2520 | .. 2521 | => 3 2522 | ``` 2523 | 2524 | Note that on the rerun, block `A` is not executed because external 2525 | events are replayed simply by reproducing their outcome, in this 2526 | case returning 2. See @REPLAYING-THE-OUTCOME. Block `B`, on the 2527 | other hand, was rerun because it had an @UNEXPECTED-OUTCOME the 2528 | first time around. This time it ran without error, a @DATA-EVENT was 2529 | triggered, and SYNC-FN was invoked. 2530 | 2531 | If we were to invoke the now completed `RUN-WITH-DB` again, it would 2532 | simply return 3 without ever invoking SYNC-FN: 2533 | 2534 | ``` 2535 | (run-with-db) 2536 | => 3 2537 | ``` 2538 | 2539 | With JOURNAL-REPLAY-MISMATCH, SYNC-FN can be optimized to to reuse 2540 | the sequence of events in the replay journal up until the point of 2541 | divergence. 2542 | 2543 | #### Synchronization with file journals 2544 | 2545 | For FILE-JOURNALs, SYNC determines when the events written to the 2546 | RECORD-JOURNAL and its JOURNAL-STATE will be persisted durably in 2547 | the file. Syncing to the file involves two calls to `fsync` and is 2548 | not cheap. 2549 | 2550 | Syncing events to files is implemented as follows. 2551 | 2552 | - When the journal file is created, its parent directory is 2553 | immediately fsynced to make sure that the file will not be lost on 2554 | @ABORTED-EXECUTION. 2555 | 2556 | - When an event is about to be written the first time after file 2557 | creation or after a sync, a transaction start marker is written to 2558 | the file. 2559 | 2560 | - Any number of events may be subsequently written until syncing is 2561 | deemed necessary (see @SYNCHRONIZATION-STRATEGIES). 2562 | 2563 | - At this point, `fsync` is called to flush all event data and state 2564 | changes to the file, and the transaction start marker is 2565 | *overwritten* with a transaction completed marker and another 2566 | `fsync` is performed. 2567 | 2568 | - When reading back this file (e.g. for replay), an open transaction 2569 | marker is treated as the end of file. 2570 | 2571 | Note that this implementation assumes that after writing the start 2572 | transaction marker, a crash cannot leave any kind of garbage bytes 2573 | around: it must leave zeros. This is not true for all filesytems. 2574 | For example, ext3/ext4 with `data=writeback` can leave garbage 2575 | around. 2576 | 2577 | ## Safety 2578 | 2579 | ##### Thread safety 2580 | 2581 | Changes to journals come in two varieties: adding an event and 2582 | changing the JOURNAL-STATE. Both are performed by JOURNALED only 2583 | unless the low-level streamlet interface is used (see 2584 | @STREAMLETS-REFERENCE). Using JOURNALED wrapped in a 2585 | WITH-JOURNALING, WITH-BUNDLE, or @LOG-RECORD without WITH-JOURNALING 2586 | is thread-safe. 2587 | 2588 | - Every journal is guaranteed to have at most a single writer active 2589 | at any time. Writers are mainly WITH-JOURNALING and WITH-BUNDLE, 2590 | but any journals directly logged to have a log writer stored in 2591 | the journal object. See @LOGGING. 2592 | 2593 | - WITH-JOURNALING and WITH-BUNDLE have dynamic extent as writers, 2594 | but log writers of journals have indefinite extent: once a journal 2595 | is used as a LOG-RECORD, there remains a writer. 2596 | 2597 | - Attempting to create a second writer triggers a JOURNAL-ERROR. 2598 | 2599 | - Writing to the same journal via @LOG-RECORD from multiple threads 2600 | concurrently is possible since this doesn't create multiple 2601 | writers. It is ensured with locking that events are written 2602 | atomically. Frames can be interleaved, but these are LOG-EVENTs, 2603 | so this does not affect replay. 2604 | 2605 | - The juggling of replay and record journals performed by 2606 | WITH-BUNDLE is also thread-safe. 2607 | 2608 | - It is ensured that there is at most one FILE-JOURNAL object in the 2609 | same Lisp image is backed by the same file. 2610 | 2611 | - Similarly, there is at most FILE-BUNDLE object for a directory. 2612 | 2613 | ##### Process safety 2614 | 2615 | Currently, there is no protection against multiple OS processes 2616 | writing the same FILE-JOURNAL or FILE-BUNDLE. 2617 | 2618 | ##### Signal safety 2619 | 2620 | Journal is *designed* to be @ASYNC-UNWIND safe but *not reentrant*. 2621 | Interrupts are disabled only for the most critical cleanup forms. If 2622 | a thread is killed without unwinding, that constitutes 2623 | @ABORTED-EXECUTION, so guarantees about @SYNCHRONIZATION apply, but 2624 | JOURNAL objects written by the thread are not safe to access, and 2625 | the Lisp should probably be restarted. 2626 | 2627 | ## Events reference 2628 | 2629 | Events are normally triggered upon entering and leaving the 2630 | dynamic extent of a JOURNALED @BLOCK (see @IN-EVENTS and 2631 | @OUT-EVENTS) and also by LOGGED. Apart from being part of the 2632 | low-level substrate of the Journal library, working with events 2633 | directly is sometimes useful when writing tests that inspect 2634 | recorded events. Otherwise, skip this entire section. 2635 | 2636 | All EVENTs have EVENT-NAME and EVENT-VERSION, which feature 2637 | prominently in @THE-REPLAY-STRATEGY. After the examples in 2638 | @IN-EVENTS and @OUT-EVENTS, the following example is a reminder of 2639 | how events look in the simplest case. 2640 | 2641 | ``` 2642 | (with-journaling (:record t) 2643 | (journaled (foo :version 1 :args '(1 2)) 2644 | (+ 1 2)) 2645 | (logged () "Oops") 2646 | (list-events)) 2647 | => ((:IN FOO :VERSION 1 :ARGS (1 2)) 2648 | (:OUT FOO :VERSION 1 :VALUES (3)) 2649 | (:LEAF "Oops")) 2650 | ``` 2651 | 2652 | So, a JOURNALED @BLOCK generates an IN-EVENT and an OUT-EVENT, which 2653 | are simple property lists. The following reference lists these 2654 | properties, their semantics and the functions to read them. 2655 | 2656 | - [type] EVENT 2657 | 2658 | An event is either an IN-EVENT, an OUT-EVENT or a LEAF-EVENT. 2659 | 2660 | - [function] EVENT= EVENT-1 EVENT-2 2661 | 2662 | Return whether EVENT-1 and EVENT-2 represent the same event. 2663 | In- and out-events belonging to the same @FRAME are *not* the same 2664 | event. EVENT-OUTCOMEs are not compared when EVENT-EXIT is :ERROR to 2665 | avoid undue dependence on implementation specific string 2666 | representations. This function is useful in conjunction with 2667 | MAKE-IN-EVENT and MAKE-OUT-EVENT to write tests. 2668 | 2669 | - [function] EVENT-NAME EVENT 2670 | 2671 | The name of an event can be of any type. It is often a symbol or a 2672 | string. When replaying, names are compared with EQUAL. All EVENTs 2673 | have names. The names of the in- and out-events belonging to the 2674 | same @FRAME are the same. 2675 | 2676 | ### Event versions 2677 | 2678 | - [function] EVENT-VERSION EVENT 2679 | 2680 | Return the version of EVENT of type EVENT-VERSION. 2681 | 2682 | - [function] LOG-EVENT-P EVENT 2683 | 2684 | See if EVENT is a LOG-EVENT. 2685 | 2686 | - [function] VERSIONED-EVENT-P EVENT 2687 | 2688 | See if EVENT is a VERSIONED-EVENT. 2689 | 2690 | - [function] EXTERNAL-EVENT-P EVENT 2691 | 2692 | See if EVENT is an EXTERNAL-EVENT. 2693 | 2694 | ### In-events 2695 | 2696 | - [type] IN-EVENT 2697 | 2698 | IN-EVENTs are triggered upon entering the dynamic extent of a 2699 | JOURNALED @BLOCK. IN-EVENTs have EVENT-NAME, 2700 | EVENT-VERSION, and EVENT-ARGS. See @IN-EVENTS for a more 2701 | introductory treatment. 2702 | 2703 | - [function] IN-EVENT-P EVENT 2704 | 2705 | See if EVENT is a IN-EVENT. 2706 | 2707 | - [function] MAKE-IN-EVENT &KEY NAME VERSION ARGS 2708 | 2709 | Create an IN-EVENT with NAME, VERSION (of type EVENT-VERSION) and 2710 | ARGS as its EVENT-NAME, EVENT-VERSION and EVENT-ARGS. 2711 | 2712 | - [function] EVENT-ARGS IN-EVENT 2713 | 2714 | Return the arguments of IN-EVENT, normally populated using the ARGS 2715 | form in JOURNALED. 2716 | 2717 | ### Out-events 2718 | 2719 | - [type] OUT-EVENT 2720 | 2721 | OUT-EVENTs are triggered upon leaving the dynamic extent of the 2722 | JOURNALED @BLOCK. OUT-EVENTs have EVENT-NAME, 2723 | EVENT-VERSION, EVENT-EXIT and EVENT-OUTCOME. 2724 | See @OUT-EVENTS for a more introductory treatment. 2725 | 2726 | - [function] OUT-EVENT-P EVENT 2727 | 2728 | See if EVENT is an OUT-EVENT. 2729 | 2730 | - [function] MAKE-OUT-EVENT &KEY NAME VERSION EXIT OUTCOME 2731 | 2732 | Create an OUT-EVENT with NAME, VERSION (of type EVENT-VERSION), 2733 | EXIT (of type EVENT-EXIT), and OUTCOME as its EVENT-NAME, 2734 | EVENT-VERSION, EVENT-EXIT and EVENT-OUTCOME. 2735 | 2736 | - [function] EVENT-EXIT OUT-EVENT 2737 | 2738 | Return how the journaled @BLOCK finished. See EVENT-EXIT 2739 | for the possible types. 2740 | 2741 | - [function] EXPECTED-OUTCOME-P OUT-EVENT 2742 | 2743 | See if OUT-EVENT has an @EXPECTED-OUTCOME. 2744 | 2745 | - [function] UNEXPECTED-OUTCOME-P OUT-EVENT 2746 | 2747 | See if OUT-EVENT has an @UNEXPECTED-OUTCOME. 2748 | 2749 | - [function] EVENT-OUTCOME OUT-EVENT 2750 | 2751 | Return the outcome of the @FRAME (or loosely speaking of a @BLOCK) 2752 | to which OUT-EVENT belongs. 2753 | 2754 | ### Leaf-events 2755 | 2756 | - [type] LEAF-EVENT 2757 | 2758 | Leaf events are triggered by LOGGED. Unlike IN-EVENTs and 2759 | OUT-EVENTs, which represent a @FRAME, leaf events represent a point 2760 | in execution thus cannot have children. They are also the poorest of 2761 | their kind: they only have an EVENT-NAME. Their VERSION is always 2762 | NIL, which makes them LOG-EVENTs. 2763 | 2764 | - [function] LEAF-EVENT-P EVENT 2765 | 2766 | See if EVENT is a LEAF-EVENT. 2767 | 2768 | - [function] MAKE-LEAF-EVENT NAME 2769 | 2770 | Create a LEAF-EVENT with NAME. 2771 | 2772 | ## Journals reference 2773 | 2774 | In @JOURNAL-BASICS, we covered the bare minimum needed to work with 2775 | journals. Here, we go into the details. 2776 | 2777 | - [class] JOURNAL 2778 | 2779 | JOURNAL is an abstract base class for a sequence of 2780 | events. In case of FILE-JOURNALs, the events are stored in a file, 2781 | while for IN-MEMORY-JOURNALs, they are in a Lisp array. When a 2782 | journal is opened, it is possible to perform I/O on it (see 2783 | @STREAMLETS-REFERENCE), which is normally taken care of by 2784 | WITH-JOURNALING. For this reason, the user's involvement with 2785 | journals normally only consists of creating and using them in 2786 | WITH-JOURNALING. 2787 | 2788 | - [reader] JOURNAL-STATE [JOURNAL][5082] (:STATE) 2789 | 2790 | Return the state of JOURNAL, which is of type 2791 | JOURNAL-STATE. 2792 | 2793 | - [reader] JOURNAL-SYNC [JOURNAL][5082] (:SYNC = NIL) 2794 | 2795 | The SYNC argument specified at instantiation. See 2796 | @SYNCHRONIZATION-STRATEGIES. 2797 | 2798 | - [function] SYNC-JOURNAL &OPTIONAL (JOURNAL (RECORD-JOURNAL)) 2799 | 2800 | Durably persist changes made to JOURNAL if JOURNAL-SYNC is T. 2801 | The changes that are persisted are 2802 | 2803 | - WRITE-EVENTs and JOURNAL-STATE changes made in an enclosing 2804 | WITH-JOURNALING; and 2805 | 2806 | - LOG-RECORDs from any thread. 2807 | 2808 | In particular, writes made in a WITH-JOURNALING in another thread 2809 | are not persisted. SYNC-JOURNAL is a noop if JOURNAL-SYNC is NIL. It 2810 | is safe to call from any thread. 2811 | 2812 | - [reader] JOURNAL-REPLAY-MISMATCH [JOURNAL][5082] (= NIL) 2813 | 2814 | If JOURNAL-DIVERGENT-P, then this is a list of two 2815 | elements: the READ-POSITIONs in the RECORD-JOURNAL and 2816 | REPLAY-JOURNAL of the first events that were different (ignoring 2817 | LOG-EVENTs). It is NIL, otherwise. 2818 | 2819 | - [function] JOURNAL-DIVERGENT-P JOURNAL 2820 | 2821 | See if WITH-JOURNALING recorded any event so far in this journal 2822 | that was not EQUAL to its @REPLAY-EVENT or it had no corresponding 2823 | replay event. This completely ignores LOG-EVENTs in both journals 2824 | being compared and can be called any time during @REPLAY. It plays a 2825 | role in WITH-BUNDLE deciding when a journal is important enough to 2826 | keep and also in @SYNCHRONIZATION-WITH-IN-MEMORY-JOURNALS. 2827 | 2828 | The position of the first mismatch is available via 2829 | JOURNAL-REPLAY-MISMATCH. 2830 | 2831 | ### Comparing journals 2832 | 2833 | After replay finished (i.e. WITH-JOURNALING completed), we can ask 2834 | whether there were any changes produced. This is answered in the 2835 | strictest sense by IDENTICAL-JOURNALS-P and somewhat more 2836 | functionally by EQUIVALENT-REPLAY-JOURNALS-P. 2837 | 2838 | Also see JOURNAL-DIVERGENT-P. 2839 | 2840 | - [generic-function] IDENTICAL-JOURNALS-P JOURNAL-1 JOURNAL-2 2841 | 2842 | Compare two journals in a strict sense: whether 2843 | they have the same JOURNAL-STATE and the lists of their events (as 2844 | in LIST-EVENTS) are EQUAL. 2845 | 2846 | - [generic-function] EQUIVALENT-REPLAY-JOURNALS-P JOURNAL-1 JOURNAL-2 2847 | 2848 | See if two journals are equivalent when used the 2849 | for REPLAY in WITH-JOURNALING. EQUIVALENT-REPLAY-JOURNALS-P is like 2850 | IDENTICAL-JOURNALS-P, but it ignores LOG-EVENTs and allows events 2851 | with EVENT-EXIT :ERROR to differ in their outcomes, which may very 2852 | well be implementation specific, anyway. Also, it considers two 2853 | groups of states as different :NEW, :REPLAYING, :MISMATCHED, :FAILED 2854 | vs :RECORDING, :LOGGING, COMPLETED. 2855 | 2856 | The rest of section is about concrete subclasses of JOURNAL. 2857 | 2858 | ### In-memory journals 2859 | 2860 | - [class] IN-MEMORY-JOURNAL JOURNAL 2861 | 2862 | IN-MEMORY-JOURNALs are backed by a non-persistent 2863 | Lisp array of events. Much quicker than FILE-JOURNALs, they are 2864 | ideal for smallish journals persisted manually (see 2865 | @SYNCHRONIZATION-WITH-IN-MEMORY-JOURNALS for an example). 2866 | 2867 | They are also useful for writing tests based on what events were 2868 | generated. They differ from FILE-JOURNALs in that events written to 2869 | IN-MEMORY-JOURNALs are not serialized (and deserialized on replay) 2870 | with the following consequences for the objects recorded by 2871 | JOURNALED (i.e. its NAME, ARGS arguments, and also the return VALUES 2872 | of the block, or the value returned by CONDITION): 2873 | 2874 | - These objects need not be @READABLE. 2875 | 2876 | - Their identity (`EQ`ness) is not lost. 2877 | 2878 | - They must **must not be mutated** in any way. 2879 | 2880 | - [function] MAKE-IN-MEMORY-JOURNAL &KEY (EVENTS NIL EVENTSP) STATE (SYNC NIL SYNCP) SYNC-FN 2881 | 2882 | Create an IN-MEMORY-JOURNAL. 2883 | 2884 | The returned journal's JOURNAL-STATE will be set to STATE. If STATE 2885 | is NIL, then it is replaced by a default value, which is :COMPLETED 2886 | if the EVENTS argument is provided, else it is :NEW. 2887 | 2888 | Thus, `(make-in-memory-journal)` creates a journal suitable for 2889 | recording, and to make a replay journal, use :STATE :COMPLETED with 2890 | some sequence of EVENTS: 2891 | 2892 | ``` 2893 | (make-in-memory-journal :events '((:in foo :version 1)) :state :completed) 2894 | ``` 2895 | 2896 | SYNC determines when SYNC-FN will be invoked on the RECORD-JOURNAL. 2897 | SYNC defaults to T if SYNC-FN, else to NIL. For a description of 2898 | possible values, see @SYNCHRONIZATION-STRATEGIES. For more 2899 | discussion, see @SYNCHRONIZATION-WITH-IN-MEMORY-JOURNALS. 2900 | 2901 | - [reader] JOURNAL-EVENTS [IN-MEMORY-JOURNAL][b668] (:EVENTS) 2902 | 2903 | A sequence of events in the journal. Not to be 2904 | mutated by client code. 2905 | 2906 | - [reader] JOURNAL-PREVIOUS-SYNC-POSITION [IN-MEMORY-JOURNAL][b668] (= 0) 2907 | 2908 | The length of JOURNAL-EVENTS at the time of the 2909 | most recent invocation of SYNC-FN. 2910 | 2911 | ### File journals 2912 | 2913 | - [class] FILE-JOURNAL JOURNAL 2914 | 2915 | A FILE-JOURNAL is a journal whose contents and 2916 | JOURNAL-STATE are persisted in a file. This is the JOURNAL 2917 | subclass with out-of-the-box persistence, but see @FILE-BUNDLES for 2918 | a more full-featured solution for repeated @REPLAYs. 2919 | 2920 | Since serialization in FILE-JOURNALs is built on top of Lisp READ 2921 | and WRITE, everything that JOURNALED records in events (i.e. its 2922 | NAME, ARGS arguments, and also the return VALUES of the block, or 2923 | the value returned by CONDITION) must be @READABLE. 2924 | 2925 | File journals are human-readable and editable by hand with some 2926 | care. When editing, the following needs to be remembered: 2927 | 2928 | - The first character of the file represents its JOURNAL-STATE. It 2929 | is a `#\Space` (for state :NEW, :REPLAYING, :MISMATCHED and 2930 | :FAILED), or a `#\Newline` (for state :RECORDING, :LOGGING and 2931 | :COMPLETED). 2932 | 2933 | - If the journal has SYNC (see @SYNCHRONIZATION-STRATEGIES), then 2934 | between two events, there may be `#\Del` (also called `#\Rubout`) 2935 | or `#\Ack` characters (CHAR-CODE 127 and 6). `#\Del` marks the end 2936 | of the journal contents that may be read back: it's kind of an 2937 | uncommitted-transaction marker for the events that follow it. 2938 | `#\Ack` characters, of which there may be many in the file, mark 2939 | the sequence of events until the next marker of either kind as 2940 | valid (or committed). `#\Ack` characters are ignored when reading 2941 | the journal. 2942 | 2943 | Thus, when editing a file, don't change the first character and 2944 | leave the `#\Del` character, if any, where it is. Also see 2945 | @SYNCHRONIZATION-WITH-FILE-JOURNALS. 2946 | 2947 | - [function] MAKE-FILE-JOURNAL PATHNAME &KEY SYNC 2948 | 2949 | Return a FILE-JOURNAL backed by the file with PATHNAME. The file is 2950 | created when the journal is opened for writing. For a description of 2951 | SYNC, see @SYNCHRONIZATION-STRATEGIES. 2952 | 2953 | If there is already an existing FILE-JOURNAL backed by the same 2954 | file, then that object is returned. If the existing object has 2955 | different options (e.g. it has SYNC T while the SYNC argument is NIL 2956 | here), then a JOURNAL-ERROR is signalled. 2957 | 2958 | If there is already an existing FILE-JOURNAL backed by the same 2959 | file, the JOURNAL-STATE is not :NEW, but the file doesn't exist, 2960 | then the existing object is **invalidated**: attempts to write will 2961 | fail with JOURNAL-ERROR. If the existing journal object is being 2962 | written, then invalidation fails with a JOURNAL-ERROR. After 2963 | invalidation, a new FILE-JOURNAL object is created. 2964 | 2965 | - [reader] PATHNAME-OF [FILE-JOURNAL][8428] (:PATHNAME) 2966 | 2967 | The pathname of the file backing the journal. 2968 | 2969 | ### Pretty-printing journals 2970 | 2971 | - [class] PPRINT-JOURNAL JOURNAL 2972 | 2973 | Events written to a PPRINT-JOURNAL have a 2974 | customizable output format. PPRINT-JOURNALs are intended for 2975 | producing prettier output for @LOGGING and @TRACING, but they do not 2976 | support reads, so they cannot be used as a REPLAY-JOURNAL or in 2977 | LIST-EVENTS, for example. On the other hand, events written to 2978 | PPRINT-JOURNALs need not be @READABLE. 2979 | 2980 | - [function] MAKE-PPRINT-JOURNAL &KEY (STREAM (MAKE-SYNONYM-STREAM '\*STANDARD-OUTPUT\*)) (PRETTY T) (PRETTIFIER 'PRETTIFY-EVENT) LOG-DECORATOR 2981 | 2982 | Creates a PPRINT-JOURNAL. 2983 | 2984 | - [accessor] PPRINT-JOURNAL-STREAM [PPRINT-JOURNAL][9150] (:STREAM = \*STANDARD-OUTPUT\*) 2985 | 2986 | The stream where events are dumped. May be set any 2987 | time to another STREAM. 2988 | 2989 | - [accessor] PPRINT-JOURNAL-PRETTY [PPRINT-JOURNAL][9150] (:PRETTY = T) 2990 | 2991 | Whether to use PPRINT-JOURNAL-PRETTIFIER or write 2992 | events in as the property lists they are. A 2993 | @BOOLEAN-VALUED-SYMBOL. 2994 | 2995 | - [accessor] PPRINT-JOURNAL-PRETTIFIER [PPRINT-JOURNAL][9150] (:PRETTIFIER = 'PRETTIFY-EVENT) 2996 | 2997 | A function like PRETTIFY-EVENT that writes an 2998 | event to a stream. Only used when PPRINT-JOURNAL-PRETTY, this is 2999 | the output format customization knob. Also see @DECORATIONs. 3000 | 3001 | ## Bundles reference 3002 | 3003 | In @BUNDLES, we covered the repeated replay problem that 3004 | WITH-BUNDLE automates. Here, we provide a reference for the bundle 3005 | classes. 3006 | 3007 | - [class] BUNDLE 3008 | 3009 | A BUNDLE consists of a sequence of journals which 3010 | are all reruns of the same code, hopefully making more and more 3011 | progress towards completion. These journals are @REPLAYs of the 3012 | previous successful one, extending it with new events. Upon 3013 | replay (see WITH-BUNDLE), the latest journal in the bundle in 3014 | JOURNAL-STATE :COMPLETED plays the role of the replay journal, and a 3015 | new journal is added to the bundle for recording. If the replay 3016 | succeeds, this new journal eventually becomes :COMPLETED and takes 3017 | over the role of the replay journal for future replays until another 3018 | replay succeeds. When the bundle is created and it has no journals 3019 | yet, the replay journal is an empty, completed one. 3020 | 3021 | This is an abstract base class. Direct subclasses are 3022 | IN-MEMORY-BUNDLE and FILE-BUNDLE. 3023 | 3024 | - [accessor] MAX-N-FAILED [BUNDLE][d9b6] (:MAX-N-FAILED = 1) 3025 | 3026 | If MAX-N-FAILED is non-NIL, and the number of 3027 | journals of JOURNAL-STATE :FAILED in the bundle exceeds 3028 | its value, then some journals (starting with the oldest) are 3029 | deleted. 3030 | 3031 | - [accessor] MAX-N-COMPLETED [BUNDLE][d9b6] (:MAX-N-COMPLETED = 1) 3032 | 3033 | If MAX-N-COMPLETED is non-NIL, and the number of 3034 | journals of JOURNAL-STATE :COMPLETED in the bundle exceeds 3035 | its value, then some journals (starting with the oldest) are 3036 | deleted. 3037 | 3038 | ### In-memory bundles 3039 | 3040 | - [class] IN-MEMORY-BUNDLE BUNDLE 3041 | 3042 | An IN-MEMORY-BUNDLE is a BUNDLE that is built on 3043 | IN-MEMORY-JOURNALs. IN-MEMORY-BUNDLEs have limited utility as a 3044 | persistence mechanism and are provided mainly for reasons of 3045 | symmetry and for testing. See 3046 | @SYNCHRONIZATION-WITH-IN-MEMORY-JOURNALS for an example of how to 3047 | achieve persistence without bundles. 3048 | 3049 | - [function] MAKE-IN-MEMORY-BUNDLE &KEY (MAX-N-FAILED 1) (MAX-N-COMPLETED 1) SYNC SYNC-FN 3050 | 3051 | Create a new IN-MEMORY-BUNDLE with MAX-N-FAILED and MAX-N-COMPLETED. SYNC and SYNC-FN 3052 | are passed on to MAKE-IN-MEMORY-JOURNAL. 3053 | 3054 | ### File bundles 3055 | 3056 | - [class] FILE-BUNDLE BUNDLE 3057 | 3058 | A FILE-BUNDLE is a BUNDLE that is built on 3059 | FILE-JOURNALs. It provides easy replay-based persistence. 3060 | 3061 | - [reader] DIRECTORY-OF [FILE-BUNDLE][1895] (:DIRECTORY) 3062 | 3063 | The directory where the files backing the 3064 | FILE-JOURNALs in the FILE-BUNDLE are kept. 3065 | 3066 | - [function] MAKE-FILE-BUNDLE DIRECTORY &KEY (MAX-N-FAILED 1) (MAX-N-COMPLETED 1) SYNC 3067 | 3068 | Return a FILE-BUNDLE object backed by FILE-JOURNALs in DIRECTORY. 3069 | See MAX-N-FAILED and 3070 | MAX-N-COMPLETED. For a description of SYNC, see 3071 | @SYNCHRONIZATION-STRATEGIES. 3072 | 3073 | If there is already a FILE-BUNDLE with the same directory (according 3074 | to TRUENAME), return that object is returned if it has the same 3075 | MAX-N-FAILED, MAX-N-COMPLETED and SYNC options, else JOURNAL-ERROR 3076 | is signalled. 3077 | 3078 | - [function] DELETE-FILE-BUNDLE DIRECTORY 3079 | 3080 | Delete all journal files (`*.jrn`) from DIRECTORY. Delete the 3081 | directory if empty after the journal files were deleted, else signal 3082 | an error. Existing FILE-BUNDLE objects are not updated, so 3083 | MAKE-FILE-JOURNAL with FORCE-RELOAD may be required. 3084 | 3085 | ## Streamlets reference 3086 | 3087 | This section is relevant mostly for implementing new kinds of 3088 | JOURNALs in addition to FILE-JOURNALs and IN-MEMORY-JOURNALs. In 3089 | normal operation, STREAMLETs are not worked with directly. 3090 | 3091 | ### Opening and closing 3092 | 3093 | - [class] STREAMLET 3094 | 3095 | A STREAMLET is a handle to perform I/O on a 3096 | JOURNAL. The high-level stuff (WITH-JOURNALING, JOURNALED, etc) is 3097 | built on top of streamlets. 3098 | 3099 | - [reader] JOURNAL [STREAMLET][7a2f] (:JOURNAL) 3100 | 3101 | The JOURNAL that was passed to OPEN-STREAMLET. 3102 | This is the journal STREAMLET operates on. 3103 | 3104 | - [generic-function] OPEN-STREAMLET JOURNAL &KEY DIRECTION 3105 | 3106 | Return a STREAMLET suitable for performing I/O on 3107 | JOURNAL. DIRECTION (defaults to :INPUT) is one of :INPUT, :OUTPUT, 3108 | :IO, and it has the same purpose as the similarly named argument of 3109 | CL:OPEN. 3110 | 3111 | - [generic-function] CLOSE-STREAMLET STREAMLET 3112 | 3113 | Close STREAMLET, which was returned by 3114 | OPEN-STREAMLET. After closing, STREAMLET may not longer be used for 3115 | IO. 3116 | 3117 | - [generic-function] MAKE-STREAMLET-FINALIZER STREAMLET 3118 | 3119 | Return NIL or a function of no arguments suitable 3120 | as a finalizer for STREAMLET. That is, a function that closes 3121 | STREAMLET but holds no reference to it. This is intended for 3122 | streamlets that are not dynamic-extent, so using WITH-OPEN-JOURNAL 3123 | is not appropriate. 3124 | 3125 | - [generic-function] OPEN-STREAMLET-P STREAMLET 3126 | 3127 | Return true if STREAMLET is open. STREAMLETs are 3128 | open until they have been explicitly closed with CLOSE-STREAMLET. 3129 | 3130 | - [function] INPUT-STREAMLET-P STREAMLET 3131 | 3132 | See if STREAMLET was opened for input (the DIRECTION argument of 3133 | OPEN-STREAMLET was :INPUT or :IO). 3134 | 3135 | - [function] OUTPUT-STREAMLET-P STREAMLET 3136 | 3137 | See if STREAMLET was opened for input (the DIRECTION argument of 3138 | OPEN-STREAMLET was :OUTPUT or :IO). 3139 | 3140 | - [macro] WITH-OPEN-JOURNAL (VAR JOURNAL &KEY (DIRECTION :INPUT)) &BODY BODY 3141 | 3142 | This is like WITH-OPEN-FILE but for JOURNALs. 3143 | Open the journal designated by JOURNAL (see TO-JOURNAL) with 3144 | OPEN-STREAMLET, passing DIRECTION along, and bind VAR to the 3145 | resulting STREAMLET. Call CLOSE-STREAMLET after BODY finishes. If 3146 | JOURNAL is NIL, then VAR is bound to NIL and no streamlet is 3147 | created. 3148 | 3149 | - [condition] STREAMLET-ERROR ERROR 3150 | 3151 | Like CL:STREAM-ERROR: failures pertaining to I/O on 3152 | a closed STREAMLET or of the wrong DIRECTION. Actual I/O errors are 3153 | *not* encapsulated in STREAMLET-ERROR. 3154 | 3155 | ### Reading from streamlets 3156 | 3157 | - [generic-function] READ-EVENT STREAMLET &OPTIONAL EOJ-ERROR-P 3158 | 3159 | Read the event at the current read position from 3160 | STREAMLET, and move the read position to the event after. If there 3161 | are no more events, signal END-OF-JOURNAL or return NIL depending on 3162 | EOJ-ERROR-P. Signals STREAMLET-ERROR if STREAMLET is not 3163 | INPUT-STREAMLET-P or not OPEN-STREAMLET-P. 3164 | 3165 | - [generic-function] READ-POSITION STREAMLET 3166 | 3167 | Return an integer that identifies the position of 3168 | the next event to be read from STREAMLET. `SETF`able, see 3169 | SET-READ-POSITION. 3170 | 3171 | - [generic-function] SET-READ-POSITION STREAMLET POSITION 3172 | 3173 | Set the read position of STREAMLET to POSITION, 3174 | which must have been acquired from READ-POSITION. 3175 | 3176 | - [macro] SAVE-EXCURSION (STREAMLET) &BODY BODY 3177 | 3178 | Save READ-POSITION of STREAMLET, execute BODY, and make sure to 3179 | restore the saved read position. 3180 | 3181 | - [generic-function] PEEK-EVENT STREAMLET 3182 | 3183 | Read the next event from STREAMLET without changing 3184 | the read position, or return NIL if there is no event to be read. 3185 | 3186 | - [method] PEEK-EVENT (STREAMLET STREAMLET) 3187 | 3188 | This is a slow default implementation, which relies on 3189 | SAVE-EXCURSION and READ-EVENT. 3190 | 3191 | ### Writing to streamlets 3192 | 3193 | - [generic-function] WRITE-EVENT EVENT STREAMLET 3194 | 3195 | Write EVENT to STREAMLET. 3196 | Writing always happens at the end of STREAMLET's journal regardless 3197 | of the READ-POSITION, and the read position is not changed. Signals 3198 | STREAMLET-ERROR if STREAMLET is not OUTPUT-STREAMLET-P or not 3199 | OPEN-STREAMLET-P. 3200 | 3201 | - [method] WRITE-EVENT EVENT (JOURNAL JOURNAL) 3202 | 3203 | For convenience, it is possible to write directly to a JOURNAL, 3204 | in which case the journal's internal output streamlet is used. 3205 | This internal streamlet is opened for :OUTPUT and may be used by 3206 | @LOG-RECORD. 3207 | 3208 | - [generic-function] WRITE-POSITION STREAMLET 3209 | 3210 | Return an integer that identifies the position of 3211 | the next event to be written to STREAMLET. 3212 | 3213 | - [generic-function] REQUEST-COMPLETED-ON-ABORT STREAMLET 3214 | 3215 | Make it so that upon @ABORTED-EXECUTION, 3216 | STREAMLET's JOURNAL will be in JOURNAL-STATE :COMPLETED when loaded 3217 | fresh (e.g. when creating a FILE-JOURNAL with an existing file). Any 3218 | previously written events must be persisted before making this 3219 | change. Before REQUEST-COMPLETED-ON-ABORT is called, a journal must 3220 | be reloaded in state :FAILED. 3221 | 3222 | It is permissible to defer carrying out this request until the next 3223 | SYNC-STREAMLET call. If the request was carried out, return true. If 3224 | it was deferred, return NIL. 3225 | 3226 | - [generic-function] SYNC-STREAMLET STREAMLET 3227 | 3228 | Durably persist the effects of all preceding 3229 | WRITE-EVENT calls made via STREAMLET to its journal and any deferred 3230 | REQUEST-COMPLETED-ON-ABORT in this order. 3231 | 3232 | ## Glossary 3233 | 3234 | - [glossary-term] async-unwind 3235 | 3236 | If an asynchronous event, say a `SIGINT` triggered by `C-c`, is 3237 | delivered to a thread running Lisp or foreign code called from Lisp, 3238 | a Lisp condition is typically signalled. If the handler for this 3239 | condition unwinds the stack, then we have an asynchronous unwind. 3240 | Another example is BT:INTERRUPT-THREAD, which, as it can execute 3241 | arbitrary code, may unwind the stack in the target thread. 3242 | 3243 | - [glossary-term] boolean-valued symbol 3244 | 3245 | Imagine writing two STREAMs with a spaghetti of functions and 3246 | wanting to have pretty-printed output on one of them. Unfortunately, 3247 | binding *PRINT-PRETTY* to T will affect writes to both streams. 3248 | 3249 | One solution would be to have streams look up their own print-pretty 3250 | flag with `(SYMBOL-VALUE (STREAM-PRETTY-PRINT STREAM))` and have the 3251 | caller specify the dynamic variable they want: 3252 | 3253 | ``` 3254 | (defvar *print-pretty-1* nil) 3255 | (setf (stream-print-pretty stream-1) '*print-pretty-1*) 3256 | (let ((*print-pretty-1* t)) 3257 | (spaghetti stream-1 stream-2)) 3258 | ``` 3259 | 3260 | Note that if the default `STREAM-PRINT-PRETTY` is `'*PRINT-PRETTY*`, 3261 | then we have the normal Common Lisp behaviour. Setting 3262 | `STREAM-PRINT-PRETTY` to NIL or T also works, because they are 3263 | self-evaluating. 3264 | 3265 | The above hypothetical example demonstrates the concept of 3266 | boolean-valued symbols on CL:STREAMs. In Journal, they are used by 3267 | MAKE-LOG-DECORATOR and PPRINT-JOURNALs. 3268 | 3269 | - [glossary-term] readable 3270 | 3271 | In Common Lisp, readable objects are those that can be printed 3272 | readably. Anything written to stream-based journals needs to 3273 | be readable. 3274 | 3275 | [1895]: #JOURNAL:FILE-BUNDLE%20CLASS "JOURNAL:FILE-BUNDLE CLASS" 3276 | [2e9b]: #JOURNAL:REPLAY-FAILURE%20CONDITION "JOURNAL:REPLAY-FAILURE CONDITION" 3277 | [3956]: #JOURNAL:JOURNALING-FAILURE%20CONDITION "JOURNAL:JOURNALING-FAILURE CONDITION" 3278 | [5082]: #JOURNAL:JOURNAL%20CLASS "JOURNAL:JOURNAL CLASS" 3279 | [7a2f]: #JOURNAL:STREAMLET%20CLASS "JOURNAL:STREAMLET CLASS" 3280 | [8428]: #JOURNAL:FILE-JOURNAL%20CLASS "JOURNAL:FILE-JOURNAL CLASS" 3281 | [9150]: #JOURNAL:PPRINT-JOURNAL%20CLASS "JOURNAL:PPRINT-JOURNAL CLASS" 3282 | [b668]: #JOURNAL:IN-MEMORY-JOURNAL%20CLASS "JOURNAL:IN-MEMORY-JOURNAL CLASS" 3283 | [d9b6]: #JOURNAL:BUNDLE%20CLASS "JOURNAL:BUNDLE CLASS" 3284 | 3285 | * * * 3286 | ###### \[generated by [MGL-PAX](https://github.com/melisgl/mgl-pax)\] 3287 | -------------------------------------------------------------------------------- /TODO.org: -------------------------------------------------------------------------------- 1 | * checkpoints 2 | We need O(1) startup. Currently the entire journal must be read and 3 | executed. Checkpoints (snapshots?) would allow state to be 4 | reconstructed cheaply. Checkpoints would be written only when it's the 5 | easiest: e.g. in T&E, at the beginning of a player's turn (not during 6 | revolts, wars, etc). 7 | 8 | How to support undoing past a checkpoint? By replaying from the 9 | previous checkpoint if any. 10 | 11 | Would restoring state from a checkpoint be a completely different code 12 | path? That's almost as fragile as mapping the program state to a DB by 13 | hand. Still, it's a bit better because it avoids the really 14 | complicated states deeply intertwined with the control flow. 15 | 16 | Can checkpointing be implemented with journal rewriters? Can we check 17 | that the resulting state is the same? Alternate code paths leading to 18 | the same state? A checkpoint is really a shortcut. Must check state is 19 | the same. 20 | * threads/processes 21 | It should be possible to start new independent "threads", each with 22 | its own (logical) journal, probably with synchronization primitives 23 | and the ability join. 24 | 25 | Maybe this could support operations whose execution order is 26 | unimportant? 27 | -------------------------------------------------------------------------------- /journal.asd: -------------------------------------------------------------------------------- 1 | ;;;; -*-mode: Lisp; coding: utf-8;-*- 2 | 3 | ;;; See JOURNAL::@JOURNAL-MANUAL for the user guide. 4 | (asdf:defsystem :journal 5 | :licence "MIT, see COPYING." 6 | :version "0.1.0" 7 | :author "Gábor Melis " 8 | :homepage "http://github.com/melisgl/journal" 9 | :bug-tracker "http://github.com/melisgl/journal/issues" 10 | :source-control (:git "https://github.com/melisgl/journal.git") 11 | :description "A library built around explicit execution traces for 12 | logging, tracing, testing and persistence." 13 | :depends-on (#:alexandria #:bordeaux-threads #:local-time 14 | #:mgl-pax #:trivial-features #:trivial-garbage 15 | (:feature (:not (:or :abcl :allegro :sbcl :cmucl)) #:osicat) 16 | (:feature :sbcl #:sb-posix)) 17 | :components ((:module "src" 18 | :serial t 19 | :components ((:file "package") 20 | (:file "interrupt") 21 | (:file "journal") 22 | (:file "doc")))) 23 | :in-order-to ((asdf:test-op (asdf:test-op "journal/test")))) 24 | 25 | (asdf:defsystem :journal/test 26 | :licence "MIT, see COPYING." 27 | :version "0.0.1" 28 | :author "Gábor Melis " 29 | :description "Tests for Journal." 30 | :depends-on (#:alexandria #:journal #:try) 31 | :components ((:module "test" 32 | :serial t 33 | :components ((:file "package") 34 | (:file "test-journal")))) 35 | :perform (asdf:test-op (o s) 36 | (uiop:symbol-call '#:journal-test '#:test))) 37 | -------------------------------------------------------------------------------- /src/doc.lisp: -------------------------------------------------------------------------------- 1 | (in-package :journal) 2 | 3 | ;;;; Register in PAX World 4 | 5 | (defun pax-sections () 6 | (list @journal-manual)) 7 | (defun pax-pages () 8 | `((:objects 9 | (, @journal-manual) 10 | :source-uri-fn ,(make-github-source-uri-fn 11 | :journal 12 | "https://github.com/melisgl/journal")))) 13 | (register-doc-in-pax-world :journal (pax-sections) (pax-pages)) 14 | 15 | ;;; Regenerate documentation 16 | #+nil 17 | (progn 18 | (asdf:load-system :journal) 19 | (update-asdf-system-readmes @journal-manual :journal 20 | :formats '(:markdown :plain)) 21 | (let ((*document-downcase-uppercase-code* t)) 22 | (update-asdf-system-html-docs @journal-manual :journal 23 | :pages (pax-pages)))) 24 | -------------------------------------------------------------------------------- /src/file-position-test-data: -------------------------------------------------------------------------------- 1 | Data for JOURNAL::*FILE-POSITION-WORKS-P*. -------------------------------------------------------------------------------- /src/interrupt.lisp: -------------------------------------------------------------------------------- 1 | (in-package :journal) 2 | 3 | ;;;; Signal handling 4 | 5 | ;;; Only used for self-documenting. 6 | (defmacro async-signal-safe (&body body) 7 | `(progn ,@body)) 8 | 9 | (defvar *without-interrupts-available* nil) 10 | (defvar *with-interrupts-available* nil) 11 | 12 | #+allegro 13 | (progn 14 | (setq *without-interrupts-available* t) 15 | (defmacro without-interrupts (&body body) 16 | `(excl:with-delayed-interrupts ,@body)) 17 | (setq *with-interrupts-available* t) 18 | (defmacro with-interrupts (&body body) 19 | `(let ((excl::*without-interrupts* nil)) 20 | ,@body))) 21 | 22 | #+ccl 23 | (progn 24 | (setq *without-interrupts-available* t) 25 | (defmacro without-interrupts (&body body) 26 | `(ccl:without-interrupts ,@body)) 27 | (setq *with-interrupts-available* t) 28 | (defmacro with-interrupts (&body body) 29 | `(ccl:with-interrupts-enabled ,@body))) 30 | 31 | #+cmucl 32 | (progn 33 | (setq *without-interrupts-available* t) 34 | (defmacro without-interrupts (&body body) 35 | `(sys:without-interrupts ,@body)) 36 | (setq *with-interrupts-available* t) 37 | (defmacro with-interrupts (&body body) 38 | `(let ((unix::*interrupts-enabled* t)) 39 | (when unix::*interrupt-pending* 40 | (unix::do-pending-interrupt)) 41 | ,@body))) 42 | 43 | #+ecl 44 | (progn 45 | (setq *without-interrupts-available* t) 46 | (defmacro without-interrupts (&body body) 47 | `(mp:without-interrupts 48 | (mp:allow-with-interrupts 49 | ,@body))) 50 | (setq *with-interrupts-available* t) 51 | (defmacro with-interrupts (&body body) 52 | `(mp:with-interrupts ,@body))) 53 | 54 | #+lispworks 55 | (progn 56 | (setq *without-interrupts-available* t) 57 | (defmacro without-interrupts (&body body) 58 | `(mp:with-interrupts-blocked ,@body)) 59 | (setq *without-interrupts-available* nil)) 60 | 61 | #+sbcl 62 | (progn 63 | (setq *without-interrupts-available* t) 64 | (defmacro without-interrupts (&body body) 65 | `(sb-sys:without-interrupts 66 | (sb-sys:allow-with-interrupts 67 | ,@body))) 68 | (setq *with-interrupts-available* t) 69 | (defmacro with-interrupts (&body body) 70 | `(sb-sys:with-interrupts ,@body))) 71 | 72 | (unless *without-interrupts-available* 73 | (format *error-output* 74 | "~&~@") 77 | ;; KLUDGE: Quicklisp calls MUFFLE-WARNING on WARNINGs, but it fails 78 | ;; "with no restart available" on some Lisps. 79 | #-(or abcl clisp) 80 | (signal 'style-warning) 81 | (defmacro without-interrupts (&body body) 82 | `(progn ,@body))) 83 | 84 | (unless *with-interrupts-available* 85 | (format *error-output* 86 | "~&~@") 88 | #-(or abcl clisp) 89 | (signal 'style-warning) 90 | (defmacro with-interrupts (&body body) 91 | `(progn ,@body))) 92 | 93 | ;;; Recompile with these when doing statistical profiling that relies 94 | ;;; on signals. Or when feeling brave. 95 | #+nil 96 | (progn 97 | (defmacro without-interrupts (&body body) 98 | `(progn ,@body)) 99 | (defmacro with-interrupts (&body body) 100 | `(progn ,@body))) 101 | 102 | (defmacro unwind-protect* (protected &body cleanup) 103 | #-ccl 104 | `(without-interrupts 105 | (unwind-protect 106 | (with-interrupts 107 | ,protected) 108 | ,@cleanup)) 109 | ;; CCL cleanups are already protected from interrupts. 110 | #+ccl 111 | `(unwind-protect 112 | ,protected 113 | ,@cleanup)) 114 | -------------------------------------------------------------------------------- /src/mgl-jrn.el: -------------------------------------------------------------------------------- 1 | ;; -*- lexical-binding: t -*- 2 | 3 | ;;;; Autoloading of JOURNAL on the Common Lisp side 4 | 5 | (defcustom mgl-jrn-autoload t 6 | "If true, then the JOURNAL ASDF system will be loaded as 7 | necessary by `slime-toggle-jtrace-fdefinition'." 8 | :type 'boolean 9 | :group 'mgl-jrn) 10 | 11 | (defvar mgl-jrn-version '(0 1 0)) 12 | 13 | (defun mgl-jrn-maybe-autoload (cont) 14 | (if mgl-jrn-autoload 15 | (slime-eval-async 16 | `(cl:progn 17 | (cl:unless (cl:find-package :journal) 18 | (cl:format t ";; Autoloading JOURNAL for Emacs ~ 19 | (mgl-jrn-autoload is t).~%") 20 | (asdf:load-system "journal") 21 | (cl:format t ";; Done autoloading JOURNAL for Emacs~%")) 22 | (cl:and (cl:find-package :journal) 23 | (cl:funcall (cl:find-symbol 24 | (cl:string '#:check-jrn-elisp-version) 25 | :journal) 26 | ',mgl-jrn-version) 27 | t)) 28 | cont) 29 | (slime-eval-async 30 | `(cl:and (cl:find-package :journal) 31 | (cl:funcall (cl:find-symbol 32 | (cl:string '#:check-jrn-elisp-version) 33 | :journal) 34 | ',mgl-jrn-version) 35 | t) 36 | cont))) 37 | 38 | (defun slime-toggle-jtrace-fdefinition (spec) 39 | "Toggle JOURNAL:JTRACE. If invoked with the empty string, then 40 | JOURNAL:JUNTRACE all." 41 | (interactive (list (slime-read-from-minibuffer 42 | "j(un)trace: " (slime-symbol-at-point)))) 43 | (mgl-jrn-maybe-autoload 44 | (lambda (loadedp) 45 | (if (not loadedp) 46 | (message "JOURNAL is not loaded. See the variable mgl-jrn-autoload.") 47 | (message "%s" (slime-eval 48 | ;; Silently fail if Journal is not loaded. 49 | `(cl:when (cl:find-package :journal) 50 | (cl:funcall 51 | (cl:find-symbol 52 | (cl:string '#:swank-toggle-jtrace) 53 | :journal) 54 | ,spec)))))))) 55 | 56 | (define-key slime-mode-map (kbd "C-c C-j") 57 | 'slime-toggle-jtrace-fdefinition) 58 | 59 | (define-key slime-repl-mode-map (kbd "C-c C-j") 60 | 'slime-toggle-jtrace-fdefinition) 61 | 62 | (provide 'mgl-jrn) 63 | -------------------------------------------------------------------------------- /src/package.lisp: -------------------------------------------------------------------------------- 1 | (mgl-pax:define-package :journal 2 | (:nicknames #:jrn) 3 | (:use #:common-lisp #:mgl-pax #:named-readtables #:pythonic-string-reader)) 4 | -------------------------------------------------------------------------------- /test/package.lisp: -------------------------------------------------------------------------------- 1 | (mgl-pax:define-package :journal-test 2 | (:use #:common-lisp #:journal #:try) 3 | (:shadowing-import-from #:journal #:event)) 4 | -------------------------------------------------------------------------------- /test/profile.lisp: -------------------------------------------------------------------------------- 1 | (in-package :cl-user) 2 | 3 | (eval-when (:compile-toplevel :load-toplevel :execute) 4 | (require :sb-sprof)) 5 | 6 | (defun profile-bundle () 7 | (jrn:make-file-bundle (asdf:system-relative-pathname 8 | :journal 9 | "test/test-bundle/") 10 | :max-n-failed 1 11 | :max-n-completed 1 12 | :sync t)) 13 | 14 | (defun fib (n) 15 | (jrn:journaled (fib :version :infinity :args `(,n)) 16 | (cond ((= n 0) 1) 17 | ((= n 1) 1) 18 | (t (+ (fib (- n 1)) 19 | (fib (- n 2))))))) 20 | 21 | (defun xxx (n) 22 | (loop for i below n do 23 | (jrn:journaled (xxx :version :infinity :args `(,i))))) 24 | 25 | #+nil 26 | (defun foo (n filep) 27 | (let ((bundle (if filep 28 | (profile-bundle) 29 | (jrn:make-in-memory-bundle 30 | :max-n-failed 1 31 | :max-n-completed 1)))) 32 | (loop repeat n do 33 | (jrn:with-bundle (bundle) 34 | (fib 10))))) 35 | 36 | (defun foo (n filep) 37 | (let ((bundle (if filep 38 | (profile-bundle) 39 | (jrn:make-in-memory-bundle 40 | :max-n-failed 1 41 | :max-n-completed 1)))) 42 | (loop repeat n do 43 | (jrn:with-bundle (bundle) 44 | (xxx 177))))) 45 | 46 | #+nil 47 | (time (loop repeat 100 do (journal::test))) 48 | 49 | #+nil 50 | (time (foo 100 t)) 51 | 52 | #+nil 53 | (sb-sprof:with-profiling (:report :graph) 54 | (time (foo 10000 nil))) 55 | -------------------------------------------------------------------------------- /test/registration/00000000.jrn: -------------------------------------------------------------------------------- 1 | 2 | (:IN "ask-username" :VERSION :INFINITY) 3 | (:OUT "ask-username" :VERSION :INFINITY :VALUES ("joe" NIL)) 4 | (:IN "get-key" :VERSION :INFINITY :ARGS ("joe")) 5 | (:OUT "get-key" :VERSION :INFINITY :VALUES (NIL NIL)) 6 | (:IN "set-key" :VERSION :INFINITY :ARGS ("joe" (:USER-OBJECT :USERNAME "joe"))) 7 | (:OUT "set-key" :VERSION :INFINITY :VALUES (NIL)) 8 | (:IN "maybe-win-the-grand-prize" :VERSION 1) 9 | (:OUT "maybe-win-the-grand-prize" :VERSION 1 :VALUES (NIL)) 10 | (:IN "get-key" :VERSION :INFINITY :ARGS ("joe")) 11 | (:OUT "get-key" :VERSION :INFINITY :VALUES ((:USER-OBJECT :USERNAME "joe") T)) 12 | (:IN "get-key" :VERSION :INFINITY :ARGS ("joe")) 13 | (:OUT "get-key" :VERSION :INFINITY :VALUES ((:USER-OBJECT :USERNAME "joe") T)) 14 | (:IN "get-key" :VERSION :INFINITY :ARGS ("joe")) 15 | (:OUT "get-key" :VERSION :INFINITY :VALUES ((:USER-OBJECT :USERNAME "joe") T)) 16 | -------------------------------------------------------------------------------- /test/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | lisp="$1" 4 | stop_on_failure="${2:-t}" 5 | debug="${3:-nil}" 6 | print="${4:-(quote (or try:unexpected try:skip))}" 7 | describe="${5:-(quote try:unexpected)}" 8 | num_passes= 9 | num_failures= 10 | 11 | function run_test_case { 12 | local test_case_name="\"$1\"" 13 | shift 14 | echo "SHTEST: Running ${test_case_name} $@" 15 | $@ 16 | local retval=$? 17 | if ((retval == 22)); then 18 | echo 19 | echo "SHTEST: ${test_case_name} PASS" 20 | num_passes=$((num_passes+1)) 21 | else 22 | echo 23 | echo "SHTEST: ${test_case_name} FAIL" 24 | num_failures=$((num_failures+1)) 25 | fi 26 | } 27 | 28 | function lisp_tests { 29 | local lisp_name="$1" 30 | shift 31 | 32 | run_test_case "lisp test suite on ${lisp_name}" $@ < 0)); then 50 | if [ "${stop_on_failure}" = "t" ]; then 51 | echo "SHTEST: Aborting with ${num_failures} failures,"\ 52 | "${num_passes} passes." 53 | exit 1 54 | fi 55 | fi 56 | echo "SHTEST: ${num_failures} failures, ${num_passes} passes." 57 | } 58 | 59 | if [ -n "${lisp}" ]; then 60 | run_tests lisp_tests ${lisp} 61 | else 62 | run_tests lisp_tests sbcl --noinform 63 | # Runs out of heap in the Express version. 64 | # run_tests lisp_tests allegro --batch --backtrace-on-error 65 | run_tests lisp_tests ccl-bin 66 | run_tests lisp_tests cmu-bin -batch 67 | run_tests lisp_tests ecl 68 | run_tests lisp_tests abcl-bin 69 | fi 70 | --------------------------------------------------------------------------------