├── .github └── workflows │ └── make.yml ├── .gitignore ├── CHANGES.md ├── LICENSE ├── README.md ├── bindings ├── kdb │ ├── .gitignore │ ├── Makefile │ ├── README.md │ ├── hpet.c │ ├── hpet.q │ ├── shm-ipc-client.q │ ├── shm-ipc-server.q │ ├── shm-recv-once.q │ ├── shm-recv.q │ ├── shm-sender.q │ ├── shm.q │ └── shmipc.c └── python │ ├── README.md │ ├── bulkwriter.py │ ├── libchronicle.py │ ├── reader.py │ ├── reader_cb.py │ └── writer.py ├── java ├── .gitignore ├── README.md ├── java.iml ├── pom.cqv4.xml ├── pom.xml ├── sample.input └── src │ └── main │ └── java │ └── mains │ ├── InputMain.java │ └── OutputMain.java └── native ├── Makefile ├── buffer.c ├── buffer.h ├── fuzzmain.c ├── libchronicle.c ├── libchronicle.h ├── mock_k.h ├── obj └── .gitignore ├── serdes_k.h ├── shm_example_reader.c ├── shm_example_writer.c ├── shmmain.c ├── test ├── .gitignore ├── cqv4-sample-input.tar.bz2 ├── cqv5-sample-input.tar.bz2 ├── fuzz_input │ └── manual1 ├── fuzz_output │ └── .gitignore ├── test_buffer.c ├── test_queue.c ├── test_wire.c └── testdata.h ├── wire.c └── wire.h /.github/workflows/make.yml: -------------------------------------------------------------------------------- 1 | name: CMake 2 | 3 | on: [push] 4 | 5 | env: 6 | BUILD_TYPE: Release 7 | 8 | jobs: 9 | build: 10 | runs-on: ubuntu-latest 11 | 12 | steps: 13 | - uses: actions/checkout@v2 14 | - name: Prep 15 | shell: bash 16 | run: sudo apt install libcmocka-dev libarchive-dev 17 | - name: Build 18 | shell: bash 19 | run: make -C native 20 | - name: Test 21 | shell: bash 22 | run: make -C native test 23 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | native/k.h 3 | native/test/*.to 4 | native/test/*.tok 5 | native/test/*.tvg 6 | __pycache__ 7 | venv 8 | -------------------------------------------------------------------------------- /CHANGES.md: -------------------------------------------------------------------------------- 1 | CHANGES 2 | ==== 3 | 4 | To track changes in the protocol, or defaults, rather than differences in any particular implementation 5 | 6 | Chronicle Queue v5 7 | ---- 8 | 9 | * FAST_XXX RollSchemes see https://github.com/OpenHFT/Chronicle-Queue/blob/ea/src/main/java/net/openhft/chronicle/queue/RollCycles.java 10 | 11 | * Default RollScheme (e.g. for SingleChronicleQueueBuilder) changes from `DAILY` to `FAST_DAILY` 12 | * Since schemes make reduced bits available for cycle value, new concept of "epoch" which appears to be a constant offset to the cycle value, ie. we now longer assume epoch is midnight 1970.01.01 13 | * New per-queue `metadata.cq4t` file replaces `directory-listing.cq4t` file. 14 | * STStore -> SCQMeta -> SCQSRoll fields need to be parsed since they are removed from queue files: 15 | * length (uint32) 16 | * epoch (uint8) 17 | * format (text) 18 | * queue metatdata wire encoder changes: 19 | * `SCQSIndexing.indexSpacing` changes from `uint8` to `uint16` 20 | * a v5 queue with no data written can be a metadata.cq4t with no queue files (relax queuefile existance checks) 21 | * SingleChronicleQueueBuilder writeText() seems to be writing a one-byte prefix to the data 22 | * data/metadata messages are now 4 byte aligned (see `pad4` in `parse_queue_block`) so the header CAS read is always aligned 23 | * only appenders can write metadata and index pages (no lazy indexing - libchronicle never supported it) 24 | 25 | Chronicle Queue v4 26 | ---- 27 | * requires at least one queuefile to learn the roll scheme, although we do not know what the rollscheme is to find the queuefile, so bootstrap using filesystem glob 28 | * queue data/metadata pages are not aligned 29 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2018 Tea Engineering Ltd. 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | ## Shared-memory communication using OpenHFT's chronicle-queue protocol 3 | 4 | [OpenHFT](https://github.com/OpenHFT) aka. [Chronicle Software Ltd.](https://chronicle.software/) provide an open-source Java '[Chronicle Queue](https://github.com/OpenHFT/Chronicle-Queue)' ipc library. [This project](https://github.com/TeaEngineering/libchronicle) is an unaffiliated, mostly-compatible, open source implementation in the C-programming language, with bindings to other non-JVM languages. To differentiate I refer to OpenHFTs implementation capitalised as 'Chronicle Queue', and the underlying procotol itself chronicle-queue, and this implementation as `libchronicle`. 5 | 6 | ## Getting started 7 | 8 | See more examples in [python](bindings/python/) or [kdb](bindings/kdb/). 9 | 10 | Our C example is in two parts. The writing side is [`native/shm_example_writer.c`](native/shm_example_writer.c) and can be built by `make`: 11 | 12 | $ git clone git@github.com:TeaEngineering/libchronicle.git 13 | $ cd libchronicle/native 14 | $ make 15 | $ mkdir /tmp/testq 16 | $ ./obj/shm_example_writer /tmp/testq 17 | 18 | ```C 19 | #include 20 | #include 21 | 22 | void append_msg(unsigned char* base, void* msg, size_t sz) { 23 | memcpy(base, msg, sz); 24 | } 25 | 26 | size_t sizeof_msg(void* msg) { 27 | return strlen(msg); 28 | } 29 | 30 | int main(const int argc, char **argv) { 31 | queue_t* queue = chronicle_init(argv[1]); 32 | chronicle_set_encoder(queue, &sizeof_msg, &append_msg); 33 | chronicle_set_version(queue, 5); 34 | chronicle_set_roll_scheme(queue, "FAST_HOURLY"); 35 | chronicle_set_create(queue, 1); 36 | if (chronicle_open(queue) != 0) exit(-1); 37 | 38 | char line[1024]; 39 | while (1) { 40 | char* g = fgets(line, 1024, stdin); 41 | if (g == NULL) break; 42 | line[strlen(line) - 1] = 0; // remove line break 43 | long int index = chronicle_append(queue, line); 44 | printf("[%" PRIu64 "] %s\n", index, (char*)g); 45 | } 46 | chronicle_cleanup(queue); 47 | } 48 | ``` 49 | 50 | Running this code creates a new v5 queue in the directory specified using `FAST_HOURLY` rollover, then writes any text input from stdin as messages. 51 | 52 | The reading side is similar, invoked as `$ ./obj/shm_example_reader /tmp/testq` with source [`shm_example_reader.c`](native/shm_example_reader.c): 53 | 54 | ```C 55 | #include 56 | #include 57 | #include 58 | 59 | static volatile int keepRunning = 1; 60 | 61 | void* parse_msg(unsigned char* base, int lim) { 62 | char* msg = calloc(1, lim+1); 63 | memcpy(msg, base, lim); 64 | return msg; 65 | } 66 | 67 | int print_msg(void* ctx, uint64_t index, void* msg) { 68 | printf("[%" PRIu64 "] %s\n", index, (char*)msg); 69 | return 0; 70 | } 71 | 72 | void sigint_handler(int dummy) { 73 | keepRunning = 0; 74 | } 75 | 76 | int main(const int argc, char **argv) { 77 | signal(SIGINT, sigint_handler); 78 | queue_t* queue = chronicle_init(argv[1]); 79 | chronicle_set_decoder(queue, &parse_msg, &free); 80 | if (chronicle_open(queue) != 0) exit(-1); 81 | chronicle_tailer(queue, &print_msg, NULL, 0); 82 | 83 | while (keepRunning) { 84 | usleep(500*1000); 85 | chronicle_peek(); 86 | } 87 | printf("exiting\n"); 88 | chronicle_cleanup(queue); 89 | } 90 | ``` 91 | 92 | ## Documenting the chronicle-queue format 93 | The format of the chronicle-queue files containing your data, the shared memory protocol, and the safe ordering of queue file maintenance is currently implementation defined. One hope is that this project will change that! 94 | 95 | A chronicle-queue has no single controlling broker process, the operating system provides persistence and hardware itself provides arbitration. Messages always flow from an 'appender' to a 'tailer' process, in one direction, with no flow control. The queue is fully contained with a standard file system directory, which should be otherwise empty. Queue files are machine independant and can be moved between machines. 96 | 97 | * An _appender_ process writes messages to a queue directory, is given 64bit value 'index' of the write 98 | * Whilst a _tailer_ subscribes to messages in a queue directory, starting from 0 or provided index 99 | 100 | Multiple appenders, multiple tailers are supported, can be added and removed at will so long as all on 101 | the same machine. All writes resolve into total order which is preserved on replay. Message length must 102 | be positive and pack into 30 bits, so the maximum size of any individually sequenced payload is `(1<<31)-1` bytes, or 1023 Mb. Typical IPC latency 1us, depending on frequency of polling. 103 | 104 | Each persisted message is identified by a sequential 64-bit index value. Settings control how the 64-bit index value is interpreted, and Chronicle Queue ships with several built-in schemes (`RollCycles`) that split the 64-bit value into two parts at a particular bit position. Upper bits are known as the 'cycle' and the remainder are the 'seqnum'. Cycle values map to a particular queuefile on disk, whilst seqnum indexes the data message within the file. 105 | 106 | The popular _DAILY_ scheme splits the 64-bit index in half, into upper 32 bit cycle and bottom 32 bit are 'seqnum'. The filenames are derived as `cycle+1970.01.01` formatted as `${yyyymmdd}.cq4`. It is critical for the writers and readers to agree on the scheme in use, therefore the scheme is written to the header in each queuefile. 107 | 108 | Queue file writers maintain a double-index structure (also stored within the queue file) to allow resuming from a 109 | particular seqnum value with a reasonable upper bound on the number of disk seeks. The first Metadata message is an array of byte positions within the file of subsequent index pages. Each of those is a Metadata message containing an array (of the same length) containing byte positions of future data messages. The _DAILY_ scheme indexes every 64th message and has 8000 entries per index page. Note that the RollCycle configures the indexing layout. 110 | 111 | Appenders sample the clock during a write to determine if the current cycle should be 'rolled' into the 112 | next. Changing cycle clears seqnum to zero, which causes the index value to "jump" correspondingly and switches the filename in use. A jump to the next cycle also occurs when the indexing structure is full, ie. all entries in the root index point to index pages that are themselves full. 113 | 114 | For performance and correctness, the queue files must be memory mapped. Kernel guarantees 115 | maps into process address space for the same file and offset and made with `MAP_SHARED` by 116 | multiple processes are mapped to the same _physical pages_, which allows shared memory primitives to be used for inter-process communication. 117 | 118 | Memory subsystem ensures correctness between CPUs and packages. To bound the `mmap()` to sensible 119 | sizes, the file is mapped in chunks of 'blocksize' bytes. If a payload is to be written or 120 | deserialised larger than blocksize, blocksize is doubled and the mmap mapping rebuilt. The kernel arranges 121 | dirty pages to be written to disk. Blocking I/O using `read()` and `write()` may see stale data 122 | however filesystem tools (e.g. `cp`) now typically use mmap, so a live copy may be taken. 123 | 124 | Messages are written to the queue with a four-byte header, containing the message size and two control 125 | bits. Writers arbitrate using compare-and-set operations (`lock; cmpxchgl`) on these four bytes to 126 | determine who takes the write lock: 127 | 128 | bits[0-29] 30 31 meaning shmipc.c constant 129 | 0 0 0 available / unallocated HD_UNALLOCATED 0x00000000 130 | size 0 0 data payload 131 | size 1 0 metadata HD_METADATA 0x40000000 132 | pid 0 1 working HD_WORKING 0x80000000 133 | 0 1 1 end-of-file HD_EOF 0xC0000000 134 | 135 | Performing `lock_cmpxchgl(&header, HD_UNALLOCATED, HD_WORKING)` takes the write lock. Data is then 136 | written and header is re-written with the size and working bit clear. x86 (64) preserves ordering 137 | of writes through to read visibility. Tailers need to use an 'mfence' between reading the header 138 | and payload to ensure payload is not prematurely fetched and decoded before the working signal 139 | is clear. mfence used in this way stops both compiler re-ordering and cpu prefetch. 140 | 141 | The `directory-listing.cq4t` file (v4) or `metadata.cq4t` file (v5) is a simple counter of the min and max cycles, which is used 142 | to avoid probing the file system execessively with `stat` system calls during a reply and around the dateroll. It is memory-mapped into all appenders and tailers to see cycle rolls in real time. 143 | 144 | ## Bindings 145 | 146 | This repository contains command line C utilities, as well as language bindings for `libchronicle.so` for: 147 | - [python](bindings/python/) 148 | - [kdb](bindings/kdb/) 149 | 150 | Planned: 151 | - nodejs 152 | 153 | ## Development and Compatability Status 154 | 155 | Queue creation and rollover is implemented for v4 and v5 queues, plus all of the standard roll schemes. 156 | So far as possible our reader and writer examples are compatable with the `InputMain` and `OutputMain` exmaples provided by Chronicle Software Ltd. 157 | 158 | The main missing feature is reading/writing the index structures maintained in the metadata pages. This allows joining an existing queue to be done very cheaply. 159 | 160 | In order to read and write queue metadata, we have a fairly complete implementation of the 'chroicle-wire' serialisation format, implemented by `wire.c` and `wire.h` that might be useful if the data in your queue is written using that format. 161 | 162 | If you can reproduce a segfault on an otherwise valid queuefile, examples would be happily recieved via. a Github Issue. 163 | 164 | ## Unit tests 165 | 166 | To build the test suite first install [cmocka](https://cmocka.org/). 167 | 168 | $ sudo apt install libcmocka-dev libarchive-dev 169 | $ cd native 170 | $ make test 171 | 172 | Some of the test cases use queues written by Chronicle Queue in Java, using test data written by [these Java files](java/). 173 | 174 | Tests can also be run under valgrind: 175 | 176 | $ sudo apt install valgrind 177 | $ make -C native grind 178 | 179 | ## Fuzzer & Coverage 180 | There is basic code coverage reporting using `clang`'s coverage suite, tested on a Mac. This uses shmmain to read and write some entries and shows line by line coverage. Try: 181 | 182 | cd native 183 | make coverage 184 | 185 | There is a very basic automated fuzzing tool (using [AFL - American Fuzzy Lop](http://lcamtuf.coredump.cx/afl/)), which follows a fuzzing script to read and write deterministic payload for a given number of bytes, then jumps the clock by an amount, repeatedly. It writes the indices and seeds for the values written. The tool under test then replays the queuefiles and expects the same payload length, index and payload data to be output. 186 | 187 | $ sudo apt install afl++ 188 | $ make fuzz 189 | 190 | The fuzzing runs quite slowly, due to disk I/O, and also most of AFLs attempts to modify the fuzzing instruction file are invalid. Looks like changing it to a binary rather than ASCII file would help a great deal. 191 | 192 | -------------------------------------------------------------------------------- /bindings/kdb/.gitignore: -------------------------------------------------------------------------------- 1 | k.h 2 | obj 3 | -------------------------------------------------------------------------------- /bindings/kdb/Makefile: -------------------------------------------------------------------------------- 1 | 2 | # k.h from https://raw.githubusercontent.com/KxSystems/kdb/master/c/c/k.h 3 | # apt-get install libcurl4-openssl-dev 4 | 5 | detected_OS := $(shell uname -s) 6 | 7 | IDIR=. 8 | CC=gcc 9 | CFLAGS=-fPIC -I$(IDIR) -Wall 10 | CDFLAGS=-shared 11 | ifeq ($(detected_OS),Darwin) # Mac OS X 12 | CDFLAGS += -undefined dynamic_lookup 13 | endif 14 | ifeq ($(detected_OS),Linux) 15 | CFLAGS += -std=gnu99 16 | endif 17 | 18 | LIBS=-lm 19 | DEPS=wire.h libchronicle.h k.h 20 | 21 | all: obj/hpet.so obj/shmipc.so 22 | 23 | k.h: 24 | wget https://raw.githubusercontent.com/KxSystems/kdb/master/c/c/k.h 25 | 26 | obj/hpet.so: hpet.c k.h 27 | $(CC) -o $@ $< $(CFLAGS) $(CDFLAGS) 28 | 29 | obj/shmipc.so: shmipc.c k.h 30 | $(CC) -o $@ $< $(CFLAGS) $(CDFLAGS) 31 | 32 | syms: 33 | nm -D ../../kdb/l64/q | grep " T " 34 | 35 | .PHONY: clean syms 36 | 37 | clean: 38 | rm -f *~ core $(INCDIR)/*~ default.prof* 39 | rm -Rf $(ODIR)/* 40 | -------------------------------------------------------------------------------- /bindings/kdb/README.md: -------------------------------------------------------------------------------- 1 | 2 | Bindings for KDB/Q 3 | === 4 | 5 | Build/install 6 | --- 7 | 8 | First build and install the libary .so and headers (libchronicle.so and libchronicle.h) by running 9 | 10 | $ cd native 11 | $ make 12 | $ sudo make install 13 | 14 | The `Makefile` in this directory assumes this has been done to be able to build. Next run 15 | 16 | $ make 17 | 18 | Which builds `obj/hpet.so` and `obj/shmipc.so`. These need to be copied into the KDB library directory (e.g. `l64`) or the full path specied the shm.q to work. 19 | 20 | If you dont have permission to write to global libs directories, use `PREFIX=~/usr make install` then customise the Makefile to use your local libs and include directories. 21 | 22 | Examples 23 | --- 24 | 25 | Example `shm-sender.q` writes the start time of the process to a queue every second. `shm-recv.q` will tail this queue and output each value. To show guaranteed delivery, `shm-recv-once.q` uses a second queue to record where in the first queue it reached: 26 | 27 | * shm-recv.q 28 | * shm-recv-once.q 29 | * shm-sender.q 30 | 31 | The second example shows a pair of queues used to push IPC commands to a server, input is in one queue and the output to another: 32 | 33 | * shm-ipc-client.q 34 | * shm-ipc.server.q 35 | 36 | Old example 37 | --- 38 | 39 | / add a tailer by using .shmipc.tailer[`:queue;cb;cycle;decoder] where cycle may be 0 to replay 40 | / from the beginning of time, and cb is the callback for each event in the queue. A replay 41 | / occurs 'inline' and any new records are dispatched by calls to .peek[] 42 | cb:{0N!(x;y)} 43 | .shmipc.tailer[`:java/queue;cb;0]; 44 | .shmipc.debug[]; 45 | 46 | / Note the standard Java wire implementations are largely ignored and returned as byte arrays 47 | / for debug tracing $ export SHMIPC_DEBUG=1 && ./q.sh native/shmipc.q 48 | 49 | / add a souce by .shmipc.appender[`:queue;data] 50 | / multiple appenders may write to the same queue, but there can be no more than one appender per 51 | / process. this is because file locks are used to synchronise some operations and these are issued 52 | / at the 'pid' level. 53 | .shmipc.appender[`:java/queue;"message"]; 54 | 55 | // tailers only advance on calls to poll[], wheres appenders advance on every call to appenders 56 | -------------------------------------------------------------------------------- /bindings/kdb/hpet.c: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Tea Engineering Ltd. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | /** 16 | * high performance event timer, uses timerfd on linux and kqueue/kevent on BSD/mac 17 | * achieves timer callback via. the main thread at an interval greater than the built in 18 | * 1ms KDB Timer. 19 | */ 20 | 21 | #define KXVER 3 22 | #include"k.h" 23 | #include 24 | #include 25 | #include 26 | #include 27 | 28 | #ifdef __linux__ 29 | #include 30 | #include 31 | #elif __APPLE__ 32 | #include 33 | #else 34 | 35 | #endif 36 | 37 | #define MAX_KX_FD 1024 38 | K klookup_cb[MAX_KX_FD]; // map fd index to callback 39 | 40 | K hpet_update(K x, K timespan); 41 | 42 | K read_cb(int fd) { 43 | 44 | K rcb = klookup_cb[fd]; 45 | if (rcb == NULL) return (K)NULL; 46 | // printf("hpet: event fd=%d\n", fd); 47 | 48 | #ifdef __linux__ 49 | uint64_t num_expirations = 0; 50 | ssize_t num_read = 0; 51 | if((num_read = read(fd, &num_expirations, sizeof(uint64_t))) == -1) { 52 | sd0(fd); 53 | return orr("read"); 54 | } 55 | #elif __APPLE__ 56 | struct kevent e; 57 | int res = kevent(fd, 0, 0, &e, 1, 0); // get the event 58 | if (res <= 0) { 59 | printf("no event %d?\n", res); 60 | return krr("no event"); 61 | } 62 | #endif 63 | 64 | // prep args and fire callback 65 | K msg = ki(fd); // don't free this, handed over to q interp 66 | K arg = knk(1, msg); 67 | K r = dot(rcb, arg); 68 | if (r) r0(r); 69 | return ki(0); 70 | } 71 | 72 | K hpet_open(K cb, K timespan) { 73 | if (cb->t != 100) return krr("cb is not function"); 74 | if (timespan->t != -KN) return krr("y timespan expected"); 75 | 76 | int fd; 77 | #ifdef __linux__ 78 | fd = timerfd_create(CLOCK_MONOTONIC, TFD_NONBLOCK | TFD_CLOEXEC); 79 | #elif __APPLE__ 80 | fd = kqueue(); 81 | #endif 82 | if (fd < 0 || fd >= MAX_KX_FD) { 83 | close(fd); 84 | return krr("hpet fd error"); 85 | } 86 | r1(cb); 87 | klookup_cb[fd] = cb; 88 | hpet_update(ki(fd), timespan); 89 | return sd1(fd, &read_cb); // does this return fd? 90 | } 91 | 92 | K hpet_close(K x) { 93 | if (x->t != -KI) return krr("x fd is not int"); 94 | int fd = x->i; 95 | 96 | if (fd < 0 || fd >= MAX_KX_FD) { 97 | close(fd); 98 | return krr("hpet fd error"); 99 | } 100 | printf("hpet: closing timer fd %d\n", fd); 101 | 102 | K rcb = klookup_cb[fd]; 103 | if(rcb == NULL) return krr("not started"); 104 | 105 | sd0(fd); // unlink from selector and close(fd) 106 | r0(rcb); // unref callback 107 | klookup_cb[fd] = NULL; 108 | return ki(fd); 109 | } 110 | 111 | K hpet_update(K x, K timespan) { 112 | if (x->t != -KI) return krr("x fd is not int"); 113 | int fd = x->i; 114 | K rcb = klookup_cb[fd]; 115 | if(rcb == NULL) return krr("x fd not started"); 116 | 117 | if (timespan->t != -KN) return krr("y timespan expected"); 118 | 119 | printf("hpet: setting fd %d timespan to %lluns\n", fd, timespan->j); 120 | 121 | #ifdef __linux__ 122 | struct itimerspec newtimer; 123 | newtimer.it_interval.tv_sec = (timespan->j) / (J)1e9; 124 | newtimer.it_interval.tv_nsec = timespan->j%(J)1e9; 125 | newtimer.it_value.tv_sec = (timespan->j) / (J)1e9; 126 | newtimer.it_value.tv_nsec = timespan->j%(J)1e9; 127 | if (timerfd_settime(fd, 0, &newtimer, NULL) != 0) { 128 | orr("hpet update error"); 129 | } 130 | #elif __APPLE__ 131 | struct kevent e; 132 | EV_SET(&e, 1, EVFILT_TIMER, EV_ADD | EV_ENABLE, NOTE_NSECONDS, timespan->j, 0); 133 | if (kevent(fd, &e, 1, 0, 0, 0) != 0) { 134 | orr("hpet update kevent error"); 135 | } 136 | #endif 137 | return (K) 0; 138 | } 139 | -------------------------------------------------------------------------------- /bindings/kdb/hpet.q: -------------------------------------------------------------------------------- 1 | / High performance event timer 2 | / uses timerfd on linux or kevent/kqueue on BSD/Mac 3 | 4 | .timer.hpet_open:`:native/obj/hpet 2:(`hpet_open;2) 5 | .timer.hpet_update:`:native/obj/hpet 2:(`hpet_update;2) 6 | .timer.hpet_close:`:native/obj/hpet 2:(`hpet_close;1) 7 | 8 | cb:{show .z.p;} 9 | 10 | // print every 0.5s 11 | fd:.timer.hpet_open[cb; 0D00:00:00.500000000]; 12 | .timer.hpet_update[fd; 0D00:00:00.250000000]; 13 | 14 | // stop printing timer after 5 seconds 15 | cb2:{.timer.hpet_close[fd];.timer.hpet_close[fd2];} 16 | fd2:.timer.hpet_open[cb2;0D00:00:05.000000000]; 17 | 18 | // exit after 10 seconds 19 | cb3:{exit 42;} 20 | fd3:.timer.hpet_open[cb3; 0D00:00:10.000000000]; 21 | 22 | 23 | -------------------------------------------------------------------------------- /bindings/kdb/shm-ipc-client.q: -------------------------------------------------------------------------------- 1 | \l shm.q 2 | 3 | .shmipc.init[`:java/in;`kx;5;0b]; 4 | .shmipc.init[`:java/out;`kx;5;0b]; 5 | 6 | / fd:.timer.hpet_open[{.shmipc.peek[]}; 0D00:00:00.010000000]; 7 | hi:.shmipc.tailer[`:java/out;{[x;y]};0]; 8 | 9 | shmquery:{[x] 10 | wr:.shmipc.append[`:java/in;x]; 11 | tmp:(-1;(0;0)); / (outseq;(inseq;result)); 12 | while[tmp[1][0]<>wr;tmp:.shmipc.collect[hi]]; 13 | :tmp[1][1] 14 | }; 15 | 16 | shmquery["0"]; / round trip to create queuefiles etc 17 | 18 | /shmquery["4+12"]; 19 | n:1000; 20 | \t:n shmquery["12+34"]; 21 | 22 | h: hopen `::12345; 23 | \t:n h"12+34"; 24 | 25 | -------------------------------------------------------------------------------- /bindings/kdb/shm-ipc-server.q: -------------------------------------------------------------------------------- 1 | \l shm.q 2 | \l hpet.q 3 | 4 | \p 12345 5 | 6 | .shmipc.init[`:java/in;`kx;5;1b]; 7 | .shmipc.init[`:java/out;`kx;5;1b]; 8 | fd:.timer.hpet_open[{do[100;.shmipc.peek[]];}; 0D00:00:00.000001000]; 9 | cb:{.shmipc.append[`:java/out;(x;value y)];} 10 | .shmipc.tailer[`:java/in;cb;0]; 11 | -------------------------------------------------------------------------------- /bindings/kdb/shm-recv-once.q: -------------------------------------------------------------------------------- 1 | \l shm.q 2 | 3 | // Recovery logic 4 | .pos:0 5 | 6 | .shmipc.init[`:java/queueout;`kx;5;1b]; 7 | rcb:{.pos:y} 8 | .shmipc.tailer[`:java/queueout;rcb;0]; 9 | .shmipc.peek[] // replay the outbound log in a "blocking" style 10 | 11 | show " " sv ("recovered position is";string .pos) 12 | .shmipc.close[`:java/queueout] // kill recovery tailer 13 | 14 | // Regular application here 15 | .shmipc.init[`:java/queue;`text;5;0b]; // input 16 | .shmipc.init[`:java/queueout;`kx;5;1b]; // output 17 | 18 | // handler for new input pushes corresponding index to output 19 | cb:{ 0N!(x;y);.shmipc.append[`:java/queueout;x]; } 20 | 21 | fd:.timer.hpet_open[{.shmipc.peek[]}; 0D00:00:00.010000000]; 22 | .shmipc.tailer[`:java/queue;cb;.pos] 23 | -------------------------------------------------------------------------------- /bindings/kdb/shm-recv.q: -------------------------------------------------------------------------------- 1 | \l shm.q 2 | 3 | .shmipc.init[`:java/queue;`text;5;0b]; 4 | 5 | fd:.timer.hpet_open[{.shmipc.peek[]}; 0D00:00:00.010000000]; 6 | 7 | cb:{0N!(x;y)} 8 | .shmipc.tailer[`:java/queue;cb;0]; 9 | -------------------------------------------------------------------------------- /bindings/kdb/shm-sender.q: -------------------------------------------------------------------------------- 1 | \l shm.q 2 | 3 | .shmipc.init[`:java/queue;`text;5;1b]; 4 | 5 | \t 1000 6 | 7 | started:string .z.p; 8 | 9 | .z.ts:{.shmipc.append[`:java/queue;" -> " sv (started;string .z.p)];} 10 | -------------------------------------------------------------------------------- /bindings/kdb/shm.q: -------------------------------------------------------------------------------- 1 | 2 | .shmipc.init2:`:native/obj/shmipc 2:(`shmipc_init;4) 3 | .shmipc.close:`:native/obj/shmipc 2:(`shmipc_close;1) 4 | .shmipc.peek:`:native/obj/shmipc 2:(`shmipc_peek;1) 5 | .shmipc.tailer:`:native/obj/shmipc 2:(`shmipc_tailer;3) 6 | .shmipc.append_ts:`:native/obj/shmipc 2:(`shmipc_append_ts;3) 7 | .shmipc.append_raw:`:native/obj/shmipc 2:(`shmipc_append;2) 8 | .shmipc.debug:`:native/obj/shmipc 2:(`shmipc_debug;1) 9 | 10 | .shmipc.dto:`long$2000.01.01D00:00:00.000-1970.01.01D00:00:00.000 11 | .shmipc.ctmillis:{(.shmipc.dto+`long$.z.p) div 1000000} 12 | 13 | .shmipc.init:{[dir;fmt;ver;create] 14 | // check if the queue directory exists, create if missing 15 | system " " sv ("mkdir";"-p";1_string dir); 16 | .shmipc.init2[dir;fmt;ver;create] 17 | } 18 | 19 | .shmipc.append:{[dir;msg] 20 | .shmipc.append_ts[dir;msg;.shmipc.ctmillis[]] 21 | } 22 | -------------------------------------------------------------------------------- /bindings/kdb/shmipc.c: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Tea Engineering Ltd. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // 16 | // This file builds a shared-object "shmipc.so" for easy integration of 17 | // libchronicle with the KDB/q intepreter from Kx. 18 | // It does not incorporate mock_k.h as the real 19 | // implementation of those symbols is provided by the KDB runtime. 20 | // 21 | // The KDB calling convention requires all functions to return a K object 22 | // and all arguments are provided as K objects, so these wrappers take care 23 | // of that. 24 | // 25 | // We provide a simple way to select the protocol of the queue, either 26 | // "text", "raw", "kx" or "wire". 27 | // 28 | // *text* writes bytes directly to the queue data messages, requiring and 29 | // returning messages bytes as characters in type KG 30 | // 31 | // *raw* writes and returns byte arrays of type KB - this is compatable with 32 | // anything Java might produce, however will require some unpickling. 33 | // 34 | // *kx* uses d9/b9 to use KDB's built in serialisers. This is your best bet 35 | // if serialising between two KDB processes, or if you want to use Kx objects 36 | // tables etc. Remember the max data size is 1GB. 37 | // 38 | // *wire* uses wire.h to write some KDB values using the Chronicle-Wire 39 | // "self-describing" format. This limits what you can do but is easiest for 40 | // cross-platform compatability. 41 | // 42 | 43 | #define _GNU_SOURCE 44 | #define KERR -128 45 | 46 | #define KXVER 3 47 | #include "k.h" 48 | 49 | #include 50 | #include 51 | #include 52 | 53 | // Usual KDB indirection via arrays, since 54 | // returning a void* to the runtime isn't an option, and there's 55 | // no nice way to store it into a K object directly. 56 | // return handle integer to index structure 57 | tailer_t** tailer_handles; 58 | int tailer_handles_count = 0; 59 | queue_t** queue_handles; 60 | int queue_handles_count = 0; 61 | 62 | COBJ parse_kx(unsigned char* base, int lim) { 63 | // prep args and fire callback 64 | K msg = ktn(KG, lim); 65 | memcpy((char*)msg->G0, base, lim); 66 | int ok = okx(msg); 67 | if (ok) { 68 | K out = d9(msg); 69 | r0(msg); 70 | return out; 71 | } else { 72 | // if (debug) printf("shmipc: caution index okx returned bytes !ok, skipping\n"); 73 | return NULL; 74 | } 75 | return 0; 76 | } 77 | 78 | // the encoding via. b9 happens in shmipc_append, so here we just 79 | // write the bytes 80 | void append_kx(unsigned char* base, COBJ msg, size_t lim) { 81 | K m = (K)msg; 82 | memcpy(base, (char*)m->G0, m->n); 83 | } 84 | 85 | size_t sizeof_kx(COBJ msg) { 86 | // if (debug) printf("shmipc: kx persist needs %lld bytes\n", msg->n); 87 | K m = (K)msg; 88 | return m->n; 89 | } 90 | 91 | void free_kx(COBJ msg) { 92 | K m = (K)msg; 93 | r0(m); 94 | } 95 | 96 | int tailer_callback_kx(void* dispatch_ctx, uint64_t index, COBJ m) { 97 | K msg = (K)m; 98 | K arg = knk(2, kj(index), msg); 99 | 100 | K r = dot((K)dispatch_ctx, arg); 101 | r0(arg); 102 | if (r == NULL) { 103 | printf(" shmipc: caution, error signalled by callback (returned NULL)\n"); 104 | return 0; 105 | } else if (r && r->t == KERR) { 106 | printf(" shmipc: callback error string: %s\n", r->s); 107 | } 108 | r0(r); 109 | return 0; 110 | } 111 | 112 | K shmipc_init(K dir, K parser, K ver, K create) { 113 | if (dir->t != -KS) return krr("dir is not symbol"); 114 | if (dir->s[0] != ':') return krr("dir is not symbol handle (starts with :)"); 115 | if (ver->t != -KJ) return krr("ver is not j"); 116 | if (create->t != -KB) return krr("create is not bool"); 117 | 118 | // if (strncmp(parser->s, "text", parser->n) == 0) { 119 | // queue->parser = &parse_text; 120 | // queue->encoder = &append_text; 121 | // queue->encodecheck = &sizeof_text; 122 | // } else if (strncmp(parser->s, "kx", parser->n) == 0) { 123 | // queue->parser = &parse_kx; 124 | // queue->encoder = &append_kx; 125 | // queue->encodecheck = &sizeof_kx; 126 | // } else { 127 | // return krr("bad format: supports `kx and `text"); 128 | // } 129 | 130 | char* dirs = &dir->s[1]; 131 | queue_t* queue = chronicle_init(dirs); 132 | if (strncmp(parser->s, "text", parser->n) == 0) { 133 | chronicle_set_decoder(queue, &parse_kx, &free_kx); 134 | chronicle_set_encoder(queue, &sizeof_kx, &append_kx); 135 | } else if (strncmp(parser->s, "kx", parser->n) == 0) { 136 | chronicle_set_decoder(queue, &parse_kx, &free_kx); 137 | chronicle_set_encoder(queue, &sizeof_kx, &append_kx); 138 | } else { 139 | return krr("bad format: supports `kx and `text"); 140 | } 141 | chronicle_set_version(queue, ver->i); 142 | chronicle_set_create(queue, create->i); 143 | chronicle_open(queue); 144 | 145 | int handle = -1; 146 | if (queue) { 147 | // maintain array for index lookup 148 | handle = queue_handles_count; 149 | queue_handles = realloc(queue_handles, ++queue_handles_count * sizeof(queue_t*)); 150 | queue_handles[handle] = queue; 151 | } 152 | return ki(handle); 153 | } 154 | 155 | K shmipc_append_ts(K queuei, K msg, K ms) { 156 | if (queuei->t != -KI) return krr("queue is not int"); 157 | if (queuei->i < 0 || queuei->i >= queue_handles_count) return krr("queue out of range"); 158 | queue_t *queue = queue_handles[queuei->i]; 159 | if (queue == NULL) return krr("queue closed"); 160 | 161 | if (ms != NULL && ms->t != -KJ) return krr("ms NULL or J milliseconds"); 162 | 163 | long millis = -1; 164 | if (ms) millis = ms->j; 165 | 166 | // convert msg based on the format selected 167 | // TODO: support text etc etc? 168 | K enc = b9(3, msg); 169 | if (enc == NULL) { 170 | return krr("shmipc: failed to serialise msg using b9"); 171 | } 172 | 173 | int j = chronicle_append_ts(queue, enc, millis); 174 | // TODO check return error 175 | r0(enc); 176 | return kj(j); 177 | } 178 | 179 | K shmipc_append(K queuei, K msg) { 180 | return shmipc_append_ts(queuei, msg, NULL); 181 | } 182 | 183 | K shmipc_tailer(K queuei, K callback, K index) { 184 | if (queuei->t != -KI) return krr("queue is not int"); 185 | if (queuei->i < 0 || queuei->i >= queue_handles_count) return krr("queue out of range"); 186 | queue_t *queue = queue_handles[queuei->i]; 187 | if (queue == NULL) return krr("queue closed"); 188 | 189 | if (index->t != -KJ) return krr("index must be J"); 190 | // if (callback-> != function) return krr("dispatcher is not a callback"); 191 | 192 | tailer_t* tailer = chronicle_tailer(queue, &tailer_callback_kx, callback, index->j); 193 | 194 | // maintain array for index lookup 195 | int handle = tailer_handles_count; 196 | tailer_handles = realloc(tailer_handles, ++tailer_handles_count * sizeof(tailer_t*)); 197 | tailer_handles[handle] = tailer; 198 | K r = ki(handle); 199 | 200 | r0(index); 201 | return r; 202 | } 203 | 204 | K shmipc_collect(K taileri) { 205 | if (taileri->t != -KI) return krr("idx is not int"); 206 | if (taileri->i < 0 || taileri->i >= tailer_handles_count) return krr("idx out of range"); 207 | tailer_t* tailer = tailer_handles[taileri->i]; 208 | collected_t result; 209 | 210 | K r = chronicle_collect(tailer, &result); 211 | return r; 212 | } 213 | 214 | K shmipc_close(K queuei) { 215 | if (queuei->t != -KI) return krr("queue is not int"); 216 | if (queuei->i < 0 || queuei->i >= queue_handles_count) return krr("queue out of range"); 217 | queue_t *queue = queue_handles[queuei->i]; 218 | if (queue == NULL) return krr("queue already closed"); 219 | 220 | queue_handles[queuei->i] = NULL; 221 | int r = chronicle_cleanup(queue); 222 | 223 | return kj(r); 224 | } 225 | 226 | K shmipc_peek(K x) { 227 | chronicle_peek(); 228 | return (K)0; 229 | } 230 | -------------------------------------------------------------------------------- /bindings/python/README.md: -------------------------------------------------------------------------------- 1 | ## Getting started 2 | 3 | If you haven't already built the native code to the shared libary `libchronicle.so`), build it with: 4 | 5 | $ git clone git@github.com:TeaEngineering/libchronicle.git 6 | $ cd libchronicle/native 7 | $ make 8 | 9 | The python bindings for `libchronicle.so` are use the `ctypes` module which is built into python. 10 | 11 | One (or more) example [writers](writer.py) can be started with: 12 | 13 | $ cd ../bindings/python/ 14 | $ python3 writer.py /tmp/q4 15 | chronicle: detected version v0 16 | hello 17 | 82841329205248 18 | world 19 | 82841329205249 20 | ^CTraceback (most recent call last): 21 | File "/home/chris/repos/libchronicle/bindings/python/writer.py", line 9, in 22 | for line in sys.stdin: 23 | KeyboardInterrupt 24 | 25 | 26 | And the [reading side](reader_cb.py) with: 27 | 28 | $ python3 reader_cb.py /tmp/q4 29 | chronicle: detected version v5 30 | [82841329205248] b'hello' 31 | [82841329205249] b'world' 32 | 33 | TODO: publish to pypi 34 | -------------------------------------------------------------------------------- /bindings/python/bulkwriter.py: -------------------------------------------------------------------------------- 1 | import libchronicle 2 | import os 3 | import sys 4 | import random 5 | import string 6 | from time import sleep 7 | 8 | path = sys.argv[1] 9 | os.makedirs(path, mode=0o777, exist_ok=True) 10 | 11 | with libchronicle.Queue(path, version=5, create=True, roll_scheme="TEST_SECONDLY") as q: 12 | for i in range(int(1e8)): 13 | N = random.randrange(1, 10000) 14 | line = "".join(random.choices(string.ascii_uppercase + string.digits, k=N)) 15 | print(q.append(line.rstrip().encode())) 16 | sleep(random.randrange(1, 100) / 1000.0) 17 | -------------------------------------------------------------------------------- /bindings/python/libchronicle.py: -------------------------------------------------------------------------------- 1 | from ctypes import ( 2 | c_void_p, 3 | c_long, 4 | c_longlong, 5 | c_int, 6 | c_char_p, 7 | POINTER, 8 | CFUNCTYPE, 9 | Structure, 10 | cdll, 11 | byref, 12 | string_at, 13 | ) 14 | from ctypes.util import find_library 15 | from typing import Optional 16 | import os 17 | 18 | lib = find_library("chronicle") 19 | if lib is None: 20 | # setup default lib location relative to script 21 | root_path = os.path.dirname( 22 | os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 23 | ) 24 | lib = os.path.join(root_path, "native", "obj", "libchronicle.so") 25 | 26 | cx = cdll.LoadLibrary(lib) 27 | 28 | 29 | # typedef struct { 30 | # COBJ msg; 31 | # size_t sz; 32 | # uint64_t index; 33 | # } collected_t; 34 | class Collected(Structure): 35 | _fields_ = [("msg", c_void_p), ("sz", c_long), ("index", c_longlong)] 36 | 37 | # ret arg0 arg1 arg2 38 | TAILER_CB = CFUNCTYPE(c_int, c_void_p, c_longlong, c_char_p) 39 | 40 | cx.chronicle_init.argtype = c_char_p 41 | cx.chronicle_init.restype = c_void_p 42 | cx.chronicle_set_version.argtypes = [c_void_p, c_int] 43 | cx.chronicle_set_version.restype = None 44 | cx.chronicle_get_version.argtypes = [c_void_p] 45 | cx.chronicle_get_version.restype = c_int 46 | 47 | cx.chronicle_set_create.argtypes = [c_void_p, c_int] 48 | cx.chronicle_set_create.restype = None 49 | 50 | cx.chronicle_set_roll_scheme.argtypes = [c_void_p, c_char_p] 51 | cx.chronicle_set_roll_scheme.restype = c_int 52 | 53 | cx.chronicle_get_roll_scheme.argtypes = [c_void_p] 54 | cx.chronicle_get_roll_scheme.restype = c_char_p 55 | 56 | cx.chronicle_open.argtypes = [c_void_p] 57 | cx.chronicle_open.restype = c_int 58 | 59 | cx.chronicle_cleanup.argtypes = [c_void_p] 60 | cx.chronicle_cleanup.restype = c_int 61 | 62 | cx.chronicle_append.argtypes = [c_void_p, c_char_p] 63 | cx.chronicle_append.restype = c_longlong 64 | 65 | cx.chronicle_tailer.argtypes = [c_void_p, c_void_p, c_void_p, c_longlong] 66 | cx.chronicle_tailer.restype = c_void_p 67 | 68 | cx.chronicle_collect.argtypes = [c_void_p, POINTER(Collected)] 69 | cx.chronicle_collect.restype = c_longlong 70 | 71 | cx.chronicle_strerror.argtypes = [] 72 | cx.chronicle_strerror.restype = c_char_p 73 | 74 | cx.chronicle_debug.argtypes = [] 75 | 76 | cx.chronicle_peek_queue.argtypes = [c_void_p] 77 | cx.chronicle_peek_queue.restype = None 78 | 79 | cx.chronicle_peek_tailer.argtypes = [c_void_p] 80 | cx.chronicle_peek_tailer.restype = None 81 | 82 | 83 | class Queue: 84 | def __init__( 85 | self, 86 | directory: str, 87 | create: bool = False, 88 | version: int = 0, 89 | roll_scheme: Optional[str] = None, 90 | ): 91 | self.q = cx.chronicle_init(directory.encode()) 92 | if create: 93 | cx.chronicle_set_create(self.q, 1) 94 | if version != 0: 95 | cx.chronicle_set_version(self.q, version) 96 | if roll_scheme is not None: 97 | rc = cx.chronicle_set_roll_scheme(self.q, roll_scheme.encode()) 98 | if rc != 0: 99 | raise ValueError(f"No such roll_scheme {roll_scheme}") 100 | 101 | def __enter__(self): 102 | rc = cx.chronicle_open(self.q) 103 | if rc != 0: 104 | raise ValueError(f"Open failed: {cx.chronicle_strerror()}") 105 | return self 106 | 107 | def __exit__(self, type, value, tb): 108 | rc = cx.chronicle_cleanup(self.q) 109 | if rc != 0: 110 | raise ValueError(f"Close failed: {cx.chronicle_strerror()}") 111 | 112 | def debug(self): 113 | cx.chronicle_debug() 114 | 115 | def append(self, data: bytes): 116 | return cx.chronicle_append(self.q, data) 117 | 118 | def tailer(self, index: int = 0, cb=None): 119 | # for now we don't use callback api, just blocking collect 120 | return Tailer(self, index, cb) 121 | 122 | def peek(self): 123 | cx.chronicle_peek_queue(self.q) 124 | 125 | 126 | class Tailer: 127 | def __init__(self, queue: Queue, index:int = 0, cb=None): 128 | self.cb_func = None 129 | if cb: 130 | self.cb_func = TAILER_CB(cb) 131 | self.tailer = cx.chronicle_tailer(queue.q, self.cb_func, None, index) 132 | self.collected = Collected() 133 | 134 | def __enter__(self): 135 | return self 136 | 137 | def __exit__(self, type, value, tb): 138 | pass 139 | 140 | def collect(self, timeout=None): 141 | # TODO: this blocks forever inside the native code, which 142 | # prevents ctl-C from working 143 | cx.chronicle_collect(self.tailer, byref(self.collected)) 144 | data = string_at(self.collected.msg, self.collected.sz) 145 | ## cx.chronicle_return(self.tailer, self.collected) 146 | return (self.collected.index, data) 147 | 148 | def peek(self): 149 | cx.chronicle_peek_tailer(self.tailer) 150 | -------------------------------------------------------------------------------- /bindings/python/reader.py: -------------------------------------------------------------------------------- 1 | import libchronicle 2 | import sys 3 | 4 | path = sys.argv[1] 5 | with libchronicle.Queue(path) as q: 6 | with q.tailer(0) as tailer: 7 | while True: 8 | index, bs = tailer.collect() 9 | print(f"[{index}] {bs}") 10 | -------------------------------------------------------------------------------- /bindings/python/reader_cb.py: -------------------------------------------------------------------------------- 1 | import libchronicle 2 | import sys 3 | 4 | 5 | def printmsg(ctx, index, bs): 6 | print(f"[{index}] {bs}") 7 | return 0 8 | 9 | 10 | path = sys.argv[1] 11 | with libchronicle.Queue(path) as q: 12 | tailer = q.tailer(0, cb=printmsg) 13 | while True: 14 | q.peek() 15 | -------------------------------------------------------------------------------- /bindings/python/writer.py: -------------------------------------------------------------------------------- 1 | import libchronicle 2 | import os 3 | import sys 4 | 5 | path = sys.argv[1] 6 | os.makedirs(path, mode=0o777, exist_ok=True) 7 | 8 | with libchronicle.Queue(path, version=5, create=True, roll_scheme="DAILY") as q: 9 | for line in sys.stdin: 10 | print(q.append(line.rstrip().encode())) 11 | -------------------------------------------------------------------------------- /java/.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | queue 3 | -------------------------------------------------------------------------------- /java/README.md: -------------------------------------------------------------------------------- 1 | 2 | ### Command line examples - Java 3 | 4 | InputMain.java and OutputMain.java are provided from: 5 | https://github.com/OpenHFT/Chronicle-Queue-Sample/tree/master/simple-input/src/main/java/net/openhft/chronicle/queue/simple/input 6 | 7 | To run an example appender: 8 | 9 | (mac) $ brew install maven 10 | (ubuntu) $ sudo apt-get install maven 11 | 12 | Then 13 | 14 | $ mvn dependency:copy-dependencies 15 | $ mvn package 16 | $ java -cp target/java-1.0-SNAPSHOT.jar:target/dependency/chronicle-queue-5.20.102.jar mains.InputMain cqv5 < sample.input 17 | type something 18 | [81346680586240] one 19 | [81346680586241] two 20 | [81346680586242] three 21 | Input stream finished 22 | $ 23 | 24 | InputMain will create a directory called 'q1' in the current directory containing a queuefile (`*.cq4`) and a `metadata.cq4t` file. The file will contain three messages in the current cycle. 25 | 26 | For the tailer: 27 | 28 | $ java -cp target/java-1.0-SNAPSHOT.jar:target/dependency/chronicle-queue-5.20.102.jar mains.OutputMain q1 2>/dev/null 29 | [81346680586240] one 30 | [81346680586241] two 31 | [81346680586242] three 32 | $ 33 | 34 | ### Interoperability 35 | 36 | Java write, libchronicle read: 37 | 38 | $ make -C ../native 39 | $ ../native/obj/shmmain :q1/ -d 2>/dev/null 40 | [81346680586240] one 41 | [81346680586241] two 42 | [81346680586242] three 43 | $ 44 | 45 | libchronicle write, Java read: 46 | 47 | TODO: support wire encoding for text round trip to work! 48 | $ ../native/obj/shmmain :q1/ -d -a HELLO 49 | $ ../native/obj/shmmain :q1/ -d 50 | $ java -cp target/java-1.0-SNAPSHOT.jar:target/dependency/chronicle-queue-5.20.102.jar mains.OutputMain q1 51 | 52 | ## Adding test data for unit tests 53 | 54 | gzip doesn't compress megabytes of zeros efficiently - see Mark Adlers reply on 55 | https://stackoverflow.com/questions/16792189/gzip-compression-ratio-for-zeros 56 | 57 | bzip2 works well, and can be unpacked by `libarchive` in unit tests: 58 | 59 | ## Generating CQV5 test data 60 | 61 | Build as above then: 62 | 63 | $ java -cp target/java-1.0-SNAPSHOT.jar:target/dependency/chronicle-queue-5.20.102.jar mains.InputMain qv5 < sample.input 64 | $ tar -cvf - qv5 | bzip2 --best > ../native/test/cqv5-sample-input.tar.bz2 65 | 66 | 67 | ### Generating CQV4 test data 68 | 69 | CQV4 needs quite an old JVM to run, JDK 10 is the maximum. Under JDK11, or without the extra `--add-exports` options it throws `java.lang.NoSuchFieldException: reservedMemory` due to the reflection calls failing. I've included a known good POM file for this combination: 70 | 71 | $ export JAVA_HOME=~/java/jdk-10.0.2/ 72 | $ mvn -B clean dependency:copy-dependencies package --file pom.cqv4.xml 73 | $ $JAVA_HOME/bin/java -cp target/java-1.0-SNAPSHOT.jar:target/dependency/chronicle-queue-4.6.109.jar --add-exports java.base/jdk.internal.ref=ALL-UNNAMED --add-exports java.base/jdk.internal.misc=ALL-UNNAMED --add-exports java.base/sun.nio.ch=ALL-UNNAMED mains.InputMain cqv4 < sample.input 74 | $ tar -cvf - cqv4 | bzip2 --best > ../native/test/cqv4-sample-input.tar.bz2 75 | 76 | -------------------------------------------------------------------------------- /java/java.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /java/pom.cqv4.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | mains 8 | java 9 | 1.0-SNAPSHOT 10 | 11 | jar 12 | 13 | 14 | 15 | 16 | net.openhft 17 | chronicle-queue 18 | 4.6.109 19 | 20 | 21 | 22 | junit 23 | junit 24 | 4.13.1 25 | test 26 | 27 | 28 | 29 | 30 | org.slf4j 31 | slf4j-simple 32 | 1.7.25 33 | test 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | org.apache.maven.plugins 42 | maven-compiler-plugin 43 | 44 | -Xlint:deprecation 45 | 1.8 46 | 1.8 47 | UTF-8 48 | 49 | 50 | 51 | 52 | -------------------------------------------------------------------------------- /java/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | mains 8 | java 9 | 1.0-SNAPSHOT 10 | 11 | jar 12 | 13 | 14 | 15 | 16 | net.openhft 17 | chronicle-queue 18 | 5.20.102 19 | 20 | 21 | 22 | net.openhft 23 | chronicle-core 24 | 2.20.102 25 | 26 | 27 | 28 | junit 29 | junit 30 | 4.13.1 31 | test 32 | 33 | 34 | 35 | 36 | org.slf4j 37 | slf4j-simple 38 | 1.7.25 39 | test 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | org.apache.maven.plugins 48 | maven-compiler-plugin 49 | 50 | -Xlint:deprecation 51 | 1.8 52 | 1.8 53 | UTF-8 54 | 55 | 56 | 57 | 58 | -------------------------------------------------------------------------------- /java/sample.input: -------------------------------------------------------------------------------- 1 | one 2 | two 3 | three 4 | a much longer item that will need encoding as variable length text -------------------------------------------------------------------------------- /java/src/main/java/mains/InputMain.java: -------------------------------------------------------------------------------- 1 | package mains; 2 | 3 | import net.openhft.chronicle.queue.ExcerptAppender; 4 | import net.openhft.chronicle.queue.impl.single.SingleChronicleQueue; 5 | import net.openhft.chronicle.queue.impl.single.SingleChronicleQueueBuilder; 6 | 7 | import java.util.Scanner; 8 | 9 | /** 10 | * Created by catherine on 17/07/2016. 11 | */ 12 | public class InputMain { 13 | public static void main(String[] args) { 14 | if (args.length < 1) { 15 | System.err.println("InputMain QUEUE"); 16 | System.exit(1); 17 | } 18 | String path = args[0]; 19 | SingleChronicleQueue queue = SingleChronicleQueueBuilder.binary(path).build(); 20 | ExcerptAppender appender = queue.acquireAppender(); 21 | Scanner read = new Scanner(System.in); 22 | System.out.println("Input text:"); 23 | while(read.hasNextLine()) { 24 | String line = read.nextLine(); 25 | if (line.isEmpty()) 26 | break; 27 | appender.writeText(line); 28 | System.out.println("[" + appender.lastIndexAppended() + "] " + line); 29 | } 30 | System.out.println("Input stream finished"); 31 | } 32 | } -------------------------------------------------------------------------------- /java/src/main/java/mains/OutputMain.java: -------------------------------------------------------------------------------- 1 | package mains; 2 | 3 | import net.openhft.chronicle.core.Jvm; 4 | import net.openhft.chronicle.queue.ExcerptTailer; 5 | import net.openhft.chronicle.queue.impl.single.SingleChronicleQueue; 6 | import net.openhft.chronicle.queue.impl.single.SingleChronicleQueueBuilder; 7 | 8 | /** 9 | * Created by catherine on 17/07/2016. 10 | */ 11 | public class OutputMain { 12 | public static void main(String[] args) { 13 | if (args.length < 1) { 14 | System.err.println("OutputMain QUEUE"); 15 | System.exit(1); 16 | } 17 | String path = args[0]; 18 | SingleChronicleQueue queue = SingleChronicleQueueBuilder.binary(path).build(); 19 | ExcerptTailer tailer = queue.createTailer(); 20 | 21 | while (true) { 22 | long index = tailer.index(); 23 | String text = tailer.readText(); 24 | if (text == null) { 25 | Jvm.pause(10); 26 | } else { 27 | System.out.println("[" + index + "] " + text); 28 | } 29 | } 30 | 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /native/Makefile: -------------------------------------------------------------------------------- 1 | 2 | # apt-get install libcurl4-openssl-dev 3 | 4 | detected_OS := $(shell uname -s) 5 | 6 | IDIR=. 7 | CC=gcc 8 | CFLAGS=libchronicle.c wire.c buffer.c -fPIC -I$(IDIR) -Wall 9 | CDFLAGS=-shared 10 | ifeq ($(detected_OS),Darwin) # Mac OS X 11 | CDFLAGS += -undefined dynamic_lookup 12 | endif 13 | ifeq ($(detected_OS),Linux) 14 | CFLAGS += -std=gnu99 15 | endif 16 | 17 | ODIR=obj 18 | LIBS=-lm 19 | DEPS=wire.h wire.c libchronicle.h libchronicle.c buffer.h buffer.c 20 | 21 | ifeq ($(PREFIX),) 22 | PREFIX := /usr 23 | endif 24 | 25 | tests := $(patsubst %.c,%.to,$(wildcard test/test*.c)) 26 | tests_ok := $(patsubst %.c,%.tok,$(wildcard test/test*.c)) 27 | tests_vg := $(patsubst %.c,%.tvg,$(wildcard test/test*.c)) 28 | 29 | all: obj/shmmain obj/shm_example_reader obj/shm_example_writer obj/libchronicle.so 30 | 31 | $(ODIR)/%.so: $(DEPS) 32 | $(CC) -o $@ $< $(CFLAGS) $(CDFLAGS) 33 | 34 | %.to: %.c test/testdata.h $(DEPS) 35 | $(CC) -o $@ $< $(CFLAGS) -g -O0 -lcmocka -larchive 36 | 37 | %.tok: %.to 38 | $< && touch $@ 39 | 40 | %.tvg: %.to 41 | valgrind --track-origins=yes --leak-check=full $< && touch $@ 42 | 43 | 44 | $(ODIR)/shmmain: shmmain.c $(DEPS) 45 | $(CC) -o $@ $< $(CFLAGS) -g -O0 46 | 47 | $(ODIR)/shm_example%: shm_example%.c $(DEPS) 48 | $(CC) -o $@ $< $(CFLAGS) -g -O0 49 | 50 | coverage: obj/shmcov 51 | rm -Rf test/coverage_queue 52 | mkdir -p test/coverage_queue 53 | obj/shmcov -v -c -a WORLD test/coverage_queue 54 | llvm-profdata merge default.profraw -output=default.profout 55 | llvm-cov show -instr-profile default.profout obj/shmcov 56 | 57 | $(ODIR)/shmcov: shmmain.c $(DEPS) 58 | clang -o $@ $< $(CFLAGS) -fprofile-instr-generate -fcoverage-mapping -O0 -g 59 | 60 | k.h: 61 | wget https://raw.githubusercontent.com/KxSystems/kdb/master/c/c/k.h 62 | 63 | $(ODIR)/fuzzmain: fuzzmain.c k.h mock_k.h $(DEPS) 64 | afl-clang -o $@ $< $(CFLAGS) -g -O0 65 | 66 | fuzz: $(ODIR)/fuzzmain 67 | # brew install afl-fuzz 68 | # sudo apt install afl++ 69 | rm -Rf test/fuzz_output/* 70 | rm -Rf test/fuzz_queue/ 71 | mkdir -p test/fuzz_output test/fuzz_queue 72 | AFL_SKIP_CPUFREQ=1 afl-fuzz -i test/fuzz_input -o test/fuzz_output $(ODIR)/fuzzmain test/fuzz_queue - 73 | 74 | 75 | .PHONY: clean grind coverage fuzz test install 76 | 77 | test: $(tests_ok) 78 | 79 | grind: $(tests_vg) 80 | 81 | clean: 82 | rm -f *~ core $(INCDIR)/*~ default.prof* 83 | rm -Rf $(ODIR)/* 84 | rm -Rf test/*.to test/*.tok test/*.tvg 85 | 86 | install: obj/libchronicle.so 87 | install -d $(DESTDIR)$(PREFIX)/lib/ 88 | install -m 644 obj/libchronicle.so $(DESTDIR)$(PREFIX)/lib/ 89 | install -d $(DESTDIR)$(PREFIX)/include/ 90 | install -m 644 libchronicle.h $(DESTDIR)$(PREFIX)/include/ 91 | -------------------------------------------------------------------------------- /native/buffer.c: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Tea Engineering Ltd. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef FILE_LIBBUFFER_SEEN 16 | #define FILE_LIBBUFFER_SEEN 17 | 18 | 19 | #include 20 | #include 21 | 22 | // Outputs non-printable characters as octal, which allows the resulting 23 | // string to be a valid C-string constant. 24 | void printbuf(char* c, int n) { 25 | printf("unsigned char* buf=\""); 26 | for (int i = 0; i < n; i++) { 27 | switch (c[i]) { 28 | 29 | case '\n': 30 | printf("\\n"); 31 | break; 32 | case '\r': 33 | printf("\\r"); 34 | break; 35 | case '\t': 36 | printf("\\t"); 37 | break; 38 | case '\\': 39 | printf("\\\\"); 40 | break; 41 | default: 42 | if ((c[i] < 0x20) || (c[i] > 0x7f)) { 43 | printf("\\%03o", (unsigned char)c[i]); 44 | } else { 45 | printf("%c", c[i]); 46 | } 47 | break; 48 | } 49 | } 50 | printf("\"\n"); 51 | } 52 | 53 | 54 | char* formatbuf(char* buf, int sz) { 55 | // nicely format a buffer to hex/ascii with a length offset. each 16 bytes 56 | // or part thereof of the input take 76 bytes include newline character. caller 57 | // must free() the buffer. 58 | // 59 | // 00000000 c7 6d 65 73 73 61 67 65 eb 48 65 6c 6c 6f 20 57 ·message ·Hello W 60 | // 00000010 6f 72 6c 64 c6 6e 75 6d 62 65 72 a6 d2 02 96 49 orld·num ber····I 61 | // 00000020 c4 63 6f 64 65 e7 53 45 43 4f 4e 44 53 c5 70 72 ·code·SE CONDS·pr 62 | // 00000030 69 63 65 90 00 00 28 41 ice···(A 63 | // 64 | // ----8--- -------------------------48--------------------- --------17-------\n 65 | const char* hexd = "0123456789abcdef"; 66 | int lines = (sz + (16-1)) / 16; // sz/16 rounded up! 67 | if (lines == 0) lines++; 68 | int osz = lines * 76 + 1; // +trailing null 69 | char* r = malloc(osz); 70 | for (int i = 0; i < lines; i++) { 71 | // this provides our trailing null! 72 | sprintf(r+i*76, "%08x %48s %17s\n", i*16, "", ""); 73 | for (int j = i*16; j < sz && j < (i+1)*16; j++) { 74 | int c = j % 16; 75 | int e = c > 7 ? 1 : 0; 76 | r[i*76+9 +c*3+e] = hexd[(buf[j] >> 4) & 0x0F]; // upper nibble 77 | r[i*76+10+c*3+e] = hexd[buf[j] & 0x0F]; // lower nibble 78 | // printable ascii? 79 | if (buf[j] >= 32 && buf[j] < 127) { 80 | r[i*76+9+49+c+e] = buf[j]; 81 | } else { 82 | r[i*76+9+49+c+e] = '.'; 83 | } 84 | } 85 | } 86 | // r[osz-1] = 0; 87 | return r; 88 | } 89 | 90 | #endif 91 | -------------------------------------------------------------------------------- /native/buffer.h: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Tea Engineering Ltd. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef FILE_LIBBUFFER_SEEN 16 | #define FILE_LIBBUFFER_SEEN 17 | 18 | void printbuf(char* c, int n); 19 | char* formatbuf(char* buf, int sz); 20 | 21 | #endif 22 | -------------------------------------------------------------------------------- /native/fuzzmain.c: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Tea Engineering Ltd. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | 16 | #define _GNU_SOURCE 17 | 18 | #define __STDC_FORMAT_MACROS 19 | #include 20 | 21 | #define KXVER 3 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include "k.h" 31 | #include "mock_k.h" 32 | #include "serdes_k.h" 33 | #include "libchronicle.h" 34 | 35 | // This is a stand-alone tool for replaying a queue for use with fuzzer 36 | int print_data = 0; 37 | int print_meta = 0; 38 | 39 | int print_msg(void* ctx, uint64_t index, void* msg) { 40 | K x = (K)msg; 41 | if (print_data) { 42 | printf("[%" PRIu64 "] KC ", index); 43 | if (x->t == KC) 44 | fwrite(kC(x), sizeof(char), x->n, stdout); 45 | printf("\n"); 46 | } 47 | return 5; 48 | } 49 | 50 | /* The state array must be initialized to not be all zero */ 51 | uint32_t xorshift128(uint32_t state[static 4]) { 52 | /* Algorithm "xor128" from p. 5 of Marsaglia, "Xorshift RNGs" */ 53 | uint32_t s, t = state[3]; 54 | t ^= t << 11; 55 | t ^= t >> 8; 56 | state[3] = state[2]; state[2] = state[1]; state[1] = s = state[0]; 57 | t ^= s; 58 | t ^= s >> 19; 59 | state[0] = t; 60 | return t; 61 | } 62 | 63 | //////// APPLE VALGRIND COMPAT //////// 64 | // valgrind has no support for mach system call under mkstemp, printing 65 | // WARNING: unhandled amd64-darwin syscall: unix:464 and 66 | // returning -1 as fp whilst setting syserr to "Function not implemented". 67 | // Due to the fail the random number generator leaks a tonne of malloc() mem. 68 | // Reported but unresolved https://www.mail-archive.com/kde-bugs-dist@kde.org/msg209613.html 69 | static inline uint64_t rdtsc(void) { 70 | uint32_t a, d; 71 | __asm__ __volatile__("rdtscp" : "=a" (a), "=d" (d)); 72 | return (((uint64_t) d << 32) | a); 73 | } 74 | int c_mkstemp(char* pattern) { 75 | uint32_t xor_state[4]; 76 | xor_state[0] = xor_state[1] = xor_state[2] = xor_state[3] = (uint32_t)rdtsc(); 77 | char* rep = strstr(pattern, "XXXXXX"); 78 | if (rep) { 79 | for (int i = 0; i < 6; i++) { 80 | uint8_t incre = xorshift128(xor_state) & 0x1F; 81 | rep[i] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"[incre]; 82 | } 83 | } 84 | int fid = open(pattern, O_RDWR | O_CREAT | O_EXCL, 0660); 85 | if (fid < 0) { 86 | printf("tmpfile %s: %s\n", pattern, strerror(errno)); 87 | abort(); 88 | } 89 | return fid; 90 | } 91 | //////// END APPLE VALGRIND COMPAT //////// 92 | 93 | int main(const int argc, char **argv) { 94 | int c; 95 | opterr = 0; 96 | int verboseflag = 0; 97 | FILE * fuzzfid = NULL; 98 | 99 | while ((c = getopt(argc, argv, "dmv:")) != -1) 100 | switch (c) { 101 | case 'd': 102 | print_data = 1; 103 | break; 104 | case 'm': 105 | print_meta = 1; 106 | break; 107 | case 'v': 108 | verboseflag = 1; 109 | break; 110 | case '?': 111 | fprintf(stderr, "Unknown option `-%c'.\n", optopt); 112 | exit(2); 113 | default: 114 | fprintf(stderr, "%c??", c); 115 | exit(3); 116 | } 117 | 118 | if (optind + 2 > argc) { 119 | printf("Missing mandatory argument.\n Expected: %s [-d] [-m] [-v] QUEUE FUZZFILE\n", argv[0]); 120 | printf(" -d print data\n"); 121 | printf(" -m print metadata\n"); 122 | printf(" -v verbose mode\n"); 123 | printf(" QUEUE queue directory\n"); 124 | printf(" FUZZFILE fuzzing script file (- stdin) providing clock and data sizes\n"); 125 | printf("\n"); 126 | printf(" FUZZFILE should contain new-line separated lines, each containing space separated values\n"); 127 | printf(" the first value `time` advances the clock by time nanoseconds.\n"); 128 | printf(" the second `bytes` appends that many random bytes as a new entry to the queue\n"); 129 | printf(" As the script is played, the random number generator seed, recieved index and byte count\n"); 130 | printf(" are written to a log. This is then re-opened to verify the data in the queue matches.\n"); 131 | 132 | exit(1); 133 | } 134 | 135 | // mandatory arguments 136 | char* dir = argv[optind]; 137 | char* fuzz = argv[optind+1]; 138 | 139 | fuzzfid = stdin; 140 | if (strcmp(fuzz, "-") == 0) { 141 | fprintf(stderr, "fuzzing from stdin\n"); 142 | } else { 143 | fuzzfid = fopen(fuzz, "r"); 144 | if (fuzzfid == NULL) { 145 | fprintf(stderr, "Unable to open %s\n", fuzz); 146 | exit(4); 147 | } 148 | } 149 | 150 | queue_t* queue = chronicle_init(dir); 151 | chronicle_set_decoder(queue, &parse_kx, &free_kx); 152 | chronicle_set_encoder(queue, &sizeof_kx, &append_kx); 153 | chronicle_set_version(queue, 5); 154 | chronicle_set_roll_scheme(queue, "FAST_HOURLY"); 155 | chronicle_set_create(queue, 1); 156 | if (chronicle_open(queue) != 0) exit(-1); 157 | 158 | tailer_t* tailer = chronicle_tailer(queue, print_msg, NULL, 0); 159 | chronicle_peek_queue(queue); 160 | 161 | if (verboseflag) { 162 | chronicle_debug(); 163 | } 164 | 165 | size_t linecap = 0; 166 | ssize_t linelen = 0; 167 | char *msgp = NULL; 168 | char *parsep = NULL; 169 | 170 | int line = 0; 171 | long long bytes; 172 | long long time; 173 | uint64_t clock = 0; 174 | uint64_t index = 0; 175 | 176 | uint32_t xor_state[4]; 177 | 178 | char* tmpfile = strdup("/tmp/shmmain.XXXXXX"); 179 | int tmpfid = c_mkstemp(tmpfile); 180 | printf("logging fuzz expectations to %s fid %d msg %s\n", tmpfile, tmpfid, strerror(errno)); 181 | FILE* tmp = fdopen(tmpfid, "w+"); 182 | printf(" tmp is %p %s\n", tmp, strerror(errno)); 183 | while ((linelen = getline(&msgp, &linecap, fuzzfid)) > 0) { 184 | line++; 185 | parsep = msgp; 186 | time = strtoll(parsep, &parsep, 0); 187 | if (*parsep == ' ') parsep++; 188 | bytes = strtoll(parsep, &parsep, 0); 189 | 190 | printf(" FUZ: %lld millis, %lld bytes\n", time, bytes); 191 | clock += time; 192 | xor_state[0] = xor_state[1] = xor_state[2] = xor_state[3] = (uint32_t)line+1; 193 | K x = ktn(KC, bytes); 194 | for (long long b = 0; b < bytes; b++) { 195 | kG(x)[b] = (uint8_t)xorshift128(xor_state); 196 | } 197 | index = chronicle_append(queue, x); 198 | fprintf(tmp, "%" PRIu64 " %lld %lld\n", clock, bytes, index); 199 | r0(x); 200 | } 201 | // we're done parsing input, now replay checking using temp file 202 | //fclose(tmp); 203 | //close(tmpfid); 204 | 205 | fflush(tmp); 206 | if (fseek(tmp, 0L, SEEK_SET) != 0) { printf("abort not fseek"); abort(); }; 207 | 208 | int rline = 0; 209 | int r; 210 | printf("replay\n"); 211 | do { 212 | r = fscanf(tmp, "%lld %lld %lld\n", &clock, &bytes, &index); 213 | if (r == 3) { 214 | // clock bytes index 215 | printf(" did %lld %lld %lld\n", clock, bytes, index); 216 | 217 | // TODO: verify!! 218 | } else if (rline != line) { 219 | printf ("error, %s r=%d at line %d!\n\n", tmpfile, r, rline); 220 | } 221 | rline++; 222 | } while (r != EOF); 223 | 224 | fclose(tmp); 225 | 226 | // unlink(tmpfile); 227 | free(tmpfile); 228 | if (msgp) free(msgp); 229 | printf("parse finished\n"); 230 | 231 | exit(rline == line ? 0 : 5); // exit with fail 232 | 233 | if (verboseflag) { 234 | chronicle_debug(); 235 | } 236 | 237 | chronicle_cleanup(queue); 238 | 239 | return 0; 240 | } 241 | -------------------------------------------------------------------------------- /native/libchronicle.c: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Tea Engineering Ltd. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #define _GNU_SOURCE 16 | 17 | #define __STDC_FORMAT_MACROS 18 | #include 19 | 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | 33 | #include "wire.h" 34 | #include "buffer.h" 35 | 36 | /** 37 | * Implementation notes 38 | * This code is not reentrant, nor does it create any threads, so this is not a problem. 39 | * Should not be used by additional threads without external locking. 40 | * Multiple processes can append and tail from a queue concurrently, and one process 41 | * may read or write from multiple queues. 42 | * 43 | * TODO: current iteration requires a Java Chronicle-Queues appender to be writing to the same 44 | * queue on a periodic timer to roll over the log files correctly and maintain the index structures. 45 | * 46 | * It should be possible to append to a queue from a tailer callback on same queue, so the fds 47 | * and mmaps used for writing are separate to those used in recovery. 48 | * 49 | * Most of the appender logic is actually just the tailer logic, followed by cas write-lock. 50 | * For interesting concurrency parts search 'lock_cmpxchgl' and 'mfence'. 51 | * 52 | * To gurantee we can read and write payloads of 'blocksize' length we map 53 | * 2x blocksize each time, aligning the map offset to be a multiple of blocksize from 54 | * the start of the file. If the block parser (parse_queue_block) is about to step 55 | * over the buffer end, it returns asking for advance. There are two outcomes: either 56 | * we are positioned in 2nd block ('overhang'), and our mmap() will advance one block 57 | * completing the parse, or having tried that the parser will stop again without moving 58 | * in which case we double the blocksize. 59 | * 60 | * The interesting caller of parse_queue_block is shmipc_peek_tailer_r, which handles 61 | * splitting the index into cycle and seqnum, determinging the filenames, opening fids, 62 | * repositioning the buffer and interpreting the return codes from the block parser. It 63 | * consists of a while (1) loop, repeating until one of the enumerated states 64 | * prevents it from doing any further work. Reasons are in tailer_state_messages and shown 65 | * in the debug output. shmipc_peek_tailer_r re-uses existing generated filenames, fids 66 | * and maps from previous invocations as much as possible. 67 | * 68 | * patch_cycles is a variable that controls compatability with Java and is only relevant 69 | * when opening an old queue that may not have been written to recently. When an appender is 70 | * started, it starts seeking for the write position from (highetCycle-patch_cycles), 71 | * which causes a replay to occur internally and ensures that any previous cycles 72 | * covered by the replay are patched up to end with an EOF marker. For symnetry, if a 73 | * reader finds itself stuck at an available marker in a queuefile for a 74 | * cycle < (highestCycle-patch_cycles), it is permitted to skip over the missing EOF. 75 | * Java acheives something similar using less-elegant timeouts. 76 | * 77 | * 78 | */ 79 | 80 | // MetaDataKeys `header`index2index`index`roll 81 | 82 | // error handling 83 | const char* cerr_msg; 84 | // error handling - trigger 85 | int chronicle_err(const char* msg) { 86 | printf("chronicle error: '%s\n", msg); 87 | errno = -1; 88 | cerr_msg = msg; 89 | return -1; 90 | } 91 | void* chronicle_perr(const char* msg) { 92 | chronicle_err(msg); 93 | return (void*)NULL; 94 | } 95 | // error handling - retrieve 96 | const char* chronicle_strerror() { 97 | return cerr_msg; 98 | } 99 | 100 | const char* tailer_state_messages[] = {"AWAITING_ENTRY", "BUSY", "AWAITING_QUEUEFILE", "E_STAT", "E_MMAP", "PEEK?", "EXTEND_FAIL", "COLLECTED"}; 101 | 102 | // structures 103 | 104 | typedef struct { 105 | unsigned char *highest_cycle; 106 | unsigned char *lowest_cycle; 107 | unsigned char *modcount; 108 | } dirlist_fields_t; 109 | 110 | struct tailer { 111 | uint64_t dispatch_after; // for resume support 112 | tailstate_t state; 113 | cdispatch_f dispatcher; 114 | void* dispatch_ctx; 115 | // to support the 'collect' operation to wait and return next item, ignoring callback 116 | collected_t* collect; 117 | 118 | int mmap_protection; // PROT_READ etc. 119 | 120 | // currently open queue file 121 | uint64_t qf_cycle_open; 122 | char* qf_fn; 123 | struct stat qf_statbuf; 124 | int qf_fd; 125 | 126 | uint64_t qf_tip; // byte position of the next header, or zero if unknown 127 | uint64_t qf_index; // seqnum of the header pointed to by qf_tip 128 | 129 | // currently mapped region: buffer, offset (from 0 in file), size 130 | unsigned char* qf_buf; 131 | uint64_t qf_mmapoff; 132 | uint64_t qf_mmapsz; 133 | 134 | struct queue* queue; 135 | 136 | struct tailer* next; 137 | struct tailer* prev; 138 | }; 139 | 140 | struct queue { 141 | char* dirname; 142 | uint blocksize; 143 | uint8_t version; 144 | uint8_t create; 145 | 146 | // directory-listing.cq4t 147 | char* dirlist_name; 148 | int dirlist_fd; 149 | struct stat dirlist_statbuf; 150 | unsigned char* dirlist; // mmap base 151 | dirlist_fields_t dirlist_fields; 152 | 153 | char* queuefile_pattern; 154 | glob_t queuefile_glob; // last glob of data files, refreshed by poll on modcount 155 | 156 | // values observed from directory-listing poll 157 | uint64_t highest_cycle; 158 | uint64_t lowest_cycle; 159 | uint64_t modcount; 160 | 161 | // roll config populated from (any) queuefile header or set on creation 162 | int roll_length; 163 | int roll_epoch; 164 | char* roll_format; 165 | char* roll_name; 166 | char* roll_strftime; 167 | int index_count; 168 | int index_spacing; 169 | 170 | int cycle_shift; 171 | uint64_t seqnum_mask; 172 | 173 | cparse_f parser; 174 | cparsefree_f parser_free; 175 | csizeof_f append_sizeof; 176 | cappend_f append_write; 177 | 178 | tailer_t* tailers; 179 | 180 | // the appender is a shared tailer, polled by append[], with writing logic 181 | // and no callback to user code for events 182 | tailer_t* appender; 183 | 184 | struct queue* next; 185 | }; 186 | 187 | 188 | typedef enum {QB_AWAITING_ENTRY, QB_BUSY, QB_REACHED_EOF, QB_NEED_EXTEND, QB_NULL_ITEM, QB_COLLECTED} parseqb_state_t; 189 | 190 | typedef parseqb_state_t (*datacallback_f)(unsigned char*,int,uint64_t,void* userdata); 191 | 192 | 193 | // paramaters that control behavior, not exposed for modification 194 | uint32_t patch_cycles = 3; 195 | long int qf_disk_sz = 83754496L; 196 | 197 | // globals 198 | int debug = 0; 199 | uint32_t pid_header = 0; 200 | queue_t* queue_head = NULL; 201 | 202 | 203 | // forward declarations that are not part of the public api 204 | void parse_dirlist(queue_t*); 205 | void parse_queuefile_meta(unsigned char*, int, queue_t*); 206 | void parse_queuefile_data(unsigned char*, int, queue_t*, tailer_t*, uint64_t); 207 | int queuefile_init(char*, queue_t*); 208 | int directory_listing_reopen(queue_t*, int, int); 209 | int directory_listing_init(queue_t*, uint64_t cycle); 210 | 211 | long chronicle_clock_ms(queue_t*); 212 | uint64_t chronicle_cycle_from_ms(queue_t*, long); 213 | int chronicle_peek_queue_tailer(queue_t*, tailer_t*); 214 | 215 | // compare and swap, 32 bits, addressed by a 64bit pointer 216 | static inline uint32_t lock_cmpxchgl(unsigned char *mem, uint32_t newval, uint32_t oldval) { 217 | __typeof (*mem) ret; 218 | __asm __volatile ("lock; cmpxchgl %2, %1" 219 | : "=a" (ret), "=m" (*mem) 220 | : "r" (newval), "m" (*mem), "0" (oldval)); 221 | return (uint32_t) ret; 222 | } 223 | 224 | static inline uint32_t lock_xadd(unsigned char* mem, uint32_t val) { 225 | __asm__ volatile("lock; xaddl %0, %1" 226 | : "+r" (val), "+m" (*mem) // input+output 227 | : // No input-only 228 | : "memory" 229 | ); 230 | return (uint32_t)val; 231 | } 232 | 233 | char* chronicle_get_cycle_fn(queue_t* queue, int cycle) { 234 | // TODO: replace with https://ideone.com/7BADb as gmtime_r leaks 235 | // time_t aka long. seconds since midnight 1970 236 | time_t rawtime = (time_t) cycle * (queue->roll_length / 1000); 237 | struct tm info; 238 | gmtime_r(&rawtime, &info); 239 | 240 | // format datetime component using prebuilt pattern 241 | char* strftime_buf = strdup(queue->roll_format); 242 | strftime(strftime_buf, strlen(strftime_buf)+1, queue->roll_strftime, &info); 243 | 244 | // join with dirname and file suffix 245 | char* fnbuf; 246 | int bufsz = asprintf(&fnbuf, "%s/%s.cq4", queue->dirname, strftime_buf); 247 | free(strftime_buf); 248 | if (bufsz < 0) return NULL; 249 | return fnbuf; 250 | } 251 | 252 | void queue_double_blocksize(queue_t* queue) { 253 | uint new_blocksize = queue->blocksize << 1; 254 | printf("shmipc: doubling blocksize from %x to %x\n", queue->blocksize, new_blocksize); 255 | queue->blocksize = new_blocksize; 256 | } 257 | 258 | 259 | queue_t* chronicle_init(char* dir) { 260 | char* debug_env = getenv("SHMIPC_DEBUG"); 261 | debug = (debug_env == NULL) ? 0 : strcmp(debug_env, "1") == 0; 262 | char* wiretrace_env = getenv("SHMIPC_WIRETRACE"); 263 | wire_trace = (wiretrace_env == NULL) ? 0 : strcmp(wiretrace_env, "1") == 0; 264 | 265 | pid_header = (getpid() & HD_MASK_LENGTH); 266 | 267 | // allocate struct, we'll link if all checks pass 268 | queue_t* queue = malloc(sizeof(queue_t)); 269 | if (queue == NULL) return chronicle_perr("m fail"); 270 | bzero(queue, sizeof(queue_t)); 271 | 272 | // wire up default 'text' parsers for data segments 273 | queue->parser = &chronicle_decoder_default_parse; 274 | queue->append_sizeof = &chronicle_encoder_default_sizeof; 275 | queue->append_write = &chronicle_encoder_default_write; 276 | 277 | // unsafe to use ref here in case caller doesn't keep in scope, so dup 278 | queue->dirname = strdup(dir); 279 | queue->blocksize = 1024*1024; // must be a power of two (single 1 bit) 280 | queue->roll_epoch = -1; 281 | 282 | // Good to use 283 | queue->next = queue_head; 284 | queue_head = queue; 285 | 286 | return queue; 287 | } 288 | 289 | int chronicle_readable(char* dirname, char* suffix) { 290 | char* dirlist_name; 291 | int dirlist_fd; 292 | 293 | // probe v4 directory-listing.cq4t 294 | asprintf(&dirlist_name, "%s/%s", dirname, suffix); 295 | if ((dirlist_fd = open(dirlist_name, O_RDONLY)) > 0) { 296 | close(dirlist_fd); 297 | free(dirlist_name); 298 | return 1; 299 | } 300 | free(dirlist_name); 301 | return 0; 302 | } 303 | 304 | int chronicle_version_detect(queue_t* queue) { 305 | if (chronicle_readable(queue->dirname, "directory-listing.cq4t")) return 4; 306 | if (chronicle_readable(queue->dirname, "metadata.cq4t")) return 5; 307 | return 0; 308 | } 309 | 310 | int chronicle_open(queue_t* queue) { 311 | 312 | if (debug) printf("shmipc: opening dir %s\n", queue->dirname); 313 | 314 | // Is this a directory 315 | struct stat statbuf; 316 | if (stat(queue->dirname, &statbuf) != 0) return chronicle_err("dir stat fail"); 317 | if (!S_ISDIR(statbuf.st_mode)) return chronicle_err("dir is not a directory"); 318 | 319 | // autodetect version 320 | int auto_version = chronicle_version_detect(queue); 321 | printf("chronicle: detected version v%d\n", auto_version); 322 | 323 | // Does queue dir contain some .cq4 files? 324 | // for V5 it is OK to have empty directory 325 | glob_t *g = &queue->queuefile_glob; 326 | g->gl_offs = 0; 327 | 328 | asprintf(&queue->queuefile_pattern, "%s/*.cq4", queue->dirname); 329 | glob(queue->queuefile_pattern, GLOB_ERR, NULL, g); 330 | if (debug) { 331 | printf("shmipc: glob %zu queue files found\n", g->gl_pathc); 332 | for (int i = 0; i < g->gl_pathc;i++) { 333 | printf(" %s\n", g->gl_pathv[i]); 334 | } 335 | } 336 | 337 | if (auto_version == 0) { 338 | if (queue->create == 0) return chronicle_err("queue should exist (no permission to create), but version detect failed"); 339 | if (queue->version == 0) return chronicle_err("queue create requires chronicle_set_version()"); 340 | if (g->gl_pathc != 0) return chronicle_err("queue create requires empty destination directory"); 341 | if (queue->roll_name == NULL) return chronicle_err("queue create requires chronicle_set_roll_scheme()"); 342 | 343 | } else if (queue->version != 0 && queue->version != auto_version) { 344 | return chronicle_err("queue version detected does not match expected set via. chronicle_set_version()"); 345 | } else { 346 | queue->version = auto_version; 347 | } 348 | 349 | // populate dirlist 350 | asprintf(&queue->dirlist_name, queue->version == 4 ? "%s/directory-listing.cq4t" : "%s/metadata.cq4t", queue->dirname); 351 | if (queue->create && auto_version == 0) { 352 | uint64_t cycle = chronicle_cycle_from_ms(queue, chronicle_clock_ms(queue)); 353 | int rc = directory_listing_init(queue, cycle); 354 | if(rc != 0) return rc; 355 | } 356 | 357 | // parse the directory listing and sanity-check all required fields exist 358 | int rc = directory_listing_reopen(queue, O_RDONLY, PROT_READ); 359 | if(rc != 0) return rc; 360 | 361 | if (queue->version == 4) { 362 | // For v4, we need to read a 'queue' header from any one of the datafiles to get the 363 | // rollover configuration. We don't know how to generate a filename from a cycle code 364 | // yet, so this needs to use the directory listing. 365 | 366 | if (g->gl_pathc < 1) { 367 | return chronicle_err("V4 and no queue files found so cannot initialise. Call chronicle_set_create(queue, 1) to allow queue creation"); 368 | } 369 | 370 | int queuefile_fd; 371 | struct stat queuefile_statbuf; 372 | uint64_t queuefile_extent; 373 | unsigned char* queuefile_buf; 374 | 375 | char* fn = queue->queuefile_glob.gl_pathv[0]; 376 | // find length of queuefile and mmap 377 | if ((queuefile_fd = open(fn, O_RDONLY)) < 0) { 378 | return chronicle_err("qfi open"); 379 | } 380 | if (fstat(queuefile_fd, &queuefile_statbuf) < 0) 381 | return chronicle_err("qfi fstat"); 382 | 383 | // only need the first block 384 | queuefile_extent = queuefile_statbuf.st_size < queue->blocksize ? queuefile_statbuf.st_size : queue->blocksize; 385 | 386 | if ((queuefile_buf = mmap(0, queuefile_extent, PROT_READ, MAP_SHARED, queuefile_fd, 0)) == MAP_FAILED) 387 | return chronicle_err("qfi mmap fail"); 388 | 389 | // we don't need a data-parser at this stage as only need values from the header 390 | if (debug) printf("shmipc: parsing queuefile %s 0..%" PRIu64 "\n", fn, queuefile_extent); 391 | parse_queuefile_meta(queuefile_buf, queuefile_extent, queue); 392 | 393 | // close queuefile 394 | munmap(queuefile_buf, queuefile_extent); 395 | close(queuefile_fd); 396 | } 397 | 398 | // check we loaded roll settings from queuefile or metadata 399 | // Note these might be the settings we wrote during create 400 | if (queue->roll_format == 0) return chronicle_err("qfi roll_format fail"); 401 | if (queue->roll_length == 0) return chronicle_err("qfi roll_length fail"); 402 | if (queue->roll_epoch == -1) return chronicle_err("qfi roll_epoch fail"); 403 | if (chronicle_set_roll_dateformat(queue, queue->roll_format) != 0) { 404 | printf("roll format detected %s\n", queue->roll_format); 405 | return chronicle_err("detected roll_format is not recognised"); 406 | } 407 | //if (queue->index_count == 0) return chronicle_err("qfi index_count fail"); 408 | //if (queue->index_spacing == 0) return chronicle_err("qfi index_spacing fail"); 409 | 410 | queue->cycle_shift = 32; 411 | queue->seqnum_mask = 0x00000000FFFFFFFF; 412 | 413 | // TODO: Logic from RollCycles.java ensures rollover occurs before we run out of index2index pages? 414 | // cycleShift = Math.max(32, Maths.intLog2(indexCount) * 2 + Maths.intLog2(indexSpacing)); 415 | 416 | // avoids a tailer registration before we have a minimum cycle 417 | chronicle_peek_queue(queue); 418 | if (debug) printf("shmipc: chronicle_open() OK\n"); 419 | 420 | return 0; 421 | } 422 | 423 | void chronicle_set_decoder(queue_t *queue, cparse_f parser, cparsefree_f parser_free) { 424 | if (debug & !parser) printf("chronicle: setting NULL parser"); 425 | queue->parser = parser; 426 | queue->parser_free = parser_free; 427 | } 428 | 429 | void chronicle_set_encoder(queue_t *queue, csizeof_f append_sizeof, cappend_f append_write) { 430 | if (debug & !append_sizeof) printf("chronicle: setting NULL append_sizeof"); 431 | if (debug & !append_write) printf("chronicle: setting NULL append_write"); 432 | queue->append_sizeof = append_sizeof; 433 | queue->append_write = append_write; 434 | } 435 | 436 | struct ROLL_SCHEME chronicle_roll_schemes[] = { 437 | // in use by cq5 438 | {"FIVE_MINUTELY", "yyyyMMdd-HHmm'V'", 5*60, 2<<10, 256}, 439 | {"TEN_MINUTELY", "yyyyMMdd-HHmm'X'", 10*60, 2<<10, 256}, 440 | {"TWENTY_MINUTELY", "yyyyMMdd-HHmm'XX'", 20*60, 2<<10, 256}, 441 | {"HALF_HOURLY", "yyyyMMdd-HHmm'H'", 30*60, 2<<10, 256}, 442 | {"FAST_HOURLY", "yyyyMMdd-HH'F'", 60*60, 4<<10, 256}, 443 | {"TWO_HOURLY", "yyyyMMdd-HH'II'", 2*60*60, 4<<10, 256}, 444 | {"FOUR_HOURLY", "yyyyMMdd-HH'IV'", 4*60*60, 4<<10, 256}, 445 | {"SIX_HOURLY", "yyyyMMdd-HH'VI'", 6*60*60, 4<<10, 256}, 446 | {"FAST_DAILY", "yyyyMMdd'F'", 24*60*60, 4<<10, 256}, 447 | // used historically by cq4 448 | {"MINUTELY", "yyyyMMdd-HHmm", 60, 2<<10, 16}, 449 | {"HOURLY", "yyyyMMdd-HH", 60*60, 4<<10, 16}, 450 | {"DAILY", "yyyyMMdd", 24*60*60, 8<<10, 64}, 451 | // minimal rolls with resulting large queue files 452 | {"LARGE_HOURLY", "yyyyMMdd-HH'L'", 60*60, 8<<10, 64}, 453 | {"LARGE_DAILY", "yyyyMMdd'L'", 24*60*60, 32<<10, 128}, 454 | {"XLARGE_DAILY", "yyyyMMdd'X'", 24*60*60, 32<<10, 256}, 455 | {"HUGE_DAILY", "yyyyMMdd'H'", 24*60*60, 32<<10, 1024}, 456 | // for tests and benchmarking with nearly no indexing 457 | {"SMALL_DAILY", "yyyyMMdd'S'", 24*60*60, 8<<10, 8}, 458 | {"LARGE_HOURLY_SPARSE", "yyyyMMdd-HH'LS'", 60*60, 4<<10, 1024}, 459 | {"LARGE_HOURLY_XSPARSE", "yyyyMMdd-HH'LX'", 60*60, 2<<10, 1<<20}, 460 | {"HUGE_DAILY_XSPARSE", "yyyyMMdd'HX'", 24*60*60, 16<<10, 1<<20}, 461 | // for tests to create smaller queue files 462 | {"TEST_SECONDLY", "yyyyMMdd-HHmmss'T'", 1, 32<<10, 4}, 463 | {"TEST4_SECONDLY", "yyyyMMdd-HHmmss'T4'", 1, 32, 4}, 464 | {"TEST_HOURLY", "yyyyMMdd-HH'T'", 60*60, 16, 4}, 465 | {"TEST_DAILY", "yyyyMMdd'T1'", 24*60*60, 8, 1}, 466 | {"TEST2_DAILY", "yyyyMMdd'T2'", 24*60*60, 16, 2}, 467 | {"TEST4_DAILY", "yyyyMMdd'T4'", 24*60*60, 32, 4}, 468 | {"TEST8_DAILY", "yyyyMMdd'T8'", 24*60*60, 128, 8}, 469 | }; 470 | 471 | void chronicle_apply_roll_scheme(queue_t* queue, struct ROLL_SCHEME x) { 472 | if (debug) printf("chronicle: chronicle_set_roll_scheme applying %s\n", x.name); 473 | free(queue->roll_name); 474 | free(queue->roll_format); 475 | free(queue->roll_strftime); 476 | queue->roll_name = strdup(x.name); 477 | queue->roll_format = strdup(x.formatstr); 478 | queue->roll_length = x.roll_length_secs * 1000; 479 | 480 | // remove appostrophe from java's format string and build 481 | // the equivelent strftime string together 482 | char* p = strdup(queue->roll_format); 483 | int px = 0; 484 | char* f = queue->roll_format; 485 | int fi = 0; 486 | int inquote = 0; 487 | while (fi < strlen(queue->roll_format)) { 488 | if (debug) { 489 | printf(" rs parser fi=%d px=%d inquote=%d buffer='%s'\n", fi, px, inquote, p); 490 | } 491 | if (inquote == 1 && f[fi] != '\'') { 492 | // copy quoted literal 493 | p[px++] = f[fi++]; 494 | } else if (f[fi] == '-') { 495 | // copy literal dash 496 | p[px++] = f[fi++]; 497 | } else if (f[fi] == '\'') { 498 | // ignore quotes, toggle flag 499 | inquote = (inquote + 1) % 2; 500 | fi++; 501 | } else if (strncmp(&f[fi], "yyyy", 4) == 0) { 502 | p[px++] = '%'; 503 | p[px++] = 'Y'; 504 | fi += 4; 505 | } else if (strncmp(&f[fi], "MM", 2) == 0) { 506 | p[px++] = '%'; 507 | p[px++] = 'm'; 508 | fi += 2; 509 | } else if (strncmp(&f[fi], "dd", 2) == 0) { 510 | p[px++] = '%'; 511 | p[px++] = 'd'; 512 | fi += 2; 513 | } else if (strncmp(&f[fi], "HH", 2) == 0) { 514 | p[px++] = '%'; 515 | p[px++] = 'H'; 516 | fi += 2; 517 | } else if (strncmp(&f[fi], "mm", 2) == 0) { 518 | p[px++] = '%'; 519 | p[px++] = 'M'; 520 | fi += 2; 521 | } else if (strncmp(&f[fi], "ss", 2) == 0) { 522 | p[px++] = '%'; 523 | p[px++] = 'S'; 524 | fi += 2; 525 | } else { 526 | printf("chronicle: parser conversion of %s exploded at fi=%d px=%d inquote=%d buffer='%s'\n", queue->roll_format, fi, px, inquote, p); 527 | return; 528 | } 529 | } 530 | p[px++] = 0; 531 | if (debug) { 532 | printf(" rs parser result='%s'\n", p); 533 | } 534 | queue->roll_strftime = p; 535 | } 536 | 537 | int chronicle_set_roll_scheme(queue_t* queue, char* scheme) { 538 | for (int i = 0; i < sizeof(chronicle_roll_schemes)/sizeof(chronicle_roll_schemes[0]); i++) { 539 | struct ROLL_SCHEME x = chronicle_roll_schemes[i]; 540 | if (strcmp(scheme, x.name) == 0) { 541 | chronicle_apply_roll_scheme(queue, x); 542 | } 543 | } 544 | return 0; 545 | } 546 | 547 | char* chronicle_get_roll_scheme(queue_t* queue) { 548 | return queue->roll_name; 549 | } 550 | 551 | char* chronicle_get_roll_format(queue_t* queue) { 552 | return queue->roll_format; 553 | } 554 | 555 | int chronicle_set_roll_dateformat(queue_t* queue, char* dateformat) { 556 | for (int i = 0; i < sizeof(chronicle_roll_schemes)/sizeof(chronicle_roll_schemes[0]); i++) { 557 | struct ROLL_SCHEME x = chronicle_roll_schemes[i]; 558 | if (strcmp(dateformat, x.formatstr) == 0) { 559 | chronicle_apply_roll_scheme(queue, x); 560 | return 0; 561 | } 562 | } 563 | return -1; 564 | } 565 | 566 | void chronicle_set_create(queue_t* queue, int create) { 567 | queue->create = create; 568 | } 569 | 570 | void chronicle_set_version(queue_t* queue, int version) { 571 | if (version == 4) { 572 | queue->version = 4; 573 | } else if (version == 5) { 574 | queue->version = 5; 575 | } 576 | } 577 | 578 | int chronicle_get_version(queue_t* queue) { 579 | return queue->version; 580 | } 581 | 582 | long chronicle_clock_ms(queue_t* queue) { 583 | // switch to custom clock etc. 584 | // default case 585 | struct timeval tv; 586 | gettimeofday(&tv, NULL); 587 | return (tv.tv_sec) * 1000 + (tv.tv_usec) / 1000 ; 588 | } 589 | 590 | uint64_t chronicle_cycle_from_ms(queue_t* queue, long ms) { 591 | return (ms - queue->roll_epoch) / queue->roll_length; 592 | } 593 | 594 | // return codes 595 | // 0 awaiting at &base 596 | // 1 we hit working 597 | // 2 we hit EOF 598 | // 3 data extent will cross base+limit 599 | // 4 hit data with no data parser 600 | // (7 collected value - from parse_data) 601 | // if any entries are read the values at basep and indexp are updated 602 | // if parse_data returns non-zero, we pause parsing after the current item 603 | // typedef enum {QB_AWAITING_ENTRY, QB_BUSY, QB_REACHED_EOF, QB_NEED_EXTEND, QB_NULL_ITEM, QB_COLLECTED} parseqb_state_t; 604 | 605 | parseqb_state_t parse_queue_block(queue_t *queue, unsigned char** basep, uint64_t *indexp, unsigned char* extent, wirecallbacks_t* hcbs, datacallback_f parse_data, void* userdata) { 606 | uint32_t header; 607 | int sz; 608 | unsigned char* base = *basep; 609 | uint64_t index = *indexp; 610 | parseqb_state_t pd = QB_AWAITING_ENTRY; 611 | while (pd == QB_AWAITING_ENTRY) { 612 | if (base+4 >= extent) return 3; 613 | memcpy(&header, base, sizeof(header)); // relax, fn optimised away 614 | // no speculative fetches before the header is read 615 | asm volatile ("mfence" ::: "memory"); 616 | 617 | if (header == HD_UNALLOCATED) { 618 | if (debug) printf(" %" PRIu64 " @%p unallocated\n", index, base); 619 | return QB_AWAITING_ENTRY; 620 | } else if ((header & HD_MASK_META) == HD_WORKING) { 621 | if (debug) printf(" @%p locked for writing by pid %d\n", base, header & HD_MASK_LENGTH); 622 | return QB_BUSY; 623 | } else if ((header & HD_MASK_META) == HD_METADATA) { 624 | sz = (header & HD_MASK_LENGTH); 625 | if (debug) printf(" @%p metadata size %x\n", base, sz); 626 | if (base+4+sz >= extent) return QB_NEED_EXTEND; 627 | wire_parse(base+4, sz, hcbs); 628 | // EventName header 629 | // switch to header parser 630 | } else if ((header & HD_MASK_META) == HD_EOF) { 631 | if (debug) printf(" @%p EOF\n", base); 632 | return QB_REACHED_EOF; 633 | } else { 634 | sz = (header & HD_MASK_LENGTH); 635 | if (debug) printf(" %" PRIu64 " @%p data size %x\n", index, base, sz); 636 | if (parse_data) { 637 | if (base+4+sz >= extent) return QB_NEED_EXTEND; 638 | pd = parse_data(base+4, sz, index, userdata); 639 | } else { 640 | // bail at first data message 641 | return QB_NULL_ITEM; 642 | } 643 | index++; 644 | *indexp = index; 645 | } 646 | int pad4 = (queue->version < 5) ? 0 : -sz & 0x03; 647 | base = base + 4 + sz + pad4; 648 | *basep = base; 649 | } 650 | return pd; 651 | } 652 | 653 | // parse data callback dispatching to wire.h parser 654 | parseqb_state_t parse_wire_data(unsigned char* base, int lim, uint64_t index, void* cbs) { 655 | wire_parse(base, lim, (wirecallbacks_t*)cbs); 656 | return QB_AWAITING_ENTRY; 657 | } 658 | 659 | 660 | // return AWAITING_ENTRY to continue dispaching, COLLECTED to signal collected item 661 | parseqb_state_t parse_data_cb(unsigned char* base, int lim, uint64_t index, void* userdata) { 662 | tailer_t* tailer = (tailer_t*)userdata; 663 | if (debug) printbuf((char*)base, lim); 664 | // prep args and fire callback 665 | if (index > tailer->dispatch_after) { 666 | 667 | COBJ msg = tailer->queue->parser(base, lim); 668 | if (msg == NULL) { 669 | if (debug) printf("chronicle: caution at index %" PRIu64 " parse function returned NULL, skipping\n", index); 670 | return QB_AWAITING_ENTRY; 671 | } 672 | // if asked to return inline, we skip dispatcher callback, user 673 | // must explicitly free parsed data with chronicle_return() 674 | if (tailer->collect) { 675 | tailer->collect->msg = msg; 676 | tailer->collect->index = index; 677 | tailer->collect->sz = lim; 678 | return QB_COLLECTED; 679 | } 680 | if (tailer->dispatcher) { 681 | tailer->dispatcher(tailer->dispatch_ctx, index, msg); 682 | } 683 | if (tailer->queue->parser_free) { 684 | tailer->queue->parser_free(msg); 685 | } 686 | } 687 | return QB_AWAITING_ENTRY; 688 | } 689 | 690 | 691 | void handle_dirlist_ptr(char* buf, int sz, unsigned char *dptr, wirecallbacks_t* cbs) { 692 | // we are preserving *pointers* within the shared directory data page 693 | // we keep the underlying mmap for life of queue 694 | queue_t* queue = (queue_t*)cbs->userdata; 695 | if (strncmp(buf, "listing.highestCycle", sz) == 0) { 696 | queue->dirlist_fields.highest_cycle = dptr; 697 | } else if (strncmp(buf, "listing.lowestCycle", sz) == 0) { 698 | queue->dirlist_fields.lowest_cycle = dptr; 699 | } else if (strncmp(buf, "listing.modCount", sz) == 0) { 700 | queue->dirlist_fields.modcount = dptr; 701 | } 702 | } 703 | 704 | void handle_dirlist_uint64(char* buf, int sz, uint64_t data, wirecallbacks_t* cbs){ 705 | queue_t* queue = (queue_t*)cbs->userdata; 706 | if (strncmp(buf, "length", sz) == 0) { 707 | if (debug) printf(" v5 roll_length set to %" PRIu64 "\n", data); 708 | queue->roll_length = data; 709 | } else if (strncmp(buf, "epoch", sz) == 0) { 710 | if (debug) printf(" v5 roll_epoch set to %" PRIu64 "\n", data); 711 | queue->roll_epoch = data; 712 | } 713 | } 714 | 715 | void handle_dirlist_text(char* buf, int sz, char* data, int dsz, wirecallbacks_t* cbs) { 716 | queue_t* queue = (queue_t*)cbs->userdata; 717 | if (strncmp(buf, "format", sz) == 0) { 718 | if (debug) printf(" v5 roll_format set to %.*s\n", dsz, data); 719 | free(queue->roll_format); 720 | queue->roll_format = strndup(data, dsz); 721 | } 722 | } 723 | 724 | void handle_qf_uint64(char* buf, int sz, uint64_t data, wirecallbacks_t* cbs){ 725 | queue_t* queue = (queue_t*)cbs->userdata; 726 | if (strncmp(buf, "length", sz) == 0) { 727 | if (debug) printf(" v4 roll_length set to %" PRIu64 "\n", data); 728 | queue->roll_length = data; 729 | } else if (strncmp(buf, "indexCount", sz) == 0) { 730 | queue->index_count = data; 731 | } else if (strncmp(buf, "indexSpacing", sz) == 0) { 732 | queue->index_spacing = data; 733 | } else if (strncmp(buf, "epoch", sz) == 0) { 734 | if (debug) printf(" v4 roll_epoch set to %" PRIu64 "\n", data); 735 | queue->roll_epoch = data; 736 | } 737 | } 738 | 739 | void handle_qf_text(char* buf, int sz, char* data, int dsz, wirecallbacks_t* cbs) { 740 | queue_t* queue = (queue_t*)cbs->userdata; 741 | if (strncmp(buf, "format", sz) == 0) { 742 | if (debug) printf(" v4 roll_format qf set to %.*s\n", dsz, data); 743 | free(queue->roll_format); 744 | queue->roll_format = strndup(data, dsz); 745 | } 746 | } 747 | 748 | void parse_dirlist(queue_t* queue) { 749 | // dirlist mmap is the size of the fstat 750 | int lim = queue->dirlist_statbuf.st_size; 751 | unsigned char* base = queue->dirlist; 752 | uint64_t index = 0; 753 | // used to dump out the test data for test_wire.c 754 | // printbuf((char*)base, lim); 755 | 756 | wirecallbacks_t cbs; 757 | bzero(&cbs, sizeof(cbs)); 758 | cbs.ptr_uint64 = &handle_dirlist_ptr; 759 | cbs.userdata = queue; 760 | 761 | wirecallbacks_t hcbs; 762 | bzero(&hcbs, sizeof(hcbs)); 763 | hcbs.field_uint64 = &handle_dirlist_uint64; 764 | hcbs.field_char = &handle_dirlist_text; 765 | hcbs.userdata = queue; 766 | parse_queue_block(queue, &base, &index, base+lim, &hcbs, &parse_wire_data, &cbs); 767 | } 768 | 769 | void parse_queuefile_meta(unsigned char* base, int limit, queue_t* queue) { 770 | uint64_t index = 0; 771 | 772 | wirecallbacks_t hcbs; 773 | bzero(&hcbs, sizeof(hcbs)); 774 | hcbs.field_uint64 = &handle_qf_uint64; 775 | hcbs.field_char = &handle_qf_text; 776 | hcbs.userdata = queue; 777 | parse_queue_block(queue, &base, &index, base+limit, &hcbs, NULL, NULL); 778 | } 779 | 780 | void chronicle_peek() { 781 | queue_t *queue = queue_head; 782 | while (queue != NULL) { 783 | chronicle_peek_queue(queue); 784 | queue = queue->next; 785 | } 786 | } 787 | 788 | void peek_queue_modcount(queue_t* queue) { 789 | // poll shared directory for modcount 790 | uint64_t modcount; 791 | memcpy(&modcount, queue->dirlist_fields.modcount, sizeof(modcount)); 792 | 793 | if (queue->modcount != modcount) { 794 | printf("shmipc: %s modcount changed from %" PRIu64 " to %" PRIu64 "\n", queue->dirname, queue->modcount, modcount); 795 | // slowpath poll 796 | memcpy(&queue->modcount, queue->dirlist_fields.modcount, sizeof(modcount)); 797 | memcpy(&queue->lowest_cycle, queue->dirlist_fields.lowest_cycle, sizeof(modcount)); 798 | memcpy(&queue->highest_cycle, queue->dirlist_fields.highest_cycle, sizeof(modcount)); 799 | } 800 | } 801 | 802 | void poke_queue_modcount(queue_t* queue) { 803 | // push modifications to lowestCycle, highestCycle to directory-listing mmap 804 | // and atomically increment the modcount 805 | uint64_t modcount; 806 | memcpy(queue->dirlist_fields.highest_cycle, &queue->highest_cycle, sizeof(modcount)); 807 | memcpy(queue->dirlist_fields.lowest_cycle, &queue->lowest_cycle, sizeof(modcount)); 808 | lock_xadd(queue->dirlist_fields.modcount, 1); 809 | printf("shmipc: bumped modcount\n"); 810 | } 811 | 812 | void chronicle_peek_queue(queue_t *queue) { 813 | if (debug) printf("peeking at %s\n", queue->dirname); 814 | peek_queue_modcount(queue); 815 | 816 | tailer_t *tailer = queue->tailers; 817 | while (tailer != NULL) { 818 | chronicle_peek_queue_tailer(queue, tailer); 819 | 820 | tailer = tailer->next; 821 | } 822 | } 823 | 824 | tailstate_t chronicle_peek_queue_tailer_r(queue_t *queue, tailer_t *tailer) { 825 | // for each cycle file { for each block { for each entry { emit }}} 826 | // this method runs like a generator, suspended in the innermost 827 | // iteration when we hit the end of the file and pick up at the next peek() 828 | wirecallbacks_t hcbs; 829 | bzero(&hcbs, sizeof(hcbs)); 830 | 831 | while (1) { 832 | 833 | uint64_t cycle = tailer->qf_index >> queue->cycle_shift; 834 | if (cycle != tailer->qf_cycle_open || tailer->qf_fn == NULL) { 835 | // free fn, mmap and fid 836 | if (tailer->qf_fn) { 837 | free(tailer->qf_fn); 838 | } 839 | if (tailer->qf_buf) { 840 | munmap(tailer->qf_buf, tailer->qf_mmapsz); 841 | tailer->qf_buf = NULL; 842 | } 843 | if (tailer->qf_fd > 0) { // close the fid if open 844 | close(tailer->qf_fd); 845 | } 846 | tailer->qf_fn = chronicle_get_cycle_fn(queue, cycle); 847 | tailer->qf_tip = 0; 848 | 849 | printf("shmipc: opening cycle %" PRIu64 " filename %s (highest_cycle %" PRIu64 ")\n", cycle, tailer->qf_fn, queue->highest_cycle); 850 | int fopen_flags = O_RDONLY; 851 | if (tailer->mmap_protection != PROT_READ) fopen_flags = O_RDWR; 852 | if ((tailer->qf_fd = open(tailer->qf_fn, fopen_flags)) < 0) { 853 | printf("shmipc: awaiting queuefile for %s open errno=%d %s\n", tailer->qf_fn, errno, strerror(errno)); 854 | 855 | // if our cycle < highCycle, permitted to skip a missing file rather than wait 856 | if (cycle < queue->highest_cycle) { 857 | uint64_t skip_to_index = (cycle + 1) << queue->cycle_shift; 858 | printf("shmipc: skipping queuefile (cycle < highest_cycle), bumping next_index from %" PRIu64 " to %" PRIu64 "\n", tailer->qf_index, skip_to_index); 859 | tailer->qf_index = skip_to_index; 860 | continue; 861 | } 862 | return TS_AWAITING_QUEUEFILE; 863 | } 864 | tailer->qf_cycle_open = cycle; 865 | 866 | // renew the stat 867 | if (fstat(tailer->qf_fd, &tailer->qf_statbuf) < 0) return 3; 868 | } 869 | 870 | // assert: we have open fid 871 | 872 | // qf_tip 873 | // file 0 v stat.sz 874 | // [-------------#######-------] 875 | // map ####### 876 | // ^ ^ 877 | // mmapoff +mmapsz 878 | // | basep 879 | // addr qf_buf 880 | // assign mmap limit and offset from tip and blocksize 881 | 882 | // note: blocksize may have changed, unroll this to a constant with care 883 | uint64_t blocksize_mask = ~(queue->blocksize-1); 884 | uint64_t mmapoff = tailer->qf_tip & blocksize_mask; 885 | 886 | // renew stat if we would otherwise map less than 2* blocksize 887 | // TODO: write needs to extend file here! 888 | if (tailer->qf_statbuf.st_size - mmapoff < 2*queue->blocksize) { 889 | if (debug) printf("shmmain: approaching file size limit, less than two blocks remain\n"); 890 | if (fstat(tailer->qf_fd, &tailer->qf_statbuf) < 0) 891 | return TS_E_STAT; 892 | // signal to extend queuefile iff we are an appending tailer 893 | if (tailer->qf_statbuf.st_size - mmapoff < 2*queue->blocksize && tailer->mmap_protection != PROT_READ) { 894 | return TS_EXTEND_FAIL; 895 | } 896 | } 897 | 898 | int limit = tailer->qf_statbuf.st_size - mmapoff > 2*queue->blocksize ? 2*queue->blocksize : tailer->qf_statbuf.st_size - mmapoff; 899 | if (debug) printf("shmipc: tip %" PRIu64 " -> mmapoff %" PRIu64 " size 0x%x blocksize_mask 0x%" PRIx64 "\n", tailer->qf_tip, mmapoff, limit, blocksize_mask); 900 | 901 | // only re-mmap if desired window has changed since last scan 902 | if (tailer->qf_buf == NULL || mmapoff != tailer->qf_mmapoff || limit != tailer->qf_mmapsz) { 903 | if ((tailer->qf_buf)) { 904 | munmap(tailer->qf_buf, tailer->qf_mmapsz); 905 | tailer->qf_buf = NULL; 906 | } 907 | 908 | tailer->qf_mmapsz = limit; 909 | tailer->qf_mmapoff = mmapoff; 910 | if ((tailer->qf_buf = mmap(0, tailer->qf_mmapsz, tailer->mmap_protection, MAP_SHARED, tailer->qf_fd, tailer->qf_mmapoff)) == MAP_FAILED) { 911 | printf("shmipc: mmap failed %s %" PRIx64 " size %" PRIx64 " error=%s\n", tailer->qf_fn, tailer->qf_mmapoff, tailer->qf_mmapsz, strerror(errno)); 912 | tailer->qf_buf = NULL; 913 | return TS_E_MMAP; 914 | } 915 | printf("shmipc: mmap offset %" PRIx64 " size %" PRIx64 " base=%p extent=%p\n", tailer->qf_mmapoff, tailer->qf_mmapsz, tailer->qf_buf, tailer->qf_buf+tailer->qf_mmapsz); 916 | } 917 | 918 | unsigned char* basep = (tailer->qf_tip - tailer->qf_mmapoff) + tailer->qf_buf; // basep within mmap 919 | unsigned char* basep_old = basep; 920 | unsigned char* extent = tailer->qf_buf+tailer->qf_mmapsz; 921 | uint64_t index = tailer->qf_index; 922 | 923 | // 0 awaiting at &base (pass) 924 | // 1 we hit working (pass) 925 | // 2 we hit EOF (handle) 926 | // 3 data extent will cross base+limit (handle) 927 | // 4 hit data with no data parser (won't happen) 928 | // 7 collected item 929 | // if any entries are read the values at basep and indexp are updated 930 | parseqb_state_t s = parse_queue_block(queue, &basep, &index, extent, &hcbs, parse_data_cb, tailer); 931 | //printf("shmipc: block parser result %d, shm %p to %p\n", s, basep_old, basep); 932 | 933 | if (s == QB_NEED_EXTEND && basep == basep_old) { 934 | queue_double_blocksize(queue); 935 | } 936 | 937 | if (basep != basep_old) { 938 | // commit result of parsing to the tailer, adjusting for the window 939 | uint64_t new_tip = basep-tailer->qf_buf + tailer->qf_mmapoff; 940 | if (debug) printf("shmipc: parser moved shm %p to %p, file %" PRIu64 " -> %" PRIu64 ", index %" PRIu64 " to %" PRIu64 "\n", basep_old, basep, tailer->qf_tip, new_tip, tailer->qf_index, index); 941 | tailer->qf_tip = new_tip; 942 | tailer->qf_index = index; 943 | } 944 | 945 | if (s == QB_BUSY) return TS_BUSY; 946 | if (s == QB_COLLECTED) return TS_COLLECTED; 947 | 948 | if (s == QB_AWAITING_ENTRY) { // awaiting at end of queuefile 949 | if (cycle < queue->highest_cycle-patch_cycles) { // allowed to fast-forward 950 | uint64_t skip_to_index = (cycle + 1) << queue->cycle_shift; 951 | printf("shmipc: missing EOF for queuefile (cycle < highest_cycle-patch_cycles), bumping next_index from %" PRIu64 " to %" PRIu64 "\n", tailer->qf_index, skip_to_index); 952 | tailer->qf_index = skip_to_index; 953 | continue; 954 | } 955 | return TS_AWAITING_ENTRY; 956 | } 957 | 958 | if (s == QB_REACHED_EOF) { 959 | // we've read an EOF marker, so the next expected index is cycle++, seqnum=0 960 | uint64_t eof_cycle = ((tailer->qf_index >> queue->cycle_shift) + 1) << queue->cycle_shift; 961 | printf("shmipc: hit EOF marker, setting next_index from %" PRIu64 " to %" PRIu64 "\n", tailer->qf_index, eof_cycle); 962 | tailer->qf_index = eof_cycle; 963 | } 964 | } 965 | } 966 | 967 | int chronicle_peek_tailer(tailer_t *tailer) { 968 | return chronicle_peek_queue_tailer(tailer->queue, tailer); 969 | } 970 | 971 | int chronicle_peek_queue_tailer(queue_t *queue, tailer_t *tailer) { 972 | return tailer->state = chronicle_peek_queue_tailer_r(queue, tailer); 973 | } 974 | 975 | void chronicle_debug() { 976 | printf("shmipc: open handles\n"); 977 | 978 | queue_t *current = queue_head; 979 | while (current != NULL) { 980 | printf(" directory %s\n", current->dirname); 981 | printf(" handle %p\n", current); 982 | printf(" blocksize %x\n", current->blocksize); 983 | printf(" version %d\n", current->version); 984 | printf(" create %d\n", current->create); 985 | printf(" dirlist_name %s\n", current->dirlist_name); 986 | printf(" dirlist_fd %d\n", current->dirlist_fd); 987 | printf(" dirlist_sz %" PRIu64 "\n", (uint64_t)current->dirlist_statbuf.st_size); 988 | printf(" dirlist %p\n", current->dirlist); 989 | printf(" cycle-low %" PRIu64 "\n", current->lowest_cycle); 990 | printf(" cycle-high %" PRIu64 "\n", current->highest_cycle); 991 | printf(" modcount %" PRIu64 "\n", current->modcount); 992 | printf(" queuefile_pattern %s\n", current->queuefile_pattern); 993 | printf(" cycle_shift %d\n", current->cycle_shift); 994 | printf(" roll_epoch %d\n", current->roll_epoch); 995 | printf(" roll_length (ms) %d\n", current->roll_length); 996 | printf(" roll_format %s\n", current->roll_format); 997 | printf(" roll_name %s\n", current->roll_name); 998 | printf(" roll_strftime %s\n", current->roll_strftime); 999 | printf(" index_count %d\n", current->index_count); 1000 | printf(" index_spacing %d\n", current->index_spacing); 1001 | 1002 | printf(" tailers:\n"); 1003 | tailer_t *tailer = current->tailers; // shortcut to save both collections 1004 | while (tailer != NULL) { 1005 | chronicle_debug_tailer(current, tailer); 1006 | tailer = tailer->next; 1007 | } 1008 | printf(" appender:\n"); 1009 | if (current->appender) 1010 | chronicle_debug_tailer(current, current->appender); 1011 | 1012 | current = current->next; 1013 | } 1014 | } 1015 | 1016 | void chronicle_debug_tailer(queue_t* queue, tailer_t* tailer) { 1017 | const char* state_text = tailer_state_messages[tailer->state]; 1018 | printf(" dispatcher %p\n", tailer->dispatcher); 1019 | uint cycle = tailer->dispatch_after >> queue->cycle_shift; 1020 | uint seqnum = tailer->dispatch_after & queue->seqnum_mask; 1021 | printf(" dispatch_after %" PRIu64 " (cycle %u, seqnum %u)\n", tailer->dispatch_after, cycle, seqnum); 1022 | printf(" state %d - %s\n", tailer->state, state_text); 1023 | printf(" qf_fn %s\n", tailer->qf_fn); 1024 | printf(" qf_fd %d\n", tailer->qf_fd); 1025 | printf(" qf_statbuf_sz %" PRIu64 "\n", (uint64_t)tailer->qf_statbuf.st_size); 1026 | printf(" qf_tip %" PRIu64 "\n", tailer->qf_tip); 1027 | cycle = tailer->qf_index >> queue->cycle_shift; 1028 | seqnum = tailer->qf_index & queue->seqnum_mask; 1029 | printf(" qf_index %" PRIu64 " (cycle %u, seqnum %u)\n", tailer->qf_index, cycle, seqnum); 1030 | printf(" qf_buf %p\n", tailer->qf_buf); 1031 | printf(" extent %p\n", tailer->qf_buf+tailer->qf_mmapsz); 1032 | printf(" qf_mmapsz %" PRIx64 "\n", tailer->qf_mmapsz); 1033 | printf(" qf_mmapoff %" PRIx64 "\n", tailer->qf_mmapoff); 1034 | } 1035 | 1036 | uint64_t chronicle_append(queue_t *queue, COBJ msg) { 1037 | long ms = chronicle_clock_ms(queue); 1038 | return chronicle_append_ts(queue, msg, ms); 1039 | } 1040 | 1041 | uint64_t chronicle_append_ts(queue_t *queue, COBJ msg, long ms) { 1042 | if (queue == NULL) return chronicle_err("queue is NULL"); 1043 | 1044 | // Appending logic 1045 | // 0) catch up to the end of the current file. 1046 | // if hit EOF then we need to wait for creation of next file, poll modcount 1047 | // then try opening filehandle 1048 | // Else we may be only writer 1049 | // a) determine if we need to roll to a new file - other writers may be idle 1050 | // call gtod and calculate current cycle 1051 | // 1052 | // - CAS loop to write EOF marker 1053 | // create new queue file 1054 | // update maxcycle and then increment modcount 1055 | // or 1056 | // b) write to current file 1057 | // Is this entry indexable? 1058 | // Is the index currently full 1059 | // Write Entry 1060 | // if mod % index, write back to index 1061 | // if index full, write new index page, write back to index2index 1062 | // when update index page, then write data 1063 | // Before write, CAS operation to put working indicator on last entry 1064 | // if fail, loop waiting for unlock. If EOF then fail is broken, retry 1065 | // if data or index, skip over them and attempt again on newest page 1066 | 1067 | // caution: encodecheck may tweak blocksize, do not redorder below shmipc_peek_tailer 1068 | size_t write_sz = queue->append_sizeof(msg); 1069 | if (write_sz < 0) return 0; 1070 | if (write_sz > HD_MASK_META) return chronicle_err("`shm msg sz > 30bit"); 1071 | while (write_sz > queue->blocksize) 1072 | queue_double_blocksize(queue); 1073 | 1074 | // refresh highest and lowest, allowing our appender to follow another appender 1075 | peek_queue_modcount(queue); 1076 | 1077 | // build a special tailer with the protection bits and file descriptor set to allow 1078 | // writing. 1079 | if (queue->appender == NULL) { 1080 | tailer_t* tailer = malloc(sizeof(tailer_t)); 1081 | if (tailer == NULL) return chronicle_err("am fail"); 1082 | bzero(tailer, sizeof(tailer_t)); 1083 | 1084 | // compat: writers do an extended lookback to patch missing EOFs 1085 | tailer->qf_index = (queue->highest_cycle - patch_cycles) << queue->cycle_shift; 1086 | tailer->dispatcher = NULL; 1087 | tailer->state = 5; 1088 | tailer->mmap_protection = PROT_READ | PROT_WRITE; 1089 | tailer->queue = queue; 1090 | queue->appender = tailer; 1091 | 1092 | // re-open directory-listing mapping in read-write mode 1093 | int x = directory_listing_reopen(queue, O_RDWR, PROT_READ | PROT_WRITE); 1094 | if (x != 0) { 1095 | printf("shmipc: rw dir listing %d %s\n", x, cerr_msg); 1096 | return -1; 1097 | } 1098 | if (debug) printf("shmipc: appender created\n"); 1099 | } 1100 | tailer_t* appender = queue->appender; 1101 | 1102 | // poll the appender 1103 | while (1) { 1104 | int r = chronicle_peek_queue_tailer(queue, appender); 1105 | // TODO: 2nd call defensive to ensure 1 whole blocksize is available to put 1106 | r = chronicle_peek_queue_tailer(queue, appender); 1107 | if (debug) printf("shmipc: writeloop appender in state %d\n", r); 1108 | 1109 | if (r == TS_AWAITING_QUEUEFILE) { 1110 | // our cycle is pointing to a queuefile that does not exist 1111 | // as we are writer, create it with temporary filename, atomically 1112 | // move it to the desired name, then bump the global highest_cycle 1113 | // value if rename succeeded 1114 | char* fn_buf; 1115 | asprintf(&fn_buf, "%s.%d.tmp", appender->qf_fn, pid_header); 1116 | 1117 | // if queuefile_init fails, re-throw the error and abort the write 1118 | if (queuefile_init(fn_buf, queue) != 0) return -1; 1119 | 1120 | if (rename(fn_buf, appender->qf_fn) != 0) { 1121 | // rename failed, maybe raced with another writer, delay and try again 1122 | printf("shmipc: create queuefile %s failed at rename, errno %d\n", fn_buf, errno); 1123 | sleep(1); 1124 | continue; 1125 | } 1126 | printf("renamed %s to %s\n", fn_buf, appender->qf_fn); 1127 | free(fn_buf); 1128 | 1129 | // if our new file higher than highest_cycle, inform listeners by bumping modcount 1130 | uint64_t cyc = appender->qf_index >> queue->cycle_shift; 1131 | if (cyc > queue->highest_cycle) { 1132 | queue->highest_cycle = cyc; 1133 | poke_queue_modcount(queue); 1134 | } 1135 | 1136 | // rename worked, we can now re-try the peek_tailer 1137 | continue; 1138 | } 1139 | 1140 | if (r == TS_EXTEND_FAIL) { 1141 | // current queuefile has less than two blocks remaining, needs extending 1142 | // should the extend fail, we are having disk issues, wait until fixed 1143 | uint64_t extend_to = appender->qf_statbuf.st_size + qf_disk_sz; 1144 | if (lseek(appender->qf_fd, extend_to - 1, SEEK_SET) == -1) { 1145 | printf("shmmain: extend queuefile %s failed at lseek: %s\n", appender->qf_fn, strerror(errno)); 1146 | sleep(1); 1147 | continue; 1148 | } 1149 | if (write(appender->qf_fd, "", 1) != 1) { 1150 | printf("shmmain: extend queuefile %s failed at write: %s\n", appender->qf_fn, strerror(errno)); 1151 | sleep(1); 1152 | continue; 1153 | } 1154 | printf("shmmain: extended queuefile %s to %" PRIu64 " bytes\n", appender->qf_fn, extend_to); 1155 | continue; 1156 | } 1157 | 1158 | // If the tailer returns 0, we are all set pointing to the next unwritten entry. 1159 | // if we write to qf_buf and the state is not zero we'll hit sigbus etc, so sleep 1160 | // and wait for availability. 1161 | if (r != TS_AWAITING_ENTRY) { 1162 | printf("shmipc: Cannot write in state %d, sleeping\n", r); 1163 | sleep(1); 1164 | continue; 1165 | } 1166 | 1167 | if ((appender->qf_tip - appender->qf_mmapoff) + write_sz > appender->qf_mmapsz) { 1168 | printf("aborting on bug: write would segfault buffer!\n"); 1169 | abort(); 1170 | } 1171 | 1172 | // Since appender->buf is pointing at the queue head, so we can 1173 | // LOCK CMPXCHG the working bit directly. If the cas failed, another writer 1174 | // has beaten us to it, we sleep poll the tailer and try again 1175 | // If the file has gone EOF, we re-visit the tailer logic which will adjust 1176 | // the maps and switch to the new file. 1177 | 1178 | // Note that we do not extended qf_buf or qf_index after the write. Let the 1179 | // tailer log handle the entry we've just written in the normal way, since that will 1180 | // adjust the buffer window/mmap for us. 1181 | unsigned char* ptr = (appender->qf_tip - appender->qf_mmapoff) + appender->qf_buf; 1182 | uint32_t ret = lock_cmpxchgl(ptr, HD_UNALLOCATED, HD_WORKING); 1183 | 1184 | // cmpxchg returns the original value in memory, so we can tell if we succeeded 1185 | // by looking for HD_UNALLOCATED. If we read a working bit or finished size, we lost. 1186 | if (ret == HD_UNALLOCATED) { 1187 | asm volatile ("mfence" ::: "memory"); 1188 | 1189 | // if given a clock, test if we should write EOF and advance cycle 1190 | if (ms > 0) { 1191 | uint64_t cyc = chronicle_cycle_from_ms(queue, ms); 1192 | if (cyc > appender->qf_index >> queue->cycle_shift) { 1193 | printf("shmipc: appender setting cycle from timestamp: current %" PRIu64 " proposed %" PRIu64 "\n", appender->qf_index >> queue->cycle_shift, cyc); 1194 | appender->qf_index = cyc << queue->cycle_shift; 1195 | 1196 | printf("shmipc: got write lock, writing EOF to start roll\n"); 1197 | uint32_t header = HD_EOF; 1198 | memcpy(ptr, &header, sizeof(header)); 1199 | continue; // retry write in next queuefile 1200 | } 1201 | } 1202 | 1203 | // Java does not patch EOF on prev files if it is down during the roll, it 1204 | // just starts a new file. As a workaround their readers 'timeout' the wait for EOF 1205 | // if a higher cycle is known for the queue. I'd like to be as correct as possible, so 1206 | // we'll patch missing EOFs during our writes if we hold the lock. This will nudge on any 1207 | // readers who haven't noticed the roll. 1208 | if (appender->qf_index < queue->highest_cycle << queue->cycle_shift) { 1209 | printf("shmipc: got write lock, but about to write to queuefile < maxcycle, writing EOF\n"); 1210 | uint32_t header = HD_EOF; 1211 | memcpy(ptr, &header, sizeof(header)); 1212 | continue; // retry write in next queuefile 1213 | } 1214 | 1215 | queue->append_write(ptr+4, msg, write_sz); 1216 | 1217 | asm volatile ("mfence" ::: "memory"); 1218 | uint32_t header = write_sz & HD_MASK_LENGTH; 1219 | memcpy(ptr, &header, sizeof(header)); 1220 | 1221 | if (debug) printf("shmipc: wrote %zu bytes as index %" PRIu64 "\n", write_sz, appender->qf_index); 1222 | 1223 | break; 1224 | } 1225 | 1226 | printf("shmipc: write lock failed, peeking again\n"); 1227 | sleep(1); 1228 | } 1229 | 1230 | return appender->qf_index; 1231 | } 1232 | 1233 | tailer_t* chronicle_tailer(queue_t *queue, cdispatch_f dispatcher, void* dispatch_ctx, uint64_t index) { 1234 | if (queue == NULL) return chronicle_perr("queue is not valid"); 1235 | 1236 | // decompose index into cycle (file) and seqnum within file 1237 | int cycle = index >> queue->cycle_shift; 1238 | int seqnum = index & queue->seqnum_mask; 1239 | 1240 | printf("shmipc: tailer added index=%" PRIu64 " (cycle=%d seqnum=%d) cb=%p\n", index, cycle, seqnum, dispatcher); 1241 | if (cycle < queue->lowest_cycle) { 1242 | index = queue->lowest_cycle << queue->cycle_shift; 1243 | } 1244 | if (cycle > queue->highest_cycle) { 1245 | index = queue->highest_cycle << queue->cycle_shift; 1246 | } 1247 | 1248 | // allocate struct, we'll link if all checks pass 1249 | tailer_t* tailer = malloc(sizeof(tailer_t)); 1250 | if (tailer == NULL) return chronicle_perr("tm fail"); 1251 | bzero(tailer, sizeof(tailer_t)); 1252 | 1253 | tailer->dispatch_after = index - 1; 1254 | tailer->qf_index = index & ~queue->seqnum_mask; // start replay from first entry in file 1255 | tailer->dispatcher = dispatcher; 1256 | tailer->dispatch_ctx = dispatch_ctx; 1257 | tailer->state = 5; 1258 | tailer->mmap_protection = PROT_READ; 1259 | 1260 | tailer->next = queue->tailers; // linked list 1261 | tailer->prev = NULL; 1262 | if (queue->tailers) queue->tailers->prev = tailer; 1263 | queue->tailers = tailer; 1264 | 1265 | tailer->queue = queue; // parent pointer 1266 | return tailer; 1267 | } 1268 | 1269 | COBJ chronicle_collect(tailer_t *tailer, collected_t *collected) { 1270 | if (tailer == NULL) return chronicle_perr("null tailer"); 1271 | if (collected == NULL) return chronicle_perr("null collected"); 1272 | tailer->collect = collected; 1273 | 1274 | uint64_t delaycount = 0; 1275 | while (1) { 1276 | int r = chronicle_peek_tailer(tailer); 1277 | if (debug) printf("collect value returns %d into object %p\n", r, tailer->collect); 1278 | if (r == TS_COLLECTED) { 1279 | break; 1280 | } 1281 | if (delaycount++ > 20) { 1282 | usleep(delaycount); 1283 | peek_queue_modcount(tailer->queue); 1284 | } 1285 | } 1286 | tailer->collect = NULL; 1287 | return collected->msg; 1288 | } 1289 | 1290 | void chronicle_return(tailer_t *tailer, collected_t *collected) { 1291 | if (tailer->queue->parser && tailer->queue->parser_free) { 1292 | tailer->queue->parser_free(collected->msg); 1293 | } 1294 | } 1295 | 1296 | tailstate_t chronicle_tailer_state(tailer_t* tailer) { 1297 | return tailer->state; 1298 | } 1299 | 1300 | uint64_t chronicle_tailer_index(tailer_t* tailer) { 1301 | return tailer->qf_index; 1302 | } 1303 | 1304 | void chronicle_tailer_close(tailer_t* tailer) { 1305 | if (tailer->qf_fn) { // if next filename cached... 1306 | free(tailer->qf_fn); 1307 | } 1308 | if (tailer->qf_buf) { // if mmap() open... 1309 | munmap(tailer->qf_buf, tailer->qf_mmapsz); 1310 | } 1311 | if (tailer->qf_fd) { // if open() open... 1312 | close(tailer->qf_fd); 1313 | } 1314 | // unlink ourselves from doubly-linked chain and update parent pointer if we were first 1315 | if (tailer->next) { 1316 | tailer->next->prev = tailer->prev; 1317 | } 1318 | if (tailer->prev) { 1319 | tailer->prev->next = tailer->next; 1320 | } else if (tailer->queue->tailers == tailer) { 1321 | tailer->queue->tailers = tailer->next; 1322 | } 1323 | free(tailer); 1324 | } 1325 | 1326 | int chronicle_cleanup(queue_t* queue_delete) { 1327 | if (queue_delete == NULL) return chronicle_err("queue is NULL"); 1328 | 1329 | // check if queue already open 1330 | queue_t **parent = &queue_head; // pointer to a queue_t pointer 1331 | queue_t *queue = queue_head; 1332 | 1333 | while (queue != NULL) { 1334 | if (queue == queue_delete) { 1335 | *parent = queue->next; // unlink 1336 | 1337 | // delete tailers 1338 | tailer_t *tailer = queue->tailers; // shortcut to save both collections 1339 | while (tailer != NULL) { 1340 | tailer_t* next_tmp = tailer->next; 1341 | chronicle_tailer_close(tailer); 1342 | tailer = next_tmp; 1343 | } 1344 | queue->tailers = NULL; 1345 | 1346 | if (queue->appender) chronicle_tailer_close(queue->appender); 1347 | 1348 | // kill queue 1349 | munmap(queue->dirlist, queue->dirlist_statbuf.st_size); 1350 | if (queue->dirlist_fd > 0) { 1351 | close(queue->dirlist_fd); 1352 | } 1353 | free(queue->dirlist_name); 1354 | free(queue->dirname); 1355 | free(queue->queuefile_pattern); 1356 | free(queue->roll_strftime); 1357 | free(queue->roll_format); 1358 | free(queue->roll_name); 1359 | globfree(&queue->queuefile_glob); 1360 | free(queue); 1361 | 1362 | return 0; 1363 | } 1364 | parent = &queue->next; 1365 | queue = queue->next; 1366 | } 1367 | return chronicle_err("chronicle_cleanup: queue not found"); 1368 | } 1369 | 1370 | int queuefile_init(char* fn, queue_t* queue) { 1371 | int fd; 1372 | int mode = 0777; 1373 | 1374 | printf("Creating %s\n", fn); 1375 | 1376 | // open/create the output file 1377 | if ((fd = open(fn, O_RDWR | O_CREAT | O_TRUNC, mode)) < 0) { 1378 | printf("can't create %s for writing", fn); 1379 | return chronicle_err("shmipc: create tmp queuefile err"); 1380 | } 1381 | 1382 | // go to the location corresponding to the last byte 1383 | if (lseek(fd, qf_disk_sz - 1, SEEK_SET) == -1) { 1384 | return chronicle_err("shmipc: lseek error"); 1385 | } 1386 | 1387 | // write a dummy byte at the last location 1388 | if (write(fd, "", 1) != 1) { 1389 | return chronicle_err("shmipc: write error"); 1390 | } 1391 | 1392 | // TODO: write header 1393 | // TODO: write index2index 1394 | printf("Created %s\n", fn); 1395 | 1396 | close(fd); 1397 | return 0; 1398 | } 1399 | 1400 | int directory_listing_init(queue_t* queue, uint64_t cycle) { 1401 | int fd; 1402 | int mode = 0777; 1403 | 1404 | if ((fd = open(queue->dirlist_name, O_RDWR | O_CREAT | O_TRUNC, mode)) < 0) { 1405 | printf("can't create %s for writing", queue->dirlist_name); 1406 | return chronicle_err("shmipc: directory_listing_init open failed"); 1407 | } 1408 | 1409 | wirepad_t* pad = wirepad_init(1024); 1410 | 1411 | // deliberately not updating queue->roll_epoch here so that we verify 1412 | // we can read them back from the directory_listing file 1413 | int roll_epoch = (queue->roll_epoch == -1) ? 0 : queue->roll_epoch; 1414 | int modcount = 1; 1415 | 1416 | // single metadata message 1417 | wirepad_qc_start(pad, 1); 1418 | wirepad_event_name(pad, "header"); 1419 | wirepad_type_prefix(pad, "STStore"); 1420 | wirepad_nest_enter(pad); //header 1421 | wirepad_field_type_enum(pad, "wireType", "WireType", "BINARY_LIGHT"); 1422 | // field metadata, type prefix SCQMeta, nesting begin 1423 | wirepad_field(pad, "metadata"); 1424 | wirepad_type_prefix(pad, "SCQMeta"); 1425 | wirepad_nest_enter(pad); 1426 | wirepad_field(pad, "roll"); 1427 | wirepad_type_prefix(pad, "SCQSRoll"); 1428 | wirepad_nest_enter(pad); 1429 | wirepad_field_varint(pad, "length", queue->roll_length); 1430 | wirepad_field_text(pad, "format", queue->roll_format); 1431 | wirepad_field_varint(pad, "epoch", roll_epoch); 1432 | wirepad_nest_exit(pad); 1433 | wirepad_field_varint(pad, "deltaCheckpointInterval", 64); 1434 | wirepad_field_varint(pad, "sourceId", 0); 1435 | wirepad_nest_exit(pad); 1436 | wirepad_pad_to_x8(pad); // feels wrong - should be automatic? 1437 | wirepad_nest_exit(pad); 1438 | wirepad_qc_finish(pad); 1439 | 1440 | // 6 data messages 1441 | wirepad_qc_start(pad, 0); 1442 | wirepad_event_name(pad, "listing.highestCycle"); 1443 | wirepad_uint64_aligned(pad, cycle); // this cannot be varint as memory mapped! explicit size 1444 | wirepad_qc_finish(pad); 1445 | 1446 | wirepad_qc_start(pad, 0); 1447 | wirepad_event_name(pad, "listing.lowestCycle"); 1448 | wirepad_uint64_aligned(pad, cycle); 1449 | wirepad_qc_finish(pad); 1450 | 1451 | wirepad_qc_start(pad, 0); 1452 | wirepad_event_name(pad, "listing.modCount"); 1453 | wirepad_uint64_aligned(pad, modcount); 1454 | wirepad_qc_finish(pad); 1455 | 1456 | wirepad_qc_start(pad, 0); 1457 | wirepad_event_name(pad, "chronicle.write.lock"); 1458 | wirepad_uint64_aligned(pad, 0x8000000000000000); 1459 | wirepad_qc_finish(pad); 1460 | 1461 | wirepad_qc_start(pad, 0); 1462 | wirepad_event_name(pad, "chronicle.lastIndexReplicated"); 1463 | wirepad_uint64_aligned(pad, -1); 1464 | wirepad_qc_finish(pad); 1465 | 1466 | wirepad_qc_start(pad, 0); 1467 | wirepad_event_name(pad, "chronicle.lastAcknowledgedIndexReplicated"); 1468 | wirepad_uint64_aligned(pad, -1); 1469 | wirepad_qc_finish(pad); 1470 | 1471 | write(fd, wirepad_base(pad), wirepad_sizeof(pad)); 1472 | close(fd); 1473 | 1474 | wirepad_free(pad); 1475 | return 0; 1476 | } 1477 | 1478 | int directory_listing_reopen(queue_t* queue, int open_flags, int mmap_prot) { 1479 | if ((queue->dirlist_fd = open(queue->dirlist_name, open_flags)) < 0) { 1480 | return chronicle_err("directory_listing_reopen open failed"); 1481 | } 1482 | 1483 | // find size of dirlist and mmap 1484 | if (fstat(queue->dirlist_fd, &queue->dirlist_statbuf) < 0) 1485 | return chronicle_err("dirlist fstat"); 1486 | if ((queue->dirlist = mmap(0, queue->dirlist_statbuf.st_size, mmap_prot, MAP_SHARED, queue->dirlist_fd, 0)) == MAP_FAILED) 1487 | return chronicle_err("dirlist mmap fail"); 1488 | 1489 | if (debug) printf("shmipc: parsing dirlist %s\n", queue->dirlist_name); 1490 | parse_dirlist(queue); 1491 | 1492 | // check the polled fields in header section were all resolved to pointers within the map 1493 | if (queue->dirlist_fields.highest_cycle == NULL || queue->dirlist_fields.lowest_cycle == NULL || 1494 | queue->dirlist_fields.modcount == NULL) { 1495 | return chronicle_err("dirlist parse hdr ptr fail"); 1496 | } 1497 | return 0; 1498 | } 1499 | 1500 | COBJ chronicle_decoder_default_parse(unsigned char* base, int lim) { 1501 | char* msg = calloc(1, lim+1); 1502 | memcpy(msg, base, lim); 1503 | return msg; 1504 | } 1505 | 1506 | size_t chronicle_encoder_default_sizeof(COBJ msg) { 1507 | return strlen(msg); 1508 | } 1509 | 1510 | void chronicle_encoder_default_write(unsigned char* base, COBJ msg, size_t sz) { 1511 | memcpy(base, msg, sz); 1512 | } 1513 | -------------------------------------------------------------------------------- /native/libchronicle.h: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Tea Engineering Ltd. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef FILE_LIBCHRONICLE_SEEN 16 | #define FILE_LIBCHRONICLE_SEEN 17 | 18 | #define _GNU_SOURCE 19 | 20 | #define __STDC_FORMAT_MACROS 21 | #include 22 | 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | 33 | #define MAXDATASIZE 1000 // max number of bytes we can get at once 34 | 35 | // Chronicle header special bits 36 | #define HD_UNALLOCATED 0x00000000 37 | #define HD_WORKING 0x80000000 38 | #define HD_METADATA 0x40000000 39 | #define HD_EOF 0xC0000000 40 | #define HD_MASK_LENGTH 0x3FFFFFFF 41 | #define HD_MASK_META HD_EOF 42 | 43 | // chronicle_init flags values (OR values together) 44 | #define CHRONICLE_FLAGS_ANY 45 | #define CHRONICLE_FLAGS_V4 46 | #define CHRONICLE_FLAGS_V5 47 | 48 | #define CHRONICLE_FLAGS_RW 49 | #define CHRONICLE_FLAGS_CREATE 50 | 51 | 52 | 53 | // Public interface 54 | // your glue code will need to cast COBJ in callbacks, by implmenting 55 | // four callbacks. 56 | typedef void* COBJ; 57 | typedef void* DISPATCH_CTX; 58 | // 59 | // cparse_f takes void* and returns custom object. Deserialise, memcpy 60 | // or return same ptr to dispatch ref valid for callback. 61 | // csizeof_f tells library how many bytes required to serialise user object 62 | // cappend_f takes custom object and writes bytes to void* 63 | // cdispatch_f takes custom object and index, delivers to application with user data 64 | typedef COBJ (*cparse_f) (unsigned char*, int); 65 | typedef void (*cparsefree_f)(COBJ); 66 | typedef size_t (*csizeof_f) (COBJ); 67 | typedef void (*cappend_f) (unsigned char*,COBJ,size_t); 68 | typedef int (*cdispatch_f) (DISPATCH_CTX,uint64_t,COBJ); 69 | 70 | // forward definition of queue 71 | typedef struct queue queue_t; 72 | typedef struct tailer tailer_t; 73 | 74 | // return codes exposed via. chronicle_tailer_state 75 | // 0 awaiting next entry 76 | // 1 hit working 77 | // 2 missing queuefile indicated, awaiting advance or creation 78 | // 3 fstat failed 79 | // 4 mmap failed (probably fatal) 80 | // 5 not yet polled 81 | // 6 queuefile at fid needs extending on disk 82 | // 7 a value was collected 83 | typedef enum {TS_AWAITING_ENTRY, TS_BUSY, TS_AWAITING_QUEUEFILE, TS_E_STAT, TS_E_MMAP, TS_PEEK, TS_EXTEND_FAIL, TS_COLLECTED} tailstate_t; 84 | 85 | // collect structure - we complete values for the caller 86 | typedef struct { 87 | COBJ msg; 88 | size_t sz; 89 | uint64_t index; 90 | } collected_t; 91 | 92 | queue_t* chronicle_init(char* dir); 93 | void chronicle_set_version(queue_t* queue, int version); 94 | int chronicle_set_roll_scheme(queue_t* queue, char* scheme); 95 | int chronicle_set_roll_dateformat(queue_t* queue, char* scheme); 96 | void chronicle_set_encoder(queue_t* queue, csizeof_f append_sizeof, cappend_f append_write); 97 | void chronicle_set_decoder(queue_t* queue, cparse_f parser, cparsefree_f parsefree); 98 | void chronicle_set_create(queue_t* queue, int create); 99 | int chronicle_open(queue_t* queue); 100 | int chronicle_cleanup(queue_t* queue); 101 | 102 | COBJ chronicle_decoder_default_parse(unsigned char*, int); 103 | size_t chronicle_encoder_default_sizeof(COBJ); 104 | void chronicle_encoder_default_write(unsigned char*,COBJ,size_t); 105 | 106 | int chronicle_get_version(queue_t* queue); 107 | char* chronicle_get_roll_scheme(queue_t* queue); 108 | char* chronicle_get_roll_format(queue_t* queue); 109 | char* chronicle_get_cycle_fn(queue_t* queue, int cycle); 110 | 111 | 112 | const char* chronicle_strerror(); 113 | 114 | tailer_t* chronicle_tailer(queue_t *queue, cdispatch_f dispatcher, DISPATCH_CTX dispatch_ctx, uint64_t index); 115 | void chronicle_tailer_close(tailer_t* tailer); 116 | tailstate_t chronicle_tailer_state(tailer_t* tailer); 117 | uint64_t chronicle_tailer_index(tailer_t* tailer); 118 | 119 | void chronicle_peek(); 120 | void chronicle_peek_queue(queue_t *queue); 121 | int chronicle_peek_tailer(tailer_t *tailer); 122 | 123 | void chronicle_debug(); 124 | void chronicle_debug_tailer(queue_t* queue, tailer_t* tailer); 125 | 126 | uint64_t chronicle_append(queue_t *queue, COBJ msg); 127 | uint64_t chronicle_append_ts(queue_t *queue, COBJ msg, long ms); 128 | 129 | COBJ chronicle_collect(tailer_t *tailer, collected_t *collect); 130 | void chronicle_return(tailer_t *tailer, collected_t *collect); 131 | 132 | struct ROLL_SCHEME { 133 | char* name; 134 | char* formatstr; 135 | uint32_t roll_length_secs; 136 | uint32_t entries; 137 | uint32_t index; 138 | }; 139 | 140 | extern struct ROLL_SCHEME chronicle_roll_schemes[]; 141 | 142 | #endif 143 | -------------------------------------------------------------------------------- /native/mock_k.h: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Tea Engineering Ltd. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // stub out kx layer (the extern function in k.h) so we can run in main, profile, 16 | // valgrind native code etc. 17 | // 18 | // It's impossible to find the source of memory leaks with the Kx slab allocator 19 | // as valgrind can't hook the individual allocations to record the stack. 20 | // 21 | 22 | #include 23 | #define KERR -128 24 | 25 | // globals 26 | int kxx_errno = 0; 27 | char* kxx_msg = NULL; 28 | 29 | K krr(const S msg) { 30 | printf("'%s\n", msg); 31 | kxx_errno = -1; 32 | kxx_msg = msg; 33 | return (K)NULL; 34 | } 35 | K orr(const S msg) { 36 | printf("%s\n", msg); 37 | kxx_errno = -1; 38 | kxx_msg = msg; 39 | return (K)NULL; 40 | } 41 | K ee(K ignored) { 42 | if (kxx_errno != 0) { 43 | K r = ktn(KERR, 1); 44 | r->s = kxx_msg; 45 | kxx_errno = 0; 46 | return r; 47 | } 48 | return ignored; 49 | } 50 | K ki(int i) { 51 | K r = ktn(-KI, 0); 52 | r->i = i; 53 | return r; 54 | } 55 | K kj(long long i) { 56 | K r = ktn(-KJ, 0); 57 | r->j = i; 58 | return r; 59 | } 60 | K kss(const char* ss) { 61 | K r = ktn(-KS, 0); 62 | r->s = (char*)ss; 63 | return r; 64 | } 65 | K dl(void* fnptr, long long n) { 66 | K r = ktn(100,0); 67 | r->s = fnptr; 68 | r->a = n; 69 | return r; 70 | } 71 | 72 | typedef K (*kfunc_1arg)(K); 73 | typedef K (*kfunc_2arg)(K,K); 74 | typedef K (*kfunc_3arg)(K,K,K); 75 | 76 | K dot(K x, K y) { // call function pointer in x with args in mixed list y 77 | if (x->t != 100) return krr("x must be fptr"); 78 | if (y->t != 0) return krr("y must be list"); 79 | if (x->a == 2) { 80 | kfunc_2arg fptr = (kfunc_2arg)x->s; 81 | return fptr(kK(y)[0], kK(y)[1]); 82 | } 83 | return ki(1); 84 | } 85 | 86 | K knk(int n, ...) { // create a mixed list from K's in varg 87 | va_list ap; 88 | K r = ktn(0, n); 89 | va_start(ap, n); //Requires the last fixed parameter (to get the address) 90 | for(int j=0; j 19) ? 8 : sizefor[abs(type)]; 101 | K r = malloc(sizeof(struct k0) + n*sz); 102 | r->r = 0; 103 | r->t = type; 104 | if (n > 0) r->n = n; // keep: trap accessing n for atom in valgrind 105 | return r; 106 | } 107 | 108 | // dummy serialiser returns a single byte array [SOH] 109 | K b9(I mode, K obj) { 110 | K r = ktn(KB,1); 111 | r->G0[0] = 1; 112 | return r; 113 | } 114 | 115 | K d9(K obj) { 116 | return ki(1); 117 | } 118 | 119 | int okx(K obj) { return 1; } 120 | // repl equivelent wrapper (protected eval, with and without gc) 121 | K pe(K x) { 122 | if (kxx_errno != 0) exit(-1); 123 | return x; 124 | } 125 | void per(K x) { 126 | pe(x); 127 | if (x != NULL) r0(x); 128 | } 129 | 130 | void r0(K x) { // Decrement the object‘s reference count 131 | if (x == 0) { printf("Bug r0 of null pointer %p\n", x); return; } 132 | if (x->r < 0) printf("Bug double-free of %p\n", x); 133 | if (x->r == 0) { 134 | if (x->t == 0) for (int i = 0; i < x->n; i++) r0(kK(x)[i]); 135 | // flip? 136 | // dict? 137 | free(x); 138 | } else { 139 | x->r--; 140 | } 141 | } 142 | K r1(K x) { // Increment the object‘s reference count 143 | x->r++; 144 | return x; 145 | } 146 | 147 | -------------------------------------------------------------------------------- /native/obj/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore everything in this directory 2 | * 3 | # Except this file 4 | !.gitignore -------------------------------------------------------------------------------- /native/serdes_k.h: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Tea Engineering Ltd. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // allows reading and writing K objects to a libchronicle queue 16 | // link with either KDBs exported functions or mock_k.h 17 | // 18 | // sample use: 19 | // 20 | // queue_t* queue = chronicle_init(dirs); 21 | // chronicle_set_decoder(queue, &parse_kx, &free_kx); 22 | // chronicle_set_encoder(queue, &sizeof_kx, &append_kx); 23 | 24 | #include "libchronicle.h" 25 | 26 | COBJ parse_kx(unsigned char* base, int lim) { 27 | // prep args and fire callback 28 | K msg = ktn(KG, lim); 29 | memcpy((char*)msg->G0, base, lim); 30 | int ok = okx(msg); 31 | if (ok) { 32 | K out = d9(msg); 33 | r0(msg); 34 | return out; 35 | } else { 36 | // if (debug) printf("shmipc: caution index okx returned bytes !ok, skipping\n"); 37 | return NULL; 38 | } 39 | return 0; 40 | } 41 | 42 | // the encoding via. b9 happens in shmipc_append, so here we just 43 | // write the bytes 44 | void append_kx(unsigned char* base, COBJ msg, size_t lim) { 45 | K m = (K)msg; 46 | memcpy(base, (char*)m->G0, m->n); 47 | } 48 | 49 | size_t sizeof_kx(COBJ msg) { 50 | // if (debug) printf("shmipc: kx persist needs %lld bytes\n", msg->n); 51 | K m = (K)msg; 52 | return m->n; 53 | } 54 | 55 | void free_kx(COBJ msg) { 56 | K m = (K)msg; 57 | r0(m); 58 | } 59 | 60 | -------------------------------------------------------------------------------- /native/shm_example_reader.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | static volatile int keepRunning = 1; 6 | 7 | // This is a stand-alone tool for reading a queue 8 | // queue data is null-terminated strings, embedded nulls will truncate printing 9 | void* parse_msg(unsigned char* base, int lim) { 10 | char* msg = calloc(1, lim+1); 11 | memcpy(msg, base, lim); 12 | return msg; 13 | } 14 | 15 | void free_msg(void* msg) { 16 | free(msg); 17 | } 18 | 19 | int print_msg(void* ctx, uint64_t index, void* msg) { 20 | printf("[%" PRIu64 "] %s\n", index, (char*)msg); 21 | return 0; 22 | } 23 | 24 | void sigint_handler(int dummy) { 25 | keepRunning = 0; 26 | } 27 | 28 | int main(const int argc, char **argv) { 29 | signal(SIGINT, sigint_handler); 30 | queue_t* queue = chronicle_init(argv[1]); 31 | chronicle_set_decoder(queue, &parse_msg, &free_msg); 32 | if (chronicle_open(queue) != 0) exit(-1); 33 | chronicle_tailer(queue, &print_msg, NULL, 0); 34 | 35 | while (keepRunning) { 36 | usleep(500*1000); 37 | chronicle_peek(); 38 | } 39 | printf("exiting\n"); 40 | chronicle_cleanup(queue); 41 | } 42 | 43 | -------------------------------------------------------------------------------- /native/shm_example_writer.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | // This is a stand-alone tool for writing a queue 5 | // queue data is null-terminated strings, embedded nulls will truncate printing 6 | void append_msg(unsigned char* base, void* msg, size_t sz) { 7 | memcpy(base, msg, sz); 8 | } 9 | 10 | size_t sizeof_msg(void* msg) { 11 | return strlen(msg); 12 | } 13 | 14 | int main(const int argc, char **argv) { 15 | queue_t* queue = chronicle_init(argv[1]); 16 | chronicle_set_encoder(queue, &sizeof_msg, &append_msg); 17 | chronicle_set_version(queue, 5); 18 | chronicle_set_roll_scheme(queue, "FAST_HOURLY"); 19 | chronicle_set_create(queue, 1); 20 | if (chronicle_open(queue) != 0) exit(-1); 21 | 22 | char line[1024]; 23 | while (1) { 24 | char* g = fgets(line, 1024, stdin); 25 | if (g == NULL) break; 26 | line[strlen(line) - 1] = 0; // remove line break 27 | long int index = chronicle_append(queue, line); 28 | printf("[%" PRIu64 "] %s\n", index, (char*)g); 29 | } 30 | chronicle_cleanup(queue); 31 | } 32 | -------------------------------------------------------------------------------- /native/shmmain.c: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Tea Engineering Ltd. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | // This is a stand-alone tool for replaying a queue, and optionally writing to it. 21 | // It is compatible with java InputMain / OutputMain, ie. the data payloads are 22 | // wire-format encoded. we use wire.h to encode/decode this. 23 | int print_msg(void* ctx, uint64_t index, COBJ y) { 24 | printf("[%" PRIu64 "] %s\n", index, (char*)y); 25 | return 0; 26 | } 27 | 28 | int main(const int argc, char **argv) { 29 | int c; 30 | opterr = 0; 31 | int verboseflag = 0; 32 | int followflag = 0; 33 | char* append = NULL; 34 | uint64_t index = 0; 35 | int can_create = 0; 36 | 37 | while ((c = getopt(argc, argv, "i:va:cf")) != -1) 38 | switch (c) { 39 | case 'i': 40 | index = strtoull(optarg, NULL, 0); 41 | break; 42 | case 'v': 43 | verboseflag = 1; 44 | break; 45 | case 'a': 46 | append = optarg; 47 | break; 48 | case 'c': 49 | can_create = 1; 50 | break; 51 | case 'f': 52 | followflag = 1; 53 | break; 54 | case '?': 55 | fprintf(stderr, "Option '-%c' missing argument\n", optopt); 56 | exit(2); 57 | default: 58 | fprintf(stderr, "Unknown option '%c'\n", c); 59 | exit(3); 60 | } 61 | 62 | if (optind + 1 > argc) { 63 | printf("Missing mandatory argument.\n Expected: %s [-d] [-m] [-i INDEX] [-v] [-a text] [-f] QUEUE\n", argv[0]); 64 | printf(" -i INDEX resume from index\n"); 65 | printf(" -v verbose mode\n"); 66 | printf(" -a TEXT write value text\n"); 67 | printf(" -c create QUEUE directory and files if it does not exist\n"); 68 | printf(" -f follow queue for more entries (rather than exit)\n"); 69 | printf(" -4 -5 set expected version"); 70 | printf("\n"); 71 | printf("shmmain opens the chronicle-queue directory QUEUE and plays all messages from INDEX\n"); 72 | printf("adding -c allows the queue to be created (and any parent directories) if it does not exist.\n"); 73 | printf("setting -d -m -v vary the amount of printing that occurs during this\n"); 74 | printf("once the end of the queue is reached, if append (-a TEXT) is set, we will write a new\n"); 75 | printf("message containing the value TEXT.\n"); 76 | printf("if follow (-f) is set, will we continue to poll for (and print) new messages, else exit.\n"); 77 | exit(1); 78 | } 79 | 80 | char* dir = argv[optind]; 81 | queue_t* queue = chronicle_init(dir); 82 | chronicle_set_encoder(queue, &wirepad_sizeof, &wirepad_write); 83 | chronicle_set_decoder(queue, &wire_parse_textonly, &free); 84 | chronicle_set_create(queue, can_create); 85 | chronicle_set_version(queue, 5); 86 | chronicle_set_roll_scheme(queue, "FAST_DAILY"); 87 | 88 | if (chronicle_open(queue) != 0) { 89 | printf("failed to open %s", chronicle_strerror()); 90 | exit(-1); 91 | } 92 | 93 | wirepad_t* pad = wirepad_init(1024); 94 | 95 | chronicle_tailer(queue, &print_msg, NULL, index); 96 | chronicle_peek(); 97 | 98 | if (append) { 99 | printf("writing %s\n", append); 100 | wirepad_clear(pad); 101 | wirepad_text(pad, append); 102 | chronicle_append(queue, pad); 103 | } 104 | 105 | chronicle_peek(); 106 | 107 | while (followflag) { 108 | usleep(500*1000); 109 | chronicle_peek(); 110 | } 111 | 112 | if (verboseflag) chronicle_debug(); 113 | 114 | chronicle_cleanup(queue); 115 | return 0; 116 | } 117 | -------------------------------------------------------------------------------- /native/test/.gitignore: -------------------------------------------------------------------------------- 1 | fuzz_queue 2 | coverage_queue 3 | -------------------------------------------------------------------------------- /native/test/cqv4-sample-input.tar.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TeaEngineering/libchronicle/33929815012639d34b87766493e7ea97ab86ba4d/native/test/cqv4-sample-input.tar.bz2 -------------------------------------------------------------------------------- /native/test/cqv5-sample-input.tar.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TeaEngineering/libchronicle/33929815012639d34b87766493e7ea97ab86ba4d/native/test/cqv5-sample-input.tar.bz2 -------------------------------------------------------------------------------- /native/test/fuzz_input/manual1: -------------------------------------------------------------------------------- 1 | 1530104836526 100 2 | 50 100 3 | 50 100 4 | 50 100 5 | 50 100 -------------------------------------------------------------------------------- /native/test/fuzz_output/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore everything in this directory 2 | * 3 | # Except this file 4 | !.gitignore -------------------------------------------------------------------------------- /native/test/test_buffer.c: -------------------------------------------------------------------------------- 1 | #define _GNU_SOURCE 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include 12 | 13 | static void test_buffer_hello(void **state) { 14 | 15 | char* f = formatbuf("hello world", 12); 16 | assert_string_equal(f, 17 | "00000000 68 65 6c 6c 6f 20 77 6f 72 6c 64 00 hello wo rld. \n"); 18 | free(f); 19 | 20 | f = formatbuf("hello", 0); 21 | assert_string_equal(f, 22 | "00000000 \n"); 23 | free(f); 24 | 25 | char* p = "The chances of a neutrino actually hitting something... are roughly " 26 | "comparable to that of dropping a ball bearing at random from a cruising " 27 | "747 and hitting, say, an egg sandwich."; 28 | f = formatbuf(p, strlen(p)); 29 | assert_string_equal(f, 30 | "00000000 54 68 65 20 63 68 61 6e 63 65 73 20 6f 66 20 61 The chan ces of a\n" 31 | "00000010 20 6e 65 75 74 72 69 6e 6f 20 61 63 74 75 61 6c neutrin o actual\n" 32 | "00000020 6c 79 20 68 69 74 74 69 6e 67 20 73 6f 6d 65 74 ly hitti ng somet\n" 33 | "00000030 68 69 6e 67 2e 2e 2e 20 61 72 65 20 72 6f 75 67 hing... are roug\n" 34 | "00000040 68 6c 79 20 63 6f 6d 70 61 72 61 62 6c 65 20 74 hly comp arable t\n" 35 | "00000050 6f 20 74 68 61 74 20 6f 66 20 64 72 6f 70 70 69 o that o f droppi\n" 36 | "00000060 6e 67 20 61 20 62 61 6c 6c 20 62 65 61 72 69 6e ng a bal l bearin\n" 37 | "00000070 67 20 61 74 20 72 61 6e 64 6f 6d 20 66 72 6f 6d g at ran dom from\n" 38 | "00000080 20 61 20 63 72 75 69 73 69 6e 67 20 37 34 37 20 a cruis ing 747 \n" 39 | "00000090 61 6e 64 20 68 69 74 74 69 6e 67 2c 20 73 61 79 and hitt ing, say\n" 40 | "000000a0 2c 20 61 6e 20 65 67 67 20 73 61 6e 64 77 69 63 , an egg sandwic\n" 41 | "000000b0 68 2e h. \n" 42 | ); 43 | free(f); 44 | } 45 | 46 | 47 | int main(void) { 48 | const struct CMUnitTest tests[] = { 49 | cmocka_unit_test(test_buffer_hello), 50 | // cmocka_unit_test(), 51 | }; 52 | return cmocka_run_group_tests(tests, NULL, NULL); 53 | } 54 | -------------------------------------------------------------------------------- /native/test/test_queue.c: -------------------------------------------------------------------------------- 1 | #define _GNU_SOURCE 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include 10 | #include 11 | #include "testdata.h" 12 | 13 | char* argv0; 14 | 15 | int print_msg(void* ctx, uint64_t index, COBJ y) { 16 | printf("[%" PRIu64 "] %s\n", index, (char*)y); 17 | return 0; 18 | } 19 | 20 | static void queue_init_cleanup(void **state) { 21 | queue_t* queue = chronicle_init("q2"); 22 | assert_non_null(queue); 23 | chronicle_cleanup(queue); 24 | } 25 | 26 | static void queue_not_exist(void **state) { 27 | queue_t* queue = chronicle_init("q2"); 28 | assert_non_null(queue); 29 | assert_int_not_equal(chronicle_open(queue), 0); 30 | assert_string_equal(chronicle_strerror(), "dir stat fail"); 31 | chronicle_cleanup(queue); 32 | } 33 | 34 | static void queue_is_file(void **state) { 35 | char* temp_dir; 36 | asprintf(&temp_dir, "%s/chronicle.test.XXXXXX", P_tmpdir); 37 | mkstemp(temp_dir); 38 | queue_t* queue = chronicle_init(temp_dir); 39 | assert_non_null(queue); 40 | assert_int_not_equal(chronicle_open(queue), 0); 41 | assert_string_equal(chronicle_strerror(), "dir is not a directory"); 42 | 43 | chronicle_cleanup(queue); 44 | delete_test_data(temp_dir); 45 | free(temp_dir); 46 | } 47 | 48 | static void queue_empty_dir_no_ver(void **state) { 49 | // this will be an empty directory, so we can't tell the version 50 | char* temp_dir; 51 | asprintf(&temp_dir, "%s/chronicle.test.XXXXXX", P_tmpdir); 52 | temp_dir = mkdtemp(temp_dir); 53 | queue_t* queue = chronicle_init(temp_dir); 54 | assert_non_null(queue); 55 | assert_int_not_equal(chronicle_open(queue), 0); 56 | assert_string_equal(chronicle_strerror(), "queue should exist (no permission to create), but version detect failed"); 57 | assert_int_equal(chronicle_get_version(queue), 0); 58 | 59 | chronicle_cleanup(queue); 60 | 61 | delete_test_data(temp_dir); 62 | free(temp_dir); 63 | } 64 | 65 | static void queue_cqv5_sample_input(void **state) { 66 | uint64_t idx = 0; 67 | collected_t result; 68 | 69 | char* test_queuedir = unpack_test_data("cqv5-sample-input.tar.bz2", argv0); 70 | assert_non_null(test_queuedir); 71 | 72 | char* queuedir; 73 | asprintf(&queuedir, "%s/qv5", test_queuedir); 74 | queue_t* queue = chronicle_init(queuedir); 75 | assert_non_null(queue); 76 | 77 | chronicle_set_decoder(queue, &wire_parse_textonly, &free); 78 | chronicle_set_encoder(queue, &wirepad_sizeof, &wirepad_write); 79 | assert_int_equal(chronicle_open(queue), 0); 80 | assert_int_equal(chronicle_get_version(queue), 5); 81 | assert_string_equal(chronicle_get_roll_scheme(queue), "FAST_DAILY"); 82 | 83 | tailer_t* tailer = chronicle_tailer(queue, NULL, NULL, 0); 84 | assert_non_null(tailer); 85 | assert_int_equal(chronicle_tailer_state(tailer), TS_PEEK); 86 | 87 | char* p = (char*)chronicle_collect(tailer, &result); 88 | assert_string_equal("one", p); 89 | assert_int_equal(result.index, 0x4A0500000000); 90 | assert_int_equal(chronicle_tailer_state(tailer), TS_COLLECTED); 91 | chronicle_return(tailer, &result); 92 | 93 | p = (char*)chronicle_collect(tailer, &result); 94 | assert_int_equal(result.index, 0x4A0500000001); 95 | assert_string_equal("two", result.msg); 96 | assert_int_equal(4, result.sz); // TODO: bug, should parser be able to re-write this? 97 | chronicle_return(tailer, &result); 98 | 99 | p = (char*)chronicle_collect(tailer, &result); 100 | assert_int_equal(result.index, 0x4A0500000002); 101 | assert_string_equal("three", result.msg); 102 | chronicle_return(tailer, &result); 103 | 104 | p = (char*)chronicle_collect(tailer, &result); 105 | assert_int_equal(result.index, 0x4A0500000003); 106 | assert_string_equal("a much longer item that will need encoding as variable length text", p); 107 | chronicle_return(tailer, &result); 108 | 109 | wirepad_t* pad = wirepad_init(1024); 110 | wirepad_text(pad, "four five"); 111 | idx = chronicle_append_ts(queue, pad, 1637267400000L); 112 | assert_int_equal(idx, 0x4A0500000004); // easier to see cycle/index split in hex 113 | 114 | // write with timestamp that will match recording of 20211118F 115 | wirepad_clear(pad); 116 | wirepad_text(pad, "six"); 117 | idx = chronicle_append_ts(queue, pad, 1637267400000L); // 20211118T203000 118 | assert_int_equal(idx, 0x4A0500000005); // easier to see cycle/seqnum split in hex 119 | 120 | // write next day's timestamp 121 | wirepad_clear(pad); 122 | wirepad_text(pad, "seven"); 123 | idx = chronicle_append_ts(queue, pad, 1637308800000L); // 20211119T080000 124 | assert_int_equal(idx, 0x4a0600000000); // cycle rolled, seqnum reset 125 | 126 | p = (char*)chronicle_collect(tailer, &result); 127 | assert_string_equal("four five", p); 128 | chronicle_return(tailer, &result); 129 | 130 | p = (char*)chronicle_collect(tailer, &result); 131 | assert_string_equal("six", p); 132 | chronicle_return(tailer, &result); 133 | 134 | chronicle_collect(tailer, &result); 135 | assert_string_equal("seven", result.msg); 136 | assert_int_equal(result.index, 0x4A0600000000); 137 | chronicle_return(tailer, &result); 138 | 139 | chronicle_tailer_close(tailer); 140 | 141 | // add a new tailer starting from midway through 142 | tailer_t* tailer2 = chronicle_tailer(queue, NULL, NULL, 0x4A0500000003); 143 | chronicle_collect(tailer2, &result); 144 | assert_int_equal(result.index, 0x4A0500000003); 145 | assert_string_equal("a much longer item that will need encoding as variable length text", result.msg); 146 | chronicle_return(tailer2, &result); 147 | 148 | chronicle_collect(tailer2, &result); 149 | assert_string_equal("four five", result.msg); 150 | chronicle_return(tailer2, &result); 151 | 152 | chronicle_collect(tailer2, &result); 153 | assert_string_equal("six", result.msg); 154 | chronicle_return(tailer2, &result); 155 | 156 | chronicle_collect(tailer2, &result); 157 | assert_string_equal("seven", result.msg); 158 | chronicle_return(tailer2, &result); 159 | 160 | chronicle_cleanup(queue); 161 | 162 | delete_test_data(test_queuedir); 163 | free(queuedir); 164 | free(test_queuedir); 165 | wirepad_free(pad); 166 | } 167 | 168 | void* parse_cqv4_textonly(unsigned char* base, int lim) { 169 | char* text_result = strndup((const char*) base, lim); 170 | return text_result; 171 | } 172 | 173 | static void queue_cqv4_sample_input(void **state) { 174 | char* test_queuedir = unpack_test_data("cqv4-sample-input.tar.bz2", argv0); 175 | assert_non_null(test_queuedir); 176 | 177 | char* queuedir; 178 | asprintf(&queuedir, "%s/cqv4", test_queuedir); 179 | queue_t* queue = chronicle_init(queuedir); 180 | assert_non_null(queue); 181 | chronicle_set_decoder(queue, &parse_cqv4_textonly, &free); 182 | chronicle_set_encoder(queue, &wirepad_sizeof, &wirepad_write); 183 | assert_int_equal(chronicle_open(queue), 0); 184 | assert_int_equal(chronicle_get_version(queue), 4); 185 | assert_string_equal(chronicle_get_roll_scheme(queue), "DAILY"); 186 | 187 | collected_t result; 188 | 189 | tailer_t* tailer = chronicle_tailer(queue, NULL, NULL, 0); 190 | assert_non_null(tailer); 191 | 192 | char* p = (char*)chronicle_collect(tailer, &result); 193 | assert_non_null(p); 194 | assert_string_equal("one", p); 195 | chronicle_return(tailer, &result); 196 | 197 | chronicle_collect(tailer, &result); 198 | assert_string_equal("two", result.msg); 199 | chronicle_return(tailer, &result); 200 | 201 | p = (char*)chronicle_collect(tailer, &result); 202 | assert_string_equal("three", p); 203 | chronicle_return(tailer, &result); 204 | 205 | p = (char*)chronicle_collect(tailer, &result); 206 | assert_string_equal("a much longer item that will need encoding as variable length text", p); 207 | chronicle_return(tailer, &result); 208 | 209 | chronicle_cleanup(queue); 210 | 211 | delete_test_data(test_queuedir); 212 | free(queuedir); 213 | free(test_queuedir); 214 | } 215 | 216 | static void queue_init_rollscheme(void **state) { 217 | queue_t* queue = chronicle_init("/tmp"); 218 | assert_non_null(queue); 219 | 220 | assert_int_equal(chronicle_get_version(queue), 0); 221 | chronicle_set_version(queue, 6); 222 | assert_int_equal(chronicle_get_version(queue), 0); 223 | chronicle_set_version(queue, 4); 224 | assert_int_equal(chronicle_get_version(queue), 4); 225 | chronicle_set_version(queue, 5); 226 | assert_int_equal(chronicle_get_version(queue), 5); 227 | 228 | assert_null(chronicle_get_roll_scheme(queue)); 229 | chronicle_set_roll_scheme(queue, "MADE_UP_SCHEME"); 230 | assert_null(chronicle_get_roll_scheme(queue)); 231 | chronicle_set_roll_scheme(queue, "FAST_HOURLY"); 232 | assert_string_equal(chronicle_get_roll_scheme(queue), "FAST_HOURLY"); 233 | assert_string_equal(chronicle_get_roll_format(queue), "yyyyMMdd-HH'F'"); 234 | char* p; 235 | p = chronicle_get_cycle_fn(queue, 0); 236 | assert_string_equal(p, "/tmp/19700101-00F.cq4"); 237 | free(p); 238 | p = chronicle_get_cycle_fn(queue, 1); 239 | assert_string_equal(p, "/tmp/19700101-01F.cq4"); 240 | free(p); 241 | p = chronicle_get_cycle_fn(queue, 24); 242 | assert_string_equal(p, "/tmp/19700102-00F.cq4"); 243 | free(p); 244 | 245 | chronicle_set_roll_scheme(queue, "FIVE_MINUTELY"); 246 | p = chronicle_get_cycle_fn(queue, 0); 247 | assert_string_equal(p, "/tmp/19700101-0000V.cq4"); 248 | free(p); 249 | p = chronicle_get_cycle_fn(queue, 1); 250 | assert_string_equal(p, "/tmp/19700101-0005V.cq4"); 251 | free(p); 252 | 253 | chronicle_set_roll_scheme(queue, "DAILY"); 254 | assert_string_equal(chronicle_get_roll_scheme(queue), "DAILY"); 255 | assert_string_equal(chronicle_get_roll_format(queue), "yyyyMMdd"); 256 | p = chronicle_get_cycle_fn(queue, 0); 257 | assert_string_equal(p, "/tmp/19700101.cq4"); 258 | free(p); 259 | p = chronicle_get_cycle_fn(queue, 1); 260 | assert_string_equal(p, "/tmp/19700102.cq4"); 261 | free(p); 262 | 263 | // bug: CodeQL cpp/integer-multiplication-cast-to-long 264 | // rawtime could overflow a 32-bit interger 265 | // index = cycle * roll_length = cycle * 24*60*60 266 | // 2B = cycle * 86400 => overflow around cycle ~24855 267 | p = chronicle_get_cycle_fn(queue, 24855); 268 | assert_string_equal(p, "/tmp/20380119.cq4"); 269 | free(p); 270 | p = chronicle_get_cycle_fn(queue, 24856); 271 | assert_string_equal(p, "/tmp/20380120.cq4"); // was: /tmp/19011214.cq4 272 | free(p); 273 | 274 | chronicle_cleanup(queue); 275 | } 276 | 277 | static void queue_cqv5_new_queue(void **state) { 278 | 279 | // create queue in an empty directory 280 | char* temp_dir; 281 | asprintf(&temp_dir, "%s/chronicle.test.XXXXXX", P_tmpdir); 282 | temp_dir = mkdtemp(temp_dir); 283 | queue_t* queue = chronicle_init(temp_dir); 284 | assert_non_null(queue); 285 | chronicle_set_version(queue, 5); 286 | chronicle_set_roll_scheme(queue, "DAILY"); 287 | chronicle_set_encoder(queue, &wirepad_sizeof, &wirepad_write); 288 | chronicle_set_create(queue, 1); 289 | assert_int_equal(chronicle_open(queue), 0); 290 | assert_int_equal(chronicle_get_version(queue), 5); 291 | assert_string_equal(chronicle_get_roll_scheme(queue), "DAILY"); 292 | 293 | // append will use local clock - we cannot predict the returned index 294 | wirepad_t* pad = wirepad_init(1024); 295 | wirepad_text(pad, "four five"); 296 | uint64_t idx = chronicle_append(queue, pad); 297 | wirepad_free(pad); 298 | 299 | chronicle_cleanup(queue); 300 | 301 | // re-open queue to check we find our text at the same index 302 | queue = chronicle_init(temp_dir); 303 | assert_non_null(queue); 304 | chronicle_set_decoder(queue, &wire_parse_textonly, &free); 305 | assert_int_equal(chronicle_open(queue), 0); 306 | assert_int_equal(chronicle_get_version(queue), 5); 307 | assert_string_equal(chronicle_get_roll_scheme(queue), "DAILY"); 308 | 309 | collected_t result; 310 | 311 | tailer_t* tailer = chronicle_tailer(queue, NULL, NULL, 0); 312 | 313 | chronicle_collect(tailer, &result); 314 | assert_string_equal("four five", result.msg); 315 | assert_true(idx == result.index); 316 | assert_int_equal(chronicle_tailer_state(tailer), TS_COLLECTED); 317 | chronicle_return(tailer, &result); 318 | 319 | chronicle_cleanup(queue); 320 | delete_test_data(temp_dir); 321 | free(temp_dir); 322 | } 323 | 324 | static void queue_cqv5_new_test4_queue_nodata(void **state) { 325 | // create queue in an empty directory 326 | char* temp_dir; 327 | asprintf(&temp_dir, "%s/chronicle.test.XXXXXX", P_tmpdir); 328 | temp_dir = mkdtemp(temp_dir); 329 | 330 | queue_t* queue = chronicle_init(temp_dir); 331 | chronicle_set_version(queue, 5); 332 | chronicle_set_roll_scheme(queue, "TEST4_SECONDLY"); 333 | chronicle_set_encoder(queue, &wirepad_sizeof, &wirepad_write); 334 | chronicle_set_create(queue, 1); 335 | assert_int_equal(chronicle_open(queue), 0); 336 | chronicle_cleanup(queue); 337 | 338 | // re-open queue to check 339 | queue = chronicle_init(temp_dir); 340 | assert_non_null(queue); 341 | chronicle_set_decoder(queue, &wire_parse_textonly, &free); 342 | assert_int_equal(chronicle_open(queue), 0); 343 | assert_int_equal(chronicle_get_version(queue), 5); 344 | assert_string_equal(chronicle_get_roll_scheme(queue), "TEST4_SECONDLY"); 345 | 346 | chronicle_cleanup(queue); 347 | delete_test_data(temp_dir); 348 | free(temp_dir); 349 | } 350 | 351 | int main(int argc, char* argv[]) { 352 | argv0 = argv[0]; 353 | const struct CMUnitTest tests[] = { 354 | cmocka_unit_test(queue_init_cleanup), 355 | cmocka_unit_test(queue_not_exist), 356 | cmocka_unit_test(queue_is_file), 357 | cmocka_unit_test(queue_empty_dir_no_ver), 358 | cmocka_unit_test(queue_init_rollscheme), 359 | cmocka_unit_test(queue_cqv4_sample_input), 360 | cmocka_unit_test(queue_cqv5_sample_input), 361 | cmocka_unit_test(queue_cqv5_new_queue), 362 | cmocka_unit_test(queue_cqv5_new_test4_queue_nodata), 363 | }; 364 | return cmocka_run_group_tests(tests, NULL, NULL); 365 | } 366 | -------------------------------------------------------------------------------- /native/test/test_wire.c: -------------------------------------------------------------------------------- 1 | #define _GNU_SOURCE 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include 12 | 13 | void handle_text(char* buf, int sz, char* data, int dsz, wirecallbacks_t* cbs) { 14 | int* res = (int*)cbs->userdata; 15 | *res = (strncmp(buf, "hello", sz) == 0) ? 5 : 1; 16 | if (wire_trace) printf(" got text cb %.*s\n", dsz, data); 17 | } 18 | 19 | static void test_wirepad_text(void **state) { 20 | wire_trace = 0; 21 | 22 | wirepad_t* pad = wirepad_init(1024); 23 | assert_non_null(pad); 24 | assert_int_equal(wirepad_sizeof(pad), 0); 25 | 26 | // the null isn't written. 1 byte descriptor, 5 bytes text 27 | wirepad_text(pad, "hello"); 28 | // this might be padding from the QC layer, perhaps shouldn't be in this test 29 | wirepad_pad_to_x8_00(pad); 30 | assert_int_equal(wirepad_sizeof(pad), 1+5+2); 31 | 32 | //wirepad_clear(pad); 33 | //assert_int_equal(wirepad_sizeof(pad), 0); 34 | char* dump = wirepad_hexformat(pad); 35 | assert_string_equal(dump, 36 | "00000000 e5 68 65 6c 6c 6f 00 00 .hello.. \n" 37 | ); 38 | free(dump); 39 | 40 | // run the parser over the pad and check we get one callback 41 | int result = 0; 42 | 43 | wirecallbacks_t hcbs; 44 | bzero(&hcbs, sizeof(hcbs)); 45 | hcbs.field_char = &handle_text; 46 | hcbs.userdata = &result; 47 | wirepad_parse(pad, &hcbs); 48 | 49 | assert_int_equal(result, 5); 50 | 51 | wirepad_free(pad); 52 | } 53 | 54 | static void test_wirepad_fields(void **state) { 55 | // example from https://github.com/OpenHFT/Chronicle-Wire#simple-use-case 56 | 57 | wirepad_t* pad = wirepad_init(1024); 58 | wirepad_field_text(pad, "message", "Hello World"); 59 | wirepad_field_varint(pad, "number", 1234567890L); 60 | wirepad_field_enum(pad, "code", "SECONDS"); 61 | wirepad_field_float64(pad, "price", 10.50); 62 | 63 | char* dump = wirepad_hexformat(pad); 64 | assert_string_equal(dump, 65 | "00000000 c7 6d 65 73 73 61 67 65 eb 48 65 6c 6c 6f 20 57 .message .Hello W\n" 66 | "00000010 6f 72 6c 64 c6 6e 75 6d 62 65 72 a6 d2 02 96 49 orld.num ber....I\n" 67 | "00000020 c4 63 6f 64 65 e7 53 45 43 4f 4e 44 53 c5 70 72 .code.SE CONDS.pr\n" 68 | "00000030 69 63 65 90 00 00 28 41 ice...(A \n" 69 | ); 70 | free(dump); 71 | wirepad_free(pad); 72 | }; 73 | 74 | 75 | static void test_wirepad_metadata(void **state) { 76 | wire_trace = 0; 77 | // extract of the data written to metadata.cq4t (v5) up to the last non-zero byte 78 | // There is additional framing in this example, an outer structure of 79 | // chronicle-queue messages surounds the wire data. 80 | // You can use wiredpad_qc_start() and wirepad_qc_finish() to write these parts, 81 | // which also manipulates the "working bit" semantics (although not with a CAS op) 82 | // so non-contested writes only at the moment. 83 | // 84 | // 00000000 ac 00 00 40 b9 06 68 65 61 64 65 72 b6 07 53 54 ...@..he ader..ST 85 | // 00000010 53 74 6f 72 65 82 96 00 00 00 c8 77 69 72 65 54 Store... ...wireT 86 | // 00000020 79 70 65 b6 08 57 69 72 65 54 79 70 65 ec 42 49 ype..Wir eType.BI 87 | // 00000030 4e 41 52 59 5f 4c 49 47 48 54 c8 6d 65 74 61 64 NARY_LIG HT.metad 88 | // 00000040 61 74 61 b6 07 53 43 51 4d 65 74 61 82 5d 00 00 ata..SCQ Meta.].. 89 | // 00000050 00 c4 72 6f 6c 6c b6 08 53 43 51 53 52 6f 6c 6c ..roll.. SCQSRoll 90 | // 00000060 82 26 00 00 00 c6 6c 65 6e 67 74 68 a6 00 5c 26 .&....le ngth..\& 91 | // 00000070 05 c6 66 6f 72 6d 61 74 eb 79 79 79 79 4d 4d 64 ..format .yyyyMMd 92 | // 00000080 64 27 46 27 c5 65 70 6f 63 68 00 d7 64 65 6c 74 d'F'.epo ch..delt 93 | // 00000090 61 43 68 65 63 6b 70 6f 69 6e 74 49 6e 74 65 72 aCheckpo intInter 94 | // 000000a0 76 61 6c 40 c8 73 6f 75 72 63 65 49 64 00 8f 8f val@.sou rceId... 95 | // 000000b0 24 00 00 00 b9 14 6c 69 73 74 69 6e 67 2e 68 69 $.....li sting.hi 96 | // 000000c0 67 68 65 73 74 43 79 63 6c 65 8e 00 00 00 00 a7 ghestCyc le...... 97 | // 000000d0 fd 49 00 00 00 00 00 00 24 00 00 00 b9 13 6c 69 .I...... $.....li 98 | // 000000e0 73 74 69 6e 67 2e 6c 6f 77 65 73 74 43 79 63 6c sting.lo westCycl 99 | // 000000f0 65 8e 01 00 00 00 00 a7 fd 49 00 00 00 00 00 00 e....... .I...... 100 | // 00000100 1c 00 00 00 b9 10 6c 69 73 74 69 6e 67 2e 6d 6f ......li sting.mo 101 | // 00000110 64 43 6f 75 6e 74 8f a7 01 00 00 00 00 00 00 00 dCount.. ........ 102 | // 00000120 24 00 00 00 b9 14 63 68 72 6f 6e 69 63 6c 65 2e $.....ch ronicle. 103 | // 00000130 77 72 69 74 65 2e 6c 6f 63 6b 8e 00 00 00 00 a7 write.lo ck...... 104 | // 00000140 00 00 00 00 00 00 00 80 2c 00 00 00 b9 1d 63 68 ........ ,.....ch 105 | // 00000150 72 6f 6e 69 63 6c 65 2e 6c 61 73 74 49 6e 64 65 ronicle. lastInde 106 | // 00000160 78 52 65 70 6c 69 63 61 74 65 64 8f 8f 8f 8f a7 xReplica ted..... 107 | // 00000170 ff ff ff ff ff ff ff ff 34 00 00 00 b9 29 63 68 ........ 4....)ch 108 | // 00000180 72 6f 6e 69 63 6c 65 2e 6c 61 73 74 41 63 6b 6e ronicle. lastAckn 109 | // 00000190 6f 77 6c 65 64 67 65 64 49 6e 64 65 78 52 65 70 owledged IndexRep 110 | // 000001a0 6c 69 63 61 74 65 64 a7 ff ff ff ff ff ff ff ff licated. ........ 111 | 112 | char* buf="\254\000\000@\271\006header\266\007STStore\202\226\000\000\000\310wireType\266\010" 113 | "WireType\354BINARY_LIGHT\310metadata\266\007SCQMeta\202]\000\000\000\304roll\266\010" 114 | "SCQSRoll\202&\000\000\000\306length\246\000\\&\005\306format\353yyyyMMdd'F'\305epoch" 115 | "\000\327deltaCheckpointInterval@\310sourceId\000\217\217$\000\000\000\271\024" 116 | "listing.highestCycle\216\000\000\000\000\247\375I\000\000\000\000\000\000$\000\000\000\271\023" 117 | "listing.lowestCycle\216\001\000\000\000\000\247\375I\000\000\000\000\000\000\034\000\000\000\271\020" 118 | "listing.modCount\217\247\001\000\000\000\000\000\000\000$\000\000\000\271\024" 119 | "chronicle.write.lock\216\000\000\000\000\247\000\000\000\000\000\000\000\200,\000\000\000\271\035" 120 | "chronicle.lastIndexReplicated\217\217\217\217\247\377\377\377\377\377\377\377\3774\000\000\000\271)" 121 | "chronicle.lastAcknowledgedIndexReplicated\247\377\377\377\377\377\377\377\377"; 122 | 123 | wirepad_t* pad = wirepad_init(1024); 124 | 125 | // single metadata message 126 | wirepad_qc_start(pad, 1); 127 | wirepad_event_name(pad, "header"); 128 | wirepad_type_prefix(pad, "STStore"); 129 | wirepad_nest_enter(pad); //header 130 | wirepad_field_type_enum(pad, "wireType", "WireType", "BINARY_LIGHT"); 131 | // field metadata, type prefix SCQMeta, nesting begin 132 | wirepad_field(pad, "metadata"); 133 | wirepad_type_prefix(pad, "SCQMeta"); 134 | wirepad_nest_enter(pad); 135 | wirepad_field(pad, "roll"); 136 | wirepad_type_prefix(pad, "SCQSRoll"); 137 | wirepad_nest_enter(pad); 138 | wirepad_field_varint(pad, "length", 86400000); 139 | wirepad_field_text(pad, "format", "yyyyMMdd'F'"); 140 | wirepad_field_varint(pad, "epoch", 0); 141 | wirepad_nest_exit(pad); 142 | wirepad_field_varint(pad, "deltaCheckpointInterval", 64); 143 | wirepad_field_varint(pad, "sourceId", 0); 144 | wirepad_nest_exit(pad); 145 | wirepad_pad_to_x8(pad); // feels wrong - should be automatic? 146 | wirepad_nest_exit(pad); 147 | wirepad_qc_finish(pad); 148 | 149 | // 6 data messages 150 | wirepad_qc_start(pad, 0); 151 | wirepad_event_name(pad, "listing.highestCycle"); 152 | wirepad_uint64_aligned(pad, 18941); // this cannot be varint as memory mapped! explicit size 153 | wirepad_qc_finish(pad); 154 | 155 | wirepad_qc_start(pad, 0); 156 | wirepad_event_name(pad, "listing.lowestCycle"); 157 | wirepad_uint64_aligned(pad, 18941); 158 | wirepad_qc_finish(pad); 159 | 160 | wirepad_qc_start(pad, 0); 161 | wirepad_event_name(pad, "listing.modCount"); 162 | wirepad_uint64_aligned(pad, 1); 163 | wirepad_qc_finish(pad); 164 | 165 | wirepad_qc_start(pad, 0); 166 | wirepad_event_name(pad, "chronicle.write.lock"); 167 | wirepad_uint64_aligned(pad, 0x8000000000000000); 168 | wirepad_qc_finish(pad); 169 | 170 | wirepad_qc_start(pad, 0); 171 | wirepad_event_name(pad, "chronicle.lastIndexReplicated"); 172 | wirepad_uint64_aligned(pad, -1); 173 | wirepad_qc_finish(pad); 174 | 175 | wirepad_qc_start(pad, 0); 176 | wirepad_event_name(pad, "chronicle.lastAcknowledgedIndexReplicated"); 177 | wirepad_uint64_aligned(pad, -1); 178 | wirepad_qc_finish(pad); 179 | 180 | assert_memory_equal(buf, wirepad_base(pad), wirepad_sizeof(pad)); 181 | 182 | wirepad_free(pad); 183 | 184 | } 185 | 186 | int main(void) { 187 | const struct CMUnitTest tests[] = { 188 | cmocka_unit_test(test_wirepad_text), 189 | cmocka_unit_test(test_wirepad_fields), 190 | cmocka_unit_test(test_wirepad_metadata), 191 | }; 192 | return cmocka_run_group_tests(tests, NULL, NULL); 193 | } 194 | -------------------------------------------------------------------------------- /native/test/testdata.h: -------------------------------------------------------------------------------- 1 | #define _GNU_SOURCE 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include 11 | #include 12 | #include 13 | 14 | static int copy_data(struct archive *ar, struct archive *aw) { 15 | int r; 16 | const void *buff; 17 | size_t size; 18 | #if ARCHIVE_VERSION_NUMBER >= 3000000 19 | int64_t offset; 20 | #else 21 | off_t offset; 22 | #endif 23 | 24 | for (;;) { 25 | r = archive_read_data_block(ar, &buff, &size, &offset); 26 | if (r == ARCHIVE_EOF) 27 | return (ARCHIVE_OK); 28 | if (r != ARCHIVE_OK) 29 | return (r); 30 | r = archive_write_data_block(aw, buff, size, offset); 31 | if (r != ARCHIVE_OK) { 32 | return (r); 33 | } 34 | } 35 | } 36 | 37 | char* unpack_test_data(char* file, char* argv0) { 38 | 39 | char* appdir = strdup(argv0); 40 | char* test_archive; 41 | asprintf(&test_archive, "%s/%s", dirname(appdir), file); 42 | 43 | char* temp_dir; 44 | asprintf(&temp_dir, "%s/chronicle.test.XXXXXX", P_tmpdir); 45 | temp_dir = mkdtemp(temp_dir); 46 | 47 | printf("Unpacking test data from %s to %s\n", test_archive, temp_dir); 48 | 49 | // https://github.com/libarchive/libarchive/wiki/Examples 50 | struct archive *a; 51 | struct archive *ext; 52 | struct archive_entry *entry; 53 | int r; 54 | 55 | a = archive_read_new(); 56 | archive_read_support_format_tar(a); 57 | archive_read_support_filter_bzip2(a); 58 | 59 | ext = archive_write_disk_new(); 60 | archive_write_disk_set_options(ext, ARCHIVE_EXTRACT_TIME); 61 | archive_write_disk_set_standard_lookup(ext); 62 | 63 | if ((r = archive_read_open_filename(a, test_archive, 10240))) { 64 | fprintf(stderr, "%s\n", archive_error_string(a)); 65 | return NULL; 66 | } 67 | char* dest_file; 68 | for (;;) { 69 | r = archive_read_next_header(a, &entry); 70 | if (r == ARCHIVE_EOF) 71 | break; 72 | if (r < ARCHIVE_OK) 73 | fprintf(stderr, "%s\n", archive_error_string(a)); 74 | if (r < ARCHIVE_WARN) 75 | return NULL; 76 | 77 | asprintf(&dest_file, "%s/%s", temp_dir, archive_entry_pathname(entry)); 78 | archive_entry_set_pathname(entry, dest_file); 79 | // printf(" writing %s\n", dest_file); 80 | 81 | r = archive_write_header(ext, entry); 82 | if (r < ARCHIVE_OK) 83 | fprintf(stderr, "%s\n", archive_error_string(ext)); 84 | else if (archive_entry_size(entry) > 0) { 85 | r = copy_data(a, ext); 86 | if (r < ARCHIVE_OK) 87 | fprintf(stderr, "%s\n", archive_error_string(ext)); 88 | if (r < ARCHIVE_WARN) 89 | exit(1); 90 | } 91 | r = archive_write_finish_entry(ext); 92 | if (r < ARCHIVE_OK) 93 | fprintf(stderr, "%s\n", archive_error_string(ext)); 94 | if (r < ARCHIVE_WARN) 95 | return NULL; 96 | 97 | free(dest_file); 98 | } 99 | archive_read_close(a); 100 | archive_read_free(a); 101 | archive_write_close(ext); 102 | archive_write_free(ext); 103 | 104 | free(appdir); 105 | free(test_archive); 106 | return temp_dir; 107 | } 108 | 109 | static int rmFiles(const char *pathname, const struct stat *sbuf, int type, struct FTW *ftwb) { 110 | if(remove(pathname) < 0) { 111 | fprintf(stderr, "ERROR: remove failed at %s", pathname); 112 | return FTW_STOP; 113 | } 114 | return FTW_CONTINUE; 115 | } 116 | 117 | int delete_test_data(char* queuedir) { 118 | // Delete the directory and its contents by traversing the tree in reverse order, without crossing mount boundaries and symbolic links 119 | if (nftw(queuedir, rmFiles, 10, FTW_DEPTH|FTW_MOUNT|FTW_PHYS|FTW_ACTIONRETVAL) < 0) { 120 | perror("ERROR: ntfw"); 121 | return 1; 122 | } 123 | return 0; 124 | } 125 | 126 | 127 | -------------------------------------------------------------------------------- /native/wire.c: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Tea Engineering Ltd. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #define __STDC_FORMAT_MACROS 16 | #include 17 | 18 | #include 19 | #include 20 | #include 21 | 22 | #include 23 | #include 24 | 25 | // enables full printf tracing during parsing of wire data (slow) 26 | int wire_trace = 0; 27 | 28 | 29 | int read_stop_uint(unsigned char* p, int *stopsz) { 30 | *stopsz = 0; 31 | int n = 0; 32 | do { 33 | n++; 34 | *stopsz = (*stopsz << 7) + (p[n-1] & 0x7F); 35 | } while ((p[n-1] & 0x80) != 0x00); 36 | // printf("stopsz %d bytes %d\n", *stopsz, n); 37 | return n; 38 | } 39 | 40 | // used for reading index and header structures as well as directory-index.cq4t 41 | void wire_parse(unsigned char* base, int lim, wirecallbacks_t* cbs) { 42 | // constants from 43 | // https://github.com/OpenHFT/Chronicle-Wire/blob/master/src/main/java/net/openhft/chronicle/wire/BinaryWireCode.java 44 | // decoder details 45 | // https://github.com/OpenHFT/Chronicle-Wire/blob/ea/src/main/java/net/openhft/chronicle/wire/BinaryWire.java 46 | unsigned char* p = base; 47 | uint8_t control; 48 | 49 | char* field_name = NULL; 50 | int field_name_sz = 0; 51 | char* ev_name = NULL; 52 | int ev_name_sz = 0; 53 | char* type_name = NULL; 54 | int type_name_sz = 0; 55 | char* field_text = NULL; 56 | int field_text_sz = 0; 57 | 58 | uint16_t padding16 = 0; 59 | uint32_t padding32 = 0; 60 | uint64_t padding64 = 0; 61 | uint64_t jlong2 = 0; 62 | float float32 = 0; 63 | 64 | // track nesting stack 65 | int nest = 0; 66 | unsigned char* pop_pos[10]; 67 | pop_pos[nest] = base+lim; 68 | if (cbs->reset_nesting) cbs->reset_nesting(); 69 | 70 | while (p < base + lim) { 71 | control = p[0]; p++; 72 | if (wire_trace) printf(" control 0x%02x\n", control); 73 | switch(control) { 74 | case 0x00 ... 0x7F: // NUM 75 | if (wire_trace) printf(" Field %.*s = %d (uint8)\n", (int)field_name_sz, field_name, control); 76 | if (cbs->field_uint64) cbs->field_uint64(field_name, field_name_sz, (uint64_t)control, cbs); 77 | break; 78 | case 0xB9: // EVENT_NAME 79 | p+= read_stop_uint(p, &ev_name_sz); 80 | ev_name = (char*)p; 81 | field_name = (char*)p; 82 | field_name_sz = ev_name_sz; 83 | p += ev_name_sz; 84 | if (cbs->event_name) cbs->event_name(ev_name, ev_name_sz, cbs); 85 | if (wire_trace) printf("Event name '%.*s'\n", ev_name_sz, ev_name); 86 | break; 87 | case 0x8F: // PADDING 88 | break; 89 | case 0x8E: // PADDING_32 90 | memcpy(&padding32, p, sizeof(padding32)); 91 | p += 4 + padding32; // not a bug, padding 4 bytes is a counter of extra bytes to skip 92 | break; 93 | case 0x82: // BYTES_LENGTH32, introduces nested structure with length 94 | memcpy(&padding32, p, sizeof(padding32)); 95 | p += 4; 96 | if (wire_trace) printf(" Enter nesting '%.*s' for %d bytes\n", field_name_sz, field_name, padding32); 97 | pop_pos[++nest] = p+padding32; 98 | break; 99 | case 0x8D: // I64_ARRAY 100 | // length/used a, then length*64-bit ints. used for index structured, so expose ptr 101 | memcpy(&padding64, p, sizeof(padding64)); 102 | memcpy(&jlong2, p+8, sizeof(jlong2)); 103 | if (wire_trace) printf(" Field %.*s = [...] (I64_ARRAY) used %" PRIu64 "/%" PRIu64 "\n", field_name_sz, field_name, jlong2, padding64); 104 | p += 16; 105 | if (cbs->ptr_uint64arr) cbs->ptr_uint64arr(field_name, field_name_sz, jlong2, padding64, p, cbs); 106 | p += 8*padding64; 107 | break; 108 | case 0x90: // FLOAT32 109 | memcpy(&float32, p, sizeof(float32)); 110 | if (wire_trace) printf(" Field %.*s = %f (FLOAT32)\n", field_name_sz, field_name, float32); 111 | // if (cbs->field_float) cbs->field_float(field_name, field_name_sz, float32, cbs); 112 | p += 4; 113 | break; 114 | //case 0x91: // FLOAT64 115 | // break; 116 | case 0xA5: // INT16 117 | memcpy(&padding16, p, sizeof(padding16)); 118 | if (wire_trace) printf(" Field %.*s = %hu (uint16)\n", (int)field_name_sz, field_name, padding16); 119 | if (cbs->field_uint64) cbs->field_uint64(field_name, field_name_sz, (uint64_t)padding16, cbs); 120 | p += 2; 121 | break; 122 | case 0xA6: // INT32 123 | memcpy(&padding32, p, sizeof(padding32)); 124 | if (wire_trace) printf(" Field %.*s = %u (uint32)\n", (int)field_name_sz, field_name, padding32); 125 | if (cbs->field_uint64) cbs->field_uint64(field_name, field_name_sz, (uint64_t)padding32, cbs); 126 | p += 4; 127 | break; 128 | case 0xA7: // INT64 129 | memcpy(&padding64, p, sizeof(padding64)); 130 | if (wire_trace) printf(" Field %.*s = %" PRIu64 " (uint64)\n", field_name_sz, field_name, padding64); 131 | if (cbs->ptr_uint64) cbs->ptr_uint64(ev_name, ev_name_sz, p, cbs); 132 | if (cbs->field_uint64) cbs->field_uint64(field_name, field_name_sz, padding64, cbs); 133 | p += 8; 134 | break; 135 | case 0xB6: // TYPE_PREFIX 136 | p+= read_stop_uint(p, &type_name_sz); 137 | type_name = (char*)p; 138 | p += type_name_sz; 139 | if (wire_trace) printf("Type prefix !%.*s\n", type_name_sz, type_name); 140 | if (cbs->type_prefix) cbs->type_prefix(type_name, type_name_sz, cbs); 141 | break; 142 | case 0xC0 ... 0xDF: // Small Field start, UTF length encoded in control 143 | field_name = (char*)p; 144 | field_name_sz = control - 0xC0; 145 | // TODO: for UTF characters in utflen, we need to skip additional bytes here 146 | // see BytesInternal.parse8bit_SB1 147 | // printf(" Field %.*s \n", field_name_sz, field_name); 148 | p += field_name_sz; 149 | break; 150 | case 0xE0 ... 0xFF: // Text field value 151 | field_text = (char*)p; 152 | field_text_sz = control - 0xE0; 153 | if (wire_trace) printf(" Field %.*s = %.*s (text)\n", (int)field_name_sz, field_name, field_text_sz, field_text); 154 | if (cbs->field_char) cbs->field_char(field_name, field_name_sz, field_text, field_text_sz, cbs); 155 | p += field_text_sz; 156 | break; 157 | case 0xB8: // Text any length 158 | p+= read_stop_uint(p, &field_text_sz); 159 | field_text = (char*)p; 160 | if (wire_trace) printf(" Field %.*s = %.*s (text)\n", (int)field_name_sz, field_name, field_text_sz, field_text); 161 | if (cbs->field_char) cbs->field_char(field_name, field_name_sz, field_text, field_text_sz, cbs); 162 | p += field_text_sz; 163 | break; 164 | default: 165 | printf("Aborted at %p (+%04lx) unknown control word %d 0x%02x\n", p-1, p-base, control, control); 166 | p = base + lim; 167 | break; 168 | } 169 | 170 | while (p >= pop_pos[nest] && nest > 0) { 171 | nest--; 172 | if (wire_trace) printf(" nesting pop\n"); 173 | } 174 | } 175 | } 176 | 177 | struct wirepad { 178 | int sz; 179 | unsigned char* pos; 180 | unsigned char* base; 181 | 182 | int nest; 183 | unsigned char* nest_enter_pos[10]; 184 | }; 185 | 186 | wirepad_t* wirepad_init(int initial_sz) { 187 | wirepad_t* pad = malloc(sizeof(wirepad_t)); 188 | pad->sz = initial_sz; 189 | pad->base = NULL; 190 | pad->base = realloc(pad->base, pad->sz); 191 | if (wire_trace) printf("wire pad created at %p sz %d\n", pad->base, pad->sz); 192 | pad->pos = pad->base; 193 | pad->nest = 0; 194 | return pad; 195 | } 196 | 197 | void wirepad_free(wirepad_t* pad) { 198 | free(pad->base); 199 | free(pad); 200 | } 201 | 202 | void wirepad_clear(wirepad_t* pad) { 203 | pad->pos = pad->base; 204 | pad->nest = 0; 205 | } 206 | 207 | void wirepad_extent(wirepad_t* pad, int sz) { 208 | int used = pad->pos - pad->base; 209 | int remain = pad->sz - used - sz; 210 | if (wire_trace) printf(" wirepad_extent base=%p used=%d need=+%d remain=%d\n", pad->base, used, sz, remain); 211 | while (remain < 0) { 212 | pad->sz = pad->sz * 2; 213 | pad->base = realloc(pad->base, pad->sz); 214 | remain = pad->sz - used - sz; 215 | } 216 | } 217 | 218 | void wirepad_text(wirepad_t* pad, char* text) { 219 | int d = strlen(text); 220 | int overhead = d < 0x1F ? 1 : 4; 221 | if (wire_trace) printf("wirepad_text pos=%p writing d=%d overhead=%d\n", pad->pos, d, overhead); 222 | wirepad_extent(pad, d+overhead); 223 | if (d < 0x1F) { 224 | pad->pos[0] = 0xE0 + d; 225 | } else { 226 | pad->pos[0] = 0xB8; 227 | // put stop bit encoded length 228 | printf("long text encoding?"); 229 | abort(); 230 | } 231 | memcpy(pad->pos+overhead, text, d); 232 | pad->pos = pad->pos + d + overhead; 233 | } 234 | 235 | void wirepad_field(wirepad_t* pad, char* text) { 236 | int d = strlen(text); 237 | int overhead = d < 0x1F ? 1 : 4; 238 | if (wire_trace) printf("wirepad_field pos=%p writing text=%s d=%d overhead=%d\n", pad->pos, text, d, overhead); 239 | wirepad_extent(pad, d+overhead); 240 | if (d < 0x1F) { 241 | pad->pos[0] = 0xC0 + d; 242 | } else { 243 | printf("long field encoding?"); 244 | abort(); 245 | } 246 | memcpy(pad->pos+overhead, text, d); 247 | pad->pos = pad->pos + overhead + d; 248 | } 249 | 250 | void wirepad_uint64_aligned(wirepad_t* pad, uint64_t v) { 251 | if (wire_trace) printf("wirepad_uint64 pos=%p writing uint64=%" PRIu64 "\n", pad->pos, v); 252 | wirepad_extent(pad, 16); 253 | 254 | // align the 8 data bytes to a multiple of 8 bytes, which 255 | // requires aligning the A7 prefix before it to the last byte of 8 256 | int padding = -((pad->pos + 1) - pad->base) & 0x7; // current (position+1), align to 8 257 | 258 | // optimise padding between 0x8F (single bytes) and 0x8E (var bytes) 259 | if (padding == 0) { 260 | // emit nothing 261 | } else if (padding < 5) { 262 | pad->pos[0] = 0x8F; // later bytes overwritten if not required 263 | pad->pos[1] = 0x8F; 264 | pad->pos[2] = 0x8F; 265 | pad->pos[3] = 0x8F; 266 | pad->pos += padding; 267 | } else { 268 | pad->pos[0] = 0x8E; 269 | uint32_t header = padding - 5; 270 | memcpy(pad->pos + 1, &header, sizeof(header)); 271 | // set possible byte 5, 6, 7 to zero 272 | for(int i = 5; i < padding; i++) pad->pos[i] = 0; 273 | pad->pos += padding; 274 | } 275 | pad->pos[0] = 0xA7; // INT64 276 | memcpy(pad->pos+1, &v, sizeof(v)); 277 | pad->pos = pad->pos + 9; 278 | } 279 | 280 | void wirepad_varint(wirepad_t* pad, uint64_t v) { 281 | // compacted value representation, unaligned 282 | 283 | if (wire_trace) printf("wirepad_varint pos=%p writing varint=%" PRIu64 "\n", pad->pos, v); 284 | wirepad_extent(pad, 9); 285 | 286 | if (v <= 0x7F) { 287 | pad->pos[0] = v & 0x7F; 288 | pad->pos = pad->pos + 1; 289 | } else if (v <= 0xFFFF) { 290 | pad->pos[0] = 0xa5; // INT16 291 | pad->pos[1] = (v >> 0) & 0xff; 292 | pad->pos[2] = (v >> 8) & 0xff; 293 | pad->pos = pad->pos + 3; 294 | } else if (v <= 0xFFFFFFFF) { 295 | // 1234567890L => 49 96 02 D2 296 | // a6 d2 02 96 49 297 | pad->pos[0] = 0xa6; // INT32 298 | pad->pos[1] = (v >> 0) & 0xff; 299 | pad->pos[2] = (v >> 8) & 0xff; 300 | pad->pos[3] = (v >> 16) & 0xff; 301 | pad->pos[4] = (v >> 24) & 0xff; 302 | pad->pos = pad->pos + 5; 303 | } else { 304 | pad->pos[0] = 0xA7; // INT64 305 | memcpy(pad->pos+1, &v, sizeof(v)); 306 | pad->pos = pad->pos + 9; 307 | } 308 | } 309 | 310 | void wirepad_field_text(wirepad_t* pad, char* field, char* text) { 311 | wirepad_field(pad, field); 312 | wirepad_text(pad, text); 313 | } 314 | 315 | void wirepad_field_enum(wirepad_t* pad, char* field, char* text) { 316 | wirepad_field(pad, field); 317 | wirepad_text(pad, text); 318 | } 319 | 320 | void wirepad_field_type_enum(wirepad_t* pad, char* field, char* type, char* text) { 321 | wirepad_field(pad, field); 322 | wirepad_type_prefix(pad, type); 323 | wirepad_text(pad, text); 324 | } 325 | 326 | void wirepad_field_uint64(wirepad_t* pad, char* field, uint64_t v) { 327 | wirepad_field(pad, field); 328 | wirepad_uint64_aligned(pad, v); 329 | } 330 | 331 | void wirepad_field_float64(wirepad_t* pad, char* field, double v) { 332 | wirepad_field(pad, field); 333 | float f = v; 334 | if (f == v) { 335 | pad->pos[0] = 0x90; // FLOAT32 336 | memcpy(pad->pos+1, &f, sizeof(f)); 337 | pad->pos = pad->pos + 5; 338 | } else { 339 | printf("float64 large encoding?"); 340 | abort(); 341 | } 342 | 343 | } 344 | 345 | void wirepad_field_varint(wirepad_t* pad, char* field, int v) { 346 | wirepad_field(pad, field); 347 | wirepad_varint(pad, v); 348 | } 349 | 350 | void wirepad_pad_to_x8(wirepad_t* pad) { 351 | int padding = -(pad->pos - pad->base) & 0x07; 352 | for (int i = 0; i < padding; i++) { 353 | pad->pos[i] = 0x8F; 354 | } 355 | pad->pos = pad->pos + padding; 356 | } 357 | 358 | void wirepad_pad_to_x8_00(wirepad_t* pad) { 359 | int padding = -(pad->pos - pad->base) & 0x07; 360 | for (int i = 0; i < padding; i++) { 361 | pad->pos[i] = 0x00; 362 | } 363 | pad->pos = pad->pos + padding; 364 | } 365 | 366 | void wirepad_event_name(wirepad_t* pad, char* event_name) { 367 | int d = strlen(event_name); 368 | if (wire_trace) printf("wirepad_event_name pos=%p name=%s\n", pad->pos, event_name); 369 | if (d > 0xFF) { 370 | printf("event_name large encoding?"); 371 | abort(); 372 | } 373 | pad->pos[0] = 0xB9; 374 | pad->pos[1] = d & 0xFF; 375 | memcpy(pad->pos+2, event_name, d); 376 | pad->pos = pad->pos + 2 + d; 377 | } 378 | 379 | void wirepad_type_prefix(wirepad_t* pad, char* type_prefix) { 380 | int d = strlen(type_prefix); 381 | if (wire_trace) printf("wirepad_type_prefix pos=%p name=%s\n", pad->pos, type_prefix); 382 | if (d > 0xFF) { 383 | printf("type_prefix large encoding?"); 384 | abort(); 385 | } 386 | pad->pos[0] = 0xB6; 387 | pad->pos[1] = d & 0xFF; 388 | memcpy(pad->pos+2, type_prefix, d); 389 | pad->pos = pad->pos + 2 + d; 390 | } 391 | 392 | const uint32_t QC_HD_WORKING = 0x80000000; 393 | const uint32_t QC_HD_METADATA = 0x40000000; 394 | 395 | void wirepad_qc_start(wirepad_t* pad, int metadata) { 396 | uint32_t header = QC_HD_WORKING | (metadata==0 ? 0 : QC_HD_METADATA); 397 | if (wire_trace) printf("wirepad_qc_start pos=%p meta=%d nest=%d header=0x%x\n", pad->pos, metadata, pad->nest, header); 398 | wirepad_extent(pad, 9); 399 | memcpy(pad->pos, &header, sizeof(header)); 400 | pad->nest_enter_pos[pad->nest++] = pad->pos; 401 | pad->pos = pad->pos + 4; 402 | } 403 | 404 | void wirepad_qc_finish(wirepad_t* pad) { 405 | pad->nest--; 406 | unsigned char* entered = pad->nest_enter_pos[pad->nest]; 407 | int len = pad->pos - entered - 4; 408 | if (wire_trace) printf("wirepad_qc_finish pos=%p entered=%p len=%d (0x%x) nest=%d\n", pad->pos, entered, len, len, pad->nest); 409 | 410 | uint32_t header = 0; 411 | memcpy(&header, entered, sizeof(header)); 412 | header = (header & ~QC_HD_WORKING) | len; // clear WORKING bit, set size 413 | // copy back 414 | memcpy(entered, &header, sizeof(header)); 415 | // pad->pos hasn't moved! 416 | } 417 | 418 | void wirepad_nest_enter(wirepad_t* pad) { 419 | if (wire_trace) printf("wirepad_nest_start pos=%p nest=%d\n", pad->pos, pad->nest); 420 | wirepad_extent(pad, 5); 421 | uint32_t header = 0; 422 | pad->pos[0] = 0x82; 423 | memcpy(pad->pos+1, &header, sizeof(header)); 424 | pad->nest_enter_pos[pad->nest++] = pad->pos+1; 425 | pad->pos = pad->pos + 5; 426 | } 427 | 428 | void wirepad_nest_exit(wirepad_t* pad) { 429 | pad->nest--; 430 | unsigned char* entered = pad->nest_enter_pos[pad->nest]; 431 | int len = pad->pos - entered - 4; 432 | if (wire_trace) printf("wirepad_nest_exit pos=%p entered=%p len=%d (0x%x) nest=%d\n", pad->pos, entered, len, len, pad->nest); 433 | 434 | uint32_t header = len; 435 | memcpy(entered, &header, sizeof(header)); 436 | } 437 | 438 | void wirepad_parse(wirepad_t* pad, wirecallbacks_t* cbs) { 439 | wire_parse(pad->base, pad->pos-pad->base, cbs); 440 | } 441 | 442 | void wirepad_dump(wirepad_t* pad) { 443 | int n = pad->pos-pad->base; 444 | printf("wirepad_dump base=%p pos=%p (+%d) sz=%d\n", pad->base, pad->pos, n, pad->sz); 445 | printbuf((char *)pad->base, n); 446 | char* p = formatbuf((char*)pad->base, n); 447 | printf("%s\n", p); 448 | free(p); 449 | } 450 | 451 | char* wirepad_hexformat(wirepad_t* pad) { 452 | int n = pad->pos-pad->base; 453 | return formatbuf((char*)pad->base, n); 454 | } 455 | 456 | unsigned char* wirepad_base(wirepad_t* pad) { 457 | return pad->base; 458 | } 459 | 460 | // Integration with queues using BinaryWire for their data pages 461 | // these are not strictly part of wire.h but ease integration with 462 | // libchronicle.c 463 | // 464 | // writer 465 | // sizeof and write take a wirepad_t argument, cast down to void* here for ease of use with libchronicle 466 | size_t wirepad_sizeof(void* msg) { 467 | wirepad_t* pad = (wirepad_t*)msg; 468 | return pad->pos - pad->base; 469 | } 470 | 471 | void wirepad_write(unsigned char* base, void* msg, size_t sz) { 472 | wirepad_t* pad = (wirepad_t*)msg; 473 | memcpy(base, pad->base, sz); 474 | } 475 | 476 | // parser 477 | // an implementation of cparse_f where we capture text and return to new object 478 | void wire_parse_textonly_text(char* field, int fsz, char* data, int dsz, wirecallbacks_t* cbs) { 479 | char** res = (char**)cbs->userdata; 480 | // avoid leak if multiple text entries - return last 481 | if (*res != NULL) free(*res); 482 | *res = strndup(data, dsz); 483 | } 484 | 485 | void* wire_parse_textonly(unsigned char* base, int lim) { 486 | wirecallbacks_t cbs; 487 | char* text_result = NULL; 488 | 489 | bzero(&cbs, sizeof(cbs)); 490 | cbs.field_char = &wire_parse_textonly_text; 491 | cbs.userdata = &text_result; 492 | wire_parse(base, lim, &cbs); 493 | 494 | return text_result; 495 | } 496 | -------------------------------------------------------------------------------- /native/wire.h: -------------------------------------------------------------------------------- 1 | // Copyright 2021 Tea Engineering Ltd. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #ifndef FILE_WIRE_SEEN 16 | #define FILE_WIRE_SEEN 17 | 18 | #include 19 | #include 20 | 21 | // this library implements the "BinaryWire" self-describing wire protocol 22 | // used for chronicle-queue metadata messages. 23 | 24 | 25 | // global options - debug trace 26 | extern int wire_trace; 27 | 28 | // wire reader - set callbacks then invoke parse_wire 29 | typedef struct wirecallbacks { 30 | void (*event_name)(char*,int,struct wirecallbacks*); 31 | void (*type_prefix)(char*,int,struct wirecallbacks*); 32 | 33 | void (*field_uint64)(char*,int,uint64_t,struct wirecallbacks*); 34 | void (*field_char)(char*,int,char*,int,struct wirecallbacks*); 35 | 36 | // direct pointer access for headers, index arrays 37 | void (*ptr_uint64)(char*,int,unsigned char*,struct wirecallbacks*); 38 | void (*ptr_uint64arr)(char*,int,uint64_t,uint64_t,unsigned char*,struct wirecallbacks*); 39 | 40 | void (*reset_nesting)(); 41 | 42 | void* userdata; 43 | } wirecallbacks_t; 44 | 45 | void wire_parse(unsigned char* base, int lim, wirecallbacks_t* cbs); 46 | 47 | // wire writer 48 | // create a wirepad using wirepad_init, then write headers and fields 49 | // print bytes wirepad_dump, write out with wirepad_write or run parse_wire 50 | // over content with wirepad_parse 51 | typedef struct wirepad wirepad_t; 52 | 53 | wirepad_t* wirepad_init(int initial_sz); 54 | void wirepad_clear(wirepad_t* pad); 55 | void wirepad_dump(wirepad_t* pad); 56 | char* wirepad_hexformat(wirepad_t* pad); 57 | void wirepad_free(wirepad_t* pad); 58 | 59 | void wirepad_pad_to_x8(wirepad_t* pad); 60 | void wirepad_pad_to_x8_00(wirepad_t* pad); 61 | void wirepad_event_name(wirepad_t* pad, char* event_name); 62 | void wirepad_type_prefix(wirepad_t* pad, char* type_prefix); 63 | 64 | void wirepad_text(wirepad_t* pad, char* text); 65 | void wirepad_uint64_aligned(wirepad_t* pad, uint64_t v); 66 | 67 | void wirepad_qc_start(wirepad_t* pad, int metadata); 68 | void wirepad_qc_finish(wirepad_t* pad); 69 | 70 | void wirepad_nest_enter(wirepad_t* pad); 71 | void wirepad_nest_exit(wirepad_t* pad); 72 | 73 | void wirepad_field(wirepad_t* pad, char* text); 74 | void wirepad_field_varint(wirepad_t* pad, char* field, int v); 75 | void wirepad_field_text(wirepad_t* pad, char* field, char* text); 76 | void wirepad_field_uint64(wirepad_t* pad, char* field, uint64_t v); 77 | void wirepad_field_float64(wirepad_t* pad, char* field, double v); 78 | void wirepad_field_enum(wirepad_t* pad, char* field, char* v); 79 | void wirepad_field_type_enum(wirepad_t* pad, char* field, char* type, char* v); 80 | 81 | unsigned char* wirepad_base(wirepad_t* pad); 82 | void wirepad_parse(wirepad_t* pad, wirecallbacks_t* cbs); 83 | 84 | // Integration with queues using BinaryWire for their data pages 85 | // 86 | // cparse_f for the common case where the wire format is just text, 87 | // and we want to capture the text into a new object. Object must 88 | // be freed by the callback cdispatch_f 89 | void* wire_parse_textonly(unsigned char* base, int lim); 90 | 91 | // sizeof_f and append_f where the object to be written is a wirepad_t 92 | // which is cast down to void* here for ease of use with libchronicle 93 | size_t wirepad_sizeof(void* pad); 94 | void wirepad_write(unsigned char* base, void* pad, size_t sz); 95 | 96 | #endif --------------------------------------------------------------------------------