├── .gitignore ├── CONCEPTS.md ├── ChangeLog.md ├── LICENSE ├── Makefile.in ├── README.md ├── V210_CHANGES.txt ├── config.guess ├── config.sub ├── configure.in ├── fassert.h ├── getopt_long.c ├── getopt_long.h ├── install-sh ├── java ├── .gitignore ├── com_omniti_labs_jlog.c ├── jlog.java └── jlogTest.java ├── jlog.c ├── jlog.h ├── jlog_change_endian.pl ├── jlog_compress.c ├── jlog_compress.h ├── jlog_config.h.in ├── jlog_hash.c ├── jlog_hash.h ├── jlog_io.c ├── jlog_io.h ├── jlog_lz4_compression_provider.h ├── jlog_null_compression_provider.h ├── jlog_private.h ├── jlog_sanity_check.pl ├── jlogctl.c ├── jlogtail.c ├── jtest.c ├── jthreadtest.c ├── mkinstalldirs ├── perl ├── Changes ├── JLog.xs ├── MANIFEST ├── Makefile.PL.in ├── README ├── lib │ ├── JLog.pm │ └── JLog │ │ ├── Reader.pm │ │ └── Writer.pm ├── ppport.h ├── t │ └── 1.t └── typemap ├── php ├── config.m4 ├── jlog.c ├── package.xml ├── package2.xml └── php_jlog.h └── python ├── cjlog.pxd ├── examples └── test_jlog.py ├── jlog.c ├── jlog.pyx └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | Makefile 2 | autom4te.cache/ 3 | jlogtail 4 | jtest 5 | perl/MYMETA.json 6 | perl/MYMETA.yml 7 | perl/Makefile.old 8 | jlog_config.h 9 | *.a 10 | *.so 11 | *.o 12 | *.lo 13 | *.dylib 14 | jlogctl 15 | jthreadtest 16 | config.log 17 | config.status 18 | configure 19 | perl/JLog.bs 20 | perl/JLog.c 21 | perl/Makefile.PL 22 | perl/blib/ 23 | perl/pm_to_blib 24 | java/libjlog.jnilib 25 | java/jlog.jar 26 | java/*.class 27 | *~ 28 | -------------------------------------------------------------------------------- /CONCEPTS.md: -------------------------------------------------------------------------------- 1 | # JLog Concepts 2 | 3 | A JLog is durable message queue; records (messages) are written to a JLog and 4 | will remain logged until subscribers have read past that message and issued a 5 | checkpoint. 6 | 7 | JLogs are implemented as a directory on disk that contains a handful of control 8 | files and a number of segment files. The segment files contain the messages 9 | that were written to the JLog; each segment file can contain multiple messages 10 | and grow to be 4MB in size. If a new message written to the JLog would cause 11 | this limit to be exceeded, a new segment file is created. Segment files are 12 | deleted automatically when all subscribers have consumed the contents of that 13 | segment file and issued a checkpoint. 14 | 15 | ## Subscribers 16 | 17 | In order to manage data in the JLog, the JLog needs to track who is 18 | subscribing. Each subscriber has a name; the name must be unique for each 19 | consuming process, otherwise the behavior of JLog is undefined as two different 20 | processes will conflict over their state. 21 | 22 | It is recommended that the writer of a jlog have knowledge of the subscriber 23 | list before it writes data to the jlog, so that segments are not pruned away 24 | before a given subscriber starts to read--such a subscriber would effectively 25 | be "late" to the party and miss out on the data. 26 | 27 | ## Using the C API 28 | 29 | Here's a quick overview of how to use the C API for writing and reading: 30 | 31 | ### Writer 32 | 33 | Here, the writer is appending data to the jlog, which we're storing at 34 | /var/log/jlogexample. We have two subscribers, named "one" and "two": 35 | 36 | jlog_ctx *ctx; 37 | const char *path = "/var/log/jlogexample"; 38 | int rv; 39 | 40 | // First, ensure that the jlog is created 41 | ctx = jlog_new(path); 42 | if (jlog_ctx_init(ctx) != 0) { 43 | if(jlog_ctx_err(ctx) != JLOG_ERR_CREATE_EXISTS) { 44 | fprintf(stderr, "jlog_ctx_init failed: %d %s\n", jlog_ctx_err(ctx), jlog_ctx_err_string(ctx)); 45 | exit(1); 46 | } 47 | // Make sure it knows about our subscriber(s) 48 | jlog_ctx_add_subscriber(ctx, "one", JLOG_BEGIN); 49 | jlog_ctx_add_subscriber(ctx, "two", JLOG_BEGIN); 50 | } 51 | 52 | // Now re-open for writing 53 | jlog_ctx_close(ctx); 54 | ctx = jlog_new(path); 55 | if (jlog_ctx_open_writer(ctx) != 0) { 56 | fprintf(stderr, "jlog_ctx_open_writer failed: %d %s\n", jlog_ctx_err(ctx), jlog_ctx_err_string(ctx)); 57 | exit(0); 58 | } 59 | 60 | // Send in some data 61 | rv = jlog_ctx_write(ctx, "hello\n", strlen("hello\n"); 62 | if (rv != 0) { 63 | fprintf(stderr, "jlog_ctx_write_message failed: %d %s\n", jlog_ctx_err(ctx), jlog_ctx_err_string(ctx)); 64 | } 65 | jlog_ctx_close(ctx); 66 | 67 | ### Reader 68 | 69 | Using the reader for subscriber "one" looks like this: 70 | 71 | jlog_ctx *ctx; 72 | const char *path = "/var/log/jlogexample"; 73 | int rv; 74 | jlog_id begin, end; 75 | int count; 76 | 77 | ctx = jlog_new(path); 78 | if (jlog_ctx_open_reader(ctx, "one") != 0) { 79 | fprintf(stderr, "jlog_ctx_open_reader failed: %d %s\n", jlog_ctx_err(ctx), jlog_ctx_err_string(ctx)); 80 | exit(1); 81 | } 82 | 83 | count = jlog_ctx_read_interval(ctx, &begin, &end); 84 | if (count > 0) { 85 | int i; 86 | jlog_message m; 87 | 88 | for (i = 0; i < count; i++, JLOG_ID_ADVANCE(&begin)) { 89 | end = begin; 90 | 91 | if (jlog_ctx_read_message(ctx, &begin, &m) == 0) { 92 | printf("Got: %.*s\n", m.mess_len, (char*)m.mess); 93 | } else { 94 | fprintf(stderr, "jlog_ctx_read_message failed: %d %s\n", jlog_ctx_err(ctx), jlog_ctx_err_string(ctx)); 95 | } 96 | } 97 | 98 | // checkpoint (commit) our read: 99 | jlog_ctx_read_checkpoint(ctx, &end); 100 | } 101 | jlog_ctx_close(ctx); 102 | -------------------------------------------------------------------------------- /ChangeLog.md: -------------------------------------------------------------------------------- 1 | # ChangeLog 2 | 3 | ## 2 4 | 5 | ### 2.6.0 (2024-05-28) 6 | 7 | * Add support for switch from `mmap` to `pread` for message reads due to 8 | bad interaction with `mmap` on certain filesystems. 9 | * Correct issue where bulk message reads did not work with compression 10 | enabled. 11 | * Fix issue where short writes could happen due to failing to check the 12 | return value of `pwritev`. 13 | 14 | ### 2.5.4 (2022-09-20) 15 | 16 | * No longer perform an unneeded ownership check on the parent directory. This 17 | silences an erroneous/misleading jlogctl error message. 18 | 19 | ### 2.5.3 (2020-07-20) 20 | 21 | * Correct issue where writing to pre-commit buffer could cause jlog to 22 | advance with a zero-sized log segment. 23 | * Make jlogctl alter -p work. 24 | * Make jlogctl repair maintain all pre-existing valid meta fields. 25 | * `jlog_ctx_read_interval` now will automatically advance past more than one 26 | missing data files. 27 | 28 | ### 2.5.2 (2020-06-11) 29 | 30 | * Add an ownership check after jlogctl runs to alert operators to potentially 31 | unusable file ownership problems. 32 | 33 | ### 2.5.1 (2020-06-08) 34 | 35 | * Add a 'meta' subcommand to assist metastore inspection. 36 | * make `jlog_ctx_repair` repair data files when run with `aggressive` != 0 37 | * Offer `jlog_err_string` 38 | 39 | ### 2.5.0 (2020-04-22) 40 | 41 | * Rework jlogctl to be subcommand-based while preserving old flags. 42 | * Fix repair to correctly leave checkpoints intact. 43 | * Add full repair (and metastore reconstruction) to jlogctl. 44 | 45 | ### 2.4.0 (2019-07-11) 46 | 47 | * Improve speed of assessing the last log id. 48 | 49 | ### 2.3.2 (2018-08-07) 50 | 51 | * Address message writes larger than a precommit. 52 | * Fix memory leak when dealing with compressed records. 53 | 54 | ### 2.3.1.1 (2018-08-06) 55 | 56 | * Avoid writes to metastore in read mode. 57 | 58 | ### 2.3.1 (2018-07-25) 59 | 60 | * Handle interrupted syscalls when opening files. 61 | 62 | ### 2.3.0 (2017-05-19) 63 | 64 | * Add `jlog_ctx_bulk_read_messages` API. 65 | 66 | ### 2.2.2.1 (2017-02-09) 67 | 68 | * Avoid compiler warnings. 69 | * Fix support on FreeBSD. 70 | 71 | ### 2.2.2 (2016-12-14) 72 | 73 | * (no changes) 74 | 75 | ### 2.2.1.3 (2016-12-14) 76 | 77 | * Fix read side indexer to not skip late writer writes into current log file 78 | 79 | ### 2.2.1.2 (2016-08-26) 80 | 81 | * Fix deadlock in case where there is pre-commit resize and there is buffer to flush 82 | 83 | ### 2.2.1.1 (2016-08-22) 84 | 85 | * Force `-D_REENTRANT` 86 | * Allow changing of precommit size on open 87 | 88 | ### 2.2.1 (2016-05-19) 89 | 90 | * Support precommit buffer for higher performance 91 | * Use pwritev for lockless writes 92 | 93 | ### 2.2.0 (2016-05-05) 94 | 95 | * Add `jlog_ctx_set_subscriber_checkpoint` API 96 | 97 | ### 2.1.3.2 (2016-03-22) 98 | 99 | * JNI load fixes. 100 | 101 | ### 2.1.3.1 (2016-03-22) 102 | 103 | * Better JNI file links. 104 | 105 | ### 2.1.3 (2016-03-21) 106 | 107 | * Code cleanup 108 | * Java true-up and install improvements. 109 | 110 | ### 2.1.2 (no release) 111 | 112 | ### 2.1.1 (2016-03-04) 113 | 114 | * Correctly notice error in file opening. 115 | 116 | ### 2.1.0 (2016-03-04) 117 | 118 | * Add `jlog_ctx_repair` API 119 | * Java true-up 120 | 121 | ### 2.0.2 (2015-10-13) 122 | 123 | * Automatically repair in read internal when checkpoint is too big. 124 | * Several memory leaks fixed on FreeBSD. 125 | * Update Python wrapper. 126 | * Add `jlog_ctx_add_subscriber_copy_checkpoint` API. 127 | 128 | ### 2.0.1 (2015-03-23) 129 | 130 | * Validate header magic in `jlog_repair_datafile` 131 | 132 | ### 2.0.0 (2015-02-19) 133 | 134 | * Make a better effort of advancing past missing files. 135 | * Add `jlog_clean` API. 136 | * Make `jlog_pending_readers` more robust. 137 | * Support header magic other than 0. 138 | * Improve error messages. 139 | * Use mmap for metastore. 140 | * Darwin support. 141 | * Add sleep backoff and simplify subscriber removal. 142 | * Add `jlogtail`. 143 | * Fix complications when using from C++. 144 | 145 | ## 1 146 | 147 | ### 1.2.2 (2013-01-03) 148 | 149 | * PHP support. 150 | * Java JNI support. 151 | 152 | ### 1.2.1 (2012-11-15) 153 | 154 | * Address locking issue on Illumos and Solaris. 155 | * Fix handling of interrupted system calls. 156 | 157 | ### 1.2 (2011-08-31) 158 | 159 | * Fix uninitialized variable and interrupted `fstat()` operation. 160 | 161 | ### 1.1 (2011-02-22) 162 | 163 | * Optimize adding subscribers with `JLOG_BEGIN`. 164 | * Code cleanup. 165 | 166 | ### 1.0 (2009-05-13) 167 | 168 | * Initial release. 169 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2005-2008, Message Systems, Inc. 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are 6 | met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | * Redistributions in binary form must reproduce the above 11 | copyright notice, this list of conditions and the following 12 | disclaimer in the documentation and/or other materials provided 13 | with the distribution. 14 | * Neither the name Message Systems, Inc. nor the names 15 | of its contributors may be used to endorse or promote products 16 | derived from this software without specific prior written 17 | permission. 18 | 19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /Makefile.in: -------------------------------------------------------------------------------- 1 | # vim:ts=2:sw=2:noet: 2 | .SUFFIXES: .lo 3 | 4 | CC=@CC@ 5 | LN_S=@LN_S@ 6 | CPPFLAGS=@CPPFLAGS@ 7 | CFLAGS=@CFLAGS@ -D_REENTRANT 8 | LDFLAGS=@LDFLAGS@ 9 | LD_LIBJLOG_VERSION=@LD_LIBJLOG_VERSION@ 10 | AR=@AR@ 11 | RANLIB=@RANLIB@ 12 | LIBS=@LIBS@ 13 | INSTALL=@INSTALL@ 14 | SHLD=@SHLD@ 15 | PERL=@PERL@ 16 | SHCFLAGS=@SHCFLAGS@ 17 | DOTSO=@DOTSO@ 18 | JAVAC=@JAVAC@ 19 | JAVAH=@JAVAH@ 20 | JAR=@JAR@ 21 | JAVA_BITS=@JAVA_BITS@ 22 | 23 | MAJOR_VERSION=2 24 | MINOR_VERSION=6 25 | PATCH_VERSION=0 26 | VERSION=$(MAJOR_VERSION).$(MINOR_VERSION).$(PATCH_VERSION) 27 | LIBSHORT=@LIBSHORT@ 28 | LIBMAJOR=@LIBMAJOR@ 29 | LIBLONG=@LIBLONG@ 30 | 31 | prefix=@prefix@ 32 | exec_prefix=@exec_prefix@ 33 | bindir=@bindir@ 34 | sbindir=@sbindir@ 35 | libdir=@libdir@ 36 | includedir=@includedir@ 37 | libexecdir=@libexecdir@ 38 | datarootdir = @datarootdir@ 39 | mandir=@mandir@ 40 | mansubdir=@mansubdir@ 41 | docdir=${prefix}/@docdir@ 42 | sysconfdir=@sysconfdir@ 43 | srcdir=@srcdir@ 44 | top_srcdir=@top_srcdir@ 45 | 46 | AOBJS= \ 47 | jlog.o jlog_hash.o jlog_io.o jlog_compress.o 48 | SOOBJS= \ 49 | jlog.lo jlog_hash.lo jlog_io.lo jlog_compress.lo 50 | 51 | all: libjlog.$(DOTSO) libjlog.a jlogctl jlogtail 52 | 53 | .c.o: 54 | $(CC) $(CPPFLAGS) $(CFLAGS) -c $< 55 | 56 | .c.lo: 57 | $(CC) $(CPPFLAGS) $(CFLAGS) $(SHCFLAGS) -c $< -o $@ 58 | 59 | test: jthreadtest jtest 60 | 61 | perl/Makefile: perl/Makefile.PL 62 | cd perl && $(PERL) Makefile.PL 63 | 64 | jlogperl: perl/Makefile 65 | @cd perl && make 66 | 67 | jlogpython: 68 | cd python && make 69 | 70 | jlogctl: libjlog.a jlogctl.o getopt_long.o 71 | $(CC) $(CFLAGS) -o jlogctl jlogctl.o getopt_long.o libjlog.a $(LDFLAGS) $(LIBS) 72 | 73 | jthreadtest: libjlog.a jthreadtest.o getopt_long.o 74 | $(CC) $(CFLAGS) -o jthreadtest jthreadtest.o getopt_long.o libjlog.a $(LDFLAGS) $(LIBS) 75 | 76 | jtest: libjlog.a jtest.o 77 | $(CC) $(CFLAGS) -o jtest jtest.o libjlog.a $(LDFLAGS) $(LIBS) 78 | 79 | jlogtail: libjlog.a jlogtail.o 80 | $(CC) $(CFLAGS) -o jlogtail jlogtail.o libjlog.a $(LDFLAGS) $(LIBS) 81 | 82 | libjlog.$(DOTSO): $(SOOBJS) 83 | $(SHLD) $(LD_LIBJLOG_VERSION) -o libjlog.$(DOTSO) $(SOOBJS) $(CFLAGS) $(LDFLAGS) $(LIBS) 84 | 85 | libjlog.a: $(AOBJS) 86 | $(AR) cr libjlog.a $(AOBJS) 87 | $(RANLIB) libjlog.a 88 | 89 | java-bits: java/jlog.jar java/libjnijlog.so java/jlogTest.class 90 | 91 | # 92 | # It should be the case that the bridge include file is 93 | # autogenerated using javah 94 | # 95 | 96 | java/jlog.jar: java/jlog.java 97 | $(JAVAC) -d java java/jlog.java && \ 98 | $(JAVAH) -d java -classpath java com.omniti.labs.jlog && \ 99 | $(JAR) -cf $@ -C java com && \ 100 | rm -rf java/com 101 | 102 | java/jlogTest.class: java/jlogTest.java java/jlog.jar 103 | cd java && $(JAVAC) -cp jlog.jar jlogTest.java 104 | 105 | java/com_omniti_labs_jlog.lo: java/com_omniti_labs_jlog.c 106 | $(CC) -I. $(CPPFLAGS) $(CFLAGS) $(SHCFLAGS) $(LDFLAGS) -c $< -o $@ 107 | 108 | java/libjnijlog.so: java/com_omniti_labs_jlog.lo $(SOOBJS) 109 | $(SHLD) -o $@ java/com_omniti_labs_jlog.lo $(SOOBJS) $(CFLAGS) $(LDFLAGS) $(LIBS) 110 | 111 | install: all 112 | $(srcdir)/mkinstalldirs $(DESTDIR)$(bindir) 113 | $(srcdir)/mkinstalldirs $(DESTDIR)$(libdir) 114 | $(srcdir)/mkinstalldirs $(DESTDIR)$(includedir) 115 | $(INSTALL) -m 0755 jlogctl $(DESTDIR)$(bindir)/jlogctl 116 | $(INSTALL) -m 0755 jlogtail $(DESTDIR)$(bindir)/jlogtail 117 | $(INSTALL) -m 0755 jlog_change_endian.pl $(DESTDIR)$(bindir)/jlog_change_endian 118 | $(INSTALL) -m 0755 jlog_sanity_check.pl $(DESTDIR)$(bindir)/jlog_sanity_check 119 | $(INSTALL) -m 0755 libjlog.a $(DESTDIR)$(libdir)/libjlog.a 120 | $(INSTALL) -m 0755 libjlog.$(DOTSO) $(DESTDIR)$(libdir)/$(LIBLONG) 121 | $(LN_S) -f $(LIBLONG) $(DESTDIR)$(libdir)/$(LIBSHORT) 122 | $(LN_S) -f $(LIBLONG) $(DESTDIR)$(libdir)/$(LIBMAJOR) 123 | $(INSTALL) -m 0644 jlog.h $(DESTDIR)$(includedir)/jlog.h 124 | $(INSTALL) -m 0644 jlog_private.h $(DESTDIR)$(includedir)/jlog_private.h 125 | $(INSTALL) -m 0644 jlog_io.h $(DESTDIR)$(includedir)/jlog_io.h 126 | $(INSTALL) -m 0644 jlog_config.h $(DESTDIR)$(includedir)/jlog_config.h 127 | 128 | java-bits-install: 129 | $(srcdir)/mkinstalldirs $(DESTDIR)$(libdir)/java 130 | $(INSTALL) -m 0644 java/jlog.jar $(DESTDIR)$(libdir)/java/jlog.jar 131 | $(INSTALL) -m 0755 java/libjnijlog.so $(DESTDIR)$(libdir)/java/libjnijlog.so 132 | 133 | install-perl: 134 | cd perl ; make install DESTDIR=$(DESTDIR) INSTALLDIRS=vendor 135 | 136 | install-python: 137 | cd python && make install 138 | 139 | clean: 140 | rm -f *.o *.lo *.$(DOTSO) *.a jthreadtest jtest jlogctl jlogtail 141 | rm -f java/*.jar java/*.jnilib java/*.lo 142 | -if test -f perl/Makefile ; then cd perl ; make clean ; fi 143 | -if test -f python/Makefile ; then cd python ; make clean ; fi 144 | 145 | distclean: clean 146 | rm -f Makefile jlog_config.h perl/Makefile.PL 147 | 148 | .SUFFIXES: .c .o .lo 149 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # JLog 2 | 3 | [Change](ChangeLog.md) 4 | 5 | JLog is short for "journaled log" and this package is really an API and implementation that is libjlog. 6 | What is libjlog? libjlog is a pure C, **very simple** durable message queue with multiple subscribers and publishers 7 | (both thread- and multi-process safe). The basic concept is that publishers can open a log and write messages to it 8 | while subscribers open the log and consume messages from it. "That sounds easy." libjlog abstracts away the need to 9 | perform log rotation or maintenance by publishing into fixed size log buffers and eliminating old log buffers when 10 | there are no more consumers pending. 11 | 12 | ## Implementation Goal 13 | 14 | The goal of libjlog is to provide a dead-simple API, and a very small, uncomplicated implementation. 15 | The goal of the JLog is to have others use this project for adding durability to some of their asynchronous 16 | notification needs. 17 | 18 | ## Sample Use Case 19 | 20 | We run an application on server A and would like to write logs to server B. If server B is down, we don't want to lose 21 | logs and we do want A to continue operating. So A writes to a JLog and B consumes from that JLog. Sounds simple, but JLog 22 | is a low-level API. The implementor must provide a service on A to which B can connect to consume the JLog messages over 23 | the network. If you want to have you cake and eat it too, become a baker and use this fine ingredient! 24 | 25 | Read through [Concepts](./CONCEPTS.md) for a bit more detail on core concepts and a brief overview of how to use the C API. 26 | 27 | ## Installing 28 | 29 | FreeBSD port `databases/jlog`: 30 | 31 | pkg install jlog 32 | 33 | MacOS via [Homebrew](http://brew.sh/): 34 | 35 | brew install jlog 36 | 37 | If JLog is not packaged by your OS vendor, read on for build instructions. 38 | 39 | The LZ4 library is required if you want support for compressed logs. 40 | 41 | Java 6 or newer is required if you want to support using JLog from a Java application. 42 | 43 | To build from source, clone this repo and then: 44 | 45 | autoconf 46 | ./configure 47 | make 48 | make install 49 | 50 | ## Team 51 | 52 | * Wez Furlong 53 | * Alec Peterson 54 | * George Schlossnagle 55 | * Theo Schlossnagle (current maintainer) 56 | * Alexey Toptygin 57 | 58 | ## License 59 | 60 | JLog is released under a new BSD license. See our [license](./LICENSE) for details. 61 | -------------------------------------------------------------------------------- /V210_CHANGES.txt: -------------------------------------------------------------------------------- 1 | This goal of these changes was (a) add a Java method and a JNI bridge so 2 | that problems with the metastore and/or checkpoint files could be fixed; 3 | (b) add diagnostics so that the actual cause could be found. This 4 | file documents the changes. 5 | 6 | 1. Revision was bumped to 2.1.0 7 | 8 | 2. Function int jlog_ctx_repair(int aggressive) added to jlog.c 9 | In non-aggressive mode this function examines the metastore file 10 | and tries to fix it. It then examines the checkpoint file and 11 | tries to fix that. If both succeed then the function returns 1. 12 | If one or both fail, and aggressive mode is set, then the 13 | function attempts to delete all files in the jlog PATH directory 14 | (except the fassert files, see below), and then removes the 15 | directory itself. 16 | 17 | The function returns 1 on success and 0 on failure. 18 | 19 | 3. A JNI bridge was built with a java function with signature 20 | 21 | public native int repair(int aggressive); 22 | 23 | 4. A macro FASSERT() was created to log assertion failures throughout 24 | the jlog.c code in an effort to track down the pesky misbehaviors 25 | that sometimes occur. Unlike assert(), FASSERT() does not crash 26 | the process. It logs assertion failures to stderr. 27 | 28 | 5. Various small changes were made to configure.in and jlog_config.in 29 | to detect the presence of a few other include files. 30 | 31 | 6. A few trivial compiler warnings were addressed. 32 | 33 | 7. This file, V210_CHANGES, was created. 34 | 35 | 8. Tests were added to jtest.c 36 | 37 | 9. Some bug fixes were made as they were observed in the code. These bug 38 | fixes are: 39 | 40 | a. #include should be bracketed with #if HAVE_SYS_PARAM_H. 41 | b. javah is now run in the java-bits build to create the bridge .h file 42 | c. a trivial makefile was created in the python directory to eliminate 43 | errors from "make clean" and "gmake clean" 44 | d. two additional directories were added to the search for jni.h 45 | -------------------------------------------------------------------------------- /configure.in: -------------------------------------------------------------------------------- 1 | dnl vim:ts=2:sw=2:et: 2 | AC_INIT(jlog.c) 3 | AC_CONFIG_HEADER(jlog_config.h) 4 | 5 | AC_SUBST(CPPFLAGS, "$CPPFLAGS -I/usr/java/include -I/usr/java/include/solaris") 6 | 7 | AC_PROG_CC 8 | AC_C_INLINE 9 | AC_CANONICAL_HOST 10 | AC_C_BIGENDIAN 11 | AC_PROG_CPP 12 | AC_PROG_RANLIB 13 | AC_PROG_INSTALL 14 | AC_PROG_LN_S 15 | AC_PATH_PROG(AR, ar) 16 | AC_PATH_PROGS(PERL, perl) 17 | AC_PATH_PROGS(JAVAC, javac) 18 | AC_PATH_PROGS(JAR, jar) 19 | AC_PATH_PROGS(JAVAH, javah) 20 | AC_SUBST(PERL) 21 | 22 | # Checks for data types 23 | AC_CHECK_SIZEOF(char, 1) 24 | AC_CHECK_SIZEOF(short int, 2) 25 | AC_CHECK_SIZEOF(int, 4) 26 | AC_CHECK_SIZEOF(long int, 4) 27 | AC_CHECK_SIZEOF(long long int, 8) 28 | AC_CHECK_SIZEOF(void *, 1) 29 | 30 | AC_CHECK_LIB(rt, sem_init, , ) 31 | AC_CHECK_LIB(posix4, sem_wait, , ) 32 | 33 | AC_MSG_CHECKING([whether sem_init works]) 34 | AC_TRY_RUN( 35 | [ 36 | #include 37 | int main(void){sem_t s;return (0 != sem_init(&s,0,0));} 38 | ], 39 | [AC_MSG_RESULT(yes)], 40 | [ 41 | AC_MSG_RESULT(no) 42 | AC_DEFINE(BROKEN_SEM_INIT) 43 | AC_MSG_WARN([****** sem_init() is broken, I'll implement one myself.]) 44 | ] 45 | ) 46 | 47 | AC_CHECK_LIB(lz4, LZ4_compress_default, , ) 48 | AC_FUNC_STRFTIME 49 | AC_CHECK_FUNC(pwritev, [AC_DEFINE(HAVE_PWRITEV)], ) 50 | 51 | # Checks for header files. 52 | AC_CHECK_HEADERS(sys/file.h sys/types.h sys/uio.h dirent.h sys/param.h libgen.h \ 53 | stdint.h fcntl.h errno.h limits.h jni.h \ 54 | sys/resource.h pthread.h semaphore.h pwd.h stdio.h stdlib.h string.h \ 55 | ctype.h unistd.h time.h sys/stat.h sys/time.h unistd.h sys/mman.h lz4.h) 56 | 57 | JAVA_BITS=java-bits 58 | if test "x$ac_cv_header_jni_h" != "xyes" ; then 59 | JAVA_BITS= 60 | AC_MSG_WARN([Not building java bits due to jni.h absence (set CPPFLAGS?)]) 61 | fi 62 | AC_SUBST(JAVA_BITS) 63 | 64 | AC_CACHE_CHECK([for u_int type], ac_cv_have_u_int, [ 65 | AC_TRY_COMPILE( 66 | [ #include ], 67 | [ u_int a; a = 1; (void)a; ], 68 | [ ac_cv_have_u_int="yes" ], 69 | [ ac_cv_have_u_int="no" ] 70 | ) 71 | ]) 72 | 73 | if test "x$GCC" = "xyes" ; then 74 | CFLAGS="$CFLAGS -Wall -Wno-int-to-pointer-cast -Werror" 75 | fi 76 | 77 | if test "x$ac_cv_have_u_int" = "xyes" ; then 78 | AC_DEFINE(HAVE_U_INT) 79 | have_u_int=1 80 | fi 81 | 82 | AC_CACHE_CHECK([for intXX_t types], ac_cv_have_intxx_t, [ 83 | AC_TRY_COMPILE( 84 | [ #include ], 85 | [ int8_t a; int16_t b; int32_t c; a = b = c = 1; (void)a; (void)b; (void)c; ], 86 | [ ac_cv_have_intxx_t="yes" ], 87 | [ ac_cv_have_intxx_t="no" ] 88 | ) 89 | ]) 90 | if test "x$ac_cv_have_intxx_t" = "xyes" ; then 91 | AC_DEFINE(HAVE_INTXX_T) 92 | have_intxx_t=1 93 | fi 94 | 95 | AC_CACHE_CHECK([for int64_t type], ac_cv_have_int64_t, [ 96 | AC_TRY_COMPILE( 97 | [ #include ], 98 | [ int64_t a; a = 1; (void)a; ], 99 | [ ac_cv_have_int64_t="yes" ], 100 | [ ac_cv_have_int64_t="no" ] 101 | ) 102 | ]) 103 | if test "x$ac_cv_have_int64_t" = "xyes" ; then 104 | AC_DEFINE(HAVE_INT64_T) 105 | have_int64_t=1 106 | fi 107 | 108 | AC_CACHE_CHECK([for u_intXX_t types], ac_cv_have_u_intxx_t, [ 109 | AC_TRY_COMPILE( 110 | [ #include ], 111 | [ u_int8_t a; u_int16_t b; u_int32_t c; a = b = c = 1; (void)a; (void)b; (void)c; ], 112 | [ ac_cv_have_u_intxx_t="yes" ], 113 | [ ac_cv_have_u_intxx_t="no" ] 114 | ) 115 | ]) 116 | if test "x$ac_cv_have_u_intxx_t" = "xyes" ; then 117 | AC_DEFINE(HAVE_U_INTXX_T) 118 | have_u_intxx_t=1 119 | fi 120 | 121 | AC_CACHE_CHECK([for u_int64_t types], ac_cv_have_u_int64_t, [ 122 | AC_TRY_COMPILE( 123 | [ #include ], 124 | [ u_int64_t a; a = 1; (void)a; ], 125 | [ ac_cv_have_u_int64_t="yes" ], 126 | [ ac_cv_have_u_int64_t="no" ] 127 | ) 128 | ]) 129 | if test "x$ac_cv_have_u_int64_t" = "xyes" ; then 130 | AC_DEFINE(HAVE_U_INT64_T) 131 | have_u_int64_t=1 132 | fi 133 | 134 | if (test -z "$have_u_intxx_t" || test -z "$have_intxx_t" && \ 135 | test "x$ac_cv_header_sys_bitypes_h" = "xyes") 136 | then 137 | AC_MSG_CHECKING([for intXX_t and u_intXX_t types in sys/bitypes.h]) 138 | AC_TRY_COMPILE( 139 | [ 140 | #include 141 | ], 142 | [ 143 | int8_t a; int16_t b; int32_t c; 144 | u_int8_t e; u_int16_t f; u_int32_t g; 145 | a = b = c = e = f = g = 1; 146 | (void)a; (void)b; (void)c; 147 | (void)e; (void)f; (void)g; 148 | ], 149 | [ 150 | AC_DEFINE(HAVE_U_INTXX_T) 151 | AC_DEFINE(HAVE_INTXX_T) 152 | AC_MSG_RESULT(yes) 153 | ], 154 | [AC_MSG_RESULT(no)] 155 | ) 156 | fi 157 | 158 | if test -z "$have_u_intxx_t" ; then 159 | AC_CACHE_CHECK([for uintXX_t types], ac_cv_have_uintxx_t, [ 160 | AC_TRY_COMPILE( 161 | [ 162 | #include 163 | ], 164 | [ uint8_t a; uint16_t b; uint32_t c; a = b = c = 1; (void)a; (void)b; (void)c; ], 165 | [ ac_cv_have_uintxx_t="yes" ], 166 | [ ac_cv_have_uintxx_t="no" ] 167 | ) 168 | ]) 169 | if test "x$ac_cv_have_uintxx_t" = "xyes" ; then 170 | AC_DEFINE(HAVE_UINTXX_T) 171 | fi 172 | fi 173 | 174 | AC_CACHE_CHECK([for socklen_t], ac_cv_have_socklen_t, [ 175 | AC_TRY_COMPILE( 176 | [ 177 | #include 178 | #include 179 | ], 180 | [socklen_t foo; foo = 1235; (void)foo; ], 181 | [ ac_cv_have_socklen_t="yes" ], 182 | [ ac_cv_have_socklen_t="no" ] 183 | ) 184 | ]) 185 | if test "x$ac_cv_have_socklen_t" = "xyes" ; then 186 | AC_DEFINE(HAVE_SOCKLEN_T) 187 | fi 188 | 189 | AC_CACHE_CHECK([for size_t], ac_cv_have_size_t, [ 190 | AC_TRY_COMPILE( 191 | [ 192 | #include 193 | ], 194 | [ size_t foo; foo = 1235; (void)foo; ], 195 | [ ac_cv_have_size_t="yes" ], 196 | [ ac_cv_have_size_t="no" ] 197 | ) 198 | ]) 199 | if test "x$ac_cv_have_size_t" = "xyes" ; then 200 | AC_DEFINE(HAVE_SIZE_T) 201 | fi 202 | 203 | AC_CACHE_CHECK([for ssize_t], ac_cv_have_ssize_t, [ 204 | AC_TRY_COMPILE( 205 | [ 206 | #include 207 | ], 208 | [ ssize_t foo; foo = 1235; (void)foo; ], 209 | [ ac_cv_have_ssize_t="yes" ], 210 | [ ac_cv_have_ssize_t="no" ] 211 | ) 212 | ]) 213 | if test "x$ac_cv_have_ssize_t" = "xyes" ; then 214 | AC_DEFINE(HAVE_SSIZE_T) 215 | fi 216 | 217 | docdir="docs" 218 | mansubdir="man" 219 | AC_SUBST(docdir) 220 | AC_SUBST(mansubdir) 221 | 222 | AC_CHECK_LIB(pthread, pthread_create) 223 | 224 | DOTSO=so 225 | LIBSHORT='libjlog.$(DOTSO)' 226 | LIBMAJOR='libjlog.$(DOTSO).$(MAJOR_VERSION)' 227 | LIBLONG='libjlog.$(DOTSO).$(VERSION)' 228 | LD_LIBJLOG_VERSION='-Wl,-soname,libjlog.$(DOTSO).$(MAJOR_VERSION)' 229 | case $host in 230 | *solaris*) 231 | RLDFLAG="-R" 232 | if test "x$ac_compiler_gnu" = "xyes" ; then 233 | # GCC 234 | SHCFLAGS="$SHCFLAGS -fPIC -D_REENTRANT" 235 | CFLAGS="$CFLAGS" 236 | SHLD="$CC -shared" 237 | else 238 | # Sun C compiler 239 | SHCFLAGS="$SHCFLAGS -KPIC -mt" 240 | CFLAGS="$CFLAGS -mt" 241 | LIBS="-mt $LIBS" 242 | SHLD="$CC -G" 243 | fi 244 | ;; 245 | *linux*) 246 | CFLAGS="$CFLAGS -D_GNU_SOURCE" 247 | RLDFLAG="-Wl,--rpath=" 248 | case `uname -m` in 249 | *x86_64*) 250 | SHCFLAGS="$SHCFLAGS -fPIC" 251 | ;; 252 | esac 253 | SHCFLAGS="$SHCFLAGS -shared" 254 | SHLD="$CC -shared -rdynamic" 255 | ;; 256 | *darwin*) 257 | DOTSO=dylib 258 | LIBSHORT='libjlog.$(DOTSO)' 259 | LIBMAJOR='libjlog.$(MAJOR_VERSION).$(DOTSO)' 260 | LIBLONG='libjlog.$(VERSION).$(DOTSO)' 261 | LD_LIBJLOG_VERSION='-current_version $(VERSION) -install_name $(libdir)/libjlog.$(MAJOR_VERSION).$(DOTSO)' 262 | SHCFLAGS="$CFLAGS" 263 | SHLD="$CC -dynamiclib -single_module -undefined dynamic_lookup" 264 | ;; 265 | *freebsd*) 266 | SHCFLAGS="$CFLAGS -fPIC -pthread" 267 | SHLD="$CC -shared -rdynamic" 268 | ;; 269 | *) 270 | AC_MSG_ERROR(need to handle this case) 271 | exit 1 272 | ;; 273 | esac 274 | AC_SUBST(DOTSO) 275 | AC_SUBST(LIBSHORT) 276 | AC_SUBST(LIBMAJOR) 277 | AC_SUBST(LIBLONG) 278 | AC_SUBST(LD_LIBJLOG_VERSION) 279 | AC_SUBST(SHLD) 280 | AC_SUBST(SHCFLAGS) 281 | AC_SUBST(LDFLAGS) 282 | AC_SUBST(RLDFLAG) 283 | 284 | AC_OUTPUT([ 285 | Makefile 286 | perl/Makefile.PL 287 | ]) 288 | -------------------------------------------------------------------------------- /fassert.h: -------------------------------------------------------------------------------- 1 | #ifndef _FASSERT_H_ 2 | #define _FASSERT_H_ 3 | 4 | /* 5 | extern void fassertx(bool tf, int ln, const char *fn, const char *str); 6 | extern void fassertxend(void); 7 | extern void fassertxsetpath(const char *path); 8 | extern const char *fassertxgetpath(void); 9 | 10 | #define FASSERT(A,B) { bool tf = (A); fassertx(tf, __LINE__, __FILE__, \ 11 | (const char *)(B)); } 12 | 13 | */ 14 | #ifdef FDEBUG 15 | #define FASSERT(CTX,A,B) { int tf = (A); if ( tf == 0 ) \ 16 | fprintf(stderr, "%s: %s %d\n", \ 17 | (const char *)(B), __FILE__, \ 18 | __LINE__); } 19 | #else 20 | #define FASSERT(ctx, predicate, B...) { \ 21 | if(!(predicate) && (ctx) && ((jlog_ctx *)ctx)->error_func) { \ 22 | ((jlog_ctx *)ctx)->error_func(((jlog_ctx *)ctx)->error_ctx, B); \ 23 | } \ 24 | } 25 | #endif 26 | 27 | #endif 28 | -------------------------------------------------------------------------------- /getopt_long.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2005-2008, Message Systems, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are 7 | * met: 8 | * 9 | * * Redistributions of source code must retain the above copyright 10 | * notice, this list of conditions and the following disclaimer. 11 | * * Redistributions in binary form must reproduce the above 12 | * copyright notice, this list of conditions and the following 13 | * disclaimer in the documentation and/or other materials provided 14 | * with the distribution. 15 | * * Neither the name Message Systems, Inc. nor the names 16 | * of its contributors may be used to endorse or promote products 17 | * derived from this software without specific prior written 18 | * permission. 19 | * 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | */ 32 | 33 | #include "jlog_config.h" 34 | #include "getopt_long.h" 35 | #include 36 | #include 37 | 38 | int optind = 1; 39 | int opterr = 1; 40 | char *optarg = NULL; 41 | 42 | static void ec_error_func(const char *msg, ...) { 43 | va_list arg; 44 | va_start(arg, msg); 45 | vfprintf(stderr, msg, arg); 46 | va_end(arg); 47 | } 48 | 49 | getopt_error_func opterrfunc = ec_error_func; 50 | 51 | static int _getopt_long(int my_argc, 52 | char * const *my_argv, 53 | const char *optstring, 54 | const struct option *longopts, 55 | int *longindex) 56 | { 57 | const char *cur = NULL; 58 | const char *cur_longopt = NULL; 59 | int cur_longopt_len; 60 | char *argument = NULL; 61 | int i; 62 | 63 | cur = my_argv[optind]; 64 | cur_longopt = cur + 2; 65 | 66 | optind++; 67 | /* check for options that have an embedded '=' in them */ 68 | if((argument = strchr(cur_longopt, '=')) != NULL) { 69 | cur_longopt_len = argument - cur_longopt; 70 | argument++; 71 | } else { 72 | cur_longopt_len = strlen(cur_longopt); 73 | } 74 | for(i = 0; longopts && longopts[i].name; i++) { 75 | if(strlen(longopts[i].name) == cur_longopt_len && 76 | !strncmp(longopts[i].name, cur_longopt, cur_longopt_len)) 77 | { 78 | switch(longopts[i].has_arg) { 79 | case no_argument: 80 | if(argument) { 81 | /* error */ 82 | opterrfunc("argument --%.*s requires no arguments\n", cur_longopt_len, cur_longopt); 83 | return GETOPT_INVALID_OPTION; 84 | } 85 | /* no break, we fall through */ 86 | break; 87 | case required_argument: 88 | optarg = argument?argument:my_argv[optind]; 89 | if(!optarg) { 90 | /* error */ 91 | opterrfunc("argument %s requires an argument\n", cur); 92 | return GETOPT_INVALID_OPTION; 93 | } 94 | if(!argument) optind++; 95 | break; 96 | case optional_argument: 97 | optarg = argument; 98 | break; 99 | } 100 | if(longindex) *longindex = i; 101 | if(longopts[i].flag) { 102 | *longopts[i].flag = longopts[i].val; 103 | return 0; 104 | } else { 105 | return longopts[i].val; 106 | } 107 | } 108 | } 109 | /* we can only reach here if we have an unknown option */ 110 | opterrfunc("unknown long option %s\n", cur); 111 | return GETOPT_INVALID_OPTION; 112 | } 113 | 114 | static int _getopt(int my_argc, char * const *my_argv, const char *optstring) 115 | { 116 | const char *cur; 117 | char *optstring_off; 118 | char cur_shortopt; 119 | 120 | cur = my_argv[optind]; 121 | /* search simple option texts */ 122 | cur_shortopt = cur[1]; 123 | if(cur_shortopt == GETOPT_MISSING_OPTION || 124 | cur_shortopt == GETOPT_INVALID_OPTION || 125 | (optstring_off = strchr(optstring, cur_shortopt)) == NULL) 126 | { 127 | /* error case, none found */ 128 | opterrfunc("unknown option: %c\n", cur_shortopt); 129 | return GETOPT_INVALID_OPTION; 130 | } 131 | optind++; 132 | if(optstring_off[1] == ':') { 133 | /* takes an option */ 134 | if(cur[2]) { 135 | /* argument is concatenated with the option */ 136 | optarg = (char *) cur + 2; 137 | } else { 138 | if(optind >= my_argc) { 139 | /* end of args */ 140 | opterrfunc("option %c requires an argument\n", cur_shortopt); 141 | return GETOPT_INVALID_OPTION; 142 | } 143 | optarg = my_argv[optind]; 144 | optind++; 145 | } 146 | } else { 147 | if(cur[2]) { 148 | /* we have a concatenated argument, but our parameter takes no args */ 149 | opterrfunc("option %c does not take an argument\n", cur_shortopt); 150 | return GETOPT_INVALID_OPTION; 151 | } 152 | /* does not take an option */ 153 | } 154 | return cur_shortopt; 155 | } 156 | 157 | int getopt_long(int my_argc, 158 | char * const *my_argv, 159 | const char *optstring, 160 | const struct option *longopts, 161 | int *longindex) 162 | { 163 | const char *cur; 164 | 165 | if(optind >= my_argc) { 166 | /* end of args */ 167 | return -1; 168 | } 169 | cur = my_argv[optind]; 170 | if(cur[0] != '-' || (cur[1] == '-' && cur[2] == '\0')) { 171 | /* end of args */ 172 | /* need to handle POSIXLY_CORRECT case */ 173 | return -1; 174 | } 175 | if(cur[1] == '-') { 176 | /* this is a long option, dispatch appropriately */ 177 | return _getopt_long(my_argc, my_argv, optstring, longopts, longindex); 178 | } else { 179 | /* short opt */ 180 | return _getopt(my_argc, my_argv, optstring); 181 | } 182 | } 183 | 184 | #if defined(_WIN32) 185 | int getopt(int my_argc, char * const *my_argv, const char *optstring) 186 | { 187 | const char *cur; 188 | 189 | if(optind >= my_argc) { 190 | /* end of args */ 191 | return -1; 192 | } 193 | cur = my_argv[optind]; 194 | if(cur[0] != '-' || (cur[1] == '-' && cur[2] == '\0')) { 195 | /* end of args */ 196 | /* need to handle POSIXLY_CORRECT case */ 197 | return -1; 198 | } 199 | return _getopt(my_argc, my_argv, optstring); 200 | } 201 | #endif 202 | -------------------------------------------------------------------------------- /getopt_long.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2005-2008, Message Systems, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are 7 | * met: 8 | * 9 | * * Redistributions of source code must retain the above copyright 10 | * notice, this list of conditions and the following disclaimer. 11 | * * Redistributions in binary form must reproduce the above 12 | * copyright notice, this list of conditions and the following 13 | * disclaimer in the documentation and/or other materials provided 14 | * with the distribution. 15 | * * Neither the name Message Systems, Inc. nor the names 16 | * of its contributors may be used to endorse or promote products 17 | * derived from this software without specific prior written 18 | * permission. 19 | * 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | */ 32 | 33 | #ifndef GETOPT_LONG_H 34 | #define GETOPT_LONG_H 35 | 36 | #include "jlog_config.h" 37 | 38 | #ifdef __cplusplus 39 | extern "C" { 40 | #endif 41 | 42 | typedef void (*getopt_error_func) (const char *msg, ...); 43 | 44 | extern int opterr; 45 | extern int optind; 46 | extern int optopt; 47 | extern int optreset; 48 | extern char *optarg; 49 | extern getopt_error_func opterrfunc; 50 | 51 | struct option { 52 | const char *name; 53 | int has_arg; 54 | int *flag; 55 | int val; 56 | }; 57 | 58 | #define no_argument 0 59 | #define required_argument 1 60 | #define optional_argument 2 61 | 62 | #define GETOPT_INVALID_OPTION (int) '?' 63 | #define GETOPT_MISSING_OPTION (int) ':' 64 | 65 | 66 | int getopt_long (int argc, char *const *argv, const char *shortopts, 67 | const struct option *longopts, int *longind); 68 | 69 | #if defined(_WIN32) 70 | int getopt(int nargc, char * const *nargv, const char *options); 71 | #endif 72 | 73 | #ifdef __cplusplus 74 | } /* Close scope of 'extern "C"' declaration which encloses file. */ 75 | #endif 76 | 77 | #endif 78 | -------------------------------------------------------------------------------- /install-sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # install - install a program, script, or datafile 4 | # This comes from X11R5 (mit/util/scripts/install.sh). 5 | # 6 | # Copyright 1991 by the Massachusetts Institute of Technology 7 | # 8 | # Permission to use, copy, modify, distribute, and sell this software and its 9 | # documentation for any purpose is hereby granted without fee, provided that 10 | # the above copyright notice appear in all copies and that both that 11 | # copyright notice and this permission notice appear in supporting 12 | # documentation, and that the name of M.I.T. not be used in advertising or 13 | # publicity pertaining to distribution of the software without specific, 14 | # written prior permission. M.I.T. makes no representations about the 15 | # suitability of this software for any purpose. It is provided "as is" 16 | # without express or implied warranty. 17 | # 18 | # Calling this script install-sh is preferred over install.sh, to prevent 19 | # `make' implicit rules from creating a file called install from it 20 | # when there is no Makefile. 21 | # 22 | # This script is compatible with the BSD install script, but was written 23 | # from scratch. It can only install one file at a time, a restriction 24 | # shared with many OS's install programs. 25 | 26 | 27 | # set DOITPROG to echo to test this script 28 | 29 | # Don't use :- since 4.3BSD and earlier shells don't like it. 30 | doit="${DOITPROG-}" 31 | 32 | 33 | # put in absolute paths if you don't have them in your path; or use env. vars. 34 | 35 | mvprog="${MVPROG-mv}" 36 | cpprog="${CPPROG-cp}" 37 | chmodprog="${CHMODPROG-chmod}" 38 | chownprog="${CHOWNPROG-chown}" 39 | chgrpprog="${CHGRPPROG-chgrp}" 40 | stripprog="${STRIPPROG-strip}" 41 | rmprog="${RMPROG-rm}" 42 | mkdirprog="${MKDIRPROG-mkdir}" 43 | 44 | transformbasename="" 45 | transform_arg="" 46 | instcmd="$mvprog" 47 | chmodcmd="$chmodprog 0755" 48 | chowncmd="" 49 | chgrpcmd="" 50 | stripcmd="" 51 | rmcmd="$rmprog -f" 52 | mvcmd="$mvprog" 53 | src="" 54 | dst="" 55 | dir_arg="" 56 | 57 | while [ x"$1" != x ]; do 58 | case $1 in 59 | -c) instcmd="$cpprog" 60 | shift 61 | continue;; 62 | 63 | -d) dir_arg=true 64 | shift 65 | continue;; 66 | 67 | -m) chmodcmd="$chmodprog $2" 68 | shift 69 | shift 70 | continue;; 71 | 72 | -o) chowncmd="$chownprog $2" 73 | shift 74 | shift 75 | continue;; 76 | 77 | -g) chgrpcmd="$chgrpprog $2" 78 | shift 79 | shift 80 | continue;; 81 | 82 | -s) stripcmd="$stripprog" 83 | shift 84 | continue;; 85 | 86 | -t=*) transformarg=`echo $1 | sed 's/-t=//'` 87 | shift 88 | continue;; 89 | 90 | -b=*) transformbasename=`echo $1 | sed 's/-b=//'` 91 | shift 92 | continue;; 93 | 94 | *) if [ x"$src" = x ] 95 | then 96 | src=$1 97 | else 98 | # this colon is to work around a 386BSD /bin/sh bug 99 | : 100 | dst=$1 101 | fi 102 | shift 103 | continue;; 104 | esac 105 | done 106 | 107 | if [ x"$src" = x ] 108 | then 109 | echo "install: no input file specified" 110 | exit 1 111 | else 112 | true 113 | fi 114 | 115 | if [ x"$dir_arg" != x ]; then 116 | dst=$src 117 | src="" 118 | 119 | if [ -d $dst ]; then 120 | instcmd=: 121 | chmodcmd="" 122 | else 123 | instcmd=mkdir 124 | fi 125 | else 126 | 127 | # Waiting for this to be detected by the "$instcmd $src $dsttmp" command 128 | # might cause directories to be created, which would be especially bad 129 | # if $src (and thus $dsttmp) contains '*'. 130 | 131 | if [ -f $src -o -d $src ] 132 | then 133 | true 134 | else 135 | echo "install: $src does not exist" 136 | exit 1 137 | fi 138 | 139 | if [ x"$dst" = x ] 140 | then 141 | echo "install: no destination specified" 142 | exit 1 143 | else 144 | true 145 | fi 146 | 147 | # If destination is a directory, append the input filename; if your system 148 | # does not like double slashes in filenames, you may need to add some logic 149 | 150 | if [ -d $dst ] 151 | then 152 | dst="$dst"/`basename $src` 153 | else 154 | true 155 | fi 156 | fi 157 | 158 | ## this sed command emulates the dirname command 159 | dstdir=`echo $dst | sed -e 's,[^/]*$,,;s,/$,,;s,^$,.,'` 160 | 161 | # Make sure that the destination directory exists. 162 | # this part is taken from Noah Friedman's mkinstalldirs script 163 | 164 | # Skip lots of stat calls in the usual case. 165 | if [ ! -d "$dstdir" ]; then 166 | defaultIFS=' 167 | ' 168 | IFS="${IFS-${defaultIFS}}" 169 | 170 | oIFS="${IFS}" 171 | # Some sh's can't handle IFS=/ for some reason. 172 | IFS='%' 173 | set - `echo ${dstdir} | sed -e 's@/@%@g' -e 's@^%@/@'` 174 | IFS="${oIFS}" 175 | 176 | pathcomp='' 177 | 178 | while [ $# -ne 0 ] ; do 179 | pathcomp="${pathcomp}${1}" 180 | shift 181 | 182 | if [ ! -d "${pathcomp}" ] ; 183 | then 184 | $mkdirprog "${pathcomp}" 185 | else 186 | true 187 | fi 188 | 189 | pathcomp="${pathcomp}/" 190 | done 191 | fi 192 | 193 | if [ x"$dir_arg" != x ] 194 | then 195 | $doit $instcmd $dst && 196 | 197 | if [ x"$chowncmd" != x ]; then $doit $chowncmd $dst; else true ; fi && 198 | if [ x"$chgrpcmd" != x ]; then $doit $chgrpcmd $dst; else true ; fi && 199 | if [ x"$stripcmd" != x ]; then $doit $stripcmd $dst; else true ; fi && 200 | if [ x"$chmodcmd" != x ]; then $doit $chmodcmd $dst; else true ; fi 201 | else 202 | 203 | # If we're going to rename the final executable, determine the name now. 204 | 205 | if [ x"$transformarg" = x ] 206 | then 207 | dstfile=`basename $dst` 208 | else 209 | dstfile=`basename $dst $transformbasename | 210 | sed $transformarg`$transformbasename 211 | fi 212 | 213 | # don't allow the sed command to completely eliminate the filename 214 | 215 | if [ x"$dstfile" = x ] 216 | then 217 | dstfile=`basename $dst` 218 | else 219 | true 220 | fi 221 | 222 | # Make a temp file name in the proper directory. 223 | 224 | dsttmp=$dstdir/#inst.$$# 225 | 226 | # Move or copy the file name to the temp name 227 | 228 | $doit $instcmd $src $dsttmp && 229 | 230 | trap "rm -f ${dsttmp}" 0 && 231 | 232 | # and set any options; do chmod last to preserve setuid bits 233 | 234 | # If any of these fail, we abort the whole thing. If we want to 235 | # ignore errors from any of these, just make sure not to ignore 236 | # errors from the above "$doit $instcmd $src $dsttmp" command. 237 | 238 | if [ x"$chowncmd" != x ]; then $doit $chowncmd $dsttmp; else true;fi && 239 | if [ x"$chgrpcmd" != x ]; then $doit $chgrpcmd $dsttmp; else true;fi && 240 | if [ x"$stripcmd" != x ]; then $doit $stripcmd $dsttmp; else true;fi && 241 | if [ x"$chmodcmd" != x ]; then $doit $chmodcmd $dsttmp; else true;fi && 242 | 243 | # Now rename the file to the real destination. 244 | 245 | $doit $rmcmd -f $dstdir/$dstfile && 246 | $doit $mvcmd $dsttmp $dstdir/$dstfile 247 | 248 | fi && 249 | 250 | 251 | exit 0 252 | -------------------------------------------------------------------------------- /java/.gitignore: -------------------------------------------------------------------------------- 1 | *.h 2 | com 3 | -------------------------------------------------------------------------------- /java/jlog.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2012, Circonus, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are 7 | * met: 8 | * 9 | * * Redistributions of source code must retain the above copyright 10 | * notice, this list of conditions and the following disclaimer. 11 | * * Redistributions in binary form must reproduce the above 12 | * copyright notice, this list of conditions and the following 13 | * disclaimer in the documentation and/or other materials provided 14 | * with the distribution. 15 | * * Neither the name Circonus, Inc. nor the names 16 | * of its contributors may be used to endorse or promote products 17 | * derived from this software without specific prior written 18 | * permission. 19 | * 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | */ 32 | package com.omniti.labs; 33 | 34 | import java.util.Date; 35 | 36 | public class jlog { 37 | public enum jlog_safety { JLOG_UNSAFE, JLOG_ALMOST_SAFE, JLOG_SAFE }; 38 | public enum jlog_position { JLOG_BEGIN, JLOG_END }; 39 | public enum jlog_err { 40 | JLOG_ERR_SUCCESS, 41 | JLOG_ERR_ILLEGAL_INIT, 42 | JLOG_ERR_ILLEGAL_OPEN, 43 | JLOG_ERR_OPEN, 44 | JLOG_ERR_NOTDIR, 45 | JLOG_ERR_CREATE_PATHLEN, 46 | JLOG_ERR_CREATE_EXISTS, 47 | JLOG_ERR_CREATE_MKDIR, 48 | JLOG_ERR_CREATE_META, 49 | JLOG_ERR_CREATE_PRE_COMMIT, 50 | JLOG_ERR_LOCK, 51 | JLOG_ERR_IDX_OPEN, 52 | JLOG_ERR_IDX_SEEK, 53 | JLOG_ERR_IDX_CORRUPT, 54 | JLOG_ERR_IDX_WRITE, 55 | JLOG_ERR_IDX_READ, 56 | JLOG_ERR_FILE_OPEN, 57 | JLOG_ERR_FILE_SEEK, 58 | JLOG_ERR_FILE_CORRUPT, 59 | JLOG_ERR_FILE_READ, 60 | JLOG_ERR_FILE_WRITE, 61 | JLOG_ERR_META_OPEN, 62 | JLOG_ERR_PRE_COMMIT_OPEN, 63 | JLOG_ERR_ILLEGAL_WRITE, 64 | JLOG_ERR_ILLEGAL_CHECKPOINT, 65 | JLOG_ERR_INVALID_SUBSCRIBER, 66 | JLOG_ERR_ILLEGAL_LOGID, 67 | JLOG_ERR_SUBSCRIBER_EXISTS, 68 | JLOG_ERR_CHECKPOINT, 69 | JLOG_ERR_NOT_SUPPORTED 70 | }; 71 | 72 | public enum jlog_compression_provider_choice { 73 | JLOG_COMPRESSION_NULL, 74 | JLOG_COMPRESSION_LZ4 75 | }; 76 | 77 | public class jlogException extends Exception { 78 | public jlogException(String a) { super(a); } 79 | } 80 | public class jlogAlreadyOpenedException extends jlogException { 81 | public jlogAlreadyOpenedException(String a) { super(a); } 82 | } 83 | public class jlogNotWriterException extends jlogException { 84 | public jlogNotWriterException(String a) { super(a); } 85 | } 86 | public class jlogNotReaderException extends jlogException { 87 | public jlogNotReaderException(String a) { super(a); } 88 | } 89 | public class jlogSubscriberExistsException extends jlogException { 90 | public jlogSubscriberExistsException(String a) { super(a); } 91 | } 92 | public class jlogInvalidSubscriberException extends jlogException { 93 | public jlogInvalidSubscriberException(String a) { super(a); } 94 | } 95 | public class jlogIOException extends jlogException { 96 | public jlogIOException(String a) { super(a); } 97 | } 98 | 99 | public class Id { 100 | private long log; 101 | private long marker; 102 | public long getLog() { return log; } 103 | public long getMarker() { return marker; } 104 | public void increment() { marker++; } 105 | public boolean equals(Id o) { 106 | return (log == o.getLog() && marker == o.getMarker()); 107 | } 108 | public int hashCode() { return (int)log ^ (int)marker; } 109 | public String toString() { 110 | return log + "/" + marker; 111 | } 112 | }; 113 | public class Interval { 114 | private Id start; 115 | private Id finish; 116 | private int count; 117 | public Id getStart() { return start; } 118 | public Id getFinish() { return finish; } 119 | public int count() { return count; } 120 | public boolean equals(Interval o) { 121 | return (start.equals(o.getStart()) && finish.equals(o.getFinish())); 122 | } 123 | public int hashCode() { return start.hashCode() ^ finish.hashCode(); } 124 | public String toString() { 125 | return start + ":" + finish; 126 | } 127 | }; 128 | public class Message { 129 | long when; 130 | byte[] data; 131 | Message(long ms, byte[] w) { 132 | when = ms; data = w; 133 | } 134 | public long getTime() { return when; } 135 | public Date getDate() { return new Date(when); } 136 | public byte[] getData() { return data; } 137 | }; 138 | 139 | private long ctx; 140 | private String subscriber; 141 | private native void jlog_ctx_new(String path); 142 | private native void jlog_ctx_close(); 143 | private native void jlog_ctx_init() throws jlogIOException; 144 | public native long raw_size(); 145 | public native Id get_checkpoint(String subscriber) 146 | throws jlogIOException; 147 | public native String[] list_subscribers(); 148 | public native jlog_err get_error(); 149 | public native String get_error_string(); 150 | public native int get_errno(); 151 | public native void open_writer() 152 | throws jlogAlreadyOpenedException, jlogIOException; 153 | public native void open_reader(String sub) 154 | throws jlogAlreadyOpenedException, jlogIOException, jlogInvalidSubscriberException; 155 | public native void close(); 156 | public native int repair(int aggressive); 157 | private native void alter_mode(int mode); 158 | private native void alter_journal_size(long size); 159 | private native void alter_safety(jlog_safety safety); 160 | public native void set_multi_process(boolean mp); 161 | public native void set_use_compression(boolean mp); 162 | public native void set_compression_provider(jlog_compression_provider_choice cp); 163 | public native void set_pre_commit_buffer_size(long size); 164 | public native void flush_pre_commit_buffer() throws jlogIOException; 165 | public native void add_subscriber(String sub, jlog_position whence) 166 | throws jlogSubscriberExistsException, jlogIOException; 167 | public native void remove_subscriber(String sub) 168 | throws jlogInvalidSubscriberException, jlogIOException; 169 | public native void write(byte[] mess, long usec_epoch) 170 | throws jlogNotWriterException, jlogIOException; 171 | public native Interval read_interval() 172 | throws jlogNotReaderException, jlogIOException; 173 | public native Message read(Id what) 174 | throws jlogNotReaderException, jlogIOException; 175 | public native void read_checkpoint(Id what) 176 | throws jlogNotReaderException, jlogIOException; 177 | public native Id advance(Id what, Id start, Id finish) 178 | throws jlogNotReaderException, jlogIOException; 179 | public native Id first_log_id(Id id); 180 | public native Id last_log_id(Id id); 181 | 182 | 183 | public jlog(String path) { 184 | jlog_ctx_new(path); 185 | } 186 | protected void finalize() { 187 | close(); 188 | } 189 | public void init() throws jlogIOException { 190 | jlog_ctx_init(); 191 | } 192 | public void init(int mode, long size, jlog_safety safety) throws jlogIOException { 193 | alter_mode(mode); 194 | alter_journal_size(size); 195 | alter_safety(safety); 196 | jlog_ctx_init(); 197 | } 198 | public Id get_checkpoint() 199 | throws jlogNotReaderException, jlogIOException { 200 | if(subscriber == null) throw new jlogNotReaderException("JLOG_ERR_ILLEGAL_OPEN"); 201 | return get_checkpoint(subscriber); 202 | } 203 | public void write(byte[] mess, Date when) throws jlogNotWriterException, jlogIOException { 204 | write(mess, when.getTime()); 205 | } 206 | public void write(String mess, Date when) throws jlogNotWriterException, jlogIOException { 207 | write(mess.getBytes(), when.getTime()); 208 | } 209 | public void write(byte[] mess) throws jlogNotWriterException, jlogIOException { 210 | write(mess, (new Date()).getTime()); 211 | } 212 | public void write(String mess) throws jlogNotWriterException, jlogIOException { 213 | write(mess.getBytes(), (new Date()).getTime()); 214 | } 215 | public Id advance(Id what, Interval interval) 216 | throws jlogNotReaderException, jlogIOException { 217 | return advance(what, interval.getStart(), interval.getFinish()); 218 | } 219 | 220 | static { 221 | try { 222 | System.loadLibrary("jnijlog"); 223 | } catch (Exception e) { 224 | System.err.println("Cannot load jlog JNI library: " + e); 225 | System.exit(-1); 226 | } 227 | } 228 | 229 | } 230 | -------------------------------------------------------------------------------- /java/jlogTest.java: -------------------------------------------------------------------------------- 1 | import com.omniti.labs.jlog; 2 | 3 | import java.io.File; 4 | import java.io.FileNotFoundException; 5 | import java.io.IOException; 6 | import java.io.PrintWriter; 7 | 8 | 9 | public class jlogTest { 10 | static final String payload = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; 11 | static final String[] names = { "00000001", "00000003", "0000010a", "cp.7473" } ; 12 | static final int pcnt = 1000000; 13 | String jlogpath = "/tmp/foo.jlog"; 14 | //String[] args; 15 | 16 | void log(String m) { System.err.println(m); } 17 | 18 | void delete(File f) throws IOException { 19 | if (f.isDirectory()) { 20 | for (File c : f.listFiles()) 21 | delete(c); 22 | } 23 | if (!f.delete()) 24 | throw new FileNotFoundException("Failed to delete file: " + f); 25 | } 26 | 27 | void fail(String msg) throws Exception { 28 | throw new Exception(msg); 29 | } 30 | 31 | void test_subscriber(String s) throws Exception { 32 | log("adding subscriber " + s); 33 | jlog ctx = new jlog(jlogpath); 34 | ctx.add_subscriber(s, jlog.jlog_position.JLOG_BEGIN); 35 | try { 36 | ctx.add_subscriber(s, jlog.jlog_position.JLOG_BEGIN); 37 | } catch(jlog.jlogSubscriberExistsException ok) { 38 | } 39 | } 40 | 41 | void initialize() throws Exception { 42 | File f = new File(jlogpath); 43 | if(f.exists()) { 44 | log("cleaning up " + jlogpath); 45 | delete(f); 46 | } 47 | log("initializing new jlog at " + jlogpath); 48 | jlog ctx = new jlog(jlogpath); 49 | ctx.init(); 50 | } 51 | 52 | void assert_subscriber(String s) throws Exception { 53 | log("checking subscriber " + s); 54 | jlog ctx = new jlog(jlogpath); 55 | String[] subs = ctx.list_subscribers(); 56 | for(String sub : subs) { 57 | if(sub.equals(s)) return; 58 | } 59 | fail("nope"); 60 | } 61 | 62 | jlog open_writer() throws Exception { 63 | log("opening writer"); 64 | jlog ctx = new jlog(jlogpath); 65 | ctx.open_writer(); 66 | return ctx; 67 | } 68 | 69 | jlog open_reader(String s) throws Exception { 70 | log("opening reader: " + s); 71 | jlog ctx = new jlog(jlogpath); 72 | ctx.open_reader(s); 73 | return ctx; 74 | } 75 | 76 | void write_payloads(int cnt) throws Exception { 77 | jlog ctx = open_writer(); 78 | log("writing out " + cnt + " " + payload.getBytes().length + " byte payloads"); 79 | for(int i = 0; i < cnt; i++) 80 | ctx.write(payload); 81 | } 82 | 83 | void read_check(String s, int expect, boolean sizeup) throws Exception { 84 | int cnt = 0; 85 | jlog ctx = open_reader(s); 86 | long start = ctx.raw_size(); 87 | while(true) { 88 | jlog.Id chkpt, cur; 89 | jlog.Interval interval = ctx.read_interval(); 90 | chkpt = cur = interval.getStart(); 91 | log("reading interval(" + cur + "): " + interval); 92 | int i, batchsize = interval.count(); 93 | if(batchsize == 0) break; 94 | for(i = 0; i < batchsize; i++) { 95 | if(i != 0) cur.increment(); 96 | @SuppressWarnings("unused") 97 | jlog.Message m = ctx.read(cur); 98 | cnt++; 99 | } 100 | ctx.read_checkpoint(chkpt); 101 | } 102 | if(cnt != expect) fail("got wrong read count: " + cnt + " != " + expect); 103 | long end = ctx.raw_size(); 104 | log("checking that size " + (sizeup ? "increased" : "decreased")); 105 | if(sizeup && (end < start)) fail("size didn't increase as exptected"); 106 | if(!sizeup && (end > start)) fail("size didn't decrease as expected"); 107 | } 108 | 109 | void addonefile(String fn, int i) throws Exception { 110 | String x = jlogpath + "/" + fn; 111 | PrintWriter pw = new PrintWriter(x); 112 | pw.write(payload, i*7, 7); 113 | pw.close(); 114 | } 115 | 116 | void corruptmetastore() throws Exception { 117 | String x = jlogpath + "/metastore"; 118 | PrintWriter pw = new PrintWriter(x); 119 | pw.write(payload, 0, 16); 120 | pw.close(); 121 | } 122 | 123 | void addsomefiles() throws Exception { 124 | for(int i=0;i 0) jlogpath = called_args[0]; 140 | } 141 | 142 | public void run() { 143 | try { 144 | initialize(); 145 | test_subscriber("testing"); 146 | assert_subscriber("testing"); 147 | test_subscriber("witness"); 148 | assert_subscriber("witness"); 149 | try { assert_subscriber("badguy"); 150 | fail("found badguy subscriber"); } 151 | catch (Exception e) { 152 | if(!e.getMessage().equals("nope")) throw e; 153 | } 154 | write_payloads(pcnt); 155 | read_check("witness", pcnt, true); 156 | read_check("testing", pcnt, false); 157 | /* reinitialize for the repair check */ 158 | initialize(); 159 | test_repair("repair"); 160 | } catch(Exception catchall) { 161 | catchall.printStackTrace(); 162 | } 163 | } 164 | 165 | public static void main(String[] args) { 166 | (new jlogTest(args)).run(); 167 | } 168 | } 169 | -------------------------------------------------------------------------------- /jlog.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2005-2008, Message Systems, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are 7 | * met: 8 | * 9 | * * Redistributions of source code must retain the above copyright 10 | * notice, this list of conditions and the following disclaimer. 11 | * * Redistributions in binary form must reproduce the above 12 | * copyright notice, this list of conditions and the following 13 | * disclaimer in the documentation and/or other materials provided 14 | * with the distribution. 15 | * * Neither the name Message Systems, Inc. nor the names 16 | * of its contributors may be used to endorse or promote products 17 | * derived from this software without specific prior written 18 | * permission. 19 | * 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | */ 32 | 33 | #ifndef _JLOG_H 34 | #define _JLOG_H 35 | 36 | #include "jlog_config.h" 37 | 38 | #ifndef JLOG_API 39 | # ifdef _WIN32 40 | # ifdef JLOG_EXPORTS 41 | # define JLOG_API(x) __declspec(dllexport) x 42 | # else 43 | # define JLOG_API(x) __declspec(dllimport) x 44 | # endif 45 | # else 46 | # ifdef __cplusplus 47 | # define JLOG_API(x) extern "C" x 48 | # else 49 | # define JLOG_API(x) x 50 | # endif 51 | # endif 52 | #endif 53 | 54 | struct _jlog_ctx; 55 | struct _jlog_message_header; 56 | struct _jlog_id; 57 | 58 | typedef struct _jlog_ctx jlog_ctx; 59 | 60 | typedef struct _jlog_message_header { 61 | u_int32_t reserved; 62 | u_int32_t tv_sec; 63 | u_int32_t tv_usec; 64 | u_int32_t mlen; 65 | } jlog_message_header; 66 | 67 | typedef struct _jlog_message_header_compressed { 68 | u_int32_t reserved; 69 | u_int32_t tv_sec; 70 | u_int32_t tv_usec; 71 | u_int32_t mlen; 72 | u_int32_t compressed_len; 73 | } jlog_message_header_compressed; 74 | 75 | typedef struct _jlog_id { 76 | u_int32_t log; 77 | u_int32_t marker; 78 | } jlog_id; 79 | 80 | #define JLOG_ID_ADVANCE(id) (id)->marker++ 81 | 82 | typedef struct _jlog_message { 83 | jlog_message_header_compressed *header; 84 | u_int32_t mess_len; 85 | void *mess; 86 | jlog_message_header_compressed aligned_header; 87 | } jlog_message; 88 | 89 | typedef enum { 90 | JLOG_BEGIN, 91 | JLOG_END 92 | } jlog_position; 93 | 94 | typedef enum { 95 | JLOG_UNSAFE, 96 | JLOG_ALMOST_SAFE, 97 | JLOG_SAFE 98 | } jlog_safety; 99 | 100 | typedef enum { 101 | JLOG_ERR_SUCCESS = 0, 102 | JLOG_ERR_ILLEGAL_INIT, 103 | JLOG_ERR_ILLEGAL_OPEN, 104 | JLOG_ERR_OPEN, 105 | JLOG_ERR_NOTDIR, 106 | JLOG_ERR_CREATE_PATHLEN, 107 | JLOG_ERR_CREATE_EXISTS, 108 | JLOG_ERR_CREATE_MKDIR, 109 | JLOG_ERR_CREATE_META, 110 | JLOG_ERR_CREATE_PRE_COMMIT, 111 | JLOG_ERR_LOCK, 112 | JLOG_ERR_IDX_OPEN, 113 | JLOG_ERR_IDX_SEEK, 114 | JLOG_ERR_IDX_CORRUPT, 115 | JLOG_ERR_IDX_WRITE, 116 | JLOG_ERR_IDX_READ, 117 | JLOG_ERR_FILE_OPEN, 118 | JLOG_ERR_FILE_SEEK, 119 | JLOG_ERR_FILE_CORRUPT, 120 | JLOG_ERR_FILE_READ, 121 | JLOG_ERR_FILE_WRITE, 122 | JLOG_ERR_META_OPEN, 123 | JLOG_ERR_PRE_COMMIT_OPEN, 124 | JLOG_ERR_ILLEGAL_WRITE, 125 | JLOG_ERR_ILLEGAL_CHECKPOINT, 126 | JLOG_ERR_INVALID_SUBSCRIBER, 127 | JLOG_ERR_ILLEGAL_LOGID, 128 | JLOG_ERR_SUBSCRIBER_EXISTS, 129 | JLOG_ERR_CHECKPOINT, 130 | JLOG_ERR_NOT_SUPPORTED, 131 | JLOG_ERR_CLOSE_LOGID, 132 | } jlog_err; 133 | 134 | typedef enum { 135 | JLOG_COMPRESSION_NULL = 0, 136 | JLOG_COMPRESSION_LZ4 = 0x01 137 | } jlog_compression_provider_choice; 138 | 139 | typedef enum { 140 | JLOG_READ_METHOD_MMAP = 0, 141 | JLOG_READ_METHOD_PREAD 142 | } jlog_read_method_type; 143 | 144 | typedef void (*jlog_error_func) (void *ctx, const char *msg, ...); 145 | 146 | JLOG_API(jlog_ctx *) jlog_new(const char *path); 147 | JLOG_API(const char *) jlog_err_string(int); 148 | JLOG_API(void) jlog_set_error_func(jlog_ctx *ctx, jlog_error_func Func, void *ptr); 149 | JLOG_API(size_t) jlog_raw_size(jlog_ctx *ctx); 150 | JLOG_API(int) jlog_ctx_init(jlog_ctx *ctx); 151 | JLOG_API(int) jlog_get_checkpoint(jlog_ctx *ctx, const char *s, jlog_id *id); 152 | JLOG_API(int) jlog_ctx_list_subscribers_dispose(jlog_ctx *ctx, char **subs); 153 | JLOG_API(int) jlog_ctx_list_subscribers(jlog_ctx *ctx, char ***subs); 154 | 155 | JLOG_API(int) jlog_ctx_err(jlog_ctx *ctx); 156 | JLOG_API(const char *) jlog_ctx_err_string(jlog_ctx *ctx); 157 | JLOG_API(int) jlog_ctx_errno(jlog_ctx *ctx); 158 | JLOG_API(int) jlog_ctx_open_writer(jlog_ctx *ctx); 159 | JLOG_API(int) jlog_ctx_open_reader(jlog_ctx *ctx, const char *subscriber); 160 | JLOG_API(int) jlog_ctx_close(jlog_ctx *ctx); 161 | 162 | JLOG_API(int) jlog_ctx_alter_mode(jlog_ctx *ctx, int mode); 163 | JLOG_API(int) jlog_ctx_alter_journal_size(jlog_ctx *ctx, size_t size); 164 | JLOG_API(int) jlog_ctx_repair(jlog_ctx *ctx, int aggressive); 165 | JLOG_API(int) jlog_ctx_alter_safety(jlog_ctx *ctx, jlog_safety safety); 166 | JLOG_API(int) jlog_ctx_alter_read_method(jlog_ctx *ctx, jlog_read_method_type method); 167 | 168 | /** 169 | * Control whether this jlog process should use multi-process safe file locks when performing 170 | * reads or writes. If you do not intend to use your jlog from multiple processes, you can 171 | * call this function with a zero for the `mproc` argument. Multi-process safety defaults to being 172 | * on. 173 | * 174 | * \sa jlog_ctx_set_pre_commit_buffer_size 175 | */ 176 | JLOG_API(int) jlog_ctx_set_multi_process(jlog_ctx *ctx, uint8_t mproc); 177 | 178 | /** 179 | * must be called after jlog_new and before the 'open' functions 180 | * defaults to using JLOG_COMPRESSION_LZ4 181 | */ 182 | JLOG_API(int) jlog_ctx_set_use_compression(jlog_ctx *ctx, uint8_t use); 183 | 184 | /** 185 | * Switch to another compression provider. This has no effect on an already created jlog 186 | * as you cannot change compression after a jlog is inited. If you want to change 187 | * the compression provider you must call this before `jlog_ctx_init`, ala: 188 | * 189 | * ```ctx = jlog_new(path); 190 | * jlog_ctx_set_use_compression(ctx, 1); 191 | * jlog_ctx_set_compression_provider(ctx, JLOG_COMPRESSION_LZ4); 192 | * jlog_ctx_alter_journal_size(ctx, 1024000); 193 | * jlog_ctx_set_pre_commit_buffer_size(ctx, 1024 * 1024); 194 | * if(jlog_ctx_init(ctx) != 0) { 195 | * ...``` 196 | * 197 | */ 198 | JLOG_API(int) jlog_ctx_set_compression_provider(jlog_ctx *ctx, jlog_compression_provider_choice provider); 199 | 200 | /** 201 | * Turn on the use of a pre-commit buffer. This will gain you increased throughput through reduction of 202 | * `pwrite/v` syscalls. Note however, care must be taken. This is only safe for single writer 203 | * setups. Merely setting multi-process to on does not protect the pre-commit space from being 204 | * corrupted by another writing process. It is safe to use the pre-commit buffer if you have multiple 205 | * reader processes and a single writer process, or if you read and write from within the same process. 206 | * 207 | * If you intend to use multiple writing processes, you need to set the pre-commit buffer size to 208 | * zero (the default for safety). 209 | * 210 | * There is a tradeoff here between throughput for jlog and read side latency. 211 | * Because reads only happen on materialized rows (rows stored in actual jlog files), a large 212 | * pre_commit buffer size will delay the materialization of the log entries in the actual 213 | * storage files and therefore delay the read side. 214 | * 215 | * You should set this appropriately for your write throughput and read latency requirements 216 | * based on the rate you expect to be writing things to the log and the size of the average 217 | * logged item. 218 | * 219 | * This must be called before `jlog_ctx_open_writer` 220 | * 221 | */ 222 | JLOG_API(int) jlog_ctx_set_pre_commit_buffer_size(jlog_ctx *ctx, size_t s); 223 | 224 | /** 225 | * Provided to deal with read side latency problem. If you intend to have a larg-ish pre-commit 226 | * buffer to have high throughput but have variability in your throughput there are times when 227 | * the rows won't be committed for the read side to see. This call is provided to flush 228 | * the pre-commit buffer whenever you want. Normally you would wire this up to a timer event. 229 | */ 230 | JLOG_API(int) jlog_ctx_flush_pre_commit_buffer(jlog_ctx *ctx); 231 | 232 | JLOG_API(int) jlog_ctx_add_subscriber(jlog_ctx *ctx, const char *subscriber, 233 | jlog_position whence); 234 | JLOG_API(int) jlog_ctx_add_subscriber_copy_checkpoint(jlog_ctx *ctx, 235 | const char *new_subscriber, 236 | const char *old_subscriber); 237 | JLOG_API(int) jlog_ctx_set_subscriber_checkpoint(jlog_ctx *ctx, const char *subscriber, 238 | const jlog_id *checkpoint); 239 | JLOG_API(int) jlog_ctx_remove_subscriber(jlog_ctx *ctx, const char *subscriber); 240 | 241 | JLOG_API(int) jlog_ctx_write(jlog_ctx *ctx, const void *message, size_t mess_len); 242 | JLOG_API(int) jlog_ctx_write_message(jlog_ctx *ctx, jlog_message *msg, struct timeval *when); 243 | JLOG_API(int) jlog_ctx_read_interval(jlog_ctx *ctx, 244 | jlog_id *first_mess, jlog_id *last_mess); 245 | JLOG_API(int) jlog_ctx_read_message(jlog_ctx *ctx, const jlog_id *, jlog_message *); 246 | JLOG_API(int) jlog_ctx_bulk_read_messages(jlog_ctx *ctx, const jlog_id *, const int, jlog_message *); 247 | JLOG_API(int) jlog_ctx_read_checkpoint(jlog_ctx *ctx, const jlog_id *checkpoint); 248 | JLOG_API(int) jlog_snprint_logid(char *buff, int n, const jlog_id *checkpoint); 249 | 250 | JLOG_API(int) jlog_pending_readers(jlog_ctx *ctx, u_int32_t log, u_int32_t *earliest_ptr); 251 | JLOG_API(int) __jlog_pending_readers(jlog_ctx *ctx, u_int32_t log); 252 | JLOG_API(int) jlog_ctx_first_log_id(jlog_ctx *ctx, jlog_id *id); 253 | JLOG_API(int) jlog_ctx_last_log_id(jlog_ctx *ctx, jlog_id *id); 254 | JLOG_API(int) jlog_ctx_last_storage_log(jlog_ctx *ctx, uint32_t *logid); 255 | JLOG_API(int) jlog_ctx_advance_id(jlog_ctx *ctx, jlog_id *cur, 256 | jlog_id *start, jlog_id *finish); 257 | JLOG_API(int) jlog_clean(const char *path); 258 | 259 | #endif 260 | -------------------------------------------------------------------------------- /jlog_change_endian.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | use strict; 3 | use warnings; 4 | use Fcntl; 5 | use File::Path; 6 | 7 | my $src = shift; 8 | 9 | my $fromle = undef; 10 | my $jlog_endian = shift; 11 | if (defined $jlog_endian) { 12 | if ($jlog_endian =~ /^(tolittle|tole)$/) { 13 | $fromle = 0; 14 | } elsif ($jlog_endian =~ /^(tobig|tobe)$/) { 15 | $fromle = 1; 16 | } 17 | } 18 | 19 | my $dst = shift; 20 | 21 | if (!defined $src or !defined $fromle or !defined $dst) { 22 | print "Usage: $0 \n"; 23 | exit 1; 24 | } 25 | 26 | sub unpack_32 { 27 | if ($fromle) { 28 | return unpack('V1', $_[0]); 29 | } else { 30 | return unpack('N1', $_[0]); 31 | } 32 | } 33 | sub repack_32 { 34 | if ($fromle) { 35 | return pack('N1', $_[0]); 36 | } else { 37 | return pack('V1', $_[0]); 38 | } 39 | } 40 | # we have to use 2 numbers to represent a 64-bit value so this works on 32-bit 41 | sub unpack_64 { 42 | if ($fromle) { 43 | return unpack('V1', substr($_[0], 4, 4)), unpack('V1', substr($_[0], 0, 4)); 44 | } else { 45 | return unpack('N1', substr($_[0], 0, 4)), unpack('N1', substr($_[0], 4, 4)); 46 | } 47 | } 48 | sub repack_64 { 49 | if ($fromle) { 50 | return pack('N2', $_[0], $_[1]); 51 | } else { 52 | return pack('V2', $_[1], $_[0]); 53 | } 54 | } 55 | 56 | mkdir($dst) or die "could not mkdir $dst: $!"; 57 | $SIG{__DIE__} = sub { rmtree $dst; }; 58 | 59 | opendir(DIR, $src) or die "could not opendir $src: $!"; 60 | my $files = [ readdir(DIR) ]; 61 | closedir DIR; 62 | 63 | my $metastore = grep /^metastore$/, @$files; 64 | $files = [ grep !/^metastore$/, @$files ]; 65 | my $checkpoints = [ grep /^cp[.]/, @$files ]; 66 | $files = [ grep !/^cp[.]/, @$files ]; 67 | my $segments = [ sort { hex $a <=> hex $b } grep /^[0-9A-Fa-f]{8}$/, @$files ]; 68 | $files = [ grep !/^[0-9A-Fa-f]{8}$/, @$files ]; 69 | my $indexes = [ grep /^[0-9A-Fa-f]{8}.idx$/, @$files ]; 70 | $files = [ grep !/^[0-9A-Fa-f]{8}.idx$/, @$files ]; 71 | 72 | if (!$metastore) { 73 | die "no metastore found\n"; 74 | } 75 | 76 | $files = [ grep !/^[.]{1,2}$/, @$files ]; 77 | foreach (@$files) { 78 | print "unexpected file found: $_ (skipping)\n"; 79 | } 80 | undef $files; 81 | 82 | if ((stat "$src/metastore")[7] != 12) { 83 | die "metastore has invalid size\n"; 84 | } 85 | my ($current_segment, $unit_limit, $safety); 86 | sysopen(META, "$src/metastore", O_RDONLY) 87 | or die "could not sysopen $src/metastore: $!"; 88 | my $data; 89 | sysread(META, $data, 4) == 4 or die "metastore read error: $!"; 90 | $current_segment = unpack_32($data); 91 | sysread(META, $data, 4) == 4 or die "metastore read error: $!"; 92 | $unit_limit = unpack_32($data); 93 | sysread(META, $data, 4) == 4 or die "metastore read error: $!"; 94 | $safety = unpack_32($data); 95 | close META; 96 | 97 | sysopen(META, "$dst/metastore", O_WRONLY|O_CREAT) 98 | or die "could not sysopen $dst/metastore: $!"; 99 | $data = repack_32($current_segment); 100 | syswrite(META, $data, 4) == 4 or die "metastore write error: $!"; 101 | $data = repack_32($unit_limit); 102 | syswrite(META, $data, 4) == 4 or die "metastore write error: $!"; 103 | $data = repack_32($safety); 104 | syswrite(META, $data, 4) == 4 or die "metastore write error: $!"; 105 | close META; 106 | 107 | if (!scalar @$checkpoints) { 108 | die "no checkpoints\n"; 109 | } 110 | my $oldest_cp_segment = 0xffffffff; 111 | my $cpbyname = {}; 112 | foreach my $cp (@$checkpoints) { 113 | my $cpname = $cp; 114 | $cpname =~ s/^cp[.]//; 115 | $cpname = pack('H*', $cpname); 116 | if ((stat "$src/$cp")[7] != 8) { 117 | die "checkpoint $cpname has invalid size\n"; 118 | } 119 | sysopen(CP, "$src/$cp", O_RDONLY) or die "could not sysopen $src/$cp: $!"; 120 | sysread(CP, $data, 4) == 4 or die "checkpoint $cpname: read error: $!"; 121 | my $segment = unpack_32($data); 122 | sysread(CP, $data, 4) == 4 or die "checkpoint $cpname: read error: $!"; 123 | my $marker = unpack_32($data); 124 | close CP; 125 | if ($segment > $current_segment) { 126 | die "checkpoint $cpname points to segment newer than current segment\n"; 127 | } 128 | sysopen(CP, "$dst/$cp", O_WRONLY|O_CREAT) or die "could not sysopen $dst/$cp: $!"; 129 | $data = repack_32($segment); 130 | syswrite(CP, $data, 4) == 4 or die "checkpoint $cpname: write error: $!"; 131 | $data = repack_32($marker); 132 | syswrite(CP, $data, 4) == 4 or die "checkpoint $cpname: write error: $!"; 133 | close CP; 134 | $oldest_cp_segment = $segment if ($segment < $oldest_cp_segment); 135 | $segment = sprintf "%08x", $segment; 136 | $cpbyname->{$cpname} = { 137 | segment => $segment, 138 | marker => $marker, 139 | }; 140 | } 141 | 142 | my $lastnum = $oldest_cp_segment; 143 | foreach my $seg (@$segments) { 144 | my $num = hex $seg; 145 | if ($num < $oldest_cp_segment) { 146 | die "segment $seg is older than any checkpoint\n"; 147 | } 148 | if ($num > $current_segment) { 149 | die "segment $seg is newer than the current segment\n"; 150 | } 151 | if ($num > $lastnum + 1) { 152 | if ($num > $lastnum + 2) { 153 | printf "segments %08x though %08x missing\n", $lastnum + 1, $num - 1; 154 | } else { 155 | printf "segment %08x missing\n", $lastnum + 1; 156 | } 157 | } 158 | if ($num > $lastnum) { 159 | $lastnum = $num; 160 | } 161 | my ($idx) = grep /$seg[.]idx/i, @$indexes; 162 | $indexes = [ grep !/$seg[.]idx/i, @$indexes ]; 163 | $seg = [ $seg, $idx ]; 164 | } 165 | if ($current_segment > $lastnum + 1) { 166 | if ($current_segment > $lastnum + 2) { 167 | printf "segments %08x though %08x missing\n", 168 | $lastnum + 1, $current_segment - 1; 169 | } else { 170 | printf "segment %08x missing\n", $lastnum + 1; 171 | } 172 | } 173 | 174 | foreach my $idx (@$indexes) { 175 | print "index $idx doesn't correspond to any segment (skipping)\n"; 176 | } 177 | 178 | foreach my $segdata (@$segments) { 179 | my ($seg, $idx) = @$segdata; 180 | my $last_marker = 0; 181 | my $warned_toobig = 0; 182 | my $data_off = 0; 183 | my $idx_off = 0; 184 | sysopen(SEG, "$src/$seg", O_RDONLY) or die "could not sysopen $src/$seg: $!"; 185 | my $data_len = (stat SEG)[7]; 186 | sysopen(WSEG, "$dst/$seg", O_WRONLY|O_CREAT) or die "could not sysopen $dst/$seg: $!"; 187 | my $idx_len; 188 | if ($idx) { 189 | sysopen(IDX, "$src/$idx", O_RDONLY) or die "could not sysopen $src/$idx: $!"; 190 | $idx_len = (stat IDX)[7]; 191 | sysopen(WIDX, "$dst/$idx", O_WRONLY|O_CREAT) or die "could not sysopen $dst/$idx: $!"; 192 | } 193 | 194 | while ($data_off < $data_len) { 195 | if (!$warned_toobig and ($data_off > $unit_limit)) { 196 | print "segment $seg has message offset larger than unit limit\n"; 197 | $warned_toobig = 1; 198 | } 199 | if ($data_off + 16 > $data_len) { 200 | die "segment $seg offset $data_off: not enough room for message header\n"; 201 | } 202 | sysread(SEG, $data, 16) == 16 203 | or die "segment $seg offset $data_off: read error: $!"; 204 | my $reserved = unpack_32(substr $data, 0, 4); 205 | my $tv_sec = unpack_32(substr $data, 4, 4); 206 | my $tv_usec = unpack_32(substr $data, 8, 4); 207 | my $mlen = unpack_32(substr $data, 12, 4); 208 | if ($reserved) { 209 | die "segment $seg offset $data_off: reserved field not 0\n"; 210 | } 211 | if ($data_off + $mlen > $data_len) { 212 | die "segment $seg offset $data_off: not enough room for message body\n"; 213 | } 214 | $data = repack_32($reserved) . repack_32($tv_sec) . 215 | repack_32($tv_usec) . repack_32($mlen); 216 | syswrite(WSEG, $data, 16) == 16 217 | or die "segment $seg offset $data_off: write error: $!"; 218 | sysread(SEG, $data, $mlen) == $mlen 219 | or die "segment $seg offset $data_off + 16: read error: $!"; 220 | syswrite(WSEG, $data, $mlen) == $mlen 221 | or die "segment $seg offset $data_off + 16: write error: $!"; 222 | $last_marker++; 223 | 224 | if ($idx) { 225 | if ($idx_off == $idx_len) { 226 | if ($current_segment > hex $seg) { 227 | print "index $idx is incomplete (not an error)\n"; 228 | } 229 | close IDX; 230 | close WIDX; 231 | undef $idx; 232 | } elsif ($idx_off + 8 > $idx_len) { 233 | die "index $idx offset $idx_off: no room for next offset\n"; 234 | } else { 235 | sysread(IDX, $data, 8) == 8 236 | or die "index $idx offset $idx_off: read error: $!"; 237 | my ($offh, $offl) = unpack_64($data); 238 | if ($offh != 0 or $offl != $data_off) { 239 | die "index $idx offset $idx_off: index points to wrong offset\n"; 240 | } else { 241 | $data = repack_64($offh, $offl); 242 | syswrite(WIDX, $data, 8) == 8 243 | or die "index $idx offset $idx_off: write error: $!"; 244 | $idx_off += 8; 245 | } 246 | } 247 | } 248 | 249 | $data_off += 16 + $mlen; 250 | } 251 | $segdata->[2] = $last_marker; 252 | 253 | close SEG; 254 | close WSEG; 255 | if ($idx) { 256 | if ($idx_off == $idx_len) { 257 | if (hex $seg < $current_segment) { 258 | print "index $idx not current or closed\n"; 259 | } 260 | } elsif ($idx_off + 8 > $idx_len) { 261 | die "index $idx offset $idx_off: no room for closing index\n"; 262 | } elsif ($idx_off + 8 < $idx_len) { 263 | die "index $idx offset $idx_off: index too long\n"; 264 | } else { 265 | sysread(IDX, $data, 8) == 8 266 | or die "index $idx offset $idx_off: read error: $!"; 267 | my ($offh, $offl) = unpack_64($data); 268 | if ($offh != 0 or $offl != 0) { 269 | die "index $idx offset $idx_off: closing offset not 0\n"; 270 | } 271 | $data = repack_64($offh, $offl); 272 | syswrite(WIDX, $data, 8) == 8 273 | or die "index $idx offset $idx_off: write error: $!"; 274 | } 275 | close IDX; 276 | close WIDX; 277 | } 278 | } 279 | 280 | foreach my $cp (keys %$cpbyname) { 281 | if ($cpbyname->{$cp}{segment} ne '00000000' or 282 | $cpbyname->{$cp}{marker} != 0) 283 | { 284 | my ($segdata) = grep { $_->[0] eq $cpbyname->{$cp}{segment} } @$segments; 285 | if (!defined $segdata) { 286 | if (hex $cpbyname->{$cp}{segment} != $current_segment) { 287 | die "checkpoint $cp points to nonexistent segment\n"; 288 | } 289 | } elsif ($cpbyname->{$cp}{marker} > $segdata->[2]) { 290 | die "checkpoint $cp points past the end of segment\n"; 291 | } 292 | } 293 | } 294 | -------------------------------------------------------------------------------- /jlog_compress.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016, Circonus, Inc. All rights reserved. 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions are 6 | * met: 7 | * 8 | * * Redistributions of source code must retain the above copyright 9 | * notice, this list of conditions and the following disclaimer. 10 | * * Redistributions in binary form must reproduce the above 11 | * copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided 13 | * with the distribution. 14 | * * Neither the name Circonus, Inc. nor the names 15 | * of its contributors may be used to endorse or promote products 16 | * derived from this software without specific prior written 17 | * permission. 18 | * 19 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | */ 31 | 32 | 33 | #include 34 | #include "jlog_config.h" 35 | #include "jlog_compress.h" 36 | #include "jlog_private.h" 37 | #include "fassert.h" 38 | 39 | #include "jlog_null_compression_provider.h" 40 | #include "jlog_lz4_compression_provider.h" 41 | 42 | static struct jlog_compression_provider *provider = &jlog_null_compression_provider; 43 | 44 | #define unlikely(x) __builtin_expect(!!(x), 0) 45 | 46 | int 47 | jlog_set_compression_provider(const jlog_compression_provider_choice jcp) 48 | { 49 | switch(jcp) { 50 | case JLOG_COMPRESSION_NULL: 51 | provider = &jlog_null_compression_provider; 52 | break; 53 | case JLOG_COMPRESSION_LZ4: 54 | { 55 | #ifdef HAVE_LZ4_H 56 | provider = &jlog_lz4_compression_provider; 57 | #else 58 | fprintf(stderr, "lz4 not detected on system, cannot set"); 59 | return -1; 60 | #endif 61 | break; 62 | } 63 | }; 64 | return 0; 65 | } 66 | 67 | int 68 | jlog_compress(const char *source, const size_t source_bytes, char **dest, size_t *dest_bytes) 69 | { 70 | FASSERT(NULL, dest != NULL, "jlog_compress: dest pointer is NULL"); 71 | size_t required = provider->compress_bound(source_bytes); 72 | 73 | if (*dest_bytes < required) { 74 | /* incoming buffer not large enough, must allocate */ 75 | *dest = malloc(required); 76 | FASSERT(NULL, *dest != NULL, "jlog_compress: malloc failed"); 77 | } 78 | 79 | int rv = provider->compress(source, *dest, source_bytes, required); 80 | if (rv > 0) { 81 | #ifdef DEBUG 82 | fprintf(stderr, "Compressed %d bytes into %d bytes\n", source_bytes, rv); 83 | #endif 84 | *dest_bytes = rv; 85 | return 0; 86 | } 87 | return rv; 88 | } 89 | 90 | int 91 | jlog_decompress(const char *source, const size_t source_bytes, char *dest, size_t dest_bytes) 92 | { 93 | if (unlikely(dest == NULL)) { 94 | return -1; 95 | } 96 | 97 | int rv = provider->decompress(source, dest, source_bytes, dest_bytes); 98 | if (rv >= 0) { 99 | #ifdef DEBUG 100 | fprintf(stderr, "Compressed %d bytes into %d bytes\n", source_bytes, rv); 101 | #endif 102 | return 0; 103 | } 104 | return rv; 105 | } 106 | -------------------------------------------------------------------------------- /jlog_compress.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016, Circonus, Inc. All rights reserved. 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions are 6 | * met: 7 | * 8 | * * Redistributions of source code must retain the above copyright 9 | * notice, this list of conditions and the following disclaimer. 10 | * * Redistributions in binary form must reproduce the above 11 | * copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided 13 | * with the distribution. 14 | * * Neither the name Circonus, Inc. nor the names 15 | * of its contributors may be used to endorse or promote products 16 | * derived from this software without specific prior written 17 | * permission. 18 | * 19 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | */ 31 | 32 | #ifndef _JLOG_COMPRESS_H 33 | #define _JLOG_COMPRESS_H 34 | 35 | #include "jlog.h" 36 | #include "jlog_config.h" 37 | 38 | struct jlog_compression_provider { 39 | /** 40 | * providers that require initialization should do so here 41 | */ 42 | void (*init)(); 43 | 44 | /** 45 | * returns the required size of the destination buffer to compress into when dealing with a source size 46 | */ 47 | size_t (*compress_bound)(const int source_size); 48 | 49 | /** 50 | * returns the number of bytes written into dest or zero on error 51 | */ 52 | int (*compress)(const char *source, char *dest, int sourceSize, int max_dest_size); 53 | 54 | /** 55 | * returns the number of bytes decompressed into dest buffer or < 0 on error 56 | */ 57 | int (*decompress)(const char *source, char *dest, int compressed_size, int max_decompressed_size); 58 | }; 59 | 60 | 61 | /** 62 | * set the provider to the chosen type. will return 0 on success, or < 0 on error 63 | */ 64 | int jlog_set_compression_provider(const jlog_compression_provider_choice jcp); 65 | 66 | /** 67 | * will allocate into 'dest' the required size based on source_bytes. It's up to caller to free dest. 68 | */ 69 | int jlog_compress(const char *source, const size_t source_bytes, char **dest, size_t *dest_bytes); 70 | 71 | /** 72 | * reverse the compression 73 | */ 74 | int jlog_decompress(const char *source, const size_t source_bytes, char *dest, size_t dest_bytes); 75 | 76 | 77 | #endif 78 | -------------------------------------------------------------------------------- /jlog_config.h.in: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2005-2008, Message Systems, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are 7 | * met: 8 | * 9 | * * Redistributions of source code must retain the above copyright 10 | * notice, this list of conditions and the following disclaimer. 11 | * * Redistributions in binary form must reproduce the above 12 | * copyright notice, this list of conditions and the following 13 | * disclaimer in the documentation and/or other materials provided 14 | * with the distribution. 15 | * * Neither the name Message Systems, Inc. nor the names 16 | * of its contributors may be used to endorse or promote products 17 | * derived from this software without specific prior written 18 | * permission. 19 | * 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | */ 32 | 33 | #ifndef __JLOG_CONFIG_H 34 | #define __JLOG_CONFIG_H 35 | 36 | /* define inline unless that is what the compiler already calls it. */ 37 | #undef inline 38 | 39 | #undef HAVE_FCNTL_H 40 | #undef HAVE_SYS_TYPES_H 41 | #undef HAVE_LIBGEN_H 42 | #undef HAVE_DIRENT_H 43 | #undef HAVE_ERRNO_H 44 | #undef HAVE_STRING_H 45 | #undef HAVE_STDLIB_H 46 | #undef HAVE_STDINT_H 47 | #undef HAVE_LZ4_H 48 | #undef HAVE_UNISTD_H 49 | #undef HAVE_SYS_PARAM_H 50 | #undef HAVE_SYS_MMAN_H 51 | #undef HAVE_TIME_H 52 | #undef HAVE_SYS_TIME_H 53 | #undef HAVE_SYS_STAT_H 54 | #undef HAVE_SYS_UIO_H 55 | #undef HAVE_PWRITEV 56 | #undef HAVE_INT64_T 57 | #undef HAVE_INTXX_T 58 | #undef HAVE_LONG_LONG_INT 59 | #undef HAVE_UINTXX_T 60 | #undef HAVE_U_INT 61 | #undef HAVE_U_INT64_T 62 | #undef HAVE_U_INTXX_T 63 | #define IFS_CH '/' 64 | 65 | #ifdef HAVE_STRING_H 66 | #include 67 | #endif 68 | #ifdef HAVE_STDLIB_H 69 | #include 70 | #endif 71 | #ifdef HAVE_STDINT_H 72 | #include 73 | #endif 74 | #ifdef HAVE_SYS_PARAM_H 75 | #include 76 | #endif 77 | #ifdef HAVE_SYS_TYPES_H 78 | #include 79 | #endif 80 | #ifdef HAVE_SYS_STAT_H 81 | #include 82 | #endif 83 | #if HAVE_LIBGEN_H 84 | #include 85 | #endif 86 | 87 | 88 | 89 | /* The number of bytes in a char. */ 90 | #undef SIZEOF_CHAR 91 | 92 | /* The number of bytes in a int. */ 93 | #undef SIZEOF_INT 94 | 95 | /* The number of bytes in a size_t. */ 96 | #undef SIZEOF_SIZE_T 97 | 98 | 99 | /* The number of bytes in a long int. */ 100 | #undef SIZEOF_LONG_INT 101 | 102 | /* The number of bytes in a long long int. */ 103 | #undef SIZEOF_LONG_LONG_INT 104 | 105 | /* The number of bytes in a short int. */ 106 | #undef SIZEOF_SHORT_INT 107 | 108 | /* The number of bytes in a void *. */ 109 | #undef SIZEOF_VOID_P 110 | 111 | #ifndef HAVE_U_INT 112 | #define HAVE_U_INT 113 | typedef unsigned int u_int; 114 | #endif 115 | 116 | #undef HAVE_INTXX_T 117 | #ifndef HAVE_INTXX_T 118 | #if (SIZEOF_CHAR == 1) 119 | typedef char int8_t; 120 | #else 121 | #error "8 bit int type not found." 122 | #endif 123 | #if (SIZEOF_SHORT_INT == 2) 124 | typedef short int int16_t; 125 | #else 126 | #ifdef _CRAY 127 | typedef long int16_t; 128 | #else 129 | #warning "16 bit int type not found." 130 | #endif /* _CRAY */ 131 | #endif 132 | #if (SIZEOF_INT == 4) 133 | typedef int int32_t; 134 | #else 135 | #ifdef _CRAY 136 | typedef long int32_t; 137 | #else 138 | #error "32 bit int type not found." 139 | #endif /* _CRAY */ 140 | #endif 141 | #endif 142 | 143 | /* If sys/types.h does not supply u_intXX_t, supply them ourselves */ 144 | #ifndef HAVE_U_INTXX_T 145 | #ifdef HAVE_UINTXX_T 146 | typedef uint8_t u_int8_t; 147 | typedef uint16_t u_int16_t; 148 | typedef uint32_t u_int32_t; 149 | #define HAVE_U_INTXX_T 1 150 | #else 151 | #if (SIZEOF_CHAR == 1) 152 | typedef unsigned char u_int8_t; 153 | #else 154 | #error "8 bit int type not found." 155 | #endif 156 | #if (SIZEOF_SHORT_INT == 2) 157 | typedef unsigned short int u_int16_t; 158 | #else 159 | #ifdef _CRAY 160 | typedef unsigned long u_int16_t; 161 | #else 162 | #warning "16 bit int type not found." 163 | #endif 164 | #endif 165 | #if (SIZEOF_INT == 4) 166 | typedef unsigned int u_int32_t; 167 | #else 168 | #ifdef _CRAY 169 | typedef unsigned long u_int32_t; 170 | #else 171 | #error "32 bit int type not found." 172 | #endif 173 | #endif 174 | #endif 175 | #endif 176 | 177 | /* 64-bit types */ 178 | #ifndef HAVE_INT64_T 179 | #if (SIZEOF_LONG_INT == 8) 180 | typedef long int int64_t; 181 | #define HAVE_INT64_T 1 182 | #else 183 | #if (SIZEOF_LONG_LONG_INT == 8) 184 | typedef long long int int64_t; 185 | #define HAVE_INT64_T 1 186 | #define HAVE_LONG_LONG_INT 187 | #endif 188 | #endif 189 | #endif 190 | #ifndef HAVE_U_INT64_T 191 | #if (SIZEOF_LONG_INT == 8) 192 | typedef unsigned long int u_int64_t; 193 | #define HAVE_U_INT64_T 1 194 | #else 195 | #if (SIZEOF_LONG_LONG_INT == 8) 196 | typedef unsigned long long int u_int64_t; 197 | #define HAVE_U_INT64_T 1 198 | #endif 199 | #endif 200 | #endif 201 | 202 | #endif 203 | -------------------------------------------------------------------------------- /jlog_hash.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2005-2008, Message Systems, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are 7 | * met: 8 | * 9 | * * Redistributions of source code must retain the above copyright 10 | * notice, this list of conditions and the following disclaimer. 11 | * * Redistributions in binary form must reproduce the above 12 | * copyright notice, this list of conditions and the following 13 | * disclaimer in the documentation and/or other materials provided 14 | * with the distribution. 15 | * * Neither the name Message Systems, Inc. nor the names 16 | * of its contributors may be used to endorse or promote products 17 | * derived from this software without specific prior written 18 | * permission. 19 | * 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | */ 32 | 33 | #include "jlog_config.h" 34 | #include "jlog_hash.h" 35 | 36 | /* This is from http://burtleburtle.net/bob/hash/doobs.html */ 37 | 38 | #define JLogHASH_INITIAL_SIZE (1<<7) 39 | 40 | #define mix(a,b,c) \ 41 | { \ 42 | a -= b; a -= c; a ^= (c>>13); \ 43 | b -= c; b -= a; b ^= (a<<8); \ 44 | c -= a; c -= b; c ^= (b>>13); \ 45 | a -= b; a -= c; a ^= (c>>12); \ 46 | b -= c; b -= a; b ^= (a<<16); \ 47 | c -= a; c -= b; c ^= (b>>5); \ 48 | a -= b; a -= c; a ^= (c>>3); \ 49 | b -= c; b -= a; b ^= (a<<10); \ 50 | c -= a; c -= b; c ^= (b>>15); \ 51 | } 52 | 53 | static inline 54 | u_int32_t __hash(const char *k, u_int32_t length, u_int32_t initval) 55 | { 56 | register u_int32_t a,b,c,len; 57 | 58 | /* Set up the internal state */ 59 | len = length; 60 | a = b = 0x9e3779b9; /* the golden ratio; an arbitrary value */ 61 | c = initval; /* the previous hash value */ 62 | 63 | /*---------------------------------------- handle most of the key */ 64 | while (len >= 12) 65 | { 66 | a += (k[0] +((u_int32_t)k[1]<<8) +((u_int32_t)k[2]<<16) +((u_int32_t)k[3]<<24)); 67 | b += (k[4] +((u_int32_t)k[5]<<8) +((u_int32_t)k[6]<<16) +((u_int32_t)k[7]<<24)); 68 | c += (k[8] +((u_int32_t)k[9]<<8) +((u_int32_t)k[10]<<16)+((u_int32_t)k[11]<<24)); 69 | mix(a,b,c); 70 | k += 12; len -= 12; 71 | } 72 | 73 | /*------------------------------------- handle the last 11 bytes */ 74 | c += length; 75 | switch(len) /* all the case statements fall through */ 76 | { 77 | case 11: c+=((u_int32_t)k[10]<<24); 78 | case 10: c+=((u_int32_t)k[9]<<16); 79 | case 9 : c+=((u_int32_t)k[8]<<8); 80 | /* the first byte of c is reserved for the length */ 81 | case 8 : b+=((u_int32_t)k[7]<<24); 82 | case 7 : b+=((u_int32_t)k[6]<<16); 83 | case 6 : b+=((u_int32_t)k[5]<<8); 84 | case 5 : b+=k[4]; 85 | case 4 : a+=((u_int32_t)k[3]<<24); 86 | case 3 : a+=((u_int32_t)k[2]<<16); 87 | case 2 : a+=((u_int32_t)k[1]<<8); 88 | case 1 : a+=k[0]; 89 | /* case 0: nothing left to add */ 90 | } 91 | mix(a,b,c); 92 | /*-------------------------------------------- report the result */ 93 | return c; 94 | } 95 | 96 | u_int32_t jlog_hash__hash(const char *k, u_int32_t length, u_int32_t initval) { 97 | return __hash(k,length,initval); 98 | } 99 | 100 | void jlog_hash_init(jlog_hash_table *h) { 101 | memset(h, 0, sizeof(jlog_hash_table)); 102 | h->initval = lrand48(); 103 | h->table_size = JLogHASH_INITIAL_SIZE; 104 | h->buckets = calloc(h->table_size, sizeof(jlog_hash_bucket *)); 105 | } 106 | 107 | jlog_hash_bucket *jlog_hash__new_bucket(const char *k, int klen, void *data) { 108 | jlog_hash_bucket *b; 109 | b = calloc(1, sizeof(jlog_hash_bucket)); 110 | b->k = k; 111 | b->klen = klen; 112 | b->data = data; 113 | return b; 114 | } 115 | void jlog_hash__rebucket(jlog_hash_table *h, int newsize) { 116 | int i, newoff; 117 | jlog_hash_bucket **newbuckets, *b, *n; 118 | 119 | if (h->dont_rebucket) return; 120 | 121 | i = newsize; 122 | while(i) { 123 | if(i & 1) break; 124 | i >>= 1; 125 | } 126 | if(i & ~1) { 127 | return; 128 | } 129 | newbuckets = calloc(newsize, sizeof(jlog_hash_bucket *)); 130 | h->num_used_buckets = 0; 131 | for(i = 0; i < h->table_size; i++) { 132 | b = h->buckets[i]; 133 | while(b) { 134 | n = b->next; 135 | newoff = __hash(b->k, b->klen, h->initval) & (newsize-1); 136 | if(newbuckets[newoff] == NULL) h->num_used_buckets++; 137 | b->next = newbuckets[newoff]; 138 | newbuckets[newoff] = b; 139 | b = n; 140 | } 141 | } 142 | free(h->buckets); 143 | h->table_size = newsize; 144 | h->buckets = newbuckets; 145 | } 146 | int jlog_hash_replace(jlog_hash_table *h, const char *k, int klen, void *data, 147 | JLogHashFreeFunc keyfree, JLogHashFreeFunc datafree) { 148 | int off; 149 | int replaced = 0; 150 | jlog_hash_bucket __b, *tr, *b = &__b; 151 | 152 | if(h->table_size == 0) jlog_hash_init(h); 153 | off = __hash(k, klen, h->initval) & (h->table_size-1); 154 | __b.next = h->buckets[off]; 155 | if(!b->next) h->num_used_buckets++; 156 | while(b->next) { 157 | if(b->next->klen == klen && !memcmp(b->next->k, k, klen)) { 158 | tr = b->next; 159 | if(keyfree) keyfree((void *)tr->k); 160 | if(datafree && tr->data) datafree((void *)tr->data); 161 | b->next = tr->next; 162 | if(tr == h->buckets[off]) h->buckets[off] = tr->next; 163 | free(tr); 164 | replaced = 1; 165 | break; 166 | } else { 167 | b = b->next; 168 | } 169 | } 170 | b = jlog_hash__new_bucket(k, klen, data); 171 | b->next = h->buckets[off]; 172 | h->buckets[off] = b; 173 | if(!replaced) h->size++; 174 | 175 | if(h->size > h->table_size - (h->table_size >> 3)) { 176 | jlog_hash__rebucket(h, h->table_size << 1); 177 | } 178 | return 1; 179 | } 180 | int jlog_hash_store(jlog_hash_table *h, const char *k, int klen, void *data) { 181 | int off; 182 | jlog_hash_bucket *b; 183 | 184 | if(h->table_size == 0) jlog_hash_init(h); 185 | off = __hash(k, klen, h->initval) & (h->table_size-1); 186 | b = h->buckets[off]; 187 | if(!b) h->num_used_buckets++; 188 | while(b) { 189 | if(b->klen == klen && !memcmp(b->k, k, klen)) return 0; 190 | b = b->next; 191 | } 192 | b = jlog_hash__new_bucket(k, klen, data); 193 | b->next = h->buckets[off]; 194 | h->buckets[off] = b; 195 | h->size++; 196 | 197 | if(h->size > h->table_size - (h->table_size >> 3)) { 198 | jlog_hash__rebucket(h, h->table_size << 1); 199 | } 200 | return 1; 201 | } 202 | int jlog_hash_retrieve(jlog_hash_table *h, const char *k, int klen, void **data) { 203 | int off; 204 | jlog_hash_bucket *b; 205 | 206 | if(h->table_size == 0) jlog_hash_init(h); 207 | off = __hash(k, klen, h->initval) & (h->table_size-1); 208 | b = h->buckets[off]; 209 | while(b) { 210 | if(b->klen == klen && !memcmp(b->k, k, klen)) break; 211 | b = b->next; 212 | } 213 | if(b) { 214 | if(data) *data = b->data; 215 | return 1; 216 | } 217 | return 0; 218 | } 219 | int jlog_hash_delete(jlog_hash_table *h, const char *k, int klen, 220 | JLogHashFreeFunc keyfree, JLogHashFreeFunc datafree) { 221 | int off; 222 | jlog_hash_bucket *b, *prev = NULL; 223 | 224 | if(h->table_size == 0) jlog_hash_init(h); 225 | off = __hash(k, klen, h->initval) & (h->table_size-1); 226 | b = h->buckets[off]; 227 | while(b) { 228 | if(b->klen == klen && !memcmp(b->k, k, klen)) break; 229 | prev = b; 230 | b = b->next; 231 | } 232 | if(!b) return 0; /* No match */ 233 | if(!prev) h->buckets[off] = h->buckets[off]->next; 234 | else prev->next = b->next; 235 | if(keyfree) keyfree((void *)b->k); 236 | if(datafree && b->data) datafree(b->data); 237 | free(b); 238 | h->size--; 239 | if(h->buckets[off] == NULL) h->num_used_buckets--; 240 | if(h->table_size > JLogHASH_INITIAL_SIZE && 241 | h->size < h->table_size >> 2) 242 | jlog_hash__rebucket(h, h->table_size >> 1); 243 | return 1; 244 | } 245 | 246 | void jlog_hash_delete_all(jlog_hash_table *h, JLogHashFreeFunc keyfree, JLogHashFreeFunc datafree) { 247 | int i; 248 | jlog_hash_bucket *b, *tofree; 249 | for(i=0; itable_size; i++) { 250 | b = h->buckets[i]; 251 | while(b) { 252 | tofree = b; 253 | b = b->next; 254 | if(keyfree) keyfree((void *)tofree->k); 255 | if(datafree && tofree->data) datafree(tofree->data); 256 | free(tofree); 257 | } 258 | h->buckets[i] = NULL; 259 | } 260 | h->num_used_buckets = 0; 261 | h->size = 0; 262 | jlog_hash__rebucket(h, JLogHASH_INITIAL_SIZE); 263 | } 264 | 265 | void jlog_hash_destroy(jlog_hash_table *h, JLogHashFreeFunc keyfree, JLogHashFreeFunc datafree) { 266 | jlog_hash_delete_all(h, keyfree, datafree); 267 | if(h->buckets) free(h->buckets); 268 | } 269 | 270 | int jlog_hash_next(jlog_hash_table *h, jlog_hash_iter *iter, 271 | const char **k, int *klen, void **data) { 272 | jlog_hash_bucket *b; 273 | next_row: 274 | if(iter->p1 < 0 || iter->p1 >= h->table_size) return 0; 275 | if(iter->p2 == NULL) iter->p2 = (void *)h->buckets[iter->p1]; 276 | if(iter->p2 == NULL) { 277 | iter->p1++; 278 | goto next_row; 279 | } 280 | b = (jlog_hash_bucket *)(iter->p2); 281 | *k = b->k; *klen = b->klen; 282 | if(data) *data = b->data; 283 | b = b->next; 284 | if(!b) iter->p1++; 285 | iter->p2 = b; 286 | return 1; 287 | } 288 | 289 | int jlog_hash_firstkey(jlog_hash_table *h, const char **k, int *klen) { 290 | int i; 291 | for(i=0;itable_size;i++) { 292 | if(h->buckets[i]) { 293 | *k = h->buckets[i]->k; 294 | *klen = h->buckets[i]->klen; 295 | return 1; 296 | } 297 | } 298 | return 0; 299 | } 300 | int jlog_hash_nextkey(jlog_hash_table *h, const char **k, int *klen, const char *lk, int lklen) { 301 | int off; 302 | jlog_hash_bucket *b; 303 | 304 | if(h->table_size == 0) return 0; 305 | off = __hash(lk, lklen, h->initval) & (h->table_size-1); 306 | b = h->buckets[off]; 307 | while(b) { 308 | if(b->klen == lklen && !memcmp(b->k, lk, lklen)) break; 309 | b = b->next; 310 | } 311 | if(b) { 312 | if(b->next) { 313 | *k = b->next->k; 314 | *klen = b->next->klen; 315 | return 1; 316 | } else { 317 | off++; 318 | for(;off < h->table_size; off++) { 319 | if(h->buckets[off]) { 320 | *k = h->buckets[off]->k; 321 | *klen = h->buckets[off]->klen; 322 | return 1; 323 | } 324 | } 325 | } 326 | } 327 | return 0; 328 | } 329 | /* vim: se sw=2 ts=2 et: */ 330 | -------------------------------------------------------------------------------- /jlog_hash.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2005-2008, Message Systems, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are 7 | * met: 8 | * 9 | * * Redistributions of source code must retain the above copyright 10 | * notice, this list of conditions and the following disclaimer. 11 | * * Redistributions in binary form must reproduce the above 12 | * copyright notice, this list of conditions and the following 13 | * disclaimer in the documentation and/or other materials provided 14 | * with the distribution. 15 | * * Neither the name Message Systems, Inc. nor the names 16 | * of its contributors may be used to endorse or promote products 17 | * derived from this software without specific prior written 18 | * permission. 19 | * 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | */ 32 | 33 | #ifndef _JLOG_HASH_H 34 | #define _JLOG_HASH_H 35 | 36 | #include "jlog_config.h" 37 | 38 | typedef void (*JLogHashFreeFunc)(void *); 39 | 40 | typedef struct _jlog_hash_bucket { 41 | const char *k; 42 | int klen; 43 | void *data; 44 | struct _jlog_hash_bucket *next; 45 | } jlog_hash_bucket; 46 | 47 | typedef struct { 48 | jlog_hash_bucket **buckets; 49 | u_int32_t table_size; 50 | u_int32_t initval; 51 | u_int32_t num_used_buckets; 52 | u_int32_t size; 53 | unsigned dont_rebucket:1; 54 | unsigned _spare:31; 55 | } jlog_hash_table; 56 | 57 | #define JLOG_HASH_EMPTY { NULL, 0, 0, 0, 0, 0, 0 } 58 | 59 | typedef struct { 60 | void *p2; 61 | int p1; 62 | } jlog_hash_iter; 63 | 64 | #define JLOG_HASH_ITER_ZERO { NULL, 0 } 65 | 66 | void jlog_hash_init(jlog_hash_table *h); 67 | /* NOTE! "k" and "data" MUST NOT be transient buffers, as the hash table 68 | * implementation does not duplicate them. You provide a pair of 69 | * JLogHashFreeFunc functions to free up their storage when you call 70 | * jlog_hash_delete(), jlog_hash_delete_all() or jlog_hash_destroy(). 71 | * */ 72 | int jlog_hash_store(jlog_hash_table *h, const char *k, int klen, void *data); 73 | int jlog_hash_replace(jlog_hash_table *h, const char *k, int klen, void *data, 74 | JLogHashFreeFunc keyfree, JLogHashFreeFunc datafree); 75 | int jlog_hash_retrieve(jlog_hash_table *h, const char *k, int klen, void **data); 76 | int jlog_hash_delete(jlog_hash_table *h, const char *k, int klen, 77 | JLogHashFreeFunc keyfree, JLogHashFreeFunc datafree); 78 | void jlog_hash_delete_all(jlog_hash_table *h, JLogHashFreeFunc keyfree, 79 | JLogHashFreeFunc datafree); 80 | void jlog_hash_destroy(jlog_hash_table *h, JLogHashFreeFunc keyfree, 81 | JLogHashFreeFunc datafree); 82 | 83 | /* This is an iterator and requires the hash to not be written to during the 84 | iteration process. 85 | To use: 86 | jlog_hash_iter iter = JLOG_HASH_ITER_ZERO; 87 | 88 | const char *k; 89 | int klen; 90 | void *data; 91 | 92 | while(jlog_hash_next(h, &iter, &k, &klen, &data)) { 93 | .... use k, klen and data .... 94 | } 95 | */ 96 | int jlog_hash_next(jlog_hash_table *h, jlog_hash_iter *iter, 97 | const char **k, int *klen, void **data); 98 | int jlog_hash_firstkey(jlog_hash_table *h, const char **k, int *klen); 99 | int jlog_hash_nextkey(jlog_hash_table *h, const char **k, int *klen, const char *lk, int lklen); 100 | 101 | /* This function serves no real API use sans calculating expected buckets 102 | for keys (or extending the hash... which is unsupported) */ 103 | u_int32_t jlog_hash__hash(const char *k, u_int32_t length, u_int32_t initval); 104 | jlog_hash_bucket *jlog_hash__new_bucket(const char *k, int klen, void *data); 105 | void jlog_hash__rebucket(jlog_hash_table *h, int newsize); 106 | #endif 107 | -------------------------------------------------------------------------------- /jlog_io.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2005-2008, Message Systems, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are 7 | * met: 8 | * 9 | * * Redistributions of source code must retain the above copyright 10 | * notice, this list of conditions and the following disclaimer. 11 | * * Redistributions in binary form must reproduce the above 12 | * copyright notice, this list of conditions and the following 13 | * disclaimer in the documentation and/or other materials provided 14 | * with the distribution. 15 | * * Neither the name Message Systems, Inc. nor the names 16 | * of its contributors may be used to endorse or promote products 17 | * derived from this software without specific prior written 18 | * permission. 19 | * 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | */ 32 | 33 | /* 34 | * We want the single unix spec, so this define is needed on 35 | * the identity crisis that is Linux. pread()/pwrite() 36 | */ 37 | #if defined(linux) || defined(__linux) || defined(__linux__) 38 | #define _XOPEN_SOURCE 500 39 | #endif 40 | 41 | #include "jlog_config.h" 42 | #include "jlog_hash.h" 43 | #include "jlog_io.h" 44 | #include 45 | #include 46 | #include 47 | #include 48 | #include 49 | #include 50 | #include 51 | 52 | static pthread_mutex_t jlog_files_lock = PTHREAD_MUTEX_INITIALIZER; 53 | static jlog_hash_table jlog_files = JLOG_HASH_EMPTY; 54 | 55 | typedef struct { 56 | dev_t st_dev; 57 | ino_t st_ino; 58 | } jlog_file_id; 59 | 60 | struct _jlog_file { 61 | jlog_file_id id; 62 | int fd; 63 | int refcnt; 64 | int locked; 65 | pthread_mutex_t lock; 66 | uint8_t multi_process; 67 | }; 68 | 69 | jlog_file *jlog_file_open(const char *path, int flags, int mode, int multi_process) 70 | { 71 | struct stat sb; 72 | jlog_file_id id; 73 | jlog_file *f = NULL; 74 | union { 75 | jlog_file *f; 76 | void *vptr; 77 | } pun; 78 | int fd, realflags = O_RDWR, rv; 79 | 80 | if (flags & O_CREAT) realflags |= O_CREAT; 81 | if (flags & O_EXCL) realflags |= O_EXCL; 82 | 83 | if (pthread_mutex_lock(&jlog_files_lock) != 0) return NULL; 84 | 85 | while ((rv = stat(path, &sb)) == -1 && errno == EINTR); 86 | if (rv == 0) { 87 | if (!S_ISREG(sb.st_mode)) goto out; 88 | memset(&id, 0, sizeof(id)); 89 | id.st_dev = sb.st_dev; 90 | id.st_ino = sb.st_ino; 91 | if (jlog_hash_retrieve(&jlog_files, (void *)&id, sizeof(jlog_file_id), 92 | &pun.vptr)) 93 | { 94 | if (!(flags & O_EXCL)) { 95 | f = pun.f; 96 | f->refcnt++; 97 | } else { 98 | errno = EEXIST; 99 | } 100 | goto out; 101 | } 102 | } 103 | 104 | while ((fd = open(path, realflags, mode)) == -1 && errno == EINTR); 105 | if (fd == -1) goto out; 106 | while ((rv = fstat(fd, &sb)) == -1 && errno == EINTR); 107 | if (rv != 0) { 108 | while (close(fd) == -1 && errno == EINTR) ; 109 | goto out; 110 | } 111 | id.st_dev = sb.st_dev; 112 | id.st_ino = sb.st_ino; 113 | if (!(f = malloc(sizeof(jlog_file)))) { 114 | while (close(fd) == -1 && errno == EINTR) ; 115 | goto out; 116 | } 117 | memset(f, 0, sizeof(jlog_file)); 118 | f->id = id; 119 | f->fd = fd; 120 | f->refcnt = 1; 121 | f->locked = 0; 122 | f->multi_process = multi_process; 123 | pthread_mutex_init(&(f->lock), NULL); 124 | if (!jlog_hash_store(&jlog_files, (void *)&f->id, sizeof(jlog_file_id), f)) { 125 | while (close(f->fd) == -1 && errno == EINTR) ; 126 | free(f); 127 | f = NULL; 128 | } 129 | out: 130 | pthread_mutex_unlock(&jlog_files_lock); 131 | return f; 132 | } 133 | 134 | int jlog_file_close(jlog_file *f) 135 | { 136 | if (pthread_mutex_lock(&jlog_files_lock) != 0) return 0; 137 | if (--f->refcnt == 0) { 138 | assert(jlog_hash_delete(&jlog_files, (void *)&f->id, sizeof(jlog_file_id), 139 | NULL, NULL)); 140 | while (close(f->fd) == -1 && errno == EINTR) ; 141 | pthread_mutex_destroy(&(f->lock)); 142 | free(f); 143 | } 144 | pthread_mutex_unlock(&jlog_files_lock); 145 | return 1; 146 | } 147 | 148 | int jlog_file_lock(jlog_file *f) 149 | { 150 | struct flock fl; 151 | int frv; 152 | 153 | if (pthread_mutex_lock(&(f->lock)) != 0) return 0; 154 | 155 | if (f->multi_process != 0) { 156 | memset(&fl, 0, sizeof(fl)); 157 | fl.l_type = F_WRLCK; 158 | fl.l_whence = SEEK_SET; 159 | fl.l_start = 0; 160 | fl.l_len = 0; 161 | 162 | while ((frv = fcntl(f->fd, F_SETLKW, &fl)) == -1 && (errno == EINTR || errno == EAGAIN)) ; 163 | if (frv != 0) { 164 | int save = errno; 165 | pthread_mutex_unlock(&(f->lock)); 166 | errno = save; 167 | return 0; 168 | } 169 | } 170 | 171 | f->locked = 1; 172 | return 1; 173 | } 174 | 175 | int jlog_file_unlock(jlog_file *f) 176 | { 177 | struct flock fl; 178 | int frv; 179 | 180 | if (!f->locked) return 0; 181 | 182 | if (f->multi_process != 0) { 183 | memset(&fl, 0, sizeof(fl)); 184 | fl.l_type = F_UNLCK; 185 | fl.l_whence = SEEK_SET; 186 | fl.l_start = 0; 187 | fl.l_len = 0; 188 | 189 | while ((frv = fcntl(f->fd, F_SETLKW, &fl)) == -1 && (errno == EINTR || errno == EAGAIN)) ; 190 | if (frv != 0) return 0; 191 | f->locked = 0; 192 | } 193 | pthread_mutex_unlock(&(f->lock)); 194 | return 1; 195 | } 196 | 197 | int jlog_file_pread(jlog_file *f, void *buf, size_t nbyte, off_t offset) 198 | { 199 | while (nbyte > 0) { 200 | ssize_t rv = pread(f->fd, buf, nbyte, offset); 201 | if (rv == -1 && errno == EINTR) continue; 202 | if (rv <= 0) return 0; 203 | nbyte -= rv; 204 | offset += rv; 205 | } 206 | return 1; 207 | } 208 | 209 | int jlog_file_pwrite(jlog_file *f, const void *buf, size_t nbyte, off_t offset) 210 | { 211 | while (nbyte > 0) { 212 | ssize_t rv = pwrite(f->fd, buf, nbyte, offset); 213 | if (rv == -1 && errno == EINTR) continue; 214 | if (rv <= 0) return 0; 215 | nbyte -= rv; 216 | offset += rv; 217 | } 218 | return 1; 219 | } 220 | 221 | int jlog_file_pwritev_verify_return_value(jlog_file *f, const struct iovec *vecs, int iov_count, off_t offset, 222 | size_t expected_length) { 223 | ssize_t rv = 0; 224 | while (1) { 225 | #ifdef HAVE_PWRITEV 226 | rv = pwritev(f->fd, vecs, iov_count, offset); 227 | #else 228 | if(lseek(f->fd, offset, SEEK_SET) < 0) return 0; 229 | rv = writev(f->fd, vecs, iov_count); 230 | #endif 231 | if (rv == -1 && errno == EINTR) continue; 232 | if (rv <= 0) return 0; 233 | if (rv > 0 && rv != expected_length) continue; 234 | break; 235 | } 236 | return 1; 237 | } 238 | 239 | int jlog_file_pwritev(jlog_file *f, const struct iovec *vecs, int iov_count, off_t offset) 240 | { 241 | int i; 242 | size_t expected_length = 0; 243 | for (i=0; i < iov_count; i++) { 244 | expected_length += vecs[i].iov_len; 245 | } 246 | if (expected_length) { 247 | return jlog_file_pwritev_verify_return_value(f, vecs, iov_count, offset, expected_length); 248 | } 249 | return 0; 250 | } 251 | 252 | int jlog_file_sync(jlog_file *f) 253 | { 254 | int rv; 255 | 256 | #ifdef HAVE_FDATASYNC 257 | while((rv = fdatasync(f->fd)) == -1 && errno == EINTR) ; 258 | #else 259 | while((rv = fsync(f->fd)) == -1 && errno == EINTR) ; 260 | #endif 261 | if (rv == 0) return 1; 262 | return 0; 263 | } 264 | 265 | int jlog_file_map_rdwr(jlog_file *f, void **base, size_t *len) 266 | { 267 | struct stat sb; 268 | void *my_map; 269 | int flags = 0; 270 | 271 | #ifdef MAP_SHARED 272 | flags = MAP_SHARED; 273 | #endif 274 | if (fstat(f->fd, &sb) != 0) return 0; 275 | my_map = mmap(NULL, sb.st_size, PROT_READ|PROT_WRITE, flags, f->fd, 0); 276 | if (my_map == MAP_FAILED) return 0; 277 | *base = my_map; 278 | *len = sb.st_size; 279 | return 1; 280 | } 281 | 282 | int jlog_file_map_read(jlog_file *f, void **base, size_t *len) 283 | { 284 | struct stat sb; 285 | void *my_map; 286 | int flags = 0; 287 | 288 | #ifdef MAP_SHARED 289 | flags = MAP_SHARED; 290 | #endif 291 | if (fstat(f->fd, &sb) != 0) return 0; 292 | my_map = mmap(NULL, sb.st_size, PROT_READ, flags, f->fd, 0); 293 | if (my_map == MAP_FAILED) return 0; 294 | *base = my_map; 295 | *len = sb.st_size; 296 | return 1; 297 | } 298 | 299 | off_t jlog_file_size(jlog_file *f) 300 | { 301 | struct stat sb; 302 | int rv; 303 | while ((rv = fstat(f->fd, &sb)) == -1 && errno == EINTR) ; 304 | if (rv != 0) return -1; 305 | return sb.st_size; 306 | } 307 | 308 | int jlog_file_truncate(jlog_file *f, off_t len) 309 | { 310 | int rv; 311 | while ((rv = ftruncate(f->fd, len)) == -1 && errno == EINTR) ; 312 | if (rv == 0) return 1; 313 | return 0; 314 | } 315 | 316 | /* vim:se ts=2 sw=2 et: */ 317 | -------------------------------------------------------------------------------- /jlog_io.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2005-2008, Message Systems, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are 7 | * met: 8 | * 9 | * * Redistributions of source code must retain the above copyright 10 | * notice, this list of conditions and the following disclaimer. 11 | * * Redistributions in binary form must reproduce the above 12 | * copyright notice, this list of conditions and the following 13 | * disclaimer in the documentation and/or other materials provided 14 | * with the distribution. 15 | * * Neither the name Message Systems, Inc. nor the names 16 | * of its contributors may be used to endorse or promote products 17 | * derived from this software without specific prior written 18 | * permission. 19 | * 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | */ 32 | 33 | #ifndef _JLOG_IO_H_ 34 | #define _JLOG_IO_H_ 35 | 36 | #ifdef __cplusplus 37 | extern "C" { 38 | #endif 39 | 40 | #include "jlog_config.h" 41 | #if HAVE_SYS_UIO_H 42 | #include 43 | #endif 44 | 45 | typedef struct _jlog_file jlog_file; 46 | 47 | /** 48 | * opens a jlog_file 49 | * 50 | * since a jlog_file is a shared handle potentially used by many threads, 51 | * the underlying open mode is always O_RDWR; only the O_CREAT and O_EXCL 52 | * flags are honored 53 | * @return pointer to jlog_file on success, NULL on failure 54 | * @internal 55 | */ 56 | jlog_file *jlog_file_open(const char *path, int flags, int mode, int multi_process); 57 | 58 | /** 59 | * closes a jlog_file 60 | * @return 1 on success, 0 on failure 61 | * @internal 62 | */ 63 | int jlog_file_close(jlog_file *f); 64 | 65 | /** 66 | * exclusively locks a jlog_file against other processes and threads 67 | * @return 1 on success, 0 on failure 68 | * @internal 69 | */ 70 | int jlog_file_lock(jlog_file *f); 71 | 72 | /** 73 | * unlocks a jlog_file 74 | * @return 1 on success, 0 on failure 75 | * @internal 76 | */ 77 | int jlog_file_unlock(jlog_file *f); 78 | 79 | /** 80 | * preads from a jlog_file, retries EINTR 81 | * @return 1 if the read was fully satisfied, 0 otherwise 82 | * @internal 83 | */ 84 | int jlog_file_pread(jlog_file *f, void *buf, size_t nbyte, off_t offset); 85 | 86 | /** 87 | * pwrites to a jlog_file, retries EINTR 88 | * @return 1 if the write was fully satisfied, 0 otherwise 89 | * @internal 90 | */ 91 | int jlog_file_pwrite(jlog_file *f, const void *buf, size_t nbyte, off_t offset); 92 | 93 | /** 94 | * pwritevs to a jlog_file, retries EINTR. takes the expected bytes written as an argument. 95 | * @return 1 if the write was fully satisfied, 0 otherwise 96 | * @internal 97 | */ 98 | int jlog_file_pwritev_verify_return_value(jlog_file *f, const struct iovec *vecs, int iov_count, off_t offset, 99 | size_t expected_length); 100 | 101 | /** 102 | * pwritevs to a jlog_file, retries EINTR. calculates the expected bytes written internally. 103 | * @return 1 if the write was fully satisfied, 0 otherwise 104 | * @internal 105 | */ 106 | int jlog_file_pwritev(jlog_file *f, const struct iovec *vec, int iov_count, off_t offset); 107 | 108 | /** 109 | * calls fdatasync (if avalable) or fsync on the underlying filehandle 110 | * @return 1 on success, 0 on failure 111 | * @internal 112 | */ 113 | int jlog_file_sync(jlog_file *f); 114 | 115 | /** 116 | * maps the entirety of a jlog_file into memory for reading and writing 117 | * @param[in] f the jlog_file on which you are operating 118 | * @param[out] base is set to the base of the mapped region 119 | * @param[out] len is set to the length of the mapped region 120 | * @return 1 on success, 0 on failure 121 | * @internal 122 | */ 123 | int jlog_file_map_rdwr(jlog_file *f, void **base, size_t *len); 124 | 125 | /** 126 | * maps the entirety of a jlog_file into memory for reading 127 | * @param[out] map is set to the base of the mapped region 128 | * @param[out] len is set to the length of the mapped region 129 | * @return 1 on success, 0 on failure 130 | * @internal 131 | */ 132 | int jlog_file_map_read(jlog_file *f, void **base, size_t *len); 133 | 134 | /** 135 | * gives the size of a jlog_file 136 | * @return size of file on success, -1 on failure 137 | * @internal 138 | */ 139 | off_t jlog_file_size(jlog_file *f); 140 | 141 | /** 142 | * truncates a jlog_file, retries EINTR 143 | * @return 1 on success, 0 on failure 144 | * @internal 145 | */ 146 | int jlog_file_truncate(jlog_file *f, off_t len); 147 | 148 | #ifdef __cplusplus 149 | } /* Close scope of 'extern "C"' declaration which encloses file. */ 150 | #endif 151 | 152 | #endif /* _JLOG_IO_H_ */ 153 | /* vim:se ts=2 sw=2 et: */ 154 | -------------------------------------------------------------------------------- /jlog_lz4_compression_provider.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016, Circonus, Inc. All rights reserved. 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions are 6 | * met: 7 | * 8 | * * Redistributions of source code must retain the above copyright 9 | * notice, this list of conditions and the following disclaimer. 10 | * * Redistributions in binary form must reproduce the above 11 | * copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided 13 | * with the distribution. 14 | * * Neither the name Circonus, Inc. nor the names 15 | * of its contributors may be used to endorse or promote products 16 | * derived from this software without specific prior written 17 | * permission. 18 | * 19 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | */ 31 | 32 | 33 | #ifndef JLOG_LZ4_COMPRESSION_PROVIDER_H 34 | #define JLOG_LZ4_COMPRESSION_PROVIDER_H 35 | 36 | #ifdef HAVE_LZ4_H 37 | #include 38 | 39 | void jlog_lz4_compression_provider_init() { 40 | } 41 | 42 | static inline size_t 43 | jlog_lz4_compression_provider_compress_bound(const int source_size) 44 | { 45 | return LZ4_compressBound(source_size); 46 | } 47 | 48 | static inline int 49 | jlog_lz4_compression_provider_compress(const char *source, char *dest, int source_size, int max_dest_size) 50 | { 51 | return LZ4_compress_default(source, dest, source_size, max_dest_size); 52 | } 53 | 54 | static inline int 55 | jlog_lz4_compression_provider_decompress(const char *source, char *dest, int compressed_size, int max_decompressed_size) 56 | { 57 | return LZ4_decompress_safe(source, dest, compressed_size, max_decompressed_size); 58 | } 59 | 60 | static struct jlog_compression_provider jlog_lz4_compression_provider = { 61 | .init = jlog_lz4_compression_provider_init, 62 | .compress_bound = jlog_lz4_compression_provider_compress_bound, 63 | .compress = jlog_lz4_compression_provider_compress, 64 | .decompress = jlog_lz4_compression_provider_decompress 65 | }; 66 | 67 | #endif 68 | #endif 69 | -------------------------------------------------------------------------------- /jlog_null_compression_provider.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016, Circonus, Inc. All rights reserved. 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions are 6 | * met: 7 | * 8 | * * Redistributions of source code must retain the above copyright 9 | * notice, this list of conditions and the following disclaimer. 10 | * * Redistributions in binary form must reproduce the above 11 | * copyright notice, this list of conditions and the following 12 | * disclaimer in the documentation and/or other materials provided 13 | * with the distribution. 14 | * * Neither the name Circonus, Inc. nor the names 15 | * of its contributors may be used to endorse or promote products 16 | * derived from this software without specific prior written 17 | * permission. 18 | * 19 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | */ 31 | 32 | 33 | #ifndef JLOG_NULL_COMPRESSION_PROVIDER_H 34 | #define JLOG_NULL_COMPRESSION_PROVIDER_H 35 | 36 | #include "jlog_compress.h" 37 | 38 | #ifndef min 39 | #define min(a,b) ((a) < (b) ? (a) : (b)) 40 | #endif 41 | 42 | void jlog_null_compression_provider_init() { 43 | } 44 | 45 | size_t jlog_null_compression_provider_compress_bound(int source_size) 46 | { 47 | return source_size; 48 | } 49 | 50 | int jlog_null_compression_provider_compress(const char *source, char *dest, int source_size, int max_dest_size) 51 | { 52 | memcpy(dest, source, min(max_dest_size, source_size)); 53 | return min(max_dest_size, source_size); 54 | } 55 | 56 | int jlog_null_compression_provider_decompress(const char *source, char *dest, int compressed_size, int max_decompressed_size) 57 | { 58 | memcpy(dest, source, min(compressed_size, max_decompressed_size)); 59 | return 0; 60 | } 61 | 62 | static struct jlog_compression_provider jlog_null_compression_provider = { 63 | .init = jlog_null_compression_provider_init, 64 | .compress_bound = jlog_null_compression_provider_compress_bound, 65 | .compress = jlog_null_compression_provider_compress, 66 | .decompress = jlog_null_compression_provider_decompress 67 | }; 68 | 69 | #endif 70 | -------------------------------------------------------------------------------- /jlog_private.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2005-2008, Message Systems, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are 7 | * met: 8 | * 9 | * * Redistributions of source code must retain the above copyright 10 | * notice, this list of conditions and the following disclaimer. 11 | * * Redistributions in binary form must reproduce the above 12 | * copyright notice, this list of conditions and the following 13 | * disclaimer in the documentation and/or other materials provided 14 | * with the distribution. 15 | * * Neither the name Message Systems, Inc. nor the names 16 | * of its contributors may be used to endorse or promote products 17 | * derived from this software without specific prior written 18 | * permission. 19 | * 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | */ 32 | 33 | #ifndef _JLOG_PRIVATE_H 34 | #define _JLOG_PRIVATE_H 35 | /* vim:se ts=2 sw=2 et: */ 36 | 37 | #include "jlog_config.h" 38 | #include "jlog.h" 39 | #include "jlog_io.h" 40 | 41 | #define DEFAULT_FILE_MODE 0640 42 | #define DEFAULT_UNIT_LIMIT (4*1024*1024) 43 | /* 4 Megabytes */ 44 | #define DEFAULT_HDR_MAGIC 0x663A7318 45 | #define DEFAULT_HDR_MAGIC_COMPRESSION 0x15106A00 46 | #define DEFAULT_SAFETY JLOG_ALMOST_SAFE 47 | #define DEFAULT_READ_MESSAGE_TYPE JLOG_READ_METHOD_MMAP 48 | #define INDEX_EXT ".idx" 49 | #define MAXLOGPATHLEN (MAXPATHLEN - (8+sizeof(INDEX_EXT))) 50 | 51 | static const char __jlog_hexchars[] = "0123456789abcdef"; 52 | 53 | typedef enum { 54 | JLOG_NEW = 0, 55 | JLOG_INIT, 56 | JLOG_READ, 57 | JLOG_APPEND, 58 | JLOG_INVALID 59 | } jlog_mode; 60 | 61 | struct _jlog_meta_info { 62 | u_int32_t storage_log; 63 | u_int32_t unit_limit; 64 | u_int32_t safety; 65 | u_int32_t hdr_magic; 66 | }; 67 | 68 | struct _jlog_ctx { 69 | struct _jlog_meta_info *meta; 70 | pthread_mutex_t write_lock; 71 | jlog_read_method_type read_method; 72 | int meta_is_mapped; 73 | int pre_commit_is_mapped; 74 | uint8_t multi_process; 75 | uint8_t pre_commit_buffer_size_specified; 76 | void *pre_commit_buffer; 77 | void *pre_commit_pos; 78 | void *pre_commit_end; 79 | size_t pre_commit_buffer_len; 80 | size_t desired_pre_commit_buffer_len; 81 | uint32_t *pre_commit_pointer; 82 | struct _jlog_meta_info pre_init; /* only used before we're opened */ 83 | jlog_mode context_mode; 84 | char *path; 85 | int file_mode; 86 | u_int32_t current_log; 87 | jlog_file *data; 88 | jlog_file *index; 89 | jlog_file *checkpoint; 90 | jlog_file *metastore; 91 | jlog_file *pre_commit; 92 | void *mmap_base; 93 | size_t mmap_len; 94 | size_t data_file_size; 95 | uint8_t reader_is_initialized; 96 | void *compressed_data_buffer; 97 | size_t compressed_data_buffer_len; 98 | char *subscriber_name; 99 | int last_error; 100 | int last_errno; 101 | jlog_error_func error_func; 102 | void *error_ctx; 103 | 104 | /** 105 | * Store the last read message in the case of use_compression == 1. 106 | * There is an expectation from jlog user's that they are handed 107 | * mmap'd memory directly and that they don't have to free it, so we 108 | * have to maintain a block of memory where we can decompress messages 109 | * and hand them back. The contract being that everytime jlog_ctx_read_message 110 | * is called, we overwrite this memory with the new message 111 | */ 112 | size_t mess_data_size; 113 | char *mess_data; 114 | }; 115 | 116 | /* macros */ 117 | 118 | #define STRLOGID(s, logid) do { \ 119 | int __i; \ 120 | for(__i=0;__i<8;__i++) \ 121 | (s)[__i] = __jlog_hexchars[((logid) >> (32 - ((__i+1)*4))) & 0xf]; \ 122 | (s)[__i] = '\0'; \ 123 | } while(0) 124 | 125 | #define STRSETDATAFILE(ctx, file, log) do { \ 126 | int __len; \ 127 | __len = strlen((ctx)->path); \ 128 | memcpy((file), (ctx)->path, __len); \ 129 | (file)[__len] = IFS_CH; \ 130 | STRLOGID((file)+(__len+1), log); \ 131 | } while(0) 132 | 133 | #define SYS_FAIL_EX(a, dowarn) do { \ 134 | if (ctx) { \ 135 | ctx->last_error = (a); \ 136 | ctx->last_errno = errno; \ 137 | if(ctx->error_func && dowarn) { \ 138 | ctx->error_func(ctx->error_ctx, \ 139 | "JLOG-%d error: %d (%s) errno: %d (%s)\n", __LINE__, \ 140 | ctx->last_error, jlog_ctx_err_string(ctx), \ 141 | ctx->last_errno, strerror(ctx->last_errno)); \ 142 | } \ 143 | } \ 144 | goto finish; \ 145 | } while(0) 146 | 147 | #define SYS_FAIL(a) SYS_FAIL_EX(a, 1) 148 | 149 | /** 150 | * repairs a damaged datafile 151 | * @return 0 OK, >0 number of damaged segments removed, -1 repair failed 152 | * @internal 153 | */ 154 | JLOG_API(int) jlog_repair_datafile(jlog_ctx *ctx, u_int32_t log); 155 | /** 156 | * prints detailed info about the log segment to stderr 157 | * @return 0 OK, 1 segment damaged, -1 other error 158 | * @internal 159 | */ 160 | JLOG_API(int) jlog_inspect_datafile(jlog_ctx *ctx, u_int32_t log, int verbose); 161 | /** 162 | * fetches the last marker in the index and the closedness thereof 163 | * @return 0 OK, -1 error 164 | * @internal 165 | */ 166 | JLOG_API(int) jlog_idx_details(jlog_ctx *ctx, u_int32_t log, 167 | u_int32_t *marker, int *closed); 168 | 169 | 170 | #ifdef _WIN32 171 | #include "ec_win32.h" 172 | #endif 173 | 174 | #endif 175 | -------------------------------------------------------------------------------- /jlog_sanity_check.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | use strict; 3 | use warnings; 4 | use Fcntl qw/SEEK_CUR SEEK_SET O_RDONLY O_WRONLY/; 5 | sub systell { sysseek($_[0], 0, SEEK_CUR) } 6 | 7 | my $le = 1; 8 | my $jlog = shift; 9 | my $jlog_endian = shift; 10 | if (defined $jlog_endian) { 11 | if ($jlog_endian =~ /^(little|le)$/) { 12 | $le = 1; 13 | } elsif ($jlog_endian =~ /^(big|be)$/) { 14 | $le = 0; 15 | } 16 | } 17 | 18 | if (!defined $le or !defined $jlog) { 19 | print "Usage: $0 \n"; 20 | exit 1; 21 | } 22 | 23 | sub unpack_32 { 24 | if ($le) { 25 | return unpack('V1', $_[0]); 26 | } else { 27 | return unpack('N1', $_[0]); 28 | } 29 | } 30 | # we have to use 2 numbers to represent a 64-bit value so this works on 32-bit 31 | sub unpack_64 { 32 | if ($le) { 33 | return unpack('V1', substr($_[0], 4, 4)), unpack('V1', substr($_[0], 0, 4)); 34 | } else { 35 | return unpack('N1', substr($_[0], 0, 4)), unpack('N1', substr($_[0], 4, 4)); 36 | } 37 | } 38 | 39 | opendir(DIR, $jlog) or die "could not opendir $jlog: $!"; 40 | my $files = [ readdir(DIR) ]; 41 | closedir DIR; 42 | 43 | my $metastore = grep /^metastore$/, @$files; 44 | $files = [ grep !/^metastore$/, @$files ]; 45 | my $checkpoints = [ grep /^cp[.]/, @$files ]; 46 | $files = [ grep !/^cp[.]/, @$files ]; 47 | my $segments = [ sort { hex $a <=> hex $b } grep /^[0-9A-Fa-f]{8}$/, @$files ]; 48 | $files = [ grep !/^[0-9A-Fa-f]{8}$/, @$files ]; 49 | my $indexes = [ grep /^[0-9A-Fa-f]{8}.idx$/, @$files ]; 50 | $files = [ grep !/^[0-9A-Fa-f]{8}.idx$/, @$files ]; 51 | 52 | if (!$metastore) { 53 | die "no metastore found\n"; 54 | } 55 | 56 | $files = [ grep !/^[.]{1,2}$/, @$files ]; 57 | foreach (@$files) { 58 | print "unexpected file found: $_ (skipping)\n"; 59 | } 60 | undef $files; 61 | 62 | if ((stat "$jlog/metastore")[7] != 16) { 63 | die "metastore has invalid size\n"; 64 | } 65 | my ($current_segment, $unit_limit, $safety, $hdr_magic); 66 | sysopen(META, "$jlog/metastore", O_RDONLY) 67 | or die "could not sysopen $jlog/metastore: $!"; 68 | my $data; 69 | sysread(META, $data, 4) == 4 or die "metastore read error: $!"; 70 | $current_segment = unpack_32($data); 71 | sysread(META, $data, 4) == 4 or die "metastore read error: $!"; 72 | $unit_limit = unpack_32($data); 73 | sysread(META, $data, 4) == 4 or die "metastore read error: $!"; 74 | $safety = unpack_32($data); 75 | sysread(META, $data, 4) == 4 or die "metastore read error: $!"; 76 | $hdr_magic = unpack_32($data); 77 | close META; 78 | 79 | my $oldest_cp_segment = 0xffffffff; 80 | my $cpbyname = {}; 81 | foreach my $cp (@$checkpoints) { 82 | my $cpname = $cp; 83 | $cpname =~ s/^cp[.]//; 84 | $cpname = pack('H*', $cpname); 85 | if ((stat "$jlog/$cp")[7] != 8) { 86 | print "checkpoint $cpname has invalid size\n"; 87 | next; 88 | } 89 | sysopen(CP, "$jlog/$cp", O_RDONLY) or die "could not sysopen $jlog/$cp: $!"; 90 | sysread(CP, $data, 4) == 4 or die "checkpoint $cpname: read error: $!"; 91 | my $segment = unpack_32($data); 92 | sysread(CP, $data, 4) == 4 or die "checkpoint $cpname: read error: $!"; 93 | my $marker = unpack_32($data); 94 | close CP; 95 | if ($segment > $current_segment) { 96 | print "checkpoint $cpname points to segment newer than current segment\n"; 97 | next; 98 | } 99 | $oldest_cp_segment = $segment if ($segment < $oldest_cp_segment); 100 | $segment = sprintf "%08x", $segment; 101 | $cpbyname->{$cpname} = { 102 | segment => $segment, 103 | marker => $marker, 104 | }; 105 | } 106 | if (!scalar keys %$cpbyname) { 107 | warn "no valid checkpoints\n"; 108 | } 109 | 110 | my $lastnum = $oldest_cp_segment; 111 | foreach my $seg (@$segments) { 112 | my $num = hex $seg; 113 | if ($num < $oldest_cp_segment) { 114 | print "segment $seg is older than any checkpoint\n"; 115 | } 116 | if ($num > $current_segment) { 117 | print "segment $seg is newer than the current segment\n"; 118 | } 119 | if ($num > $lastnum + 1) { 120 | if ($num > $lastnum + 2) { 121 | printf "segments %08x though %08x missing\n", $lastnum + 1, $num - 1; 122 | } else { 123 | printf "segment %08x missing\n", $lastnum + 1; 124 | } 125 | } 126 | if ($num > $lastnum) { 127 | $lastnum = $num; 128 | } 129 | my ($idx) = grep /$seg[.]idx/i, @$indexes; 130 | $indexes = [ grep !/$seg[.]idx/i, @$indexes ]; 131 | $seg = [ $seg, $idx ]; 132 | } 133 | if ($current_segment > $lastnum + 1) { 134 | if ($current_segment > $lastnum + 2) { 135 | printf "segments %08x though %08x missing\n", 136 | $lastnum + 1, $current_segment - 1; 137 | } else { 138 | printf "segment %08x missing\n", $lastnum + 1; 139 | } 140 | } 141 | 142 | foreach my $idx (@$indexes) { 143 | print "index $idx doesn't correspond to any segment (skipping)\n"; 144 | } 145 | 146 | foreach my $segdata (@$segments) { 147 | my ($seg, $idx) = @$segdata; 148 | my $last_marker = 0; 149 | my $last_tv_sec = 0; 150 | my $last_tv_usec = 0; 151 | my $warned_timewarp = 0; 152 | my $warned_toobig = 0; 153 | my $data_off = 0; 154 | my $idx_off = 0; 155 | sysopen(SEG, "$jlog/$seg", O_RDONLY) or die "could not sysopen $jlog/$seg: $!"; 156 | my $data_len = (stat SEG)[7]; 157 | my $idx_len; 158 | if ($idx) { 159 | sysopen(IDX, "$jlog/$idx", O_RDONLY) or die "could not sysopen $jlog/$idx: $!"; 160 | $idx_len = (stat IDX)[7]; 161 | } 162 | 163 | my $nrec = 0; 164 | my @fixers = (); 165 | while ($data_off < $data_len) { 166 | if (!$warned_toobig and ($data_off > $unit_limit)) { 167 | print "segment $seg has message offset larger than unit limit\n"; 168 | $warned_toobig = 1; 169 | } 170 | if ($data_off + 16 > $data_len) { 171 | print "segment $seg offset $data_off: not enough room for message header\n"; 172 | last; 173 | } 174 | my $offset = systell(*SEG); 175 | sysread(SEG, $data, 16) == 16 176 | or die "segment $seg offset $data_off: read error: $!"; 177 | my $reserved = unpack_32(substr $data, 0, 4); 178 | my $tv_sec = unpack_32(substr $data, 4, 4); 179 | my $tv_usec = unpack_32(substr $data, 8, 4); 180 | my $mlen = unpack_32(substr $data, 12, 4); 181 | if ($reserved != $hdr_magic) { 182 | printf "segment $seg offset $data_off: reserved field (%08x != %08x)\n", 183 | $reserved, $hdr_magic; 184 | push @fixers, $offset if ($reserved == 0); 185 | } 186 | if (!$warned_timewarp) { 187 | if ($tv_sec < $last_tv_sec or 188 | ($tv_sec == $last_tv_sec and $tv_usec < $last_tv_usec)) 189 | { 190 | print "segment $seg offset $data_off: time goes backwards\n"; 191 | $warned_timewarp = 1; 192 | } else { 193 | $last_tv_sec = $tv_sec; 194 | $last_tv_usec = $tv_usec; 195 | } 196 | } 197 | if ($data_off + $mlen > $data_len) { 198 | print "segment $seg offset $data_off: not enough room for message body\n"; 199 | last; 200 | } 201 | sysread(SEG, $data, $mlen) == $mlen 202 | or die "segment $seg offset $data_off + 16: read error: $!"; 203 | $last_marker++; 204 | $nrec++; 205 | if ($idx) { 206 | if ($idx_off == $idx_len) { 207 | if ($current_segment > hex $seg) { 208 | print "index $idx is incomplete (not an error)\n"; 209 | } 210 | close IDX; 211 | undef $idx; 212 | } elsif ($idx_off + 8 > $idx_len) { 213 | print "index $idx offset $idx_off: no room for next offset\n"; 214 | close IDX; 215 | undef $idx; 216 | } else { 217 | sysread(IDX, $data, 8) == 8 218 | or die "index $idx offset $idx_off: read error: $!"; 219 | my ($offh, $offl) = unpack_64($data); 220 | if ($offh != 0 or $offl != $data_off) { 221 | print "index $idx offset $idx_off: index points to wrong offset\n"; 222 | close IDX; 223 | undef $idx; 224 | } else { 225 | $idx_off += 8; 226 | } 227 | } 228 | } 229 | 230 | $data_off += 16 + $mlen; 231 | } 232 | if ($data_off == $data_len) { 233 | $segdata->[2] = $last_marker; 234 | foreach my $offset (@fixers) { 235 | printf "writing new hdr_magic at off: %d\n", $offset; 236 | sysopen(FIX, "$jlog/$seg", O_WRONLY) or die "could not sysopen $jlog/$seg: $!"; 237 | sysseek(FIX, $offset, SEEK_SET); 238 | my $hdr_blob = pack('V1', $hdr_magic); 239 | die unless length($hdr_blob) == 4; 240 | syswrite(FIX, pack('V1', $hdr_magic), 4); 241 | close FIX; 242 | } 243 | } else { 244 | close IDX; 245 | undef $idx; 246 | } 247 | 248 | close SEG; 249 | if ($idx) { 250 | if ($idx_off == $idx_len) { 251 | if (hex $seg < $current_segment) { 252 | print "index $idx not current or closed\n"; 253 | } 254 | } elsif ($idx_off + 8 > $idx_len) { 255 | print "index $idx offset $idx_off: no room for closing index\n"; 256 | } elsif ($idx_off + 8 < $idx_len) { 257 | print "index $idx offset $idx_off: index too long\n"; 258 | } else { 259 | sysread(IDX, $data, 8) == 8 260 | or die "index $idx offset $idx_off: read error: $!"; 261 | my ($offh, $offl) = unpack_64($data); 262 | if ($offh != 0 or $offl != 0) { 263 | print "index $idx offset $idx_off: closing offset not 0\n"; 264 | } 265 | } 266 | close IDX; 267 | } 268 | printf "segment $seg has %d records\n", $nrec; 269 | } 270 | 271 | foreach my $cp (keys %$cpbyname) { 272 | if ($cpbyname->{$cp}{segment} ne '00000000' or 273 | $cpbyname->{$cp}{marker} != 0) 274 | { 275 | my ($segdata) = grep { $_->[0] eq $cpbyname->{$cp}{segment} } @$segments; 276 | if (!defined $segdata) { 277 | if (hex $cpbyname->{$cp}{segment} != $current_segment) { 278 | printf "checkpoint $cp points to nonexistent segment\n"; 279 | } 280 | } elsif (!defined $segdata->[2]) { 281 | print "checkpoint $cp points to a damaged segment\n"; 282 | } elsif ($cpbyname->{$cp}{marker} > $segdata->[2]) { 283 | print "checkpoint $cp points past the end of segment\n"; 284 | } 285 | } 286 | } 287 | -------------------------------------------------------------------------------- /jlogtail.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include "jlog.h" 6 | 7 | #define SLEEP_10MS_IN_US 10000 8 | #define SLEEP_1S_IN_US 1000000 9 | jlog_ctx* ctx; 10 | const char *lf = "\n"; 11 | char subscriber[32] = "jlog-tail"; 12 | 13 | int main(int argc, char** argv) { 14 | const char* path; 15 | jlog_id begin, end; 16 | int count, two_times_a_charm = 0; 17 | int sleeptime = SLEEP_10MS_IN_US; 18 | 19 | if(argc != 2) { 20 | fprintf(stderr, "usage: %s /path/to/jlog\n", argv[0]); 21 | exit(1); 22 | } 23 | path = argv[1]; 24 | 25 | snprintf(subscriber, sizeof(subscriber), "jlog-tail-%d", (int)getpid()); 26 | ctx = jlog_new(path); 27 | jlog_ctx_add_subscriber(ctx, subscriber, JLOG_END); 28 | 29 | if (jlog_ctx_open_reader(ctx, subscriber) != 0) { 30 | fprintf(stderr, "jlog_ctx_open_reader failed: %d %s\n", jlog_ctx_err(ctx), jlog_ctx_err_string(ctx)); 31 | exit(1); 32 | } 33 | 34 | jlog_ctx_remove_subscriber(ctx, subscriber); 35 | 36 | while(1) { 37 | count = jlog_ctx_read_interval(ctx, &begin, &end); 38 | if (count > 0) { 39 | int i; 40 | jlog_message m; 41 | 42 | two_times_a_charm = 0; 43 | for (i = 0; i < count; i++, JLOG_ID_ADVANCE(&begin)) { 44 | end = begin; 45 | 46 | if (jlog_ctx_read_message(ctx, &begin, &m) == 0) { 47 | if(m.mess_len > 0) { 48 | const char *use_lf = lf; 49 | if(((char *)m.mess)[m.mess_len-1] == '\n') use_lf = ""; 50 | printf("%.*s%s", m.mess_len, (char*)m.mess, use_lf); 51 | fflush(stdout); 52 | } 53 | else { 54 | printf("... empty message ...\n"); 55 | } 56 | } else { 57 | fprintf(stderr, "jlog_ctx_read_message failed: %d %s\n", jlog_ctx_err(ctx), jlog_ctx_err_string(ctx)); 58 | } 59 | } 60 | 61 | // checkpoint (commit) our read: 62 | jlog_ctx_read_checkpoint(ctx, &end); 63 | } 64 | if(two_times_a_charm > 1) { 65 | sleeptime *= 2; 66 | if(sleeptime > SLEEP_1S_IN_US) sleeptime = SLEEP_1S_IN_US; 67 | usleep(sleeptime); 68 | } 69 | else sleeptime = SLEEP_10MS_IN_US; 70 | two_times_a_charm++; 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /jthreadtest.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2005-2008, Message Systems, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are 7 | * met: 8 | * 9 | * * Redistributions of source code must retain the above copyright 10 | * notice, this list of conditions and the following disclaimer. 11 | * * Redistributions in binary form must reproduce the above 12 | * copyright notice, this list of conditions and the following 13 | * disclaimer in the documentation and/or other materials provided 14 | * with the distribution. 15 | * * Neither the name Message Systems, Inc. nor the names 16 | * of its contributors may be used to endorse or promote products 17 | * derived from this software without specific prior written 18 | * permission. 19 | * 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | */ 32 | 33 | #include 34 | #include 35 | #include 36 | #include 37 | 38 | #include "jlog_config.h" 39 | #include "jlog.h" 40 | #include "jlog_private.h" 41 | 42 | #define MASTER "master" 43 | #define LOGNAME "/tmp/jtest.foo" 44 | 45 | int writer_done = 0; 46 | int only_read = 0; 47 | int only_write = 0; 48 | int error = 0; 49 | 50 | static void _croak(int lineno) 51 | { 52 | fprintf(stderr, "croaked at line %d\n", lineno); 53 | error=1; 54 | } 55 | #define croak() _croak(__LINE__) 56 | 57 | void jcreate(jlog_safety s) { 58 | jlog_ctx *ctx; 59 | const char *label = NULL; 60 | 61 | switch (s) { 62 | case JLOG_ALMOST_SAFE: label = "almost safe"; break; 63 | case JLOG_UNSAFE: label = "unsafe"; break; 64 | case JLOG_SAFE: label = "safe"; break; 65 | } 66 | fprintf(stderr, "jcreate %s in %s mode\n", LOGNAME, label); 67 | 68 | ctx = jlog_new(LOGNAME); 69 | jlog_ctx_alter_journal_size(ctx, 102400); 70 | jlog_ctx_alter_safety(ctx, s); 71 | if(jlog_ctx_init(ctx) != 0) { 72 | fprintf(stderr, "jlog_ctx_init failed: %d %s\n", jlog_ctx_err(ctx), jlog_ctx_err_string(ctx)); 73 | if(jlog_ctx_err(ctx) != JLOG_ERR_CREATE_EXISTS) exit(0); 74 | } else { 75 | jlog_ctx_add_subscriber(ctx, MASTER, JLOG_BEGIN); 76 | } 77 | jlog_ctx_close(ctx); 78 | } 79 | 80 | void *writer(void *unused) { 81 | jlog_ctx *ctx; 82 | int i; 83 | char foo[72]; 84 | ctx = jlog_new(LOGNAME); 85 | memset(foo, 'X', sizeof(foo)-1); 86 | foo[sizeof(foo)-1] = '\0'; 87 | if(jlog_ctx_open_writer(ctx) != 0) { 88 | fprintf(stderr, "jlog_ctx_open_writer failed: %d %s\n", jlog_ctx_err(ctx), jlog_ctx_err_string(ctx)); 89 | croak(); 90 | } 91 | 92 | #ifdef TEST_UNIT_LIMIT 93 | newsize = 1024*1024 + (((intptr_t)unused) % (1024 * 1024)); 94 | fprintf(stderr, "writer setting new unit_limit to %d\n", (int)newsize); 95 | jlog_ctx_alter_journal_size(ctx, newsize); 96 | #endif 97 | for(i=0;i<10000;i++) { 98 | int rv; 99 | rv = jlog_ctx_write(ctx, foo, strlen(foo)); 100 | if(rv != 0) { 101 | fprintf(stderr, "jlog_ctx_write_message failed: %d %s\n", jlog_ctx_err(ctx), jlog_ctx_err_string(ctx)); 102 | /* abort(); */ 103 | } 104 | usleep(100); 105 | } 106 | fprintf(stderr, "writer thinks unit_limit is %d\n", ctx->meta->unit_limit); 107 | jlog_ctx_close(ctx); 108 | writer_done = 1; 109 | return 0; 110 | } 111 | 112 | void *reader(void *unused) { 113 | jlog_ctx *ctx; 114 | char subname[32]; 115 | int tcount = 0, fcount = 0; 116 | int prev_err = 0; 117 | int subno = (int)(uintptr_t)unused; 118 | snprintf(subname, sizeof(subname), "sub-%02d", subno); 119 | reader_retry: 120 | ctx = jlog_new(LOGNAME); 121 | if(jlog_ctx_open_reader(ctx, subname) != 0) { 122 | if(prev_err == 0) { 123 | prev_err = jlog_ctx_err(ctx); 124 | jlog_ctx_close(ctx); 125 | ctx = jlog_new(LOGNAME); 126 | if(prev_err == JLOG_ERR_INVALID_SUBSCRIBER) { 127 | fprintf(stderr, "[%02d] invalid subscriber, init...\n", subno); 128 | if(jlog_ctx_open_writer(ctx) != 0) { 129 | fprintf(stderr, "[%02d] jlog_ctx_open_writer failed: %d %s\n", subno, jlog_ctx_err(ctx), jlog_ctx_err_string(ctx)); 130 | } else { 131 | if(jlog_ctx_add_subscriber(ctx, subname, JLOG_BEGIN) != 0) { 132 | fprintf(stderr, "[%02d] jlog_ctx_add_subscriber failed: %d %s\n", subno, jlog_ctx_err(ctx), jlog_ctx_err_string(ctx)); 133 | } else { 134 | jlog_ctx_close(ctx); 135 | goto reader_retry; 136 | } 137 | } 138 | } 139 | } 140 | fprintf(stderr, "[%02d] jlog_ctx_open_reader failed: %d %s\n", subno, jlog_ctx_err(ctx), jlog_ctx_err_string(ctx)); 141 | croak(); 142 | } 143 | fprintf(stderr, "[%02d] reader started\n", subno); 144 | while(1) { 145 | char begins[20], ends[20]; 146 | jlog_id begin, end; 147 | int count; 148 | jlog_message message; 149 | if((count = jlog_ctx_read_interval(ctx, &begin, &end)) == -1) { 150 | fprintf(stderr, "jlog_ctx_read_interval failed: %d %s\n", jlog_ctx_err(ctx), jlog_ctx_err_string(ctx)); 151 | croak(); 152 | } 153 | jlog_snprint_logid(begins, sizeof(begins), &begin); 154 | jlog_snprint_logid(ends, sizeof(ends), &end); 155 | if(count > 0) { 156 | int i; 157 | //fprintf(stderr, "[%02d] reader (%s, %s] count: %d\n", subno, begins, ends, count); 158 | for(i=0; i 3) { 232 | usage(); 233 | } 234 | 235 | jcreate(safety); 236 | 237 | if(toremove) { 238 | jlog_ctx *ctx; 239 | ctx = jlog_new(LOGNAME); 240 | if(jlog_ctx_open_writer(ctx) != 0) { 241 | fprintf(stderr, "jlog_ctx_open_writer failed: %d %s\n", jlog_ctx_err(ctx), jlog_ctx_err_string(ctx)); 242 | croak(); 243 | } 244 | jlog_ctx_remove_subscriber(ctx, argv[2]); 245 | jlog_ctx_close(ctx); 246 | exit(0); 247 | } 248 | if(!only_write) { 249 | for(i=0; i 7 | # Created: 1993-05-16 8 | # Public domain. 9 | # 10 | # This file is maintained in Automake, please report 11 | # bugs to or send patches to 12 | # . 13 | 14 | errstatus=0 15 | dirmode= 16 | 17 | usage="\ 18 | Usage: mkinstalldirs [-h] [--help] [--version] [-m MODE] DIR ... 19 | 20 | Create each directory DIR (with mode MODE, if specified), including all 21 | leading file name components. 22 | 23 | Report bugs to ." 24 | 25 | # process command line arguments 26 | while test $# -gt 0 ; do 27 | case $1 in 28 | -h | --help | --h*) # -h for help 29 | echo "$usage" 30 | exit $? 31 | ;; 32 | -m) # -m PERM arg 33 | shift 34 | test $# -eq 0 && { echo "$usage" 1>&2; exit 1; } 35 | dirmode=$1 36 | shift 37 | ;; 38 | --version) 39 | echo "$0 $scriptversion" 40 | exit $? 41 | ;; 42 | --) # stop option processing 43 | shift 44 | break 45 | ;; 46 | -*) # unknown option 47 | echo "$usage" 1>&2 48 | exit 1 49 | ;; 50 | *) # first non-opt arg 51 | break 52 | ;; 53 | esac 54 | done 55 | 56 | for file 57 | do 58 | if test -d "$file"; then 59 | shift 60 | else 61 | break 62 | fi 63 | done 64 | 65 | case $# in 66 | 0) exit 0 ;; 67 | esac 68 | 69 | # Solaris 8's mkdir -p isn't thread-safe. If you mkdir -p a/b and 70 | # mkdir -p a/c at the same time, both will detect that a is missing, 71 | # one will create a, then the other will try to create a and die with 72 | # a "File exists" error. This is a problem when calling mkinstalldirs 73 | # from a parallel make. We use --version in the probe to restrict 74 | # ourselves to GNU mkdir, which is thread-safe. 75 | case $dirmode in 76 | '') 77 | if mkdir -p --version . >/dev/null 2>&1 && test ! -d ./--version; then 78 | echo "mkdir -p -- $*" 79 | exec mkdir -p -- "$@" 80 | else 81 | # On NextStep and OpenStep, the `mkdir' command does not 82 | # recognize any option. It will interpret all options as 83 | # directories to create, and then abort because `.' already 84 | # exists. 85 | test -d ./-p && rmdir ./-p 86 | test -d ./--version && rmdir ./--version 87 | fi 88 | ;; 89 | *) 90 | if mkdir -m "$dirmode" -p --version . >/dev/null 2>&1 && 91 | test ! -d ./--version; then 92 | echo "mkdir -m $dirmode -p -- $*" 93 | exec mkdir -m "$dirmode" -p -- "$@" 94 | else 95 | # Clean up after NextStep and OpenStep mkdir. 96 | for d in ./-m ./-p ./--version "./$dirmode"; 97 | do 98 | test -d $d && rmdir $d 99 | done 100 | fi 101 | ;; 102 | esac 103 | 104 | for file 105 | do 106 | case $file in 107 | /*) pathcomp=/ ;; 108 | *) pathcomp= ;; 109 | esac 110 | oIFS=$IFS 111 | IFS=/ 112 | set fnord $file 113 | shift 114 | IFS=$oIFS 115 | 116 | for d 117 | do 118 | test "x$d" = x && continue 119 | 120 | pathcomp=$pathcomp$d 121 | case $pathcomp in 122 | -*) pathcomp=./$pathcomp ;; 123 | esac 124 | 125 | if test ! -d "$pathcomp"; then 126 | echo "mkdir $pathcomp" 127 | 128 | mkdir "$pathcomp" || lasterr=$? 129 | 130 | if test ! -d "$pathcomp"; then 131 | errstatus=$lasterr 132 | else 133 | if test ! -z "$dirmode"; then 134 | echo "chmod $dirmode $pathcomp" 135 | lasterr= 136 | chmod "$dirmode" "$pathcomp" || lasterr=$? 137 | 138 | if test ! -z "$lasterr"; then 139 | errstatus=$lasterr 140 | fi 141 | fi 142 | fi 143 | fi 144 | 145 | pathcomp=$pathcomp/ 146 | done 147 | done 148 | 149 | exit $errstatus 150 | 151 | # Local Variables: 152 | # mode: shell-script 153 | # sh-indentation: 2 154 | # eval: (add-hook 'write-file-hooks 'time-stamp) 155 | # time-stamp-start: "scriptversion=" 156 | # time-stamp-format: "%:y-%02m-%02d.%02H" 157 | # time-stamp-end: "$" 158 | # End: 159 | -------------------------------------------------------------------------------- /perl/Changes: -------------------------------------------------------------------------------- 1 | Revision history for Perl extension JLog. 2 | 3 | 0.01 Sun Jul 2 15:54:08 2006 4 | - original version; created by h2xs 1.23 with options 5 | -A -n JLog 6 | 7 | -------------------------------------------------------------------------------- /perl/JLog.xs: -------------------------------------------------------------------------------- 1 | #include "EXTERN.h" 2 | #include "perl.h" 3 | #include "XSUB.h" 4 | 5 | #include "ppport.h" 6 | #include "jlog.h" 7 | 8 | typedef struct { 9 | jlog_ctx *ctx; 10 | char *path; 11 | jlog_id start; 12 | jlog_id last; 13 | jlog_id prev; 14 | jlog_id end; 15 | int auto_checkpoint; 16 | int error; 17 | } jlog_obj; 18 | 19 | typedef jlog_obj * JLog; 20 | typedef jlog_obj * JLog_Writer; 21 | typedef jlog_obj * JLog_Reader; 22 | 23 | #define FREE_JLOG_OBJ(my_obj) do { \ 24 | if(my_obj->ctx) { \ 25 | jlog_ctx_close(my_obj->ctx); \ 26 | } \ 27 | if(my_obj->path){ \ 28 | free(my_obj->path); \ 29 | } \ 30 | free(my_obj); \ 31 | } while(0) 32 | 33 | #define SYS_CROAK(message) do { \ 34 | croak(message "; error: %d (%s) errno: %d (%s)", \ 35 | jlog_ctx_err(my_obj->ctx), jlog_ctx_err_string(my_obj->ctx), \ 36 | jlog_ctx_errno(my_obj->ctx), strerror(jlog_ctx_errno(my_obj->ctx))); \ 37 | } while (0) 38 | 39 | MODULE = JLog PACKAGE = JLog PREFIX=JLOG_ 40 | 41 | SV *JLOG_new(classname, path, ...) 42 | char *classname; 43 | char *path; 44 | CODE: 45 | { 46 | jlog_obj *my_obj; 47 | int options = O_CREAT; 48 | size_t size = 0; 49 | my_obj = calloc(1, sizeof(*my_obj)); 50 | my_obj->ctx = jlog_new(path); 51 | my_obj->path = strdup(path); 52 | if(items > 2) { 53 | options = SvIV(ST(2)); 54 | if(items > 3) { 55 | size = SvIV(ST(3)); 56 | } 57 | } 58 | 59 | if(!my_obj->ctx) { 60 | FREE_JLOG_OBJ(my_obj); 61 | croak("jlog_new(%s) failed", path); 62 | } 63 | if(options & O_CREAT) { 64 | if(size) { 65 | jlog_ctx_alter_journal_size(my_obj->ctx, size); 66 | } 67 | if(jlog_ctx_init(my_obj->ctx) != 0) { 68 | if(jlog_ctx_err(my_obj->ctx) == JLOG_ERR_CREATE_EXISTS) { 69 | if(options & O_EXCL) { 70 | FREE_JLOG_OBJ(my_obj); 71 | croak("file already exists: %s", path); 72 | } 73 | } else { 74 | int err = jlog_ctx_err(my_obj->ctx); 75 | const char *err_string = jlog_ctx_err_string(my_obj->ctx); 76 | FREE_JLOG_OBJ(my_obj); 77 | croak("error initializing jlog: %d %s", err, err_string); 78 | } 79 | } 80 | jlog_ctx_close(my_obj->ctx); 81 | my_obj->ctx = jlog_new(path); 82 | if(!my_obj->ctx) { 83 | FREE_JLOG_OBJ(my_obj); 84 | croak("jlog_new(%s) failed after successful init", path); 85 | } 86 | } 87 | RETVAL = newSV(0); 88 | sv_setref_pv(RETVAL, classname, (void *)my_obj); 89 | } 90 | OUTPUT: 91 | RETVAL 92 | 93 | SV *JLOG_JLOG_BEGIN() 94 | CODE: 95 | { 96 | RETVAL = newSViv(JLOG_BEGIN); 97 | } 98 | OUTPUT: 99 | RETVAL 100 | 101 | SV *JLOG_JLOG_END() 102 | CODE: 103 | { 104 | RETVAL = newSViv(JLOG_END); 105 | } 106 | OUTPUT: 107 | RETVAL 108 | 109 | 110 | SV *JLOG_add_subscriber(my_obj, subscriber, ...) 111 | JLog my_obj; 112 | char *subscriber; 113 | CODE: 114 | { 115 | int whence = JLOG_BEGIN; 116 | if(items > 2) { 117 | whence = SvIV(ST(2)); 118 | } 119 | if(!my_obj || !my_obj->ctx || 120 | jlog_ctx_add_subscriber(my_obj->ctx, subscriber, whence) != 0) 121 | { 122 | RETVAL = &PL_sv_no; 123 | } else { 124 | RETVAL = &PL_sv_yes; 125 | } 126 | } 127 | OUTPUT: 128 | RETVAL 129 | 130 | SV *JLOG_remove_subscriber(my_obj, subscriber) 131 | JLog my_obj; 132 | char *subscriber; 133 | CODE: 134 | { 135 | if(!my_obj || !my_obj->ctx || 136 | jlog_ctx_remove_subscriber(my_obj->ctx, subscriber) != 0) 137 | { 138 | RETVAL = &PL_sv_no; 139 | } else { 140 | RETVAL = &PL_sv_yes; 141 | } 142 | } 143 | OUTPUT: 144 | RETVAL 145 | 146 | void JLOG_list_subscribers(my_obj) 147 | JLog my_obj; 148 | PPCODE: 149 | { 150 | char **list; 151 | int i; 152 | if(!my_obj || !my_obj->ctx) { 153 | croak("invalid jlog context"); 154 | } 155 | jlog_ctx_list_subscribers(my_obj->ctx, &list); 156 | for(i=0; list[i]; i++) { 157 | XPUSHs(sv_2mortal(newSVpv(list[i], 0))); 158 | } 159 | jlog_ctx_list_subscribers_dispose(my_obj->ctx, list); 160 | } 161 | 162 | SV *JLOG_alter_journal_size(my_obj, size) 163 | JLog my_obj; 164 | size_t size; 165 | CODE: 166 | { 167 | if(!my_obj || !my_obj->ctx) { 168 | croak("invalid jlog context"); 169 | } 170 | /* calling jlog_ctx_alter_journal_size here will never have any 171 | * effect, it's either too late or too early. Make this return 172 | * failure and deprecate it */ 173 | RETVAL = &PL_sv_no; 174 | } 175 | OUTPUT: 176 | RETVAL 177 | 178 | SV *JLOG_raw_size(my_obj) 179 | JLog my_obj; 180 | CODE: 181 | { 182 | size_t size; 183 | if(!my_obj || !my_obj->ctx) { 184 | croak("invalid jlog context"); 185 | } 186 | size = jlog_raw_size(my_obj->ctx); 187 | RETVAL = newSViv(size); 188 | } 189 | OUTPUT: 190 | RETVAL 191 | 192 | void JLOG_close(my_obj) 193 | JLog my_obj; 194 | CODE: 195 | { 196 | if(!my_obj || !my_obj->ctx) { return; } 197 | jlog_ctx_close(my_obj->ctx); 198 | my_obj->ctx = NULL; 199 | } 200 | 201 | SV* JLOG_inspect(my_obj) 202 | JLog my_obj; 203 | CODE: 204 | { 205 | HV *rh; 206 | char start[20], last[20], prev[20], end[20]; 207 | rh = (HV *)sv_2mortal((SV *)newHV()); 208 | jlog_snprint_logid(start, sizeof(start), &my_obj->start); 209 | hv_store(rh, "start", sizeof("start") - 1, newSVpv(start, 0), 0); 210 | 211 | jlog_snprint_logid(last, sizeof(last), &my_obj->last); 212 | hv_store(rh, "last", sizeof("last") - 1, newSVpv(last, 0), 0); 213 | 214 | jlog_snprint_logid(prev, sizeof(prev), &my_obj->prev); 215 | hv_store(rh, "prev", sizeof("prev") - 1, newSVpv(prev, 0), 0); 216 | 217 | jlog_snprint_logid(end, sizeof(end), &my_obj->end); 218 | hv_store(rh, "end", sizeof("end") - 1, newSVpv(end, 0), 0); 219 | 220 | hv_store(rh, "path", sizeof("path") - 1, newSVpv(my_obj->path, 0), 0); 221 | RETVAL = newRV((SV *)rh); 222 | } 223 | OUTPUT: 224 | RETVAL 225 | 226 | void JLOG_DESTROY(my_obj) 227 | JLog my_obj; 228 | CODE: 229 | { 230 | if(!my_obj) return; 231 | FREE_JLOG_OBJ(my_obj); 232 | } 233 | 234 | 235 | MODULE = JLog PACKAGE = JLog::Writer PREFIX=JLOG_W_ 236 | 237 | 238 | SV *JLOG_W_open(my_obj) 239 | JLog_Writer my_obj; 240 | CODE: 241 | { 242 | if(!my_obj || !my_obj->ctx) { 243 | croak("invalid jlog context"); 244 | } 245 | if(jlog_ctx_open_writer(my_obj->ctx) != 0) { 246 | SYS_CROAK("jlog_ctx_open_writer failed"); 247 | } else { 248 | RETVAL = newSVsv(ST(0)); 249 | } 250 | } 251 | OUTPUT: 252 | RETVAL 253 | 254 | SV *JLOG_W_write(my_obj, buffer_sv, ...) 255 | JLog_Writer my_obj; 256 | SV *buffer_sv; 257 | CODE: 258 | { 259 | char *buffer; 260 | int ts = 0; 261 | jlog_message m; 262 | struct timeval t; 263 | STRLEN buffer_len; 264 | 265 | if(!my_obj || !my_obj->ctx) { 266 | croak("invalid jlog context"); 267 | } 268 | if(items > 2) { 269 | ts = (time_t) SvIV(ST(2)); 270 | } 271 | 272 | buffer = SvPVx(buffer_sv, buffer_len); 273 | m.mess = buffer; 274 | m.mess_len = buffer_len; 275 | t.tv_sec = ts; 276 | t.tv_usec = 0; 277 | 278 | if(jlog_ctx_write_message(my_obj->ctx, &m, ts?&t:NULL) < 0) { 279 | RETVAL = &PL_sv_no; 280 | } else { 281 | RETVAL = &PL_sv_yes; 282 | } 283 | } 284 | OUTPUT: 285 | RETVAL 286 | 287 | 288 | MODULE = JLog PACKAGE = JLog::Reader PREFIX=JLOG_R_ 289 | 290 | 291 | SV *JLOG_R_open(my_obj, subscriber) 292 | JLog_Reader my_obj; 293 | char *subscriber; 294 | CODE: 295 | { 296 | if(!my_obj || !my_obj->ctx) { 297 | croak("invalid jlog context"); 298 | } 299 | if(jlog_ctx_open_reader(my_obj->ctx, subscriber) != 0) { 300 | SYS_CROAK("jlog_ctx_open_reader failed"); 301 | } else { 302 | RETVAL = newSVsv(ST(0)); 303 | } 304 | } 305 | OUTPUT: 306 | RETVAL 307 | 308 | SV * JLOG_R_read(my_obj) 309 | JLog_Reader my_obj; 310 | CODE: 311 | { 312 | const jlog_id epoch = { 0, 0 }; 313 | jlog_id cur; 314 | jlog_message message; 315 | int cnt; 316 | if(!my_obj || !my_obj->ctx) { 317 | croak("invalid jlog context"); 318 | } 319 | /* if start is unset, we need to read the interval (again) */ 320 | if(my_obj->error || !memcmp(&my_obj->start, &epoch, sizeof(jlog_id))) 321 | { 322 | my_obj->error = 0; 323 | cnt = jlog_ctx_read_interval(my_obj->ctx, &my_obj->start, &my_obj->end); 324 | if(cnt == 0 || (cnt == -1 && jlog_ctx_err(my_obj->ctx) == JLOG_ERR_FILE_OPEN)) { 325 | my_obj->start = epoch; 326 | my_obj->end = epoch; 327 | RETVAL = &PL_sv_undef; 328 | goto end; 329 | } 330 | else if(cnt == -1) SYS_CROAK("jlog_ctx_read_interval failed"); 331 | } 332 | /* if last is unset, start at the beginning */ 333 | if(!memcmp(&my_obj->last, &epoch, sizeof(jlog_id))) { 334 | cur = my_obj->start; 335 | } else { 336 | /* if we've already read the end, return; otherwise advance */ 337 | cur = my_obj->last; 338 | if(!memcmp(&my_obj->prev, &my_obj->end, sizeof(jlog_id))) { 339 | my_obj->start = epoch; 340 | my_obj->end = epoch; 341 | RETVAL = &PL_sv_undef; 342 | goto end; 343 | } 344 | jlog_ctx_advance_id(my_obj->ctx, &my_obj->last, &cur, &my_obj->end); 345 | if(!memcmp(&my_obj->last, &cur, sizeof(jlog_id))) { 346 | my_obj->start = epoch; 347 | my_obj->end = epoch; 348 | RETVAL = &PL_sv_undef; 349 | goto end; 350 | } 351 | } 352 | if(jlog_ctx_read_message(my_obj->ctx, &cur, &message) != 0) { 353 | if(jlog_ctx_err(my_obj->ctx) == JLOG_ERR_FILE_OPEN) { 354 | my_obj->error = 1; 355 | RETVAL = &PL_sv_undef; 356 | goto end; 357 | } 358 | /* read failed; croak, but recover if the read is retried */ 359 | my_obj->error = 1; 360 | SYS_CROAK("read failed"); 361 | } 362 | if(my_obj->auto_checkpoint) { 363 | if(jlog_ctx_read_checkpoint(my_obj->ctx, &cur) != 0) 364 | SYS_CROAK("checkpoint failed"); 365 | /* we have to re-read the interval after a checkpoint */ 366 | my_obj->last = epoch; 367 | my_obj->prev = epoch; 368 | my_obj->start = epoch; 369 | my_obj->end = epoch; 370 | } else { 371 | /* update last */ 372 | my_obj->prev = my_obj->last; 373 | my_obj->last = cur; 374 | /* if we've reaached the end, clear interval so we'll re-read it */ 375 | } 376 | RETVAL = newSVpv(message.mess, message.mess_len); 377 | end: 378 | ; 379 | } 380 | OUTPUT: 381 | RETVAL 382 | 383 | SV *JLOG_R_rewind(my_obj) 384 | JLog_Reader my_obj; 385 | CODE: 386 | { 387 | if(!my_obj || !my_obj->ctx) { 388 | croak("invalid jlog context"); 389 | } 390 | my_obj->last = my_obj->prev; 391 | RETVAL = newSVsv(ST(0)); 392 | } 393 | OUTPUT: 394 | RETVAL 395 | 396 | SV *JLOG_R_checkpoint(my_obj) 397 | JLog_Reader my_obj; 398 | CODE: 399 | { 400 | jlog_id epoch = { 0, 0 }; 401 | if(!my_obj || !my_obj->ctx) { 402 | croak("invalid jlog context"); 403 | } 404 | if(memcmp(&my_obj->last, &epoch, sizeof(jlog_id))) 405 | { 406 | jlog_ctx_read_checkpoint(my_obj->ctx, &my_obj->last); 407 | /* we have to re-read the interval after a checkpoint */ 408 | my_obj->last = epoch; 409 | my_obj->start = epoch; 410 | my_obj->end = epoch; 411 | } 412 | RETVAL = newSVsv(ST(0)); 413 | } 414 | OUTPUT: 415 | RETVAL 416 | 417 | SV *JLOG_R_auto_checkpoint(my_obj, ...) 418 | JLog_Reader my_obj; 419 | CODE: 420 | { 421 | if(!my_obj || !my_obj->ctx) { 422 | croak("invalid jlog context"); 423 | } 424 | if(items > 1) { 425 | int ac = SvIV(ST(1)); 426 | my_obj->auto_checkpoint = ac; 427 | } 428 | RETVAL = newSViv(my_obj->auto_checkpoint); 429 | } 430 | OUTPUT: 431 | RETVAL 432 | -------------------------------------------------------------------------------- /perl/MANIFEST: -------------------------------------------------------------------------------- 1 | Changes 2 | JLog.xs 3 | Makefile.PL 4 | MANIFEST 5 | ppport.h 6 | README 7 | t/1.t 8 | lib/JLog.pm 9 | lib/JLog/Writer.pm 10 | lib/JLog/Reader.pm 11 | typemap 12 | META.yml Module meta-data (added by MakeMaker) 13 | -------------------------------------------------------------------------------- /perl/Makefile.PL.in: -------------------------------------------------------------------------------- 1 | use 5.008005; 2 | use ExtUtils::MakeMaker; 3 | use Config; 4 | use Cwd qw/abs_path/; 5 | 6 | # See lib/ExtUtils/MakeMaker.pm for details of how to influence 7 | # the contents of the Makefile that is written. 8 | 9 | my $extra_lddflags = ("@RLDFLAG@" ne "") ? "@RLDFLAG@@libdir@" : ""; 10 | WriteMakefile( 11 | NAME => 'JLog', 12 | VERSION_FROM => 'lib/JLog.pm', 13 | CCFLAGS => "$ENV{'CFLAGS'}", 14 | PREREQ_PM => {}, 15 | ($] >= 5.005 ? 16 | (ABSTRACT_FROM => 'lib/JLog.pm', 17 | AUTHOR => 'George Schlossnagle ') : ()), 18 | LDDLFLAGS => "$Config{lddlflags} $extra_lddflags", 19 | LIBS => ["-L@libdir@ -L.. -ljlog"], 20 | INC => '-I.. -I. -I../..', 21 | # Un-comment this if you add C files to link with later: 22 | # OBJECT => '$(O_FILES)', # link all the C files too 23 | ); 24 | -------------------------------------------------------------------------------- /perl/README: -------------------------------------------------------------------------------- 1 | JLog version 0.01 2 | ================= 3 | 4 | The README is used to introduce the module and provide instructions on 5 | how to install the module, any machine dependencies it may have (for 6 | example C compilers and installed libraries) and any other information 7 | that should be provided before the module is installed. 8 | 9 | A README file is required for CPAN modules since CPAN extracts the 10 | README file from a module distribution so that people browsing the 11 | archive can use it get an idea of the modules uses. It is usually a 12 | good idea to provide version information here so that people can 13 | decide whether fixes for the module are worth downloading. 14 | 15 | INSTALLATION 16 | 17 | To install this module type the following: 18 | 19 | perl Makefile.PL 20 | make 21 | make test 22 | make install 23 | 24 | DEPENDENCIES 25 | 26 | This module requires these other modules and libraries: 27 | 28 | blah blah blah 29 | 30 | COPYRIGHT AND LICENCE 31 | 32 | Copyright (C) 2006-2008 Message Systems, Inc. 33 | 34 | This library is free software; you can redistribute it and/or modify 35 | it under the same terms as Perl itself, either Perl version 5.8.5 or, 36 | at your option, any later version of Perl 5 you may have available. 37 | 38 | 39 | -------------------------------------------------------------------------------- /perl/lib/JLog.pm: -------------------------------------------------------------------------------- 1 | package JLog; 2 | 3 | use 5.008005; 4 | use strict; 5 | use warnings; 6 | 7 | require Exporter; 8 | 9 | our @ISA = qw(Exporter); 10 | 11 | # Items to export into callers namespace by default. Note: do not export 12 | # names by default without a very good reason. Use EXPORT_OK instead. 13 | # Do not simply export all your public functions/methods/constants. 14 | 15 | # This allows declaration use JLog ':all'; 16 | # If you do not need this, moving things directly into @EXPORT or @EXPORT_OK 17 | # will save memory. 18 | our %EXPORT_TAGS = ( 'all' => [ qw( 19 | 20 | ) ] ); 21 | 22 | our @EXPORT_OK = ( @{ $EXPORT_TAGS{'all'} } ); 23 | 24 | our @EXPORT = qw( 25 | 26 | ); 27 | 28 | our $VERSION = '1.0'; 29 | 30 | require XSLoader; 31 | XSLoader::load('JLog', $VERSION); 32 | 33 | # Preloaded methods go here. 34 | 35 | 1; 36 | __END__ 37 | 38 | =head1 NAME 39 | 40 | JLog - Perl extension for the jlog journaled queueing system 41 | 42 | =head1 DESCRIPTION 43 | 44 | Parent class for JLog::Reader and JLog::Writer. You probably want to 45 | be looking at those instead. JLog is a durable, reliable, 46 | publish-and-subscribe queueing system. 47 | 48 | =head1 INTERFACE 49 | 50 | =head2 Subscriber Management 51 | 52 | A JLog must have subscribers to be functional. Without a subscriber, 53 | a queue may be purged, as there are no interested readers. For this 54 | reason it is highly recommended that you add a subscriber before 55 | writing to a log. 56 | 57 | =head3 add_subscriber 58 | 59 | $w->add_subscriber( $name, [ $flag ] ); 60 | 61 | Add a subscriber to the JLog queue. 62 | 63 | =over 4 64 | 65 | =item $name 66 | 67 | The name of the subscriber. 68 | 69 | =item $flag 70 | 71 | An optional flag dictating where the subscriber should be marked interested 72 | from. The default is JLog::JLOG_BEGIN. The other available option is 73 | JLog::JLOG_END. 74 | 75 | =back 76 | 77 | =head3 remove_subscriber 78 | 79 | $w->remove_subscriber ( $name ); 80 | 81 | Remove a subscriber to the JLog queue. 82 | 83 | =over 4 84 | 85 | =item $name 86 | 87 | The name of the subscriber. 88 | 89 | =back 90 | 91 | =head3 list_subscribers 92 | 93 | @subscribers = $w->list_subscribers; 94 | 95 | Return a list of all the subscribers to a JLog queue. 96 | 97 | =head2 Internals 98 | 99 | =head3 alter_journal_size 100 | 101 | This function is a stub provided for backwards compatibility. It will always 102 | return false. 103 | 104 | =over 4 105 | 106 | =item $size 107 | 108 | The desired size in bytes. 109 | 110 | =back 111 | 112 | =head3 raw_size 113 | 114 | $size = $w->raw_size; 115 | 116 | The size of the existing journal (including checkpointed but unpurged messages 117 | in the current journal file), in bytes. 118 | 119 | =head1 SEE ALSO 120 | 121 | JLog::Reader 122 | JLog::Writer 123 | 124 | =head1 COPYRIGHT AND LICENSE 125 | 126 | Copyright (C) 2006-2008 by Message Systems, Inc. 127 | 128 | =cut 129 | -------------------------------------------------------------------------------- /perl/lib/JLog/Reader.pm: -------------------------------------------------------------------------------- 1 | package JLog::Reader; 2 | 3 | require DynaLoader; 4 | 5 | use JLog; 6 | use strict; 7 | 8 | use vars qw/@ISA/; 9 | 10 | @ISA = qw(Exporter DynaLoader JLog); 11 | 12 | 1; 13 | __END__ 14 | 15 | =head1 NAME 16 | 17 | JLog::Reader - Perl extension for reading to a jlog journal. 18 | 19 | =head1 SUMMARY 20 | 21 | use JLog::Reader; 22 | # create a new reader off the log directory 23 | $r = JLog::Reader->new($log); 24 | # open the log as the indicated subscriber 25 | $r->open($subscriber); 26 | while(my $line = $r->read) { 27 | # work with $line 28 | } 29 | # mark the seen records as read 30 | $r->checkpoint; 31 | 32 | or 33 | 34 | use JLog::Reader; 35 | $r = JLog::Reader->new($log); 36 | $r->open($subscriber); 37 | # mark lines read as they are pulled off the queue 38 | $r->auto_checkpoint(1); 39 | while(my $line = $r->read) { 40 | # work with $line 41 | } 42 | 43 | 44 | =head1 DESCRIPTION 45 | 46 | JLog::Reader allows you to access a jlog queue for reader. 47 | 48 | =head1 INTERFACE 49 | 50 | =head2 Constructor 51 | 52 | =head3 new 53 | 54 | $w = JLog::Reader->new( $path_to_jlog, [ $flags [, $size ] ] ); 55 | 56 | Instantiates a JLog writer object associated with the JLog directory. 57 | 58 | =over 4 59 | 60 | =item $path_to_jlog 61 | 62 | The directory for the JLog queue. 63 | 64 | =item $flags 65 | 66 | Optional flags, from 'Fcntl'. The default is O_CREAT. 67 | 68 | =item $size 69 | 70 | Optional size of the individual journal files. 71 | 72 | =back 73 | 74 | =head2 Subscriber Management 75 | 76 | These functions are inherited from JLog 77 | 78 | =head3 add_subscriber 79 | 80 | $w->add_subscriber( $name, [ $flag ] ); 81 | 82 | Add a subscriber to the JLog queue. 83 | 84 | =over 4 85 | 86 | =item $name 87 | 88 | The name of the subscriber. 89 | 90 | =item $flag 91 | 92 | An optional flag dictating where the subscriber should be marked interested 93 | from. The default is JLog::JLOG_BEGIN. The other available option is 94 | JLog::JLOG_END. 95 | 96 | =back 97 | 98 | =head3 remove_subscriber 99 | 100 | $w->remove_subscriber ( $name ); 101 | 102 | Remove a subscriber to the JLog queue. 103 | 104 | =over 4 105 | 106 | =item $name 107 | 108 | The name of the subscriber. 109 | 110 | =back 111 | 112 | =head2 Reading From The Queue 113 | 114 | =head3 open 115 | 116 | $w->open( $subscriber_name ); 117 | 118 | Opens the JLog for reading. 119 | 120 | =over 4 121 | 122 | =item $subscriber_name 123 | 124 | The name we want to subscribe under. This must previously have been 125 | registered as a log subscriber via add_subscriber(). 126 | 127 | =back 128 | 129 | =head3 read 130 | 131 | $message = $w->read; 132 | 133 | Read the next message from the JLog queue. 134 | 135 | =head3 checkpoint 136 | 137 | $r->checkpoint; 138 | 139 | Checkpoint your read. This will notify the JLog that you have successfully 140 | read logs up to this point. If all registered subscribers have read to 141 | a certain point, the JLog system can remove the underlying data for the 142 | read messages. 143 | 144 | =head2 auto_checkpoint( [ $val ] ) 145 | 146 | Returns (and optionally sets) the auto_checkpoint property. With 147 | auto-checkpointing enabled, JLog::Reader will automatically 148 | checkpoint whenever you call read(). 149 | 150 | =over 4 151 | 152 | =item $val 153 | 154 | The value you wish to set auto_checkpointing to. 155 | 156 | =back 157 | 158 | =head2 Internals 159 | 160 | =head3 alter_journal_size 161 | 162 | $r->alter_journal_size( $size ); 163 | 164 | Set the size of the individual journal files. 165 | 166 | =over 4 167 | 168 | =item $size 169 | 170 | The desired size in bytes. 171 | 172 | =back 173 | 174 | =head3 raw_size 175 | 176 | $size = $r->raw_size; 177 | 178 | The size of the existing journal (including checkpointed but unpurged messages 179 | in the current journal file), in bytes. 180 | 181 | =head rewind 182 | 183 | $r->rewind; 184 | 185 | Rewind the jlog to the previous transaction id (when in an uncommitted state). 186 | This is useful for implementing a 'peek' style action. 187 | 188 | =back 189 | 190 | 191 | =head1 SEE ALSO 192 | 193 | JLog 194 | JLog::Writer 195 | 196 | =head1 COPYRIGHT 197 | 198 | Copyright (C) 2006-2008 by Message Systems, Inc. 199 | 200 | =cut 201 | -------------------------------------------------------------------------------- /perl/lib/JLog/Writer.pm: -------------------------------------------------------------------------------- 1 | package JLog::Writer; 2 | 3 | require DynaLoader; 4 | 5 | use JLog; 6 | use strict; 7 | 8 | use vars qw/@ISA/; 9 | 10 | @ISA = qw(Exporter DynaLoader JLog); 11 | 12 | 1; 13 | __END__ 14 | 15 | =head1 NAME 16 | 17 | JLog::Writer - Perl extension for writing to a jlog journal. 18 | 19 | =head1 SUMMARY 20 | 21 | use JLog::Writer; 22 | use Fcntl qw/:DEFAULT/; 23 | 24 | my $sub = "testsubscriber"; 25 | my $log = "foo.jlog"; 26 | 27 | # open a log - this respects stander Fcntl flags 28 | my $w = JLog::Writer->new($log, O_CREAT); 29 | 30 | # add a subscriber - without this there is danger that 31 | # a log may be expired without the unnamed subscriber 32 | # stating its intention to read. 33 | $w->add_subscriber($sub); 34 | 35 | # open for writing 36 | $w->open; 37 | 38 | foreach (1 ... 3) { 39 | # write to the queue 40 | $w->write("foo $_"); 41 | } 42 | # close the queue 43 | $w->close; 44 | 45 | =head1 DESCRIPTION 46 | 47 | JLog::Writer allows you to access a jlog queue for writing. 48 | 49 | =head1 INTERFACE 50 | 51 | =head2 Constructor 52 | 53 | =head3 new 54 | 55 | $w = JLog::Writer->new( $path_to_jlog, [ $flags [, $size ] ] ); 56 | 57 | Instantiates a JLog writer object associated with the JLog directory. 58 | 59 | =over 4 60 | 61 | =item $path_to_jlog 62 | 63 | The directory for the JLog queue. 64 | 65 | =item $flags 66 | 67 | Optional flags, from 'Fcntl'. The default is O_CREAT. 68 | 69 | =item $size 70 | 71 | Optional size of the individual journal files. 72 | 73 | =back 74 | 75 | =head2 Subscriber Management 76 | 77 | These functions are inherited from JLog 78 | 79 | =head3 add_subscriber 80 | 81 | $w->add_subscriber( $name, [ $flag ] ); 82 | 83 | Add a subscriber to the JLog queue. 84 | 85 | =over 4 86 | 87 | =item $name 88 | 89 | The name of the subscriber. 90 | 91 | =item $flag 92 | 93 | An optional flag dictating where the subscriber should be marked interested 94 | from. The default is JLog::JLOG_BEGIN. The other available option is 95 | JLog::JLOG_END. 96 | 97 | =back 98 | 99 | =head3 remove_subscriber 100 | 101 | $w->remove_subscriber ( $name ); 102 | 103 | Remove a subscriber to the JLog queue. 104 | 105 | =over 4 106 | 107 | =item $name 108 | 109 | The name of the subscriber. 110 | 111 | =back 112 | 113 | =head2 Writing to the Queue 114 | 115 | =head3 open 116 | 117 | $w->open(); 118 | 119 | Opens the JLog for writing. 120 | 121 | =head3 write 122 | 123 | $w->write( $message [, $ts ] ); 124 | 125 | Write a message to the JLog. 126 | 127 | =over 4 128 | 129 | =item $message 130 | 131 | The message to write. 132 | 133 | =item $ts 134 | 135 | The timestamp (in epoch seconds) to set the record timestamp to. The default is time(). 136 | 137 | =back 138 | 139 | =head2 Internals 140 | 141 | =head3 alter_journal_size 142 | 143 | $w->alter_journal_size( $size ); 144 | 145 | Set the size of the individual journal files. 146 | 147 | =over 4 148 | 149 | =item $size 150 | 151 | The desired size in bytes. 152 | 153 | =back 154 | 155 | =head3 raw_size 156 | 157 | $size = $w->raw_size; 158 | 159 | The size of the existing journal (including checkpointed but unpurged messages 160 | in the current journal file), in bytes. 161 | 162 | =head1 SEE ALSO 163 | 164 | JLog 165 | JLog::Reader 166 | 167 | =head1 COPYRIGHT AND LICENSE 168 | 169 | Copyright (C) 2006-2008 by Message Systems, Inc. 170 | 171 | =cut 172 | 173 | -------------------------------------------------------------------------------- /perl/t/1.t: -------------------------------------------------------------------------------- 1 | # Before `make install' is performed this script should be runnable with 2 | # `make test'. After `make install' it should work as `perl JLog.t' 3 | 4 | ######################### 5 | 6 | use Fcntl; 7 | use Test::More tests => 7; 8 | BEGIN { 9 | use_ok('JLog::Writer'); 10 | use_ok('JLog::Reader'); 11 | }; 12 | 13 | my $sub = "testsubscriber"; 14 | my $log = "foo.jlog"; 15 | 16 | system("rm -rf $log"); 17 | my $w = JLog::Writer->new($log, O_CREAT|O_EXCL, 1024); 18 | ok !$w->alter_journal_size(1024), "set journal size should fail"; 19 | 20 | $w->add_subscriber($sub); 21 | $w->open; 22 | foreach (1 ... 3) { 23 | $w->write("foo $_"); 24 | } 25 | $w->close; 26 | 27 | my $r = JLog::Reader->new($log); 28 | 29 | my @subs = $r->list_subscribers; 30 | is_deeply(\@subs, [ $sub ], 'list_subscribers'); 31 | 32 | $r->open($sub); 33 | 34 | my @lines; 35 | my $i; 36 | 37 | while(my $line = $r->read) { 38 | push @lines, $line; 39 | $i++; 40 | } 41 | is_deeply(\@lines, [ 'foo 1', 'foo 2', 'foo 3' ], "in == out"); 42 | 43 | # checkpoint 44 | undef $r; 45 | 46 | $r = JLog::Reader->new($log); 47 | $r->open($sub); 48 | @lines = (); 49 | $i = 0; 50 | while(my $line = $r->read) { 51 | push @lines, $line; 52 | $i++; 53 | } 54 | is_deeply(\@lines, [ 'foo 1', 'foo 2', 'foo 3' ], "in == out"); 55 | 56 | $r->checkpoint; 57 | 58 | $r = JLog::Reader->new($log); 59 | $r->open($sub); 60 | is $r->read, undef, "our checkpoint cleared things"; 61 | -------------------------------------------------------------------------------- /perl/typemap: -------------------------------------------------------------------------------- 1 | JLog T_PTROBJ_SPECIAL 2 | JLog_Writer T_PTROBJ_SPECIAL 3 | JLog_Reader T_PTROBJ_SPECIAL 4 | 5 | INPUT 6 | T_PTROBJ_SPECIAL 7 | if (sv_derived_from($arg, \"${(my $ntt=$ntype)=~s/_/::/g;\$ntt}\")) { 8 | IV tmp = SvIV((SV*)SvRV($arg)); 9 | $var = ($type) tmp; 10 | } 11 | else 12 | croak(\"$var is not of type ${(my $ntt=$ntype)=~s/_/::/g;\$ntt}\") 13 | 14 | 15 | OUTPUT 16 | T_PTROBJ_SPECIAL 17 | sv_setref_pv($arg, \"${(my $ntt=$ntype)=~s/_/::/g;\$ntt}\", 18 | (void*)$var); 19 | 20 | -------------------------------------------------------------------------------- /php/config.m4: -------------------------------------------------------------------------------- 1 | dnl 2 | dnl $ Id: $ 3 | dnl 4 | 5 | PHP_ARG_WITH(jlog, jlog,[ --with-jlog[=DIR] With jlog support]) 6 | 7 | 8 | if test "$PHP_JLOG" != "no"; then 9 | if test "$PHP_JLOG" == "yes"; then 10 | if test -d /opt/msys ; then 11 | PHP_JLOG="/opt/msys/jlog" 12 | else 13 | PHP_JLOG="/opt/ecelerity" 14 | fi 15 | fi 16 | CPPFLAGS="$CPPFLAGS $INCLUDES -DHAVE_JLOG" 17 | 18 | PHP_ADD_INCLUDE(..) 19 | PHP_ADD_INCLUDE(../..) 20 | PHP_ADD_INCLUDE(.) 21 | PHP_ADD_INCLUDE($PHP_JLOG/include) 22 | case $PHP_JLOG in 23 | *ecelerity*) 24 | dnl has architecture specific include dir 25 | archdir=`uname -p` 26 | PHP_ADD_INCLUDE($PHP_JLOG/include/$archdir) 27 | ;; 28 | esac 29 | OLD_CPPFLAGS="$CPPFLAGS" 30 | CPPFLAGS="$CPPFLAGS $INCLUDES -DHAVE_JLOG" 31 | CPPFLAGS="$OLD_CPPFLAGS" 32 | PHP_SUBST(JLOG_SHARED_LIBADD) 33 | 34 | PHP_ADD_LIBRARY_WITH_PATH(jlog, $PHP_JLOG/lib64, JLOG_SHARED_LIBADD) 35 | 36 | 37 | PHP_SUBST(JLOG_SHARED_LIBADD) 38 | AC_DEFINE(HAVE_JLOG, 1, [ ]) 39 | PHP_NEW_EXTENSION(jlog, jlog.c , $ext_shared) 40 | 41 | fi 42 | 43 | -------------------------------------------------------------------------------- /php/package.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | jlog 6 | A sample PHP extension 7 | 8 | This is a sample extension specification 9 | showing how to use CodeGen_PECL for 10 | extension generation. 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 0.0.1 19 | devel 20 | 2007-07-02 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | -------------------------------------------------------------------------------- /php/package2.xml: -------------------------------------------------------------------------------- 1 | 2 | 9 | 10 | jlog 11 | pecl.php.net 12 | 13 | A sample PHP extension 14 | 15 | 16 | This is a sample extension specification 17 | showing how to use CodeGen_PECL for 18 | extension generation. 19 | 20 | 21 | 22 | 2007-07-02 23 | 24 | 0.0.1 25 | 0.0.1 26 | 27 | 28 | devel 29 | devel 30 | 31 | 32 | unknown 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 5.0.0 66 | 67 | 68 | 1.4.0a1 69 | 70 | unix 71 | 72 | 73 | 74 | jlog 75 | 76 | 77 | 78 | 79 | 80 | 81 | -------------------------------------------------------------------------------- /php/php_jlog.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2005-2008, Message Systems, Inc. 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are 7 | * met: 8 | * 9 | * * Redistributions of source code must retain the above copyright 10 | * notice, this list of conditions and the following disclaimer. 11 | * * Redistributions in binary form must reproduce the above 12 | * copyright notice, this list of conditions and the following 13 | * disclaimer in the documentation and/or other materials provided 14 | * with the distribution. 15 | * * Neither the name Message Systems, Inc. nor the names 16 | * of its contributors may be used to endorse or promote products 17 | * derived from this software without specific prior written 18 | * permission. 19 | * 20 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | */ 32 | 33 | #ifndef PHP_JLOG_H 34 | #define PHP_JLOG_H 35 | 36 | #ifdef __cplusplus 37 | extern "C" { 38 | #endif 39 | 40 | #ifdef HAVE_CONFIG_H 41 | #include "config.h" 42 | #endif 43 | 44 | #include 45 | 46 | #ifdef HAVE_JLOG 47 | 48 | #include 49 | #include 50 | #include 51 | #include 52 | #ifdef __cplusplus 53 | } // extern "C" 54 | #endif 55 | #include 56 | #ifdef __cplusplus 57 | extern "C" { 58 | #endif 59 | 60 | extern zend_module_entry jlog_module_entry; 61 | #define phpext_jlog_ptr &jlog_module_entry 62 | 63 | #ifdef PHP_WIN32 64 | #define PHP_JLOG_API __declspec(dllexport) 65 | #else 66 | #define PHP_JLOG_API 67 | #endif 68 | 69 | PHP_MINIT_FUNCTION(jlog); 70 | PHP_MSHUTDOWN_FUNCTION(jlog); 71 | PHP_RINIT_FUNCTION(jlog); 72 | PHP_RSHUTDOWN_FUNCTION(jlog); 73 | PHP_MINFO_FUNCTION(jlog); 74 | 75 | #ifdef ZTS 76 | #include "TSRM.h" 77 | #endif 78 | 79 | #define FREE_RESOURCE(resource) zend_list_delete(Z_LVAL_P(resource)) 80 | 81 | #define PROP_GET_LONG(name) Z_LVAL_P(zend_read_property(_this_ce, _this_zval, #name, strlen(#name), 1 TSRMLS_CC)) 82 | #define PROP_SET_LONG(name, l) zend_update_property_long(_this_ce, _this_zval, #name, strlen(#name), l TSRMLS_CC) 83 | 84 | #define PROP_GET_DOUBLE(name) Z_DVAL_P(zend_read_property(_this_ce, _this_zval, #name, strlen(#name), 1 TSRMLS_CC)) 85 | #define PROP_SET_DOUBLE(name, d) zend_update_property_double(_this_ce, _this_zval, #name, strlen(#name), d TSRMLS_CC) 86 | 87 | #define PROP_GET_STRING(name) Z_STRVAL_P(zend_read_property(_this_ce, _this_zval, #name, strlen(#name), 1 TSRMLS_CC)) 88 | #define PROP_GET_STRLEN(name) Z_STRLEN_P(zend_read_property(_this_ce, _this_zval, #name, strlen(#name), 1 TSRMLS_CC)) 89 | #define PROP_SET_STRING(name, s) zend_update_property_string(_this_ce, _this_zval, #name, strlen(#name), s TSRMLS_CC) 90 | #define PROP_SET_STRINGL(name, s, l) zend_update_property_string(_this_ce, _this_zval, #name, strlen(#name), s, l TSRMLS_CC) 91 | 92 | 93 | PHP_METHOD(Jlog, __construct); 94 | ZEND_BEGIN_ARG_INFO(Jlog____construct_args, 0) 95 | ZEND_ARG_INFO(0, options) 96 | ZEND_END_ARG_INFO() 97 | PHP_METHOD(Jlog, add_subscriber); 98 | ZEND_BEGIN_ARG_INFO(Jlog__add_subscriber_args, 0) 99 | ZEND_ARG_INFO(0, subscriber) 100 | ZEND_ARG_INFO(0, whence) 101 | ZEND_END_ARG_INFO() 102 | PHP_METHOD(Jlog, remove_subscriber); 103 | ZEND_BEGIN_ARG_INFO(Jlog__remove_subscriber_args, 0) 104 | ZEND_ARG_INFO(0, subscriber) 105 | ZEND_END_ARG_INFO() 106 | PHP_METHOD(Jlog, list_subscribers); 107 | ZEND_BEGIN_ARG_INFO(Jlog__list_subscribers_args, 0) 108 | ZEND_END_ARG_INFO() 109 | PHP_METHOD(Jlog, alter_journal_size); 110 | ZEND_BEGIN_ARG_INFO(Jlog__alter_journal_size_args, 0) 111 | ZEND_ARG_INFO(0, size) 112 | ZEND_END_ARG_INFO() 113 | PHP_METHOD(Jlog, raw_size); 114 | ZEND_BEGIN_ARG_INFO(Jlog__raw_size_args, 0) 115 | ZEND_END_ARG_INFO() 116 | PHP_METHOD(Jlog, close); 117 | ZEND_BEGIN_ARG_INFO(Jlog__close_args, 0) 118 | ZEND_END_ARG_INFO() 119 | PHP_METHOD(Jlog_Writer, open); 120 | ZEND_BEGIN_ARG_INFO(Jlog_Writer__open_args, 0) 121 | ZEND_END_ARG_INFO() 122 | PHP_METHOD(Jlog_Writer, write); 123 | ZEND_BEGIN_ARG_INFO(Jlog_Writer__write_args, 0) 124 | ZEND_ARG_INFO(0, buffer) 125 | ZEND_END_ARG_INFO() 126 | PHP_METHOD(Jlog_Reader, open); 127 | ZEND_BEGIN_ARG_INFO(Jlog_Reader__open_args, 0) 128 | ZEND_ARG_INFO(0, subscriber) 129 | ZEND_END_ARG_INFO() 130 | PHP_METHOD(Jlog_Reader, read); 131 | ZEND_BEGIN_ARG_INFO(Jlog_Reader__read_args, 0) 132 | ZEND_END_ARG_INFO() 133 | PHP_METHOD(Jlog_Reader, checkpoint); 134 | ZEND_BEGIN_ARG_INFO(Jlog_Reader__checkpoint_args, 0) 135 | ZEND_END_ARG_INFO() 136 | PHP_METHOD(Jlog_Reader, auto_checkpoint); 137 | ZEND_BEGIN_ARG_INFO(Jlog_Reader__auto_checkpoint_args, 0) 138 | ZEND_ARG_INFO(0, state) 139 | ZEND_END_ARG_INFO() 140 | #ifdef __cplusplus 141 | } // extern "C" 142 | #endif 143 | 144 | #endif /* PHP_HAVE_JLOG */ 145 | 146 | #endif /* PHP_JLOG_H */ 147 | 148 | 149 | /* 150 | * Local variables: 151 | * tab-width: 4 152 | * c-basic-offset: 4 153 | * End: 154 | * vim600: noet sw=4 ts=4 fdm=marker 155 | * vim<600: noet sw=4 ts=4 156 | */ 157 | -------------------------------------------------------------------------------- /python/cjlog.pxd: -------------------------------------------------------------------------------- 1 | # for jlog_ctx_write_message 2 | cdef extern struct timeval: 3 | long tv_sec 4 | long tv_usec 5 | 6 | cdef extern from "jlog.h": 7 | 8 | ctypedef struct jlog_ctx: 9 | pass 10 | 11 | ctypedef struct jlog_message_header: 12 | int reserved 13 | int tv_sec 14 | int tv_usec 15 | int mlen 16 | 17 | ctypedef struct jlog_id: 18 | int log 19 | int marker 20 | 21 | void JLOG_ID_ADVANCE(jlog_id *id) 22 | 23 | ctypedef struct jlog_message: 24 | jlog_message_header *header 25 | int mess_len 26 | void *mess 27 | jlog_message_header aligned_header 28 | 29 | ctypedef enum jlog_position: 30 | JLOG_BEGIN, 31 | JLOG_END 32 | 33 | ctypedef enum jlog_safety: 34 | JLOG_UNSAFE, 35 | JLOG_ALMOST_SAFE, 36 | JLOG_SAFE 37 | 38 | ctypedef enum jlog_err: 39 | JLOG_ERR_SUCCESS = 0, 40 | JLOG_ERR_ILLEGAL_INIT, 41 | JLOG_ERR_ILLEGAL_OPEN, 42 | JLOG_ERR_OPEN, 43 | JLOG_ERR_NOTDIR, 44 | JLOG_ERR_CREATE_PATHLEN, 45 | JLOG_ERR_CREATE_EXISTS, 46 | JLOG_ERR_CREATE_MKDIR, 47 | JLOG_ERR_CREATE_META, 48 | JLOG_ERR_CREATE_PRE_COMMIT, 49 | JLOG_ERR_LOCK, 50 | JLOG_ERR_IDX_OPEN, 51 | JLOG_ERR_IDX_SEEK, 52 | JLOG_ERR_IDX_CORRUPT, 53 | JLOG_ERR_IDX_WRITE, 54 | JLOG_ERR_IDX_READ, 55 | JLOG_ERR_FILE_OPEN, 56 | JLOG_ERR_FILE_SEEK, 57 | JLOG_ERR_FILE_CORRUPT, 58 | JLOG_ERR_FILE_READ, 59 | JLOG_ERR_FILE_WRITE, 60 | JLOG_ERR_META_OPEN, 61 | JLOG_ERR_PRE_COMMIT_OPEN, 62 | JLOG_ERR_ILLEGAL_WRITE, 63 | JLOG_ERR_ILLEGAL_CHECKPOINT, 64 | JLOG_ERR_INVALID_SUBSCRIBER, 65 | JLOG_ERR_ILLEGAL_LOGID, 66 | JLOG_ERR_SUBSCRIBER_EXISTS, 67 | JLOG_ERR_CHECKPOINT, 68 | JLOG_ERR_NOT_SUPPORTED, 69 | JLOG_ERR_CLOSE_LOGID 70 | 71 | ctypedef enum jlog_compression_provider_choice: 72 | JLOG_COMPRESSION_NULL = 0, 73 | JLOG_COMPRESSION_LZ4 = 0x01 74 | 75 | 76 | ctypedef void (*jlog_error_func) (void *ctx, char *msg, ...) 77 | 78 | jlog_ctx *jlog_new(char *path) 79 | void jlog_set_error_func(jlog_ctx *ctx, jlog_error_func Func, void *ptr) 80 | long jlog_raw_size(jlog_ctx *ctx) 81 | int jlog_ctx_init(jlog_ctx *ctx) 82 | int jlog_get_checkpoint(jlog_ctx *ctx, char *s, jlog_id *id) 83 | int jlog_ctx_list_subscribers_dispose(jlog_ctx *ctx, char **subs) 84 | int jlog_ctx_list_subscribers(jlog_ctx *ctx, char ***subs) 85 | 86 | int jlog_ctx_err(jlog_ctx *ctx) 87 | char *jlog_ctx_err_string(jlog_ctx *ctx) 88 | int jlog_ctx_errno(jlog_ctx *ctx) 89 | int jlog_ctx_open_writer(jlog_ctx *ctx) 90 | int jlog_ctx_open_reader(jlog_ctx *ctx, char *subscriber) 91 | int jlog_ctx_close(jlog_ctx *ctx) 92 | 93 | int jlog_ctx_alter_mode(jlog_ctx *ctx, int mode) 94 | int jlog_ctx_alter_journal_size(jlog_ctx *ctx, long size) 95 | int jlog_ctx_alter_safety(jlog_ctx *ctx, jlog_safety safety) 96 | int jlog_ctx_set_multi_process(jlog_ctx *ctx, int mproc); 97 | int jlog_ctx_set_use_compression(jlog_ctx *ctx, int use); 98 | int jlog_ctx_set_compression_provider(jlog_ctx *ctx, jlog_compression_provider_choice provider); 99 | int jlog_ctx_set_pre_commit_buffer_size(jlog_ctx *ctx, long s); 100 | int jlog_ctx_flush_pre_commit_buffer(jlog_ctx *ctx); 101 | int jlog_ctx_add_subscriber(jlog_ctx *ctx, char *subscriber, jlog_position whence) 102 | int jlog_ctx_remove_subscriber(jlog_ctx *ctx, char *subscriber) 103 | 104 | int jlog_ctx_write(jlog_ctx *ctx, void *message, long mess_len) 105 | int jlog_ctx_write_message(jlog_ctx *ctx, jlog_message *msg, timeval *when) 106 | int jlog_ctx_read_interval(jlog_ctx *ctx, jlog_id *first_mess, jlog_id *last_mess) 107 | int jlog_ctx_read_message(jlog_ctx *ctx, jlog_id *, jlog_message *) 108 | int jlog_ctx_read_checkpoint(jlog_ctx *ctx, jlog_id *checkpoint) 109 | int jlog_snprint_logid(char *buff, int n, jlog_id *checkpoint) 110 | 111 | int jlog_pending_readers(jlog_ctx *ctx, int log, int *earliest_ptr) 112 | int __jlog_pending_readers(jlog_ctx *ctx, int log) 113 | int jlog_ctx_first_log_id(jlog_ctx *ctx, jlog_id *id) 114 | int jlog_ctx_last_log_id(jlog_ctx *ctx, jlog_id *id) 115 | int jlog_ctx_advance_id(jlog_ctx *ctx, jlog_id *cur, jlog_id *start, jlog_id *finish) 116 | int jlog_clean(char *path) 117 | -------------------------------------------------------------------------------- /python/examples/test_jlog.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import unittest 4 | import os 5 | import tempfile 6 | import shutil 7 | 8 | from jlog import * 9 | 10 | # initial setting gets overriden in main() 11 | g_tempdir = None 12 | 13 | # We can't make these configurable because of the way unittest works. If you 14 | # need to debug a test set this to true 15 | g_debug = False 16 | 17 | if 'TEST_DEBUG' in os.environ and os.environ['TEST_DEBUG'] == 'true': 18 | g_debug = True 19 | 20 | class TestJLog(unittest.TestCase): 21 | 22 | def test_jlog(self): 23 | 24 | jlog_dir = g_tempdir + 'test_jlog.jlog' 25 | 26 | # setup writer 27 | writer = JLogWriter(jlog_dir) 28 | self.assertTrue(writer.ctx_initialized) 29 | self.assertTrue(writer.add_subscriber("testsub")) 30 | # 1-100 31 | for i in xrange(1, 101): 32 | writer.write("test" + str(i)) 33 | 34 | # setup reader 35 | reader = JLogReader(jlog_dir, "testsub") 36 | self.assertTrue(reader.ctx_initialized) 37 | self.assertEqual(reader.subscriber, "testsub") 38 | self.assertEqual(len(reader), 100) 39 | 40 | # read 5 entries 41 | count = 0 42 | for msg in reader: 43 | self.assertEqual(msg, "test" + str(count + 1)) 44 | count = count + 1 45 | if count >= 5: 46 | break 47 | 48 | self.assertEqual(len(reader), 100 - count) 49 | 50 | # truncate log 51 | reader.truncate(2) 52 | self.assertEqual(len(reader), 2) 53 | # XXX - this next part is broken. I think it's an issue with 54 | # jlog_ctx_read_interval reporting an invalid size. It will still work 55 | # for now 56 | msg = next(reader) 57 | #self.assertEqual(msg, "test99") 58 | msg = next(reader) 59 | #self.assertEqual(msg, "test100") 60 | 61 | 62 | def setUpModule(): 63 | global g_tempdir 64 | g_tempdir = tempfile.mkdtemp() + '/' 65 | 66 | def tearDownModule(): 67 | if g_debug is True: 68 | print "Test state saved for debugging in %s" % g_tempdir 69 | else: 70 | print "To debug tests set TEST_DEBUG=true in env" 71 | # clean up the mess 72 | shutil.rmtree(g_tempdir) 73 | 74 | if __name__ == '__main__': 75 | # run tests 76 | unittest.main() 77 | -------------------------------------------------------------------------------- /python/jlog.pyx: -------------------------------------------------------------------------------- 1 | cimport cjlog 2 | from cpython cimport PyString_FromStringAndSize 3 | from cpython cimport PyString_FromString 4 | from cpython cimport PyString_AsString 5 | from cpython cimport PyString_AsStringAndSize 6 | 7 | JLOG_BEGIN = cjlog.JLOG_BEGIN 8 | JLOG_END = cjlog.JLOG_END 9 | 10 | JLOG_UNSAFE = cjlog.JLOG_UNSAFE 11 | JLOG_ALMOST_SAFE = cjlog.JLOG_ALMOST_SAFE 12 | JLOG_SAFE = cjlog.JLOG_SAFE 13 | 14 | JLOG_ERR_SUCCESS = cjlog.JLOG_ERR_SUCCESS 15 | JLOG_ERR_ILLEGAL_INIT = cjlog.JLOG_ERR_ILLEGAL_INIT 16 | JLOG_ERR_ILLEGAL_OPEN = cjlog.JLOG_ERR_ILLEGAL_OPEN 17 | JLOG_ERR_OPEN = cjlog.JLOG_ERR_OPEN 18 | JLOG_ERR_NOTDIR = cjlog.JLOG_ERR_NOTDIR 19 | JLOG_ERR_CREATE_PATHLEN = cjlog.JLOG_ERR_CREATE_PATHLEN 20 | JLOG_ERR_CREATE_EXISTS = cjlog.JLOG_ERR_CREATE_EXISTS 21 | JLOG_ERR_CREATE_MKDIR = cjlog.JLOG_ERR_CREATE_MKDIR 22 | JLOG_ERR_CREATE_META = cjlog.JLOG_ERR_CREATE_META 23 | JLOG_ERR_CREATE_PRE_COMMIT = cjlog.JLOG_ERR_CREATE_PRE_COMMIT 24 | JLOG_ERR_LOCK = cjlog.JLOG_ERR_LOCK 25 | JLOG_ERR_IDX_OPEN = cjlog.JLOG_ERR_IDX_OPEN 26 | JLOG_ERR_IDX_SEEK = cjlog.JLOG_ERR_IDX_SEEK 27 | JLOG_ERR_IDX_CORRUPT = cjlog.JLOG_ERR_IDX_CORRUPT 28 | JLOG_ERR_IDX_WRITE = cjlog.JLOG_ERR_IDX_WRITE 29 | JLOG_ERR_IDX_READ = cjlog.JLOG_ERR_IDX_READ 30 | JLOG_ERR_FILE_OPEN = cjlog.JLOG_ERR_FILE_OPEN 31 | JLOG_ERR_FILE_SEEK = cjlog.JLOG_ERR_FILE_SEEK 32 | JLOG_ERR_FILE_CORRUPT = cjlog.JLOG_ERR_FILE_CORRUPT 33 | JLOG_ERR_FILE_READ = cjlog.JLOG_ERR_FILE_READ 34 | JLOG_ERR_FILE_WRITE = cjlog.JLOG_ERR_FILE_WRITE 35 | JLOG_ERR_META_OPEN = cjlog.JLOG_ERR_META_OPEN 36 | JLOG_ERR_PRE_COMMIT_OPEN = cjlog.JLOG_ERR_PRE_COMMIT_OPEN 37 | JLOG_ERR_ILLEGAL_WRITE = cjlog.JLOG_ERR_ILLEGAL_WRITE 38 | JLOG_ERR_ILLEGAL_CHECKPOINT = cjlog.JLOG_ERR_ILLEGAL_CHECKPOINT 39 | JLOG_ERR_INVALID_SUBSCRIBER = cjlog.JLOG_ERR_INVALID_SUBSCRIBER 40 | JLOG_ERR_ILLEGAL_LOGID = cjlog.JLOG_ERR_ILLEGAL_LOGID 41 | JLOG_ERR_SUBSCRIBER_EXISTS = cjlog.JLOG_ERR_SUBSCRIBER_EXISTS 42 | JLOG_ERR_CHECKPOINT = cjlog.JLOG_ERR_CHECKPOINT 43 | JLOG_ERR_NOT_SUPPORTED = cjlog.JLOG_ERR_NOT_SUPPORTED 44 | JLOG_ERR_CLOSE_LOGID = cjlog.JLOG_ERR_CLOSE_LOGID 45 | 46 | class JLogError(Exception): 47 | def __init__(self, message, reason = None): 48 | if not reason: 49 | self.message = message 50 | else: 51 | self.message = message + ": " + reason 52 | 53 | def __str__(self): 54 | return self.message 55 | 56 | 57 | cpdef jlog_add_subscriber(path, subscriber, position = JLOG_BEGIN): 58 | cdef cjlog.jlog_ctx *ctx 59 | ctx = cjlog.jlog_new(path) 60 | if ctx is NULL: 61 | return False 62 | error = cjlog.jlog_ctx_add_subscriber(ctx, subscriber, position) 63 | if error: 64 | return False 65 | else: 66 | return True 67 | 68 | 69 | cpdef jlog_remove_subscriber(path, subscriber): 70 | cdef cjlog.jlog_ctx *ctx 71 | ctx = cjlog.jlog_new(path) 72 | if ctx is NULL: 73 | return False 74 | error = cjlog.jlog_ctx_remove_subscriber(ctx, subscriber) 75 | if error: 76 | return False 77 | else: 78 | return True 79 | 80 | 81 | # Keep methods and variables that work for both readers and writers here 82 | cdef class BaseJLog: 83 | cdef public object ctx_initialized 84 | cdef public object path 85 | cdef cjlog.jlog_ctx *ctx 86 | 87 | def add_subscriber(self, subscriber, position = JLOG_BEGIN): 88 | if not self.ctx_initialized: 89 | return False 90 | error = cjlog.jlog_ctx_add_subscriber(self.ctx, subscriber, position) 91 | if error: 92 | return False 93 | else: 94 | return True 95 | 96 | def remove_subscriber(self, subscriber): 97 | if not self.ctx_initialized: 98 | return False 99 | error = cjlog.jlog_ctx_remove_subscriber(self.ctx, subscriber) 100 | if error: 101 | return False 102 | else: 103 | return True 104 | 105 | 106 | cdef class JLogReader(BaseJLog): 107 | cdef public object subscriber 108 | cdef cjlog.jlog_id begin 109 | cdef cjlog.jlog_id end 110 | cdef int count 111 | 112 | 113 | # path - the directory path to the jlog directory 114 | # subscriber - the subscriber for the jlog 115 | def __cinit__(self, path, subscriber): 116 | self.path = path 117 | self.subscriber = subscriber 118 | self.ctx = cjlog.jlog_new(path) 119 | self.count = 0 120 | self.ctx_initialized = False 121 | 122 | if self.ctx is NULL: 123 | raise JLogError("jlog failed to initialize") 124 | else: 125 | self.ctx_initialized = True 126 | 127 | error = cjlog.jlog_ctx_open_reader(self.ctx, subscriber) 128 | if error: 129 | raise JLogError("jlog reader failed to open subscriber %s@%s" % \ 130 | (subscriber, path), JLogReader.error_msg()) 131 | 132 | 133 | def __dealloc__(self): 134 | if self.ctx_initialized: 135 | cjlog.jlog_ctx_close(self.ctx) 136 | self.ctx_initialized = False 137 | 138 | 139 | # python iterable interface. 140 | def __next__(self): 141 | cdef cjlog.jlog_message msg 142 | 143 | # Stop iteration if we reached -1 (end of data) on the last 144 | # iteration. Set count to 0 so iteration can try again 145 | if self.count is -1: 146 | self.count = 0 147 | raise StopIteration() 148 | 149 | if not self.count: 150 | # check and see if there's anything to read since the last batch 151 | self.count = cjlog.jlog_ctx_read_interval(self.ctx, &self.begin, \ 152 | &self.end) 153 | 154 | # nothing to read 155 | if self.count < 1: 156 | raise StopIteration() 157 | 158 | error = cjlog.jlog_ctx_read_message(self.ctx, &self.begin, &msg) 159 | if error: 160 | raise JLogError("jlog_ctx_read_message", JLogReader.error_msg()) 161 | 162 | # checkpoint the read since we don't know when the caller will stop or 163 | # we could have an exception 164 | # XXX - make checkpoint happen on exceptions, dealloc. Calling it every 165 | # read is slow 166 | cjlog.jlog_ctx_read_checkpoint(self.ctx, &self.begin) 167 | 168 | pystr = PyString_FromStringAndSize(msg.mess, msg.mess_len) 169 | 170 | self.count -= 1 171 | if not self.count: 172 | self.count = -1 173 | else: 174 | cjlog.JLOG_ID_ADVANCE(&self.begin) 175 | 176 | return pystr 177 | 178 | 179 | def __iter__(self): 180 | return self 181 | 182 | 183 | def __len__(self): 184 | # use our own temporary begin/end so as not to disturb the object state 185 | cdef cjlog.jlog_id begin 186 | cdef cjlog.jlog_id end 187 | count = cjlog.jlog_ctx_read_interval(self.ctx, &begin, &end) 188 | return count 189 | 190 | 191 | # truncate the subscriber to size entries or 0 if size is not specified. If 192 | # there are no other pending readers the files will be cleaned up. 193 | def truncate(self, size = None): 194 | if size is not None and size > 0: 195 | count = cjlog.jlog_ctx_read_interval(self.ctx, &self.begin, &self.end) 196 | if count > size: 197 | self.begin.log = self.end.log 198 | self.begin.marker = self.end.marker - size 199 | else: 200 | return False 201 | else: 202 | self.begin = self.end 203 | cjlog.JLOG_ID_ADVANCE(&self.begin) 204 | 205 | # checkpointing the file will cause all log segments without readers 206 | # to be cleaned up 207 | cjlog.jlog_ctx_read_checkpoint(self.ctx, &self.begin) 208 | return True 209 | 210 | 211 | def error_msg(self): 212 | return PyString_FromString(cjlog.jlog_ctx_err_string(self.ctx)) 213 | 214 | 215 | cdef class JLogWriter(BaseJLog): 216 | 217 | # path - the directory path to the jlog directory 218 | # subscriber - the subscriber for the jlog 219 | def __cinit__(self, path): 220 | self.path = path 221 | self.ctx = cjlog.jlog_new(path) 222 | self.ctx_initialized = False 223 | 224 | if self.ctx is NULL: 225 | JLogError("jlog writer failed to initialize") 226 | 227 | error = cjlog.jlog_ctx_init(self.ctx) 228 | if error and cjlog.jlog_ctx_err(self.ctx) is cjlog.JLOG_ERR_CREATE_EXISTS: 229 | raise JLogError("jlog writer failed to open %s" % path, 230 | self.error_msg()) 231 | 232 | # jlog requires a clean handle to open even after an init 233 | error = cjlog.jlog_ctx_close(self.ctx) 234 | if error: 235 | JLogError("jlog writer failed reinit after creation") 236 | self.ctx = cjlog.jlog_new(path) 237 | self.ctx_initialized = True 238 | 239 | error = cjlog.jlog_ctx_open_writer(self.ctx) 240 | if error: 241 | raise JLogError("jlog writer failed to open %s" % path, self.error_msg()) 242 | 243 | 244 | def write(self, msg_py): 245 | cdef char *msg 246 | cdef Py_ssize_t mlen 247 | PyString_AsStringAndSize(msg_py, &msg, &mlen) 248 | error = cjlog.jlog_ctx_write(self.ctx, msg, mlen) 249 | if error: 250 | raise JLogError("jlog write error", self.error_msg()) 251 | 252 | 253 | def __dealloc__(self): 254 | if self.ctx_initialized: 255 | cjlog.jlog_ctx_close(self.ctx) 256 | self.ctx_initialized = False 257 | 258 | 259 | def error_msg(self): 260 | return PyString_FromString(cjlog.jlog_ctx_err_string(self.ctx)) 261 | -------------------------------------------------------------------------------- /python/setup.py: -------------------------------------------------------------------------------- 1 | from distutils.core import setup 2 | from distutils.extension import Extension 3 | from Cython.Build import cythonize 4 | 5 | ext_modules=[ 6 | Extension( 7 | "jlog", 8 | sources = ["jlog.pyx"], 9 | libraries = ["jlog"], 10 | include_dirs = [ 11 | "/usr/local/include", 12 | "/usr/include"], 13 | library_dirs = [ 14 | "/usr/local/lib", 15 | "/usr/lib"] 16 | ) 17 | ] 18 | 19 | setup( 20 | name = "jlog", 21 | version = "1.0", 22 | description = "JLog Python Library", 23 | url = "https://labs.omniti.com/labs/jlog", 24 | ext_modules = cythonize(ext_modules) 25 | ) 26 | --------------------------------------------------------------------------------