├── AUTHORS ├── COPYING ├── ChangeLog ├── INSTALL ├── LICENSE ├── Makefile.am ├── NEWS ├── README ├── c6 ├── Makefile.am ├── bench │ ├── Makefile │ ├── oneday.dat │ ├── oneday.gpl │ ├── oneday.hist │ ├── oneday.m │ └── oneday.pdf ├── commands.c ├── commands.h ├── compress.c ├── hashtable │ ├── AUTHORS │ ├── COPYING │ ├── ChangeLog │ ├── INSTALL │ ├── Makefile.am │ ├── NEWS │ ├── README │ ├── configure.ac │ ├── hashtable-0.1.tar.gz │ ├── src │ │ ├── Makefile.am │ │ ├── hashtable.c │ │ ├── hashtable.h │ │ └── hashtable_private.h │ └── tests │ │ ├── Makefile.am │ │ ├── check_hashtable.1.c │ │ └── check_hashtable.c ├── intervals.c ├── intervals.h ├── logging.c ├── logging.h ├── mkcron.sh ├── mkmonit.sh ├── pbuf │ ├── Makefile.am │ └── rdb.proto ├── rdb.c ├── rdb.h ├── reading-client.c ├── reading-server.c ├── readingcount.txt ├── readingdb.c ├── readingdb.h ├── readingdb_py.h ├── rpc.c ├── rpc.h ├── sketch.c ├── sketch.h ├── stats-sock.c ├── stats.h ├── util-db.c ├── util.c └── util.h ├── configure.ac ├── debian ├── changelog ├── control ├── postinst └── rules ├── iface_bin ├── MANIFEST.in ├── Makefile ├── __init__.py ├── autotest.py ├── dataloader.py ├── debian │ ├── changelog │ ├── compat │ ├── control │ ├── copyright │ └── rules ├── fetch.py ├── import.py ├── import_tsdb.py ├── multi.c ├── readingdb.c ├── readingdb.i ├── readingdb_py.h ├── setup.py ├── simple.py └── test.py └── s3backup.sh /AUTHORS: -------------------------------------------------------------------------------- 1 | 2 | Stephen Dawson-Haggerty 3 | 4 | -------------------------------------------------------------------------------- /COPYING: -------------------------------------------------------------------------------- 1 | LICENSE -------------------------------------------------------------------------------- /ChangeLog: -------------------------------------------------------------------------------- 1 | 2012-04-20 Stephen Dawson-Haggerty 2 | * multi.c: add file with parallel downloader. 3 | 4 | * readingdb.i: change/break client API. db_setup() is now required to 5 | set connection params; whole query api is db_prev/db_next/db_query. 6 | Auto connects and returns lists of numpy matrices (creating them in 7 | python was really slow). 8 | 9 | 2011-05-25 Stephen Dawson-Haggerty 10 | * compress.c (val_compress, val_decompress): remove btree 11 | compression routines and replace with these since we're moving 12 | compression external to the library. 13 | 14 | * util-db.c (get, put, get_partial): enable compression before 15 | writing page to bdb. This changes the on-disk format. 16 | 17 | * reading-server.c (start_threads, deadlock_thread, 18 | checkpoint_thread): start deadlock detector, checkpointer, and 19 | archiver in-process to avoid the need for external scripts. 20 | 21 | 22 | 2011-05-14 Stephen Dawson-Haggerty 23 | * compress.c: add file with compression routines. Use combined 24 | delta-encoding + huffman tree from zlib. 25 | 26 | 2011-05-01 Stephen Dawson-Haggerty 27 | * reading-server.c (process_request, process_pbuf): switch to 28 | using google protocol buffers for wire format. deprecate and 29 | disable ascii protocol. 30 | 31 | 2011-03-30 Stephen Dawson-Haggerty 32 | * reading-server.c: Finally enable transaction updates to 33 | hopefully get rid of data losses. 34 | 35 | -------------------------------------------------------------------------------- /INSTALL: -------------------------------------------------------------------------------- 1 | 2 | Building and running the readingdb server 3 | ========================================= 4 | 5 | Build this project using automake as usual: 6 | 7 | $ ./configure 8 | $ make 9 | $ sudo make install 10 | 11 | This installs the `reading-server` binary. By default, it places the 12 | database volumes in /var/lib/readingdb. You can change it using an 13 | option to `configure`, or by passing a command-line argument. 14 | 15 | The install process also installs a monit script in /etc/monit/conf.d 16 | which can be used to start and stop the server if you have monit 17 | installed; a `monit start readingdb` will work. 18 | 19 | reading-server will attempt to drop to a "readingdb" user when 20 | started; this is convenient since monit often runs as root. If you 21 | want this to happen, you'll need to create this user (for instance, by 22 | doing `adduser --system readingdb`) if you are not using the binary 23 | package. You'll have to make sure that the data directory is readable 24 | and writable by this user if you do this. 25 | 26 | Building and installing the python bindings 27 | =========================================== 28 | 29 | In order to connect to the database, you'll need the python libraries. 30 | They must be installed separately: 31 | 32 | $ cd iface_bin 33 | $ sudo make install 34 | 35 | This will build and install the python bindings in your system-wide 36 | python install. 37 | 38 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2011, 2012, Regents of the University of California 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions 7 | * are met: 8 | * 9 | * - Redistributions of source code must retain the above copyright 10 | * notice, this list of conditions and the following disclaimer. 11 | * - Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the 14 | * distribution. 15 | * 16 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 19 | * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL 20 | * THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 21 | * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 22 | * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 23 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 25 | * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED 27 | * OF THE POSSIBILITY OF SUCH DAMAGE. 28 | */ 29 | 30 | -------------------------------------------------------------------------------- /Makefile.am: -------------------------------------------------------------------------------- 1 | 2 | SUBDIRS = c6 3 | EXTRA_DIST = iface_bin 4 | 5 | debbuild: 6 | dpkg-buildpackage -i -I -rfakeroot -uc -us -S 7 | -------------------------------------------------------------------------------- /NEWS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stevedh/readingdb/5ebc06c567becb84a7eeaade9d5ebce3f13210ef/NEWS -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | 2 | readingdb is a time-series database designed for efficiency and speed. 3 | 4 | Time series data is any data which is a series of (time, sequence, 5 | value) points. ReadingDB buckets and compresses your data using a 6 | delta encoding and zlib, and then writes this into a bdb installation 7 | with a bdb index. It uses the bdb transaction manager for write-ahead 8 | logging so that the volume will not become corrupted. 9 | 10 | To use it, first follow the instructions in INSTALL to build the 11 | database and the python interface module. The key objects in 12 | readingdb are "streams", which have an integer id. The readingdb 13 | python module lets you talk to the server process; traffic between the 14 | client and server is encoded using google protocol buffer definitions 15 | found in c6/pbuf. 16 | 17 | A simple python script for inserting data would be: 18 | 19 | -- 20 | import readingdb as rdb 21 | 22 | # specify default host/port 23 | rdb.db_setup('localhost', 4242) 24 | 25 | # create a connection 26 | db = rdb.db_open('localhost') 27 | 28 | # add data. the tuples are (timestamp, seqno, value) 29 | rdb.db_add(db, 1, [(x, 0, x) for x in xrange(0, 100)]) 30 | 31 | # read back the data we just wrote using the existing connection 32 | # the args are streamid, start_timestamp, end_timestamp 33 | print rdb.db_query(1, 0, 100, conn=db) 34 | # close 35 | rdb.db_close(db) 36 | 37 | # read back the data again using a connection pool this time. You can 38 | # specify a list of streamids to range-query multiple streams at once. 39 | rdb.db_query([1], 0, 100) 40 | -- 41 | 42 | ReadingDB supports efficient range querying and interation using the 43 | db_query, db_prev, and db_next operations; you can delete data with 44 | db_del. 45 | 46 | As you can see, db_query can re-use an existing connection if desired. 47 | As of April 20, 2012, the result of querying the TSDB is returned as a 48 | list of numpy matrices. This is because this is a very 49 | memory-efficient data structure, and creating numpy matrices using the 50 | c API is a lot more efficient than later doing it in python. 51 | 52 | If no connection is specified for db_query/db_prev/db_next, new 53 | connections will be opened to the host/port specified with db_setup. 54 | The most efficient way of downloading data from a large number of 55 | streams is to specify a list of streamids, which allows the client 56 | library to conduct multiple parallel downloads of the data. Using 57 | this approach we have observed readingdb easily saturating a 100Mb 58 | NIC. 59 | 60 | Sketches 61 | -------- 62 | 63 | As of 0.7.0 (October, 2014), readingdb supports computing sketches 64 | over the data in order to allow for more interactive performance on 65 | very large data sets. As of this release, it supports precomputing 66 | min, max, mean, and count at 5-minute, 15-minute, and 1-hour resolutions. 67 | 68 | By default, the behavor is the same as before. If sketches are 69 | enabled on reading-server (by starting it with the -r flag), it will 70 | stream a log of regions of streams which have new data to disk; these 71 | are the places where the sketches should be updated. 72 | 73 | The sketches may be updated using the new reading-sketch program; when 74 | run, it reads in the log, computes new sketches for the regions of 75 | time which have changed, and exits. This should be called 76 | periodically (e.g., by cron); the debian package installes a disabled 77 | crontab for this purpose into /etc/cron.d/readingdb. 78 | 79 | Clients may request sketches through the new sketch kwarg to rb_query. 80 | For instance: 81 | 82 | -- 83 | # load hourly minimia from stream number 2, over all time. 84 | min_data = rdb.db_query([2], 0, int(time.time()), sketch=("min", 3600))[0] 85 | -- 86 | 87 | The value should be a tuple of (sketchname, resolution (in seconds)). 88 | The server will raise an exception on the client if an invalid sketch 89 | is passed. 90 | 91 | Note: older servers may instead return the underlying data. older 92 | client libraries will raise an exception since sketch is not a valid 93 | kwarg there. 94 | 95 | Dependencies 96 | ------------ 97 | 98 | For reading-server 99 | libdb4.8, libdb4.8-dev (berkeley database) 100 | libprotobuf, libprotobuf-dev (google protocol buffers) 101 | libprotoc6, libprotoc6-dev (c bindings for protobufs) 102 | zlib, zlib-dev (for compression) 103 | gcc, make, automake 104 | 105 | For python bindings 106 | python, python-dev, python-numpy (python deps) 107 | libdb4.8-dev 108 | swig (interface generator) 109 | -------------------------------------------------------------------------------- /c6/Makefile.am: -------------------------------------------------------------------------------- 1 | 2 | SUBDIRS = hashtable pbuf 3 | 4 | bin_PROGRAMS = reading-server reading-sketch 5 | 6 | # reading_server_LDFLAGS = -lz 7 | reading_server_LDADD = hashtable/src/libhashtable.a 8 | reading_server_CFLAGS = -Ihashtable/src 9 | reading_sketch_LDADD = hashtable/src/libhashtable.a 10 | reading_sketch_CFLAGS = -Ihashtable/src 11 | reading_server_SOURCES = util.c util-db.c reading-server.c logging.c stats-sock.c \ 12 | rpc.c pbuf/rdb.pb-c.c compress.c commands.c rdb.c 13 | reading_sketch_SOURCES = util.c util-db.c logging.c stats-sock.c \ 14 | rpc.c pbuf/rdb.pb-c.c compress.c commands.c rdb.c sketch.c intervals.c 15 | noinst_HEADERS = config.h logging.h readingdb.h readingdb_py.h \ 16 | rpc.h stats.h util.h pbuf/rdb.proto commands.h rdb.h 17 | 18 | # noinst_PROGRAMS = compress 19 | # compress_SOURCES = compress.c pbuf/rdb.pb-c.c rpc.c util.c logging.c 20 | # compress_LDFLAGS = -lprotobuf-c -lz 21 | 22 | install-data-local: 23 | mkdir -p $(DESTDIR)/$(localstatedir)/lib/readingdb 24 | mkdir -p $(DESTDIR)/$(sysconfdir)/monit/conf.d 25 | mkdir -p $(DESTDIR)/$(sysconfdir)/cron.d 26 | sh mkmonit.sh $(localstatedir) $(bindir) > $(DESTDIR)/$(sysconfdir)/monit/conf.d/readingdb 27 | sh mkcron.sh $(localstatedir) $(bindir) > $(DESTDIR)/$(sysconfdir)/cron.d/readingdb 28 | -------------------------------------------------------------------------------- /c6/bench/Makefile: -------------------------------------------------------------------------------- 1 | 2 | oneday.pdf: oneday.hist oneday.gpl 3 | gnuplot oneday.gpl 4 | 5 | oneday.hist: oneday.dat oneday.m 6 | ./oneday.m 7 | -------------------------------------------------------------------------------- /c6/bench/oneday.gpl: -------------------------------------------------------------------------------- 1 | #!/opt/local/bin/gnuplot -persist 2 | # 3 | # 4 | # G N U P L O T 5 | # Version 4.2 patchlevel 5 6 | # last modified Mar 2009 7 | # System: Darwin 10.5.0 8 | # 9 | # Copyright (C) 1986 - 1993, 1998, 2004, 2007 - 2009 10 | # Thomas Williams, Colin Kelley and many others 11 | # 12 | # Type `help` to access the on-line reference manual. 13 | # The gnuplot FAQ is available from http://www.gnuplot.info/faq/ 14 | # 15 | # Send bug reports and suggestions to 16 | # 17 | set terminal pdf noenhanced fname 'Helvetica' fsize 6 linewidth 1.0 18 | set output 'oneday.pdf' 19 | unset clip points 20 | set clip one 21 | unset clip two 22 | set bar 1.000000 23 | set border 31 front linetype -1 linewidth 1.000 24 | set xdata 25 | set ydata 26 | set zdata 27 | set x2data 28 | set y2data 29 | set timefmt x "%d/%m/%y,%H:%M" 30 | set timefmt y "%d/%m/%y,%H:%M" 31 | set timefmt z "%d/%m/%y,%H:%M" 32 | set timefmt x2 "%d/%m/%y,%H:%M" 33 | set timefmt y2 "%d/%m/%y,%H:%M" 34 | set timefmt cb "%d/%m/%y,%H:%M" 35 | set boxwidth 36 | set style fill empty border 37 | set style rectangle back fc lt -3 fillstyle solid 1.00 border -1 38 | set dummy x,y 39 | set format x "% g" 40 | set format y "% g" 41 | set format x2 "% g" 42 | set format y2 "% g" 43 | set format z "% g" 44 | set format cb "% g" 45 | set angles radians 46 | set grid nopolar 47 | set grid xtics nomxtics ytics nomytics noztics nomztics \ 48 | nox2tics nomx2tics noy2tics nomy2tics nocbtics nomcbtics 49 | set grid layerdefault linetype 0 linewidth 1.000, linetype 0 linewidth 1.000 50 | set key title "" 51 | unset key 52 | unset label 53 | unset arrow 54 | set style increment default 55 | unset style line 56 | unset style arrow 57 | set style histogram clustered gap 2 title offset character 0, 0, 0 58 | unset logscale 59 | set offsets 0, 0, 0, 0 60 | set pointsize 1 61 | set encoding default 62 | unset polar 63 | unset parametric 64 | unset decimalsign 65 | set view 60, 30, 1, 1 66 | set view 67 | set samples 100, 100 68 | set isosamples 10, 10 69 | set surface 70 | unset contour 71 | set clabel '%8.3g' 72 | set mapping cartesian 73 | set datafile separator whitespace 74 | unset hidden3d 75 | set cntrparam order 4 76 | set cntrparam linear 77 | set cntrparam levels auto 5 78 | set cntrparam points 5 79 | set size ratio 0 1,1 80 | set origin 0,0 81 | set style data points 82 | set style function lines 83 | set xzeroaxis linetype -2 linewidth 1.000 84 | set yzeroaxis linetype -2 linewidth 1.000 85 | set zzeroaxis linetype -2 linewidth 1.000 86 | set x2zeroaxis linetype -2 linewidth 1.000 87 | set y2zeroaxis linetype -2 linewidth 1.000 88 | set ticslevel 0.5 89 | set mxtics default 90 | set mytics default 91 | set mztics default 92 | set mx2tics default 93 | set my2tics default 94 | set mcbtics default 95 | set xtics border in scale 1,0.5 mirror norotate offset character 0, 0, 0 96 | set xtics autofreq norangelimit 97 | set ytics border in scale 1,0.5 mirror norotate offset character 0, 0, 0 98 | set ytics autofreq norangelimit 99 | set ztics border in scale 1,0.5 nomirror norotate offset character 0, 0, 0 100 | set ztics autofreq norangelimit 101 | set nox2tics 102 | set noy2tics 103 | set cbtics border in scale 1,0.5 mirror norotate offset character 0, 0, 0 104 | set cbtics autofreq norangelimit 105 | set title "" 106 | set title offset character 0, 0, 0 font "" norotate 107 | set timestamp bottom 108 | set timestamp "" 109 | set timestamp offset character 0, 0, 0 font "" norotate 110 | set rrange [ * : * ] noreverse nowriteback # (currently [0.00000:10.0000] ) 111 | set trange [ * : * ] noreverse nowriteback # (currently [-5.00000:5.00000] ) 112 | set urange [ * : * ] noreverse nowriteback # (currently [-5.00000:5.00000] ) 113 | set vrange [ * : * ] noreverse nowriteback # (currently [-5.00000:5.00000] ) 114 | set xlabel "Compression achieved" 115 | set xlabel offset character 0, 0, 0 font "" textcolor lt -1 norotate 116 | set x2label "" 117 | set x2label offset character 0, 0, 0 font "" textcolor lt -1 norotate 118 | set xrange [ * : * ] noreverse nowriteback # (currently [-10.0000:10.0000] ) 119 | set x2range [ * : * ] noreverse nowriteback # (currently [-10.0000:10.0000] ) 120 | set ylabel "Number of streams" 121 | set ylabel offset character 0, 0, 0 font "" textcolor lt -1 rotate by 90 122 | set y2label "" 123 | set y2label offset character 0, 0, 0 font "" textcolor lt -1 rotate by 90 124 | set yrange [ * : * ] noreverse nowriteback # (currently [-10.0000:10.0000] ) 125 | set y2range [ * : * ] noreverse nowriteback # (currently [-10.0000:10.0000] ) 126 | set zlabel "" 127 | set zlabel offset character 0, 0, 0 font "" textcolor lt -1 norotate 128 | set zrange [ * : * ] noreverse nowriteback # (currently [-10.0000:10.0000] ) 129 | set cblabel "" 130 | set cblabel offset character 0, 0, 0 font "" textcolor lt -1 rotate by 90 131 | set cbrange [ * : * ] noreverse nowriteback # (currently [-10.0000:10.0000] ) 132 | set zero 1e-08 133 | set lmargin -1 134 | set bmargin -1 135 | set rmargin -1 136 | set tmargin -1 137 | set locale "C" 138 | set pm3d explicit at s 139 | set pm3d scansautomatic 140 | set pm3d interpolate 1,1 flush begin noftriangles nohidden3d corners2color mean 141 | set palette positive nops_allcF maxcolors 0 gamma 1.5 color model RGB 142 | set palette rgbformulae 7, 5, 15 143 | set colorbox default 144 | set colorbox vertical origin screen 0.9, 0.2, 0 size screen 0.05, 0.6, 0 front bdefault 145 | set loadpath 146 | set fontpath 147 | set fit noerrorvariables 148 | GNUTERM = "aqua" 149 | plot "oneday.hist" with boxes 150 | # EOF 151 | -------------------------------------------------------------------------------- /c6/bench/oneday.hist: -------------------------------------------------------------------------------- 1 | # Created by Octave 3.2.2, Thu May 26 16:05:11 2011 PDT 2 | # name: M 3 | # type: matrix 4 | # rows: 100 5 | # columns: 2 6 | 0.024442920659252 316 7 | 0.036899664058556 699 8 | 0.04935640745786 328 9 | 0.061813150857164 355 10 | 0.07426989425646799 298 11 | 0.08672663765577199 282 12 | 0.09918338105507601 217 13 | 0.11164012445438 165 14 | 0.124096867853684 106 15 | 0.136553611252988 81 16 | 0.149010354652292 57 17 | 0.161467098051596 76 18 | 0.1739238414509 48 19 | 0.186380584850204 39 20 | 0.198837328249508 63 21 | 0.211294071648812 62 22 | 0.223750815048116 74 23 | 0.23620755844742 418 24 | 0.248664301846724 1824 25 | 0.261121045246028 1220 26 | 0.273577788645332 314 27 | 0.286034532044636 158 28 | 0.29849127544394 109 29 | 0.310948018843244 92 30 | 0.323404762242548 64 31 | 0.335861505641852 56 32 | 0.348318249041156 48 33 | 0.36077499244046 64 34 | 0.373231735839764 75 35 | 0.385688479239068 76 36 | 0.398145222638372 87 37 | 0.410601966037676 90 38 | 0.42305870943698 87 39 | 0.435515452836284 109 40 | 0.447972196235588 119 41 | 0.460428939634892 91 42 | 0.472885683034196 88 43 | 0.4853424264335 106 44 | 0.497799169832804 95 45 | 0.5102559132321079 110 46 | 0.522712656631412 85 47 | 0.5351694000307159 53 48 | 0.5476261434300199 64 49 | 0.5600828868293239 66 50 | 0.572539630228628 59 51 | 0.584996373627932 55 52 | 0.597453117027236 52 53 | 0.6099098604265399 63 54 | 0.622366603825844 38 55 | 0.634823347225148 48 56 | 0.6472800906244519 62 57 | 0.6597368340237559 45 58 | 0.67219357742306 47 59 | 0.684650320822364 43 60 | 0.697107064221668 31 61 | 0.709563807620972 33 62 | 0.7220205510202758 19 63 | 0.7344772944195799 19 64 | 0.7469340378188839 21 65 | 0.7593907812181879 19 66 | 0.771847524617492 26 67 | 0.784304268016796 29 68 | 0.7967610114160999 27 69 | 0.8092177548154039 29 70 | 0.8216744982147079 16 71 | 0.834131241614012 21 72 | 0.846587985013316 26 73 | 0.85904472841262 25 74 | 0.8715014718119241 19 75 | 0.8839582152112279 9 76 | 0.8964149586105319 5 77 | 0.9088717020098359 1 78 | 0.9213284454091399 1 79 | 0.933785188808444 1 80 | 0.946241932207748 0 81 | 0.9586986756070519 0 82 | 0.9711554190063559 0 83 | 0.98361216240566 0 84 | 0.996068905804964 0 85 | 1.008525649204268 1 86 | 1.020982392603572 2 87 | 1.033439136002876 0 88 | 1.04589587940218 0 89 | 1.058352622801484 0 90 | 1.070809366200788 0 91 | 1.083266109600092 1 92 | 1.095722852999396 0 93 | 1.1081795963987 0 94 | 1.120636339798004 0 95 | 1.133093083197308 0 96 | 1.145549826596612 0 97 | 1.158006569995916 0 98 | 1.17046331339522 0 99 | 1.182920056794524 0 100 | 1.195376800193828 0 101 | 1.207833543593132 2 102 | 1.220290286992436 0 103 | 1.23274703039174 0 104 | 1.245203773791044 2 105 | 1.257660517190348 1 106 | -------------------------------------------------------------------------------- /c6/bench/oneday.m: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env octave -q 2 | # -*- octave -*- 3 | 4 | x = load("oneday.dat"); 5 | [s, h] = hist(x(:,2), 100); 6 | M = [h' s']; 7 | save "oneday.hist" M 8 | -------------------------------------------------------------------------------- /c6/bench/oneday.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stevedh/readingdb/5ebc06c567becb84a7eeaade9d5ebce3f13210ef/c6/bench/oneday.pdf -------------------------------------------------------------------------------- /c6/commands.c: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | #include "readingdb.h" 8 | #include "util.h" 9 | #include "logging.h" 10 | #include "pbuf/rdb.pb-c.h" 11 | #include "commands.h" 12 | 13 | 14 | extern DB_ENV *env; 15 | extern sig_atomic_t do_shutdown; 16 | 17 | void query(DB *dbp, Query *q, Response *r, enum query_action action) { 18 | int ret; 19 | DBC *cursorp; 20 | struct rec_key k; 21 | struct rec_val v; 22 | unsigned long long starttime, endtime; 23 | int streamid; 24 | int cursor_flags = 0; 25 | 26 | streamid = q->streamid; 27 | starttime = q->starttime; 28 | endtime = q->endtime; 29 | debug("starting query id: %i start: %i end: %i\n", streamid, starttime, endtime); 30 | 31 | /* set up the query key */ 32 | k.stream_id = streamid; 33 | k.timestamp = starttime - (starttime % bucket_sizes[NBUCKETSIZES-1]); 34 | 35 | switch (action) { 36 | case QUERY_COUNT: 37 | /* initialize our return with one guy in it */ 38 | reading__init(r->data->data[0]); 39 | reading__init(r->data->data[1]); 40 | r->data->data[0]->timestamp = 41 | r->data->data[1]->timestamp = 42 | time(NULL); 43 | r->data->n_data = 2; 44 | break; 45 | case QUERY_DATA: 46 | break; 47 | default: 48 | /* invalid request type */ 49 | r->error = RESPONSE__ERROR_CODE__FAIL_PARAM; 50 | return; 51 | } 52 | 53 | ret = dbp->cursor(dbp, NULL, &cursorp, cursor_flags); 54 | if (cursorp == NULL) { 55 | dbp->err(dbp, ret, "cursor"); 56 | r->error = RESPONSE__ERROR_CODE__FAIL; 57 | return; 58 | } 59 | 60 | if (get_partial(cursorp, DB_SET_RANGE, &k, &v, sizeof(struct rec_val), 0) != 0) { 61 | goto done; 62 | } 63 | 64 | do { 65 | int i; 66 | int read_recs = min(v.n_valid, MAXRECS - r->data->n_data); 67 | struct point bucket[MAXBUCKETRECS + NBUCKETSIZES]; 68 | debug("examining record start: 0x%x length: %i streamid: %i\n", 69 | k.timestamp, v.period_length, k.stream_id); 70 | if (streamid != k.stream_id) break; 71 | if (k.timestamp >= endtime) break; 72 | if (r->data->n_data >= MAXRECS) break; 73 | if (!valid_bucketsize(v.period_length) || v.n_valid > MAXBUCKETRECS + NBUCKETSIZES) { 74 | warn("length is invalid: %i! streamid: %i start: %i nvalid: %i\n", 75 | v.period_length, k.stream_id, k.timestamp, v.n_valid); 76 | goto next; 77 | } 78 | 79 | switch (action) { 80 | case QUERY_DATA: 81 | if (get_partial(cursorp, DB_SET, &k, bucket, 82 | sizeof(struct point) * read_recs, 83 | sizeof(struct rec_val)) < 0) { 84 | goto next; 85 | } 86 | for (i = 0; i < read_recs; i++) { 87 | if (bucket[i].timestamp >= starttime && 88 | bucket[i].timestamp < endtime) { 89 | _rpc_copy_reading(r->data->data[r->data->n_data++], &bucket[i]); 90 | } 91 | } 92 | debug("query: added %i/%i records\n", r->data->n_data, v.n_valid); 93 | break; 94 | case QUERY_COUNT: 95 | r->data->data[0]->value += v.n_valid; 96 | r->data->data[1]->value += 1; 97 | break; 98 | } 99 | next: 100 | ; 101 | } while (get_partial(cursorp, DB_NEXT, &k, &v, 102 | sizeof(struct rec_val), 0) == 0); 103 | 104 | debug("returning %i records\n", r->data->n_data); 105 | r->error = RESPONSE__ERROR_CODE__OK; 106 | 107 | done: 108 | cursorp->close(cursorp); 109 | } 110 | 111 | void query_nearest(DB *dbp, Nearest *n, Response *rs, enum query_action action) { 112 | int ret; 113 | DBC *cursorp; 114 | struct rec_key k; 115 | struct rec_val v; 116 | unsigned long long starttime; 117 | int streamid, i, rc; 118 | int direction = n->direction == NEAREST__DIRECTION__NEXT ? DB_NEXT : DB_PREV; 119 | int ret_n = n->has_n ? min(MAXRECS, n->n) : 1; 120 | 121 | streamid = n->streamid; 122 | starttime = n->reference; 123 | debug("starting nearest query id: %i reference: %i\n", streamid, starttime); 124 | 125 | /* set up the query key */ 126 | k.stream_id = streamid; 127 | if (direction == DB_NEXT) { 128 | k.timestamp = starttime - (starttime % bucket_sizes[NBUCKETSIZES-1]); 129 | } else if (direction == DB_PREV) { 130 | k.timestamp = starttime - (starttime % bucket_sizes[0]); 131 | } 132 | 133 | rs->error = RESPONSE__ERROR_CODE__OK; 134 | rs->data->n_data = 0; 135 | 136 | ret = dbp->cursor(dbp, NULL, &cursorp, 0); 137 | if (cursorp == NULL) { 138 | dbp->err(dbp, ret, "cursor"); 139 | return; 140 | } 141 | 142 | if ((rc = get_partial(cursorp, DB_SET_RANGE, &k, &v, sizeof(struct rec_val), 0)) < 0) { 143 | // this might put us at the end of the database... 144 | if (direction == DB_PREV && rc == DB_NOTFOUND) goto next; 145 | else goto done; 146 | } 147 | 148 | do { 149 | struct point bucket[MAXBUCKETRECS + NBUCKETSIZES]; 150 | debug("examining record start: %i length: %i streamid: %i\n", 151 | k.timestamp, v.period_length, k.stream_id); 152 | if ((direction == DB_NEXT && k.stream_id > streamid) || 153 | (direction == DB_PREV && k.stream_id < streamid)) break; 154 | if ((direction == DB_NEXT && k.timestamp + v.period_length < starttime) || 155 | (direction == DB_PREV && k.timestamp > starttime)) goto next; 156 | if (get_partial(cursorp, DB_SET, &k, bucket, 157 | sizeof(struct point) * v.n_valid, 158 | sizeof(struct rec_val)) < 0) { 159 | goto next; 160 | } 161 | 162 | for (i = (direction == DB_NEXT ? 0 : v.n_valid - 1); 163 | (direction == DB_NEXT ? i < v.n_valid : i >= 0); 164 | (direction == DB_NEXT ? i++ : i--)) { 165 | if (k.stream_id == streamid && 166 | ((direction == DB_NEXT && bucket[i].timestamp > starttime) || 167 | (direction == DB_PREV && bucket[i].timestamp < starttime))) { 168 | /* return */ 169 | _rpc_copy_reading(rs->data->data[rs->data->n_data++], &bucket[i]); 170 | if (rs->data->n_data >= ret_n) 171 | goto done; 172 | } 173 | } 174 | 175 | next: 176 | ; 177 | } while (get_partial(cursorp, direction, &k, &v, 178 | sizeof(struct rec_val), 0) == 0); 179 | done: 180 | cursorp->close(cursorp); 181 | } 182 | 183 | int del(DB *dbp, Delete *d) { 184 | int ret; 185 | DBC *cursorp; 186 | DB_TXN *tid = NULL; 187 | struct rec_key k; 188 | struct rec_val v; 189 | unsigned long long starttime, endtime; 190 | int streamid; 191 | 192 | streamid = d->streamid; 193 | starttime = d->starttime; 194 | endtime = d->endtime; 195 | debug("starting query id: %i start: %i end: %i\n", streamid, starttime, endtime); 196 | 197 | /* set up the query key */ 198 | k.stream_id = streamid; 199 | k.timestamp = starttime - (starttime % bucket_sizes[NBUCKETSIZES-1]); 200 | 201 | if ((ret = env->txn_begin(env, NULL, &tid, 0)) != 0) { 202 | error("txn_begin: %s\n", db_strerror(ret)); 203 | return -1; 204 | } 205 | 206 | ret = dbp->cursor(dbp, tid, &cursorp, 0); 207 | if (cursorp == NULL) { 208 | dbp->err(dbp, ret, "cursor"); 209 | goto abort; 210 | } 211 | 212 | 213 | if (get_partial(cursorp, DB_SET_RANGE, &k, &v, sizeof(struct rec_val), 0) != 0) { 214 | goto abort; 215 | } 216 | 217 | do { 218 | debug("delete: examining record start: 0x%x length: %i streamid: %i=%i\n", 219 | k.timestamp, v.period_length, k.stream_id, streamid); 220 | if (k.stream_id < streamid) goto next; 221 | else if (k.stream_id > streamid) break; 222 | if (k.timestamp < starttime) goto next;; 223 | if (k.timestamp + v.period_length > endtime) break; 224 | if (!valid_bucketsize(v.period_length) || v.n_valid > MAXBUCKETRECS + NBUCKETSIZES) { 225 | warn("delete: length is invalid: %i! streamid: %i start: %i nvalid: %i\n", 226 | v.period_length, k.stream_id, k.timestamp, v.n_valid); 227 | } 228 | 229 | if (cursorp->c_del(cursorp, 0) != 0) { 230 | goto abort; 231 | } 232 | debug("delete: removed %i records\n", v.n_valid); 233 | next: 234 | debug("checking next\n"); 235 | } while (get_partial(cursorp, DB_NEXT, &k, &v, 236 | sizeof(struct rec_val), 0) == 0); 237 | 238 | cursorp->close(cursorp); 239 | 240 | if ((ret = tid->commit(tid, 0)) != 0) { 241 | fatal("transaction commit failed: %s\n", db_strerror(ret)); 242 | do_shutdown = 1; 243 | } 244 | 245 | return 0; 246 | 247 | abort: 248 | cursorp->close(cursorp); 249 | warn("delete: Aborting transaction\n"); 250 | 251 | if ((ret = tid->abort(tid)) != 0) { 252 | fatal("delete: Could not abort transaction: %s\n", db_strerror(ret)); 253 | // do_shutdown = 1; 254 | assert(0); 255 | } 256 | return -1; 257 | 258 | } 259 | -------------------------------------------------------------------------------- /c6/commands.h: -------------------------------------------------------------------------------- 1 | #ifndef COMMANDS_H 2 | #define COMMANDS_H 3 | 4 | #include "pbuf/rdb.pb-c.h" 5 | 6 | #define MAXRECS 10000 // (((MAXQUERYSET) < (IPC_MAX_RECORDS)) ? (MAXQUERYSET) : (IPC_MAX_RECORDS)) 7 | #define min(X,Y) ((X) < (Y) ? (X) : (Y)) 8 | 9 | enum query_action { 10 | QUERY_DATA = 1, 11 | QUERY_COUNT = 2, 12 | }; 13 | 14 | /* 15 | * Conduct a range query of time-series data 16 | */ 17 | void query(DB *dbp, Query *q, Response *r, enum query_action action); 18 | 19 | /* 20 | * Conduct a query relative to a reference time 21 | */ 22 | void query_nearest(DB *dbp, Nearest *n, Response *rs, enum query_action action); 23 | 24 | /* 25 | * Delete data within a range for a stream 26 | */ 27 | int del(DB *dbp, Delete *d); 28 | 29 | #endif 30 | -------------------------------------------------------------------------------- /c6/compress.c: -------------------------------------------------------------------------------- 1 | 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #include "config.h" 13 | #if HAVE_ENDIAN_H 14 | #include 15 | #elif HAVE_LIBKERN_OSBYTEORDER_H 16 | #include 17 | #define htobe64(X) OSSwapHostToBigInt64(X) 18 | #else 19 | #error "No endian header found" 20 | #endif 21 | 22 | 23 | #include "readingdb.h" 24 | #include "rpc.h" 25 | #include "pbuf/rdb.pb-c.h" 26 | #include "logging.h" 27 | 28 | #define ZLIB_LEVEL Z_HUFFMAN_ONLY 29 | 30 | static void _free_db_rec(DatabaseRecord *db) { 31 | free(db->deltas[0]); 32 | free(db->deltas); 33 | if (db->first) free(db->first); 34 | free(db); 35 | } 36 | 37 | static DatabaseRecord * _alloc_db_rec(int n) { 38 | DatabaseRecord *rec; 39 | DatabaseDelta *deltas; 40 | DatabaseDelta **deltas_vec; 41 | Reading *reading; 42 | int i; 43 | 44 | rec = malloc(sizeof(DatabaseRecord)); 45 | deltas = malloc(sizeof(DatabaseDelta) * n); 46 | deltas_vec = malloc(sizeof(DatabaseDelta *) * n); 47 | reading = malloc(sizeof(Reading)); 48 | if (!rec || !deltas || !deltas_vec || !reading) { 49 | if (rec) free(rec); 50 | if (deltas) free(deltas); 51 | if (deltas_vec) free(deltas_vec); 52 | if (reading) free(reading); 53 | return NULL; 54 | } 55 | 56 | database_record__init(rec); 57 | reading__init(reading); 58 | rec->first = reading; 59 | rec->deltas = deltas_vec; 60 | rec->n_deltas = n; 61 | 62 | for (i = 0; i < n; i++) { 63 | database_delta__init(&deltas[i]); 64 | deltas_vec[i] = &deltas[i]; 65 | } 66 | return rec; 67 | } 68 | 69 | static inline int64_t DOUBLE_DELTA(double x, double y) { 70 | int64_t i1, i2; 71 | memcpy(&i1, &x, sizeof(i1)); 72 | memcpy(&i2, &y, sizeof(i2)); 73 | i1 = htobe64(i1); 74 | i2 = htobe64(i2); 75 | 76 | // pf(x), pf(y), printf("\n"); 77 | // pi(i1 - i2), printf("\n"); 78 | 79 | return i1 - i2; 80 | } 81 | 82 | void pf(double delta) { 83 | int i; 84 | for (i = 0; i < 8; i++) { 85 | printf("%02x ", ((uint8_t *)&delta)[i]); 86 | } 87 | } 88 | 89 | void pi(uint64_t delta) { 90 | int i; 91 | for (i = 0; i < 8; i++) { 92 | printf("%02x ", ((uint8_t *)&delta)[i]); 93 | } 94 | } 95 | 96 | static inline double DOUBLE_ADD(double x, int64_t delta) { 97 | int64_t i1; 98 | double ret; 99 | memcpy(&i1, &x, sizeof(i1)); 100 | // pi(i1), pi(delta), printf("\n"); 101 | i1 = htobe64(i1); 102 | i1 += delta; 103 | i1 = htobe64(i1); 104 | // pi(i1), printf("\n"); 105 | memcpy(&ret, &i1, sizeof(ret)); 106 | return ret; 107 | } 108 | 109 | // #define DOUBLE_DELTA(X,Y) (((uint64_t)((X) - (Y)))) 110 | #define HAS_MAX(X) ((X) < ((double)(LLONG_MAX - 1))) 111 | #define HAS_MIN(X) ((X) < ((double)(LLONG_MIN + 1))) 112 | 113 | static int pbuf_compress(struct rec_val *v, void *buf, int outbuf) { 114 | DatabaseRecord *db = _alloc_db_rec(v->n_valid + 1); 115 | double bottom = LLONG_MIN + 1, top = LLONG_MAX - 1; 116 | int i; 117 | uint32_t last_ts = 0; 118 | int64_t last_delta = 0; 119 | int pack_size; 120 | if (!db) return 0; 121 | 122 | // fprintf(stderr, "pbuf_compress: %lu %lu\n", ntohl(k->stream_id), ntohl(k->timestamp)); 123 | db->period_length = v->period_length; 124 | 125 | // printf("here:, n_valid: %i\n", v->n_valid); 126 | 127 | if (v->n_valid == 0) { 128 | free(db->first); 129 | db->first = NULL; 130 | db->n_deltas = 0; 131 | } else { 132 | db->n_deltas = v->n_valid - 1; 133 | db->first->timestamp = v->data[0].timestamp; 134 | db->first->value = v->data[0].reading; 135 | 136 | if (v->data[0].reading_sequence != 0) { 137 | db->first->has_seqno = 1; 138 | db->first->seqno = v->data[0].reading_sequence; 139 | } 140 | if (v->data[0].min > bottom) { 141 | db->first->has_min = 1; 142 | db->first->min = v->data[0].min; 143 | } 144 | if (v->data[0].max < top) { 145 | db->first->has_max = 1; 146 | db->first->max = v->data[0].max; 147 | } 148 | 149 | for (i = 1; i < v->n_valid; i++) { 150 | /* include timestamp deltas if they're different */ 151 | if (v->data[i].timestamp - v->data[i-1].timestamp != last_ts) { 152 | last_ts = db->deltas[i-1]->timestamp = v->data[i].timestamp - v->data[i-1].timestamp; 153 | db->deltas[i-1]->has_timestamp = 1; 154 | } 155 | 156 | /* the reading is also packed if we're not the same */ 157 | if (DOUBLE_DELTA(v->data[i].reading, v->data[i-1].reading) != last_delta) { 158 | last_delta = db->deltas[i-1]->value = 159 | DOUBLE_DELTA(v->data[i].reading, v->data[i-1].reading); 160 | db->deltas[i-1]->has_value = 1; 161 | } 162 | 163 | if (v->data[i].reading_sequence != 0) { 164 | db->deltas[i-1]->has_seqno = 1; 165 | if (v->data[i - 1].reading_sequence != 0) 166 | db->deltas[i-1]->seqno = v->data[i].reading_sequence - 167 | v->data[i-1].reading_sequence; 168 | else 169 | db->deltas[i-1]->seqno = v->data[i].reading_sequence; 170 | } 171 | 172 | /* include deltas or full readings for min and max */ 173 | if (HAS_MIN(v->data[i].min)) { 174 | if (HAS_MIN(v->data[i-1].min)) { 175 | // printf("HAS MIN"); 176 | db->deltas[i-1]->min_delta = DOUBLE_DELTA(v->data[i].min, v->data[i-1].min); 177 | db->deltas[i-1]->has_min_delta = 1; 178 | } else { 179 | db->deltas[i-1]->min = v->data[i].min; 180 | db->deltas[i-1]->has_min = 1; 181 | } 182 | } 183 | if (HAS_MAX(v->data[i].max)) { 184 | if (HAS_MAX(v->data[i-1].max)) { 185 | // printf("HAS MAX\n"); 186 | db->deltas[i-1]->max_delta = DOUBLE_DELTA(v->data[i].max, v->data[i-1].max); 187 | db->deltas[i-1]->has_max_delta = 1; 188 | } else { 189 | db->deltas[i-1]->max = v->data[i].max; 190 | db->deltas[i-1]->has_max = 1; 191 | } 192 | } 193 | } 194 | } 195 | 196 | pack_size = database_record__get_packed_size(db); 197 | if (outbuf < pack_size) { 198 | _free_db_rec(db); 199 | return - pack_size; 200 | } 201 | database_record__pack(db, buf); 202 | _free_db_rec(db); 203 | return pack_size; 204 | } 205 | 206 | 207 | int val_compress(struct rec_val *v, void *buf, int len) { 208 | char pbuf_buf[COMPRESS_WORKING_BUF]; 209 | int c_sz, ret; 210 | 211 | c_sz = pbuf_compress(v, pbuf_buf, sizeof(pbuf_buf)); 212 | // fprintf(stderr, "compressed-size: %i bound: %i len: %i\n", c_sz, compressBound(c_sz), len); 213 | if (c_sz < 0) { 214 | assert(0); 215 | } 216 | 217 | assert(compressBound(c_sz) < len); 218 | uLongf sz = len; 219 | if ((ret = compress2(buf, &sz, pbuf_buf, c_sz, ZLIB_LEVEL)) < 0) { 220 | fatal("compress2 failed: %i (dest: %lu src: %lu bound: %lu)\n", 221 | ret, sz, c_sz, compressBound(c_sz)); 222 | assert(0); 223 | } 224 | debug("compress2: %i -> %i\n", c_sz, sz); 225 | 226 | return (int)sz; 227 | } 228 | 229 | static int pbuf_decompress(void *buf, int len, struct rec_val *val, int sz) { 230 | DatabaseRecord *rec; 231 | uint32_t last_ts = 0; 232 | int64_t last_delta = 0; 233 | int i, n_present; 234 | int unpack_sz; 235 | rec = database_record__unpack(NULL, len, buf); 236 | unpack_sz = sizeof(struct rec_val) + (sizeof(struct point) * 237 | ((rec->first == NULL ? 0 : 1) + 238 | rec->n_deltas)); 239 | 240 | if (!rec) 241 | assert(0); 242 | 243 | // fprintf(stderr, "pbuf_decompress: %lu %lu\n", rec->k_streamid, rec->k_timestamp); 244 | debug("unpack_sz: %i, sz: %i\n", unpack_sz, sz); 245 | 246 | val->period_length = rec->period_length; 247 | val->n_valid = rec->n_deltas + (rec->first == NULL ? 0 : 1); 248 | 249 | // printf("read back n_valid: %i\n", val->n_valid); 250 | 251 | if (val->n_valid == 0) { 252 | database_record__free_unpacked(rec, NULL); 253 | return unpack_sz; 254 | } 255 | 256 | /* if there's at least one record first is present */ 257 | _rpc_copy_records(&val->data[0], &rec->first, 1); 258 | 259 | for (i = 0; i < rec->n_deltas; i++) { 260 | if (rec->deltas[i]->has_timestamp) { 261 | last_ts = rec->deltas[i]->timestamp; 262 | } 263 | val->data[i+1].timestamp = val->data[i].timestamp + last_ts; 264 | 265 | if (rec->deltas[i]->has_value) { 266 | last_delta = rec->deltas[i]->value; 267 | } 268 | val->data[i+1].reading = DOUBLE_ADD(val->data[i].reading, last_delta); 269 | 270 | if (rec->deltas[i]->has_seqno) { 271 | if (val->data[i].reading_sequence == 0) { 272 | val->data[i+1].reading_sequence = rec->deltas[i]->seqno; 273 | } else { 274 | val->data[i+1].reading_sequence = val->data[i].reading_sequence + 275 | rec->deltas[i]->seqno; 276 | } 277 | } else { 278 | val->data[i+1].reading_sequence = 0; 279 | } 280 | 281 | val->data[i+1].min = LLONG_MIN; 282 | val->data[i+1].max = LLONG_MAX; 283 | } 284 | val->tail_timestamp = val->data[i].timestamp; 285 | database_record__free_unpacked(rec, NULL); 286 | return unpack_sz; 287 | } 288 | 289 | int val_decompress(void *cmpr, int cmpr_len, struct rec_val *v, int v_len) { 290 | char zbuf[COMPRESS_WORKING_BUF]; 291 | uLongf destLen = sizeof(zbuf); 292 | int bufsz; 293 | if (uncompress(zbuf, &destLen, cmpr, cmpr_len) != Z_OK) { 294 | warn("inflate failed!\n"); 295 | assert(0); 296 | } 297 | 298 | debug("inflate: %i -> %i\n", cmpr_len, destLen); 299 | 300 | bufsz = pbuf_decompress(zbuf, destLen, v, v_len); 301 | if (bufsz < 0) { 302 | assert(0); 303 | } else { 304 | return 0; 305 | } 306 | assert(0); 307 | } 308 | 309 | 310 | #if 0 311 | int main(int argc, char **argv) { 312 | #define NPTS 10000 313 | struct point p[NPTS]; 314 | int i = 0; 315 | memset(p, 0, sizeof(p)); 316 | 317 | FILE *fp = fopen(argv[1], "r"); 318 | while (i < NPTS && fscanf(fp, "%lu,%lf\n", &p[i].timestamp, &p[i].reading) == 2) { 319 | p[i].reading_sequence = i; 320 | p[i].max = LLONG_MAX; 321 | p[i].min = LLONG_MIN; 322 | i++; 323 | } 324 | fclose(fp); 325 | 326 | struct timeval start, end, delta; 327 | gettimeofday(&start, NULL); 328 | 329 | char cbuf[300000]; 330 | unsigned int sz = pbuf_compress(p, i, cbuf, sizeof(cbuf)); 331 | //gettimeofday(&end, NULL); 332 | //timeval_subtract(&delta, &end, &start); 333 | //printf("pbuf compress: %i.%06i\n", delta.tv_sec, delta.tv_usec); 334 | 335 | //gettimeofday(&start, NULL); 336 | // printf("read %i\n", i); 337 | uLongf gzsz = compressBound(sz); 338 | void *gzbuf = malloc(gzsz); 339 | 340 | compress2(gzbuf, &gzsz, cbuf, sz, ZLIB_LEVEL); 341 | free(gzbuf); 342 | gettimeofday(&end, NULL); 343 | timeval_subtract(&delta, &end, &start); 344 | printf("deflate compress: %i.%06i\n", delta.tv_sec, delta.tv_usec); 345 | 346 | printf("%u %i %i %.04f %04f %.02f\n", 347 | sizeof(struct point) * i, 348 | i * 12, 349 | sz, 350 | (float)sz / (sizeof(struct point) * i), 351 | (float)sz / (i * 12), 352 | (float)sz / i); 353 | 354 | printf("%u %i %i %.04f %04f %.02f\n", 355 | sizeof(struct point) * i, 356 | i * 12, 357 | gzsz, 358 | (float)gzsz / (sizeof(struct point) * i), 359 | (float)gzsz / (i * 12), 360 | (float)gzsz / i); 361 | 362 | 363 | struct point recon[NPTS]; 364 | int read= 365 | pbuf_decompress(cbuf, sz, recon, NPTS); 366 | // printf("%i\n", read); 367 | assert(read == i); 368 | 369 | 370 | 371 | int j; 372 | ReadingSet *rs = _rpc_alloc_rs(i); 373 | for (j = 0; j < i; j++) { 374 | reading__init(rs->data[j]); 375 | _rpc_copy_reading(rs->data[j], &p[j]); 376 | } 377 | rs->n_data = i; 378 | void * simple; int simple_len; 379 | simple_len = reading_set__get_packed_size(rs); 380 | simple = malloc(simple_len); 381 | printf("simple len: %i\n", simple_len); 382 | reading_set__pack(rs, simple); 383 | 384 | gzsz = compressBound(simple_len); 385 | gzbuf = malloc(gzsz); 386 | 387 | compress2(gzbuf, &gzsz, simple, simple_len, 0); 388 | printf("%u %i %i %.04f %04f %.02f\n", 389 | sizeof(struct point) * i, 390 | i * 12, 391 | gzsz, 392 | (float)gzsz / (sizeof(struct point) * i), 393 | (float)gzsz / (i * 12), 394 | (float)gzsz / i); 395 | 396 | 397 | /* for (i = 0; i < 20; i++) { */ 398 | /* printf("%lu\t%lu\t%lf\t\t%lu\t%lu\t%lf\n", */ 399 | /* p[i].timestamp, p[i].reading_sequence, p[i].reading, */ 400 | /* recon[i].timestamp, p[i].reading_sequence, recon[i].reading); */ 401 | /* } */ 402 | 403 | assert(memcmp(recon, p, sizeof(struct point) * read) == 0); 404 | return 0; 405 | } 406 | #endif 407 | -------------------------------------------------------------------------------- /c6/hashtable/AUTHORS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stevedh/readingdb/5ebc06c567becb84a7eeaade9d5ebce3f13210ef/c6/hashtable/AUTHORS -------------------------------------------------------------------------------- /c6/hashtable/ChangeLog: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stevedh/readingdb/5ebc06c567becb84a7eeaade9d5ebce3f13210ef/c6/hashtable/ChangeLog -------------------------------------------------------------------------------- /c6/hashtable/INSTALL: -------------------------------------------------------------------------------- 1 | Installation Instructions 2 | ************************* 3 | 4 | Copyright (C) 1994, 1995, 1996, 1999, 2000, 2001, 2002, 2004, 2005, 5 | 2006, 2007, 2008, 2009 Free Software Foundation, Inc. 6 | 7 | Copying and distribution of this file, with or without modification, 8 | are permitted in any medium without royalty provided the copyright 9 | notice and this notice are preserved. This file is offered as-is, 10 | without warranty of any kind. 11 | 12 | Basic Installation 13 | ================== 14 | 15 | Briefly, the shell commands `./configure; make; make install' should 16 | configure, build, and install this package. The following 17 | more-detailed instructions are generic; see the `README' file for 18 | instructions specific to this package. Some packages provide this 19 | `INSTALL' file but do not implement all of the features documented 20 | below. The lack of an optional feature in a given package is not 21 | necessarily a bug. More recommendations for GNU packages can be found 22 | in *note Makefile Conventions: (standards)Makefile Conventions. 23 | 24 | The `configure' shell script attempts to guess correct values for 25 | various system-dependent variables used during compilation. It uses 26 | those values to create a `Makefile' in each directory of the package. 27 | It may also create one or more `.h' files containing system-dependent 28 | definitions. Finally, it creates a shell script `config.status' that 29 | you can run in the future to recreate the current configuration, and a 30 | file `config.log' containing compiler output (useful mainly for 31 | debugging `configure'). 32 | 33 | It can also use an optional file (typically called `config.cache' 34 | and enabled with `--cache-file=config.cache' or simply `-C') that saves 35 | the results of its tests to speed up reconfiguring. Caching is 36 | disabled by default to prevent problems with accidental use of stale 37 | cache files. 38 | 39 | If you need to do unusual things to compile the package, please try 40 | to figure out how `configure' could check whether to do them, and mail 41 | diffs or instructions to the address given in the `README' so they can 42 | be considered for the next release. If you are using the cache, and at 43 | some point `config.cache' contains results you don't want to keep, you 44 | may remove or edit it. 45 | 46 | The file `configure.ac' (or `configure.in') is used to create 47 | `configure' by a program called `autoconf'. You need `configure.ac' if 48 | you want to change it or regenerate `configure' using a newer version 49 | of `autoconf'. 50 | 51 | The simplest way to compile this package is: 52 | 53 | 1. `cd' to the directory containing the package's source code and type 54 | `./configure' to configure the package for your system. 55 | 56 | Running `configure' might take a while. While running, it prints 57 | some messages telling which features it is checking for. 58 | 59 | 2. Type `make' to compile the package. 60 | 61 | 3. Optionally, type `make check' to run any self-tests that come with 62 | the package, generally using the just-built uninstalled binaries. 63 | 64 | 4. Type `make install' to install the programs and any data files and 65 | documentation. When installing into a prefix owned by root, it is 66 | recommended that the package be configured and built as a regular 67 | user, and only the `make install' phase executed with root 68 | privileges. 69 | 70 | 5. Optionally, type `make installcheck' to repeat any self-tests, but 71 | this time using the binaries in their final installed location. 72 | This target does not install anything. Running this target as a 73 | regular user, particularly if the prior `make install' required 74 | root privileges, verifies that the installation completed 75 | correctly. 76 | 77 | 6. You can remove the program binaries and object files from the 78 | source code directory by typing `make clean'. To also remove the 79 | files that `configure' created (so you can compile the package for 80 | a different kind of computer), type `make distclean'. There is 81 | also a `make maintainer-clean' target, but that is intended mainly 82 | for the package's developers. If you use it, you may have to get 83 | all sorts of other programs in order to regenerate files that came 84 | with the distribution. 85 | 86 | 7. Often, you can also type `make uninstall' to remove the installed 87 | files again. In practice, not all packages have tested that 88 | uninstallation works correctly, even though it is required by the 89 | GNU Coding Standards. 90 | 91 | 8. Some packages, particularly those that use Automake, provide `make 92 | distcheck', which can by used by developers to test that all other 93 | targets like `make install' and `make uninstall' work correctly. 94 | This target is generally not run by end users. 95 | 96 | Compilers and Options 97 | ===================== 98 | 99 | Some systems require unusual options for compilation or linking that 100 | the `configure' script does not know about. Run `./configure --help' 101 | for details on some of the pertinent environment variables. 102 | 103 | You can give `configure' initial values for configuration parameters 104 | by setting variables in the command line or in the environment. Here 105 | is an example: 106 | 107 | ./configure CC=c99 CFLAGS=-g LIBS=-lposix 108 | 109 | *Note Defining Variables::, for more details. 110 | 111 | Compiling For Multiple Architectures 112 | ==================================== 113 | 114 | You can compile the package for more than one kind of computer at the 115 | same time, by placing the object files for each architecture in their 116 | own directory. To do this, you can use GNU `make'. `cd' to the 117 | directory where you want the object files and executables to go and run 118 | the `configure' script. `configure' automatically checks for the 119 | source code in the directory that `configure' is in and in `..'. This 120 | is known as a "VPATH" build. 121 | 122 | With a non-GNU `make', it is safer to compile the package for one 123 | architecture at a time in the source code directory. After you have 124 | installed the package for one architecture, use `make distclean' before 125 | reconfiguring for another architecture. 126 | 127 | On MacOS X 10.5 and later systems, you can create libraries and 128 | executables that work on multiple system types--known as "fat" or 129 | "universal" binaries--by specifying multiple `-arch' options to the 130 | compiler but only a single `-arch' option to the preprocessor. Like 131 | this: 132 | 133 | ./configure CC="gcc -arch i386 -arch x86_64 -arch ppc -arch ppc64" \ 134 | CXX="g++ -arch i386 -arch x86_64 -arch ppc -arch ppc64" \ 135 | CPP="gcc -E" CXXCPP="g++ -E" 136 | 137 | This is not guaranteed to produce working output in all cases, you 138 | may have to build one architecture at a time and combine the results 139 | using the `lipo' tool if you have problems. 140 | 141 | Installation Names 142 | ================== 143 | 144 | By default, `make install' installs the package's commands under 145 | `/usr/local/bin', include files under `/usr/local/include', etc. You 146 | can specify an installation prefix other than `/usr/local' by giving 147 | `configure' the option `--prefix=PREFIX', where PREFIX must be an 148 | absolute file name. 149 | 150 | You can specify separate installation prefixes for 151 | architecture-specific files and architecture-independent files. If you 152 | pass the option `--exec-prefix=PREFIX' to `configure', the package uses 153 | PREFIX as the prefix for installing programs and libraries. 154 | Documentation and other data files still use the regular prefix. 155 | 156 | In addition, if you use an unusual directory layout you can give 157 | options like `--bindir=DIR' to specify different values for particular 158 | kinds of files. Run `configure --help' for a list of the directories 159 | you can set and what kinds of files go in them. In general, the 160 | default for these options is expressed in terms of `${prefix}', so that 161 | specifying just `--prefix' will affect all of the other directory 162 | specifications that were not explicitly provided. 163 | 164 | The most portable way to affect installation locations is to pass the 165 | correct locations to `configure'; however, many packages provide one or 166 | both of the following shortcuts of passing variable assignments to the 167 | `make install' command line to change installation locations without 168 | having to reconfigure or recompile. 169 | 170 | The first method involves providing an override variable for each 171 | affected directory. For example, `make install 172 | prefix=/alternate/directory' will choose an alternate location for all 173 | directory configuration variables that were expressed in terms of 174 | `${prefix}'. Any directories that were specified during `configure', 175 | but not in terms of `${prefix}', must each be overridden at install 176 | time for the entire installation to be relocated. The approach of 177 | makefile variable overrides for each directory variable is required by 178 | the GNU Coding Standards, and ideally causes no recompilation. 179 | However, some platforms have known limitations with the semantics of 180 | shared libraries that end up requiring recompilation when using this 181 | method, particularly noticeable in packages that use GNU Libtool. 182 | 183 | The second method involves providing the `DESTDIR' variable. For 184 | example, `make install DESTDIR=/alternate/directory' will prepend 185 | `/alternate/directory' before all installation names. The approach of 186 | `DESTDIR' overrides is not required by the GNU Coding Standards, and 187 | does not work on platforms that have drive letters. On the other hand, 188 | it does better at avoiding recompilation issues, and works well even 189 | when some directory options were not specified in terms of `${prefix}' 190 | at `configure' time. 191 | 192 | Optional Features 193 | ================= 194 | 195 | If the package supports it, you can cause programs to be installed 196 | with an extra prefix or suffix on their names by giving `configure' the 197 | option `--program-prefix=PREFIX' or `--program-suffix=SUFFIX'. 198 | 199 | Some packages pay attention to `--enable-FEATURE' options to 200 | `configure', where FEATURE indicates an optional part of the package. 201 | They may also pay attention to `--with-PACKAGE' options, where PACKAGE 202 | is something like `gnu-as' or `x' (for the X Window System). The 203 | `README' should mention any `--enable-' and `--with-' options that the 204 | package recognizes. 205 | 206 | For packages that use the X Window System, `configure' can usually 207 | find the X include and library files automatically, but if it doesn't, 208 | you can use the `configure' options `--x-includes=DIR' and 209 | `--x-libraries=DIR' to specify their locations. 210 | 211 | Some packages offer the ability to configure how verbose the 212 | execution of `make' will be. For these packages, running `./configure 213 | --enable-silent-rules' sets the default to minimal output, which can be 214 | overridden with `make V=1'; while running `./configure 215 | --disable-silent-rules' sets the default to verbose, which can be 216 | overridden with `make V=0'. 217 | 218 | Particular systems 219 | ================== 220 | 221 | On HP-UX, the default C compiler is not ANSI C compatible. If GNU 222 | CC is not installed, it is recommended to use the following options in 223 | order to use an ANSI C compiler: 224 | 225 | ./configure CC="cc -Ae -D_XOPEN_SOURCE=500" 226 | 227 | and if that doesn't work, install pre-built binaries of GCC for HP-UX. 228 | 229 | On OSF/1 a.k.a. Tru64, some versions of the default C compiler cannot 230 | parse its `' header file. The option `-nodtk' can be used as 231 | a workaround. If GNU CC is not installed, it is therefore recommended 232 | to try 233 | 234 | ./configure CC="cc" 235 | 236 | and if that doesn't work, try 237 | 238 | ./configure CC="cc -nodtk" 239 | 240 | On Solaris, don't put `/usr/ucb' early in your `PATH'. This 241 | directory contains several dysfunctional programs; working variants of 242 | these programs are available in `/usr/bin'. So, if you need `/usr/ucb' 243 | in your `PATH', put it _after_ `/usr/bin'. 244 | 245 | On Haiku, software installed for all users goes in `/boot/common', 246 | not `/usr/local'. It is recommended to use the following options: 247 | 248 | ./configure --prefix=/boot/common 249 | 250 | Specifying the System Type 251 | ========================== 252 | 253 | There may be some features `configure' cannot figure out 254 | automatically, but needs to determine by the type of machine the package 255 | will run on. Usually, assuming the package is built to be run on the 256 | _same_ architectures, `configure' can figure that out, but if it prints 257 | a message saying it cannot guess the machine type, give it the 258 | `--build=TYPE' option. TYPE can either be a short name for the system 259 | type, such as `sun4', or a canonical name which has the form: 260 | 261 | CPU-COMPANY-SYSTEM 262 | 263 | where SYSTEM can have one of these forms: 264 | 265 | OS 266 | KERNEL-OS 267 | 268 | See the file `config.sub' for the possible values of each field. If 269 | `config.sub' isn't included in this package, then this package doesn't 270 | need to know the machine type. 271 | 272 | If you are _building_ compiler tools for cross-compiling, you should 273 | use the option `--target=TYPE' to select the type of system they will 274 | produce code for. 275 | 276 | If you want to _use_ a cross compiler, that generates code for a 277 | platform different from the build platform, you should specify the 278 | "host" platform (i.e., that on which the generated programs will 279 | eventually be run) with `--host=TYPE'. 280 | 281 | Sharing Defaults 282 | ================ 283 | 284 | If you want to set default values for `configure' scripts to share, 285 | you can create a site shell script called `config.site' that gives 286 | default values for variables like `CC', `cache_file', and `prefix'. 287 | `configure' looks for `PREFIX/share/config.site' if it exists, then 288 | `PREFIX/etc/config.site' if it exists. Or, you can set the 289 | `CONFIG_SITE' environment variable to the location of the site script. 290 | A warning: not all `configure' scripts look for a site script. 291 | 292 | Defining Variables 293 | ================== 294 | 295 | Variables not defined in a site shell script can be set in the 296 | environment passed to `configure'. However, some packages may run 297 | configure again during the build, and the customized values of these 298 | variables may be lost. In order to avoid this problem, you should set 299 | them in the `configure' command line, using `VAR=value'. For example: 300 | 301 | ./configure CC=/usr/local2/bin/gcc 302 | 303 | causes the specified `gcc' to be used as the C compiler (unless it is 304 | overridden in the site shell script). 305 | 306 | Unfortunately, this technique does not work for `CONFIG_SHELL' due to 307 | an Autoconf bug. Until the bug is fixed you can use this workaround: 308 | 309 | CONFIG_SHELL=/bin/bash /bin/bash ./configure CONFIG_SHELL=/bin/bash 310 | 311 | `configure' Invocation 312 | ====================== 313 | 314 | `configure' recognizes the following options to control how it 315 | operates. 316 | 317 | `--help' 318 | `-h' 319 | Print a summary of all of the options to `configure', and exit. 320 | 321 | `--help=short' 322 | `--help=recursive' 323 | Print a summary of the options unique to this package's 324 | `configure', and exit. The `short' variant lists options used 325 | only in the top level, while the `recursive' variant lists options 326 | also present in any nested packages. 327 | 328 | `--version' 329 | `-V' 330 | Print the version of Autoconf used to generate the `configure' 331 | script, and exit. 332 | 333 | `--cache-file=FILE' 334 | Enable the cache: use and save the results of the tests in FILE, 335 | traditionally `config.cache'. FILE defaults to `/dev/null' to 336 | disable caching. 337 | 338 | `--config-cache' 339 | `-C' 340 | Alias for `--cache-file=config.cache'. 341 | 342 | `--quiet' 343 | `--silent' 344 | `-q' 345 | Do not print messages saying which checks are being made. To 346 | suppress all normal output, redirect it to `/dev/null' (any error 347 | messages will still be shown). 348 | 349 | `--srcdir=DIR' 350 | Look for the package's source code in directory DIR. Usually 351 | `configure' can determine that directory automatically. 352 | 353 | `--prefix=DIR' 354 | Use DIR as the installation prefix. *note Installation Names:: 355 | for more details, including other options available for fine-tuning 356 | the installation locations. 357 | 358 | `--no-create' 359 | `-n' 360 | Run the configure checks, but stop before creating any output 361 | files. 362 | 363 | `configure' also accepts some other, not widely useful, options. Run 364 | `configure --help' for more details. 365 | 366 | -------------------------------------------------------------------------------- /c6/hashtable/Makefile.am: -------------------------------------------------------------------------------- 1 | 2 | SUBDIRS = src . tests 3 | -------------------------------------------------------------------------------- /c6/hashtable/NEWS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stevedh/readingdb/5ebc06c567becb84a7eeaade9d5ebce3f13210ef/c6/hashtable/NEWS -------------------------------------------------------------------------------- /c6/hashtable/README: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stevedh/readingdb/5ebc06c567becb84a7eeaade9d5ebce3f13210ef/c6/hashtable/README -------------------------------------------------------------------------------- /c6/hashtable/configure.ac: -------------------------------------------------------------------------------- 1 | # -*- Autoconf -*- 2 | # Process this file with autoconf to produce a configure script. 3 | 4 | AC_PREREQ(2.61) 5 | AC_INIT(hashtable, 0.1, stevedh@eecs.berkeley.edu) 6 | AC_CONFIG_SRCDIR([src/hashtable.c]) 7 | AC_CONFIG_HEADER([config.h]) 8 | AC_CONFIG_MACRO_DIR([m4]) 9 | 10 | # Checks for programs. 11 | AM_INIT_AUTOMAKE([-Wall]) 12 | AC_PROG_CC 13 | AM_PROG_CC_C_O 14 | AC_PROG_RANLIB 15 | 16 | PKG_CHECK_MODULES([CHECK], [check >= 0.9.4]) 17 | # Checks for libraries. 18 | 19 | # Checks for header files. 20 | AC_HEADER_STDC 21 | AC_CHECK_HEADERS([stdlib.h string.h]) 22 | 23 | # Checks for typedefs, structures, and compiler characteristics. 24 | AC_C_CONST 25 | AC_C_INLINE 26 | 27 | # Checks for library functions. 28 | AC_FUNC_MALLOC 29 | AC_FUNC_REALLOC 30 | AC_CHECK_FUNCS([memset]) 31 | 32 | AC_CONFIG_FILES([Makefile 33 | src/Makefile 34 | tests/Makefile]) 35 | AC_OUTPUT 36 | -------------------------------------------------------------------------------- /c6/hashtable/hashtable-0.1.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stevedh/readingdb/5ebc06c567becb84a7eeaade9d5ebce3f13210ef/c6/hashtable/hashtable-0.1.tar.gz -------------------------------------------------------------------------------- /c6/hashtable/src/Makefile.am: -------------------------------------------------------------------------------- 1 | 2 | noinst_LIBRARIES = libhashtable.a 3 | 4 | noinst_HEADERS = hashtable.h 5 | libhashtable_a_SOURCES = hashtable.c hashtable_private.h $(noinst_HEADERS) 6 | -------------------------------------------------------------------------------- /c6/hashtable/src/hashtable.c: -------------------------------------------------------------------------------- 1 | /* Copyright (C) 2004 Christopher Clark */ 2 | 3 | #include "hashtable.h" 4 | #include "hashtable_private.h" 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #define iceil(X) ((unsigned int)((X) + 1)) 11 | 12 | /* 13 | Credit for primes table: Aaron Krowne 14 | http://br.endernet.org/~akrowne/ 15 | http://planetmath.org/encyclopedia/GoodHashTablePrimes.html 16 | */ 17 | static const unsigned int primes[] = { 18 | 53, 97, 193, 389, 19 | 769, 1543, 3079, 6151, 20 | 12289, 24593, 49157, 98317, 21 | 196613, 393241, 786433, 1572869, 22 | 3145739, 6291469, 12582917, 25165843, 23 | 50331653, 100663319, 201326611, 402653189, 24 | 805306457, 1610612741 25 | }; 26 | const unsigned int prime_table_length = sizeof(primes)/sizeof(primes[0]); 27 | const float max_load_factor = 0.65; 28 | 29 | /*****************************************************************************/ 30 | struct hashtable * 31 | create_hashtable(unsigned int minsize, 32 | unsigned int (*hashf) (void*), 33 | int (*eqf) (void*,void*)) 34 | { 35 | struct hashtable *h; 36 | unsigned int pindex, size = primes[0]; 37 | /* Check requested hashtable isn't too large */ 38 | if (minsize > (1u << 30)) return NULL; 39 | /* Enforce size as prime */ 40 | for (pindex=0; pindex < prime_table_length; pindex++) { 41 | if (primes[pindex] > minsize) { size = primes[pindex]; break; } 42 | } 43 | h = (struct hashtable *)malloc(sizeof(struct hashtable)); 44 | if (NULL == h) return NULL; /*oom*/ 45 | h->table = (struct entry **)malloc(sizeof(struct entry*) * size); 46 | if (NULL == h->table) { free(h); return NULL; } /*oom*/ 47 | memset(h->table, 0, size * sizeof(struct entry *)); 48 | h->tablelength = size; 49 | h->primeindex = pindex; 50 | h->entrycount = 0; 51 | h->hashfn = hashf; 52 | h->eqfn = eqf; 53 | h->loadlimit = (unsigned int) iceil(size * max_load_factor); 54 | return h; 55 | } 56 | 57 | /*****************************************************************************/ 58 | unsigned int 59 | hash(struct hashtable *h, void *k) 60 | { 61 | /* Aim to protect against poor hash functions by adding logic here 62 | * - logic taken from java 1.4 hashtable source */ 63 | unsigned int i = h->hashfn(k); 64 | i += ~(i << 9); 65 | i ^= ((i >> 14) | (i << 18)); /* >>> */ 66 | i += (i << 4); 67 | i ^= ((i >> 10) | (i << 22)); /* >>> */ 68 | return i; 69 | } 70 | 71 | /*****************************************************************************/ 72 | static int 73 | hashtable_expand(struct hashtable *h) 74 | { 75 | /* Double the size of the table to accomodate more entries */ 76 | struct entry **newtable; 77 | struct entry *e; 78 | struct entry **pE; 79 | unsigned int newsize, i, index; 80 | /* Check we're not hitting max capacity */ 81 | if (h->primeindex == (prime_table_length - 1)) return 0; 82 | newsize = primes[++(h->primeindex)]; 83 | newtable = (struct entry **)malloc(sizeof(struct entry*) * newsize); 84 | if (NULL != newtable) 85 | { 86 | memset(newtable, 0, newsize * sizeof(struct entry *)); 87 | /* This algorithm is not 'stable'. ie. it reverses the list 88 | * when it transfers entries between the tables */ 89 | for (i = 0; i < h->tablelength; i++) { 90 | while (NULL != (e = h->table[i])) { 91 | h->table[i] = e->next; 92 | index = indexFor(newsize,e->h); 93 | e->next = newtable[index]; 94 | newtable[index] = e; 95 | } 96 | } 97 | free(h->table); 98 | h->table = newtable; 99 | } 100 | /* Plan B: realloc instead */ 101 | else 102 | { 103 | newtable = (struct entry **) 104 | realloc(h->table, newsize * sizeof(struct entry *)); 105 | if (NULL == newtable) { (h->primeindex)--; return 0; } 106 | h->table = newtable; 107 | memset(newtable[h->tablelength], 0, newsize - h->tablelength); 108 | for (i = 0; i < h->tablelength; i++) { 109 | for (pE = &(newtable[i]), e = *pE; e != NULL; e = *pE) { 110 | index = indexFor(newsize,e->h); 111 | if (index == i) 112 | { 113 | pE = &(e->next); 114 | } 115 | else 116 | { 117 | *pE = e->next; 118 | e->next = newtable[index]; 119 | newtable[index] = e; 120 | } 121 | } 122 | } 123 | } 124 | h->tablelength = newsize; 125 | h->loadlimit = (unsigned int) iceil(newsize * max_load_factor); 126 | return -1; 127 | } 128 | 129 | /*****************************************************************************/ 130 | unsigned int 131 | hashtable_count(struct hashtable *h) 132 | { 133 | return h->entrycount; 134 | } 135 | 136 | /*****************************************************************************/ 137 | int 138 | hashtable_insert(struct hashtable *h, void *k, void *v) 139 | { 140 | /* This method allows duplicate keys - but they shouldn't be used */ 141 | unsigned int index; 142 | struct entry *e; 143 | if (++(h->entrycount) > h->loadlimit) 144 | { 145 | /* Ignore the return value. If expand fails, we should 146 | * still try cramming just this value into the existing table 147 | * -- we may not have memory for a larger table, but one more 148 | * element may be ok. Next time we insert, we'll try expanding again.*/ 149 | hashtable_expand(h); 150 | } 151 | e = (struct entry *)malloc(sizeof(struct entry)); 152 | if (NULL == e) { --(h->entrycount); return 0; } /*oom*/ 153 | e->h = hash(h,k); 154 | index = indexFor(h->tablelength,e->h); 155 | e->k = k; 156 | e->v = v; 157 | e->next = h->table[index]; 158 | h->table[index] = e; 159 | return -1; 160 | } 161 | 162 | /*****************************************************************************/ 163 | void * /* returns value associated with key */ 164 | hashtable_search(struct hashtable *h, void *k) 165 | { 166 | struct entry *e; 167 | unsigned int hashvalue, index; 168 | hashvalue = hash(h,k); 169 | index = indexFor(h->tablelength,hashvalue); 170 | e = h->table[index]; 171 | while (NULL != e) 172 | { 173 | /* Check hash value to short circuit heavier comparison */ 174 | if ((hashvalue == e->h) && (h->eqfn(k, e->k))) return e->v; 175 | e = e->next; 176 | } 177 | return NULL; 178 | } 179 | 180 | void * 181 | hashtable_next(struct hashtable *h, void **k) { 182 | struct entry *e; 183 | unsigned int hashvalue, index, next = 0; 184 | if (!*k) { 185 | /* return the first key, value pair in the table */ 186 | index = 0; 187 | 188 | if (h->entrycount == 0) 189 | return NULL; 190 | 191 | while (h->table[index] == NULL) 192 | index ++; 193 | *k = h->table[index]->k; 194 | return h->table[index]->v; 195 | } 196 | 197 | hashvalue = hash(h, *k); 198 | index = indexFor(h->tablelength, hashvalue); 199 | e = h->table[index]; 200 | 201 | while (e != NULL) { 202 | if ((hashvalue == e->h) && h->eqfn(*k, e->k)) { 203 | next = 1; 204 | } else if (next) { 205 | *k = e->k; 206 | return e->v; 207 | } 208 | e = e->next; 209 | if (e == NULL) { 210 | index++; 211 | while (index < h->tablelength && h->table[index] == NULL) 212 | index++; 213 | if (index < h->tablelength) 214 | e = h->table[index]; 215 | } 216 | } 217 | return NULL; 218 | } 219 | 220 | /*****************************************************************************/ 221 | void * /* returns value associated with key */ 222 | hashtable_remove(struct hashtable *h, void *k) 223 | { 224 | /* TODO: consider compacting the table when the load factor drops enough, 225 | * or provide a 'compact' method. */ 226 | 227 | struct entry *e; 228 | struct entry **pE; 229 | void *v; 230 | unsigned int hashvalue, index; 231 | 232 | hashvalue = hash(h,k); 233 | index = indexFor(h->tablelength,hash(h,k)); 234 | pE = &(h->table[index]); 235 | e = *pE; 236 | while (NULL != e) 237 | { 238 | /* Check hash value to short circuit heavier comparison */ 239 | if ((hashvalue == e->h) && (h->eqfn(k, e->k))) 240 | { 241 | *pE = e->next; 242 | h->entrycount--; 243 | v = e->v; 244 | freekey(e->k); 245 | free(e); 246 | return v; 247 | } 248 | pE = &(e->next); 249 | e = e->next; 250 | } 251 | return NULL; 252 | } 253 | 254 | /*****************************************************************************/ 255 | /* destroy */ 256 | void 257 | hashtable_destroy(struct hashtable *h, int free_values) 258 | { 259 | unsigned int i; 260 | struct entry *e, *f; 261 | struct entry **table = h->table; 262 | if (free_values) 263 | { 264 | for (i = 0; i < h->tablelength; i++) 265 | { 266 | e = table[i]; 267 | while (NULL != e) 268 | { f = e; e = e->next; freekey(f->k); free(f->v); free(f); } 269 | } 270 | } 271 | else 272 | { 273 | for (i = 0; i < h->tablelength; i++) 274 | { 275 | e = table[i]; 276 | while (NULL != e) 277 | { f = e; e = e->next; freekey(f->k); free(f); } 278 | } 279 | } 280 | free(h->table); 281 | free(h); 282 | } 283 | 284 | /* 285 | * Copyright (c) 2002, Christopher Clark 286 | * All rights reserved. 287 | * 288 | * Redistribution and use in source and binary forms, with or without 289 | * modification, are permitted provided that the following conditions 290 | * are met: 291 | * 292 | * * Redistributions of source code must retain the above copyright 293 | * notice, this list of conditions and the following disclaimer. 294 | * 295 | * * Redistributions in binary form must reproduce the above copyright 296 | * notice, this list of conditions and the following disclaimer in the 297 | * documentation and/or other materials provided with the distribution. 298 | * 299 | * * Neither the name of the original author; nor the names of any contributors 300 | * may be used to endorse or promote products derived from this software 301 | * without specific prior written permission. 302 | * 303 | * 304 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 305 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 306 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 307 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER 308 | * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 309 | * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 310 | * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 311 | * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 312 | * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 313 | * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 314 | * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 315 | */ 316 | -------------------------------------------------------------------------------- /c6/hashtable/src/hashtable.h: -------------------------------------------------------------------------------- 1 | /* Copyright (C) 2002 Christopher Clark */ 2 | /* hashtable_next updated 2010 Stephen Dawson-Haggerty */ 3 | 4 | #ifndef __HASHTABLE_CWC22_H__ 5 | #define __HASHTABLE_CWC22_H__ 6 | 7 | struct hashtable; 8 | 9 | /* Example of use: 10 | * 11 | * struct hashtable *h; 12 | * struct some_key *k; 13 | * struct some_value *v; 14 | * 15 | * static unsigned int hash_from_key_fn( void *k ); 16 | * static int keys_equal_fn ( void *key1, void *key2 ); 17 | * 18 | * h = create_hashtable(16, hash_from_key_fn, keys_equal_fn); 19 | * k = (struct some_key *) malloc(sizeof(struct some_key)); 20 | * v = (struct some_value *) malloc(sizeof(struct some_value)); 21 | * 22 | * (initialise k and v to suitable values) 23 | * 24 | * if (! hashtable_insert(h,k,v) ) 25 | * { exit(-1); } 26 | * 27 | * if (NULL == (found = hashtable_search(h,k) )) 28 | * { printf("not found!"); } 29 | * 30 | * if (NULL == (found = hashtable_remove(h,k) )) 31 | * { printf("Not found\n"); } 32 | * 33 | */ 34 | 35 | /* Macros may be used to define type-safe(r) hashtable access functions, with 36 | * methods specialized to take known key and value types as parameters. 37 | * 38 | * Example: 39 | * 40 | * Insert this at the start of your file: 41 | * 42 | * DEFINE_HASHTABLE_INSERT(insert_some, struct some_key, struct some_value); 43 | * DEFINE_HASHTABLE_SEARCH(search_some, struct some_key, struct some_value); 44 | * DEFINE_HASHTABLE_REMOVE(remove_some, struct some_key, struct some_value); 45 | * 46 | * This defines the functions 'insert_some', 'search_some' and 'remove_some'. 47 | * These operate just like hashtable_insert etc., with the same parameters, 48 | * but their function signatures have 'struct some_key *' rather than 49 | * 'void *', and hence can generate compile time errors if your program is 50 | * supplying incorrect data as a key (and similarly for value). 51 | * 52 | * Note that the hash and key equality functions passed to create_hashtable 53 | * still take 'void *' parameters instead of 'some key *'. This shouldn't be 54 | * a difficult issue as they're only defined and passed once, and the other 55 | * functions will ensure that only valid keys are supplied to them. 56 | * 57 | * The cost for this checking is increased code size and runtime overhead 58 | * - if performance is important, it may be worth switching back to the 59 | * unsafe methods once your program has been debugged with the safe methods. 60 | * This just requires switching to some simple alternative defines - eg: 61 | * #define insert_some hashtable_insert 62 | * 63 | */ 64 | 65 | /***************************************************************************** 66 | * create_hashtable 67 | 68 | * @name create_hashtable 69 | * @param minsize minimum initial size of hashtable 70 | * @param hashfunction function for hashing keys 71 | * @param key_eq_fn function for determining key equality 72 | * @return newly created hashtable or NULL on failure 73 | */ 74 | 75 | struct hashtable * 76 | create_hashtable(unsigned int minsize, 77 | unsigned int (*hashfunction) (void*), 78 | int (*key_eq_fn) (void*,void*)); 79 | 80 | /***************************************************************************** 81 | * hashtable_insert 82 | 83 | * @name hashtable_insert 84 | * @param h the hashtable to insert into 85 | * @param k the key - hashtable claims ownership and will free on removal 86 | * @param v the value - does not claim ownership 87 | * @return non-zero for successful insertion 88 | * 89 | * This function will cause the table to expand if the insertion would take 90 | * the ratio of entries to table size over the maximum load factor. 91 | * 92 | * This function does not check for repeated insertions with a duplicate key. 93 | * The value returned when using a duplicate key is undefined -- when 94 | * the hashtable changes size, the order of retrieval of duplicate key 95 | * entries is reversed. 96 | * If in doubt, remove before insert. 97 | */ 98 | 99 | int 100 | hashtable_insert(struct hashtable *h, void *k, void *v); 101 | 102 | #define DEFINE_HASHTABLE_INSERT(fnname, keytype, valuetype) \ 103 | int fnname (struct hashtable *h, keytype *k, valuetype *v) \ 104 | { \ 105 | return hashtable_insert(h,k,v); \ 106 | } 107 | 108 | /***************************************************************************** 109 | * hashtable_search 110 | 111 | * @name hashtable_search 112 | * @param h the hashtable to search 113 | * @param k the key to search for - does not claim ownership 114 | * @return the value associated with the key, or NULL if none found 115 | */ 116 | 117 | void * 118 | hashtable_search(struct hashtable *h, void *k); 119 | 120 | #define DEFINE_HASHTABLE_SEARCH(fnname, keytype, valuetype) \ 121 | valuetype * fnname (struct hashtable *h, keytype *k) \ 122 | { \ 123 | return (valuetype *) (hashtable_search(h,k)); \ 124 | } 125 | 126 | 127 | /***************************************************************************** 128 | * hashtable_next 129 | 130 | * @name hashtable_next 131 | * @param h the hashtable to search 132 | * @param k key to search for. if NULL the first (k,v) pair is returned 133 | * @return the value "after" the key passed in. k is updated. 134 | */ 135 | void * 136 | hashtable_next(struct hashtable *h, void **k); 137 | 138 | /***************************************************************************** 139 | * hashtable_remove 140 | 141 | * @name hashtable_remove 142 | * @param h the hashtable to remove the item from 143 | * @param k the key to search for - does not claim ownership 144 | * @return the value associated with the key, or NULL if none found 145 | */ 146 | 147 | void * /* returns value */ 148 | hashtable_remove(struct hashtable *h, void *k); 149 | 150 | #define DEFINE_HASHTABLE_REMOVE(fnname, keytype, valuetype) \ 151 | valuetype * fnname (struct hashtable *h, keytype *k) \ 152 | { \ 153 | return (valuetype *) (hashtable_remove(h,k)); \ 154 | } 155 | 156 | 157 | /***************************************************************************** 158 | * hashtable_count 159 | 160 | * @name hashtable_count 161 | * @param h the hashtable 162 | * @return the number of items stored in the hashtable 163 | */ 164 | unsigned int 165 | hashtable_count(struct hashtable *h); 166 | 167 | 168 | /***************************************************************************** 169 | * hashtable_destroy 170 | 171 | * @name hashtable_destroy 172 | * @param h the hashtable 173 | * @param free_values whether to call 'free' on the remaining values 174 | */ 175 | 176 | void 177 | hashtable_destroy(struct hashtable *h, int free_values); 178 | 179 | #endif /* __HASHTABLE_CWC22_H__ */ 180 | 181 | /* 182 | * Copyright (c) 2002, Christopher Clark 183 | * All rights reserved. 184 | * 185 | * Redistribution and use in source and binary forms, with or without 186 | * modification, are permitted provided that the following conditions 187 | * are met: 188 | * 189 | * * Redistributions of source code must retain the above copyright 190 | * notice, this list of conditions and the following disclaimer. 191 | * 192 | * * Redistributions in binary form must reproduce the above copyright 193 | * notice, this list of conditions and the following disclaimer in the 194 | * documentation and/or other materials provided with the distribution. 195 | * 196 | * * Neither the name of the original author; nor the names of any contributors 197 | * may be used to endorse or promote products derived from this software 198 | * without specific prior written permission. 199 | * 200 | * 201 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 202 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 203 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 204 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER 205 | * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 206 | * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 207 | * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 208 | * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 209 | * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 210 | * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 211 | * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 212 | */ 213 | -------------------------------------------------------------------------------- /c6/hashtable/src/hashtable_private.h: -------------------------------------------------------------------------------- 1 | /* 2 | * "Copyright (c) 2008 The Regents of the University of California. 3 | * All rights reserved." 4 | * 5 | * Permission to use, copy, modify, and distribute this software and its 6 | * documentation for any purpose, without fee, and without written agreement is 7 | * hereby granted, provided that the above copyright notice, the following 8 | * two paragraphs and the author appear in all copies of this software. 9 | * 10 | * IN NO EVENT SHALL THE UNIVERSITY OF CALIFORNIA BE LIABLE TO ANY PARTY FOR 11 | * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING OUT 12 | * OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF THE UNIVERSITY OF 13 | * CALIFORNIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | * 15 | * THE UNIVERSITY OF CALIFORNIA SPECIFICALLY DISCLAIMS ANY WARRANTIES, 16 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY 17 | * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS 18 | * ON AN "AS IS" BASIS, AND THE UNIVERSITY OF CALIFORNIA HAS NO OBLIGATION TO 19 | * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS." 20 | * 21 | */ 22 | /* Copyright (C) 2002, 2004 Christopher Clark */ 23 | 24 | #ifndef __HASHTABLE_PRIVATE_CWC22_H__ 25 | #define __HASHTABLE_PRIVATE_CWC22_H__ 26 | 27 | #include "hashtable.h" 28 | 29 | /*****************************************************************************/ 30 | struct entry 31 | { 32 | void *k, *v; 33 | unsigned int h; 34 | struct entry *next; 35 | }; 36 | 37 | struct hashtable { 38 | unsigned int tablelength; 39 | struct entry **table; 40 | unsigned int entrycount; 41 | unsigned int loadlimit; 42 | unsigned int primeindex; 43 | unsigned int (*hashfn) (void *k); 44 | int (*eqfn) (void *k1, void *k2); 45 | }; 46 | 47 | struct hashtable_iterator { 48 | unsigned int table_index; 49 | unsigned int bucket_index; 50 | }; 51 | 52 | /*****************************************************************************/ 53 | unsigned int 54 | hash(struct hashtable *h, void *k); 55 | 56 | /*****************************************************************************/ 57 | /* indexFor */ 58 | static inline unsigned int 59 | indexFor(unsigned int tablelength, unsigned int hashvalue) { 60 | return (hashvalue % tablelength); 61 | }; 62 | 63 | /* Only works if tablelength == 2^N */ 64 | /*static inline unsigned int 65 | indexFor(unsigned int tablelength, unsigned int hashvalue) 66 | { 67 | return (hashvalue & (tablelength - 1u)); 68 | } 69 | */ 70 | 71 | /*****************************************************************************/ 72 | #define freekey(X) free(X) 73 | /*define freekey(X) ; */ 74 | 75 | 76 | /*****************************************************************************/ 77 | 78 | #endif /* __HASHTABLE_PRIVATE_CWC22_H__*/ 79 | 80 | /* 81 | * Copyright (c) 2002, Christopher Clark 82 | * All rights reserved. 83 | * 84 | * Redistribution and use in source and binary forms, with or without 85 | * modification, are permitted provided that the following conditions 86 | * are met: 87 | * 88 | * * Redistributions of source code must retain the above copyright 89 | * notice, this list of conditions and the following disclaimer. 90 | * 91 | * * Redistributions in binary form must reproduce the above copyright 92 | * notice, this list of conditions and the following disclaimer in the 93 | * documentation and/or other materials provided with the distribution. 94 | * 95 | * * Neither the name of the original author; nor the names of any contributors 96 | * may be used to endorse or promote products derived from this software 97 | * without specific prior written permission. 98 | * 99 | * 100 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 101 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 102 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 103 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER 104 | * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 105 | * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 106 | * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 107 | * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 108 | * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 109 | * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 110 | * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 111 | */ 112 | -------------------------------------------------------------------------------- /c6/hashtable/tests/Makefile.am: -------------------------------------------------------------------------------- 1 | 2 | TESTS = check_hashtable 3 | check_PROGRAMS = check_hashtable 4 | check_hashtable_SOURCES = check_hashtable.c $(top_builddir)/src/hashtable.h 5 | check_hashtable_CFLAGS = @CHECK_CFLAGS@ 6 | check_hashtable_LDADD = $(top_builddir)/src/libhashtable.a @CHECK_LIBS@ 7 | -------------------------------------------------------------------------------- /c6/hashtable/tests/check_hashtable.1.c: -------------------------------------------------------------------------------- 1 | 2 | #include "../src/hashtable.h" 3 | 4 | START_TEST(test_hashtable_create) 5 | { 6 | 7 | } 8 | -------------------------------------------------------------------------------- /c6/hashtable/tests/check_hashtable.c: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | #include 4 | #include "../src/hashtable.h" 5 | 6 | unsigned int int_hashfunction(void *h) { 7 | unsigned int *v = h; 8 | return *v; 9 | } 10 | unsigned int int_eq_fn(void *k1, void *k2) { 11 | unsigned int *i1 = k1; 12 | unsigned int *i2 = k2; 13 | return (*i1 == *i2); 14 | } 15 | 16 | START_TEST(test_hashtable_create) 17 | { 18 | int *k, *v; 19 | struct hashtable *h = create_hashtable(12, int_hashfunction, int_eq_fn); 20 | fail_unless(hashtable_count(h) == 0, 21 | "Table is not empty on creation"); 22 | hashtable_destroy(h, 1); 23 | } 24 | END_TEST 25 | 26 | START_TEST(test_hashtable_insert_count) 27 | { 28 | int i, *k, *v; 29 | struct hashtable *h = create_hashtable(12, int_hashfunction, int_eq_fn); 30 | for (i = 0; i < 10; i++) { 31 | k = malloc(sizeof(int)); 32 | v = malloc(sizeof(int)); 33 | *k = i; 34 | fail_unless(hashtable_count(h) == i, 35 | "Table count is not %i", i); 36 | hashtable_insert(h, k, v); 37 | } 38 | hashtable_destroy(h, 1); 39 | } 40 | END_TEST 41 | 42 | START_TEST(test_hashtable_next_empty) 43 | { 44 | void *k, *v; 45 | struct hashtable *h = create_hashtable(12, int_hashfunction, int_eq_fn); 46 | k = NULL; 47 | v = hashtable_next(h, &k); 48 | fail_unless(v == NULL, 49 | "hashtable_next must return NULL on an empty table"); 50 | hashtable_destroy(h, 1); 51 | } 52 | END_TEST 53 | 54 | START_TEST(test_hashtable_next_one) 55 | { 56 | int *k, *v, *qk, *qv; 57 | struct hashtable *h = create_hashtable(12, int_hashfunction, int_eq_fn); 58 | k = malloc(sizeof(int)); 59 | v = malloc(sizeof(int)); 60 | *k = 12; 61 | hashtable_insert(h, k, v); 62 | qk = NULL; 63 | qv = hashtable_next(h, &qk); 64 | fail_unless(qk == k, 65 | "hashtable_next did not return only key"); 66 | fail_unless(qv == v, 67 | "hashtable_next did not return correct value"); 68 | qv = hashtable_next(h, &qk); 69 | fail_unless(qv == NULL, 70 | "hashtable_next did not empty"); 71 | hashtable_destroy(h, 1); 72 | } 73 | END_TEST 74 | 75 | /* check that we get all the keys back, exactly once */ 76 | START_TEST(test_hashtable_next_several) 77 | { 78 | int i, *k, *v, *qk, *qv; 79 | int read_vals = 0, read_n = 0; 80 | struct hashtable *h = create_hashtable(12, int_hashfunction, int_eq_fn); 81 | for (i = 0; i < 16; i++) { 82 | k = malloc(sizeof(int)); 83 | v = malloc(sizeof(int)); 84 | *k = i; 85 | *v = i; 86 | hashtable_insert(h, k, v); 87 | } 88 | fail_unless(hashtable_count(h) == 16, 89 | "Did not insert all 16 keys"); 90 | qk = NULL; 91 | qv = hashtable_next(h, &qk); 92 | while (qv != NULL) { 93 | fail_unless(*qk == *qv, 94 | "next did not return matching keys: %i %i", *qk, *qv); 95 | read_vals |= 1 << *qk; 96 | read_n ++; 97 | if (read_n > 30) 98 | break; 99 | qv = hashtable_next(h, &qk); 100 | } 101 | fail_unless(read_n == 16, 102 | "hashtable_next did not return exactly 16 values (%i)", read_n); 103 | fail_unless(read_vals == 0xffff, 104 | "hashtable_next did not return all 16 values"); 105 | hashtable_destroy(h, 1); 106 | } 107 | END_TEST 108 | 109 | Suite * 110 | hashtable_suite(void) { 111 | Suite *s = suite_create("Hashtable"); 112 | TCase *tc_core = tcase_create("Core"); 113 | TCase *tc_next = tcase_create("Next"); 114 | 115 | /* core tests */ 116 | tcase_add_test(tc_core, test_hashtable_create); 117 | tcase_add_test(tc_core, test_hashtable_insert_count); 118 | suite_add_tcase(s, tc_core); 119 | 120 | /* next tests */ 121 | tcase_add_test(tc_next, test_hashtable_next_empty); 122 | tcase_add_test(tc_next, test_hashtable_next_one); 123 | tcase_add_test(tc_next, test_hashtable_next_several); 124 | suite_add_tcase(s, tc_next); 125 | 126 | return s; 127 | } 128 | 129 | int main(void) 130 | { 131 | int number_failed; 132 | Suite *s = hashtable_suite(); 133 | SRunner *sr = srunner_create(s); 134 | srunner_run_all(sr, CK_NORMAL); 135 | number_failed = srunner_ntests_failed(sr); 136 | srunner_free(sr); 137 | return (number_failed == 0) ? EXIT_SUCCESS : EXIT_FAILURE; 138 | } 139 | -------------------------------------------------------------------------------- /c6/intervals.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Module for computing a covering set of all dirty regions. 3 | * 4 | * the main process (reading-server) append to a log of dirty regions. 5 | * This has the form of a (streamid, starttime, endtime) for each 6 | * write. It's relatively inefficient to naively run the sketch 7 | * updater on all of these tuples, because (a) you have to fetch at 8 | * least +/- 1 hour of data around each write, and (b) most writes are 9 | * close together in time. This file has routines to converting that 10 | * into a set of intervals which are more than OVERLAP_SLACK apart. 11 | * 12 | * The algorithm is basically from 13 | * 14 | * http://www.geeksforgeeks.org/merging-intervals/ 15 | * 16 | * ported, to C, and with the addition of the slack parameter. It's 17 | * runtime is whatever qsort provides on your platform, and then 18 | * linear in the input + output size. 19 | */ 20 | /* 21 | * @author Stephen Dawson-Haggerty 22 | */ 23 | 24 | #include 25 | #include 26 | #include 27 | 28 | #include "intervals.h" 29 | 30 | /* merge intervals that are less than this apart, even though they 31 | don't overlap */ 32 | #define OVERLAP_SLACK 3600 33 | 34 | struct stack { 35 | struct interval *stack; 36 | int tail; 37 | int size; 38 | }; 39 | 40 | void stack_init(struct stack *s, int n) { 41 | s->stack = malloc(n * sizeof(struct interval)); 42 | s->tail = 0; 43 | s->size = n; 44 | } 45 | 46 | void stack_push(struct stack *s, struct interval ival) { 47 | if (s->tail == s->size) { 48 | s->size *= 2; 49 | s->stack = realloc(s->stack, s->size * sizeof(struct interval)); 50 | } 51 | s->stack[s->tail++] = ival; 52 | } 53 | 54 | int stack_pop(struct stack *s, struct interval *out) { 55 | if (s->tail > 0) { 56 | *out = s->stack[--s->tail]; 57 | return 1; 58 | } else { 59 | return 0; 60 | } 61 | }; 62 | 63 | struct interval *stack_top(struct stack *s) { 64 | if (s->tail > 0) { 65 | return &s->stack[s->tail - 1]; 66 | } else { 67 | return NULL; 68 | } 69 | } 70 | 71 | int cmp_interval(const void *a, const void *b) { 72 | const struct interval *ai = a, *bi = b; 73 | 74 | if (ai->stream_id != bi->stream_id) { 75 | return ai->stream_id - bi->stream_id; 76 | } else { 77 | if (ai->start != bi->start) { 78 | return ai->start - bi->start; 79 | } else { 80 | return ai->end - bi->end; 81 | } 82 | } 83 | }; 84 | 85 | /* load a dirty region file into memory and sort it. */ 86 | struct interval *parse_file(const char *filename, int *n) { 87 | FILE *fp = fopen(filename, "rx"); 88 | int cur_size = 128, cur_idx = 0; 89 | struct interval *rv; 90 | 91 | if (!fp) { 92 | return NULL; 93 | } 94 | 95 | rv = malloc(cur_size * sizeof(struct interval)); 96 | 97 | while (fscanf(fp, "%u\t%u\t%u\n", 98 | &rv[cur_idx].stream_id, 99 | &rv[cur_idx].start, 100 | &rv[cur_idx].end) == 3) { 101 | cur_idx ++; 102 | if (cur_idx == cur_size) { 103 | cur_size *= 2; 104 | rv = realloc(rv, cur_size * sizeof(struct interval)); 105 | } 106 | } 107 | *n = cur_idx; 108 | 109 | qsort(rv, cur_idx, sizeof(struct interval), cmp_interval); 110 | 111 | return rv; 112 | } 113 | 114 | /* take a sorted list of intervals 115 | 116 | return a list of intervals which are further than OVERLAP_SLACK 117 | apart, and cover all input intervals 118 | */ 119 | struct interval *merge_intervals(const struct interval *input, int n, int *out_n) { 120 | /* this is as big as the output can be if it's all unique */ 121 | struct interval *output = malloc(n * sizeof(struct interval)); 122 | int current_output = 0, i; 123 | struct stack stk; 124 | 125 | stack_init(&stk, 16); 126 | 127 | for (i = 0; i <= n; i++) { 128 | struct interval *top; 129 | struct interval cur = input[i]; 130 | cur.end += OVERLAP_SLACK; 131 | 132 | /* if we're into a new stream, output and restart */ 133 | if (i == 0 || i == n || input[i-1].stream_id != input[i].stream_id) { 134 | /* add the resulting intervals to the output */ 135 | while (stack_pop(&stk, &output[current_output]) != 0) { 136 | current_output ++; 137 | } 138 | if (i == n) { 139 | break; 140 | } else { 141 | /* restart on the next stream */ 142 | stack_push(&stk, cur); 143 | } 144 | } 145 | 146 | top = stack_top(&stk); 147 | if (top->end + 1 < cur.start) { 148 | stack_push(&stk, cur); 149 | } else if (top->end < cur.end) { 150 | top->end = cur.end; 151 | } 152 | } 153 | *out_n = current_output; 154 | return output; 155 | }; 156 | -------------------------------------------------------------------------------- /c6/intervals.h: -------------------------------------------------------------------------------- 1 | struct interval { 2 | uint32_t stream_id; 3 | uint32_t start; 4 | uint32_t end; 5 | }; 6 | 7 | struct interval *parse_file(const char *filename, int *n); 8 | struct interval *merge_intervals(const struct interval *input, int n, int *out_n); 9 | -------------------------------------------------------------------------------- /c6/logging.c: -------------------------------------------------------------------------------- 1 | /* 2 | * "Copyright (c) 2008 The Regents of the University of California. 3 | * All rights reserved." 4 | * 5 | * Permission to use, copy, modify, and distribute this software and its 6 | * documentation for any purpose, without fee, and without written agreement is 7 | * hereby granted, provided that the above copyright notice, the following 8 | * two paragraphs and the author appear in all copies of this software. 9 | * 10 | * IN NO EVENT SHALL THE UNIVERSITY OF CALIFORNIA BE LIABLE TO ANY PARTY FOR 11 | * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING OUT 12 | * OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF THE UNIVERSITY OF 13 | * CALIFORNIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | * 15 | * THE UNIVERSITY OF CALIFORNIA SPECIFICALLY DISCLAIMS ANY WARRANTIES, 16 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY 17 | * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS 18 | * ON AN "AS IS" BASIS, AND THE UNIVERSITY OF CALIFORNIA HAS NO OBLIGATION TO 19 | * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS." 20 | * 21 | */ 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include "logging.h" 31 | 32 | loglevel_t log_level; 33 | FILE *log_dest; 34 | 35 | 36 | const char *log_names[5] = {"DEBUG", 37 | "INFO", 38 | "WARN", 39 | "ERROR", 40 | "FATAL"}; 41 | 42 | /* buffers for log history */ 43 | static int log_size, log_end; 44 | static char log_history[LOGHISTSIZ][LOGLINESIZ]; 45 | 46 | #define LOG_INCR do { \ 47 | log_end = (log_end + 1) % LOGHISTSIZ; \ 48 | log_size = (log_size < LOGHISTSIZ) ? (log_size + 1) : log_size; \ 49 | } while (0); 50 | 51 | static int timestamp(char *buf, int len){ 52 | struct timeval tv; 53 | struct tm *ltime; 54 | int rv = 0; 55 | gettimeofday(&tv, NULL); 56 | // automatically calls tzset() 57 | ltime = localtime(&tv.tv_sec); 58 | rv += snprintf(buf, len, ISO8601_FMT(ltime, &tv)); 59 | rv += snprintf(buf + rv, len - rv, ": "); 60 | return rv; 61 | } 62 | 63 | loglevel_t log_setlevel(loglevel_t l) { 64 | loglevel_t old_lvl = log_level; 65 | log_level = l; 66 | return old_lvl; 67 | } 68 | 69 | loglevel_t log_getlevel() { 70 | return log_level; 71 | } 72 | 73 | void log_init() { 74 | log_level = LOGLVL_DEBUG; 75 | if (isatty(fileno(stderr))) { 76 | log_dest = stderr; 77 | } else { 78 | log_dest = NULL; 79 | } 80 | log_end = log_size = 0; 81 | } 82 | 83 | void log_addentry(char *buf) { 84 | strncpy(log_history[log_end], buf, LOGLINESIZ); 85 | log_history[log_end][LOGLINESIZ-1] = '\0'; 86 | LOG_INCR; 87 | } 88 | 89 | void log_log (loglevel_t level, const char *fmt, ...) { 90 | char buf[1024]; 91 | int buf_used = 0; 92 | va_list ap; 93 | va_start(ap, fmt); 94 | 95 | if (log_level > level) return; 96 | 97 | buf_used += timestamp(buf, 1024 - buf_used); 98 | buf_used += snprintf(buf + buf_used, 1024 - buf_used, "%s: ", log_names[level]); 99 | buf_used += vsnprintf(buf + buf_used, 1024 - buf_used, fmt, ap); 100 | 101 | if (log_dest) { 102 | fputs(buf, log_dest); 103 | } 104 | log_addentry(buf); 105 | 106 | va_end(ap); 107 | } 108 | 109 | void log_fatal_perror(const char *msg) { 110 | char buf[1024]; 111 | int in_errno = errno; 112 | if (in_errno < 0) return; 113 | if (!log_dest) return; 114 | timestamp(buf, 1024); 115 | fputs(buf, log_dest); 116 | 117 | fprintf(log_dest, "%s: ", log_names[LOGLVL_FATAL]); 118 | if (msg != NULL) fprintf(log_dest, "%s: ", msg); 119 | fprintf(log_dest, "%s\n", strerror(in_errno)); 120 | } 121 | 122 | void log_clear (loglevel_t level, const char *fmt, ...) { 123 | char buf[1024]; 124 | if (log_level > level) return; 125 | if (!log_dest) return; 126 | va_list ap; 127 | va_start(ap, fmt); 128 | vsnprintf(buf, 1024, fmt, ap); 129 | fputs(buf, log_dest); 130 | log_addentry(buf); 131 | va_end(ap); 132 | } 133 | 134 | /* print char* in hex format */ 135 | void log_dump_serial_packet(unsigned char *packet, const int len) { 136 | int i; 137 | if (log_level > LOGLVL_DEBUG) return; 138 | 139 | printf("len: %d\n", len); 140 | if (!packet) 141 | return; 142 | for (i = 0; i < len; i++) { 143 | printf("%02x ", packet[i]); 144 | } 145 | putchar('\n'); 146 | } 147 | 148 | -------------------------------------------------------------------------------- /c6/logging.h: -------------------------------------------------------------------------------- 1 | /* 2 | * "Copyright (c) 2008 The Regents of the University of California. 3 | * All rights reserved." 4 | * 5 | * Permission to use, copy, modify, and distribute this software and its 6 | * documentation for any purpose, without fee, and without written agreement is 7 | * hereby granted, provided that the above copyright notice, the following 8 | * two paragraphs and the author appear in all copies of this software. 9 | * 10 | * IN NO EVENT SHALL THE UNIVERSITY OF CALIFORNIA BE LIABLE TO ANY PARTY FOR 11 | * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING OUT 12 | * OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF THE UNIVERSITY OF 13 | * CALIFORNIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 14 | * 15 | * THE UNIVERSITY OF CALIFORNIA SPECIFICALLY DISCLAIMS ANY WARRANTIES, 16 | * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY 17 | * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS 18 | * ON AN "AS IS" BASIS, AND THE UNIVERSITY OF CALIFORNIA HAS NO OBLIGATION TO 19 | * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS." 20 | * 21 | */ 22 | #ifndef LOGGING_H_ 23 | #define LOGGING_H_ 24 | 25 | #include 26 | 27 | // SDH : log levels defined here 28 | // also edit the log names in logging.h 29 | typedef enum { 30 | LOGLVL_DEBUG = 0, 31 | LOGLVL_INFO = 1, 32 | LOGLVL_WARN = 2, 33 | LOGLVL_ERROR = 3, 34 | LOGLVL_FATAL = 4, 35 | } loglevel_t; 36 | 37 | extern const char *log_names[5]; 38 | extern loglevel_t log_level; 39 | extern FILE *log_dest; 40 | 41 | void log_init(); 42 | 43 | loglevel_t log_setlevel(loglevel_t l); 44 | loglevel_t log_getlevel(); 45 | 46 | void log_log (loglevel_t level, const char *fmt, ...); 47 | void log_fatal_perror(const char *msg); 48 | void log_clear (loglevel_t level, const char *fmt, ...); 49 | 50 | #define debug(fmt, args...) \ 51 | log_log(LOGLVL_DEBUG, fmt, ## args) 52 | #define info(fmt, args...) \ 53 | log_log(LOGLVL_INFO, fmt, ## args) 54 | #define warn(fmt, args...) \ 55 | log_log(LOGLVL_WARN, fmt, ## args) 56 | #define error(fmt, args...) \ 57 | log_log(LOGLVL_ERROR, fmt, ## args) 58 | #define fatal(fmt, args...) \ 59 | log_log(LOGLVL_FATAL, fmt, ## args) 60 | 61 | #define log_fprintf(X, FMT, ...) ; 62 | 63 | 64 | #define ISO8601_FMT(ltime, tv) "%04d-%02d-%02dT%02d:%02d:%02d.%03d%s", \ 65 | (ltime)->tm_year+1900, (ltime)->tm_mon+1, (ltime)->tm_mday, \ 66 | (ltime)->tm_hour, (ltime)->tm_min, (ltime)->tm_sec, (int)(tv)->tv_usec/ 1000, \ 67 | tzname[0] 68 | 69 | void log_dump_serial_packet(unsigned char *packet, const int len); 70 | 71 | /* the number of lines of log history to buffer */ 72 | #define LOGHISTSIZ 500 73 | /* the length of each buffered log line; longer entries are truncated */ 74 | #define LOGLINESIZ 100 75 | 76 | /* shell command for log history */ 77 | void log_show_log(int fd, int argc, char **argv); 78 | 79 | #endif 80 | -------------------------------------------------------------------------------- /c6/mkcron.sh: -------------------------------------------------------------------------------- 1 | 2 | VAR=$1 3 | BINDIR=$2 4 | 5 | cat < 5 | 6 | #include "pbuf/rdb.pb-c.h" 7 | #include "logging.h" 8 | #include "readingdb.h" 9 | 10 | struct config { 11 | long commit_interval; /* seconds */ 12 | loglevel_t loglevel; 13 | char data_dir[FILENAME_MAX]; 14 | unsigned short port; 15 | long cache_size; 16 | long deadlock_interval; 17 | long checkpoint_interval; 18 | int sketch; 19 | char sketch_log[FILENAME_MAX]; 20 | int ipv6; 21 | unsigned long compute_sketches; 22 | }; 23 | 24 | struct subdb { 25 | DB *dbp; 26 | char dbfile[128]; 27 | 28 | /* dirty data */ 29 | struct hashtable *dirty_data; 30 | pthread_mutex_t lock; 31 | }; 32 | 33 | void db_open(struct config *conf); 34 | void db_close(); 35 | int get_bucket(DBC *cursorp, struct rec_key *k, struct rec_val *v); 36 | int split_bucket(DB *dbp, DBC *cursorp, DB_TXN *tid, struct rec_key *k); 37 | int valid_bucketsize(int length); 38 | int add(struct config *c, DB *dbp, ReadingSet *rs); 39 | int add_enqueue(struct config *c, ReadingSet *rs, Response *reply); 40 | void commit_data(struct config *conf); 41 | 42 | void checkpoint_thread(void *p); 43 | void deadlock_thread(void *p); 44 | pthread_t **start_threads(struct config *c); 45 | void stop_threads(pthread_t **threads); 46 | 47 | 48 | #endif 49 | -------------------------------------------------------------------------------- /c6/reading-client.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | #include "readingdb.h" 15 | #include "util.h" 16 | 17 | 18 | int main() { 19 | IPC *ipp = ipc_open(0, 0); 20 | struct ipc_command *c = ipp->shm; 21 | struct ipc_reply *r = ipp->shm; 22 | int i; 23 | 24 | sem_wait(ipp->mutex_caller); 25 | printf("GOT CALLER\n"); 26 | c->command = COMMAND_SYNC; 27 | sem_post(ipp->mutex_server); 28 | sem_wait(ipp->mutex_reply); 29 | sem_post(ipp->mutex_caller); 30 | 31 | // try querying 32 | sem_wait(ipp->mutex_caller); 33 | c->command = COMMAND_QUERY; 34 | c->args.query.streamid = 320; 35 | c->args.query.starttime = 0; 36 | c->args.query.endtime = time(NULL); 37 | 38 | sem_post(ipp->mutex_server); 39 | sem_wait(ipp->mutex_reply); 40 | printf("got reply: %i records\n", r->data.query.nrecs); 41 | for (i = 0; i < r->data.query.nrecs; i++) { 42 | printf("%i %f\n", r->data.query.pts[i].timestamp, 43 | r->data.query.pts[i].reading); 44 | } 45 | sem_post(ipp->mutex_caller); 46 | ipc_close(ipp); 47 | return 0; 48 | } 49 | -------------------------------------------------------------------------------- /c6/readingcount.txt: -------------------------------------------------------------------------------- 1 | 1 373210 2 | 2 373710 3 | 3 373717 4 | 4 373721 5 | 5 373721 6 | 6 373721 7 | 7 373711 8 | 8 373713 9 | 9 373709 10 | 10 373714 11 | 11 373707 12 | 12 373712 13 | 13 373709 14 | 14 373712 15 | 15 373709 16 | 16 373709 17 | 17 373705 18 | 18 373707 19 | 19 373710 20 | 20 373707 21 | 21 373709 22 | 22 373712 23 | 23 373709 24 | 24 373712 25 | 25 373716 26 | 26 373718 27 | 27 373714 28 | 28 373714 29 | 29 373716 30 | 30 373711 31 | 31 373716 32 | 32 373713 33 | 33 373710 34 | 34 373713 35 | 35 373714 36 | 36 373718 37 | 37 373711 38 | 38 373715 39 | 39 373716 40 | 40 373711 41 | 41 373713 42 | 42 373714 43 | 43 373712 44 | 44 387339 45 | 45 387831 46 | 46 387838 47 | 47 387835 48 | 48 387829 49 | 49 387830 50 | 50 387832 51 | 51 387834 52 | 52 387837 53 | 53 387833 54 | 54 387827 55 | 55 387827 56 | 56 387820 57 | 57 387825 58 | 58 387831 59 | 59 387827 60 | 60 387827 61 | 61 387826 62 | 62 387835 63 | 63 387834 64 | 64 387833 65 | 65 387832 66 | 66 387833 67 | 67 387829 68 | 68 387831 69 | 69 387831 70 | 70 387834 71 | 71 387828 72 | 72 387830 73 | 73 387835 74 | 74 387829 75 | 75 387828 76 | 76 387819 77 | 77 387824 78 | 78 387828 79 | 79 387825 80 | 80 387830 81 | 81 387826 82 | 82 387826 83 | 83 387826 84 | 84 387827 85 | 85 387828 86 | 86 387822 87 | 87 387588 88 | 88 388080 89 | 89 388085 90 | 90 388080 91 | 91 388086 92 | 92 388083 93 | 93 388082 94 | 94 388084 95 | 95 388088 96 | 96 388082 97 | 97 388084 98 | 98 388081 99 | 99 388087 100 | 100 388085 101 | 101 388085 102 | 102 388078 103 | 103 388078 104 | 104 388076 105 | 105 388072 106 | 106 388080 107 | 107 388076 108 | 108 388080 109 | 109 388072 110 | 110 388076 111 | 111 388079 112 | 112 388071 113 | 113 388070 114 | 114 388070 115 | 115 388078 116 | 116 388069 117 | 117 388069 118 | 118 388063 119 | 119 388059 120 | 120 388065 121 | 121 388068 122 | 122 388072 123 | 123 388063 124 | 124 388059 125 | 125 388065 126 | 126 388062 127 | 127 388065 128 | 128 388063 129 | 129 388060 130 | 130 387828 131 | 131 388317 132 | 132 388320 133 | 133 388318 134 | 134 388320 135 | 135 388317 136 | 136 388317 137 | 137 388316 138 | 138 388313 139 | 139 388310 140 | 140 388314 141 | 141 388318 142 | 142 388313 143 | 143 388315 144 | 144 388314 145 | 145 388314 146 | 146 388322 147 | 147 388321 148 | 148 388315 149 | 149 388313 150 | 150 388322 151 | 151 388323 152 | 152 388322 153 | 153 388319 154 | 154 388322 155 | 155 388322 156 | 156 388319 157 | 157 388320 158 | 158 388318 159 | 159 388322 160 | 160 388319 161 | 161 388321 162 | 162 388319 163 | 163 388315 164 | 164 388314 165 | 165 388311 166 | 166 388315 167 | 167 388318 168 | 168 388319 169 | 169 388313 170 | 170 388306 171 | 171 388312 172 | 172 388314 173 | 173 387178 174 | 174 387670 175 | 175 387678 176 | 176 387676 177 | 177 387677 178 | 178 387683 179 | 179 387669 180 | 180 387676 181 | 181 387676 182 | 182 387671 183 | 183 387674 184 | 184 387675 185 | 185 387679 186 | 186 387673 187 | 187 387682 188 | 188 387676 189 | 189 387674 190 | 190 387669 191 | 191 387675 192 | 192 387668 193 | 193 387671 194 | 194 387663 195 | 195 387669 196 | 196 387669 197 | 197 387663 198 | 198 387672 199 | 199 387665 200 | 200 387672 201 | 201 387671 202 | 202 387670 203 | 203 387671 204 | 204 387669 205 | 205 387666 206 | 206 387668 207 | 207 387666 208 | 208 387674 209 | 209 387666 210 | 210 387667 211 | 211 387668 212 | 212 387672 213 | 213 387669 214 | 214 387666 215 | 215 387671 216 | 216 387990 217 | 217 388484 218 | 218 388487 219 | 219 388490 220 | 220 388479 221 | 221 388478 222 | 222 388483 223 | 223 388480 224 | 224 388483 225 | 225 388482 226 | 226 388479 227 | 227 388482 228 | 228 388486 229 | 229 388486 230 | 230 388483 231 | 231 388477 232 | 232 388477 233 | 233 388480 234 | 234 388477 235 | 235 388484 236 | 236 388478 237 | 237 388478 238 | 238 388482 239 | 239 388484 240 | 240 388479 241 | 241 388480 242 | 242 388477 243 | 243 388475 244 | 244 388483 245 | 245 388475 246 | 246 388482 247 | 247 388490 248 | 248 388480 249 | 249 388486 250 | 250 388487 251 | 251 388490 252 | 252 388487 253 | 253 388470 254 | 254 388476 255 | 255 388484 256 | 256 388485 257 | 257 388486 258 | 258 388478 259 | 259 388025 260 | 260 388502 261 | 261 388503 262 | 262 388503 263 | 263 388502 264 | 264 388498 265 | 265 388501 266 | 266 388491 267 | 267 388497 268 | 268 388495 269 | 269 388495 270 | 270 388495 271 | 271 388494 272 | 272 388496 273 | 273 388494 274 | 274 388497 275 | 275 388497 276 | 276 388494 277 | 277 388501 278 | 278 388500 279 | 279 388500 280 | 280 388500 281 | 281 388501 282 | 282 388504 283 | 283 388508 284 | 284 388503 285 | 285 388507 286 | 286 388501 287 | 287 388503 288 | 288 388504 289 | 289 388507 290 | 290 388514 291 | 291 388511 292 | 292 388511 293 | 293 388503 294 | 294 388512 295 | 295 388512 296 | 296 388510 297 | 297 388505 298 | 298 388505 299 | 299 388499 300 | 300 388502 301 | 301 388513 302 | 302 387861 303 | 303 388341 304 | 304 388344 305 | 305 388345 306 | 306 388342 307 | 307 388342 308 | 308 388333 309 | 309 388341 310 | 310 388339 311 | 311 388348 312 | 312 388345 313 | 313 388346 314 | 314 388347 315 | 315 388342 316 | 316 388341 317 | 317 388344 318 | 318 388343 319 | 319 388340 320 | 320 388343 321 | 321 388346 322 | 322 388341 323 | 323 388342 324 | 324 388346 325 | 325 388343 326 | 326 388347 327 | 327 388344 328 | 328 388344 329 | 329 388342 330 | 330 388342 331 | 331 388337 332 | 332 388338 333 | 333 388341 334 | 334 388333 335 | 335 388342 336 | 336 388332 337 | 337 388329 338 | 338 388333 339 | 339 388327 340 | 340 388333 341 | 341 388326 342 | 342 388330 343 | 343 388330 344 | 344 388323 345 | 345 384638 346 | 346 385138 347 | 347 385145 348 | 348 385142 349 | 349 385151 350 | 350 385151 351 | 351 385144 352 | 352 385142 353 | 353 385147 354 | 354 385145 355 | 355 385147 356 | 356 385142 357 | 357 385145 358 | 358 385147 359 | 359 385143 360 | 360 385145 361 | 361 385143 362 | 362 385148 363 | 363 385148 364 | 364 385142 365 | 365 385144 366 | 366 385141 367 | 367 385146 368 | 368 385143 369 | 369 385139 370 | 370 385136 371 | 371 385139 372 | 372 385140 373 | 373 385136 374 | 374 385138 375 | 375 385138 376 | 376 385138 377 | 377 385135 378 | 378 385132 379 | 379 385134 380 | 380 385130 381 | 381 385136 382 | 382 385137 383 | 383 385135 384 | 384 385134 385 | 385 385135 386 | 386 385140 387 | 387 385139 388 | 388 387860 389 | 389 388376 390 | 390 388378 391 | 391 388385 392 | 392 388380 393 | 393 388383 394 | 394 388378 395 | 395 388383 396 | 396 388379 397 | 397 388382 398 | 398 388378 399 | 399 388381 400 | 400 388376 401 | 401 388375 402 | 402 388381 403 | 403 388381 404 | 404 388377 405 | 405 388376 406 | 406 388379 407 | 407 388380 408 | 408 388381 409 | 409 388377 410 | 410 388376 411 | 411 388375 412 | 412 388375 413 | 413 388372 414 | 414 388373 415 | 415 388375 416 | 416 388376 417 | 417 388377 418 | 418 388374 419 | 419 388375 420 | 420 388372 421 | 421 388371 422 | 422 388372 423 | 423 388373 424 | 424 388379 425 | 425 388370 426 | 426 388367 427 | 427 388366 428 | 428 388365 429 | 429 388366 430 | 430 388367 431 | 431 389474 432 | 432 389967 433 | 433 389978 434 | 434 389980 435 | 435 389973 436 | 436 389977 437 | 437 389975 438 | 438 389975 439 | 439 389977 440 | 440 389974 441 | 441 389971 442 | 442 389974 443 | 443 389976 444 | 444 389977 445 | 445 389975 446 | 446 389979 447 | 447 389978 448 | 448 389974 449 | 449 389975 450 | 450 389976 451 | 451 389976 452 | 452 389969 453 | 453 389975 454 | 454 389975 455 | 455 389976 456 | 456 389971 457 | 457 389974 458 | 458 389971 459 | 459 389971 460 | 460 389973 461 | 461 389973 462 | 462 389965 463 | 463 389964 464 | 464 389970 465 | 465 389970 466 | 466 389967 467 | 467 389970 468 | 468 389971 469 | 469 389974 470 | 470 389969 471 | 471 389969 472 | 472 389974 473 | 473 389971 474 | 474 389357 475 | 475 389874 476 | 476 389866 477 | 477 389868 478 | 478 389868 479 | 479 389863 480 | 480 389865 481 | 481 389864 482 | 482 389862 483 | 483 389864 484 | 484 389859 485 | 485 389693 486 | 486 389843 487 | 487 389862 488 | 488 389862 489 | 489 389862 490 | 490 389864 491 | 491 389863 492 | 492 389859 493 | 493 389861 494 | 494 389857 495 | 495 389857 496 | 496 389866 497 | 497 389858 498 | 498 389863 499 | 499 389869 500 | 500 389869 501 | 501 389879 502 | 502 389876 503 | 503 389876 504 | 504 389874 505 | 505 389871 506 | 506 389869 507 | 507 389876 508 | 508 389875 509 | 509 389872 510 | 510 389876 511 | 511 389868 512 | 512 389876 513 | 513 389874 514 | 514 389879 515 | 515 389882 516 | 516 389880 517 | 517 -------------------------------------------------------------------------------- /c6/readingdb.c: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #include "readingdb.h" 13 | #include "readingdb_py.h" 14 | #include "util.h" 15 | 16 | #define TIMEOUT_SECS 10 17 | #define SIGREPLACE SIGTERM 18 | 19 | int call_start(IPC *ipp) { 20 | struct timespec ts; 21 | struct sigaction act; 22 | 23 | if (ipp == NULL) 24 | return -1; 25 | /* abort if we can't lock ourselves fast enough */ 26 | /* however, after we get this lock we must finish the protocol... */ 27 | if (clock_gettime(CLOCK_REALTIME, &ts) == -1) { 28 | PyErr_SetString(PyExc_IOError, "clock_gettime failed"); 29 | return -1; 30 | } 31 | ts.tv_sec += TIMEOUT_SECS; 32 | 33 | /* check if SA_RESTART is set on the SIGREPLACE handler. This could 34 | cause us to hang on exit */ 35 | ipp->sa_flags = 0; 36 | if (sigaction(SIGREPLACE, NULL, &act) < 0) 37 | return -1; 38 | if (act.sa_flags & SA_RESTART) { 39 | ipp->sa_flags = act.sa_flags; 40 | act.sa_flags &= ~SA_RESTART; 41 | if (sigaction(SIGREPLACE, &act, NULL) < 0) { 42 | ipp->sa_flags = 0; 43 | return -1; 44 | } 45 | } 46 | 47 | if (sem_timedwait(ipp->mutex_caller, &ts) < 0) { 48 | act.sa_flags = ipp->sa_flags; 49 | sigaction(SIGREPLACE, &act, NULL); 50 | return -1; 51 | } 52 | 53 | return 0; 54 | } 55 | 56 | int call_end(IPC *ipp) { 57 | struct sigaction act; 58 | while (sem_post(ipp->mutex_caller) < 0) { 59 | fprintf(stderr, "sem_post returned in call_end\n"); 60 | } 61 | 62 | if (ipp->sa_flags & SA_RESTART) { 63 | /* *shrug* */ 64 | if (sigaction(SIGREPLACE, NULL, &act) < 0) return -1; 65 | act.sa_flags = ipp->sa_flags; 66 | sigaction(SIGREPLACE, &act, NULL); 67 | } 68 | return 0; 69 | } 70 | 71 | IPC *db_open(void) { 72 | IPC *ipp = ipc_open(0, O_APPEND); 73 | if (ipp == NULL) { 74 | PyErr_SetString(PyExc_IOError, strerror(errno)); 75 | } 76 | return ipp; 77 | } 78 | 79 | void db_substream(IPC *ipp, int substream) { 80 | ipp->substream = substream; 81 | } 82 | 83 | void db_close(IPC *ipp) { 84 | if (ipp) { 85 | ipc_close(ipp); 86 | free(ipp); 87 | } 88 | } 89 | 90 | int db_sync(IPC *ipp) { 91 | struct ipc_command *c = ipp->shm; 92 | struct ipc_reply *r = ipp->shm; 93 | int rv = 0; 94 | 95 | if (call_start(ipp) < 0) { 96 | return -1; 97 | } 98 | 99 | c->command = COMMAND_SYNC; 100 | c->dbid = ipp->dbid; 101 | sem_post(ipp->mutex_server); 102 | while (sem_wait(ipp->mutex_reply) < 0); 103 | if (r->reply == REPLY_OK) { 104 | rv = -1; 105 | PyErr_Format(PyExc_IOError, "Error during query: %i", r->data.error); 106 | } 107 | call_end(ipp); 108 | return rv; 109 | } 110 | 111 | int db_add(IPC *ipp, int streamid, 112 | PyObject *values) { 113 | struct ipc_command *c = ipp->shm; 114 | struct ipc_reply *r = ipp->shm; 115 | int i, rv = 0; 116 | 117 | if (!PyList_Check(values)) { 118 | return -1; 119 | } 120 | 121 | if (PyList_Size(values) > MAXQUERYSET) { 122 | return -1; 123 | } 124 | 125 | if (call_start(ipp) < 0) 126 | return -1; 127 | 128 | c->command = COMMAND_ADD; 129 | c->dbid = ipp->dbid; 130 | c->streamid = streamid; 131 | c->args.add.n = PyList_Size(values); 132 | 133 | for (i = 0; i < c->args.add.n; i++) { 134 | PyObject *tuple = PyList_GetItem(values, i); 135 | 136 | if (PyArg_ParseTuple(tuple, "iiddd", 137 | &c->args.add.v[i].timestamp, 138 | &c->args.add.v[i].reading_sequence, 139 | &c->args.add.v[i].reading, 140 | &c->args.add.v[i].min, 141 | &c->args.add.v[i].max) < 0) { 142 | c->args.add.n = i; 143 | break; 144 | } 145 | } 146 | 147 | // printf("addinging ts: %i %f\n", readingtime, reading); 148 | 149 | sem_post(ipp->mutex_server); 150 | while (sem_wait(ipp->mutex_reply) < 0); 151 | if (r->reply == REPLY_OK) { 152 | rv = -1; 153 | PyErr_Format(PyExc_IOError, "Error during query: %i", r->data.error); 154 | } 155 | call_end(ipp); 156 | return rv; 157 | } 158 | 159 | PyObject *db_query(IPC *ipp, unsigned long long streamid, 160 | unsigned long long starttime, 161 | unsigned long long endtime) { 162 | struct ipc_command *c = ipp->shm; 163 | struct ipc_reply *r = ipp->shm; 164 | PyObject *ret = NULL, *record; 165 | int i; 166 | 167 | if (call_start(ipp) < 0) 168 | return NULL; 169 | 170 | c->command = COMMAND_QUERY; 171 | c->dbid = ipp->dbid; 172 | c->streamid = streamid; 173 | c->args.query.starttime = starttime; 174 | c->args.query.endtime = endtime; 175 | 176 | sem_post(ipp->mutex_server); 177 | while (sem_wait(ipp->mutex_reply) < 0); 178 | 179 | if (r->reply == REPLY_ERROR) { 180 | PyErr_Format(PyExc_IOError, "Error during query: %i", r->data.error); 181 | ret = NULL; 182 | } else if (r->reply == REPLY_QUERY) { 183 | ret = PyList_New(r->data.query.nrecs); 184 | if (ret == NULL) { 185 | PyErr_SetString(PyExc_IOError, "couldn't allocate list"); 186 | return NULL; 187 | } 188 | for (i = 0; i < r->data.query.nrecs; i++) { 189 | record = Py_BuildValue("iiddd", r->data.query.pts[i].timestamp, 190 | r->data.query.pts[i].reading_sequence, 191 | r->data.query.pts[i].reading, 192 | r->data.query.pts[i].min, 193 | r->data.query.pts[i].max); 194 | PyList_SetItem(ret, i, record); 195 | } 196 | } 197 | call_end(ipp); 198 | return ret; 199 | } 200 | -------------------------------------------------------------------------------- /c6/readingdb.h: -------------------------------------------------------------------------------- 1 | #ifndef _READINGDB_H_ 2 | #define _READINGDB_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | /* types used for shared memory communication */ 10 | #include 11 | #include 12 | #include 13 | 14 | #define MAX_SUBSTREAMS 14 15 | 16 | /* max number of client threads */ 17 | #define MAXCONCURRENCY 150 18 | 19 | #define NBUCKETSIZES 3 20 | #define MAXBUCKETRECS (60 * 5) /* has to be >= the number of records 21 | in the smallest bucket */ 22 | 23 | 24 | #define COMPRESS_WORKING_BUF 64000 25 | 26 | 27 | #define DEFAULT_PAGESIZE 16384 28 | /* must be sorted */ 29 | extern int bucket_sizes[NBUCKETSIZES]; 30 | 31 | #define MAXQUERYSET 128 32 | 33 | /* record definitions for the bdb instance */ 34 | struct rec_key { 35 | /* these are in network byte-order */ 36 | uint32_t stream_id; 37 | uint32_t timestamp; 38 | }; 39 | 40 | struct point { 41 | uint32_t timestamp; 42 | uint32_t reading_sequence; 43 | double reading; 44 | double min; 45 | double max; 46 | } __attribute__((packed)); 47 | 48 | #define POINT_OFF(IDX) ((((IDX) > 0 ? (IDX) : 0) * (sizeof(struct point))) + (sizeof (struct rec_val))) 49 | struct rec_val { 50 | uint32_t n_valid; 51 | uint32_t period_length; 52 | uint32_t tail_timestamp; 53 | struct point data[0]; 54 | }; 55 | 56 | #define SMALL_POINTS 128 57 | struct point_list { 58 | int n; 59 | struct point v[SMALL_POINTS]; 60 | }; 61 | 62 | struct sock_request { 63 | int sock; 64 | FILE *sock_fp; 65 | int substream; 66 | }; 67 | 68 | struct ipc_command { 69 | enum { 70 | COMMAND_QUERY = 1, 71 | COMMAND_ADD = 2, 72 | COMMAND_SYNC = 3, 73 | } command; 74 | int dbid; 75 | unsigned long long streamid; 76 | union { 77 | struct { 78 | unsigned long long starttime; 79 | unsigned long long endtime; 80 | } query; 81 | struct point_list add; 82 | } args; 83 | }; 84 | 85 | struct ipc_reply { 86 | enum { 87 | REPLY_OK = 0, 88 | REPLY_QUERY = 1, 89 | REPLY_ERROR = 2, 90 | } reply; 91 | union { 92 | struct { 93 | int nrecs; 94 | struct point pts[0]; 95 | } query; 96 | enum { 97 | E_INVAL_SUBSTREAM = 1, 98 | } error; 99 | } data;; 100 | }; 101 | 102 | struct pbuf_header { 103 | uint32_t message_type; 104 | uint32_t body_length; 105 | }; 106 | 107 | #define MAX_PBUF_MESSAGE 1000000 108 | 109 | /* compression functions in compress.c */ 110 | int val_compress(struct rec_val *v, void *buf, int len); 111 | int val_decompress(void *cmpr, int cmpr_len, struct rec_val *v, int v_len); 112 | 113 | typedef enum { 114 | FALSE = 0, TRUE = 1 115 | } bool_t; 116 | 117 | #endif 118 | -------------------------------------------------------------------------------- /c6/readingdb_py.h: -------------------------------------------------------------------------------- 1 | #ifndef _READINGDB_PY_H_ 2 | #define _READINGDB_PY_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #include "readingdb.h" 9 | 10 | /* exported python definition */ 11 | IPC *db_open(void); 12 | void db_substream(IPC *dpb, int substream); 13 | void db_close(IPC *dbp); 14 | int db_sync(IPC *dbp); 15 | PyObject *db_query(IPC *dbp, unsigned long long streamid, 16 | unsigned long long starttime, 17 | unsigned long long endtime) ; 18 | int db_add(IPC *dbp, int streamid, PyObject *values); 19 | 20 | #endif 21 | -------------------------------------------------------------------------------- /c6/rpc.c: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | #include 4 | #include 5 | 6 | #include "readingdb.h" 7 | #include "pbuf/rdb.pb-c.h" 8 | 9 | void _rpc_copy_records(struct point *dest, Reading **src, int n) { 10 | int i; 11 | for (i = 0; i < n; i++) { 12 | dest[i].timestamp = src[i]->timestamp; 13 | dest[i].reading = src[i]->value; 14 | 15 | if (src[i]->has_seqno) 16 | dest[i].reading_sequence = src[i]->seqno; 17 | else 18 | dest[i].reading_sequence = 0; 19 | 20 | if (src[i]->has_max) 21 | dest[i].max = src[i]->max; 22 | else 23 | dest[i].max = LLONG_MAX; 24 | 25 | if (src[i]->has_min) 26 | dest[i].min = src[i]->min; 27 | else 28 | dest[i].min = LLONG_MIN; 29 | } 30 | } 31 | 32 | void _rpc_copy_reading(Reading *dest, struct point *src) { 33 | double bottom = LLONG_MIN + 1, top = LLONG_MAX - 1; 34 | reading__init(dest); 35 | 36 | dest->timestamp = src->timestamp; 37 | dest->value = src->reading; 38 | 39 | /* optional fields */ 40 | if (src->reading_sequence != 0) 41 | dest->has_seqno = 1; 42 | if (src->min > bottom) 43 | dest->has_min = 1; 44 | if (src->max < top) 45 | dest->has_max = 1; 46 | 47 | dest->seqno = src->reading_sequence; 48 | dest->min = src->min; 49 | dest->max = src->max; 50 | } 51 | 52 | ReadingSet *_rpc_alloc_rs(int n) { 53 | int i; 54 | ReadingSet *points; 55 | Reading *points_data; 56 | Reading **points_vec; 57 | 58 | /* allocate a copy of the input reading set */ 59 | points = malloc(sizeof(ReadingSet)); 60 | points_vec = malloc(sizeof(Reading *) * n); 61 | points_data = malloc(sizeof(Reading) * n); 62 | if (!points || !points_vec || !points_data) { 63 | if (points) free(points); 64 | if (points_vec) free(points_vec); 65 | if (points_data) free(points_data); 66 | return NULL; 67 | } 68 | 69 | reading_set__init(points); 70 | points->n_data = 0; 71 | points->data = points_vec; 72 | for (i = 0; i < n; i++) 73 | points_vec[i] = &points_data[i]; 74 | 75 | return points; 76 | } 77 | 78 | void _rpc_free_rs(ReadingSet *rs) { 79 | free(rs->data[0]); 80 | free(rs->data); 81 | free(rs); 82 | } 83 | 84 | int rpc_send_reply(struct sock_request *request, 85 | Response *response) { 86 | void *reply; 87 | int reply_len, i_alloced = 0; 88 | if (response->data == NULL) { 89 | response->data = _rpc_alloc_rs(1); 90 | i_alloced = 1; 91 | } 92 | 93 | reply_len = response__get_packed_size(response); 94 | reply = malloc(reply_len); 95 | if (reply) { 96 | struct pbuf_header h; 97 | h.message_type = htonl(MESSAGE_TYPE__RESPONSE); 98 | h.body_length = htonl(reply_len); 99 | response__pack(response, reply); 100 | if (fwrite(&h, sizeof(h), 1, request->sock_fp) <= 0) 101 | goto fail; 102 | if (fwrite(reply, reply_len, 1, request->sock_fp) <= 0) 103 | goto fail; 104 | // info("wrote %i bytes\n", sizeof(h) + reply_len); 105 | fflush(request->sock_fp); 106 | free(reply); 107 | if (i_alloced) 108 | _rpc_free_rs(response->data); 109 | return 0; 110 | } 111 | fail: 112 | if (i_alloced) 113 | _rpc_free_rs(response->data); 114 | return -1; 115 | } 116 | 117 | -------------------------------------------------------------------------------- /c6/rpc.h: -------------------------------------------------------------------------------- 1 | #ifndef _RPC_H_ 2 | #define _RPC_H_ 3 | 4 | #include "pbuf/rdb.pb-c.h" 5 | 6 | void _rpc_copy_records(struct point *dest, Reading **src, int n); 7 | void _rpc_copy_reading(Reading *dest, struct point *src); 8 | ReadingSet *_rpc_alloc_rs(int n); 9 | void _rpc_free_rs(ReadingSet *rs); 10 | void rpc_send_reply(struct sock_request *request, 11 | Response *response); 12 | 13 | #endif 14 | -------------------------------------------------------------------------------- /c6/sketch.c: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include "stats.h" 12 | #include "rdb.h" 13 | #include "rpc.h" 14 | #include "pbuf/rdb.pb-c.h" 15 | #include "commands.h" 16 | #include "sketch.h" 17 | #include "intervals.h" 18 | #include "config.h" 19 | 20 | sig_atomic_t do_shutdown = 0; 21 | pthread_mutex_t stats_lock = PTHREAD_MUTEX_INITIALIZER; 22 | struct stats stats = {0}; 23 | extern struct subdb dbs[MAX_SUBSTREAMS]; 24 | extern DB_ENV *env; 25 | 26 | 27 | /* 28 | * Return a list of computed stats over a window. 29 | */ 30 | ReadingSet **w_stats(ReadingSet *window, 31 | unsigned long long start, 32 | unsigned long long end, 33 | int windowlen) { 34 | ReadingSet **rv = malloc(sizeof(ReadingSet *) * 4); 35 | int i = 0, window_idx = 0, j; 36 | unsigned long long current; 37 | 38 | for (j = 0; j < 4; j++) { 39 | rv[j] = _rpc_alloc_rs(((end - start) / windowlen) + 1); 40 | rv[j]->n_data = 0; 41 | } 42 | 43 | /* iterate over the sub windows */ 44 | for (current = start; current < end && i < window->n_data; current += windowlen) { 45 | int count = 0; 46 | double sum = 0., min = INFINITY, max = -INFINITY; 47 | 48 | for (; i < window->n_data && 49 | window->data[i]->timestamp >= current && 50 | window->data[i]->timestamp < current + windowlen; 51 | i++) { 52 | sum += window->data[i]->value; 53 | count += 1; 54 | if (window->data[i]->value < min) { 55 | min = window->data[i]->value; 56 | } 57 | if (window->data[i]->value > max) { 58 | max = window->data[i]->value; 59 | } 60 | }; 61 | 62 | if (count > 0) { 63 | rv[SKETCH__SKETCH_TYPE__COUNT-1]->data[window_idx]->timestamp = current; 64 | rv[SKETCH__SKETCH_TYPE__COUNT-1]->data[window_idx]->value = count; 65 | 66 | rv[SKETCH__SKETCH_TYPE__MEAN-1]->data[window_idx]->timestamp = current; 67 | rv[SKETCH__SKETCH_TYPE__MEAN-1]->data[window_idx]->value = (sum / count); 68 | 69 | rv[SKETCH__SKETCH_TYPE__MIN-1]->data[window_idx]->timestamp = current; 70 | rv[SKETCH__SKETCH_TYPE__MIN-1]->data[window_idx]->value = min; 71 | 72 | rv[SKETCH__SKETCH_TYPE__MAX-1]->data[window_idx]->timestamp = current; 73 | rv[SKETCH__SKETCH_TYPE__MAX-1]->data[window_idx]->value = max; 74 | 75 | window_idx ++; 76 | } 77 | }; 78 | rv[SKETCH__SKETCH_TYPE__COUNT-1]->n_data = window_idx; 79 | rv[SKETCH__SKETCH_TYPE__MEAN-1]->n_data = window_idx; 80 | rv[SKETCH__SKETCH_TYPE__MIN-1]->n_data = window_idx; 81 | rv[SKETCH__SKETCH_TYPE__MAX-1]->n_data = window_idx; 82 | return rv; 83 | } 84 | 85 | 86 | /* 87 | * Update the sketches for a streamid in the provided window. 88 | */ 89 | void update_sketches(struct config *c, 90 | unsigned int streamid, 91 | unsigned int start, 92 | unsigned int end) { 93 | Query q = QUERY__INIT; 94 | Response r = RESPONSE__INIT; 95 | unsigned long int current; 96 | unsigned long int fetch_period = 3600; /* SDH : this should be computed from the sketches list */ 97 | int i, j; 98 | 99 | r.data = _rpc_alloc_rs(MAXRECS); 100 | memset(&q, 0, sizeof(q)); 101 | q.streamid = streamid; 102 | q.substream = 0; 103 | 104 | /* iterate over our fetch chunks */ 105 | for (current = start - (start % fetch_period); current <= end; current += fetch_period) { 106 | int cursubstream = 1; 107 | /* pull the data */ 108 | r.data->n_data = 0; 109 | q.starttime = current; 110 | q.endtime = current + fetch_period; 111 | query(dbs[0].dbp, &q, &r, QUERY_DATA); 112 | debug("found %i records\n", r.data->n_data); 113 | 114 | /* iterate over the sketches we maintain */ 115 | for (i = 0; i < 3; i++) { 116 | ReadingSet **rv = 117 | w_stats(r.data, current, current + fetch_period, sketches[i].period); 118 | /* add the substreams back as data in the right substream*/ 119 | for (j = 0; j < sketches[i].nsubstreams; j++) { 120 | if ((1 << cursubstream) & c->compute_sketches && rv[j] && rv[j]->n_data) { 121 | debug("got %i records from filter, %i %i\n", rv[j]->n_data, cursubstream, j); 122 | rv[j]->streamid = streamid; 123 | rv[j]->substream = cursubstream; 124 | if (add(c, dbs[cursubstream].dbp, rv[j]) < 0) { 125 | warn("adding data failed...\n"); 126 | } 127 | } 128 | if (rv[j]) { 129 | _rpc_free_rs(rv[j]); 130 | } 131 | cursubstream ++; 132 | assert(cursubstream < MAX_SUBSTREAMS); 133 | } 134 | free(rv); 135 | } 136 | } 137 | for (i = 1; i < MAX_SUBSTREAMS; i ++) { 138 | dbs[i].dbp->sync(dbs[i].dbp, 0); 139 | } 140 | _rpc_free_rs(r.data); 141 | } 142 | 143 | void usage(char *progname) { 144 | fprintf(stderr, 145 | "\n\t%s [options]\n" 146 | "\t\t-V print version and exit\n" 147 | "\t\t-v verbose\n" 148 | "\t\t-h help\n" 149 | "\t\t-s cache size (32MB)\n" 150 | "\t\t-d set data directory (%s)\n\n", 151 | progname, DATA_DIR); 152 | } 153 | 154 | void default_config(struct config *c) { 155 | char *cur; 156 | c->loglevel = LOGLVL_INFO; 157 | 158 | strcpy(c->data_dir, DATA_DIR); 159 | cur = stpncpy(c->sketch_log, c->data_dir, sizeof(c->sketch_log) - 1); 160 | *cur++ = '/'; 161 | stpncpy(cur, DIRTY_SKETCH_LOGFILE, 162 | sizeof(c->sketch_log) - (cur - c->sketch_log)); 163 | 164 | c->cache_size = 32; 165 | c->sketch = 0; 166 | c->compute_sketches = 0; 167 | } 168 | 169 | int optparse(int argc, char **argv, struct config *c) { 170 | char o; 171 | char *endptr; 172 | long int temp; 173 | while ((o = getopt(argc, argv, "Vvhd:s:r:")) != -1) { 174 | switch (o) { 175 | case 'h': 176 | usage(argv[0]); 177 | return -1; 178 | break; 179 | case 'V': 180 | printf("%s\n", PACKAGE_STRING); 181 | return -1; 182 | case 'v': 183 | c->loglevel = LOGLVL_DEBUG; 184 | break; 185 | case 'd': 186 | strncpy(c->data_dir, optarg, FILENAME_MAX); 187 | endptr = stpncpy(c->sketch_log, optarg, FILENAME_MAX); 188 | *endptr++ = '/'; 189 | stpncpy(endptr, DIRTY_SKETCH_LOGFILE, sizeof(c->sketch_log) - (endptr - c->sketch_log)); 190 | break; 191 | case 's': 192 | c->cache_size = strtol(optarg, &endptr, 10); 193 | if (endptr == optarg) { 194 | fatal("Invalid cache size\n"); 195 | return -1; 196 | } 197 | case 'r': 198 | temp = strtol(optarg, &endptr, 10); 199 | if (endptr == optarg || temp < 0 || temp >= 12) { 200 | fatal("Invalid sketch number\n"); 201 | return -1; 202 | } 203 | c->compute_sketches |= 1 << temp; 204 | break; 205 | 206 | } 207 | } 208 | if (c->compute_sketches == 0) { 209 | info("Updating all sketches\n"); 210 | c->compute_sketches = 0xffff; 211 | } else { 212 | for (temp = 1; temp <= 12; temp++) { 213 | if (c->compute_sketches & (1 << temp)) { 214 | info("Updating sketch %i: %s(%i)\n", 215 | temp, 216 | sketch_names[(temp % 4)], 217 | sketches[(temp - 1) / 4].period); 218 | } 219 | } 220 | } 221 | return 0; 222 | } 223 | 224 | 225 | /* 226 | * Run to process the log file from reading-server, and recompute 227 | * sketches on dirty regions. 228 | * 229 | * Holds an advisory lock on the log file, so it safe to call multiple 230 | * times. 231 | */ 232 | int update_from_log(struct config *c) { 233 | FILE *lock_fp; 234 | char lockfile[1024], workfile[1024], *cur; 235 | struct stat sb; 236 | int ret, input_recs, merged_recs, i; 237 | unsigned int streamid, starttime, endtime; 238 | struct interval *input, *merged; 239 | 240 | /* 241 | Locking protocol: 242 | 243 | 1. Open the current work log and obtain an advisory lock on it. 244 | If this fails, some other process must be working so we exit. 245 | 246 | 2. Check if there's an existing work file. If there was, someone 247 | else must not have finished, so use that one to avoid loosing 248 | data. 249 | 250 | 3. If there isn't one, the last guy must have finished 251 | successfully, so rename the current log to a workfile and process 252 | that. 253 | 254 | TODO: 255 | 256 | This would be made much more efficient by first computing the 257 | non-overlapping intervals and then iterating over those. 258 | */ 259 | 260 | /* name the log file name */ 261 | memcpy(workfile, c->sketch_log, sizeof(workfile)); 262 | strcpy(workfile + strlen(c->sketch_log), ".work"); 263 | info("Updating sketches from logfile %s\n", c->sketch_log);; 264 | 265 | memcpy(lockfile, c->sketch_log, sizeof(lockfile)); 266 | strcpy(lockfile + strlen(c->sketch_log), ".lock"); 267 | 268 | lock_fp = fopen(lockfile, "a"); 269 | if (!lock_fp) { 270 | warn("Could not open logfile; cannot proceed\n"); 271 | return -1; 272 | } 273 | 274 | if (flock(fileno(lock_fp), LOCK_EX | LOCK_NB) < 0) { 275 | warn("Log file is locked; aborting\n"); 276 | return -1; 277 | } 278 | 279 | if (stat(workfile, &sb) == 0) { 280 | /* there's a work file already there (from a previous invocation?) 281 | so process it without copying it. */ 282 | info("Work file exists, so proceeding using it\n"); 283 | } else { 284 | info ("Copying log file to work file\n"); 285 | if (rename(c->sketch_log, workfile) < 0) { 286 | error("Moving logfile failed: aborting: %s\n", strerror(errno)); 287 | return -1; 288 | } 289 | } 290 | 291 | /* Workfile now contains the filename we want to process. We will keep 292 | the logfile open to hold the write lock; we'll close that and 293 | release the lock on exit. */ 294 | input = parse_file(workfile, &input_recs); 295 | if (!input) { 296 | error("Can't open work file (although it must exist?): %s\n", strerror(errno)); 297 | return -1; 298 | } 299 | 300 | merged = merge_intervals(input, input_recs, &merged_recs); 301 | 302 | info("merged %i regions into %i\n", input_recs, merged_recs); 303 | 304 | for (i = 0; i < merged_recs; i++) { 305 | info("updating sketches for streamid: %i from: %i starttime to: %i: endtime\n", 306 | merged[i].stream_id, merged[i].start, merged[i].end); 307 | update_sketches(c, merged[i].stream_id, merged[i].start, merged[i].end); 308 | } 309 | free(merged); 310 | free(input); 311 | 312 | if ((ret = env->txn_checkpoint(env, 10, 0, 0)) != 0) { 313 | warn("txn_checkpoint: %s\n", db_strerror(ret)); 314 | } 315 | 316 | remove(workfile); 317 | return 0; 318 | }; 319 | 320 | 321 | int main(int argc, char **argv) { 322 | struct config c; 323 | 324 | default_config(&c); 325 | log_init(); 326 | 327 | if (optparse(argc, argv, &c) < 0) { 328 | exit(1); 329 | } 330 | 331 | drop_priv(); 332 | 333 | log_setlevel(c.loglevel); 334 | 335 | db_open(&c); 336 | 337 | update_from_log(&c); 338 | 339 | db_close(); 340 | } 341 | -------------------------------------------------------------------------------- /c6/sketch.h: -------------------------------------------------------------------------------- 1 | #ifndef SKETCH_H 2 | #define SKETCH_H 3 | 4 | #include 5 | 6 | #include "pbuf/rdb.pb-c.h" 7 | 8 | /* sketch definitions. 9 | * 10 | * These must be sorted smallest to largest, and all window sizes must 11 | * divide the largest window size. The results will be placed into 12 | * the substream corresponding to the sketch definition index in this 13 | * array. 14 | */ 15 | struct sketch { 16 | /* period of sketch computation, seconds */ 17 | int period; 18 | /* function that computes the window */ 19 | /* a window function gets passed a readingset, start, and end value */ 20 | /* it should return a new ReadingSet of window values it wants to update */ 21 | int nsubstreams; 22 | } sketches [] = { 23 | { 300, 4}, /* returns count, mean, min, max */ 24 | { 900, 4}, /* returns count, mean, min, max */ 25 | { 3600, 4}, 26 | }; 27 | 28 | /* these match the sketch type enum in rdb.proto */ 29 | char *sketch_names[] = { 30 | "null", 31 | "count", 32 | "mean", 33 | "min", 34 | "max", 35 | "first", 36 | "median" 37 | }; 38 | 39 | #define DIRTY_SKETCH_LOGFILE "dirty_sketches" 40 | 41 | /* map a sketch to a substream, badly... */ 42 | int get_sketch_substream(Sketch *s) { 43 | if (!(s->type > 0 && s->type <= SKETCH__SKETCH_TYPE__MAX)) { 44 | return -1; 45 | } 46 | switch (s->window) { 47 | case 300: 48 | return ((int)s->type); 49 | case 900: 50 | return ((int)s->type) + 4; 51 | case 3600: 52 | return ((int)s->type) + 8; 53 | default: 54 | return -1; 55 | } 56 | }; 57 | 58 | #endif 59 | -------------------------------------------------------------------------------- /c6/stats-sock.c: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #include "logging.h" 13 | #include "stats.h" 14 | 15 | static int sock = -1; 16 | 17 | void stats_init(unsigned short p) { 18 | struct sockaddr_in6 dest = { 19 | .sin6_family = AF_INET6, 20 | .sin6_addr = IN6ADDR_LOOPBACK_INIT, 21 | .sin6_port = htons(p), 22 | }; 23 | sock = socket(AF_INET6, SOCK_DGRAM, 0); 24 | if (sock < 0) { 25 | warn("stats_init: socket: %s\n", strerror(errno)); 26 | return; 27 | } 28 | 29 | if (connect(sock, (struct sockaddr *)&dest, sizeof(dest)) < 0) { 30 | warn("stats_init: connect: %s\n", strerror(errno)); 31 | return; 32 | } 33 | } 34 | 35 | void stats_report(struct stats *s, struct timeval *when) { 36 | char buf[1024]; 37 | int n; 38 | 39 | if (sock < 0) { 40 | return; 41 | } 42 | 43 | n = snprintf(buf, sizeof(buf), 44 | "{\"timestamp\": %li.%06li, " 45 | "\"queries\": %i, \"adds\": %i, " 46 | "\"failed_adds\": %i, " 47 | "\"connects\": %i, \"disconnects\": %i," 48 | "\"nearest\": %i }\n", 49 | when->tv_sec, when->tv_usec, 50 | s->queries, s->adds, s->failed_adds, 51 | s->connects, s->disconnects, 52 | s->nearest); 53 | 54 | if (send(sock, buf, n, MSG_DONTWAIT) < 0) { 55 | debug("stats_report: send: %s\n", strerror(errno)); 56 | } 57 | } 58 | 59 | void stats_close() { 60 | close(sock); 61 | } 62 | -------------------------------------------------------------------------------- /c6/stats.h: -------------------------------------------------------------------------------- 1 | #ifndef STATS_H_ 2 | #define STATS_H_ 3 | 4 | #include 5 | 6 | struct stats { 7 | int queries, adds, failed_adds, connects, disconnects, nearest, deletes; 8 | }; 9 | #define INCR_STAT(STAT) { pthread_mutex_lock(&stats_lock); \ 10 | stats.STAT ++; \ 11 | pthread_mutex_unlock(&stats_lock); } 12 | 13 | /* open the socket once */ 14 | void stats_init(unsigned short port); 15 | 16 | /* report the statistics to any listener */ 17 | void stats_report(struct stats *s, struct timeval *ts); 18 | 19 | /* exit */ 20 | void stats_close(); 21 | 22 | extern pthread_mutex_t stats_lock; 23 | extern struct stats stats; 24 | 25 | #endif 26 | -------------------------------------------------------------------------------- /c6/util-db.c: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | #include "logging.h" 22 | #include "readingdb.h" 23 | #include "config.h" 24 | 25 | #ifdef WRITE_COMPRESSION_LOG 26 | #define CMPR_LOG "cmpr.log" 27 | static void write_cmprlog(unsigned long long streamid, 28 | unsigned long long substream, 29 | unsigned long long base, 30 | int nvalid, 31 | size_t uncmpr, size_t cmpr) { 32 | FILE *fp = fopen(CMPR_LOG, "a"); 33 | if (!fp) return; 34 | fprintf(fp, "%llu,%llu,%llu,%i,%llu,%llu\n", 35 | streamid, substream, base, nvalid, uncmpr, cmpr); 36 | fclose(fp); 37 | } 38 | #endif 39 | 40 | int put(DB *dbp, DB_TXN *txn, struct rec_key *k, struct rec_val *v) { 41 | struct rec_key put_k; 42 | int ret; 43 | DBT key, data; 44 | int len; 45 | void *data_buf; 46 | 47 | #ifdef USE_COMPRESSION 48 | char cmpr_buf[COMPRESS_WORKING_BUF]; 49 | if ((ret = val_compress(v, cmpr_buf, sizeof(cmpr_buf))) < 0) { 50 | warn("Compression failed!\n"); 51 | assert (0); 52 | } 53 | 54 | debug("compress: streamid: %i timestamp: 0x%x n_valid: %i %lu -> %i\n", 55 | k->stream_id, k->timestamp, 56 | v->n_valid, POINT_OFF(v->n_valid), ret); 57 | 58 | #ifdef WRITE_COMPRESSION_LOG 59 | write_cmprlog(k->stream_id, 0, k->timestamp, 60 | v->n_valid, POINT_OFF(v->n_valid), ret); 61 | #endif 62 | 63 | data_buf = cmpr_buf; 64 | len = ret; 65 | #else 66 | #warn Database compression disabled 67 | data_buf = v; 68 | len = POINT_OFF(v->n_valid); 69 | #endif 70 | 71 | bzero(&key, sizeof(key)); 72 | bzero(&data, sizeof(data)); 73 | 74 | debug("put stream_id: %i timestamp: 0x%x len: %i\n", 75 | k->stream_id, k->timestamp, len); 76 | 77 | put_k.stream_id = htonl(k->stream_id); 78 | put_k.timestamp = htonl(k->timestamp); 79 | 80 | key.data = &put_k; 81 | key.size = key.ulen = sizeof(struct rec_key); 82 | key.flags = DB_DBT_USERMEM; 83 | 84 | data.data = data_buf; 85 | data.size = data.ulen = len; 86 | data.flags = DB_DBT_USERMEM; 87 | 88 | if ((ret = dbp->put(dbp, txn, &key, &data, 0)) != 0) { 89 | error("db put: %s\n", db_strerror(ret)); 90 | return -1; 91 | } 92 | return 0; 93 | } 94 | 95 | /** Lookup the key passed in using an exact match 96 | * @cursorp cursor to use 97 | * @k (input) the key to lookup 98 | * @buf output buffer of length 99 | * @len 100 | */ 101 | int get(DBC *cursorp, int flags, struct rec_key *k, struct rec_val *v, int len) { 102 | int ret; 103 | struct rec_key get_k; 104 | DBT key, data; 105 | char zbuf[COMPRESS_WORKING_BUF]; 106 | bzero(&key, sizeof(key)); 107 | bzero(&data, sizeof(data)); 108 | 109 | debug("get stream_id: %i timestamp: 0x%x\n", k->stream_id, k->timestamp); 110 | 111 | get_k.stream_id = htonl(k->stream_id); 112 | get_k.timestamp = htonl(k->timestamp); 113 | 114 | key.data = &get_k; 115 | key.size = key.ulen = sizeof(struct rec_key); 116 | key.flags = DB_DBT_USERMEM; 117 | 118 | #ifdef USE_COMPRESSION 119 | data.data = zbuf; 120 | data.size = data.ulen = sizeof(zbuf); 121 | data.flags = DB_DBT_USERMEM; 122 | #else 123 | data.data = v; 124 | data.size = data.ulen = len; 125 | data.flags = DB_DBT_USERMEM; 126 | #endif 127 | 128 | if ((ret = cursorp->get(cursorp, &key, &data, flags)) == 0) { 129 | if (flags & DB_NEXT || flags & DB_SET_RANGE) { 130 | k->stream_id = ntohl(get_k.stream_id); 131 | k->timestamp = ntohl(get_k.timestamp); 132 | } 133 | 134 | #ifdef USE_COMPRESSION 135 | val_decompress(zbuf, data.size, v, len); 136 | 137 | debug("decompress: streamid: %i timestamp: 0x%x n_valid: %i, %u -> %u\n", 138 | k->stream_id, k->timestamp, v->n_valid, 139 | data.size, POINT_OFF(v->n_valid)); 140 | 141 | #endif 142 | return 0; 143 | } 144 | if (ret != DB_NOTFOUND) 145 | warn("Get failed: %s\n", db_strerror(ret)); 146 | return ret; 147 | } 148 | 149 | int get_partial(DBC *cursorp, int flags, struct rec_key *k, 150 | void *buf, int len, int off) { 151 | char unpacked[COMPRESS_WORKING_BUF]; 152 | int ret; 153 | 154 | debug("get_partial stream_id: %i timestamp: 0x%x\n", 155 | k->stream_id, k->timestamp); 156 | 157 | if ((ret = get(cursorp, flags, k, (struct rec_val *)unpacked, sizeof(unpacked))) < 0) { 158 | return ret; 159 | } 160 | 161 | memcpy(buf, unpacked + off, len); 162 | 163 | return ret; 164 | } 165 | -------------------------------------------------------------------------------- /c6/util.c: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | #include "readingdb.h" 21 | #include "logging.h" 22 | #include "config.h" 23 | 24 | #if 0 25 | IPC *ipc_open(int semflags, int memflags) { 26 | IPC *ipp = malloc(sizeof(IPC)); 27 | memset(ipp, 0, sizeof(IPC)); 28 | char buf[4096]; 29 | int written_len = 0; 30 | 31 | // FILE *log = stdout; 32 | 33 | //fprintf(log, "logging\n"); 34 | if (semflags & O_CREAT) { 35 | sem_unlink(SEM_NAME "CALLER"); 36 | sem_unlink(SEM_NAME "SERVER"); 37 | sem_unlink(SEM_NAME "REPLY"); 38 | 39 | ipp->mutex_caller = sem_open(SEM_NAME "CALLER", semflags, 0666, 1); 40 | ipp->mutex_server = sem_open(SEM_NAME "SERVER", semflags, 0666, 0); 41 | ipp->mutex_reply = sem_open(SEM_NAME "REPLY", semflags, 0666, 0); 42 | } else { 43 | ipp->mutex_caller = sem_open(SEM_NAME "CALLER", semflags); 44 | ipp->mutex_server = sem_open(SEM_NAME "SERVER", semflags); 45 | ipp->mutex_reply = sem_open(SEM_NAME "REPLY", semflags); 46 | } 47 | if (ipp->mutex_caller == SEM_FAILED || 48 | ipp->mutex_server == SEM_FAILED || 49 | ipp->mutex_reply == SEM_FAILED) { 50 | error("Open semaphore failed: %s: %s\n", SEM_NAME, strerror(errno)); 51 | goto sem_fail; 52 | } 53 | 54 | ipp->fd = open(SHM_PATH, memflags | O_RDWR, 0600); 55 | if (ipp->fd < 0) { 56 | error("Error opening shared file: %s: %s\n", SHM_PATH, strerror(errno)); 57 | goto sem_fail; 58 | } 59 | 60 | if (memflags & O_CREAT) { 61 | memset(buf, 0, sizeof(buf)); 62 | while (written_len < SHM_SIZ) { 63 | write(ipp->fd, buf, sizeof(buf)); 64 | written_len += sizeof(buf); 65 | } 66 | } 67 | lseek(ipp->fd, 0, SEEK_SET); 68 | 69 | ipp->shm = mmap(NULL, SHM_SIZ, PROT_READ | PROT_WRITE, MAP_SHARED, ipp->fd, 0); 70 | if (ipp->shm == MAP_FAILED) { 71 | error("mmap: %s\n", strerror(errno)); 72 | goto sem_fail; 73 | } 74 | close(ipp->fd); 75 | ipp->dbid = 0; 76 | return ipp; 77 | 78 | sem_fail: 79 | //fprintf(log, "FAILURE CONDITION\n"); 80 | free(ipp); 81 | //fprintf(log, "DONE\n"); 82 | /* SEM_DESTROY(ipp->mutex_caller, "CALLER") */ 83 | /* SEM_DESTROY(ipp->mutex_server, "SERVER") */ 84 | /* SEM_DESTROY(ipp->mutex_reply, "REPLY") */ 85 | // fclose(log); 86 | return NULL; 87 | } 88 | 89 | void ipc_close(IPC *ipp) { 90 | sem_close(ipp->mutex_caller); 91 | sem_close(ipp->mutex_server); 92 | sem_close(ipp->mutex_reply); 93 | munmap(ipp->shm, SHM_SIZ); 94 | } 95 | #endif 96 | 97 | void drop_priv(void) { 98 | const char *user = READINGDB_USER; 99 | struct passwd *user_pwd = getpwnam(user); 100 | info("trying to drop privileges to '%s'\n", user); 101 | if (user_pwd != NULL) { 102 | if (setuid(user_pwd->pw_uid) < 0) { 103 | error("setuid: %s\n", strerror(errno)); 104 | return; 105 | } 106 | setgid(user_pwd->pw_gid); 107 | } else { 108 | error("no such user '%s'\n", user); 109 | } 110 | } 111 | 112 | int timeval_subtract(struct timeval *result, 113 | struct timeval *x, 114 | struct timeval *y) { 115 | /* Perform the carry for the later subtraction by updating y. */ 116 | if (x->tv_usec < y->tv_usec) { 117 | int nsec = (y->tv_usec - x->tv_usec) / 1000000 + 1; 118 | y->tv_usec -= 1000000 * nsec; 119 | y->tv_sec += nsec; 120 | } 121 | if (x->tv_usec - y->tv_usec > 1000000) { 122 | int nsec = (x->tv_usec - y->tv_usec) / 1000000; 123 | y->tv_usec += 1000000 * nsec; 124 | y->tv_sec -= nsec; 125 | } 126 | 127 | /* Compute the time remaining to wait. 128 | tv_usec is certainly positive. */ 129 | result->tv_sec = x->tv_sec - y->tv_sec; 130 | result->tv_usec = x->tv_usec - y->tv_usec; 131 | 132 | /* Return 1 if result is negative. */ 133 | return x->tv_sec < y->tv_sec; 134 | } 135 | 136 | print_buf(void *buf, int len) { 137 | int i; 138 | uint8_t *cur = buf; 139 | printf("----\n "); 140 | for (i = 0; i < len; i++) { 141 | printf("0x%02x ", cur[i]); 142 | if (i % 16 == 0 && i > 0) 143 | printf("\n "); 144 | } 145 | printf("\nbuffer[%i]\n\n", len); 146 | } 147 | -------------------------------------------------------------------------------- /c6/util.h: -------------------------------------------------------------------------------- 1 | #ifndef _UTIL_H 2 | #define _UTIL_H 3 | 4 | /* IPC *ipc_open(int semflags, int memflags); */ 5 | /* void ipc_close(IPC *ipp); */ 6 | void drop_priv(void); 7 | 8 | int timeval_subtract (struct timeval *result, struct timeval *x, struct timeval *y); 9 | 10 | int put(DB *dbp, DB_TXN *txn, struct rec_key *k, void *buf, int len); 11 | int put_partial(DB *dbp, DB_TXN *txn, struct rec_key *k, void *buf, int len, int off); 12 | int get_partial(DBC *cursorp, int flags, struct rec_key *k, void *buf, int len, int off); 13 | int get(DBC *cursorp, int flags, struct rec_key *k, void *buf, int len); 14 | 15 | #define FREELIST(type, name) ; 16 | /* #define FREELIST(type, name) \ */ 17 | /* static type *freelist_ ## name = NULL; \ */ 18 | /* static type *freelist_ ## name ## _tmp; */ 19 | 20 | #define FREELIST_GET(type, name) ((type *)malloc(sizeof(type))) 21 | /* #define FREELIST_GET(type, name) \ */ 22 | /* ((freelist_ ## name) == NULL ? ((type *)(malloc(sizeof(type)))) : \ */ 23 | /* ((freelist_ ## name ## _tmp) = freelist_ ## name, \ */ 24 | /* freelist_ ## name = *((type **)(freelist_ ## name)), \ */ 25 | /* freelist_ ## name ## _tmp)) */ 26 | 27 | #define FREELIST_PUT(type, name, val) free(val) 28 | 29 | /* #define FREELIST_PUT(type, name, val) {\ */ 30 | /* *(type **)(val) = freelist_ ## name; \ */ 31 | /* freelist_ ## name = val; \ */ 32 | /* } */ 33 | #endif 34 | -------------------------------------------------------------------------------- /configure.ac: -------------------------------------------------------------------------------- 1 | # -*- Autoconf -*- 2 | # Process this file with autoconf to produce a configure script. 3 | 4 | AC_PREREQ(2.61) 5 | AC_INIT(readingdb, 0.7.2, steve@buildingrobotics.com) 6 | AC_CONFIG_HEADER([c6/config.h]) 7 | AC_CONFIG_MACRO_DIR([m4]) 8 | 9 | # Checks for programs. 10 | AM_INIT_AUTOMAKE([-Wall]) 11 | AC_PROG_CC 12 | AM_PROG_CC_C_O 13 | AC_PROG_RANLIB 14 | dnl AC_PROG_PYTHON 15 | dnl AC_PROG_SWIG 16 | 17 | # PKG_CHECK_MODULES([CHECK], [check >= 0.9.4]) 18 | # Checks for libraries. 19 | 20 | AC_ARG_ENABLE([compression], 21 | AS_HELP_STRING([--disable-compression], [Disable database compression])) 22 | AS_IF([test "x$enable_compression" != "xno"], AC_DEFINE([USE_COMPRESSION], [], [Use compressed database])) 23 | 24 | AC_ARG_ENABLE([compression-log], 25 | AS_HELP_STRING([--enable-compression-log], [Write a log for each write of compression statistics])) 26 | AS_IF([test "x$enable_compression_log" = "xyes"], AC_DEFINE([WRITE_COMPRESSION_LOG], [], [Write compression log])) 27 | 28 | dnl user to run as 29 | AC_MSG_CHECKING(user to run as) 30 | AC_ARG_WITH([username], 31 | [AC_HELP_STRING([--with-username], [User to run readingdb database as (readingdb)])], 32 | READINGDB_USER=$withval, 33 | READINGDB_USER=readingdb) 34 | AC_MSG_RESULT($READINGDB_USER) 35 | AC_DEFINE_UNQUOTED([READINGDB_USER], ["$READINGDB_USER"], [User to run as]) 36 | 37 | dnl find datadir 38 | AC_MSG_CHECKING(default data directory) 39 | AC_ARG_WITH([datadir], 40 | [AC_HELP_STRING([--with-datadir], [Default data directory (/var/lib/readingdb)])], 41 | DATA_DIR=$withval, 42 | DATA_DIR=/var/lib/readingdb) 43 | AC_MSG_RESULT($DATA_DIR) 44 | AC_DEFINE_UNQUOTED([DATA_DIR], ["$DATA_DIR"], [Default data directory]) 45 | 46 | 47 | # Checks for header files. 48 | AC_HEADER_STDC 49 | AC_CHECK_HEADERS([stdlib.h string.h endian.h libkern/OSByteOrder.h]) 50 | 51 | # Checks for typedefs, structures, and compiler characteristics. 52 | AC_C_CONST 53 | AC_C_INLINE 54 | 55 | # Checks for library functions. 56 | AC_FUNC_MALLOC 57 | AC_FUNC_REALLOC 58 | AC_CHECK_FUNCS([memset]) 59 | 60 | # bdb 61 | AC_SEARCH_LIBS(db_strerror, [db], , AC_MSG_ERROR([ERROR: can't link against berkeley db])) 62 | dnl AC_MSG_CHECKING(for acceptable berkeley db version) 63 | dnl # there's got to be a better way to check library versions... 64 | dnl # we require >= 4.8.28 since (a) they changed the on-disk format then 65 | dnl # and I don't want to deal with the old one, and (b) pretty sure 66 | dnl # that's when they added compression. 67 | dnl AC_COMPILE_IFELSE(AC_LANG_PROGRAM([#include ], [ 68 | dnl # if DB_VERSION_MAJOR >= 4 69 | dnl # if DB_VERSION_MINOR >= 8 70 | dnl # if DB_VERSION_PATCH >= 28 71 | dnl # else 72 | dnl # error 73 | dnl # endif 74 | dnl # else 75 | dnl # error 76 | dnl # endif 77 | dnl # else 78 | dnl # error 79 | dnl # endif 80 | dnl ]), AC_MSG_RESULT([yes]), AC_MSG_ERROR([Wrong bdb version: >= 4.8.28 required])) 81 | 82 | # assume they have pthreads for now... 83 | AC_SEARCH_LIBS(pthread_create, [pthread], ,AC_MSG_ERROR([ERROR: can't find pthreds])) 84 | AC_SEARCH_LIBS(compress2, [z], , AC_MSG_ERROR([ERROR: can't find zlib])) 85 | 86 | # check for protoc-c compiler 87 | AC_PATH_PROG(PROTOC, protoc-c) 88 | if test "x$PROTOC" = x; then 89 | echo "ERROR: cannot find protobuf-c, required" 90 | fi 91 | # check for proto-c libraries 92 | AC_CHECK_HEADERS(google/protobuf-c/protobuf-c.h) 93 | AC_SEARCH_LIBS(protobuf_c_message_pack, [protobuf-c], , AC_MSG_ERROR([ERROR: can't link against protobuf-c library])) 94 | 95 | AC_CONFIG_FILES([Makefile c6/Makefile c6/pbuf/Makefile]) 96 | AC_CONFIG_SUBDIRS([c6/hashtable/]) 97 | AC_OUTPUT 98 | 99 | -------------------------------------------------------------------------------- /debian/changelog: -------------------------------------------------------------------------------- 1 | readingdb (0.7.3-precise1) precise; urgency=low 2 | 3 | * Add support for only computing certain sketces 4 | 5 | -- Stephen Dawson-Haggerty Fri, 13 Nov 2015 16:54:23 -0800 6 | 7 | readingdb (0.7.2-1) precise; urgency=low 8 | 9 | * Add support for IPv4-only environments 10 | 11 | -- Stephen Dawson-Haggerty Mon, 27 Oct 2014 15:40:00 -0700 12 | 13 | readingdb (0.7.1-1) precise; urgency=low 14 | 15 | * Add support for fetching substream sketches in db_prev/db_next 16 | 17 | -- Stephen Dawson-Haggerty Thu, 09 Oct 2014 01:07:00 +0200 18 | 19 | readingdb (0.7.0-1) precise; urgency=low 20 | 21 | * Add support for sketching 22 | 23 | -- Stephen Dawson-Haggerty Sat, 04 Oct 2014 17:22:19 +0200 24 | 25 | readingdb (0.6.0-4) precise; urgency=low 26 | 27 | * Initial release. 28 | 29 | -- Stephen Dawson-Haggerty Thu, 17 May 2012 15:52:32 -0700 30 | -------------------------------------------------------------------------------- /debian/control: -------------------------------------------------------------------------------- 1 | Source: readingdb 2 | Section: database 3 | Priority: optional 4 | Maintainer: Stephen Dawson-Haggerty 5 | Standards-Version: 3.8.4 6 | Build-Depends: debhelper (>= 7.0.50~), dh-autoreconf, zlib1g-dev, libdb4.8, 7 | libdb4.8-dev, cdbs (>= 0.4.49), 8 | protobuf-c-compiler, libprotobuf-c0-dev, libprotobuf-c0, check, 9 | pkg-config 10 | 11 | Package: readingdb 12 | Architecture: any 13 | Section: database 14 | Priority: optional 15 | Depends: zlib1g, libdb4.8, libprotobuf-c0 16 | Suggests: monit, readingdb-python 17 | Homepage: https://github.com/stevedh/readingdb 18 | Description: readingdb is a time-series database designed for 19 | efficiency and speed. 20 | -------------------------------------------------------------------------------- /debian/postinst: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # postinst script for readingdb 3 | # 4 | # set -e 5 | 6 | case "$1" in 7 | configure) 8 | adduser --system readingdb 2>/dev/null 9 | 10 | chown readingdb /var/lib/readingdb 11 | ;; 12 | esac 13 | 14 | #DEBHELPER# 15 | #echo "---- ending postinst $@" 16 | exit 0 17 | -------------------------------------------------------------------------------- /debian/rules: -------------------------------------------------------------------------------- 1 | #!/usr/bin/make -f 2 | # -*- makefile -*- 3 | # Sample debian/rules that uses debhelper. 4 | # 5 | # This file was originally written by Joey Hess and Craig Small. 6 | # As a special exception, when this file is copied by dh-make into a 7 | # dh-make output file, you may use that output file without restriction. 8 | # This special exception was added by Craig Small in version 0.37 of dh-make. 9 | # 10 | # Modified to make a template file for a multi-binary package with separated 11 | # build-arch and build-indep targets by Bill Allombert 2001 12 | 13 | # Uncomment this to turn on verbose mode. 14 | #export DH_VERBOSE=1 15 | 16 | # This has to be exported to make some magic below work. 17 | export DH_OPTIONS 18 | 19 | # include /usr/share/cdbs/1/rules/debhelper.mk 20 | 21 | %: 22 | dh $@ --with autoreconf 23 | -------------------------------------------------------------------------------- /iface_bin/MANIFEST.in: -------------------------------------------------------------------------------- 1 | recursive-include c6 *.c 2 | recursive-include c6 *.h 3 | include *.c 4 | include *.h 5 | -------------------------------------------------------------------------------- /iface_bin/Makefile: -------------------------------------------------------------------------------- 1 | 2 | CFLAGS+=-Wall 3 | LDFLAGS+= 4 | 5 | EXTRA_DIST=setup.py readingdb.i readingdb_py.h readingdb.c 6 | SWIG_OUTPUTS = readingdb_wrap.c readingdb.py 7 | 8 | PROJECT=readingdb-python 9 | VERSION=0.6.0-3 10 | 11 | all-local: module 12 | 13 | install: module 14 | python setup.py install 15 | 16 | prep: readingdb.i $(SWIG_OUTPUTS) ../c6/pbuf/rdb.pb-c.c 17 | mkdir -p c6/ 18 | cp -r ../c6/pbuf/ c6/pbuf 19 | cp ../c6/readingdb.h ../c6/rpc.c ../c6/rpc.h ../c6/commands.h ../c6/sketch.h c6 20 | 21 | module: prep 22 | python setup.py build_ext --inplace 23 | 24 | dist: module 25 | python setup.py sdist --formats gztar 26 | 27 | builddeb: distclean prep 28 | # python setup.py sdist $(COMPILE) --dist-dir=../ --prune 29 | mkdir ../$(PROJECT)-$(VERSION) 30 | cp -r * ../$(PROJECT)-$(VERSION) 31 | rm -rf ../$(PROJECT)-$(VERSION)/dist 32 | tar zcvf ../$(PROJECT)_$(VERSION).orig.tar.gz ../$(PROJECT)-$(VERSION) 33 | rm -rf ../$(PROJECT)-$(VERSION) 34 | dpkg-buildpackage -i -I -rfakeroot -uc -us -S 35 | 36 | $(SWIG_OUTPUTS): readingdb.i ../c6/readingdb.h readingdb_py.h 37 | swig -python -keyword -Wall readingdb.i 38 | 39 | ../c6/pbuf/rdb.pb-c.c: ../c6/pbuf/rdb.proto 40 | cd ../c6/pbuf/ && make 41 | 42 | clean: 43 | -rm -rf ../$(PROJECT)_$(VERSION).orig.tar.gz ../$(PROJECT)-$(VERSION) 44 | -rm -f _readingdb.so *.pyc 45 | -python setup.py clean 46 | -rm -rf c6 47 | 48 | distclean: clean 49 | -rm -f $(SWIG_OUTPUTS) 50 | -rm -rf dist 51 | -------------------------------------------------------------------------------- /iface_bin/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/stevedh/readingdb/5ebc06c567becb84a7eeaade9d5ebce3f13210ef/iface_bin/__init__.py -------------------------------------------------------------------------------- /iface_bin/autotest.py: -------------------------------------------------------------------------------- 1 | 2 | import os 3 | import glob 4 | import subprocess 5 | import random 6 | import unittest 7 | import time 8 | import numpy as np 9 | import readingdb as rdb 10 | 11 | # make sure we're testing the version of readingdb in this dir. 12 | assert os.path.dirname(os.path.abspath(rdb.__file__)) == \ 13 | os.path.dirname(os.path.abspath(__file__)) 14 | 15 | datadir = '_testdata' 16 | readingdb = '../c6/reading-server' 17 | log = '/dev/null' 18 | port = int(random.random() * 5000) + 20000 19 | 20 | class TestIface(unittest.TestCase): 21 | def setUp(self): 22 | try: 23 | os.makedirs(datadir) 24 | except OSError, e: 25 | if e.errno != os.errno.EEXIST: 26 | raise 27 | cmd = [readingdb, '-p', str(port), '-d', datadir, '-c', '1'] 28 | self.log = open(log, 'a') 29 | self.db = subprocess.Popen(cmd, stderr=subprocess.PIPE, stdout=self.log) 30 | 31 | # wait for startup or a fatal message 32 | for x in xrange(0, 20): 33 | l = self.db.stderr.readline() 34 | if 'FATAL' in l: 35 | raise Exception(l) 36 | elif 'listening' in l: 37 | break 38 | 39 | self.conn = rdb.db_open('localhost', port) 40 | 41 | def tearDown(self): 42 | rdb.db_close(self.conn) 43 | self.db.terminate() 44 | self.db.wait() 45 | self.log.close() 46 | 47 | for f in glob.glob(os.path.join(datadir, '*')): 48 | os.remove(f) 49 | os.removedirs(datadir) 50 | 51 | def infill_stream(self, stream): 52 | for i in range(0, 1000): 53 | data = [(x, x, x) for x in xrange(i * 100, i * 100 + 100)] 54 | self.assertEqual(rdb.db_add(self.conn, stream, data), 1) 55 | 56 | 57 | def test_simple(self): 58 | self.infill_stream(1) 59 | 60 | d = rdb.db_query(self.conn, 1, 0, 10000) 61 | self.assertEqual(len(d), 10000) 62 | for i in xrange(0, 10000): 63 | self.assertEqual(d[i][0], i) 64 | self.assertEqual(d[i][1], i) 65 | 66 | def test_multi(self): 67 | streams = range(1, int(1e4), int(1e3)) 68 | for i in streams: 69 | self.infill_stream(i) 70 | 71 | rdb.db_setup('localhost', port) 72 | fetch = random.sample(streams, 3) 73 | data = rdb.db_multiple(fetch, 0, 10000) 74 | 75 | # check grabbing three random streams 76 | self.assertEqual(len(data), 3) 77 | for dv in data: 78 | self.assertEqual(dv.shape, (10000, 2)) 79 | self.assertEqual(np.sum(dv[:, 0] - np.arange(0, 10000)), 0) 80 | self.assertEqual(np.sum(dv[:, 1] - np.arange(0, 10000)), 0) 81 | 82 | # grab some streams without data 83 | data = rdb.db_multiple([2,3,4,6], 0, 10000) 84 | self.assertEqual(len(data), 4) 85 | for dv in data: 86 | self.assertEqual(dv.shape, (0, 2)) 87 | 88 | 89 | if __name__ == '__main__': 90 | unittest.main() 91 | -------------------------------------------------------------------------------- /iface_bin/dataloader.py: -------------------------------------------------------------------------------- 1 | 2 | import sys 3 | sys.path.append("..") 4 | 5 | import numpy as np 6 | import time 7 | import threading 8 | import Queue 9 | import readingdb as rdb4 10 | 11 | 12 | MAX_THREADS = 5 13 | 14 | class DataLoader: 15 | """A query frontend which makes parallel requests on the readingdb server 16 | 17 | This can lead to lower latency when fetching data, in some cases. 18 | """ 19 | class DataQueryThread(threading.Thread): 20 | def __init__(self, parent, host=None, full=False): 21 | threading.Thread.__init__(self) 22 | self.parent = parent 23 | self.host = host 24 | self.full = full 25 | self.daemon = True 26 | 27 | def run(self): 28 | """Dequeue requests until we're done working""" 29 | if self.host == None: 30 | db = rdb4.db_open() 31 | else: 32 | db = rdb4.db_open(host=self.host[0], port=self.host[1]) 33 | while True: 34 | try: 35 | request = self.parent.requests.get_nowait() 36 | except Queue.Empty: 37 | break 38 | if request.has_key("substream"): 39 | rdb4.db_substream(db, request['substream']) 40 | else: 41 | rdb4.db_substream(db, 0) 42 | 43 | first = True 44 | result = [] 45 | last = [] 46 | 47 | while first or len(last) == 10000: 48 | last = rdb4.db_query(db, int(request['streamid']), 49 | int(request['starttime']), 50 | int(request['endtime'])) 51 | first = False 52 | print len(last) 53 | result.extend(last) 54 | if not self.full: break 55 | 56 | if self.parent.as_numpy and len(result) > 0: 57 | result = np.array(result) 58 | result = result[:,[0,2]] 59 | self.parent.returns[request['streamid']] = result 60 | 61 | rdb4.db_close(db) 62 | 63 | def __init__(self, requests, threads=MAX_THREADS, as_numpy=False, host=None, full=False): 64 | """Get a new DataLoader. 65 | 66 | requests: a list of streams to load. Each should be a dict 67 | with "streamid", "starttime", and "endtime" keys; a 68 | "substream" key is optional as well. 69 | """ 70 | 71 | self.requests = Queue.Queue() 72 | self.as_numpy = as_numpy 73 | self.returns = {} 74 | self.host = host 75 | self.full = full 76 | for r in requests: 77 | self.requests.put(r) 78 | self.n_threads = min(len(requests), threads) 79 | 80 | def run(self): 81 | """Fire off the load. 82 | 83 | Returns a dict where keys are streamids, and values are the 84 | data returned by readingdb. 85 | """ 86 | threads = [] 87 | for i in range(0, self.n_threads): 88 | th = self.DataQueryThread(self, host=self.host, full=self.full) 89 | th.start() 90 | threads.append(th) 91 | for th in threads: 92 | th.join() 93 | return self.returns 94 | 95 | if __name__ == '__main__': 96 | ids = [6100, 6126, 6048, 6074, 552, 3805] 97 | ids = [{'starttime' : time.time() - (3600 * 72), 98 | 'endtime' : time.time(), 99 | 'streamid': x} for x in ids] 100 | 101 | loader = DataLoader(ids, as_numpy=False) 102 | loader.run() 103 | 104 | -------------------------------------------------------------------------------- /iface_bin/debian/changelog: -------------------------------------------------------------------------------- 1 | readingdb-python (0.7.1) precise; urgency=low 2 | 3 | * Add support for fetching sketches from db_prev/db_next 4 | 5 | -- Stephen Dawson-Haggerty Thu, 09 Oct 2014 01:08:00 +0200 6 | 7 | readingdb-python (0.7.0-1) precise; urgency=low 8 | 9 | * Add support for sketches to python client 10 | 11 | -- Stephen Dawson-Haggerty Sat, 04 Oct 2014 20:17:31 +0200 12 | 13 | readingdb-python (0.6.0-3) oneiric; urgency=low 14 | 15 | * Initial release. 16 | 17 | -- readingdb-python Sun, 20 May 2012 15:12:38 -0700 18 | -------------------------------------------------------------------------------- /iface_bin/debian/compat: -------------------------------------------------------------------------------- 1 | 8 2 | -------------------------------------------------------------------------------- /iface_bin/debian/control: -------------------------------------------------------------------------------- 1 | Source: readingdb-python 2 | Section: database 3 | Priority: optional 4 | Maintainer: Stephen Dawson-Haggerty 5 | Build-Depends: debhelper (>= 7.0.50~), dh-autoreconf, cdbs (>= 0.4.49), 6 | python-all-dev, python-support (>= 0.6), python-numpy-dev, 7 | python-numpy, libprotobuf-c0-dev, libprotobuf-c0, 8 | libdb4.8-dev 9 | XS-Python-Version: >= 2.6 10 | Standards-Version: 3.8.4 11 | 12 | Package: readingdb-python 13 | Architecture: any 14 | Section: database 15 | Priority: optional 16 | XB-Python-Version: ${python:Versions} 17 | Depends: ${misc:Depends}, ${python:Depends}, python-numpy, 18 | libprotobuf-c0 19 | Homepage: https://github.com/stevedh/readingdb 20 | Description: python bindings for accessing a readingdb database 21 | -------------------------------------------------------------------------------- /iface_bin/debian/copyright: -------------------------------------------------------------------------------- 1 | Copyright (c) 2011, 2012, Regents of the University of California 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions 6 | are met: 7 | 8 | - Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | - Redistributions in binary form must reproduce the above copyright 11 | notice, this list of conditions and the following disclaimer in the 12 | documentation and/or other materials provided with the 13 | distribution. 14 | 15 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 16 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 17 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 18 | FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL 19 | THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 20 | INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 22 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 | HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 24 | STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED 26 | OF THE POSSIBILITY OF SUCH DAMAGE. 27 | -------------------------------------------------------------------------------- /iface_bin/debian/rules: -------------------------------------------------------------------------------- 1 | #!/usr/bin/make -f 2 | # -*- makefile -*- 3 | 4 | DEB_PYTHON_SYSTEM := pysupport 5 | 6 | include /usr/share/cdbs/1/rules/debhelper.mk 7 | include /usr/share/cdbs/1/class/python-distutils.mk 8 | 9 | clean:: 10 | rm -rf build build-stamp configure-stamp build/ MANIFEST 11 | dh_clean 12 | -------------------------------------------------------------------------------- /iface_bin/fetch.py: -------------------------------------------------------------------------------- 1 | 2 | import sys 3 | import os 4 | import optparse 5 | import csv 6 | 7 | import dataloader 8 | 9 | if __name__ == '__main__': 10 | parser = optparse.OptionParser(description="Fetch data from readingdb", 11 | usage="usage: %prog [options] stream specs ..") 12 | parser.add_option('-o', '--output', dest='output', default='-', 13 | help="base name for output files") 14 | opts, args = parser.parse_args() 15 | 16 | reqvec = [] 17 | for stream in args: 18 | reqvec.append({'streamid': int(stream), 19 | 'starttime': 0, 20 | 'endtime': 2**32 - 10}) 21 | loader = dataloader.DataLoader(reqvec, host=("gecko.cs.berkeley.edu", 4243), full=True) 22 | data = loader.run() 23 | if opts.output == '-': 24 | print data 25 | pass 26 | else: 27 | for k,d in data.iteritems(): 28 | with open(opts.output + '.' + str(k), 'w') as fp: 29 | writer = csv.writer(fp) 30 | writer.writerows(d) 31 | -------------------------------------------------------------------------------- /iface_bin/import.py: -------------------------------------------------------------------------------- 1 | 2 | import re 3 | import time 4 | import sys 5 | from optparse import OptionParser 6 | import numpy as np 7 | 8 | import readingdb as rdb4 9 | 10 | def tic(): 11 | return time.time() 12 | def toc(v, p=True): 13 | delta = (time.time() - v) 14 | if p: 15 | print "%.3f" % delta 16 | return delta 17 | 18 | IMPORT_START = int(time.time() - 3600 * 35) 19 | def parse_netloc(loc): 20 | split = loc.rfind(':') 21 | if split > 0: 22 | return loc[:split], int(loc[split+1:]) 23 | else: 24 | return loc, 4246 25 | 26 | if __name__ == '__main__': 27 | parser = OptionParser(description="Import data from one readingdb to another", 28 | usage="usage: %prog [options] db1 db2") 29 | parser.add_option('-s', '--substream', dest='substream', default='0', 30 | help='which substream to import') 31 | parser.add_option('-a', '--ago', dest='ago', default='1', 32 | help='how long ago to begin or end importing from (hours)') 33 | parser.add_option('-z', '--zero', dest='zero', default=False, action='store_true', 34 | help='start import at time zero') 35 | parser.add_option('-b', '--begin-streamid', dest='startid', default='0', 36 | help='what streamid to start with (default=0') 37 | parser.add_option('-m', '--max-streamid', dest='maxid', default='10000', 38 | help='what streamid to stop at (default=10000') 39 | parser.add_option('-d', '--delay', dest='delay', default='0.1', 40 | help='how long to wait between each query-insert (default=0.1s)') 41 | parser.add_option('-n', '--no-action', dest='noop', default=False, action='store_true', 42 | help='don\'t actually insert the data') 43 | parser.add_option('-f', '--map-file', dest='mapfile', default=False, 44 | help='import using a map file') 45 | opts, hosts = parser.parse_args() 46 | if len(hosts) != 2: 47 | parser.print_help() 48 | sys.exit(1) 49 | 50 | old_db = parse_netloc(hosts[0]) 51 | new_db = parse_netloc(hosts[1]) 52 | 53 | print "Importing data from %s:%i to %s:%i" % (old_db + new_db) 54 | print "substream: %i" % int(opts.substream) 55 | 56 | rdb4.db_setup(old_db[0], old_db[1]) 57 | db0 = rdb4.db_open(host=old_db[0], port=old_db[1]) 58 | db1 = rdb4.db_open(host=new_db[0], port=new_db[1]) 59 | 60 | # rdb4.db_substream(db0, int(opts.substream)) 61 | # rdb4.db_substream(db1, int(opts.substream)) 62 | 63 | if not opts.zero: 64 | IMPORT_START = int(time.time()) - (int(opts.ago) * 3600) 65 | IMPORT_STOP = 2**32 - 10 66 | else: 67 | IMPORT_START = 1 68 | IMPORT_STOP = int(time.time()) - (int(opts.ago) * 3600) 69 | print "Importing from %i to %i" % (IMPORT_START, IMPORT_STOP) 70 | 71 | if not opts.mapfile: 72 | start = int(opts.startid) 73 | import_map = [(x, x) for x in xrange(start, int(opts.maxid))] 74 | else: 75 | import_map = [] 76 | with open(opts.mapfile, "r") as fp: 77 | for line in fp.readlines(): 78 | line = re.sub('\#.*$', '', line) 79 | ids = re.split('[ \t]+', line) 80 | for id in ids[1:]: 81 | import_map.append((int(ids[0]), int(id))) 82 | 83 | for to_stream, from_stream in import_map: 84 | print "starting %i <- %i" % (to_stream, from_stream) 85 | first = True 86 | vec = [(IMPORT_START,)] 87 | t = tic() 88 | data = rdb4.db_query(from_stream, IMPORT_START, IMPORT_STOP, limit=100000000, conn=db0) 89 | if not len(data): continue 90 | data = data[0] 91 | print "received", data.shape 92 | toc(t) 93 | 94 | t = tic() 95 | if opts.noop: continue 96 | bound = (int(data.shape[0]) / 100) + 1 97 | for i in xrange(0, bound): 98 | vec = (data[(i*100):(i*100) + 100, :]).tolist() 99 | # print time.ctime(vec[0][0]) 100 | rdb4.db_add(db1, to_stream, map(tuple, vec)) 101 | if len(vec) < 100: break 102 | print "inserted", to_stream 103 | toc(t) 104 | time.sleep(float(opts.delay)) 105 | # rdb4.db_sync(db4) 106 | #rdb4.db_close(db0) 107 | rdb4.db_close(db1) 108 | -------------------------------------------------------------------------------- /iface_bin/import_tsdb.py: -------------------------------------------------------------------------------- 1 | 2 | import sys 3 | import time 4 | import random 5 | import MySQLdb as mq 6 | import threading 7 | import Queue 8 | import socket 9 | 10 | from powerdb.smap.models import * 11 | import powerdb.settings as settings 12 | 13 | 14 | import readingdb 15 | 16 | OLDSTREAM=34 17 | NEWSTREAM=34 18 | 19 | # DBHOST='jackalope.cs.berkeley.edu' 20 | # DBNAME='powerdb' 21 | # DBUSER='p' 22 | # DBPASS='410soda' 23 | assert settings.DATABASE_ENGINE == 'mysql' 24 | 25 | class TsdbLoader(threading.Thread): 26 | def __init__(self, host='green.millennium.berkeley.edu', port=4242): 27 | threading.Thread.__init__(self) 28 | self.q = Queue.Queue(maxsize=10) 29 | self.sockaddr = (host, port) 30 | self.daemon = True 31 | 32 | def run(self): 33 | s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 34 | s.connect(self.sockaddr) 35 | while True: 36 | add = self.q.get() 37 | if add['method'] == 'put': 38 | print "adding", len(add['data']), add['extra'] 39 | print add['id'] 40 | for i in xrange(0, len(add['data'])): 41 | a = "put %i %i %f %s\n" % (add['id'], 42 | add['data'][i][0], 43 | add['data'][i][2], 44 | add['extra']) 45 | s.send(a) 46 | elif add['method'] == 'quit': 47 | break 48 | s.close() 49 | 50 | if __name__ == '__main__': 51 | db = mq.connect(passwd=settings.DATABASE_PASSWORD, 52 | db=settings.DATABASE_NAME, 53 | host=settings.DATABASE_HOST, 54 | user=settings.DATABASE_USER) 55 | rdb = readingdb.db_open() 56 | loader = TsdbLoader() 57 | loader.start() 58 | 59 | for s in Stream.objects.all().order_by('id'): 60 | if s.id <= 1: 61 | next 62 | first = True 63 | data = [] 64 | startts = 0 65 | extra = 'source=%s path=%s' % (s.subscription, s.path()) 66 | print extra 67 | while first or len(data) == 10000: 68 | first = False 69 | data = readingdb.db_query(rdb, s.id, startts, 2 ** 31) 70 | 71 | if len(data) > 0: 72 | startts = data[-1][0] 73 | loader.q.put({'method': 'put', 74 | 'id': s.id, 75 | 'data': data, 76 | 'extra': extra}) 77 | 78 | print startts 79 | # print data 80 | break 81 | loader.q.put({'method': 'quit'}) 82 | loader.join() 83 | 84 | readingdb.db_close(rdb) 85 | db.close() 86 | 87 | -------------------------------------------------------------------------------- /iface_bin/multi.c: -------------------------------------------------------------------------------- 1 | 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | #include "readingdb_py.h" 19 | 20 | #include "c6/readingdb.h" 21 | #include "c6/rpc.h" 22 | #include "c6/commands.h" 23 | 24 | static char host[512]; 25 | static short port; 26 | static int workers; 27 | static int substream; 28 | static int setup = 0; 29 | 30 | void db_setup(char *a_host, 31 | short a_port, 32 | int a_workers, 33 | int a_substream) { 34 | strncpy(host, a_host, sizeof(host)); 35 | port = a_port; 36 | workers = a_workers; 37 | substream = a_substream; 38 | setup = 1; 39 | import_array(); 40 | } 41 | 42 | struct np_point { 43 | double ts; 44 | double val; 45 | }; 46 | 47 | struct request { 48 | pthread_mutex_t lock; 49 | struct sock_request *conn; 50 | 51 | // server conf 52 | const char *host; 53 | const short port; 54 | 55 | // work queue 56 | int next_streamid; 57 | int n_streams; 58 | 59 | // request params 60 | const struct request_desc *r; 61 | 62 | struct np_point **return_data; 63 | int *return_data_len; 64 | int errors; 65 | }; 66 | 67 | int setup_request(struct sock_request *conn, 68 | const struct request_desc *req, 69 | unsigned long long streamid, 70 | unsigned long long starttime) { 71 | switch (req->type) { 72 | case REQ_QUERY: 73 | return db_query_all(conn, streamid, starttime, 74 | req->endtime, req->substream, 75 | &req->sketch, QUERY_DATA); 76 | case REQ_ITER: 77 | return db_iter(conn, streamid, starttime, req); 78 | default: 79 | return -1; 80 | } 81 | } 82 | 83 | int read_numpy_resultset(struct sock_request *ipp, 84 | struct np_point **buf, int *off) { 85 | Response *r; 86 | struct pbuf_header h; 87 | void *reply; 88 | int len, i, rv; 89 | 90 | /* read the reply */ 91 | if (fread(&h, sizeof(h), 1, ipp->sock_fp) <= 0) { 92 | // PyErr_Format(PyExc_IOError, "read_resultset: error reading from socket: %s", strerror(errno)); 93 | return -errno; 94 | } 95 | len = ntohl(h.body_length); 96 | reply = malloc(len); 97 | if (!reply) return -1; 98 | if (fread(reply, len, 1, ipp->sock_fp) <= 0) { 99 | free(reply); 100 | // PyErr_Format(PyExc_IOError, "read_resultset: error reading from socket: %s", strerror(errno)); 101 | return -errno; 102 | } 103 | r = response__unpack(NULL, len, reply); 104 | free(reply); 105 | if (!r) { 106 | // PyErr_Format(PyExc_IOError, "read_resultset: error unpacking"); 107 | return -errno; 108 | } 109 | /* printf("Received reply code: %i results: %li len: %i\n", */ 110 | /* r->error, r->data->n_data, len); */ 111 | if (r->error != RESPONSE__ERROR_CODE__OK) { 112 | // PyErr_Format(PyExc_Exception, "read_resultset: received error from server: %i", r->error); 113 | response__free_unpacked(r, NULL); 114 | return -errno; 115 | } 116 | 117 | /* build the python data structure */ 118 | if (*buf == NULL || *off == 0) { 119 | len = sizeof(struct np_point) * r->data->n_data; 120 | *buf = malloc(len); 121 | } else { 122 | len = sizeof(struct np_point) * (r->data->n_data + *off); 123 | *buf = realloc(*buf, len); 124 | // printf("reallocing\n"); 125 | } 126 | if (!*buf) { 127 | fprintf(stderr, "Alloc/realloc failed: request: %i\n", len); 128 | response__free_unpacked(r, NULL); 129 | return -1; 130 | } 131 | 132 | for (i = *off; i < *off + r->data->n_data; i++) { 133 | ((struct np_point *)(*buf))[i].ts = r->data->data[i - *off]->timestamp; 134 | ((struct np_point *)(*buf))[i].val = r->data->data[i - *off]->value; 135 | } 136 | *off += r->data->n_data; 137 | rv = r->data->n_data; 138 | response__free_unpacked(r, NULL); 139 | 140 | return rv; 141 | } 142 | 143 | void *worker_thread(void *ptr) { 144 | struct sock_request *conn; 145 | int id, idx, limit, rv = 0, conn_error = 0; 146 | unsigned long long starttime; 147 | struct request *req = ptr; 148 | int substream; 149 | 150 | // don't need a lock, this might block 151 | if (req->conn != NULL) { 152 | conn = req->conn; 153 | } else { 154 | conn = __db_open(req->host, req->port, &conn_error); 155 | if (!conn || conn_error) { 156 | req->errors = conn_error; 157 | return NULL; 158 | } 159 | } 160 | 161 | while (1) { 162 | pthread_mutex_lock(&req->lock); 163 | idx = req->next_streamid; 164 | if (idx < req->n_streams) { 165 | id = req->r->streamids[req->next_streamid++]; 166 | } else { 167 | id = 0; 168 | } 169 | pthread_mutex_unlock(&req->lock); 170 | starttime = req->r->starttime; 171 | limit = req->r->limit; 172 | substream = req->r->substream; 173 | 174 | // printf("starting load of %i (%i)\n", id, idx); 175 | if (id == 0) { 176 | break; 177 | } 178 | 179 | // read all the range data from a single stream 180 | while (1) { 181 | rv = setup_request(conn, req->r, id, starttime); 182 | if (rv < 0) { 183 | fprintf(stderr, "Error from DB: %s\n", strerror(-rv)); 184 | req->errors = -rv; 185 | goto done; 186 | } 187 | rv = read_numpy_resultset(conn, &req->return_data[idx], &req->return_data_len[idx]); 188 | if (rv < 0) { 189 | fprintf(stderr, "Error reading results: %s\n", strerror(-rv)); 190 | req->errors = -rv; 191 | goto done; 192 | } else if (rv < 10000 || limit - rv <= 0) { 193 | break; 194 | } 195 | limit -= rv; 196 | starttime = req->return_data[idx][req->return_data_len[idx]-1].ts + 1; 197 | } 198 | } 199 | done: 200 | if (!req->conn) { 201 | db_close(conn); 202 | } 203 | return NULL; 204 | } 205 | 206 | PyObject *make_numpy_list(struct request *req) { 207 | PyObject *a; 208 | PyObject *r = PyList_New(req->n_streams); 209 | int i; 210 | if (!r) { 211 | return PyErr_NoMemory(); 212 | } 213 | for (i = 0; i < req->n_streams; i++) { 214 | npy_intp dims[2]; 215 | int length = min(req->r->limit, req->return_data_len[i]); 216 | 217 | if (req->return_data && length > 0) { 218 | dims[0] = length; dims[1] = 2; 219 | // memcpy into a new array because there's apparently no way to 220 | // pass off the buffer that will be safe... 221 | a = PyArray_SimpleNew(2, dims, NPY_DOUBLE); 222 | if (!a) { 223 | Py_DECREF(r); 224 | free(req->return_data[i]); 225 | return PyErr_NoMemory(); 226 | } else { 227 | memcpy(PyArray_DATA(a), req->return_data[i], 228 | (length * sizeof(struct np_point))); 229 | free(req->return_data[i]); 230 | 231 | // donate the ref we got 232 | PyList_SetItem(r, i, a); 233 | } 234 | } else { 235 | npy_intp dims[2] = {0, 2}; 236 | // otherwise return an empty array with the proper dimension. 237 | // n.b. PyArray_SimpleNew segfaults if any dimensions are zero... 238 | PyList_SetItem(r, i, PyArray_EMPTY(2, dims, NPY_DOUBLE, 0)); 239 | } 240 | } 241 | return r; 242 | } 243 | 244 | PyObject *db_multiple(struct sock_request *ipp, const struct request_desc *r) { 245 | // set up a request; 246 | PyThreadState *_save; 247 | PyObject *rv; 248 | struct request req = { 249 | .conn = ipp, 250 | .host = host, 251 | .port = port, 252 | .r = r, 253 | .errors = 0, 254 | }; 255 | pthread_t threads[workers]; 256 | int n_streams, i; 257 | 258 | if (!setup) { 259 | PyErr_SetString(PyExc_IOError, 260 | "ERROR: you must call db_setup before using the API\n"); 261 | return NULL; 262 | } 263 | 264 | _save = PyEval_SaveThread(); 265 | for (n_streams = 0; req.r->streamids[n_streams] != 0; n_streams++); 266 | // printf("loading %i streams limit %i\n", n_streams, r->limit); 267 | 268 | pthread_mutex_init(&req.lock, NULL); 269 | req.next_streamid = 0; 270 | req.n_streams = n_streams; 271 | req.return_data = malloc(sizeof(PyObject *) * n_streams); 272 | if (!req.return_data) return NULL; 273 | req.return_data_len = malloc(sizeof(int) * n_streams); 274 | if (!req.return_data) { 275 | free(req.return_data); 276 | PyEval_RestoreThread(_save); 277 | return PyErr_NoMemory(); 278 | } 279 | memset(req.return_data_len, 0, sizeof(int) * n_streams); 280 | for (i = 0; i < n_streams; i++) req.return_data[i] = NULL; 281 | 282 | if (n_streams == 1 || ipp) { 283 | // printf("not starting threads because connection is provided: %p\n", ipp); 284 | worker_thread(&req); 285 | } else { 286 | int my_workers = min(n_streams, workers); 287 | // printf("starting %i workers\n", my_workers); 288 | for (i = 0; i < my_workers; i++) { 289 | pthread_create(&threads[i], NULL, worker_thread, &req); 290 | } 291 | for (i = 0; i < my_workers; i++) { 292 | void *code; 293 | pthread_join(threads[i], &code); 294 | } 295 | } 296 | 297 | PyEval_RestoreThread(_save); 298 | // printf("req errors %i\n", req.errors); 299 | if (!req.errors) { 300 | rv = make_numpy_list(&req); 301 | free(req.return_data); 302 | free(req.return_data_len); 303 | return rv; 304 | } else { 305 | for (i = 0; i < n_streams; i++) { 306 | if (req.return_data[i]) 307 | free(req.return_data[i]); 308 | } 309 | free(req.return_data); 310 | free(req.return_data_len); 311 | PyErr_Format(PyExc_Exception, "error reading data: last error: %s", 312 | strerror(req.errors)); 313 | return NULL; 314 | } 315 | } 316 | -------------------------------------------------------------------------------- /iface_bin/readingdb.c: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | #include "readingdb_py.h" 16 | 17 | #include "c6/readingdb.h" 18 | #include "c6/rpc.h" 19 | #include "c6/commands.h" 20 | #include "c6/sketch.h" 21 | #define TIMEOUT_SECS 10 22 | #define SIGREPLACE SIGTERM 23 | 24 | 25 | // version of db_open which can be called without holding the GIL 26 | struct sock_request *__db_open(const char *host, const short port, int *rc) { 27 | struct sock_request *req = malloc(sizeof(struct sock_request)); 28 | struct addrinfo *res, hints; 29 | struct sockaddr_in dest; 30 | struct timeval timeout; 31 | 32 | *rc = 0; 33 | 34 | if (!req) { 35 | *rc = ENOMEM; 36 | return NULL; 37 | }; 38 | 39 | memset(&hints, 0, sizeof(hints)); 40 | hints.ai_family = AF_INET; 41 | if (getaddrinfo(host, NULL, &hints, &res) != 0) { 42 | *rc = ENOENT; 43 | return NULL; 44 | } 45 | 46 | memcpy(&dest, res->ai_addr, res->ai_addrlen); 47 | dest.sin_port = htons(port); 48 | freeaddrinfo(res); 49 | 50 | req->sock = socket(AF_INET, SOCK_STREAM, 0); 51 | if (req->sock < 0) { 52 | *rc = errno; 53 | goto cleanup; 54 | } 55 | 56 | timeout.tv_sec = 30; 57 | timeout.tv_usec = 0; 58 | if (setsockopt(req->sock, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout)) < 0) { 59 | *rc = errno; 60 | close(req->sock); 61 | goto cleanup; 62 | } 63 | 64 | if (connect(req->sock, (struct sockaddr *)&dest, sizeof(dest)) < 0) { 65 | *rc = errno; 66 | goto cleanup; 67 | } 68 | 69 | req->sock_fp = fdopen(req->sock, "r+"); 70 | if (req->sock_fp == NULL) { 71 | *rc = errno; 72 | close(req->sock); 73 | goto cleanup; 74 | } 75 | req->substream = 0; 76 | return req; 77 | cleanup: 78 | free(req); 79 | return NULL; 80 | } 81 | 82 | Sketch *parse_sketch(PyObject *sketch, Sketch *out) { 83 | char *sketch_type; 84 | int sketch_window, i, sketch_type_val = -1; 85 | 86 | sketch__init(out); 87 | out->type = SKETCH__SKETCH_TYPE__NULL; 88 | out->window = 0; 89 | printf("parsing sketch\n"); 90 | 91 | if (sketch && sketch != Py_None) { 92 | if (!PyArg_ParseTuple(sketch, "si", &sketch_type, &sketch_window)) { 93 | PyErr_SetString(PyExc_ValueError, "db_query: invalid sketch definition"); 94 | return NULL; 95 | } 96 | for (i = 0; i < sizeof(sketch_names) / sizeof(sketch_names[0]); i ++) { 97 | if (strcmp(sketch_names[i], sketch_type) == 0) { 98 | sketch_type_val = i; 99 | break; 100 | } 101 | } 102 | if (sketch_type_val < 0) { 103 | PyErr_SetString(PyExc_ValueError, "db_query: invalid sketch name"); 104 | return NULL; 105 | } 106 | out->type = sketch_type_val; 107 | out->window = sketch_window; 108 | printf("sketch: %i %i\n", out->type, out->window); 109 | } 110 | return out; 111 | } 112 | 113 | // friendly db_open which raises python exceptions 114 | struct sock_request *db_open(const char *host, const short port) { 115 | int rc; 116 | void *rv; 117 | rv = __db_open(host, port, &rc); 118 | if (rc == 0) { 119 | return rv; 120 | } else if (rc == ENOMEM) { 121 | return (struct sock_request *)PyErr_NoMemory(); 122 | } else { 123 | PyErr_Format(PyExc_IOError, "db_open: %s", strerror(rc)); 124 | return NULL; 125 | } 126 | } 127 | 128 | int db_add(struct sock_request *ipp, int streamid, PyObject *values) { 129 | int i, len; 130 | ReadingSet *r; 131 | Response *response; 132 | struct pbuf_header h; 133 | unsigned char *buf; 134 | void *reply; 135 | 136 | r = _rpc_alloc_rs(SMALL_POINTS); 137 | if (!r) { 138 | PyErr_SetNone(PyExc_MemoryError); 139 | return 0; 140 | } 141 | if (!ipp) { 142 | PyErr_SetString(PyExc_ValueError, "db_add: conn is NULL"); 143 | return 0; 144 | } 145 | 146 | r->streamid = streamid; 147 | r->substream = ipp->substream; 148 | 149 | if (!PyList_Check(values)) { 150 | _rpc_free_rs(r); 151 | return 0; 152 | } 153 | 154 | if (PyList_Size(values) > SMALL_POINTS) { 155 | _rpc_free_rs(r); 156 | return 0; 157 | } 158 | 159 | for (i = 0; i < PyList_Size(values); i++) { 160 | PyObject *tuple = PyList_GetItem(values, i); 161 | reading__init(r->data[i]); 162 | if (PyTuple_Size(tuple) == 5) { 163 | if (PyArg_ParseTuple(tuple, "llddd", 164 | &r->data[i]->timestamp, 165 | &r->data[i]->seqno, 166 | &r->data[i]->value, 167 | &r->data[i]->min, 168 | &r->data[i]->max) < 0) 169 | break; 170 | r->data[i]->has_min = 1; 171 | r->data[i]->has_max = 1; 172 | } else if (PyTuple_Size(tuple) == 3) { 173 | if (PyArg_ParseTuple(tuple, "lld", 174 | &r->data[i]->timestamp, 175 | &r->data[i]->seqno, 176 | &r->data[i]->value) < 0) 177 | break; 178 | } else if (PyTuple_Size(tuple) == 2) { 179 | if (PyArg_ParseTuple(tuple, "ld", 180 | &r->data[i]->timestamp, 181 | &r->data[i]->value) < 0) 182 | break; 183 | r->data[i]->seqno = 0; 184 | } else { 185 | _rpc_free_rs(r); 186 | PyErr_Format(PyExc_ValueError, 187 | "Invalid data passed: must be a list of tuples"); 188 | return 0; 189 | } 190 | if (r->data[i]->seqno != 0) 191 | r->data[i]->has_seqno = 1; 192 | r->n_data ++; 193 | } 194 | 195 | len = reading_set__get_packed_size(r); 196 | buf = malloc(len); 197 | if (!buf) { 198 | _rpc_free_rs(r); 199 | PyErr_SetNone(PyExc_MemoryError); 200 | return 0; 201 | } 202 | reading_set__pack(r, buf); 203 | _rpc_free_rs(r); 204 | 205 | h.body_length = htonl(len); 206 | h.message_type = htonl(MESSAGE_TYPE__READINGSET); 207 | 208 | if (fwrite(&h, sizeof(h), 1, ipp->sock_fp) <= 0) { 209 | free(buf); 210 | PyErr_Format(PyExc_IOError, "error writing to socket: %s", strerror(errno)); 211 | return 0; 212 | } 213 | if (fwrite(buf, len, 1, ipp->sock_fp) <= 0) { 214 | free(buf); 215 | PyErr_Format(PyExc_IOError, "error writing to socket: %s", strerror(errno)); 216 | return 0; 217 | } 218 | free(buf); 219 | 220 | /* add sent. now wait for ack */ 221 | if (fread(&h, sizeof(h), 1, ipp->sock_fp) <= 0) { 222 | PyErr_Format(PyExc_IOError, "error reading reply: %s", strerror(errno)); 223 | return 0; 224 | } 225 | 226 | len = ntohl(h.body_length); 227 | reply = malloc(len); 228 | if (!reply) return -1; 229 | if (fread(reply, len, 1, ipp->sock_fp) <= 0) { 230 | PyErr_Format(PyExc_IOError, "error reading reply code: %s", strerror(errno)); 231 | free(reply); 232 | return 0; 233 | } 234 | 235 | response = response__unpack(NULL, len, reply); 236 | if (response->error != RESPONSE__ERROR_CODE__OK) { 237 | PyErr_Format(PyExc_IOError, "server error adding data: %i", response->error); 238 | free(reply); 239 | response__free_unpacked(response, NULL); 240 | return 0; 241 | } 242 | 243 | free(reply); 244 | response__free_unpacked(response, NULL); 245 | 246 | return 1; 247 | } 248 | 249 | int db_query_all(struct sock_request *ipp, unsigned long long streamid, 250 | unsigned long long starttime, 251 | unsigned long long endtime, 252 | int substream, 253 | const Sketch *sketch, 254 | enum query_action action) { 255 | Query q = QUERY__INIT; 256 | struct pbuf_header h; 257 | unsigned char buf [512]; 258 | int len; 259 | 260 | q.streamid = streamid; 261 | q.substream = substream; 262 | q.starttime = starttime; 263 | q.endtime = endtime; 264 | q.has_action = 1; 265 | q.action = action; 266 | 267 | if (sketch && sketch->type != SKETCH__SKETCH_TYPE__NULL) { 268 | q.sketch = sketch; 269 | } else { 270 | q.sketch = NULL; 271 | } 272 | 273 | if ((len = query__get_packed_size(&q)) < sizeof(buf)) { 274 | /* pack the request */ 275 | query__pack(&q, buf); 276 | h.message_type = htonl(MESSAGE_TYPE__QUERY); 277 | h.body_length = htonl(len); 278 | /* send it */ 279 | if (fwrite(&h, sizeof(h), 1, ipp->sock_fp) <= 0) 280 | goto write_error; 281 | if (fwrite(buf, len , 1, ipp->sock_fp) <= 0) 282 | goto write_error; 283 | fflush(ipp->sock_fp); 284 | 285 | return 0; 286 | } 287 | write_error: 288 | return -errno; 289 | } 290 | 291 | int db_iter(struct sock_request *ipp, int streamid, 292 | unsigned long long reference, 293 | const struct request_desc *req) { 294 | struct pbuf_header h; 295 | Nearest n = NEAREST__INIT; 296 | unsigned char buf[512]; 297 | int len; 298 | 299 | n.streamid = streamid; 300 | n.reference = reference; 301 | n.direction = req->direction; 302 | 303 | if (req->limit > 1) { 304 | n.has_n = 1; 305 | n.n = req->limit; 306 | } 307 | 308 | if (req->sketch.type != SKETCH__SKETCH_TYPE__NULL) { 309 | n.sketch = &req->sketch; 310 | } 311 | 312 | if ((len = nearest__get_packed_size(&n)) > sizeof(buf)) { 313 | return -ENOMEM; 314 | } 315 | 316 | h.message_type = htonl(MESSAGE_TYPE__NEAREST); 317 | h.body_length = htonl(len); 318 | nearest__pack(&n, buf); 319 | 320 | if (fwrite(&h, sizeof(h), 1, ipp->sock_fp) <= 0) 321 | goto write_error; 322 | if (fwrite(buf, len, 1, ipp->sock_fp) <= 0) 323 | goto write_error; 324 | fflush(ipp->sock_fp); 325 | 326 | return 0; 327 | 328 | write_error: 329 | return -errno; 330 | } 331 | 332 | 333 | PyObject *db_query(unsigned long long *streamids, 334 | unsigned long long starttime, 335 | unsigned long long endtime, 336 | int limit, 337 | int substream, 338 | PyObject *sketch, 339 | struct sock_request *ipp) { 340 | struct request_desc d; 341 | d.streamids = streamids; 342 | d.substream = substream; 343 | d.type = REQ_QUERY; 344 | d.starttime = starttime; 345 | d.endtime = endtime; 346 | d.limit = limit > 0 ? limit : 1e6; 347 | /* sets an error indicator */ 348 | if (!parse_sketch(sketch, &d.sketch)) 349 | return NULL; 350 | if (substream && d.sketch.type != SKETCH__SKETCH_TYPE__NULL) { 351 | PyErr_SetString(PyExc_ValueError, "db_query: cannot request both sketch and substream"); 352 | return NULL; 353 | } 354 | 355 | return db_multiple(ipp, &d); 356 | } 357 | 358 | PyObject *db_next(unsigned long long *streamids, 359 | unsigned long long reference, int n, 360 | PyObject *sketch, 361 | struct sock_request *ipp) { 362 | struct request_desc d; 363 | d.streamids = streamids; 364 | d.type = REQ_ITER; 365 | d.starttime = reference; 366 | d.direction = NEAREST__DIRECTION__NEXT; 367 | d.limit = n > 0 ? n : 1; 368 | 369 | /* sets an error indicator */ 370 | if (!parse_sketch(sketch, &d.sketch)) 371 | return NULL; 372 | 373 | return db_multiple(ipp, &d); 374 | // return db_iter(ipp, streamid, reference, NEAREST__DIRECTION__NEXT, n); 375 | } 376 | 377 | PyObject *db_prev(unsigned long long *streamids, 378 | unsigned long long reference, int n, 379 | PyObject *sketch, 380 | struct sock_request *ipp) { 381 | struct request_desc d; 382 | PyObject *numpy, *flipud, *p, *rv; 383 | int i; 384 | d.streamids = streamids; 385 | d.type = REQ_ITER; 386 | d.starttime = reference; 387 | d.direction = NEAREST__DIRECTION__PREV; 388 | d.limit = n > 0 ? n : 1; 389 | 390 | /* sets an error indicator */ 391 | if (!parse_sketch(sketch, &d.sketch)) 392 | return NULL; 393 | 394 | // load the data 395 | rv = db_multiple(ipp, &d); 396 | if (rv == NULL) { 397 | return NULL; 398 | } 399 | 400 | // use the numpy flipud to return a view on the data which has it in 401 | // the right order (ascending timestamps). 402 | numpy = PyImport_ImportModule("numpy"); 403 | if (!numpy) { 404 | // ignore the import error 405 | PyErr_Clear(); 406 | return rv; 407 | } 408 | flipud = PyObject_GetAttrString(numpy, "flipud"); 409 | if (!flipud) { 410 | PyErr_Clear(); 411 | Py_DECREF(numpy); 412 | return rv; 413 | } 414 | 415 | // try to flip all of the data vectors so they are ascending 416 | if (!PyList_Check(rv)) goto done; 417 | for (i = 0; i < PyList_Size(rv); i++) { 418 | p = PyList_GetItem(rv, i); 419 | PyList_SetItem(rv, i, PyObject_CallFunctionObjArgs(flipud, p, NULL)); 420 | // the way I think this works is we call flipud, which creates a 421 | // "view" into the original array as a new object. Since the 422 | // reference to the original array disappears, we don't need to 423 | // incref/decref it; we essentially donate our ref to the view. 424 | } 425 | done: 426 | Py_DECREF(numpy); 427 | Py_DECREF(flipud); 428 | return rv; 429 | } 430 | 431 | void db_del(struct sock_request *ipp, 432 | unsigned long long streamid, 433 | unsigned long long starttime, 434 | unsigned long long endtime) { 435 | struct pbuf_header h; 436 | Delete d = DELETE__INIT; 437 | unsigned char buf[512]; 438 | int len; 439 | d.streamid = streamid; 440 | d.starttime = starttime; 441 | d.endtime = endtime; 442 | 443 | if ((len = delete__get_packed_size(&d)) > sizeof(buf)) { 444 | goto error; 445 | } 446 | 447 | delete__pack(&d, buf); 448 | h.message_type = htonl(MESSAGE_TYPE__DELETE); 449 | h.body_length = htonl(len); 450 | /* send it */ 451 | if (fwrite(&h, sizeof(h), 1, ipp->sock_fp) <= 0) 452 | goto error; 453 | if (fwrite(buf, len , 1, ipp->sock_fp) <= 0) 454 | goto error; 455 | fflush(ipp->sock_fp); 456 | return; 457 | error: 458 | PyErr_SetString(PyExc_IOError, "db_del: generic error"); 459 | return; 460 | } 461 | 462 | 463 | void db_close(struct sock_request *ipp) { 464 | fclose(ipp->sock_fp); 465 | free(ipp); 466 | } 467 | 468 | /* PyObject *db_sketches(struct sock_request *ipp) { */ 469 | /* struct pbuf_header h; */ 470 | /* Response *r; */ 471 | /* int len, i; */ 472 | /* void *reply; */ 473 | /* PyObject *rv = NULL; */ 474 | 475 | /* h.message_type = htonl(MESSAGE_TYPE__SKETCHLIST); */ 476 | /* h.body_length = htonl(0); */ 477 | /* printf("%i\n", MESSAGE_TYPE__SKETCHLIST); */ 478 | /* if (fwrite(&h, sizeof(h), 1, ipp->sock_fp) <= 0) */ 479 | /* goto error; */ 480 | 481 | /* /\* read the reply *\/ */ 482 | /* if (fread(&h, sizeof(h), 1, ipp->sock_fp) <= 0) { */ 483 | /* PyErr_Format(PyExc_IOError, "db_sketches: error reading from socket: %s", strerror(errno)); */ 484 | /* return NULL; */ 485 | /* } */ 486 | /* len = ntohl(h.body_length); */ 487 | /* reply = malloc(len); */ 488 | /* if (!reply) */ 489 | /* return PyErr_NoMemory(); */ 490 | 491 | /* if (fread(reply, len, 1, ipp->sock_fp) <= 0) { */ 492 | /* free(reply); */ 493 | /* PyErr_Format(PyExc_IOError, "db_sketches: error reading from socket: %s", strerror(errno)); */ 494 | /* return NULL; */ 495 | /* } */ 496 | 497 | /* r = response__unpack(NULL, len, reply); */ 498 | /* free(reply); */ 499 | /* if (!r) { */ 500 | /* PyErr_Format(PyExc_IOError, "db_sketches: error unpacking"); */ 501 | /* return NULL; */ 502 | /* } */ 503 | 504 | /* if (r->error != RESPONSE__ERROR_CODE__OK) { */ 505 | /* PyErr_Format(PyExc_Exception, "read_resultset: received error from server: %i", r->error); */ 506 | /* response__free_unpacked(r, NULL); */ 507 | /* return NULL; */ 508 | /* } */ 509 | 510 | 511 | /* rv = PyList_New(r->sketches->n_sketches); */ 512 | /* if (!rv) { */ 513 | /* response__free_unpacked(r, NULL); */ 514 | /* return PyErr_NoMemory(); */ 515 | /* } */ 516 | /* for (i = 0; i < r->sketches->n_sketches; i++) { */ 517 | /* char *name = "unknown"; */ 518 | /* if (r->sketches->sketches[i]->type >= 0 && */ 519 | /* r->sketches->sketches[i]->type < sizeof(sketch_names) / sizeof(sketch_names[0])) { */ 520 | /* name = sketch_names[r->sketches->sketches[i]->type]; */ 521 | /* } */ 522 | /* PyList_SetItem(rv, i, Py_BuildValue("si", */ 523 | /* name, */ 524 | /* r->sketches->sketches[i]->window)); */ 525 | /* } */ 526 | 527 | /* response__free_unpacked(r, NULL); */ 528 | /* return rv; */ 529 | 530 | /* free_error: */ 531 | /* error: */ 532 | /* PyErr_SetString(PyExc_IOError, "db_sketches: server error"); */ 533 | /* return NULL; */ 534 | /* } */ 535 | -------------------------------------------------------------------------------- /iface_bin/readingdb.i: -------------------------------------------------------------------------------- 1 | 2 | %module readingdb 3 | %{ 4 | #include "readingdb_py.h" 5 | %} 6 | 7 | %exception db_open { 8 | $action 9 | if (!result) return NULL; 10 | } 11 | struct sock_request *db_open(char *host="localhost", short port=4242); 12 | void db_close(struct sock_request *dbp); 13 | 14 | 15 | 16 | // set up with the right database 17 | void db_setup(char *a_host, 18 | short a_port, 19 | int a_workers=5, 20 | int a_substream=0); 21 | 22 | // convert a python list of ints to a array of long longs. 23 | // if it's just one int, make that work too 24 | %typemap(in) unsigned long long * { 25 | if (PyList_Check($input)) { 26 | int size = PyList_Size($input); 27 | int i = 0; 28 | $1 = (unsigned long long *)malloc(sizeof(unsigned long long) * (size + 1)); 29 | if (!$1) return NULL; 30 | for (i = 0; i < size; i++) { 31 | PyObject *o = PyList_GetItem ($input, i); 32 | if (PyInt_Check(o)) { 33 | $1[i] = PyInt_AsUnsignedLongMask(o); 34 | } else { 35 | PyErr_SetString(PyExc_TypeError, "Stream IDs must all be Integers"); 36 | free($1); 37 | return NULL; 38 | } 39 | } 40 | $1[i] = 0; 41 | } else if (PyInt_Check($input)) { 42 | $1 = (unsigned long long *)malloc(sizeof(unsigned long long) * 2); 43 | if (!$1) return NULL; 44 | $1[0] = PyInt_AsLong($input); 45 | $1[1] = 0; 46 | } else { 47 | PyErr_SetString(PyExc_TypeError, "Not a list of streamids!"); 48 | return NULL; 49 | } 50 | } 51 | %typemap(freearg) unsigned long long * { 52 | free($1); 53 | } 54 | 55 | %exception db_query { 56 | $action 57 | if (!result) return NULL; 58 | } 59 | PyObject *db_query(unsigned long long *streamid, 60 | unsigned long long starttime, 61 | unsigned long long endtime, 62 | int limit = 10000, 63 | int substream = 0, 64 | PyObject *sketch = NULL, 65 | struct sock_request *conn = NULL); 66 | 67 | %exception db_next { 68 | $action 69 | if (!result) return NULL; 70 | } 71 | PyObject * db_next(unsigned long long *streamids, 72 | unsigned long long reference, 73 | int n = 1, 74 | PyObject *sketch = NULL, 75 | struct sock_request *conn = NULL); 76 | 77 | %exception db_prev { 78 | $action 79 | if (!result) return NULL; 80 | } 81 | PyObject * db_prev(unsigned long long *streamids, 82 | unsigned long long reference, 83 | int n = 1, 84 | PyObject *sketch = NULL, 85 | struct sock_request *conn = NULL); 86 | 87 | %exception db_add { 88 | $action 89 | if (!result) return NULL; 90 | } 91 | int db_add(struct sock_request *dbp, int streamid, PyObject *data); 92 | 93 | void db_del(struct sock_request *dbp, int streamid, 94 | unsigned long long starttime, 95 | unsigned long long endtime); 96 | 97 | // %exception db_sketches { 98 | // $action 99 | // if (!result) return NULL; 100 | // } 101 | // PyObject * db_sketches(struct sock_request *dbp); 102 | -------------------------------------------------------------------------------- /iface_bin/readingdb_py.h: -------------------------------------------------------------------------------- 1 | #ifndef _READINGDB_PY_H_ 2 | #define _READINGDB_PY_H_ 3 | 4 | #include 5 | #include 6 | 7 | #include "c6/readingdb.h" 8 | #include "c6/rpc.h" 9 | #include "c6/commands.h" 10 | 11 | struct request_desc { 12 | enum { 13 | REQ_QUERY, REQ_ITER, 14 | } type; 15 | unsigned long long *streamids; 16 | int substream; 17 | unsigned long long starttime; 18 | unsigned long long endtime; 19 | int direction; 20 | int limit; 21 | Sketch sketch; 22 | }; 23 | 24 | struct pyerr { 25 | PyObject *exception; 26 | char message[512]; 27 | }; 28 | 29 | /* exported python definition */ 30 | struct sock_request *db_open(const char *host, const short port); 31 | struct sock_request *__db_open(const char *host, const short port, int *error); 32 | void db_close(struct sock_request *dbp); 33 | void db_setup(char *a_host, 34 | short a_port, 35 | int a_workers, 36 | int a_substream); 37 | 38 | /* queries */ 39 | PyObject *db_query(unsigned long long *streamids, 40 | unsigned long long starttime, 41 | unsigned long long endtime, 42 | int limit, 43 | int substream, 44 | PyObject *sketch, 45 | struct sock_request *ipp) ; 46 | PyObject *db_next(unsigned long long *streamid, 47 | unsigned long long reference, 48 | int n, 49 | PyObject *sketch, 50 | struct sock_request *ipp); 51 | PyObject *db_prev(unsigned long long *streamid, 52 | unsigned long long reference, 53 | int n, 54 | PyObject *sketch, 55 | struct sock_request *ipp); 56 | 57 | /* modifications */ 58 | int db_add(struct sock_request *dbp, int streamid, PyObject *values); 59 | void db_del(struct sock_request *ipp, 60 | unsigned long long streamid, 61 | unsigned long long starttime, 62 | unsigned long long endtime); 63 | 64 | 65 | 66 | /* non-api functions */ 67 | 68 | /* request-generating functions -- write a request out to the socket */ 69 | int db_query_all(struct sock_request *ipp, unsigned long long streamid, 70 | unsigned long long starttime, 71 | unsigned long long endtime, 72 | int substream, 73 | const Sketch *sketch, 74 | enum query_action action); 75 | int db_iter(struct sock_request *ipp, int streamid, 76 | unsigned long long reference, 77 | const struct request_desc *req); 78 | 79 | PyObject *db_multiple(struct sock_request *ipp, const struct request_desc *req); 80 | // PyObject *db_sketches(struct sock_request *dbp); 81 | 82 | #endif 83 | -------------------------------------------------------------------------------- /iface_bin/setup.py: -------------------------------------------------------------------------------- 1 | 2 | from distutils.core import setup, Extension 3 | import numpy as np 4 | 5 | readingdb_module = Extension('_readingdb', 6 | sources=['readingdb_wrap.c', 'readingdb.c', 'multi.c', 7 | 'c6/pbuf/rdb.pb-c.c', 'c6/rpc.c'], 8 | libraries=['protobuf-c']) 9 | 10 | setup (name='readingdb-python', 11 | version='0.6.0', 12 | author='Stephen Dawson-Haggerty', 13 | author_email='stevedh@eecs.berkeley.edu', 14 | description=('Python interface to the readingdb time series database'), 15 | license='BSD', 16 | requires=['numpy'], 17 | url="https://github.com/stevedh/readingdb", 18 | ext_modules=[readingdb_module], 19 | include_dirs = [np.get_include()], 20 | py_modules=('readingdb',)) 21 | 22 | # MANIFEST.in 23 | # Makefile 24 | # Makefile.am 25 | # autotest.py 26 | # c6/pbuf/Makefile 27 | # c6/pbuf/Makefile.am 28 | # c6/pbuf/Makefile.in 29 | # c6/pbuf/rdb.proto 30 | # dataloader.py 31 | # fetch.py 32 | # import.py 33 | # import_tsdb.py 34 | # readingdb.i 35 | # test.py 36 | # test_multi.py 37 | -------------------------------------------------------------------------------- /iface_bin/simple.py: -------------------------------------------------------------------------------- 1 | 2 | import readingdb as rdb 3 | rdb.db_setup('localhost', 4242) 4 | a = rdb.db_open('localhost') 5 | 6 | rdb.db_add(a, 1, [(x, 0, x) for x in xrange(0, 100)]) 7 | print rdb.db_query(1, 0, 100, conn=a) 8 | rdb.db_close(a) 9 | -------------------------------------------------------------------------------- /iface_bin/test.py: -------------------------------------------------------------------------------- 1 | 2 | import sys 3 | import time 4 | import readingdb as rdb 5 | import _readingdb 6 | 7 | import numpy as np 8 | 9 | print "using readingdb", rdb.__file__ 10 | print _readingdb.__file__ 11 | 12 | end = 1304102690 13 | PORT = 4242 14 | 15 | rdb.db_setup('localhost', PORT) 16 | db = rdb.db_open(host='localhost', port=PORT) 17 | # db = rdb.db_open() 18 | 19 | def next(id, ref, n=1): 20 | return rdb.db_next(id, ref, n=n, conn=db)[0].tolist() 21 | 22 | def prev(id, ref, n=1, conn=db): 23 | return rdb.db_prev(id, ref, n=n, conn=db)[0].tolist() 24 | 25 | S1MAX = 1000 * 100 26 | if len(sys.argv) == 1: 27 | print "%s [-a | -r | -n | -d | -c]" % sys.argv[0] 28 | elif sys.argv[1] == '-a': 29 | # substream 1 has every bucket filled 30 | import random 31 | start = random.randint(0, end) 32 | for i in range(0, 1000): 33 | data = [(x + start, x + start, x + start) for x in xrange(i * 100, i * 100 + 100)] 34 | rdb.db_add(db, 1, data) 35 | 36 | # substream 2 has points one hour apart 37 | for i in range(start / 3600, start / 3600 + 10000): 38 | rdb.db_add(db, 2, [(i * 3600, 0, i * 3600)]) 39 | elif sys.argv[1] == '-r': 40 | # test that we read back what we wrote 41 | d = rdb.db_query(1, 0, 10000) 42 | assert len(d) == 1 43 | d = d[0].tolist() 44 | assert len(d) == 10000 45 | 46 | for i in xrange(0, 10000): 47 | assert d[i][0] == i 48 | assert d[i][1] == i 49 | 50 | d = rdb.db_query(2, 0, 3600 * 10000) 51 | assert len(d) == 1 52 | d = d[0].tolist() 53 | print d[0:10] 54 | for i in xrange(0, 10000): 55 | assert d[i][0] == i * 3600 56 | elif sys.argv[1] == '-d': 57 | # test we can delete some data 58 | d = rdb.db_del(db, 1, 0, 10000) 59 | d = rdb.db_del(db, 2, 0, 1000000) 60 | elif sys.argv[1] == '-n': 61 | # test that db_next and db_prev iterate correctly through the data 62 | for i in xrange(0, 10000): 63 | d = next(1, i) 64 | assert d[0][0] == i+1 65 | # print i 66 | d = prev(1, i) 67 | if i == 0: 68 | assert len(d) == 0 69 | else: 70 | assert d[0][0] == i - 1 71 | if not i % 100: 72 | print '.', 73 | # print "done with test 1" 74 | for i in xrange(1, 100000): 75 | d = next(2, i) 76 | assert d[0][0] == (i + 3600 - (i % 3600)) 77 | d = prev(2, i) 78 | p = i - 3600 + (3600 - (i % 3600)) 79 | if i % 3600 == 0: p -= 3600 80 | assert d[0][0] == p 81 | if not i % 1000: 82 | print '.', 83 | elif sys.argv[1] == '-s': 84 | for i in xrange(1, 2000): 85 | s = time.time() 86 | x = prev(i, int(time.time()), n=10) 87 | print len(x), (time.time() - s) 88 | elif sys.argv[1] == '-l': 89 | for f in sys.argv[2:]: 90 | with open(f, 'r') as fp: 91 | add_vec = [] 92 | streamid = int(f[f.rindex('.')+1:]) 93 | for line in fp.readlines(): 94 | parts = line.strip().split(',') 95 | print parts 96 | assert len(parts) == 2 97 | tuple = (int(parts[0]), 0, float(parts[1])) 98 | add_vec.append(tuple) 99 | if len(add_vec) == 100: 100 | rdb.db_add(db, streamid, add_vec) 101 | add_vec = [] 102 | elif sys.argv[1] == '-m': 103 | d = rdb.db_query([1, 2], 0, 100000000000, limit=-1) 104 | d1, d2 = d 105 | assert np.shape(d1) == (1e5, 2) 106 | assert np.shape(d2) == (1e4, 2) 107 | assert np.sum(d1[:, 0] - np.arange(0, 1e5)) == 0 108 | assert np.sum(d1[:, 1] - np.arange(0, 1e5)) == 0 109 | assert np.sum(d2[:, 0] - np.arange(0, 3600 * 1e4, 3600)) == 0 110 | assert np.sum(d2[:, 1] - np.arange(0, 3600 * 1e4, 3600)) == 0 111 | else: 112 | print "invalid argument" 113 | 114 | # for i in xrange(0, 1000): 115 | # x = rdb.db_next(db, 1, i) 116 | # assert x[0][0] == i+1 117 | 118 | rdb.db_close(db) 119 | -------------------------------------------------------------------------------- /s3backup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Reference: 4 | # http://docs.oracle.com/cd/E17275_01/html/programmer_reference/transapp_archival.html 5 | 6 | # see the "hot backup" steps. 7 | # 8 | # The approach used here is to first upload the database volumes to s3 9 | # simply by copying the database pages, reading them atomically in 10 | # units of at least the database page size. 11 | # 12 | # We then incrementally upload log files until we make a new snapshot. 13 | # 14 | # This technique requires pausing the automatic archiving normal 15 | # deployed with readingdb. Instead, we will manually delete the files 16 | # using this script once we have successfully archived them to s3. 17 | 18 | BUCKET=dev-backup-1 19 | BASE_BACKUP=testbackup 20 | HNAME=$(hostname) 21 | UPLOADPATH="/readingdb/$HNAME/$BASE_BACKUP" 22 | CONTENTTYPE="application/x-compressed-tar" 23 | S3KEY=XXX 24 | S3SECRET=XXX 25 | DB_ARCHIVE="/usr/local/bin/db_archive -h data" 26 | DB_CHECKPOINT="/usr/local/bin/db_checkpoint -h data" 27 | 28 | function upload() { 29 | FILE=$1 30 | DATE=$(date) 31 | SIGNSTRING="PUT\n\n$CONTENTTYPE\n$DATE\n/$BUCKET$UPLOADPATH/$FILE" 32 | SIGNATURE=`echo -en ${SIGNSTRING} | openssl sha1 -hmac ${S3SECRET} -binary | base64` 33 | 34 | echo $SIGNSTRING 35 | 36 | echo curl -XPUT -T "$FILE" \ 37 | -H "Host: $BUCKET.s3.amazonaws.com" \ 38 | -H "Date: $DATE" \ 39 | -H "Content-Type: $CONTENTTYPE" \ 40 | -H "Authorization: AWS $S3KEY:$SIGNATURE" \ 41 | https://$BUCKET.sw.amazon.aws.com/$UPLOADPATH/$FILE 42 | } 43 | 44 | if [ -z $1 ]; then 45 | echo 46 | echo "\tUsage: $0 [base|incremental]" 47 | echo 48 | 49 | exit 1 50 | fi 51 | 52 | echo "Creating checkpoint" 53 | 54 | 55 | if [ $1 == "base" ]; then 56 | # upload the database volumes 57 | for file in $($DB_ARCHIVE -s); do 58 | upload $file 59 | done 60 | fi 61 | 62 | for file in $(DB_ARCHIVE); do 63 | # upload the logs 64 | upload $file 65 | # if the upload succeeded, we can remove the log. 66 | fi 67 | --------------------------------------------------------------------------------