├── .gitignore ├── LICENSE ├── Makefile.am ├── README ├── autogen.sh ├── bindings ├── README.md ├── kvjava │ ├── KVJava.c │ ├── KVJava.java │ ├── Makefile │ ├── TODO │ └── TestKVJava.java ├── kvperl │ ├── .gitignore │ ├── Changes │ ├── KVSpool.xs │ ├── MANIFEST │ ├── Makefile │ ├── Makefile.PL │ ├── README │ ├── const-c.inc │ ├── const-xs.inc │ ├── fallback │ │ ├── const-c.inc │ │ └── const-xs.inc │ ├── lib │ │ └── KVSpool.pm │ ├── ppport.h │ └── t │ │ └── TestKVSpool.t └── kvpy │ ├── .gitignore │ ├── Makefile │ ├── build.py │ ├── kvspool.c │ └── tests │ ├── README │ ├── read.py │ ├── read_all.py │ ├── runtest.sh │ ├── speed.py │ └── write.py ├── config ├── compare.m4 ├── perl.m4 └── python.m4 ├── configure.ac ├── doc ├── LICENSE.txt ├── Makefile ├── index.html ├── index.txt ├── pub-sub.png ├── pub-sub.txt ├── reader-writer.png └── reader-writer.txt ├── include ├── kvspool.h ├── kvspool_internal.h ├── tpl.h ├── utarray.h ├── uthash.h └── utstring.h ├── src ├── Makefile.am ├── kvspool.c ├── kvspoolr.c ├── kvspoolw.c └── tpl.c └── utils ├── .gitignore ├── Makefile.am ├── kvsp-bcat.c ├── kvsp-bconfig.c ├── kvsp-bconfig.h ├── kvsp-bpub.c ├── kvsp-bshr.c ├── kvsp-bsub.c ├── kvsp-concen.c ├── kvsp-init.c ├── kvsp-kkpub.c ├── kvsp-mod.c ├── kvsp-mpub.c ├── kvsp-npub.c ├── kvsp-nsub.c ├── kvsp-pub.c ├── kvsp-rewind.c ├── kvsp-speed.c ├── kvsp-spr.c ├── kvsp-spw.c ├── kvsp-status.c ├── kvsp-sub.c ├── kvsp-tee.c ├── kvsp-tpub.c ├── kvsp-tsub.c ├── kvsp-upub.c ├── ramdisk.c ├── ringbuf.c ├── ringbuf.h ├── spr-cast.cfg ├── ts.c └── ts.h /.gitignore: -------------------------------------------------------------------------------- 1 | Makefile 2 | *.cache 3 | *.log 4 | *.o 5 | *.a 6 | .deps 7 | Makefile.in 8 | config.status 9 | config 10 | configure 11 | aclocal.m4 12 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | © 2011 The Johns Hopkins University Applied Physics Laboratory LLC. All Rights Reserved. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /Makefile.am: -------------------------------------------------------------------------------- 1 | SUBDIRS = src utils 2 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | kvspool data streaming utilities 2 | by Troy D. Hanson 3 | 4 | Documentation for kvspool is at: 5 | 6 | http://troydhanson.github.io/kvspool 7 | 8 | -------------------------------------------------------------------------------- /autogen.sh: -------------------------------------------------------------------------------- 1 | autoreconf -ifv 2 | -------------------------------------------------------------------------------- /bindings/README.md: -------------------------------------------------------------------------------- 1 | Currently deprecated 2 | -------------------------------------------------------------------------------- /bindings/kvjava/KVJava.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "KVJava.h" 4 | #include "kvspool.h" 5 | 6 | typedef struct { 7 | void *spr; 8 | void *spw; 9 | void *set; 10 | char dir[PATH_MAX]; 11 | } kvsp_handle_t; 12 | 13 | /* retrieve the handle from the object, creating it if need be */ 14 | static kvsp_handle_t *get_handle(JNIEnv *env, jobject obj) { 15 | jclass cls = (*env)->GetObjectClass(env, obj); 16 | jfieldID id = (*env)->GetFieldID(env, cls, "kvsp_handle", "J"); 17 | jlong kvsp_handle_jlong = (*env)->GetLongField(env, obj, id); 18 | /* doing the cast through tmp silences a compiler warning on 32-bit */ 19 | intptr_t tmp = (intptr_t)kvsp_handle_jlong; 20 | kvsp_handle_t *kvsp_handle = (kvsp_handle_t*)tmp; 21 | if (kvsp_handle == NULL) { 22 | kvsp_handle = calloc(1, sizeof(kvsp_handle_t)); 23 | kvsp_handle->set = kv_set_new(); 24 | (*env)->SetLongField(env, obj, id, (long)kvsp_handle); 25 | } 26 | return kvsp_handle; 27 | } 28 | 29 | static const char *get_dir(JNIEnv *env, jobject obj) { 30 | kvsp_handle_t *handle = get_handle(env, obj); 31 | jclass cls = (*env)->GetObjectClass(env, obj); 32 | jfieldID id = (*env)->GetFieldID(env, cls, "dir", "Ljava/lang/String;"); 33 | jstring dirj = (jstring) (*env)->GetObjectField(env, obj, id); 34 | kvsp_handle_t *h = get_handle(env, obj); 35 | const char* dir = (*env)->GetStringUTFChars(env, dirj, 0); 36 | strncpy(h->dir, dir, sizeof(h->dir)); 37 | (*env)->ReleaseStringUTFChars(env, dirj, dir); 38 | return h->dir; 39 | } 40 | 41 | static unsigned char get_blocking(JNIEnv *env, jobject obj) { 42 | jclass cls = (*env)->GetObjectClass(env, obj); 43 | jfieldID id = (*env)->GetFieldID(env, cls, "blocking", "Z"); 44 | jboolean blockingj = (jboolean) (*env)->GetBooleanField(env, obj, id); 45 | unsigned char blocking = (unsigned char)blockingj; 46 | return blocking; 47 | } 48 | 49 | jint throwIoError(JNIEnv *env, char* message) { 50 | jclass exClass; 51 | char *className = "java/io/IOException"; 52 | 53 | exClass = (*env)->FindClass(env, className); 54 | if (exClass == NULL) { 55 | //return throwNoClassDefError(env, className); 56 | return -1; 57 | } 58 | 59 | return (*env)->ThrowNew(env, exClass, message); 60 | } 61 | 62 | jobject kvs_to_map(JNIEnv* env, void *set) { 63 | jclass mapClass = NULL; 64 | static jmethodID init = NULL; 65 | static jmethodID put = NULL; 66 | //if (mapClass == NULL) { 67 | mapClass = (*env)->FindClass(env, "java/util/HashMap"); 68 | //} 69 | if (mapClass == NULL) { 70 | return NULL; 71 | } 72 | 73 | jsize map_len = kv_len(set); 74 | if (init == NULL) { 75 | init = (*env)->GetMethodID(env, mapClass, "", "(I)V"); 76 | } 77 | jobject obj = (*env)->NewObject(env, mapClass, init, map_len); 78 | if (put == NULL) { 79 | put = (*env)->GetMethodID(env, mapClass, "put", "(Ljava/lang/Object;Ljava/lang/Object;)Ljava/lang/Object;"); 80 | } 81 | 82 | kv_t *kv = NULL; 83 | while ((kv = kv_next(set, kv))) { 84 | jstring key = (*env)->NewStringUTF(env, kv->key); 85 | jstring val = (*env)->NewStringUTF(env, kv->val); 86 | (*env)->CallObjectMethod(env, obj, put, key, val); 87 | } 88 | 89 | return obj; 90 | } 91 | 92 | JNIEXPORT jobject JNICALL Java_KVJava_read(JNIEnv * env, jobject obj) { 93 | kvsp_handle_t *h = get_handle(env, obj); 94 | jobject ret=NULL; 95 | 96 | if (h->spr == NULL) { 97 | const char *dir = get_dir(env, obj); 98 | h->spr = kv_spoolreader_new(dir); 99 | if (h->spr == NULL) { 100 | throwIoError(env,"Failed to open spool"); 101 | return NULL; 102 | } 103 | } 104 | int blocking = get_blocking(env, obj); 105 | int rc = kv_spool_read(h->spr, h->set, blocking); 106 | if (rc < 0) { 107 | throwIoError(env,"Spool reader error"); 108 | } else if (rc == 0) { 109 | // non blocking mode, but no data available; return null below 110 | } else { 111 | ret = kvs_to_map(env, h->set); 112 | } 113 | return ret; 114 | } 115 | 116 | int map_to_kvs(JNIEnv* env, jobject obj, void *set) { 117 | jclass mapClass = NULL; 118 | jclass setClass = NULL; 119 | jclass iteratorClass = NULL; 120 | static jmethodID get = NULL; 121 | 122 | static jmethodID keySet = NULL; 123 | static jmethodID iterator = NULL; 124 | static jmethodID hasNext = NULL; 125 | static jmethodID next = NULL; 126 | //if (mapClass == NULL) { 127 | mapClass = (*env)->FindClass(env, "java/util/HashMap"); 128 | 129 | setClass = (*env)->FindClass(env, "java/util/Set"); 130 | iteratorClass = (*env)->FindClass(env, "java/util/Iterator"); 131 | //} 132 | 133 | if (get == NULL) { 134 | get = (*env)->GetMethodID(env, mapClass, "get", "(Ljava/lang/Object;)Ljava/lang/Object;"); 135 | } 136 | if (keySet == NULL) { 137 | keySet = (*env)->GetMethodID(env, mapClass, "keySet", "()Ljava/util/Set;"); 138 | } 139 | if (iterator == NULL) { 140 | iterator = (*env)->GetMethodID(env, setClass, "iterator", "()Ljava/util/Iterator;"); 141 | } 142 | if (hasNext == NULL) { 143 | hasNext = (*env)->GetMethodID(env, iteratorClass, "hasNext", "()Z"); 144 | } 145 | if (next == NULL) { 146 | next = (*env)->GetMethodID(env, iteratorClass, "next", "()Ljava/lang/Object;"); 147 | } 148 | jobject sets = (*env)->CallObjectMethod(env, obj, keySet); 149 | jobject iter = (*env)->CallObjectMethod(env, sets, iterator); 150 | jmethodID toStringK = NULL; 151 | jmethodID toStringV = NULL; 152 | jclass cls = NULL; 153 | while ((*env)->CallBooleanMethod(env, iter, hasNext)) { 154 | 155 | 156 | jobject key = (*env)->CallObjectMethod(env, iter, next); 157 | jobject val = (*env)->CallObjectMethod(env, obj, get, key); 158 | cls = (*env)->GetObjectClass(env, key); 159 | toStringK = (*env)->GetMethodID(env, cls, "toString", "()Ljava/lang/String;"); 160 | key = (*env)->CallObjectMethod(env, key, toStringK); 161 | cls = (*env)->GetObjectClass(env, val); 162 | toStringV = (*env)->GetMethodID(env, cls, "toString", "()Ljava/lang/String;"); 163 | val = (*env)->CallObjectMethod(env, val, toStringV); 164 | const char* keyc = (*env)->GetStringUTFChars(env, key, 0); 165 | 166 | const char* valc = (*env)->GetStringUTFChars(env, val, 0); 167 | kv_add(set, keyc, (*env)->GetStringUTFLength(env, key), valc, (*env)->GetStringUTFLength(env, val)); 168 | 169 | (*env)->ReleaseStringUTFChars(env, key, keyc); 170 | 171 | (*env)->ReleaseStringUTFChars(env, val, valc); 172 | } 173 | 174 | //return obj; 175 | return -1; 176 | } 177 | 178 | JNIEXPORT void JNICALL Java_KVJava_write(JNIEnv *env, jobject obj, jobject map) { 179 | kvsp_handle_t *h = get_handle(env, obj); 180 | if (h->spw == NULL) { 181 | const char *dir = get_dir(env, obj); 182 | h->spw = kv_spoolwriter_new(dir); 183 | if (h->spw == NULL) { 184 | throwIoError(env,"Failed to open spool"); 185 | return; 186 | } 187 | } 188 | map_to_kvs(env, map, h->set); 189 | kv_spool_write(h->spw, h->set); 190 | } 191 | 192 | JNIEXPORT void JNICALL Java_KVJava_close(JNIEnv *env, jobject obj) { 193 | jclass cls = (*env)->GetObjectClass(env, obj); 194 | jfieldID kvsp_handle_id = (*env)->GetFieldID(env, cls, "kvsp_handle", "J"); 195 | jlong kvsp_handle_jlong = (*env)->GetLongField(env, obj, kvsp_handle_id); 196 | /* doing the cast through tmp silences a compiler warning on 32-bit */ 197 | intptr_t tmp = (intptr_t)kvsp_handle_jlong; 198 | kvsp_handle_t *kvsp_handle = (kvsp_handle_t*)tmp; 199 | 200 | if (kvsp_handle == NULL) return; 201 | if (kvsp_handle->set) kv_set_free(kvsp_handle->set); 202 | if (kvsp_handle->spr) kv_spoolreader_free(kvsp_handle->spr); 203 | if (kvsp_handle->spw) kv_spoolwriter_free(kvsp_handle->spw); 204 | (*env)->SetLongField(env, obj, kvsp_handle_id, 0); 205 | } 206 | -------------------------------------------------------------------------------- /bindings/kvjava/KVJava.java: -------------------------------------------------------------------------------- 1 | 2 | /* the KVJava object has methods implemented as JNI. This file is the Java 3 | * part of the class, which is compiled as usual with javac, then a 4 | * C header is generated from it (using javah -jni) for the JNI functions. 5 | * The implementation of the JNI functions in C is in KVJava.c. This gets 6 | * built into a shared library (which must be installed to a place the 7 | * dynamic linker can find it at runtime). 8 | */ 9 | 10 | import java.util.HashMap; 11 | import java.lang.String; 12 | 13 | public class KVJava { 14 | static { 15 | System.loadLibrary("KVJava"); // dynamically load native libKVJava.so 16 | } 17 | 18 | public boolean blocking; 19 | 20 | private String dir; /* the spool directory */ 21 | private long kvsp_handle; /* a C structure address, opaque to Java */ 22 | 23 | public KVJava(String _dir) { 24 | dir = _dir; 25 | blocking = true; 26 | } 27 | 28 | public native HashMap read(); 29 | public native void write(HashMap map); 30 | public native void close(); 31 | } 32 | -------------------------------------------------------------------------------- /bindings/kvjava/Makefile: -------------------------------------------------------------------------------- 1 | all: KVJava.class libKVJava.so 2 | #CFLAGS = -g -O0 3 | JNIINC=/usr/lib/jvm/java-6-openjdk/include/ 4 | #JNILIB=/usr/lib/jvm/java-6-openjdk/jre/lib/i386/ 5 | JNILIB=/usr/lib/jvm/java-6-openjdk/jre/lib/amd64/ 6 | 7 | KVJava.class: KVJava.java 8 | javac $< 9 | 10 | KVJava.h: KVJava.class 11 | javah -jni KVJava 12 | 13 | libKVJava.so: KVJava.h KVJava.class KVJava.c 14 | $(CC) $(CFLAGS) -o $@ -fPIC -shared -I. -I${JNIINC} -I../include KVJava.c -L../src/ -L.. -lkvspool -L${JNILIB} -ljava -static-libgcc 15 | 16 | .PHONY: clean test 17 | 18 | test: 19 | mkdir -p /tmp/spool 20 | javac TestKVJava.java 21 | java TestKVJava 22 | 23 | clean: 24 | rm -f KVJava.h *.class *.so 25 | -------------------------------------------------------------------------------- /bindings/kvjava/TODO: -------------------------------------------------------------------------------- 1 | JNIINC and JNILIB need to be done in autoconf test 2 | detect jni 3 | -------------------------------------------------------------------------------- /bindings/kvjava/TestKVJava.java: -------------------------------------------------------------------------------- 1 | import java.util.HashMap; 2 | import java.util.Map; 3 | import java.lang.String; 4 | 5 | public class TestKVJava { 6 | 7 | private static void printMap(HashMap m) { 8 | for (Map.Entry entry : m.entrySet()) { 9 | String key = entry.getKey(); 10 | String val = entry.getValue(); 11 | System.out.println(key + ": " + val); 12 | } 13 | } 14 | 15 | public static void main(String args[]) { 16 | KVJava kv = new KVJava("/tmp/spool"); 17 | HashMap h = new HashMap(); 18 | 19 | /* 20 | * test the writer 21 | */ 22 | h.put("user", "troy"); 23 | h.put("id", "100"); 24 | kv.write(h); 25 | h.put("user", "bob"); 26 | h.put("id", "101"); 27 | kv.write(h); 28 | 29 | /* 30 | * test the reader 31 | */ 32 | HashMap m; 33 | m = kv.read(); 34 | printMap(m); 35 | 36 | /* test non blocking mode */ 37 | kv.blocking = false; 38 | m = kv.read(); 39 | if (m != null) printMap(m); 40 | else System.out.println("non-blocking read: no data"); 41 | 42 | m = kv.read(); 43 | if (m != null) printMap(m); 44 | else System.out.println("non-blocking read: no data"); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /bindings/kvperl/.gitignore: -------------------------------------------------------------------------------- 1 | KVSpool.bs 2 | KVSpool.c 3 | Makefile.gen 4 | blib/ 5 | pm_to_blib 6 | -------------------------------------------------------------------------------- /bindings/kvperl/Changes: -------------------------------------------------------------------------------- 1 | Revision history for Perl extension KVSpool. 2 | 3 | 0.01 Wed Jun 15 13:02:15 2011 4 | - original version; created by h2xs 1.23 with options 5 | -n KVSpool kvspool.h 6 | 7 | -------------------------------------------------------------------------------- /bindings/kvperl/KVSpool.xs: -------------------------------------------------------------------------------- 1 | #include "EXTERN.h" 2 | #include "perl.h" 3 | #include "XSUB.h" 4 | 5 | #include "ppport.h" 6 | 7 | #include 8 | 9 | #include "const-c.inc" 10 | 11 | 12 | int hash_to_kvs(HV *hash, void *set) { 13 | SV *pv; 14 | int rc = -1; 15 | char *k, *v; 16 | int len = 0; 17 | 18 | while ((pv = hv_iternextsv(hash,&k,(I32*)&len))!= 0) { 19 | STRLEN le; 20 | v = SvPV(pv,le); 21 | if (!k || !v) goto done; 22 | kv_add(set, k,len, v,le); 23 | } 24 | rc = 0; 25 | done: 26 | return rc; 27 | } 28 | 29 | SV** kvs_to_hash(HV **hash, void *set) { 30 | SV *pv; //,*pk; 31 | SV **rc = NULL; 32 | 33 | *hash = newHV(); 34 | 35 | kv_t *kv=NULL; 36 | while ( (kv= kv_next(set,kv))) { 37 | 38 | pv = newSVpv(kv->val,kv->vlen); 39 | rc = hv_store(*hash,kv->key,kv->klen,pv,0); 40 | 41 | if (rc == NULL) break; 42 | } 43 | 44 | if (rc == NULL) { 45 | hv_undef(*hash); 46 | *hash = NULL; 47 | } 48 | return rc; 49 | } 50 | /* 51 | */ 52 | 53 | 54 | 55 | MODULE = KVSpool PACKAGE = KVSpool 56 | 57 | INCLUDE: const-xs.inc 58 | 59 | 60 | SV* 61 | makeset() 62 | PREINIT: 63 | void *set; 64 | CODE: 65 | 66 | set = kv_set_new(); 67 | if ( set == NULL) { 68 | croak("cannot initialize set"); 69 | } 70 | SV* pv = newSViv(PTR2IV(set)); 71 | RETVAL = pv; 72 | OUTPUT: 73 | RETVAL 74 | 75 | 76 | 77 | SV* 78 | makersp(char * dir) 79 | PREINIT: 80 | void *sp; 81 | CODE: 82 | 83 | if ( (sp = kv_spoolreader_new(dir)) == NULL) { 84 | croak("cannot initialize spool reader"); 85 | } 86 | //IV pv = PTR2IV(sp); 87 | SV* pv = newSViv(PTR2IV(sp)); 88 | // printf("%i %i\n",SvUV(pv),sp); 89 | //printf("IV is %"IVdf" %i\n", pv,sp); 90 | 91 | RETVAL = pv;//newRV(pv); 92 | OUTPUT: 93 | RETVAL 94 | 95 | 96 | 97 | 98 | SV* 99 | makewsp(char * dir) 100 | PREINIT: 101 | void *sp; 102 | CODE: 103 | 104 | if ( (sp = kv_spoolwriter_new(dir)) == NULL) { 105 | croak("cannot initialize spool reader"); 106 | } 107 | SV* pv = newSViv(PTR2IV(sp)); 108 | 109 | RETVAL = pv;//newRV(pv); 110 | OUTPUT: 111 | RETVAL 112 | 113 | void 114 | freeset(IV pos) 115 | INIT: 116 | void *set; 117 | CODE: 118 | set = INT2PTR(void*,pos); 119 | kv_set_free(set); 120 | 121 | 122 | 123 | void 124 | freersp(IV pos) 125 | INIT: 126 | void *sp; 127 | CODE: 128 | sp = INT2PTR(void*,pos); 129 | kv_spoolreader_free(sp); 130 | 131 | 132 | void 133 | freewsp(IV pos) 134 | INIT: 135 | void *sp; 136 | CODE: 137 | sp = INT2PTR(void*,pos); 138 | kv_spoolwriter_free(sp); 139 | 140 | 141 | 142 | HV* 143 | kvread(IV pos,void *set, int block) 144 | INIT: 145 | void *sp; 146 | HV *hash = NULL; 147 | int rc =0; 148 | SV** val; 149 | CODE: 150 | sp = INT2PTR(void*,pos); 151 | 152 | if ( (rc=kv_spool_read(sp,set,block)) > 0) { 153 | kvs_to_hash(&hash, set); 154 | RETVAL = hash; 155 | } else if (rc == 0) { /* non blocking read, no data available */ 156 | XSRETURN_UNDEF; 157 | } else if (rc < 0) { 158 | croak("internal error in spool reader"); 159 | } 160 | OUTPUT: 161 | RETVAL 162 | 163 | 164 | void 165 | kvwrite(IV pos, void *set, HV* hash) 166 | INIT: 167 | void *sp; 168 | CODE: 169 | sp = INT2PTR(void*,pos); 170 | if (hash_to_kvs(hash, set) == -1) { 171 | croak("non-string key or value"); 172 | } 173 | kv_spool_write(sp,set); 174 | 175 | 176 | 177 | HV* 178 | kv_stat(char * dir) 179 | INIT: 180 | int sc, i; 181 | SV** rc = NULL; 182 | kv_stat_t stats; 183 | HV *hash = NULL; 184 | SV *pv; 185 | CODE: 186 | 187 | sc = kv_stat(dir,&stats); 188 | if (sc == -1) { 189 | croak("kv_stat failed"); 190 | } 191 | 192 | hash = newHV(); 193 | pv = sv_2mortal(newSViv((long)(stats.pct_consumed))); 194 | rc = hv_store(hash,"pct",strlen("pct"),pv,0); 195 | if (rc == NULL) { 196 | sv_2mortal((SV*)hash); 197 | hash = NULL; 198 | } 199 | 200 | RETVAL = hash; 201 | OUTPUT: 202 | RETVAL 203 | 204 | 205 | 206 | -------------------------------------------------------------------------------- /bindings/kvperl/MANIFEST: -------------------------------------------------------------------------------- 1 | Changes 2 | KVSpool.xs 3 | Makefile.PL 4 | MANIFEST 5 | ppport.h 6 | README 7 | t/KVSpool.t 8 | fallback/const-c.inc 9 | fallback/const-xs.inc 10 | lib/KVSpool.pm 11 | -------------------------------------------------------------------------------- /bindings/kvperl/Makefile: -------------------------------------------------------------------------------- 1 | all distclean clean install: 2 | perl -f Makefile.PL 3 | make -f Makefile.gen $@ 4 | -------------------------------------------------------------------------------- /bindings/kvperl/Makefile.PL: -------------------------------------------------------------------------------- 1 | use 5.010001; 2 | use ExtUtils::MakeMaker; 3 | # See lib/ExtUtils/MakeMaker.pm for details of how to influence 4 | # the contents of the Makefile that is written. 5 | WriteMakefile( 6 | FIRST_MAKEFILE => 'Makefile.gen', 7 | NAME => 'KVSpool', 8 | VERSION_FROM => 'lib/KVSpool.pm', # finds $VERSION 9 | PREREQ_PM => {}, # e.g., Module::Name => 1.1 10 | ($] >= 5.005 ? ## Add these new keywords supported since 5.005 11 | (ABSTRACT_FROM => 'lib/KVSpool.pm', # retrieve abstract from module 12 | AUTHOR => 'Trevor Adams ') : ()), 13 | LIBS => ['-L../src -lkvspool'], # e.g., '-lm' 14 | DEFINE => '', # e.g., '-DHAVE_SOMETHING' 15 | INC => '-I. -I../include -I..', # e.g., '-I. -I/usr/include/other' 16 | # Un-comment this if you add C files to link with later: 17 | # OBJECT => '$(O_FILES)', # link all the C files too 18 | ); 19 | if (1) { 20 | use File::Copy; 21 | use File::Spec; 22 | foreach my $file ('const-c.inc', 'const-xs.inc') { 23 | my $fallback = File::Spec->catfile('fallback', $file); 24 | copy ($fallback, $file) or die "Can't copy $fallback to $file: $!"; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /bindings/kvperl/README: -------------------------------------------------------------------------------- 1 | KVSpool version 0.01 2 | ==================== 3 | 4 | The README is used to introduce the module and provide instructions on 5 | how to install the module, any machine dependencies it may have (for 6 | example C compilers and installed libraries) and any other information 7 | that should be provided before the module is installed. 8 | 9 | A README file is required for CPAN modules since CPAN extracts the 10 | README file from a module distribution so that people browsing the 11 | archive can use it get an idea of the modules uses. It is usually a 12 | good idea to provide version information here so that people can 13 | decide whether fixes for the module are worth downloading. 14 | 15 | INSTALLATION 16 | 17 | To install this module type the following: 18 | 19 | perl Makefile.PL 20 | make 21 | make test 22 | make install 23 | 24 | DEPENDENCIES 25 | 26 | This module requires these other modules and libraries: 27 | 28 | blah blah blah 29 | 30 | COPYRIGHT AND LICENCE 31 | 32 | Put the correct copyright and licence information here. 33 | 34 | Copyright (C) 2011 by A. U. Thor 35 | 36 | This library is free software; you can redistribute it and/or modify 37 | it under the same terms as Perl itself, either Perl version 5.10.1 or, 38 | at your option, any later version of Perl 5 you may have available. 39 | 40 | 41 | -------------------------------------------------------------------------------- /bindings/kvperl/const-c.inc: -------------------------------------------------------------------------------- 1 | #define PERL_constant_NOTFOUND 1 2 | #define PERL_constant_NOTDEF 2 3 | #define PERL_constant_ISIV 3 4 | #define PERL_constant_ISNO 4 5 | #define PERL_constant_ISNV 5 6 | #define PERL_constant_ISPV 6 7 | #define PERL_constant_ISPVN 7 8 | #define PERL_constant_ISSV 8 9 | #define PERL_constant_ISUNDEF 9 10 | #define PERL_constant_ISUV 10 11 | #define PERL_constant_ISYES 11 12 | 13 | #ifndef NVTYPE 14 | typedef double NV; /* 5.6 and later define NVTYPE, and typedef NV to it. */ 15 | #endif 16 | #ifndef aTHX_ 17 | #define aTHX_ /* 5.6 or later define this for threading support. */ 18 | #endif 19 | #ifndef pTHX_ 20 | #define pTHX_ /* 5.6 or later define this for threading support. */ 21 | #endif 22 | 23 | static int 24 | constant (pTHX_ const char *name, STRLEN len, IV *iv_return) { 25 | /* Initially switch on the length of the name. */ 26 | /* When generated this function returned values for the list of names given 27 | in this section of perl code. Rather than manually editing these functions 28 | to add or remove constants, which would result in this comment and section 29 | of code becoming inaccurate, we recommend that you edit this section of 30 | code, and use it to regenerate a new set of constant functions which you 31 | then use to replace the originals. 32 | 33 | Regenerate these constant functions by feeding this entire source file to 34 | perl -x 35 | 36 | #!/usr/bin/perl -w 37 | use ExtUtils::Constant qw (constant_types C_constant XS_constant); 38 | 39 | my $types = {map {($_, 1)} qw(IV)}; 40 | my @names = (qw(KV_BASE_MAX)); 41 | 42 | print constant_types(), "\n"; # macro defs 43 | foreach (C_constant ("KVSpool", 'constant', 'IV', $types, undef, 3, @names) ) { 44 | print $_, "\n"; # C constant subs 45 | } 46 | print "\n#### XS Section:\n"; 47 | print XS_constant ("KVSpool", $types); 48 | __END__ 49 | */ 50 | 51 | switch (len) { 52 | case 11: 53 | if (memEQ(name, "KV_BASE_MAX", 11)) { 54 | #ifdef KV_BASE_MAX 55 | *iv_return = KV_BASE_MAX; 56 | return PERL_constant_ISIV; 57 | #else 58 | return PERL_constant_NOTDEF; 59 | #endif 60 | } 61 | break; 62 | } 63 | return PERL_constant_NOTFOUND; 64 | } 65 | 66 | -------------------------------------------------------------------------------- /bindings/kvperl/const-xs.inc: -------------------------------------------------------------------------------- 1 | void 2 | constant(sv) 3 | PREINIT: 4 | #ifdef dXSTARG 5 | dXSTARG; /* Faster if we have it. */ 6 | #else 7 | dTARGET; 8 | #endif 9 | STRLEN len; 10 | int type; 11 | IV iv; 12 | /* NV nv; Uncomment this if you need to return NVs */ 13 | /* const char *pv; Uncomment this if you need to return PVs */ 14 | INPUT: 15 | SV * sv; 16 | const char * s = SvPV(sv, len); 17 | PPCODE: 18 | /* Change this to constant(aTHX_ s, len, &iv, &nv); 19 | if you need to return both NVs and IVs */ 20 | type = constant(aTHX_ s, len, &iv); 21 | /* Return 1 or 2 items. First is error message, or undef if no error. 22 | Second, if present, is found value */ 23 | switch (type) { 24 | case PERL_constant_NOTFOUND: 25 | sv = 26 | sv_2mortal(newSVpvf("%s is not a valid KVSpool macro", s)); 27 | PUSHs(sv); 28 | break; 29 | case PERL_constant_NOTDEF: 30 | sv = sv_2mortal(newSVpvf( 31 | "Your vendor has not defined KVSpool macro %s, used", 32 | s)); 33 | PUSHs(sv); 34 | break; 35 | case PERL_constant_ISIV: 36 | EXTEND(SP, 1); 37 | PUSHs(&PL_sv_undef); 38 | PUSHi(iv); 39 | break; 40 | /* Uncomment this if you need to return NOs 41 | case PERL_constant_ISNO: 42 | EXTEND(SP, 1); 43 | PUSHs(&PL_sv_undef); 44 | PUSHs(&PL_sv_no); 45 | break; */ 46 | /* Uncomment this if you need to return NVs 47 | case PERL_constant_ISNV: 48 | EXTEND(SP, 1); 49 | PUSHs(&PL_sv_undef); 50 | PUSHn(nv); 51 | break; */ 52 | /* Uncomment this if you need to return PVs 53 | case PERL_constant_ISPV: 54 | EXTEND(SP, 1); 55 | PUSHs(&PL_sv_undef); 56 | PUSHp(pv, strlen(pv)); 57 | break; */ 58 | /* Uncomment this if you need to return PVNs 59 | case PERL_constant_ISPVN: 60 | EXTEND(SP, 1); 61 | PUSHs(&PL_sv_undef); 62 | PUSHp(pv, iv); 63 | break; */ 64 | /* Uncomment this if you need to return SVs 65 | case PERL_constant_ISSV: 66 | EXTEND(SP, 1); 67 | PUSHs(&PL_sv_undef); 68 | PUSHs(sv); 69 | break; */ 70 | /* Uncomment this if you need to return UNDEFs 71 | case PERL_constant_ISUNDEF: 72 | break; */ 73 | /* Uncomment this if you need to return UVs 74 | case PERL_constant_ISUV: 75 | EXTEND(SP, 1); 76 | PUSHs(&PL_sv_undef); 77 | PUSHu((UV)iv); 78 | break; */ 79 | /* Uncomment this if you need to return YESs 80 | case PERL_constant_ISYES: 81 | EXTEND(SP, 1); 82 | PUSHs(&PL_sv_undef); 83 | PUSHs(&PL_sv_yes); 84 | break; */ 85 | default: 86 | sv = sv_2mortal(newSVpvf( 87 | "Unexpected return type %d while processing KVSpool macro %s, used", 88 | type, s)); 89 | PUSHs(sv); 90 | } 91 | -------------------------------------------------------------------------------- /bindings/kvperl/fallback/const-c.inc: -------------------------------------------------------------------------------- 1 | #define PERL_constant_NOTFOUND 1 2 | #define PERL_constant_NOTDEF 2 3 | #define PERL_constant_ISIV 3 4 | #define PERL_constant_ISNO 4 5 | #define PERL_constant_ISNV 5 6 | #define PERL_constant_ISPV 6 7 | #define PERL_constant_ISPVN 7 8 | #define PERL_constant_ISSV 8 9 | #define PERL_constant_ISUNDEF 9 10 | #define PERL_constant_ISUV 10 11 | #define PERL_constant_ISYES 11 12 | 13 | #ifndef NVTYPE 14 | typedef double NV; /* 5.6 and later define NVTYPE, and typedef NV to it. */ 15 | #endif 16 | #ifndef aTHX_ 17 | #define aTHX_ /* 5.6 or later define this for threading support. */ 18 | #endif 19 | #ifndef pTHX_ 20 | #define pTHX_ /* 5.6 or later define this for threading support. */ 21 | #endif 22 | 23 | static int 24 | constant (pTHX_ const char *name, STRLEN len, IV *iv_return) { 25 | /* Initially switch on the length of the name. */ 26 | /* When generated this function returned values for the list of names given 27 | in this section of perl code. Rather than manually editing these functions 28 | to add or remove constants, which would result in this comment and section 29 | of code becoming inaccurate, we recommend that you edit this section of 30 | code, and use it to regenerate a new set of constant functions which you 31 | then use to replace the originals. 32 | 33 | Regenerate these constant functions by feeding this entire source file to 34 | perl -x 35 | 36 | #!/usr/bin/perl -w 37 | use ExtUtils::Constant qw (constant_types C_constant XS_constant); 38 | 39 | my $types = {map {($_, 1)} qw(IV)}; 40 | my @names = (qw(KV_BASE_MAX)); 41 | 42 | print constant_types(), "\n"; # macro defs 43 | foreach (C_constant ("KVSpool", 'constant', 'IV', $types, undef, 3, @names) ) { 44 | print $_, "\n"; # C constant subs 45 | } 46 | print "\n#### XS Section:\n"; 47 | print XS_constant ("KVSpool", $types); 48 | __END__ 49 | */ 50 | 51 | switch (len) { 52 | case 11: 53 | if (memEQ(name, "KV_BASE_MAX", 11)) { 54 | #ifdef KV_BASE_MAX 55 | *iv_return = KV_BASE_MAX; 56 | return PERL_constant_ISIV; 57 | #else 58 | return PERL_constant_NOTDEF; 59 | #endif 60 | } 61 | break; 62 | } 63 | return PERL_constant_NOTFOUND; 64 | } 65 | 66 | -------------------------------------------------------------------------------- /bindings/kvperl/fallback/const-xs.inc: -------------------------------------------------------------------------------- 1 | void 2 | constant(sv) 3 | PREINIT: 4 | #ifdef dXSTARG 5 | dXSTARG; /* Faster if we have it. */ 6 | #else 7 | dTARGET; 8 | #endif 9 | STRLEN len; 10 | int type; 11 | IV iv; 12 | /* NV nv; Uncomment this if you need to return NVs */ 13 | /* const char *pv; Uncomment this if you need to return PVs */ 14 | INPUT: 15 | SV * sv; 16 | const char * s = SvPV(sv, len); 17 | PPCODE: 18 | /* Change this to constant(aTHX_ s, len, &iv, &nv); 19 | if you need to return both NVs and IVs */ 20 | type = constant(aTHX_ s, len, &iv); 21 | /* Return 1 or 2 items. First is error message, or undef if no error. 22 | Second, if present, is found value */ 23 | switch (type) { 24 | case PERL_constant_NOTFOUND: 25 | sv = 26 | sv_2mortal(newSVpvf("%s is not a valid KVSpool macro", s)); 27 | PUSHs(sv); 28 | break; 29 | case PERL_constant_NOTDEF: 30 | sv = sv_2mortal(newSVpvf( 31 | "Your vendor has not defined KVSpool macro %s, used", 32 | s)); 33 | PUSHs(sv); 34 | break; 35 | case PERL_constant_ISIV: 36 | EXTEND(SP, 1); 37 | PUSHs(&PL_sv_undef); 38 | PUSHi(iv); 39 | break; 40 | /* Uncomment this if you need to return NOs 41 | case PERL_constant_ISNO: 42 | EXTEND(SP, 1); 43 | PUSHs(&PL_sv_undef); 44 | PUSHs(&PL_sv_no); 45 | break; */ 46 | /* Uncomment this if you need to return NVs 47 | case PERL_constant_ISNV: 48 | EXTEND(SP, 1); 49 | PUSHs(&PL_sv_undef); 50 | PUSHn(nv); 51 | break; */ 52 | /* Uncomment this if you need to return PVs 53 | case PERL_constant_ISPV: 54 | EXTEND(SP, 1); 55 | PUSHs(&PL_sv_undef); 56 | PUSHp(pv, strlen(pv)); 57 | break; */ 58 | /* Uncomment this if you need to return PVNs 59 | case PERL_constant_ISPVN: 60 | EXTEND(SP, 1); 61 | PUSHs(&PL_sv_undef); 62 | PUSHp(pv, iv); 63 | break; */ 64 | /* Uncomment this if you need to return SVs 65 | case PERL_constant_ISSV: 66 | EXTEND(SP, 1); 67 | PUSHs(&PL_sv_undef); 68 | PUSHs(sv); 69 | break; */ 70 | /* Uncomment this if you need to return UNDEFs 71 | case PERL_constant_ISUNDEF: 72 | break; */ 73 | /* Uncomment this if you need to return UVs 74 | case PERL_constant_ISUV: 75 | EXTEND(SP, 1); 76 | PUSHs(&PL_sv_undef); 77 | PUSHu((UV)iv); 78 | break; */ 79 | /* Uncomment this if you need to return YESs 80 | case PERL_constant_ISYES: 81 | EXTEND(SP, 1); 82 | PUSHs(&PL_sv_undef); 83 | PUSHs(&PL_sv_yes); 84 | break; */ 85 | default: 86 | sv = sv_2mortal(newSVpvf( 87 | "Unexpected return type %d while processing KVSpool macro %s, used", 88 | type, s)); 89 | PUSHs(sv); 90 | } 91 | -------------------------------------------------------------------------------- /bindings/kvperl/lib/KVSpool.pm: -------------------------------------------------------------------------------- 1 | package KVSpool; 2 | 3 | use 5.010001; 4 | use strict; 5 | use warnings; 6 | use Carp; 7 | use Data::Dumper; 8 | 9 | require Exporter; 10 | 11 | our @ISA = qw(Exporter); 12 | 13 | # Items to export into callers namespace by default. Note: do not export 14 | # names by default without a very good reason. Use EXPORT_OK instead. 15 | # Do not simply export all your public functions/methods/constants. 16 | 17 | # This allows declaration use KVSpool ':all'; 18 | # If you do not need this, moving things directly into @EXPORT or @EXPORT_OK 19 | # will save memory. 20 | our %EXPORT_TAGS = ( 'all' => [ qw( 21 | ) ] ); 22 | 23 | our @EXPORT_OK = ( @{ $EXPORT_TAGS{'all'} } ); 24 | 25 | our @EXPORT = qw( 26 | ); 27 | 28 | our $VERSION = '0.01'; 29 | 30 | sub AUTOLOAD { 31 | # This AUTOLOAD is used to 'autoload' constants from the constant() 32 | # XS function. 33 | 34 | my $constname; 35 | our $AUTOLOAD; 36 | ($constname = $AUTOLOAD) =~ s/.*:://; 37 | croak "&KVSpool::constant not defined" if $constname eq 'constant'; 38 | my ($error, $val) = constant($constname); 39 | if ($error) { croak $error; } 40 | { 41 | no strict 'refs'; 42 | # Fixed between 5.005_53 and 5.005_61 43 | #XXX if ($] >= 5.00561) { 44 | #XXX *$AUTOLOAD = sub () { $val }; 45 | #XXX } 46 | #XXX else { 47 | *$AUTOLOAD = sub { $val }; 48 | #XXX } 49 | } 50 | goto &$AUTOLOAD; 51 | } 52 | 53 | require XSLoader; 54 | XSLoader::load('KVSpool', $VERSION); 55 | 56 | # Preloaded methods go here. 57 | 58 | sub new { 59 | my $class = shift; 60 | my $dir = shift; 61 | my $self = { 62 | 'rsp' => 0, 63 | 'wsp' => 0, 64 | 'set' => 0, 65 | 'dir' => $dir, 66 | 'blocking' => 1, 67 | }; 68 | bless $self,$class; 69 | return $self; 70 | } 71 | sub read { 72 | my $self = shift; 73 | $self->{'rsp'} = makersp($self->{'dir'}) if($self->{'rsp'} == 0); 74 | $self->{'set'} = makeset() if($self->{'set'} == 0); 75 | return kvread($self->{'rsp'},$self->{'set'},$self->{'blocking'}); 76 | } 77 | sub write { 78 | my $self = shift; 79 | my $hash = shift; 80 | $self->{'wsp'} = makewsp($self->{'dir'}) if($self->{'wsp'} == 0); 81 | $self->{'set'} = makeset() if($self->{'set'} == 0); 82 | return kvwrite($self->{'wsp'},$self->{'set'},$hash); 83 | } 84 | sub stat { 85 | my $self = shift; 86 | return kv_stat($self->{'dir'}); 87 | } 88 | 89 | 90 | sub DESTROY { 91 | my $self = shift; 92 | freersp($self->{'rsp'}) if($self->{'rsp'} != 0); 93 | freewsp($self->{'wsp'}) if($self->{'wsp'} != 0); 94 | freeset($self->{'set'}) if($self->{'set'} != 0) ; 95 | } 96 | # Autoload methods go after =cut, and are processed by the autosplit program. 97 | 98 | 1; 99 | __END__ 100 | # Below is stub documentation for your module. You'd better edit it! 101 | 102 | =head1 NAME 103 | 104 | KVSpool - Perl extension for kvspool library 105 | 106 | =head1 SYNOPSIS 107 | 108 | use KVSpool; 109 | my $v = KVSpool->new("spool"); 110 | my $h = {'day' => 'Wednesday', 'user' => 'Troy'}; 111 | $v->write($h); 112 | # or for a spool reader: 113 | $h = $v->read(); 114 | 115 | =head1 DESCRIPTION 116 | 117 | This is the Perl binding for the kvspool library. 118 | 119 | =head2 EXPORT 120 | 121 | None by default. 122 | 123 | =head1 SEE ALSO 124 | 125 | See http://troydhanson.github.io 126 | 127 | =head1 AUTHOR 128 | 129 | Trevor Adams 130 | Troy D. Hanson 131 | 132 | =head1 COPYRIGHT AND LICENSE 133 | 134 | Copyright (c) 2011 The Johns Hopkins University/Applied Physics Laboratory 135 | 136 | This software was developed at The Johns Hopkins University/Applied Physics 137 | Laboratory ("JHU/APL") that is the author thereof under the "work made for 138 | hire" provisions of the copyright law. Permission is hereby granted, free of 139 | charge, to any person obtaining a copy of this software and associated 140 | documentation (the "Software"), to use the Software without restriction, 141 | including without limitation the rights to copy, modify, merge, publish, 142 | distribute, sublicense, and/or sell copies of the Software, and to permit 143 | others to do so, subject to the following conditions: 144 | 145 | 1. This LICENSE AND DISCLAIMER, including the copyright notice, shall be 146 | included in all copies of the Software, including copies of substantial 147 | portions of the Software; 148 | 149 | 2. JHU/APL assumes no obligation to provide support of any kind with regard 150 | to the Software. This includes no obligation to provide assistance in using 151 | the Software nor to provide updated versions of the Software; and 152 | 153 | 3. THE SOFTWARE AND ITS DOCUMENTATION ARE PROVIDED AS IS AND WITHOUT ANY 154 | EXPRESS OR IMPLIED WARRANTIES WHATSOEVER. ALL WARRANTIES INCLUDING, BUT NOT 155 | LIMITED TO, PERFORMANCE, MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, 156 | AND NONINFRINGEMENT ARE HEREBY DISCLAIMED. USERS ASSUME THE ENTIRE RISK AND 157 | LIABILITY OF USING THE SOFTWARE. USERS ARE ADVISED TO TEST THE SOFTWARE 158 | THOROUGHLY BEFORE RELYING ON IT. IN NO EVENT SHALL THE JOHNS HOPKINS 159 | UNIVERSITY BE LIABLE FOR ANY DAMAGES WHATSOEVER, INCLUDING, WITHOUT 160 | LIMITATION, ANY LOST PROFITS, LOST SAVINGS OR OTHER INCIDENTAL OR 161 | CONSEQUENTIAL DAMAGES, ARISING OUT OF THE USE OR INABILITY TO USE THE 162 | SOFTWARE. 163 | 164 | =cut 165 | -------------------------------------------------------------------------------- /bindings/kvperl/t/TestKVSpool.t: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | use strict; 3 | use warnings; 4 | 5 | `mkdir spool` unless -d "spool"; 6 | 7 | use KVSpool; 8 | my $kv = KVSpool->new("spool"); 9 | 10 | my $h = {'day' => 'Wednesday', 'user' => 'Troy'}; 11 | $kv->write($h); 12 | 13 | my $d = $kv->read(); 14 | print "$_: $d->{$_}\n" for keys %$d; 15 | 16 | # test non blocking 17 | $kv->{blocking}=0; 18 | $d = $kv->read(); 19 | if (not defined $d) { print "non blocking read: no data\n"; } 20 | else { print "$_: $d->{$_}\n" for keys %$d; } 21 | 22 | -------------------------------------------------------------------------------- /bindings/kvpy/.gitignore: -------------------------------------------------------------------------------- 1 | build 2 | -------------------------------------------------------------------------------- /bindings/kvpy/Makefile: -------------------------------------------------------------------------------- 1 | all: kvspool 2 | 3 | kvspool: kvspool.c ../src/libkvspool.a 4 | rm -rf build 5 | python build.py build 6 | 7 | install: 8 | @echo "Installing (typically in /usr/local/lib/python2.6/dist-packages/)" 9 | python build.py install 10 | 11 | .PHONY: clean distclean 12 | clean: 13 | @python build.py $@ 14 | rm -rf build 15 | 16 | distclean: clean 17 | -------------------------------------------------------------------------------- /bindings/kvpy/build.py: -------------------------------------------------------------------------------- 1 | from distutils.core import setup, Extension 2 | 3 | module1 = Extension('kvspool', 4 | sources = ['kvspool.c'], 5 | include_dirs=['../include'], 6 | library_dirs = ['../src'], 7 | libraries = ['kvspool']) 8 | 9 | setup (name = 'kvspool', 10 | version = '1.0', 11 | description = 'Python interface to kvspool', 12 | ext_modules = [module1]) 13 | -------------------------------------------------------------------------------- /bindings/kvpy/kvspool.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "kvspool.h" 4 | 5 | /* A Python object for using kvspool (as reader or writer) 6 | * 7 | * USAGE: 8 | * 9 | * import kvspool 10 | * sp = kvspool.Kvspool("/tmp/spool") 11 | * dict = sp.read() 12 | * or 13 | * dict = {'day': 'today', 'weather':'sun'} 14 | * sp.write(dict) 15 | * 16 | * Implementation based on 17 | * http://docs.python.org/extending/newtypes.html#defining-new-types 18 | * 19 | */ 20 | 21 | /* TODO 22 | * - add tostr so that kvspool object can be printed (include r or w mode) 23 | * - add support for python sequence protocol in reader mode, c.f.: 24 | * http://docs.python.org/extending/newtypes.html#abstract-protocol-support 25 | * 26 | * 27 | */ 28 | 29 | 30 | typedef struct { 31 | PyObject_HEAD 32 | PyObject *dir; 33 | void *set; 34 | void *spr; 35 | void *spw; 36 | int blocking; 37 | } Kvspool; 38 | 39 | static void 40 | Kvspool_dealloc(Kvspool *self) 41 | { 42 | Py_XDECREF(self->dir); 43 | if (self->spr) kv_spoolreader_free(self->spr); 44 | if (self->spw) kv_spoolwriter_free(self->spw); 45 | if (self->set) kv_set_free(self->set); 46 | self->ob_type->tp_free((PyObject*)self); 47 | } 48 | 49 | /* the 'new' function is intended to create the space; later the 'init' function 50 | * fills in the new object. it's more complicated than this when pickling etc */ 51 | static PyObject * 52 | Kvspool_new(PyTypeObject *type, PyObject *args, PyObject *kwds) 53 | { 54 | Kvspool *self; 55 | self = (Kvspool*)type->tp_alloc(type,0); 56 | if (!self) goto done; 57 | 58 | self->dir = PyString_FromString(""); 59 | if (self->dir == NULL) { Py_DECREF(self); return NULL; } 60 | self->set = NULL; 61 | self->spr = NULL; 62 | self->spw = NULL; 63 | 64 | done: 65 | return (PyObject *)self; 66 | } 67 | 68 | static int 69 | Kvspool_init(Kvspool *self, PyObject *args, PyObject *kwds) 70 | { 71 | PyObject *dir = NULL, *tmp; 72 | static char *kwlist[] = {"dir", NULL}; 73 | 74 | if (! PyArg_ParseTupleAndKeywords(args, kwds, "S", kwlist, &dir)) return -1; 75 | 76 | if (dir) { 77 | tmp = self->dir; // the justification for this way of setting dir is to 78 | Py_INCREF(dir); // handle cases of subclasses with certain kinds of 79 | self->dir = dir; // deallocators - it's from the Python extending docs. 80 | Py_XDECREF(tmp); // Whew. 81 | } 82 | self->set = kv_set_new(); 83 | self->blocking = 1; 84 | return 0; 85 | } 86 | 87 | /* here's how we expose certain instance variables to python (e.g. kv.dir) */ 88 | static PyMemberDef Kvspool_members[] = { 89 | {"dir", T_OBJECT_EX, offsetof(Kvspool, dir), READONLY, "spool directory"}, 90 | {"blocking", T_INT, offsetof(Kvspool, blocking), 0, "mode"}, 91 | {NULL} 92 | }; 93 | 94 | /* add a kv.name() method. right now the name is the spool directory. 95 | * I'm not sure what circumstances name() is used or if its just an 96 | * example from the Python extension docs; it seems to me that tostr 97 | * is a more useful thing to implement since that supports 'print kv' */ 98 | static PyObject * 99 | Kvspool_name(Kvspool *self) 100 | { 101 | static PyObject *format; 102 | PyObject *args, *result; 103 | 104 | format = PyString_FromString("%s"); 105 | if (format == NULL) return NULL; 106 | 107 | if (self->dir == NULL) { 108 | PyErr_SetString(PyExc_AttributeError, "dir"); 109 | return NULL; 110 | } 111 | 112 | args = Py_BuildValue("O", self->dir); 113 | if (args == NULL) return NULL; 114 | 115 | result = PyString_Format(format, args); 116 | Py_DECREF(args); 117 | 118 | return result; 119 | } 120 | 121 | static int dictionary_to_kvs(PyObject *dict, void *set) { 122 | PyObject *pk, *pv; 123 | ssize_t pos = 0; /* Python 2.4 lacked Py_ssize_t */ 124 | int rc = -1; 125 | char *k, *v; 126 | 127 | kv_set_clear(set); 128 | 129 | while (PyDict_Next(dict, &pos, &pk, &pv)) { 130 | k = PyString_AsString(pk); 131 | v = PyString_AsString(pv); 132 | if (!k || !v) goto done; 133 | kv_adds(set, k, v); 134 | } 135 | rc = 0; 136 | done: 137 | return rc; 138 | } 139 | 140 | static int kvs_to_dictionary(PyObject **_dict, void *set) { 141 | PyObject *pk, *pv; 142 | int rc = -1; 143 | 144 | PyObject *dict = PyDict_New(); 145 | 146 | kv_t *kv=NULL; 147 | while ( (kv= kv_next(set,kv))) { 148 | 149 | pk = Py_BuildValue("s#",kv->key,(int)kv->klen); 150 | pv = Py_BuildValue("s#",kv->val,(int)kv->vlen); 151 | rc = PyDict_SetItem(dict,pk,pv); 152 | Py_DECREF(pk); Py_DECREF(pv); 153 | if (rc == -1) break; 154 | } 155 | 156 | if (rc == -1) { 157 | Py_DECREF(dict); 158 | dict = NULL; 159 | } 160 | *_dict = dict; 161 | return rc; 162 | } 163 | 164 | PyDoc_STRVAR(Kvspool_read__doc__, "read() -> dictionary"); 165 | PyDoc_STRVAR(Kvspool_write__doc__, "write(dictionary)"); 166 | 167 | static PyObject * 168 | Kvspool_read(Kvspool *self, PyObject *args) 169 | { 170 | PyObject *dict = NULL; 171 | char *dir; 172 | int rc; 173 | 174 | 175 | /* first time reading? open spool reader, save the handle. */ 176 | if (self->spr == NULL) { 177 | dir = PyString_AsString(self->dir); 178 | if ( (self->spr = kv_spoolreader_new(dir)) == NULL) { 179 | PyErr_SetString(PyExc_RuntimeError, "can't init spool reader"); 180 | return NULL; 181 | } 182 | } 183 | 184 | /* try to read a spool frame. */ 185 | if ( (rc = kv_spool_read(self->spr, self->set, self->blocking)) > 0) { 186 | kvs_to_dictionary(&dict, self->set); 187 | } else if (rc == 0) { 188 | /* only happens in non-blocking mode when no data is available */ 189 | dict = Py_BuildValue(""); // builds "None" in Python 190 | } else if (rc < 0) { 191 | PyErr_SetString(PyExc_RuntimeError, "internal error"); 192 | } 193 | 194 | return dict; 195 | } 196 | 197 | static PyObject * 198 | Kvspool_write(Kvspool *self, PyObject *args) 199 | { 200 | PyObject *dict; 201 | char *dir; 202 | 203 | if (!PyArg_ParseTuple(args, "O:write", &dict)) return NULL; 204 | 205 | /* first time writing? open spool writer, save the handle. */ 206 | if (self->spw == NULL) { 207 | dir = PyString_AsString(self->dir); 208 | if ( (self->spw = kv_spoolwriter_new(dir)) == NULL) { 209 | PyErr_SetString(PyExc_RuntimeError, "can't init spool writer"); 210 | return NULL; 211 | } 212 | } 213 | 214 | if (dictionary_to_kvs(dict, self->set) == -1) { 215 | PyErr_SetString(PyExc_RuntimeError, "non-string key or value"); 216 | return NULL; 217 | } 218 | 219 | if (kv_spool_write(self->spw, self->set) != 0) { 220 | PyErr_SetString(PyExc_RuntimeError, "write error"); 221 | return NULL; 222 | } 223 | return Py_BuildValue(""); /* builds None (because NULL would throw error) */ 224 | } 225 | 226 | static PyMethodDef Kvspool_methods[] = { 227 | {"read", (PyCFunction)Kvspool_read, METH_NOARGS, Kvspool_read__doc__}, 228 | {"write", (PyCFunction)Kvspool_write, METH_VARARGS, Kvspool_write__doc__}, 229 | {"name", (PyCFunction)Kvspool_name, METH_NOARGS, "get spool directory"}, 230 | {NULL, NULL} 231 | }; 232 | 233 | /* this kahuna defines a new python type; the type of our Kvspool object */ 234 | static PyTypeObject KvspoolType = { 235 | PyObject_HEAD_INIT(NULL) 236 | 0, /* ob_size */ 237 | "kvspool.Kvspool", /* tp_name */ 238 | sizeof(Kvspool), /* tp_basicsize */ 239 | 0, /* tp_itemsize */ 240 | (destructor)Kvspool_dealloc, /* tp_dealloc */ 241 | 0, /* tp_print */ 242 | 0, /* tp_getattr */ 243 | 0, /* tp_setattr */ 244 | 0, /* tp_compare */ 245 | 0, /* tp_repr */ 246 | 0, /* tp_as_number */ 247 | 0, /* tp_as_sequence */ 248 | 0, /* tp_as_mapping */ 249 | 0, /* tp_hash */ 250 | 0, /* tp_call */ 251 | 0, /* tp_str */ 252 | 0, /* tp_getattro */ 253 | 0, /* tp_setattro */ 254 | 0, /* tp_as_buffer */ 255 | Py_TPFLAGS_DEFAULT, /* tp_flags */ 256 | "Kvspool object", /* tp_doc */ 257 | 0, /* tp_traverse */ 258 | 0, /* tp_clear */ 259 | 0, /* tp_richcompare */ 260 | 0, /* tp_weaklistoffset */ 261 | 0, /* tp_iter */ 262 | 0, /* tp_iternext */ 263 | Kvspool_methods, /* to_methods */ 264 | Kvspool_members, /* to_members */ 265 | 0, /* tp_getset */ 266 | 0, /* tp_base */ 267 | 0, /* tp_dict */ 268 | 0, /* tp_descr_get */ 269 | 0, /* tp_descr_set */ 270 | 0, /* tp_dictoffset */ 271 | (initproc)Kvspool_init, /* tp_init */ 272 | 0, /* tp_alloc */ 273 | Kvspool_new, /* tp_new */ 274 | }; 275 | 276 | #ifndef PyMODINIT_FUNC /* declarations for DLL import/export */ 277 | #define PyMODINIT_FUNC void 278 | #endif 279 | PyMODINIT_FUNC 280 | initkvspool(void) 281 | { 282 | PyObject *m; 283 | KvspoolType.tp_new = PyType_GenericNew; 284 | if (PyType_Ready(&KvspoolType) < 0) return; 285 | m = Py_InitModule3("kvspool", NULL, "Kvspool module"); 286 | Py_INCREF(&KvspoolType); 287 | PyModule_AddObject(m, "Kvspool", (PyObject *)&KvspoolType); 288 | } 289 | 290 | -------------------------------------------------------------------------------- /bindings/kvpy/tests/README: -------------------------------------------------------------------------------- 1 | These scripts exercise the kvpy module which reads key-value spools within Python. 2 | 3 | read.py : tries to read one frame from the spool subdirectory 4 | read_all.py: tries to read all frames from the spool subdirectory 5 | speed.py : basic read/write speed test 6 | runtest.sh : invokes another utility to write a frame to the spool then runs read.py 7 | -------------------------------------------------------------------------------- /bindings/kvpy/tests/read.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | # so we don't have to install kvpy.so (or we could 4 | # have set PYTHONPATH to include its dir beforehand) 5 | import sys 6 | sys.path.append("../build/lib.linux-i686-2.6") 7 | 8 | import kvspool 9 | kv = kvspool.Kvspool("spool") 10 | d = kv.read() 11 | for key in d.keys(): 12 | print "key: " + key + " value: " + d[key] 13 | -------------------------------------------------------------------------------- /bindings/kvpy/tests/read_all.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | # so we don't have to install kvpy.so (or we could 4 | # have set PYTHONPATH to include its dir beforehand) 5 | import sys 6 | sys.path.append("../build/lib.linux-i686-2.6") 7 | 8 | import kvspool 9 | kv = kvspool.Kvspool("spool") 10 | kv.blocking = 0 11 | while True: 12 | d = kv.read() 13 | if (d == None): 14 | break 15 | for key in d.keys(): 16 | print "key: " + key + " value: " + d[key] 17 | -------------------------------------------------------------------------------- /bindings/kvpy/tests/runtest.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | SPW=../../utils/kvsp-spw 4 | export PYTHONPATH="../build/lib.linux-i686-2.6" 5 | 6 | # sanity checks 7 | if [ ! -d spool ]; then mkdir spool; fi 8 | if [ ! -x ${SPW} ]; then echo "script requires ${SPW}"; exit -1; fi 9 | 10 | # write out a frame and read it from python 11 | ${SPW} spool 12 | ./read.py 13 | -------------------------------------------------------------------------------- /bindings/kvpy/tests/speed.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | # so we don't have to install kvpy.so (or we could 4 | # have set PYTHONPATH to include its dir beforehand) 5 | import sys 6 | sys.path.append("../build/lib.linux-i686-2.6") 7 | 8 | from datetime import datetime; 9 | 10 | import kvspool 11 | kv = kvspool.Kvspool("/tmp/spool"); 12 | 13 | d = {"key":"value","key2":"value2"} 14 | 15 | # write test 16 | t1 = datetime.now() 17 | for i in range(100000): 18 | kv.write(d); 19 | t2 = datetime.now() 20 | t3 = t2 - t1 21 | print "write:", int(100 / (t3.seconds + (t3.microseconds/1000000.0))), "kfps" 22 | 23 | # read test 24 | t1 = datetime.now() 25 | for i in range(100000): 26 | d = kv.read() 27 | t2 = datetime.now() 28 | t3 = t2 - t1 29 | print "read: ", int(100 / (t3.seconds + (t3.microseconds/1000000.0))), "kfps" 30 | 31 | -------------------------------------------------------------------------------- /bindings/kvpy/tests/write.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | # so we don't have to install kvpy.so (or we could 4 | # have set PYTHONPATH to include its dir beforehand) 5 | import sys 6 | sys.path.append("../build/lib.linux-i686-2.6") 7 | 8 | import kvspool 9 | kv = kvspool.Kvspool("spool") 10 | d = {"key":"value","key2":"value2"} 11 | kv.write(d) 12 | -------------------------------------------------------------------------------- /config/compare.m4: -------------------------------------------------------------------------------- 1 | # =========================================================================== 2 | # http://www.gnu.org/software/autoconf-archive/ax_compare_version.html 3 | # =========================================================================== 4 | # 5 | # SYNOPSIS 6 | # 7 | # AX_COMPARE_VERSION(VERSION_A, OP, VERSION_B, [ACTION-IF-TRUE], [ACTION-IF-FALSE]) 8 | # 9 | # DESCRIPTION 10 | # 11 | # This macro compares two version strings. Due to the various number of 12 | # minor-version numbers that can exist, and the fact that string 13 | # comparisons are not compatible with numeric comparisons, this is not 14 | # necessarily trivial to do in a autoconf script. This macro makes doing 15 | # these comparisons easy. 16 | # 17 | # The six basic comparisons are available, as well as checking equality 18 | # limited to a certain number of minor-version levels. 19 | # 20 | # The operator OP determines what type of comparison to do, and can be one 21 | # of: 22 | # 23 | # eq - equal (test A == B) 24 | # ne - not equal (test A != B) 25 | # le - less than or equal (test A <= B) 26 | # ge - greater than or equal (test A >= B) 27 | # lt - less than (test A < B) 28 | # gt - greater than (test A > B) 29 | # 30 | # Additionally, the eq and ne operator can have a number after it to limit 31 | # the test to that number of minor versions. 32 | # 33 | # eq0 - equal up to the length of the shorter version 34 | # ne0 - not equal up to the length of the shorter version 35 | # eqN - equal up to N sub-version levels 36 | # neN - not equal up to N sub-version levels 37 | # 38 | # When the condition is true, shell commands ACTION-IF-TRUE are run, 39 | # otherwise shell commands ACTION-IF-FALSE are run. The environment 40 | # variable 'ax_compare_version' is always set to either 'true' or 'false' 41 | # as well. 42 | # 43 | # Examples: 44 | # 45 | # AX_COMPARE_VERSION([3.15.7],[lt],[3.15.8]) 46 | # AX_COMPARE_VERSION([3.15],[lt],[3.15.8]) 47 | # 48 | # would both be true. 49 | # 50 | # AX_COMPARE_VERSION([3.15.7],[eq],[3.15.8]) 51 | # AX_COMPARE_VERSION([3.15],[gt],[3.15.8]) 52 | # 53 | # would both be false. 54 | # 55 | # AX_COMPARE_VERSION([3.15.7],[eq2],[3.15.8]) 56 | # 57 | # would be true because it is only comparing two minor versions. 58 | # 59 | # AX_COMPARE_VERSION([3.15.7],[eq0],[3.15]) 60 | # 61 | # would be true because it is only comparing the lesser number of minor 62 | # versions of the two values. 63 | # 64 | # Note: The characters that separate the version numbers do not matter. An 65 | # empty string is the same as version 0. OP is evaluated by autoconf, not 66 | # configure, so must be a string, not a variable. 67 | # 68 | # The author would like to acknowledge Guido Draheim whose advice about 69 | # the m4_case and m4_ifvaln functions make this macro only include the 70 | # portions necessary to perform the specific comparison specified by the 71 | # OP argument in the final configure script. 72 | # 73 | # LICENSE 74 | # 75 | # Copyright (c) 2008 Tim Toolan 76 | # 77 | # Copying and distribution of this file, with or without modification, are 78 | # permitted in any medium without royalty provided the copyright notice 79 | # and this notice are preserved. This file is offered as-is, without any 80 | # warranty. 81 | 82 | #serial 11 83 | 84 | dnl ######################################################################### 85 | AC_DEFUN([AX_COMPARE_VERSION], [ 86 | AC_REQUIRE([AC_PROG_AWK]) 87 | 88 | # Used to indicate true or false condition 89 | ax_compare_version=false 90 | 91 | # Convert the two version strings to be compared into a format that 92 | # allows a simple string comparison. The end result is that a version 93 | # string of the form 1.12.5-r617 will be converted to the form 94 | # 0001001200050617. In other words, each number is zero padded to four 95 | # digits, and non digits are removed. 96 | AS_VAR_PUSHDEF([A],[ax_compare_version_A]) 97 | A=`echo "$1" | sed -e 's/\([[0-9]]*\)/Z\1Z/g' \ 98 | -e 's/Z\([[0-9]]\)Z/Z0\1Z/g' \ 99 | -e 's/Z\([[0-9]][[0-9]]\)Z/Z0\1Z/g' \ 100 | -e 's/Z\([[0-9]][[0-9]][[0-9]]\)Z/Z0\1Z/g' \ 101 | -e 's/[[^0-9]]//g'` 102 | 103 | AS_VAR_PUSHDEF([B],[ax_compare_version_B]) 104 | B=`echo "$3" | sed -e 's/\([[0-9]]*\)/Z\1Z/g' \ 105 | -e 's/Z\([[0-9]]\)Z/Z0\1Z/g' \ 106 | -e 's/Z\([[0-9]][[0-9]]\)Z/Z0\1Z/g' \ 107 | -e 's/Z\([[0-9]][[0-9]][[0-9]]\)Z/Z0\1Z/g' \ 108 | -e 's/[[^0-9]]//g'` 109 | 110 | dnl # In the case of le, ge, lt, and gt, the strings are sorted as necessary 111 | dnl # then the first line is used to determine if the condition is true. 112 | dnl # The sed right after the echo is to remove any indented white space. 113 | m4_case(m4_tolower($2), 114 | [lt],[ 115 | ax_compare_version=`echo "x$A 116 | x$B" | sed 's/^ *//' | sort -r | sed "s/x${A}/false/;s/x${B}/true/;1q"` 117 | ], 118 | [gt],[ 119 | ax_compare_version=`echo "x$A 120 | x$B" | sed 's/^ *//' | sort | sed "s/x${A}/false/;s/x${B}/true/;1q"` 121 | ], 122 | [le],[ 123 | ax_compare_version=`echo "x$A 124 | x$B" | sed 's/^ *//' | sort | sed "s/x${A}/true/;s/x${B}/false/;1q"` 125 | ], 126 | [ge],[ 127 | ax_compare_version=`echo "x$A 128 | x$B" | sed 's/^ *//' | sort -r | sed "s/x${A}/true/;s/x${B}/false/;1q"` 129 | ],[ 130 | dnl Split the operator from the subversion count if present. 131 | m4_bmatch(m4_substr($2,2), 132 | [0],[ 133 | # A count of zero means use the length of the shorter version. 134 | # Determine the number of characters in A and B. 135 | ax_compare_version_len_A=`echo "$A" | $AWK '{print(length)}'` 136 | ax_compare_version_len_B=`echo "$B" | $AWK '{print(length)}'` 137 | 138 | # Set A to no more than B's length and B to no more than A's length. 139 | A=`echo "$A" | sed "s/\(.\{$ax_compare_version_len_B\}\).*/\1/"` 140 | B=`echo "$B" | sed "s/\(.\{$ax_compare_version_len_A\}\).*/\1/"` 141 | ], 142 | [[0-9]+],[ 143 | # A count greater than zero means use only that many subversions 144 | A=`echo "$A" | sed "s/\(\([[0-9]]\{4\}\)\{m4_substr($2,2)\}\).*/\1/"` 145 | B=`echo "$B" | sed "s/\(\([[0-9]]\{4\}\)\{m4_substr($2,2)\}\).*/\1/"` 146 | ], 147 | [.+],[ 148 | AC_WARNING( 149 | [illegal OP numeric parameter: $2]) 150 | ],[]) 151 | 152 | # Pad zeros at end of numbers to make same length. 153 | ax_compare_version_tmp_A="$A`echo $B | sed 's/./0/g'`" 154 | B="$B`echo $A | sed 's/./0/g'`" 155 | A="$ax_compare_version_tmp_A" 156 | 157 | # Check for equality or inequality as necessary. 158 | m4_case(m4_tolower(m4_substr($2,0,2)), 159 | [eq],[ 160 | test "x$A" = "x$B" && ax_compare_version=true 161 | ], 162 | [ne],[ 163 | test "x$A" != "x$B" && ax_compare_version=true 164 | ],[ 165 | AC_WARNING([illegal OP parameter: $2]) 166 | ]) 167 | ]) 168 | 169 | AS_VAR_POPDEF([A])dnl 170 | AS_VAR_POPDEF([B])dnl 171 | 172 | dnl # Execute ACTION-IF-TRUE / ACTION-IF-FALSE. 173 | if test "$ax_compare_version" = "true" ; then 174 | m4_ifvaln([$4],[$4],[:])dnl 175 | m4_ifvaln([$5],[else $5])dnl 176 | fi 177 | ]) dnl AX_COMPARE_VERSION 178 | -------------------------------------------------------------------------------- /config/perl.m4: -------------------------------------------------------------------------------- 1 | # =========================================================================== 2 | # http://www.gnu.org/software/autoconf-archive/ax_prog_perl_version.html 3 | # =========================================================================== 4 | # 5 | # SYNOPSIS 6 | # 7 | # AX_PROG_PERL_VERSION([VERSION],[ACTION-IF-TRUE],[ACTION-IF-FALSE]) 8 | # 9 | # DESCRIPTION 10 | # 11 | # Makes sure that perl supports the version indicated. If true the shell 12 | # commands in ACTION-IF-TRUE are executed. If not the shell commands in 13 | # ACTION-IF-FALSE are run. Note if $PERL is not set (for example by 14 | # running AC_CHECK_PROG or AC_PATH_PROG) the macro will fail. 15 | # 16 | # Example: 17 | # 18 | # AC_PATH_PROG([PERL],[perl]) 19 | # AX_PROG_PERL_VERSION([5.8.0],[ ... ],[ ... ]) 20 | # 21 | # This will check to make sure that the perl you have supports at least 22 | # version 5.8.0. 23 | # 24 | # NOTE: This macro uses the $PERL variable to perform the check. 25 | # AX_WITH_PERL can be used to set that variable prior to running this 26 | # macro. The $PERL_VERSION variable will be valorized with the detected 27 | # version. 28 | # 29 | # LICENSE 30 | # 31 | # Copyright (c) 2009 Francesco Salvestrini 32 | # 33 | # Copying and distribution of this file, with or without modification, are 34 | # permitted in any medium without royalty provided the copyright notice 35 | # and this notice are preserved. This file is offered as-is, without any 36 | # warranty. 37 | 38 | #serial 11 39 | 40 | AC_DEFUN([AX_PROG_PERL_VERSION],[ 41 | AC_REQUIRE([AC_PROG_SED]) 42 | AC_REQUIRE([AC_PROG_GREP]) 43 | 44 | AS_IF([test -n "$PERL"],[ 45 | ax_perl_version="$1" 46 | 47 | AC_MSG_CHECKING([for perl version]) 48 | changequote(<<,>>) 49 | perl_version=`$PERL --version 2>&1 | $GREP "This is perl" | $SED -e 's/.* v\([0-9]*\.[0-9]*\.[0-9]*\) .*/\1/'` 50 | changequote([,]) 51 | AC_MSG_RESULT($perl_version) 52 | 53 | AC_SUBST([PERL_VERSION],[$perl_version]) 54 | 55 | AX_COMPARE_VERSION([$ax_perl_version],[le],[$perl_version],[ 56 | : 57 | $2 58 | ],[ 59 | : 60 | $3 61 | ]) 62 | ],[ 63 | AC_MSG_WARN([could not find the perl interpreter]) 64 | $3 65 | ]) 66 | ]) 67 | -------------------------------------------------------------------------------- /config/python.m4: -------------------------------------------------------------------------------- 1 | # =========================================================================== 2 | # http://www.gnu.org/software/autoconf-archive/ax_python_devel.html 3 | # =========================================================================== 4 | # 5 | # SYNOPSIS 6 | # 7 | # AX_PYTHON_DEVEL([version]) 8 | # 9 | # DESCRIPTION 10 | # 11 | # Note: Defines as a precious variable "PYTHON_VERSION". Don't override it 12 | # in your configure.ac. 13 | # 14 | # This macro checks for Python and tries to get the include path to 15 | # 'Python.h'. It provides the $(PYTHON_CPPFLAGS) and $(PYTHON_LDFLAGS) 16 | # output variables. It also exports $(PYTHON_EXTRA_LIBS) and 17 | # $(PYTHON_EXTRA_LDFLAGS) for embedding Python in your code. 18 | # 19 | # You can search for some particular version of Python by passing a 20 | # parameter to this macro, for example ">= '2.3.1'", or "== '2.4'". Please 21 | # note that you *have* to pass also an operator along with the version to 22 | # match, and pay special attention to the single quotes surrounding the 23 | # version number. Don't use "PYTHON_VERSION" for this: that environment 24 | # variable is declared as precious and thus reserved for the end-user. 25 | # 26 | # This macro should work for all versions of Python >= 2.1.0. As an end 27 | # user, you can disable the check for the python version by setting the 28 | # PYTHON_NOVERSIONCHECK environment variable to something else than the 29 | # empty string. 30 | # 31 | # If you need to use this macro for an older Python version, please 32 | # contact the authors. We're always open for feedback. 33 | # 34 | # LICENSE 35 | # 36 | # Copyright (c) 2009 Sebastian Huber 37 | # Copyright (c) 2009 Alan W. Irwin 38 | # Copyright (c) 2009 Rafael Laboissiere 39 | # Copyright (c) 2009 Andrew Collier 40 | # Copyright (c) 2009 Matteo Settenvini 41 | # Copyright (c) 2009 Horst Knorr 42 | # 43 | # This program is free software: you can redistribute it and/or modify it 44 | # under the terms of the GNU General Public License as published by the 45 | # Free Software Foundation, either version 3 of the License, or (at your 46 | # option) any later version. 47 | # 48 | # This program is distributed in the hope that it will be useful, but 49 | # WITHOUT ANY WARRANTY; without even the implied warranty of 50 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 51 | # Public License for more details. 52 | # 53 | # You should have received a copy of the GNU General Public License along 54 | # with this program. If not, see . 55 | # 56 | # As a special exception, the respective Autoconf Macro's copyright owner 57 | # gives unlimited permission to copy, distribute and modify the configure 58 | # scripts that are the output of Autoconf when processing the Macro. You 59 | # need not follow the terms of the GNU General Public License when using 60 | # or distributing such scripts, even though portions of the text of the 61 | # Macro appear in them. The GNU General Public License (GPL) does govern 62 | # all other use of the material that constitutes the Autoconf Macro. 63 | # 64 | # This special exception to the GPL applies to versions of the Autoconf 65 | # Macro released by the Autoconf Archive. When you make and distribute a 66 | # modified version of the Autoconf Macro, you may extend this special 67 | # exception to the GPL to apply to your modified version as well. 68 | 69 | #serial 8 70 | 71 | AU_ALIAS([AC_PYTHON_DEVEL], [AX_PYTHON_DEVEL]) 72 | AC_DEFUN([AX_PYTHON_DEVEL],[ 73 | # 74 | # Allow the use of a (user set) custom python version 75 | # 76 | AC_ARG_VAR([PYTHON_VERSION],[The installed Python 77 | version to use, for example '2.3'. This string 78 | will be appended to the Python interpreter 79 | canonical name.]) 80 | 81 | AC_PATH_PROG([PYTHON],[python[$PYTHON_VERSION]]) 82 | if test -z "$PYTHON"; then 83 | AC_MSG_ERROR([Cannot find python$PYTHON_VERSION in your system path]) 84 | PYTHON_VERSION="" 85 | fi 86 | 87 | # 88 | # Check for a version of Python >= 2.1.0 89 | # 90 | AC_MSG_CHECKING([for a version of Python >= '2.1.0']) 91 | ac_supports_python_ver=`$PYTHON -c "import sys; \ 92 | ver = sys.version.split ()[[0]]; \ 93 | print (ver >= '2.1.0')"` 94 | if test "$ac_supports_python_ver" != "True"; then 95 | if test -z "$PYTHON_NOVERSIONCHECK"; then 96 | AC_MSG_RESULT([no]) 97 | AC_MSG_WARN([ 98 | This version of the AC@&t@_PYTHON_DEVEL macro 99 | doesn't work properly with versions of Python before 100 | 2.1.0. You may need to re-run configure, setting the 101 | variables PYTHON_CPPFLAGS, PYTHON_LDFLAGS, PYTHON_SITE_PKG, 102 | PYTHON_EXTRA_LIBS and PYTHON_EXTRA_LDFLAGS by hand. 103 | Moreover, to disable this check, set PYTHON_NOVERSIONCHECK 104 | to something else than an empty string. 105 | ]) 106 | else 107 | AC_MSG_RESULT([skip at user request]) 108 | fi 109 | else 110 | AC_MSG_RESULT([yes]) 111 | fi 112 | 113 | # 114 | # if the macro parameter ``version'' is set, honour it 115 | # 116 | if test -n "$1"; then 117 | AC_MSG_CHECKING([for a version of Python $1]) 118 | ac_supports_python_ver=`$PYTHON -c "import sys; \ 119 | ver = sys.version.split ()[[0]]; \ 120 | print (ver $1)"` 121 | if test "$ac_supports_python_ver" = "True"; then 122 | AC_MSG_RESULT([yes]) 123 | else 124 | AC_MSG_RESULT([no]) 125 | AC_MSG_ERROR([this package requires Python $1. 126 | If you have it installed, but it isn't the default Python 127 | interpreter in your system path, please pass the PYTHON_VERSION 128 | variable to configure. See ``configure --help'' for reference. 129 | ]) 130 | PYTHON_VERSION="" 131 | fi 132 | fi 133 | 134 | # 135 | # Check if you have distutils, else fail 136 | # 137 | AC_MSG_CHECKING([for the distutils Python package]) 138 | ac_distutils_result=`$PYTHON -c "import distutils" 2>&1` 139 | if test -z "$ac_distutils_result"; then 140 | AC_MSG_RESULT([yes]) 141 | else 142 | AC_MSG_RESULT([no]) 143 | AC_MSG_WARN([cannot import Python module "distutils". 144 | Please check your Python installation. The error was: 145 | $ac_distutils_result]) 146 | PYTHON_VERSION="" 147 | fi 148 | 149 | # 150 | # Check for Python include path 151 | # 152 | AC_MSG_CHECKING([for Python include path]) 153 | if test -z "$PYTHON_CPPFLAGS"; then 154 | python_path=`$PYTHON -c "import distutils.sysconfig; \ 155 | print (distutils.sysconfig.get_python_inc ());"` 156 | if test -n "${python_path}"; then 157 | python_path="-I$python_path" 158 | fi 159 | PYTHON_CPPFLAGS=$python_path 160 | fi 161 | AC_MSG_RESULT([$PYTHON_CPPFLAGS]) 162 | AC_SUBST([PYTHON_CPPFLAGS]) 163 | 164 | # 165 | # Check for Python library path 166 | # 167 | AC_MSG_CHECKING([for Python library path]) 168 | if test -z "$PYTHON_LDFLAGS"; then 169 | # (makes two attempts to ensure we've got a version number 170 | # from the interpreter) 171 | ac_python_version=`cat<]], 298 | [[Py_Initialize();]]) 299 | ],[pythonexists=yes],[pythonexists=no]) 300 | AC_LANG_POP([C]) 301 | # turn back to default flags 302 | CPPFLAGS="$ac_save_CPPFLAGS" 303 | LIBS="$ac_save_LIBS" 304 | 305 | AC_MSG_RESULT([$pythonexists]) 306 | 307 | if test ! "x$pythonexists" = "xyes"; then 308 | AC_MSG_WARN([ 309 | Could not link test program to Python. Maybe the main Python library has been 310 | installed in some non-standard library path. If so, pass it to configure, 311 | via the LDFLAGS environment variable. 312 | Example: ./configure LDFLAGS="-L/usr/non-standard-path/python/lib" 313 | ============================================================================ 314 | Skipping the build of the Python kvspool module. 315 | 316 | You may need to install your platform's package equivalents for 317 | 318 | python2.6-dev or other version 319 | libssl-dev 320 | 321 | ============================================================================ 322 | ]) 323 | PYTHON_VERSION="" 324 | fi 325 | 326 | # 327 | # all done! 328 | # 329 | ]) 330 | 331 | -------------------------------------------------------------------------------- /configure.ac: -------------------------------------------------------------------------------- 1 | AC_PREREQ(2.59) 2 | 3 | AC_INIT([kvspool], [1.0], [tdh@tkhanson.net]) 4 | AC_CONFIG_SRCDIR(src/kvspool.c) 5 | AC_CONFIG_AUX_DIR(config) 6 | AC_CONFIG_HEADERS(config/config.h) 7 | AM_INIT_AUTOMAKE([foreign subdir-objects]) 8 | AC_PROG_CC 9 | AC_PROG_RANLIB 10 | 11 | have_shr_header=n 12 | have_shr_lib=n 13 | AC_CHECK_HEADERS([shr.h],[have_shr_header=y]) 14 | AC_CHECK_LIB(shr,shr_readv,[have_shr_lib=y]) 15 | if test "x${have_shr_header}${have_shr_lib}" != xyy 16 | then 17 | AC_MSG_ERROR([ 18 | ----------------------------------------------------- 19 | The libshr build prerequisite was not found. Please 20 | see the build instructions, install libshr and retry. 21 | link to libshr: https://github.com/troydhanson/shr 22 | ----------------------------------------------------- 23 | ]) 24 | fi 25 | 26 | # is Jansson installed 27 | AC_CHECK_LIB(jansson,json_string, 28 | AM_CONDITIONAL(HAVE_JANSSON,true), 29 | AM_CONDITIONAL(HAVE_JANSSON,false)) 30 | 31 | # is nanomsg installed 32 | AC_CHECK_LIB(nanomsg,nn_socket, 33 | AM_CONDITIONAL(HAVE_NANOMSG,true), 34 | AM_CONDITIONAL(HAVE_NANOMSG,false)) 35 | 36 | # is zeromq (0MQ) installed 37 | AC_CHECK_LIB(zmq,zmq_init, 38 | AM_CONDITIONAL(HAVE_ZEROMQ,true), 39 | AM_CONDITIONAL(HAVE_ZEROMQ,false)) 40 | 41 | # is libpcre3 installed 42 | AC_CHECK_LIB(pcre,pcre_exec, 43 | AM_CONDITIONAL(HAVE_PCRE,true), 44 | AM_CONDITIONAL(HAVE_PCRE,false)) 45 | 46 | # is librdkafka installed 47 | AC_CHECK_LIB(rdkafka,rd_kafka_new, 48 | AM_CONDITIONAL(HAVE_RDKAFKA,true), 49 | AM_CONDITIONAL(HAVE_RDKAFKA,false)) 50 | 51 | AC_CONFIG_FILES(Makefile src/Makefile utils/Makefile) 52 | AC_OUTPUT 53 | 54 | -------------------------------------------------------------------------------- /doc/LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2011-2016 Troy D. Hanson 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /doc/Makefile: -------------------------------------------------------------------------------- 1 | all: reader-writer.png pub-sub.png index.html 2 | DITAA=java -jar ditaa0_9.jar -o 3 | 4 | reader-writer.png: reader-writer.txt 5 | $(DITAA) $< 6 | 7 | pub-sub.png: pub-sub.txt 8 | $(DITAA) $< 9 | 10 | index.html: index.txt 11 | asciidoc -a toc2 $< 12 | 13 | TMP=/tmp/kvspool-gh-pages 14 | stage: 15 | mkdir -p ${TMP} 16 | rm -if ${TMP}/* 17 | cp LICENSE.txt ${TMP} 18 | cp *.html ${TMP} 19 | cp *.png ${TMP} 20 | 21 | .PHONY: clean 22 | 23 | clean: 24 | rm -f index.html 25 | -------------------------------------------------------------------------------- /doc/pub-sub.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/troydhanson/kvspool/dc211bbff2af3218e51fa5a925f6b3ba5730ac89/doc/pub-sub.png -------------------------------------------------------------------------------- /doc/pub-sub.txt: -------------------------------------------------------------------------------- 1 | 2 | | 3 | /----------\ | /----------\ 4 | | | +-------+ | +--------+ | | 5 | | writer |-->| spool | | | spool |-->| reader | 6 | | | +-------+ | +--------+ | | 7 | \----------/ | | ^ \----------/ 8 | v | | 9 | kvsp-pub------>kvsp-sub 10 | | 11 | COMPUTER A | COMPUTER B 12 | -------------------------------------------------------------------------------- /doc/reader-writer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/troydhanson/kvspool/dc211bbff2af3218e51fa5a925f6b3ba5730ac89/doc/reader-writer.png -------------------------------------------------------------------------------- /doc/reader-writer.txt: -------------------------------------------------------------------------------- 1 | 2 | /----------\ /----------\ 3 | | | +-------+ | | 4 | | writer |-->| spool |-->| reader | 5 | | | +-------+ | | 6 | \----------/ \----------/ 7 | -------------------------------------------------------------------------------- /include/kvspool.h: -------------------------------------------------------------------------------- 1 | #ifndef _KVSPOOL_H_ 2 | #define _KVSPOOL_H_ 3 | 4 | #include 5 | #include "uthash.h" 6 | 7 | /* kvspool: an API for dealing with a set of key-value pairs */ 8 | 9 | /****************************************************************************** 10 | * key-value set API 11 | *****************************************************************************/ 12 | typedef struct { 13 | char *key; 14 | int klen; 15 | char *val; 16 | int vlen; 17 | UT_hash_handle hh; /* internal */ 18 | } kv_t; 19 | 20 | 21 | #if defined __cplusplus 22 | extern "C" { 23 | #endif 24 | 25 | void* kv_set_new(void); 26 | void kv_set_free(void*); 27 | void kv_set_clear(void*); 28 | void kv_set_dump(void *set,FILE *out); 29 | void kv_add(void*set, const char *key, int klen, const char *val, int vlen); 30 | kv_t *kv_get(void*set, char *key); 31 | #define kv_adds(set, key, val) kv_add(set,key,strlen(key),val,strlen(val)) 32 | int kv_len(void*set); 33 | kv_t *kv_next(void*set,kv_t *kv); 34 | 35 | /****************************************************************************** 36 | * spooling API 37 | *****************************************************************************/ 38 | void *kv_spoolreader_new(const char *dir); 39 | void *kv_spoolreader_new_nb(const char *dir, int *fd); 40 | int kv_spool_read(void*sp, void *set, int blocking); 41 | int kv_spool_readN(void*sp, void **set, int *nset); 42 | void kv_spoolreader_free(void*); 43 | 44 | void *kv_spoolwriter_new(const char *dir); 45 | int kv_spool_write(void*sp, void *set); 46 | int kv_spool_writeN(void *sp, void **setv, int nset); 47 | void kv_spoolwriter_free(void*); 48 | 49 | /****************************************************************************** 50 | * special purpose API 51 | *****************************************************************************/ 52 | typedef struct { int pct_consumed; time_t last_write; off_t spool_sz;} kv_stat_t; 53 | int kv_stat(const char *dir, kv_stat_t *stats); 54 | 55 | #if defined __cplusplus 56 | } 57 | #endif 58 | 59 | #endif 60 | -------------------------------------------------------------------------------- /include/kvspool_internal.h: -------------------------------------------------------------------------------- 1 | #ifndef _KVSPOOL_INTERNAL_H_ 2 | #define _KVSPOOL_INTERNAL_H_ 3 | 4 | #include "kvspool.h" 5 | 6 | typedef struct { 7 | kv_t *kvs; 8 | } kvset_t; 9 | 10 | #endif 11 | -------------------------------------------------------------------------------- /include/tpl.h: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2005-2010, Troy D. Hanson http://tpl.sourceforge.net 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS 12 | IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 13 | TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A 14 | PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER 15 | OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 16 | EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 17 | PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 18 | PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 19 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 20 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 21 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 22 | */ 23 | 24 | #ifndef TPL_H 25 | #define TPL_H 26 | 27 | #include /* size_t */ 28 | 29 | #ifdef __INTEL_COMPILER 30 | #include 31 | #endif /* Intel Compiler efficient memcpy etc */ 32 | 33 | #ifdef _MSC_VER 34 | typedef unsigned int uint32_t; 35 | #else 36 | #include /* uint32_t */ 37 | #endif 38 | 39 | #if defined __cplusplus 40 | extern "C" { 41 | #endif 42 | 43 | #ifdef _WIN32 44 | #ifdef TPL_EXPORTS 45 | #define TPL_API __declspec(dllexport) 46 | #else /* */ 47 | #ifdef TPL_NOLIB 48 | #define TPL_API 49 | #else 50 | #define TPL_API __declspec(dllimport) 51 | #endif /* TPL_NOLIB */ 52 | #endif /* TPL_EXPORTS*/ 53 | #else 54 | #define TPL_API 55 | #endif 56 | 57 | /* bit flags (external) */ 58 | #define TPL_FILE (1 << 0) 59 | #define TPL_MEM (1 << 1) 60 | #define TPL_PREALLOCD (1 << 2) 61 | #define TPL_EXCESS_OK (1 << 3) 62 | #define TPL_FD (1 << 4) 63 | #define TPL_UFREE (1 << 5) 64 | #define TPL_DATAPEEK (1 << 6) 65 | #define TPL_FXLENS (1 << 7) 66 | #define TPL_GETSIZE (1 << 8) 67 | /* do not add flags here without renumbering the internal flags! */ 68 | 69 | /* flags for tpl_gather mode */ 70 | #define TPL_GATHER_BLOCKING 1 71 | #define TPL_GATHER_NONBLOCKING 2 72 | #define TPL_GATHER_MEM 3 73 | 74 | /* Hooks for error logging, memory allocation functions and fatal */ 75 | typedef int (tpl_print_fcn)(const char *fmt, ...); 76 | typedef void *(tpl_malloc_fcn)(size_t sz); 77 | typedef void *(tpl_realloc_fcn)(void *ptr, size_t sz); 78 | typedef void (tpl_free_fcn)(void *ptr); 79 | typedef void (tpl_fatal_fcn)(char *fmt, ...); 80 | 81 | typedef struct tpl_hook_t { 82 | tpl_print_fcn *oops; 83 | tpl_malloc_fcn *malloc; 84 | tpl_realloc_fcn *realloc; 85 | tpl_free_fcn *free; 86 | tpl_fatal_fcn *fatal; 87 | size_t gather_max; 88 | } tpl_hook_t; 89 | 90 | typedef struct tpl_node { 91 | int type; 92 | void *addr; 93 | void *data; /* r:tpl_root_data*. A:tpl_atyp*. ow:szof type */ 94 | int num; /* length of type if its a C array */ 95 | size_t ser_osz; /* serialization output size for subtree */ 96 | struct tpl_node *children; /* my children; linked-list */ 97 | struct tpl_node *next,*prev; /* my siblings (next child of my parent) */ 98 | struct tpl_node *parent; /* my parent */ 99 | } tpl_node; 100 | 101 | /* used when un/packing 'B' type (binary buffers) */ 102 | typedef struct tpl_bin { 103 | void *addr; 104 | uint32_t sz; 105 | } tpl_bin; 106 | 107 | /* for async/piecemeal reading of tpl images */ 108 | typedef struct tpl_gather_t { 109 | char *img; 110 | int len; 111 | } tpl_gather_t; 112 | 113 | /* Callback used when tpl_gather has read a full tpl image */ 114 | typedef int (tpl_gather_cb)(void *img, size_t sz, void *data); 115 | 116 | /* Prototypes */ 117 | TPL_API tpl_node *tpl_map(char *fmt,...); /* define tpl using format */ 118 | TPL_API void tpl_free(tpl_node *r); /* free a tpl map */ 119 | TPL_API int tpl_pack(tpl_node *r, int i); /* pack the n'th packable */ 120 | TPL_API int tpl_unpack(tpl_node *r, int i); /* unpack the n'th packable */ 121 | TPL_API int tpl_dump(tpl_node *r, int mode, ...); /* serialize to mem/file */ 122 | TPL_API int tpl_load(tpl_node *r, int mode, ...); /* set mem/file to unpack */ 123 | TPL_API int tpl_Alen(tpl_node *r, int i); /* array len of packable i */ 124 | TPL_API char* tpl_peek(int mode, ...); /* sneak peek at format string */ 125 | TPL_API int tpl_gather( int mode, ...); /* non-blocking image gather */ 126 | TPL_API int tpl_jot(int mode, ...); /* quick write a simple tpl */ 127 | 128 | #if defined __cplusplus 129 | } 130 | #endif 131 | 132 | #endif /* TPL_H */ 133 | 134 | -------------------------------------------------------------------------------- /src/Makefile.am: -------------------------------------------------------------------------------- 1 | srcdir = @srcdir@ 2 | 3 | AM_CFLAGS = -fPIC -I$(srcdir)/../include 4 | lib_LIBRARIES = libkvspool.a 5 | libkvspool_a_SOURCES = kvspool.c kvspoolw.c kvspoolr.c tpl.c 6 | include_HEADERS = ../include/kvspool.h ../include/uthash.h 7 | 8 | -------------------------------------------------------------------------------- /src/kvspool.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include "kvspool.h" 5 | #include "kvspool_internal.h" 6 | 7 | void sp_oom(void) { 8 | fprintf(stderr, "out of memory\n"); 9 | exit(-1); 10 | } 11 | 12 | void* kv_set_new(void) { 13 | kvset_t *set; 14 | if ( (set = malloc(sizeof(kvset_t))) == NULL) sp_oom(); 15 | memset(set,0,sizeof(*set)); 16 | return set; 17 | } 18 | 19 | void kv_set_clear(void*_set) { 20 | kvset_t *set = (kvset_t*)_set; 21 | kv_t *kv, *tmp; 22 | HASH_ITER(hh, set->kvs, kv, tmp) { 23 | HASH_DEL(set->kvs, kv); 24 | free(kv->key); free(kv->val); free(kv); 25 | } 26 | } 27 | 28 | void kv_set_dump(void*_set,FILE *out) { 29 | kvset_t *set = (kvset_t*)_set; 30 | kv_t *kv; 31 | char c; 32 | int i; 33 | 34 | kv=NULL; 35 | while ( (kv=kv_next(set,kv))) { 36 | fprintf(out," %.*s: ", kv->klen, kv->key); 37 | fprintf(out, "%.*s\n", kv->vlen, kv->val); 38 | } 39 | } 40 | 41 | void kv_set_free(void*_set) { 42 | kvset_t *set = (kvset_t*)_set; 43 | kv_t *kv, *tmp; 44 | HASH_ITER(hh, set->kvs, kv, tmp) { 45 | HASH_DEL(set->kvs, kv); 46 | free(kv->key); free(kv->val); 47 | free(kv); 48 | } 49 | assert(set->kvs == NULL); 50 | free(set); 51 | } 52 | 53 | kv_t *kv_get(void*_set, char *key) { 54 | kv_t *kv; 55 | kvset_t *set = (kvset_t*)_set; 56 | HASH_FIND(hh, set->kvs, key, strlen(key), kv); 57 | return kv; 58 | } 59 | 60 | void kv_add(void*_set, const char *key, int klen, const char *val, int vlen) { 61 | kvset_t *set = (kvset_t*)_set; 62 | assert(klen); //assert(vlen); 63 | kv_t *kv; 64 | 65 | /* check if we're replacing an existing key */ 66 | HASH_FIND(hh, set->kvs, key, klen, kv); 67 | if (kv) { /* yes, free the old value and replace it */ 68 | free(kv->val); 69 | if ( (kv->val = malloc(vlen+1)) == NULL) sp_oom(); kv->vlen = vlen; 70 | memcpy(kv->val, val, vlen); kv->val[vlen]='\0'; 71 | return; 72 | } 73 | /* new key. deep copy the key/val and add it, null term for convenience */ 74 | if ( (kv = malloc(sizeof(*kv))) == NULL) sp_oom(); 75 | if ( (kv->key = malloc(klen+1)) == NULL) sp_oom(); kv->klen = klen; 76 | if ( (kv->val = malloc(vlen+1)) == NULL) sp_oom(); kv->vlen = vlen; 77 | memcpy(kv->key, key, klen); kv->key[klen]='\0'; 78 | memcpy(kv->val, val, vlen); kv->val[vlen]='\0'; 79 | HASH_ADD_KEYPTR(hh,set->kvs,kv->key,kv->klen,kv); 80 | } 81 | 82 | int kv_len(void*_set) { 83 | kvset_t *set = (kvset_t*)_set; 84 | return set->kvs ? (HASH_COUNT(set->kvs)) : 0; 85 | } 86 | 87 | kv_t *kv_next(void*_set,kv_t *kv) { 88 | kvset_t *set = (kvset_t*)_set; 89 | if (!kv) return set->kvs; /* get first element */ 90 | return kv->hh.next; 91 | } 92 | 93 | -------------------------------------------------------------------------------- /src/kvspoolr.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include "kvspool.h" 12 | #include "kvspool_internal.h" 13 | #include "utstring.h" 14 | #include "tpl.h" 15 | #include "shr.h" 16 | 17 | static void fill_set(void *img, size_t sz, kvset_t *set) { 18 | tpl_node *tn; 19 | char *key; 20 | char *val; 21 | 22 | kv_set_clear(set); 23 | 24 | tn = tpl_map("A(ss)", &key, &val); 25 | if (tpl_load(tn, TPL_MEM, img, sz) == -1) { 26 | fprintf(stderr, "tpl_load failed (sz %d)\n", (int)sz); 27 | return; 28 | } 29 | while( tpl_unpack(tn,1) > 0 ) { 30 | kv_adds(set, key, val); 31 | free(key); 32 | free(val); 33 | } 34 | tpl_free(tn); 35 | } 36 | 37 | /******************************************************************************* 38 | * Spool reader API 39 | ******************************************************************************/ 40 | void *kv_spoolreader_new(const char *dir) { 41 | char path[PATH_MAX]; 42 | struct shr *shr; 43 | snprintf(path, PATH_MAX, "%s/%s", dir, "data"); 44 | shr = shr_open(path, SHR_RDONLY); 45 | return shr; 46 | } 47 | 48 | void *kv_spoolreader_new_nb(const char *dir, int *fd) { 49 | char path[PATH_MAX]; 50 | struct shr *shr=NULL; 51 | int rc = -1; 52 | snprintf(path, PATH_MAX, "%s/%s", dir, "data"); 53 | shr = shr_open(path, SHR_RDONLY|SHR_NONBLOCK); 54 | if (shr == NULL) goto done; 55 | 56 | if (fd) *fd = shr_get_selectable_fd(shr); 57 | 58 | rc = 0; 59 | 60 | done: 61 | return shr; 62 | } 63 | 64 | /* returns 1 if frame ready, 0 = no data (nonblocking), or -1 on error 65 | * whether or not its a blocking or non-blocking read depends on the 66 | * way it was opened (kv_spoolreader_new or with _nb suffix) 67 | */ 68 | int kv_spool_read(void *_sp, void *_set, int obsolete_blocking_flag) { 69 | struct shr *shr = (struct shr *)_sp; 70 | kvset_t *set = (kvset_t*)_set; 71 | char buf[4096]; 72 | ssize_t sc; 73 | 74 | sc = shr_read(shr, buf, sizeof(buf)); 75 | if (sc > 0) { 76 | fill_set(buf, sc, set); 77 | return 1; 78 | } 79 | return sc; /* negative (error) or 0 (no data) case */ 80 | } 81 | 82 | int kv_spool_readN(void *_sp, void **_setv, int *nset) { 83 | struct shr *shr = (struct shr *)_sp; 84 | kvset_t **setv = (kvset_t**)_setv; 85 | ssize_t sc; 86 | char *tmp=NULL; 87 | 88 | int i; 89 | size_t iovcnt = *nset; 90 | struct iovec iov[iovcnt]; 91 | *nset = 0; 92 | 93 | int tmpsz = 10*1024*1024; 94 | tmp = malloc(tmpsz); 95 | if (tmp == NULL) { 96 | fprintf(stderr, "out of memory\n"); 97 | goto done; 98 | } 99 | 100 | sc = shr_readv(shr, tmp, tmpsz, iov, &iovcnt); 101 | if (sc <= 0) goto done; 102 | 103 | for(i=0; i < iovcnt; i++) { 104 | fill_set(iov[i].iov_base, iov[i].iov_len, setv[i]); 105 | } 106 | *nset = iovcnt; 107 | 108 | done: 109 | if (tmp) free(tmp); 110 | return sc; 111 | } 112 | 113 | void kv_spoolreader_free(void *_sp) { 114 | struct shr *shr = (struct shr *)_sp; 115 | shr_close(shr); 116 | } 117 | 118 | /* get the percentage consumed for dir 119 | returns -1 on error */ 120 | int kv_stat(const char *dir, kv_stat_t *stats) { 121 | struct shr *shr=NULL; 122 | char path[PATH_MAX]; 123 | struct shr_stat s; 124 | int rc = -1; 125 | 126 | struct stat st; 127 | snprintf(path, PATH_MAX, "%s/%s", dir, "data"); 128 | if (stat(path, &st) < 0) { 129 | fprintf(stderr, "stat: %s\n", strerror(errno)); 130 | goto done; 131 | } 132 | 133 | shr = shr_open(path, SHR_RDONLY); 134 | if (shr == NULL) goto done; 135 | if (shr_stat(shr, &s, NULL) < 0) goto done; 136 | 137 | stats->pct_consumed = 100 - ((100.0 * s.bu) / s.bn); 138 | /* last_write is actually last_mtime (read OR write) */ 139 | stats->last_write = st.st_mtim.tv_sec; 140 | stats->spool_sz = s.bn; 141 | 142 | rc = 0; 143 | 144 | done: 145 | if (shr) shr_close(shr); 146 | return rc; 147 | } 148 | -------------------------------------------------------------------------------- /src/kvspoolw.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include "utstring.h" 12 | #include "utarray.h" 13 | #include "tpl.h" 14 | #include "shr.h" 15 | 16 | #include "kvspool.h" 17 | #include "kvspool_internal.h" 18 | 19 | 20 | /******************************************************************************* 21 | * Spool writer API 22 | ******************************************************************************/ 23 | void *kv_spoolwriter_new(const char *dir) { 24 | char path[PATH_MAX]; 25 | struct shr *shr; 26 | snprintf(path, PATH_MAX, "%s/%s", dir, "data"); 27 | shr = shr_open(path, SHR_WRONLY); 28 | return shr; 29 | } 30 | 31 | int kv_spool_write(void*_sp, void *_set) { 32 | struct shr *shr = (struct shr *)_sp; 33 | kvset_t *set = (kvset_t*)_set; 34 | tpl_node *tn = NULL; 35 | char *buf=NULL; 36 | char *key, *val; 37 | size_t len; 38 | ssize_t sc; 39 | int rc=-1; 40 | 41 | /* generate frame */ 42 | tn = tpl_map("A(ss)", &key, &val); 43 | kv_t *kv = NULL; 44 | while ( (kv = kv_next(set, kv))) { 45 | key = kv->key; 46 | val = kv->val; 47 | tpl_pack(tn,1); 48 | } 49 | 50 | tpl_dump(tn, TPL_MEM, &buf, &len); 51 | sc = shr_write(shr, buf, len); 52 | if (sc <= 0) { 53 | fprintf(stderr, "shr_write: error\n"); 54 | goto done; 55 | } 56 | 57 | rc=0; 58 | 59 | done: 60 | tpl_free(tn); 61 | if (buf) free(buf); 62 | return rc; 63 | } 64 | 65 | int kv_spool_writeN(void *_sp, void **_setv, int nset) { 66 | struct shr *shr = (struct shr *)_sp; 67 | kvset_t **setv = (kvset_t**)_setv; 68 | struct iovec *iov=NULL; 69 | tpl_node *tn = NULL; 70 | int i, rc = -1, sc; 71 | char *key, *val; 72 | 73 | iov = malloc(sizeof(struct iovec) * nset); 74 | if (iov == NULL) { 75 | fprintf(stderr, "out of memory\n"); 76 | goto done; 77 | } 78 | 79 | 80 | for(i=0; i < nset; i++) { 81 | tn = tpl_map("A(ss)", &key, &val); 82 | kv_t *kv = NULL; 83 | while ( (kv = kv_next(setv[i], kv))) { 84 | key = kv->key; 85 | val = kv->val; 86 | tpl_pack(tn,1); 87 | } 88 | tpl_dump(tn, TPL_MEM, &iov[i].iov_base, &iov[i].iov_len); 89 | tpl_free(tn); 90 | tn = NULL; 91 | } 92 | 93 | sc = shr_writev(shr, iov, nset); 94 | if (sc <= 0) { 95 | fprintf(stderr, "shr_writev: error\n"); 96 | goto done; 97 | } 98 | 99 | rc = 0; 100 | 101 | done: 102 | if (iov) { 103 | for(i=0; i < nset; i++) free(iov[i].iov_base); 104 | free(iov); 105 | } 106 | return rc; 107 | } 108 | 109 | void kv_spoolwriter_free(void*_sp) { 110 | struct shr *shr = (struct shr *)_sp; 111 | shr_close(shr); 112 | } 113 | -------------------------------------------------------------------------------- /utils/.gitignore: -------------------------------------------------------------------------------- 1 | kvsp-cast 2 | kvsp-concen 3 | kvsp-tee 4 | kvsp-init 5 | kvsp-sub 6 | kvsp-pub 7 | kvsp-speed 8 | kvsp-status 9 | kvsp-mod 10 | kvsp-rewind 11 | kvsp-spr 12 | kvsp-spw 13 | *.o 14 | *.a 15 | .deps 16 | Makefile.in 17 | ramdisk 18 | kvsp-bpub 19 | kvsp-bsub 20 | .gdb_history 21 | -------------------------------------------------------------------------------- /utils/Makefile.am: -------------------------------------------------------------------------------- 1 | srcdir = @srcdir@ 2 | 3 | AM_CFLAGS = -I$(srcdir)/.. -I$(srcdir)/../include 4 | LIBSPOOL = -L../src -lkvspool -lshr 5 | bin_PROGRAMS = kvsp-spr kvsp-spw kvsp-init kvsp-status \ 6 | kvsp-speed kvsp-mod kvsp-rewind \ 7 | ramdisk kvsp-bcat kvsp-bshr kvsp-tsub kvsp-tpub 8 | 9 | kvsp_spr_LDADD = $(LIBSPOOL) 10 | kvsp_spw_LDADD = $(LIBSPOOL) 11 | kvsp_tee_LDADD = $(LIBSPOOL) 12 | kvsp_init_LDADD = $(LIBSPOOL) 13 | kvsp_status_LDADD = $(LIBSPOOL) 14 | kvsp_speed_LDADD = $(LIBSPOOL) 15 | kvsp_mod_LDADD = $(LIBSPOOL) 16 | kvsp_rewind_LDADD = $(LIBSPOOL) 17 | kvsp_bcat_LDADD = $(LIBSPOOL) 18 | kvsp_bshr_LDADD = $(LIBSPOOL) 19 | kvsp_tsub_LDADD = $(LIBSPOOL) 20 | kvsp_tpub_LDADD = $(LIBSPOOL) 21 | kvsp_bpub_LDADD = $(LIBSPOOL) 22 | kvsp_bsub_LDADD = $(LIBSPOOL) 23 | kvsp_npub_LDADD = $(LIBSPOOL) 24 | kvsp_nsub_LDADD = $(LIBSPOOL) 25 | kvsp_pub_LDADD = $(LIBSPOOL) 26 | kvsp_sub_LDADD = $(LIBSPOOL) 27 | kvsp_concen_LDADD = $(LIBSPOOL) 28 | kvsp_upub_LDADD = $(LIBSPOOL) 29 | kvsp_kkpub_LDADD = $(LIBSPOOL) 30 | 31 | kvsp_bcat_SOURCES = kvsp-bcat.c kvsp-bconfig.c 32 | kvsp_bshr_SOURCES = kvsp-bshr.c kvsp-bconfig.c 33 | kvsp_tsub_SOURCES = kvsp-tsub.c kvsp-bconfig.c 34 | kvsp_tpub_SOURCES = kvsp-tpub.c kvsp-bconfig.c ringbuf.c 35 | kvsp_bpub_SOURCES = kvsp-bpub.c kvsp-bconfig.c 36 | kvsp_bsub_SOURCES = kvsp-bsub.c kvsp-bconfig.c 37 | kvsp_npub_SOURCES = kvsp-npub.c kvsp-bconfig.c 38 | kvsp_nsub_SOURCES = kvsp-nsub.c kvsp-bconfig.c 39 | 40 | if HAVE_PCRE 41 | bin_PROGRAMS += kvsp-tee 42 | kvsp_tee_LDADD += -lpcre 43 | endif 44 | 45 | if HAVE_NANOMSG 46 | bin_PROGRAMS += kvsp-npub kvsp-nsub 47 | kvsp_npub_LDADD += -lnanomsg 48 | kvsp_nsub_LDADD += -lnanomsg 49 | endif 50 | 51 | if HAVE_RDKAFKA 52 | if HAVE_JANSSON 53 | if HAVE_NANOMSG 54 | bin_PROGRAMS += kvsp-kkpub 55 | kvsp_kkpub_SOURCES = kvsp-kkpub.c ts.c ts.h 56 | kvsp_kkpub_CFLAGS = ${AM_CFLAGS} -pthread 57 | kvsp_kkpub_LDADD += -lrdkafka -ljansson -lnanomsg 58 | endif 59 | endif 60 | endif 61 | 62 | if HAVE_ZEROMQ 63 | bin_PROGRAMS += kvsp-bpub kvsp-bsub 64 | kvsp_bpub_LDADD += -lzmq 65 | kvsp_bsub_LDADD += -lzmq 66 | endif 67 | 68 | if HAVE_ZEROMQ 69 | if HAVE_JANSSON 70 | bin_PROGRAMS += kvsp-sub kvsp-pub kvsp-concen 71 | kvsp_pub_LDADD += -lzmq -ljansson 72 | kvsp_sub_LDADD += -lzmq -ljansson 73 | kvsp_concen_LDADD += -lzmq -ljansson 74 | endif 75 | endif 76 | if HAVE_JANSSON 77 | bin_PROGRAMS += kvsp-upub 78 | kvsp_upub_LDADD += -ljansson 79 | endif 80 | 81 | -------------------------------------------------------------------------------- /utils/kvsp-bcat.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include "utarray.h" 15 | #include "utstring.h" 16 | #include "kvspool.h" 17 | #include "kvsp-bconfig.h" 18 | 19 | /* read spool, cast to binary (like bpub, tpub), write to stdout */ 20 | 21 | int verbose; 22 | int oneshot=0; 23 | char *spool; 24 | 25 | void usage(char *prog) { 26 | fprintf(stderr, "usage: %s -[vo] -b -d spool\n", prog); 27 | fprintf(stderr, " -o (oneshot) read spool and exit\n"); 28 | exit(-1); 29 | } 30 | 31 | int main(int argc, char *argv[]) { 32 | void *sp=NULL; 33 | void *set=NULL; 34 | int opt,rc=-1; 35 | char *config_file, *b; 36 | size_t l; 37 | set = kv_set_new(); 38 | utarray_new(output_keys, &ut_str_icd); 39 | utarray_new(output_defaults, &ut_str_icd); 40 | utarray_new(output_types,&ut_int_icd); 41 | UT_string *tmp; 42 | utstring_new(tmp); 43 | 44 | while ( (opt = getopt(argc, argv, "v+d:b:o")) != -1) { 45 | switch (opt) { 46 | case 'v': verbose++; break; 47 | case 'o': oneshot=1; break; 48 | case 'd': spool=strdup(optarg); break; 49 | case 'b': config_file=strdup(optarg); break; 50 | default: usage(argv[0]); break; 51 | } 52 | } 53 | if (spool == NULL) usage(argv[0]); 54 | if (parse_config(config_file) < 0) goto done; 55 | 56 | sp = kv_spoolreader_new(spool); 57 | if (!sp) goto done; 58 | 59 | while (kv_spool_read(sp,set,1) > 0) { 60 | if (set_to_binary(set,tmp) < 0) goto done; 61 | 62 | b = utstring_body(tmp); 63 | l = utstring_len(tmp); 64 | if (write(STDOUT_FILENO, b, l) != l) { 65 | fprintf(stderr,"write: %s\n", l>0 ? 66 | "incomplete" : strerror(errno)); 67 | } 68 | if (oneshot) break; 69 | } 70 | 71 | rc = 0; 72 | 73 | done: 74 | if (sp) kv_spoolreader_free(sp); 75 | kv_set_free(set); 76 | utarray_free(output_keys); 77 | utarray_free(output_defaults); 78 | utarray_free(output_types); 79 | utstring_free(tmp); 80 | 81 | return 0; 82 | } 83 | 84 | -------------------------------------------------------------------------------- /utils/kvsp-bconfig.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include "utarray.h" 7 | #include "utstring.h" 8 | #include "kvsp-bconfig.h" 9 | 10 | UT_array /* of string */ *output_keys; 11 | UT_array /* of string */ *output_defaults; 12 | UT_array /* of int */ *output_types; 13 | 14 | #define x(t) #t, 15 | char *supported_types_str[] = { TYPES }; 16 | #undef x 17 | 18 | int parse_config(char *config_file) { 19 | char line[100]; 20 | FILE *file; 21 | int rc=-1; 22 | int type,t; 23 | char *sp,*nl,*def; 24 | if ( (file = fopen(config_file,"r")) == NULL) { 25 | fprintf(stderr,"can't open %s: %s\n", config_file, strerror(errno)); 26 | goto done; 27 | } 28 | while (fgets(line,sizeof(line),file) != NULL) { 29 | sp = strchr(line,' '); 30 | if (!sp) { 31 | fprintf(stderr,"syntax error in %s\n", config_file); 32 | goto done; 33 | } 34 | nl = strchr(line,'\n'); if (nl) *nl='\0'; 35 | for(t=0; t= adim(supported_types_str)){ 39 | fprintf(stderr,"unknown type %s\n",line); 40 | goto done; 41 | } 42 | char *id = sp+1; 43 | sp = strchr(id,' '); 44 | if (sp) *sp = '\0'; 45 | def = sp ? sp+1 : NULL; 46 | utarray_push_back(output_types,&t); 47 | utarray_push_back(output_keys,&id); 48 | utarray_push_back(output_defaults,&def); 49 | } 50 | rc = 0; 51 | done: 52 | if (file) fclose(file); 53 | return rc; 54 | } 55 | 56 | int set_to_binary(void *set, UT_string *bin) { 57 | uint32_t l, u, a,b,c,d,e,f, abcd; 58 | struct in_addr ia4; 59 | struct in6_addr ia6; 60 | uint16_t s; 61 | uint8_t g; 62 | double h; 63 | utstring_clear(bin); 64 | l=0; utstring_bincpy(bin,&l,sizeof(l)); // placeholder for size prefix 65 | int rc=-1,i=0,*t; 66 | kv_t *kv, kvdef; 67 | char **k=NULL,**def; 68 | while( (k=(char**)utarray_next(output_keys,k))) { 69 | kv = kv_get(set,*k); 70 | t = (int*)utarray_eltptr(output_types,i); assert(t); 71 | def = (char**)utarray_eltptr(output_defaults,i); assert(def); 72 | if (kv==NULL) { /* no such key */ 73 | kv=&kvdef; 74 | if (*def) {kv->val=*def; kv->vlen=strlen(*def);} /* default */ 75 | else if (*t == str) {kv->val=NULL; kv->vlen=0;} /* zero len string */ 76 | else { 77 | fprintf(stderr,"required key %s not present in spool frame\n", *k); 78 | goto done; 79 | } 80 | } 81 | switch(*t) { 82 | case d64: h=atof(kv->val); utstring_bincpy(bin,&h,sizeof(h)); break; 83 | case i8: g=atoi(kv->val); utstring_bincpy(bin,&g,sizeof(g)); break; 84 | case i16: s=atoi(kv->val); utstring_bincpy(bin,&s,sizeof(s)); break; 85 | case i32: u=atoi(kv->val); utstring_bincpy(bin,&u,sizeof(u)); break; 86 | case str8: 87 | g=kv->vlen; utstring_bincpy(bin,&g,sizeof(g)); /* length prefix */ 88 | utstring_bincpy(bin,kv->val,g); /* string itself */ 89 | break; 90 | case str: 91 | l=kv->vlen; utstring_bincpy(bin,&l,sizeof(l)); /* length prefix */ 92 | utstring_bincpy(bin,kv->val,kv->vlen); /* string itself */ 93 | break; 94 | case mac: 95 | if ((sscanf(kv->val,"%x:%x:%x:%x:%x:%x",&a,&b,&c,&d,&e,&f) != 6) || 96 | (a > 255 || b > 255 || c > 255 || d > 255 || e > 255 || f > 255)) { 97 | fprintf(stderr,"invalid MAC for key %s: %s\n",*k,kv->val); 98 | goto done; 99 | } 100 | g=a; utstring_bincpy(bin,&g,sizeof(g)); 101 | g=b; utstring_bincpy(bin,&g,sizeof(g)); 102 | g=c; utstring_bincpy(bin,&g,sizeof(g)); 103 | g=d; utstring_bincpy(bin,&g,sizeof(g)); 104 | g=e; utstring_bincpy(bin,&g,sizeof(g)); 105 | g=f; utstring_bincpy(bin,&g,sizeof(g)); 106 | break; 107 | case ipv46: 108 | memset(&ia4, 0, sizeof(ia4)); 109 | memset(&ia6, 0, sizeof(ia6)); 110 | if (inet_pton(AF_INET, kv->val, &ia4) == 1) { 111 | assert( 4 == sizeof(struct in_addr)); 112 | g=4; utstring_bincpy(bin,&g,sizeof(g)); 113 | utstring_bincpy(bin, &ia4, sizeof(struct in_addr)); 114 | } else if (inet_pton(AF_INET6, kv->val, &ia6) == 1) { 115 | assert( 16 == sizeof(struct in6_addr)); 116 | g=16; utstring_bincpy(bin,&g,sizeof(g)); 117 | utstring_bincpy(bin, &ia6, sizeof(struct in6_addr)); 118 | } else { 119 | fprintf(stderr,"invalid IP for key %s: %s\n",*k,kv->val); 120 | goto done; 121 | } 122 | break; 123 | case ipv4: 124 | if ((sscanf(kv->val,"%u.%u.%u.%u",&a,&b,&c,&d) != 4) || 125 | (a > 255 || b > 255 || c > 255 || d > 255)) { 126 | fprintf(stderr,"invalid IP for key %s: %s\n",*k,kv->val); 127 | goto done; 128 | } 129 | abcd = (a << 24) | (b << 16) | (c << 8) | d; 130 | abcd = htonl(abcd); 131 | utstring_bincpy(bin,&abcd,sizeof(abcd)); 132 | break; 133 | default: assert(0); break; 134 | } 135 | i++; 136 | } 137 | uint32_t len = utstring_len(bin); len -= sizeof(len); // length does not include itself 138 | char *length_prefix = utstring_body(bin); 139 | memcpy(length_prefix, &len, sizeof(len)); 140 | 141 | rc = 0; 142 | 143 | done: 144 | return rc; 145 | } 146 | 147 | static int get(void **msg_data,size_t *msg_len,void *dst,size_t len) { 148 | if (*msg_len < len) { 149 | fprintf(stderr,"received message shorter than expected\n"); 150 | return -1; 151 | } 152 | memcpy(dst,*msg_data,len); 153 | *(char**)msg_data += len; 154 | *msg_len -= len; 155 | return 0; 156 | } 157 | 158 | int binary_to_frame(void *sp, void *set, void *msg_data, size_t msg_len, UT_string *tmp) { 159 | int rc=-1,i=0,*t; 160 | const char *key; 161 | struct in_addr ia; 162 | char dst[INET6_ADDRSTRLEN]; 163 | char src[16]; 164 | 165 | uint32_t l, u, a,b,c,d, abcd; 166 | uint16_t s; 167 | uint8_t g; 168 | double h; 169 | uint8_t m[6]; 170 | 171 | kv_set_clear(set); 172 | char **k = NULL; 173 | while ( (k=(char**)utarray_next(output_keys,k))) { 174 | t = (int*)utarray_eltptr(output_types,i); assert(t); 175 | // type is *t and key is *k 176 | utstring_clear(tmp); 177 | switch(*t) { 178 | case d64: if (get(&msg_data,&msg_len,&h,sizeof(h))<0) goto done; utstring_printf(tmp,"%f",h); break; 179 | case i8: if (get(&msg_data,&msg_len,&g,sizeof(g))<0) goto done; utstring_printf(tmp,"%d",(int)g); break; 180 | case i16: if (get(&msg_data,&msg_len,&s,sizeof(s))<0) goto done; utstring_printf(tmp,"%d",(int)s); break; 181 | case i32: if (get(&msg_data,&msg_len,&u,sizeof(u))<0) goto done; utstring_printf(tmp,"%d",u); break; 182 | case str8: 183 | if (get(&msg_data,&msg_len,&g,sizeof(g)) < 0) goto done; 184 | utstring_reserve(tmp,g); 185 | if (get(&msg_data,&msg_len,utstring_body(tmp),g) < 0) goto done; 186 | tmp->i += g; 187 | break; 188 | case str: 189 | if (get(&msg_data,&msg_len,&l,sizeof(l)) < 0) goto done; 190 | utstring_reserve(tmp,l); 191 | if (get(&msg_data,&msg_len,utstring_body(tmp),l) < 0) goto done; 192 | tmp->i += l; 193 | break; 194 | case ipv4: 195 | if (get(&msg_data,&msg_len,&abcd,sizeof(abcd)) < 0) goto done; 196 | ia.s_addr = abcd; 197 | utstring_printf(tmp,"%s", inet_ntoa(ia)); 198 | break; 199 | case ipv46: 200 | if (get(&msg_data,&msg_len,&g,sizeof(g)) < 0) goto done; 201 | assert((g == 4) || (g == 16)); 202 | if (get(&msg_data,&msg_len,src,g) < 0) goto done; 203 | if (inet_ntop((g == 4) ? AF_INET : AF_INET6, src, dst, sizeof(dst)) == NULL) { 204 | fprintf(stderr, "inet_ntop: %s\n", strerror(errno)); 205 | goto done; 206 | } 207 | utstring_printf(tmp,"%s", dst); 208 | break; 209 | case mac: 210 | if (get(&msg_data,&msg_len,m,sizeof(m)) < 0) goto done; 211 | utstring_printf(tmp,"%2.2x:%2.2x:%2.2x:%2.2x:%2.2x:%2.2x", 212 | m[0], m[1], m[2], m[3], m[4], m[5]); 213 | break; 214 | default: assert(0); break; 215 | } 216 | i++; 217 | key = *k; 218 | kv_add(set, key, strlen(key), utstring_body(tmp), utstring_len(tmp)); 219 | } 220 | kv_spool_write(sp, set); 221 | 222 | rc = 0; 223 | 224 | done: 225 | if (rc) fprintf(stderr,"binary frame mismatches expected message length\n"); 226 | return rc; 227 | } 228 | -------------------------------------------------------------------------------- /utils/kvsp-bconfig.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "utarray.h" 4 | #include "utstring.h" 5 | #include "kvspool_internal.h" 6 | 7 | extern UT_array /* of string */ *output_keys; 8 | extern UT_array /* of string */ *output_defaults; 9 | extern UT_array /* of int */ *output_types; 10 | 11 | int parse_config(char *); 12 | int set_to_binary(void *set, UT_string *bin); 13 | int binary_to_frame(void *sp, void *set, void *msg_data, size_t msg_len, UT_string *tmp); 14 | 15 | 16 | extern char *supported_types_str[]; 17 | 18 | #define TYPES x(i16) x(i32) x(ipv4) x(ipv46) x(str) x(str8) x(i8) x(d64) x(mac) 19 | #define x(t) t, 20 | enum supported_types { TYPES }; 21 | #undef x 22 | #define adim(a) (sizeof(a)/sizeof(*a)) 23 | -------------------------------------------------------------------------------- /utils/kvsp-bpub.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include "utarray.h" 16 | #include "utstring.h" 17 | #include "kvspool.h" 18 | #include "kvsp-bconfig.h" 19 | 20 | #if ZMQ_VERSION_MAJOR == 2 21 | #define zmq_sendmsg zmq_send 22 | #define zmq_recvmsg zmq_recv 23 | #define zmq_hwm_t uint64_t 24 | #define ZMQ_SNDHWM ZMQ_HWM 25 | #else 26 | #define zmq_hwm_t int 27 | #endif 28 | 29 | const zmq_hwm_t hwm = 10000; /* high water mark: max messages pub will buffer */ 30 | char *pub_transport; /* clients connect to us on this transport */ 31 | void *pub_socket; 32 | void *pub_context; 33 | int verbose; 34 | char *spool; 35 | int push_mode; 36 | UT_string *tmp; 37 | 38 | void usage(char *prog) { 39 | fprintf(stderr, "usage: %s [-v] -b [-s] -d spool \n", prog); 40 | fprintf(stderr, " -s runs in push-pull mode instead of lossy pub/sub\n"); 41 | fprintf(stderr, " is a 0mq path e.g. tcp://127.0.0.1:1234\n"); 42 | exit(-1); 43 | } 44 | 45 | int main(int argc, char *argv[]) { 46 | void *sp=NULL; 47 | void *set=NULL; 48 | int opt,rc=-1; 49 | char *config_file, *bin; 50 | size_t len; 51 | set = kv_set_new(); 52 | utarray_new(output_keys, &ut_str_icd); 53 | utarray_new(output_defaults, &ut_str_icd); 54 | utarray_new(output_types,&ut_int_icd); 55 | utstring_new(tmp); 56 | 57 | while ( (opt = getopt(argc, argv, "v+d:b:s")) != -1) { 58 | switch (opt) { 59 | case 'v': verbose++; break; 60 | case 's': push_mode++; break; 61 | case 'd': spool=strdup(optarg); break; 62 | case 'b': config_file=strdup(optarg); break; 63 | default: usage(argv[0]); break; 64 | } 65 | } 66 | if (optind < argc) pub_transport = argv[optind++]; 67 | if (!pub_transport) usage(argv[0]); 68 | if (spool == NULL) usage(argv[0]); 69 | if (parse_config(config_file) < 0) goto done; 70 | 71 | if ( !(pub_context = zmq_init(1))) goto done; 72 | if ( !(pub_socket = zmq_socket(pub_context, push_mode?ZMQ_PUSH:ZMQ_PUB))) goto done; 73 | if (zmq_setsockopt(pub_socket, ZMQ_SNDHWM, &hwm, sizeof(hwm))) goto done; 74 | if (zmq_bind(pub_socket, pub_transport) == -1) goto done; 75 | 76 | sp = kv_spoolreader_new(spool); 77 | if (!sp) goto done; 78 | 79 | while (kv_spool_read(sp,set,1) > 0) { /* read til interrupted by signal */ 80 | 81 | if (set_to_binary(set,tmp) < 0) goto done; 82 | len = utstring_len(tmp); 83 | bin = utstring_body(tmp); 84 | 85 | /* skip length preamble */ 86 | if (len <= sizeof(uint32_t)) goto done; 87 | len -= sizeof(uint32_t); 88 | bin += sizeof(uint32_t); 89 | 90 | zmq_msg_t part; 91 | rc = zmq_msg_init_size(&part,len); 92 | if (rc) goto done; 93 | memcpy(zmq_msg_data(&part), bin, len); 94 | rc = zmq_sendmsg(pub_socket, &part, 0); 95 | 96 | zmq_msg_close(&part); 97 | if (rc == -1) goto done; 98 | } 99 | 100 | rc = 0; 101 | 102 | done: 103 | if (rc) fprintf(stderr,"zmq: %s %s\n", pub_transport, zmq_strerror(errno)); 104 | if (pub_socket) zmq_close(pub_socket); 105 | if (pub_context) zmq_term(pub_context); 106 | if (sp) kv_spoolreader_free(sp); 107 | kv_set_free(set); 108 | utarray_free(output_keys); 109 | utarray_free(output_defaults); 110 | utarray_free(output_types); 111 | utstring_free(tmp); 112 | 113 | return 0; 114 | } 115 | 116 | -------------------------------------------------------------------------------- /utils/kvsp-bshr.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include "utarray.h" 15 | #include "utstring.h" 16 | #include "kvspool.h" 17 | #include "kvsp-bconfig.h" 18 | #include "shr.h" 19 | 20 | /* read spool, cast to binary, write to shr */ 21 | 22 | int verbose; 23 | int oneshot=0; 24 | char *spool; 25 | char *shr_file; 26 | 27 | void usage(char *prog) { 28 | fprintf(stderr, "usage: %s -[vo] -b -d -s \n", prog); 29 | fprintf(stderr, " -o (oneshot) process one frame and exit\n"); 30 | exit(-1); 31 | } 32 | 33 | int main(int argc, char *argv[]) { 34 | struct shr *shr = NULL; 35 | void *sp=NULL; 36 | void *set=NULL; 37 | int opt,rc=-1,sc; 38 | char *config_file, *b; 39 | size_t l; 40 | set = kv_set_new(); 41 | utarray_new(output_keys, &ut_str_icd); 42 | utarray_new(output_defaults, &ut_str_icd); 43 | utarray_new(output_types,&ut_int_icd); 44 | UT_string *tmp; 45 | utstring_new(tmp); 46 | 47 | while ( (opt = getopt(argc, argv, "v+d:b:os:")) != -1) { 48 | switch (opt) { 49 | case 'v': verbose++; break; 50 | case 'o': oneshot=1; break; 51 | case 'd': spool=strdup(optarg); break; 52 | case 's': shr_file=strdup(optarg); break; 53 | case 'b': config_file=strdup(optarg); break; 54 | default: usage(argv[0]); break; 55 | } 56 | } 57 | if (spool == NULL) usage(argv[0]); 58 | if (shr_file == NULL) usage(argv[0]); 59 | if (parse_config(config_file) < 0) goto done; 60 | shr = shr_open(shr_file, SHR_WRONLY); 61 | if (shr == NULL) goto done; 62 | 63 | sp = kv_spoolreader_new(spool); 64 | if (!sp) goto done; 65 | 66 | while (kv_spool_read(sp,set,1) > 0) { 67 | if (set_to_binary(set,tmp) < 0) goto done; 68 | 69 | b = utstring_body(tmp); 70 | l = utstring_len(tmp); 71 | sc = shr_write(shr, b, l); 72 | if (sc <= 0) { 73 | fprintf(stderr,"shr_write: error %d\n", sc); 74 | goto done; 75 | } 76 | if (oneshot) break; 77 | } 78 | 79 | rc = 0; 80 | 81 | done: 82 | if (sp) kv_spoolreader_free(sp); 83 | if (shr) shr_close(shr); 84 | kv_set_free(set); 85 | utarray_free(output_keys); 86 | utarray_free(output_defaults); 87 | utarray_free(output_types); 88 | utstring_free(tmp); 89 | 90 | return 0; 91 | } 92 | 93 | -------------------------------------------------------------------------------- /utils/kvsp-bsub.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | 12 | #include "kvspool_internal.h" 13 | #include "utstring.h" 14 | #include "uthash.h" 15 | #include "kvsp-bconfig.h" 16 | 17 | void *sp; 18 | int verbose; 19 | int pull_mode; 20 | char *dir; 21 | char *remotes_file; 22 | void *context; 23 | void *zsocket; 24 | UT_string *tmp; 25 | 26 | 27 | void usage(char *exe) { 28 | fprintf(stderr,"usage: %s [-v] -b [-s] -d [-f file | ...]\n", exe); 29 | fprintf(stderr," -s runs in push-pull mode instead of lossy pub-sub\n"); 30 | exit(-1); 31 | } 32 | 33 | int read_lines(char *file, UT_array *lines) { 34 | char line[200]; 35 | int rc = -1; 36 | char *c; 37 | FILE *f = fopen(file,"r"); 38 | if (f==NULL) { 39 | fprintf(stderr,"fopen %s: %s\n", file, strerror(errno)); 40 | goto done; 41 | } 42 | while (fgets(line,sizeof(line),f) != NULL) { 43 | for(c=line; (c < line+sizeof(line)) && (*c != '\0'); c++) { 44 | if (*c == '\n') *c='\0'; 45 | if (*c == ' ') *c='\0'; 46 | } 47 | c = line; 48 | if (strlen(c) == 0) continue; 49 | utarray_push_back(lines,&c); 50 | } 51 | rc = 0; 52 | 53 | done: 54 | if (f) fclose(f); 55 | return rc; 56 | } 57 | 58 | #if ZMQ_VERSION_MAJOR == 2 59 | #define zmq_sendmsg zmq_send 60 | #define zmq_recvmsg zmq_recv 61 | #define zmq_rcvmore_t int64_t 62 | #else 63 | #define zmq_rcvmore_t int 64 | #endif 65 | 66 | int main(int argc, char *argv[]) { 67 | 68 | zmq_rcvmore_t more; size_t more_sz = sizeof(more); 69 | char *exe = argv[0], *filter = ""; 70 | int part_num,opt,rc=-1; 71 | void *msg_data, *sp, *set=NULL; 72 | char *config_file, **endpoint; 73 | UT_array *endpoints; 74 | size_t msg_len; 75 | zmq_msg_t part; 76 | utstring_new(tmp); 77 | utarray_new(output_keys, &ut_str_icd); 78 | utarray_new(output_defaults, &ut_str_icd); 79 | utarray_new(output_types,&ut_int_icd); 80 | utarray_new(endpoints,&ut_str_icd); 81 | 82 | while ( (opt = getopt(argc, argv, "sd:b:f:v+")) != -1) { 83 | switch (opt) { 84 | case 'v': verbose++; break; 85 | case 's': pull_mode++; break; 86 | case 'd': dir=strdup(optarg); break; 87 | case 'b': config_file=strdup(optarg); break; 88 | case 'f': remotes_file=strdup(optarg); break; 89 | default: usage(exe); break; 90 | } 91 | } 92 | if (!dir) usage(exe); 93 | if (parse_config(config_file) < 0) goto done; 94 | 95 | sp = kv_spoolwriter_new(dir); 96 | if (!sp) usage(exe); 97 | set = kv_set_new(); 98 | 99 | /* connect socket to each publisher. yes, zeromq lets you connect n times */ 100 | if ( !(context = zmq_init(1))) goto done; 101 | if ( !(zsocket = zmq_socket(context, pull_mode?ZMQ_PULL:ZMQ_SUB))) goto done; 102 | if (remotes_file) if (read_lines(remotes_file,endpoints)) goto done; 103 | while (optind < argc) utarray_push_back(endpoints,&argv[optind++]); 104 | if (utarray_len(endpoints) == 0) usage(argv[0]); 105 | endpoint=NULL; 106 | while ( (endpoint=(char**)utarray_next(endpoints,endpoint))) { 107 | if (verbose) fprintf(stderr,"connecting to %s\n", *endpoint); 108 | if (zmq_connect(zsocket, *endpoint)) goto done; 109 | } 110 | if (!pull_mode) { 111 | if (zmq_setsockopt(zsocket, ZMQ_SUBSCRIBE, filter, strlen(filter))) goto done; 112 | } 113 | 114 | while(1) { 115 | 116 | /* receive a multi-part message */ 117 | part_num=1; 118 | do { 119 | if ( (rc= zmq_msg_init(&part))) goto done; 120 | if ( ((rc= zmq_recvmsg(zsocket, &part, 0)) == -1) || 121 | ((rc= zmq_getsockopt(zsocket, ZMQ_RCVMORE, &more,&more_sz)) != 0)) { 122 | zmq_msg_close(&part); 123 | goto done; 124 | } 125 | 126 | msg_data = zmq_msg_data(&part); 127 | msg_len = zmq_msg_size(&part); 128 | 129 | switch(part_num) { /* part 1 has serialized frame */ 130 | case 1: if (binary_to_frame(sp,set,msg_data,msg_len,tmp)) goto done; break; 131 | default: assert(0); 132 | } 133 | 134 | zmq_msg_close(&part); 135 | part_num++; 136 | } while(more); 137 | } 138 | rc = 0; /* not reached TODO under clean shutdown on signal */ 139 | 140 | 141 | done: 142 | if (rc) fprintf(stderr,"%s: %s\n", exe, zmq_strerror(errno)); 143 | if(zsocket) zmq_close(zsocket); 144 | if(context) zmq_term(context); 145 | kv_spoolwriter_free(sp); 146 | if (set) kv_set_free(set); 147 | utarray_free(output_keys); 148 | utarray_free(output_defaults); 149 | utarray_free(output_types); 150 | utarray_free(endpoints); 151 | utstring_free(tmp); 152 | return rc; 153 | } 154 | 155 | -------------------------------------------------------------------------------- /utils/kvsp-concen.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include "utarray.h" 14 | #include "kvspool_internal.h" 15 | #include "tpl.h" 16 | 17 | /******************************************************************************* 18 | * spool concentrator 19 | * 20 | * reads several source spools at once using subprocesses; use a zeromq device to 21 | * send the spool events as messages to a single spool writer (the concentrator) 22 | *******************************************************************************/ 23 | 24 | #if ZMQ_VERSION_MAJOR == 2 25 | #define zmq_sendmsg zmq_send 26 | #define zmq_recvmsg zmq_recv 27 | #define zmq_hwm_t uint64_t 28 | #define ZMQ_SNDHWM ZMQ_HWM 29 | #else 30 | #define zmq_hwm_t int 31 | #endif 32 | 33 | typedef struct { 34 | pid_t pid; 35 | time_t start; 36 | char *dir; 37 | char *transport; 38 | } worker_t; 39 | 40 | #define SHORT_DELAY 10 41 | const zmq_hwm_t hwm = 10000; /* high water mark: max messages pub will buffer */ 42 | 43 | worker_t *workers; 44 | int wn=1; /* workers needed: 1 device + 1 publisher per spool */ 45 | int verbose; 46 | char *file; 47 | char *ospool; 48 | void *osp; 49 | UT_array *dirs; 50 | 51 | void usage(char *prog) { 52 | fprintf(stderr, "usage: %s [-v] [-d dir [-d dir ...]] \n", prog); 53 | exit(-1); 54 | } 55 | 56 | void configure_worker(int n) { 57 | worker_t *w = &workers[n]; 58 | char transport[100]; 59 | snprintf(transport,sizeof(transport),"ipc:///tmp/kvsp-concen-%u:%u",(int)getpid(),n); 60 | w->transport = strdup(transport); 61 | w->dir = strdup(*(char**)utarray_eltptr(dirs,n-1)); 62 | if (verbose) fprintf(stderr,"setting transport %s\n", w->transport); 63 | } 64 | 65 | void fill_set(void *img, size_t sz, kvset_t *set) { 66 | tpl_node *tn; 67 | char *key; 68 | char *val; 69 | 70 | kv_set_clear(set); 71 | 72 | tn = tpl_map("A(ss)", &key, &val); 73 | if (tpl_load(tn, TPL_MEM, img, sz) == -1) { 74 | fprintf(stderr, "tpl_load failed (sz %d)\n", (int)sz); 75 | return; 76 | } 77 | while( tpl_unpack(tn,1) > 0 ) { 78 | kv_adds(set, key, val); 79 | free(key); 80 | free(val); 81 | } 82 | tpl_free(tn); 83 | } 84 | 85 | 86 | /* one special sub-process runs the 'device': subscriber/central republisher */ 87 | void device(void) { 88 | char *img; 89 | size_t len; 90 | kvset_t *set=NULL; 91 | int n,rc=-1; 92 | void *dev_context=NULL; 93 | void *pull_socket=NULL; 94 | if ( !(dev_context = zmq_init(1))) goto done; 95 | if ( !(pull_socket = zmq_socket(dev_context, ZMQ_PULL))) goto done; 96 | 97 | set = kv_set_new(); 98 | 99 | /* connect the subscriber socket to each of the workers. then subscribe it */ 100 | for(n=1;n 0) { /* parent. */ 146 | /* record worker */ 147 | workers[w].pid = pid; 148 | workers[w].start = time(NULL); 149 | return; 150 | } 151 | 152 | /* child here */ 153 | 154 | /* unblock all signals */ 155 | sigset_t all; 156 | sigemptyset(&all); 157 | sigprocmask(SIG_SETMASK,&all,NULL); 158 | 159 | char name[16]; 160 | snprintf(name,sizeof(name),"kvsp-concen: %s", w ? workers[w].dir : "device"); 161 | prctl(PR_SET_NAME,name); 162 | prctl(PR_SET_PDEATHSIG, SIGHUP); 163 | 164 | if (w == 0) device(); // never returns 165 | 166 | void *_set = kv_set_new(); 167 | void *sp = kv_spoolreader_new(workers[w].dir); 168 | if (!sp) { 169 | fprintf(stderr,"failed to open spool %s\n", workers[w].dir); 170 | goto done; 171 | } 172 | 173 | /* prepare for ZMQ publishing */ 174 | void *pub_context; 175 | void *pub_socket; 176 | if ( (!(pub_context = zmq_init(1))) || 177 | ( (!(pub_socket = zmq_socket(pub_context, ZMQ_PUSH)))) || 178 | (zmq_bind(pub_socket, workers[w].transport) == -1)) { 179 | goto done; 180 | } 181 | 182 | while (kv_spool_read(sp,_set,1) > 0) { /* read til interrupted by signal */ 183 | kvset_t *set = (kvset_t*)_set; 184 | char *key, *val; 185 | 186 | /* generate frame */ 187 | tn = tpl_map("A(ss)", &key, &val); 188 | kv_t *kv = NULL; 189 | while ( (kv = kv_next(set, kv))) { 190 | key = kv->key; 191 | val = kv->val; 192 | tpl_pack(tn,1); 193 | } 194 | char *buf=NULL; 195 | size_t len = 0; 196 | tpl_dump(tn, TPL_MEM, &buf, &len); 197 | if (buf == NULL) goto done; 198 | tpl_free(tn); 199 | zmq_msg_t part; 200 | rc = zmq_msg_init_size(&part, len); 201 | memcpy(zmq_msg_data(&part), buf, len); 202 | free(buf); 203 | rc = zmq_sendmsg(pub_socket, &part, 0); 204 | zmq_msg_close(&part); 205 | if(rc == -1) goto done; 206 | } 207 | fprintf(stderr,"kv_spool_read exited (signal?)\n"); 208 | 209 | rc=0; 210 | 211 | done: 212 | 213 | if (rc) fprintf(stderr,"zmq: %s\n", zmq_strerror(errno)); 214 | if (pub_socket) zmq_close(pub_socket); 215 | if (pub_context) zmq_term(pub_context); 216 | if (sp) kv_spoolreader_free(sp); 217 | kv_set_free(_set); 218 | exit(rc); /* do not return */ 219 | } 220 | 221 | void run_workers() { 222 | int n; 223 | for(n=0; n < wn; n++) { 224 | if (workers[n].pid) continue; 225 | worker(n); 226 | } 227 | } 228 | 229 | 230 | void fini_workers() { 231 | int n,es; 232 | for(n=0; n < wn; n++) { 233 | worker_t *w = &workers[n]; 234 | if (w->pid == 0) goto release; 235 | kill(w->pid, SIGTERM); 236 | if (waitpid(w->pid, &es, WNOHANG) == w->pid) w->pid = 0; 237 | else { /* child didn't exit. give it a moment, then force quit. */ 238 | sleep(1); 239 | kill(w->pid, SIGKILL); 240 | if (waitpid(w->pid, &es, WNOHANG) == w->pid) w->pid = 0; 241 | } 242 | if(w->pid) fprintf(stderr, "can't terminate pid %d (%s): %s\n",(int)w->pid,w->dir,strerror(errno)); 243 | else if (WIFSIGNALED(es)) fprintf(stderr,"exited on signal %d", (int)WTERMSIG(es)); 244 | else if (WIFEXITED(es)) fprintf(stderr,"exit status %d", (int)WEXITSTATUS(es)); 245 | 246 | release: 247 | free(w->dir); 248 | free(w->transport); 249 | } 250 | } 251 | 252 | void read_conf(char *file) { 253 | char line[200], *linep = line; 254 | int len; 255 | FILE *f; 256 | 257 | if ( (f = fopen(file,"r")) == NULL) { 258 | fprintf(stderr,"can't open %s: %s\n", file, strerror(errno)); 259 | exit(-1); 260 | } 261 | while (fgets(line,sizeof(line),f) != NULL) { 262 | len = strlen(line); 263 | if (len && (line[len-1]=='\n')) line[--len] = '\0'; 264 | if (len) utarray_push_back(dirs,&linep); 265 | } 266 | } 267 | 268 | int main(int argc, char *argv[]) { 269 | char *file, *dir; 270 | int n,opt,es,defer_restart; 271 | pid_t pid; 272 | 273 | utarray_new(dirs,&ut_str_icd); 274 | 275 | while ( (opt = getopt(argc, argv, "v+sf:d:")) != -1) { 276 | switch (opt) { 277 | case 'v': verbose++; break; 278 | case 'f': file=optarg; read_conf(file); break; 279 | case 'd': dir=optarg; utarray_push_back(dirs,&dir); break; 280 | default: usage(argv[0]); break; 281 | } 282 | } 283 | if (optind < argc) ospool = argv[optind++]; 284 | if (!ospool) usage(argv[0]); 285 | if ( (wn += utarray_len(dirs)) == 1) { 286 | fprintf(stderr,"error: no input spools\n"); 287 | usage(argv[0]); 288 | } 289 | osp = kv_spoolwriter_new(ospool); 290 | if (!osp) { 291 | fprintf(stderr,"failed to open output spool %s\n", ospool); 292 | usage(argv[0]); 293 | } 294 | 295 | if ( (workers = calloc(wn,sizeof(worker_t))) == NULL) exit(-1); 296 | for(n=1; n 0) { 312 | for(n=0; n < wn; n++) if (workers[n].pid==pid) break; 313 | assert(n != wn); 314 | int elapsed = time(NULL) - workers[n].start; 315 | if (elapsed < SHORT_DELAY) defer_restart=1; 316 | printf("worker %d (%d) exited after %d seconds: ", n, (int)pid, elapsed); 317 | if (WIFEXITED(es)) printf("exit status %d\n", (int)WEXITSTATUS(es)); 318 | else if (WIFSIGNALED(es)) printf("signal %d\n", (int)WTERMSIG(es)); 319 | workers[n].pid = 0; 320 | } 321 | if (defer_restart) { 322 | fprintf(stderr,"workers restarting too fast, delaying\n"); 323 | alarm(SHORT_DELAY); 324 | } 325 | else run_workers(); 326 | break; 327 | case SIGALRM: 328 | run_workers(); 329 | break; 330 | default: 331 | printf("got signal %d\n", signo); 332 | goto done; 333 | break; 334 | } 335 | } 336 | 337 | done: 338 | kv_spoolwriter_free(osp); 339 | fini_workers(); 340 | free(workers); 341 | utarray_free(dirs); 342 | return 0; 343 | } 344 | -------------------------------------------------------------------------------- /utils/kvsp-init.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include "kvspool.h" 11 | #include "shr.h" 12 | #include "utstring.h" 13 | 14 | void usage(char *prog) { 15 | fprintf(stderr, "usage: %s [-v] -s spool [ spool ... ]\n", prog); 16 | fprintf(stderr, " is directory max size e.g. 1G (units KMGT)\n"); 17 | exit(-1); 18 | } 19 | 20 | int main(int argc, char * argv[]) { 21 | int opt,verbose=0, rc = -1, sc; 22 | char path[PATH_MAX]; 23 | long dirmax=10*1024*1024; 24 | /* input spool */ 25 | char *dir=NULL,unit,*sz; 26 | UT_string *s; 27 | utstring_new(s); 28 | 29 | while ( (opt = getopt(argc, argv, "v+s:")) != -1) { 30 | switch (opt) { 31 | default: usage(argv[0]); break; 32 | case 'v': verbose++; break; 33 | case 's': 34 | sz = strdup(optarg); 35 | switch (sscanf(sz, "%ld%c", &dirmax, &unit)) { 36 | case 2: /* check unit */ 37 | switch (unit) { 38 | case 't': case 'T': dirmax *= 1024; /* FALLTHRU */ 39 | case 'g': case 'G': dirmax *= 1024; /* FALLTHRU */ 40 | case 'm': case 'M': dirmax *= 1024; /* FALLTHRU */ 41 | case 'k': case 'K': dirmax *= 1024; /* FALLTHRU */ 42 | case '\r': case '\n': case ' ': case '\t': break; 43 | default: usage(argv[0]); break; 44 | } 45 | case 1: /* just a number in bytes */ break; 46 | default: usage(argv[0]); break; 47 | } 48 | break; 49 | } 50 | } 51 | if (optind >= argc) usage(argv[0]); 52 | if (!dirmax) usage(argv[0]); 53 | umask(0); 54 | 55 | while (optind < argc) { 56 | dir = argv[optind++]; 57 | snprintf(path, PATH_MAX, "%s/%s", dir, "data"); 58 | sc = shr_init(path, dirmax, SHR_KEEPEXIST|SHR_MESSAGES|SHR_DROP); 59 | if (sc < 0) goto done; 60 | sc = chmod(path, 0666); 61 | if (sc < 0) { 62 | fprintf(stderr, "chmod: %s\n", strerror(errno)); 63 | } 64 | } 65 | 66 | rc = 0; 67 | 68 | done: 69 | utstring_free(s); 70 | return rc; 71 | } 72 | 73 | -------------------------------------------------------------------------------- /utils/kvsp-mod.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include "kvspool.h" 5 | #include "utarray.h" 6 | #include "utstring.h" 7 | 8 | int verbose=0; 9 | char *dir = NULL; 10 | char *conf = NULL; 11 | char *odir = "."; 12 | UT_array *keys; 13 | 14 | void *osp = NULL; 15 | 16 | void usage(char *exe) { 17 | fprintf(stderr,"usage: %s [-v] -k " 18 | "[-k ...] [-o outdir] \n", exe); 19 | exit(-1); 20 | } 21 | 22 | int main(int argc, char *argv[]) { 23 | 24 | kv_t *kv; 25 | int opt,rc; 26 | char *exe = argv[0]; 27 | utarray_new(keys,&ut_str_icd); 28 | 29 | while ( (opt = getopt(argc, argv, "v+k:c:n:o:")) != -1) { 30 | switch (opt) { 31 | case 'v': verbose++; break; 32 | case 'c': conf=strdup(optarg); break; 33 | case 'o': odir=strdup(optarg); break; 34 | case 'k': utarray_push_back(keys,&optarg); break; 35 | default: usage(exe); break; 36 | } 37 | } 38 | if (optind < argc) dir=argv[optind++]; 39 | else usage(exe); 40 | 41 | osp = kv_spoolwriter_new(odir); 42 | if (!osp) exit(-1); 43 | 44 | void *sp = kv_spoolreader_new(dir); 45 | if (!sp) exit(-1); 46 | 47 | void *set = kv_set_new(); 48 | while ( (rc=kv_spool_read(sp,set,1)) > 0) { 49 | /* calculate hash value of keys */ 50 | char *key, *val, **k=NULL; 51 | while ( (k=(char**)utarray_next(keys,k))) { 52 | key = *k; 53 | kv = kv_get(set, key); 54 | if (!kv) {if (verbose) fprintf(stderr,"no such key: %s; skipping\n", key); continue;} 55 | /* replace the value with a hash of the value */ 56 | unsigned hv=0, vlen=kv->vlen; val = kv->val; 57 | while(vlen--) hv = hv * 33 + *val++; 58 | // we could write it in binary but we change it to text 59 | //kv_add(set, kv->key, kv->klen, (void*)&hv, sizeof(hv)); 60 | char hv_str[10]; 61 | snprintf(hv_str,sizeof(hv_str),"%u",hv); 62 | kv_add(set, kv->key, kv->klen, hv_str, strlen(hv_str)); 63 | } 64 | if (kv_spool_write(osp, set) != 0) {printf("output error\n"); break; } 65 | } 66 | 67 | /* clean up */ 68 | kv_spoolwriter_free(osp); 69 | kv_spoolreader_free(sp); 70 | kv_set_free(set); 71 | utarray_free(keys); 72 | return 0; 73 | } 74 | -------------------------------------------------------------------------------- /utils/kvsp-mpub.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include "utarray.h" 14 | #include "kvspool_internal.h" 15 | 16 | /******************************************************************************* 17 | * This program is currently out of date as it needs a trivial change to alter 18 | * the message content it sends to use the JSON format rather than frame binary. 19 | * 20 | * The difference between kvsp-mpub and kvsp-pub is that mpub reads several 21 | * source spools at once (using threads); however the threads and use of a zeromq 22 | * device to unify those readers into one output stream makes mpub more complex. 23 | *******************************************************************************/ 24 | 25 | #if ZMQ_VERSION_MAJOR == 2 26 | #define zmq_sendmsg zmq_send 27 | #define zmq_recvmsg zmq_recv 28 | #define zmq_hwm_t uint64_t 29 | #define ZMQ_SNDHWM ZMQ_HWM 30 | #else 31 | #define zmq_hwm_t int 32 | #endif 33 | /****************************************************************************** 34 | * kvsp-publish 35 | * This program monitors multiple spools using a pool of readers 36 | * Whenever a spool has a frame that's readable it is published to subscribers 37 | *****************************************************************************/ 38 | 39 | typedef struct { 40 | pid_t pid; 41 | time_t start; 42 | char *dir; 43 | char *transport; 44 | } worker_t; 45 | 46 | #define SHORT_DELAY 10 47 | const zmq_hwm_t hwm = 10000; /* high water mark: max messages pub will buffer */ 48 | 49 | worker_t *workers; 50 | int wn=1; /* workers needed: 1 device + 1 publisher per spool */ 51 | int verbose; 52 | int push_mode; 53 | char *file; 54 | char *pub_transport; /* client kvsp-sub's connect to us on this transport */ 55 | UT_array *dirs; 56 | 57 | void usage(char *prog) { 58 | fprintf(stderr, "usage: %s [-v] [-s] [-d dir [-d dir ...]] \n", prog); 59 | fprintf(stderr, " -s runs in push mode instead of lossy pub-sub\n"); 60 | fprintf(stderr, " is a 0mq path e.g. tcp://127.0.0.1:1234\n"); 61 | exit(-1); 62 | } 63 | 64 | void configure_worker(int n) { 65 | worker_t *w = &workers[n]; 66 | char transport[100]; 67 | snprintf(transport,sizeof(transport),"ipc://kvsp-pub-%u:%u",(int)getpid(),n); 68 | w->transport = strdup(transport); 69 | w->dir = strdup(*(char**)utarray_eltptr(dirs,n-1)); 70 | fprintf(stderr,"setting transport %s\n", w->transport); 71 | } 72 | 73 | /* one special sub-process runs the 'device': subscriber/central republisher */ 74 | void device(void) { 75 | int n,rc=-1; 76 | void *dev_context=NULL; 77 | void *pull_socket=NULL,*pub_socket=NULL; 78 | if ( !(dev_context = zmq_init(1))) goto done; 79 | if ( !(pull_socket = zmq_socket(dev_context, ZMQ_PULL))) goto done; 80 | 81 | if (push_mode) { 82 | if ( !(pub_socket = zmq_socket(dev_context, ZMQ_PUSH))) goto done; 83 | } else { 84 | if ( !(pub_socket = zmq_socket(dev_context, ZMQ_PUB))) goto done; 85 | } 86 | 87 | /* connect the subscriber socket to each of the workers. then subscribe it */ 88 | for(n=1;n 0) { /* parent. */ 135 | /* record worker */ 136 | workers[w].pid = pid; 137 | workers[w].start = time(NULL); 138 | return; 139 | } 140 | 141 | /* child here */ 142 | 143 | /* unblock all signals */ 144 | sigset_t all; 145 | sigemptyset(&all); 146 | sigprocmask(SIG_SETMASK,&all,NULL); 147 | 148 | char name[16]; 149 | snprintf(name,sizeof(name),"kvsp-pub: %s", w ? workers[w].dir : "device"); 150 | prctl(PR_SET_NAME,name); 151 | prctl(PR_SET_PDEATHSIG, SIGHUP); // TODO clean shutdown on HUP 152 | 153 | if (w == 0) device(); // never returns 154 | 155 | void *_set = kv_set_new(); 156 | void *sp = kv_spoolreader_new(workers[w].dir); 157 | if (!sp) { 158 | fprintf(stderr,"failed to open spool %s\n", workers[w].dir); 159 | goto done; 160 | } 161 | 162 | /* prepare for ZMQ publishing */ 163 | void *pub_context; 164 | void *pub_socket; 165 | if ( (!(pub_context = zmq_init(1))) || 166 | ( (!(pub_socket = zmq_socket(pub_context, ZMQ_PUSH)))) || 167 | (zmq_bind(pub_socket, workers[w].transport) == -1)) { 168 | goto done; 169 | } 170 | 171 | while (kv_spool_read(sp,_set,1) > 0) { /* read til interrupted by signal */ 172 | kvset_t *set = (kvset_t*)_set; 173 | assert(set->img && set->len); 174 | 175 | zmq_msg_t part; 176 | 177 | rc = zmq_msg_init_size(&part, set->len); assert(!rc); 178 | memcpy(zmq_msg_data(&part), set->img, set->len); 179 | rc = zmq_sendmsg(pub_socket, &part, 0); 180 | zmq_msg_close(&part); 181 | if(rc == -1) goto done; 182 | } 183 | fprintf(stderr,"kv_spool_read exited (signal?)\n"); 184 | 185 | rc=0; 186 | 187 | done: 188 | // TODO avoid printing to parent stderr (kv lib does; so dup fd to logging) 189 | if (rc) fprintf(stderr,"zmq: %s %s\n", pub_transport, zmq_strerror(errno)); 190 | if (pub_socket) zmq_close(pub_socket); 191 | if (pub_context) zmq_term(pub_context); 192 | if (sp) kv_spoolreader_free(sp); 193 | kv_set_free(_set); 194 | exit(rc); /* do not return */ 195 | } 196 | 197 | void run_workers() { 198 | int n; 199 | for(n=0; n < wn; n++) { 200 | if (workers[n].pid) continue; 201 | worker(n); 202 | } 203 | } 204 | 205 | 206 | void fini_workers() { 207 | int n,es; 208 | for(n=0; n < wn; n++) { 209 | worker_t *w = &workers[n]; 210 | if (w->pid == 0) goto release; 211 | kill(w->pid, SIGTERM); 212 | if (waitpid(w->pid, &es, WNOHANG) == w->pid) w->pid = 0; 213 | else { /* child didn't exit. give it a moment, then force quit. */ 214 | sleep(1); 215 | kill(w->pid, SIGKILL); 216 | if (waitpid(w->pid, &es, WNOHANG) == w->pid) w->pid = 0; 217 | } 218 | if(w->pid) fprintf(stderr, "can't terminate pid %d (%s): %s\n",(int)w->pid,w->dir,strerror(errno)); 219 | else if (WIFSIGNALED(es)) fprintf(stderr,"exited on signal %d", (int)WTERMSIG(es)); 220 | else if (WIFEXITED(es)) fprintf(stderr,"exit status %d", (int)WEXITSTATUS(es)); 221 | 222 | release: 223 | free(w->dir); 224 | free(w->transport); 225 | } 226 | } 227 | 228 | void read_conf(char *file) { 229 | char line[200], *linep = line; 230 | int len; 231 | FILE *f; 232 | 233 | if ( (f = fopen(file,"r")) == NULL) { 234 | fprintf(stderr,"can't open %s: %s\n", file, strerror(errno)); 235 | exit(-1); 236 | } 237 | while (fgets(line,sizeof(line),f) != NULL) { 238 | len = strlen(line); 239 | if (len && (line[len-1]=='\n')) line[--len] = '\0'; 240 | if (len) utarray_push_back(dirs,&linep); 241 | } 242 | } 243 | 244 | int main(int argc, char *argv[]) { 245 | char *file, *dir; 246 | int n,opt,es,defer_restart; 247 | pid_t pid; 248 | 249 | utarray_new(dirs,&ut_str_icd); 250 | 251 | while ( (opt = getopt(argc, argv, "v+sf:d:")) != -1) { 252 | switch (opt) { 253 | case 'v': verbose++; break; 254 | case 's': push_mode++; break; 255 | case 'f': file=optarg; read_conf(file); break; 256 | case 'd': dir=optarg; utarray_push_back(dirs,&dir); break; 257 | default: usage(argv[0]); break; 258 | } 259 | } 260 | if (optind < argc) pub_transport = argv[optind++]; 261 | if (!pub_transport) usage(argv[0]); 262 | if ( (wn += utarray_len(dirs)) == 1) { 263 | fprintf(stderr,"no directories configured\n"); 264 | usage(argv[0]); 265 | } 266 | 267 | if ( (workers = calloc(wn,sizeof(worker_t))) == NULL) exit(-1); 268 | for(n=1; n 0) { 284 | for(n=0; n < wn; n++) if (workers[n].pid==pid) break; 285 | assert(n != wn); 286 | int elapsed = time(NULL) - workers[n].start; 287 | if (elapsed < SHORT_DELAY) defer_restart=1; 288 | printf("worker %d (%d) exited after %d seconds: ", n, (int)pid, elapsed); 289 | if (WIFEXITED(es)) printf("exit status %d\n", (int)WEXITSTATUS(es)); 290 | else if (WIFSIGNALED(es)) printf("signal %d\n", (int)WTERMSIG(es)); 291 | workers[n].pid = 0; 292 | } 293 | if (defer_restart) { 294 | fprintf(stderr,"workers restarting too fast, delaying\n"); 295 | alarm(SHORT_DELAY); 296 | } 297 | else run_workers(); 298 | break; 299 | case SIGALRM: 300 | run_workers(); 301 | break; 302 | default: 303 | printf("got signal %d\n", signo); 304 | goto done; 305 | break; 306 | } 307 | } 308 | 309 | done: 310 | fini_workers(); 311 | free(workers); 312 | utarray_free(dirs); 313 | return 0; 314 | } 315 | -------------------------------------------------------------------------------- /utils/kvsp-npub.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include "utarray.h" 17 | #include "utstring.h" 18 | #include "kvspool.h" 19 | #include "kvsp-bconfig.h" 20 | 21 | char *remotes_file; 22 | char *config_file; 23 | UT_string *tmp; 24 | int sock,eid; 25 | int verbose; 26 | char *spool; 27 | void *set; 28 | void *sp; 29 | 30 | void usage(char *prog) { 31 | fprintf(stderr, "usage: %s [-v] -b -d spool [-f ] [ ...]\n", prog); 32 | fprintf(stderr, " is the nsub peer e.g. tcp://127.0.0.1:1234\n"); 33 | exit(-1); 34 | } 35 | 36 | int read_lines(char *file, UT_array *lines) { 37 | char line[200]; 38 | int rc = -1; 39 | char *c; 40 | FILE *f = fopen(file,"r"); 41 | if (f==NULL) { 42 | fprintf(stderr,"fopen %s: %s\n", file, strerror(errno)); 43 | goto done; 44 | } 45 | while (fgets(line,sizeof(line),f) != NULL) { 46 | for(c=line; (c < line+sizeof(line)) && (*c != '\0'); c++) { 47 | if (*c == '\n') *c='\0'; 48 | if (*c == ' ') *c='\0'; 49 | } 50 | c = line; 51 | if (strlen(c) == 0) continue; 52 | utarray_push_back(lines,&c); 53 | } 54 | rc = 0; 55 | 56 | done: 57 | if (f) fclose(f); 58 | return rc; 59 | } 60 | 61 | int main(int argc, char *argv[]) { 62 | int opt,rc=-1; 63 | size_t len; 64 | void *buf; 65 | UT_array *endpoints; 66 | char **endpoint; 67 | 68 | set = kv_set_new(); 69 | utarray_new(output_keys, &ut_str_icd); 70 | utarray_new(output_defaults, &ut_str_icd); 71 | utarray_new(output_types,&ut_int_icd); 72 | utstring_new(tmp); 73 | utarray_new(endpoints,&ut_str_icd); 74 | 75 | while ( (opt = getopt(argc, argv, "v+d:b:f:h")) != -1) { 76 | switch (opt) { 77 | case 'v': verbose++; break; 78 | case 'd': spool=strdup(optarg); break; 79 | case 'b': config_file=strdup(optarg); break; 80 | case 'f': remotes_file=strdup(optarg); break; 81 | case 'h': default: usage(argv[0]); break; 82 | } 83 | } 84 | if (remotes_file) if (read_lines(remotes_file,endpoints)) goto done; 85 | while (optind < argc) utarray_push_back(endpoints,&argv[optind++]); 86 | if (utarray_len(endpoints) == 0) { 87 | fprintf(stderr, "%s: no remotes\n", argv[0]); 88 | goto done; 89 | } 90 | if (spool == NULL) usage(argv[0]); 91 | if (parse_config(config_file) < 0) goto done; 92 | if ( !(sp = kv_spoolreader_new(spool))) goto done; 93 | rc = -2; 94 | 95 | if ( (sock = nn_socket(AF_SP, NN_PUSH)) < 0) goto done; 96 | endpoint=NULL; 97 | while ( (endpoint=(char**)utarray_next(endpoints,endpoint))) { 98 | if (verbose) fprintf(stderr,"connecting to %s\n", *endpoint); 99 | if ( (eid = nn_connect(sock, *endpoint)) < 0) goto done; 100 | } 101 | 102 | while (kv_spool_read(sp,set,1) > 0) { /* read til interrupted by signal */ 103 | if (set_to_binary(set,tmp) < 0) goto done; 104 | buf = utstring_body(tmp); 105 | len = utstring_len(tmp); 106 | /* skip length preamble */ 107 | if (len <= sizeof(uint32_t)) goto done; 108 | len -= sizeof(uint32_t); 109 | buf += sizeof(uint32_t); 110 | 111 | rc = nn_send(sock, buf, len, 0); 112 | if (rc == -1) goto done; 113 | } 114 | 115 | rc = 0; 116 | 117 | done: 118 | if (rc==-2) fprintf(stderr,"nano: %s\n", nn_strerror(errno)); 119 | if (sock) nn_close(sock); 120 | if (sp) kv_spoolreader_free(sp); 121 | kv_set_free(set); 122 | utarray_free(output_keys); 123 | utarray_free(output_defaults); 124 | utarray_free(output_types); 125 | utstring_free(tmp); 126 | utarray_free(endpoints); 127 | 128 | return 0; 129 | } 130 | 131 | -------------------------------------------------------------------------------- /utils/kvsp-nsub.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | 13 | #include "kvspool_internal.h" 14 | #include "utstring.h" 15 | #include "uthash.h" 16 | #include "kvsp-bconfig.h" 17 | 18 | char *config_file; 19 | UT_string *tmp; 20 | int sock,eid; 21 | int verbose; 22 | char *local; 23 | char *dir; 24 | void *set; 25 | void *sp; 26 | 27 | void usage(char *exe) { 28 | fprintf(stderr,"usage: %s [-v] -b -d \n", exe); 29 | fprintf(stderr," is our local address e.g. tcp://0.0.0.0:1234\n"); 30 | exit(-1); 31 | } 32 | 33 | int main(int argc, char *argv[]) { 34 | int opt,rc=-1,len; 35 | char *buf; 36 | 37 | set = kv_set_new(); 38 | utstring_new(tmp); 39 | utarray_new(output_keys, &ut_str_icd); 40 | utarray_new(output_defaults, &ut_str_icd); 41 | utarray_new(output_types,&ut_int_icd); 42 | 43 | while ( (opt = getopt(argc, argv, "d:b:v+")) != -1) { 44 | switch (opt) { 45 | case 'v': verbose++; break; 46 | case 'd': dir=strdup(optarg); break; 47 | case 'b': config_file=strdup(optarg); break; 48 | default: usage(argv[0]); break; 49 | } 50 | } 51 | 52 | if (!dir) usage(argv[0]); 53 | if (parse_config(config_file) < 0) goto done; 54 | if ( !(sp = kv_spoolwriter_new(dir))) goto done; 55 | if (optind < argc) local = argv[optind++]; 56 | if (!local) usage(argv[0]); 57 | rc = -2; 58 | 59 | if ( (sock = nn_socket(AF_SP, NN_PULL)) < 0) goto done; 60 | if ( (eid = nn_bind(sock, local)) < 0) goto done; 61 | 62 | while(1) { 63 | 64 | if ( (len = nn_recv(sock, &buf, NN_MSG, 0)) == -1) goto done; 65 | if (binary_to_frame(sp,set,buf,len,tmp)) {rc=-3; goto done;} 66 | nn_freemsg(buf); 67 | } 68 | 69 | rc = 0; /* not reached TODO under clean shutdown on signal */ 70 | 71 | 72 | done: 73 | if (rc==-2) fprintf(stderr,"%s: %s\n", local, nn_strerror(errno)); 74 | if(sock) nn_shutdown(sock,eid); 75 | kv_spoolwriter_free(sp); 76 | if (set) kv_set_free(set); 77 | utarray_free(output_keys); 78 | utarray_free(output_defaults); 79 | utarray_free(output_types); 80 | utstring_free(tmp); 81 | return rc; 82 | } 83 | 84 | -------------------------------------------------------------------------------- /utils/kvsp-pub.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include "kvspool.h" 16 | 17 | #if ZMQ_VERSION_MAJOR == 2 18 | #define zmq_sendmsg zmq_send 19 | #define zmq_recvmsg zmq_recv 20 | #define zmq_hwm_t uint64_t 21 | #define ZMQ_SNDHWM ZMQ_HWM 22 | #else 23 | #define zmq_hwm_t int 24 | #endif 25 | 26 | const zmq_hwm_t hwm = 10000; /* high water mark: max messages pub will buffer */ 27 | char *pub_transport; /* clients connect to us on this transport */ 28 | void *pub_socket; 29 | void *pub_context; 30 | int verbose; 31 | char *spool; 32 | int push_mode; 33 | 34 | void usage(char *prog) { 35 | fprintf(stderr, "usage: %s [-v] [-s] -d spool \n", prog); 36 | fprintf(stderr, " -s runs in push-pull mode instead of lossy pub/sub\n"); 37 | fprintf(stderr, " is a 0mq path e.g. tcp://127.0.0.1:1234\n"); 38 | exit(-1); 39 | } 40 | 41 | int main(int argc, char *argv[]) { 42 | void *sp=NULL; 43 | void *set=NULL; 44 | int opt,rc=-1; 45 | json_t *o = NULL; 46 | 47 | while ( (opt = getopt(argc, argv, "v+d:s")) != -1) { 48 | switch (opt) { 49 | case 'v': verbose++; break; 50 | case 's': push_mode++; break; 51 | case 'd': spool=strdup(optarg); break; 52 | default: usage(argv[0]); break; 53 | } 54 | } 55 | if (optind < argc) pub_transport = argv[optind++]; 56 | if (!pub_transport) usage(argv[0]); 57 | if (spool == NULL) usage(argv[0]); 58 | 59 | if ( !(pub_context = zmq_init(1))) goto done; 60 | if ( !(pub_socket = zmq_socket(pub_context, push_mode?ZMQ_PUSH:ZMQ_PUB))) goto done; 61 | if (zmq_setsockopt(pub_socket, ZMQ_SNDHWM, &hwm, sizeof(hwm))) goto done; 62 | if (zmq_bind(pub_socket, pub_transport) == -1) goto done; 63 | 64 | set = kv_set_new(); 65 | sp = kv_spoolreader_new(spool); 66 | if (!sp) goto done; 67 | o = json_object(); 68 | 69 | while (kv_spool_read(sp,set,1) > 0) { /* read til interrupted by signal */ 70 | zmq_msg_t part; 71 | json_object_clear(o); 72 | kv_t *kv = NULL; 73 | while (kv = kv_next(set,kv)) { 74 | json_t *jval = json_string(kv->val); 75 | json_object_set_new(o, kv->key, jval); 76 | } 77 | if (verbose) json_dumpf(o, stderr, JSON_INDENT(1)); 78 | char *json = json_dumps(o, JSON_INDENT(1)); 79 | size_t len = strlen(json); 80 | rc = zmq_msg_init_size(&part,len); if (rc) goto done; 81 | memcpy(zmq_msg_data(&part), json, len); 82 | free(json); 83 | rc = zmq_sendmsg(pub_socket, &part, 0); 84 | zmq_msg_close(&part); 85 | if (rc == -1) goto done; 86 | } 87 | 88 | rc = 0; 89 | 90 | done: 91 | if (rc) fprintf(stderr,"zmq: %s %s\n", pub_transport, zmq_strerror(errno)); 92 | if (pub_socket) zmq_close(pub_socket); 93 | if (pub_context) zmq_term(pub_context); 94 | if (sp) kv_spoolreader_free(sp); 95 | kv_set_free(set); 96 | if(o) json_decref(o); 97 | 98 | return 0; 99 | } 100 | 101 | -------------------------------------------------------------------------------- /utils/kvsp-rewind.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include "kvspool.h" 5 | 6 | char *dir; 7 | int verbose; 8 | 9 | void usage(char *exe) { 10 | fprintf(stderr,"usage: %s \n", exe); 11 | exit(-1); 12 | } 13 | 14 | int main(int argc, char *argv[]) { 15 | 16 | int opt; 17 | char *exe = argv[0]; 18 | while ( (opt = getopt(argc, argv, "v+")) != -1) { 19 | switch (opt) { 20 | case 'v': verbose++; break; 21 | default: usage(exe); break; 22 | } 23 | } 24 | if (optind < argc) dir=argv[optind++]; 25 | else usage(exe); 26 | 27 | fprintf(stderr, "This utility is no longer supported.\n"); 28 | return -1; 29 | } 30 | 31 | -------------------------------------------------------------------------------- /utils/kvsp-speed.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include "shr.h" 8 | #include "kvspool.h" 9 | 10 | int frames=100000; 11 | int verbose=0; 12 | char *dir = "/dev/shm"; 13 | char path[PATH_MAX]; 14 | char *exe; 15 | 16 | void usage(char *exe) { 17 | fprintf(stderr,"usage: %s [-v] [-i iterations] []\n", exe); 18 | exit(-1); 19 | } 20 | 21 | int individual_frames_test() { 22 | long elapsed_usec_w, elapsed_usec_r; 23 | struct timeval t1, t2; 24 | int i; 25 | 26 | void *sp = kv_spoolwriter_new(dir); 27 | if (!sp) exit(-1); 28 | 29 | char timebuf[100], iterbuf[10]; 30 | time_t t = time(NULL); 31 | snprintf(timebuf,sizeof(timebuf),"%s",ctime(&t)); 32 | timebuf[strlen(timebuf)-1] = '\0'; /* trim \n */ 33 | snprintf(iterbuf,sizeof(iterbuf),"%d",frames); 34 | 35 | void *set = kv_set_new(); 36 | kv_adds(set, "from", exe); 37 | kv_adds(set, "time", timebuf); 38 | kv_adds(set, "iter", iterbuf); 39 | printf("writing and reading individual frames:\n"); 40 | 41 | /* write test */ 42 | gettimeofday(&t1,NULL); 43 | for(i=0; i 2 | #include 3 | #include 4 | #include 5 | #include "kvspool_internal.h" 6 | 7 | int verbose=0; 8 | char *dir = NULL; 9 | enum {mode_out,mode_nop} mode = mode_out; 10 | 11 | void usage(char *exe) { 12 | fprintf(stderr,"usage: %s [-f] \n", exe); 13 | fprintf(stderr," -f (discard mode)\n"); 14 | exit(-1); 15 | } 16 | 17 | /****************************************************************************** 18 | * kvsp-spr: a spool reader, for manually inspecting contents of a spool 19 | *****************************************************************************/ 20 | int main(int argc, char *argv[]) { 21 | 22 | char *exe = argv[0]; 23 | int opt; 24 | 25 | while ( (opt = getopt(argc, argv, "fv+")) != -1) { 26 | switch (opt) { 27 | case 'v': verbose++; break; 28 | case 'f': mode=mode_nop; break; 29 | default: usage(exe); break; 30 | } 31 | } 32 | if (optind < argc) dir=argv[optind++]; 33 | else usage(exe); 34 | 35 | void *sp = kv_spoolreader_new(dir); 36 | if (!sp) exit(-1); 37 | 38 | 39 | void *set = kv_set_new(); 40 | while (kv_spool_read(sp,set,1) > 0) { 41 | switch(mode) { 42 | case mode_nop: continue; 43 | case mode_out: kv_set_dump(set,stdout); printf("\n"); break; 44 | default: assert(0); break; 45 | } 46 | } 47 | 48 | done: 49 | kv_spoolreader_free(sp); 50 | kv_set_free(set); 51 | return 0; 52 | } 53 | -------------------------------------------------------------------------------- /utils/kvsp-spw.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include "kvspool.h" 6 | 7 | int iterations=1; 8 | int iter_delay=10; 9 | int verbose=0; 10 | char *dir = NULL; 11 | 12 | void usage(char *exe) { 13 | fprintf(stderr,"usage: %s [-v] [-f] [-i iterations] [-d delay] \n", exe); 14 | exit(-1); 15 | } 16 | 17 | int main(int argc, char *argv[]) { 18 | 19 | int opt; 20 | char *exe = argv[0]; 21 | void *set; 22 | 23 | while ( (opt = getopt(argc, argv, "i:d:v+")) != -1) { 24 | switch (opt) { 25 | case 'v': verbose++; break; 26 | case 'i': iterations=atoi(optarg); break; 27 | case 'd': iter_delay=atoi(optarg); break; 28 | default: usage(exe); break; 29 | } 30 | } 31 | if (optind < argc) dir=argv[optind++]; 32 | else usage(exe); 33 | 34 | void *sp = kv_spoolwriter_new(dir); 35 | if (!sp) exit(-1); 36 | 37 | char timebuf[100], iterbuf[10]; 38 | 39 | while(iterations--) { 40 | time_t t = time(NULL); 41 | unsigned t32 = (unsigned)t; 42 | snprintf(timebuf,sizeof(timebuf),"%s",ctime(&t)); 43 | timebuf[strlen(timebuf)-1] = '\0'; /* trim \n */ 44 | snprintf(iterbuf,sizeof(iterbuf),"%d",iterations); 45 | 46 | set = kv_set_new(); 47 | kv_adds(set, "from", exe); 48 | /* kv_adds(set, "when", timebuf); */ 49 | kv_adds(set, "iter", iterbuf); 50 | 51 | /* put one of every kind of data */ 52 | kv_adds(set, "test_i8", iterbuf); 53 | kv_adds(set, "test_i16", iterbuf); 54 | kv_adds(set, "test_i32", iterbuf); 55 | kv_adds(set, "test_ipv4", "192.168.1.1"); 56 | kv_adds(set, "test1_ipv46", "fe80::20c:29ff:fe99:c21b"); 57 | kv_adds(set, "test2_ipv46", "ff02::1"); 58 | kv_adds(set, "test3_ipv46", "224.0.0.1"); 59 | kv_adds(set, "test_str", "hello"); 60 | kv_adds(set, "test_str8", "world!"); 61 | kv_adds(set, "test_d64", "3.14159"); 62 | kv_adds(set, "test_mac", "00:11:22:33:44:55:66"); 63 | 64 | kv_spool_write(sp,set); 65 | kv_set_free(set); 66 | if (iterations) sleep(iter_delay); 67 | } 68 | 69 | kv_spoolwriter_free(sp); 70 | return 0; 71 | } 72 | 73 | -------------------------------------------------------------------------------- /utils/kvsp-status.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include "kvspool.h" 19 | #include "kvspool_internal.h" 20 | #include "utarray.h" 21 | #include "utstring.h" 22 | 23 | int verbose; 24 | enum {console,network} mode = console; 25 | /* these are used in network mode to specify the udp destination */ 26 | in_addr_t addr; 27 | int port; 28 | char *iface; // used for multicast with explicitly specified nic 29 | int interval=10; /* udp report interval (seconds) */ 30 | char hostname[255]; /* used for udp reporting */ 31 | char *iface_addr_str = "0.0.0.0"; /* gets filled out with eth0's IP if they use the @eth0 multicast egress syntax */ 32 | 33 | void usage(char *prog) { 34 | fprintf(stderr, "display mode: %s [-v] spool ...\n", prog); 35 | fprintf(stderr, "network mode: %s -u udp://239.0.0.1:9999[@eth2] [-t 10] spool ...\n", prog); 36 | exit(-1); 37 | } 38 | 39 | static int parse_spec(char *spec) { 40 | char *proto = spec, *colon, *host, *at; 41 | struct hostent *h; 42 | int hlen, rc=-1; 43 | 44 | if (strncmp(proto, "udp://", 6)) goto done; 45 | host = &spec[6]; 46 | 47 | if ( !(colon = strrchr(spec, ':'))) goto done; 48 | port = atoi(colon+1); 49 | if ((port < 0) || (port > 65535)) goto done; 50 | 51 | if ( (at = strrchr(spec, '@')) != NULL) { // trailing @eth2 52 | iface = at+1; 53 | } 54 | 55 | /* dns lookup. */ 56 | *colon = '\0'; 57 | h = gethostbyname(host); 58 | hlen = strlen(host); 59 | *colon = ':'; 60 | if (!h) { 61 | fprintf(stderr, "lookup [%.*s]: %s", hlen, host, hstrerror(h_errno)); 62 | rc = -2; 63 | goto done; 64 | } 65 | 66 | addr = ((struct in_addr*)h->h_addr)->s_addr; 67 | rc = 0; /* success */ 68 | 69 | done: 70 | if (rc == -1) fprintf(stderr,"required format: udp://1.2.3.4:5678[@eth2]"); 71 | return rc; 72 | } 73 | 74 | void log_status(char*dir) { 75 | int fd=-1,sc,i; 76 | char **f, *file; 77 | time_t now, elapsed; 78 | UT_array *files; 79 | utarray_new(files,&ut_str_icd); 80 | struct stat sb; 81 | uint32_t sz, spsz; 82 | 83 | printf("%-40s ", dir); 84 | 85 | kv_stat_t stats; 86 | sc = kv_stat(dir,&stats); 87 | if (sc == -1) { 88 | fprintf(stderr,"kv_stat error in %s\n", dir); 89 | goto done; 90 | } 91 | UT_string *s; 92 | utstring_new(s); 93 | utstring_printf(s,"%3u%% ", stats.pct_consumed); 94 | long lz = stats.spool_sz; 95 | char *unit = ""; 96 | if (lz > 1024*1024*1024){unit="gb "; lz/=(1024*1024*1024); } 97 | else if (lz > 1024*1024) {unit="mb "; lz/=(1024*1024); } 98 | else if (lz > 1024) {unit="kb "; lz/=(1024); } 99 | else {unit="b "; } 100 | utstring_printf(s,"%10lu%s", (long)lz, unit); 101 | 102 | unit=""; 103 | now = time(NULL); 104 | elapsed = now - stats.last_write; 105 | if (elapsed > 60*60*24) {unit="days"; elapsed/=(60*60*24);} 106 | else if (elapsed > 60*60) {unit="hours"; elapsed/=(60*60); } 107 | else if (elapsed > 60) {unit="mins"; elapsed/=(60); } 108 | else if (elapsed >= 0) {unit="secs"; } 109 | if (stats.last_write == 0) utstring_printf(s,"%10snever",""); 110 | else utstring_printf(s,"%10lu%s", (long)elapsed, unit); 111 | printf("%s\n", utstring_body(s)); 112 | utstring_free(s); 113 | 114 | done: 115 | utarray_free(files); 116 | } 117 | 118 | void udp_status(UT_array *dirs) { 119 | int rc; 120 | kv_stat_t stats; 121 | char **dir; 122 | 123 | UT_string *s; 124 | utstring_new(s); 125 | utstring_printf(s,"kvsp-status %s %s\n", hostname, iface_addr_str); 126 | 127 | dir=NULL; 128 | while ( (dir=(char**)utarray_next(dirs,dir))) { 129 | rc = kv_stat(*dir,&stats); 130 | if (rc == -1) { 131 | fprintf(stderr,"kv_stat error in %s\n", *dir); 132 | goto done; 133 | } 134 | long lz = stats.spool_sz; 135 | utstring_printf(s,"%s ", *dir); 136 | utstring_printf(s,"%u ", stats.pct_consumed); 137 | utstring_printf(s,"%lu ", (long)lz); 138 | time_t now = time(NULL); 139 | time_t elapsed = now - stats.last_write; 140 | if (stats.last_write == 0) utstring_printf(s,"-1\n"); 141 | else utstring_printf(s,"%lu\n", (long)elapsed); 142 | } 143 | char *buf = utstring_body(s); 144 | int len = utstring_len(s); 145 | 146 | 147 | /********************************************************** 148 | * create an IPv4/UDP socket, not yet bound to any address 149 | *********************************************************/ 150 | int fd = socket(AF_INET, SOCK_DGRAM, 0); 151 | if (fd == -1) { 152 | fprintf(stderr,"socket: %s\n", strerror(errno)); 153 | goto done; 154 | } 155 | 156 | /* use a specific NIC if one was specified, supported here for multicast */ 157 | if (iface) { 158 | int l = strlen(iface); 159 | if (l+1 >IFNAMSIZ) {fprintf(stderr,"interface too long\n"); goto done;} 160 | 161 | struct ifreq ifr; 162 | ifr.ifr_addr.sa_family = AF_INET; 163 | memcpy(ifr.ifr_name, iface, l+1); 164 | 165 | /* does this interface support multicast? */ 166 | if (ioctl(fd, SIOCGIFFLAGS, &ifr)) {fprintf(stderr,"ioctl: %s\n", strerror(errno)); goto done;} 167 | if (!(ifr.ifr_flags & IFF_MULTICAST)) {fprintf(stderr,"%s does not multicast\n",iface); goto done;} 168 | 169 | /* get the interface IP address */ 170 | struct in_addr iface_addr; 171 | if (ioctl(fd, SIOCGIFADDR, &ifr)) {fprintf(stderr,"ioctl: %s\n", strerror(errno)); goto done;} 172 | iface_addr = (((struct sockaddr_in *)&ifr.ifr_addr)->sin_addr); 173 | char *volatile_str = inet_ntoa(iface_addr); 174 | iface_addr_str = strdup(volatile_str); 175 | 176 | /* ask kernel to use its IP address for outgoing multicast */ 177 | if (setsockopt(fd, IPPROTO_IP, IP_MULTICAST_IF, &iface_addr, sizeof(iface_addr))) { 178 | fprintf(stderr,"setsockopt: %s\n", strerror(errno)); 179 | goto done; 180 | } 181 | } 182 | 183 | /********************************************************** 184 | * internet socket address structure, for the remote side 185 | *********************************************************/ 186 | struct sockaddr_in sin; 187 | sin.sin_family = AF_INET; 188 | sin.sin_addr.s_addr = addr; 189 | sin.sin_port = htons(port); 190 | 191 | if (sin.sin_addr.s_addr == INADDR_NONE) { 192 | fprintf(stderr,"invalid remote IP\n"); 193 | goto done; 194 | } 195 | 196 | /********************************************************** 197 | * UDP is connectionless; connect only sets dest for writes 198 | *********************************************************/ 199 | if (connect(fd, (struct sockaddr*)&sin, sizeof(sin)) == -1) { 200 | fprintf(stderr,"connect: %s\n", strerror(errno)); 201 | goto done; 202 | } 203 | 204 | if (verbose) { 205 | fprintf(stderr,"udp packet:\n%.*s\n", len, buf); 206 | } 207 | 208 | if ( (rc=write(fd,buf,len)) != len) { 209 | fprintf(stderr,"write: %s\n", (rc<0)?strerror(errno):"incomplete"); 210 | goto done; 211 | } 212 | 213 | done: 214 | if (fd != -1) close(fd); 215 | utstring_free(s); 216 | return; 217 | } 218 | 219 | 220 | int main(int argc, char * argv[]) { 221 | char **dir; 222 | int i, opt; 223 | UT_array *dirs; 224 | utarray_new(dirs,&ut_str_icd); 225 | 226 | while ( (opt = getopt(argc, argv, "v+u:t:h")) != -1) { 227 | switch (opt) { 228 | case 'v': verbose++; break; 229 | case 'u': mode=network; parse_spec(optarg); break; 230 | case 't': interval = atoi(optarg); break; 231 | case 'h': default: usage(argv[0]); break; 232 | } 233 | } 234 | if (optind >= argc) usage(argv[0]); 235 | while(optind < argc) utarray_push_back(dirs, &argv[optind++]); 236 | gethostname(hostname,sizeof(hostname)); 237 | 238 | switch(mode) { 239 | case console: 240 | dir=NULL; 241 | while ( ( dir=(char**)utarray_next(dirs,dir))) log_status(*dir); 242 | break; 243 | case network: 244 | while(1) { 245 | udp_status(dirs); 246 | sleep(interval); 247 | } 248 | break; 249 | } 250 | 251 | done: 252 | utarray_free(dirs); 253 | return 0; 254 | } 255 | 256 | -------------------------------------------------------------------------------- /utils/kvsp-sub.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include "kvspool_internal.h" 10 | #include "utstring.h" 11 | #include "uthash.h" 12 | #include "utarray.h" 13 | 14 | void *sp; 15 | 16 | int verbose; 17 | int pull_mode; 18 | char *dir; 19 | char *remotes_file; 20 | void *context; 21 | void *socket; 22 | 23 | 24 | void usage(char *exe) { 25 | fprintf(stderr,"usage: %s [-v] [-s] -d [-f file | ...]\n", exe); 26 | fprintf(stderr," -s runs in push-pull mode instead of lossy pub-sub\n"); 27 | exit(-1); 28 | } 29 | 30 | int read_lines(char *file, UT_array *lines) { 31 | char line[200]; 32 | int rc = -1; 33 | char *c; 34 | FILE *f = fopen(file,"r"); 35 | if (f==NULL) { 36 | fprintf(stderr,"fopen %s: %s\n", file, strerror(errno)); 37 | goto done; 38 | } 39 | while (fgets(line,sizeof(line),f) != NULL) { 40 | for(c=line; (c < line+sizeof(line)) && (*c != '\0'); c++) { 41 | if (*c == '\n') *c='\0'; 42 | if (*c == ' ') *c='\0'; 43 | } 44 | c = line; 45 | if (strlen(c) == 0) continue; 46 | utarray_push_back(lines,&c); 47 | } 48 | rc = 0; 49 | 50 | done: 51 | if (f) fclose(f); 52 | return rc; 53 | } 54 | 55 | #if ZMQ_VERSION_MAJOR == 2 56 | #define zmq_sendmsg zmq_send 57 | #define zmq_recvmsg zmq_recv 58 | #define zmq_rcvmore_t int64_t 59 | #else 60 | #define zmq_rcvmore_t int 61 | #endif 62 | 63 | int json_to_frame(void *sp, void *set, void *msg_data, size_t msg_len) { 64 | int rc=-1; 65 | json_t *json; 66 | json_error_t error; 67 | const char *key; 68 | json_t *value; 69 | 70 | json = json_loadb(msg_data, msg_len, 0, &error); 71 | if (!json) { 72 | fprintf(stderr,"JSON decoding error: %s\n", error.text); 73 | goto done; 74 | } 75 | 76 | kv_set_clear(set); 77 | json_object_foreach(json, key, value) { 78 | kv_adds(set, key, json_string_value(value)); 79 | } 80 | kv_spool_write(sp, set); 81 | 82 | rc = 0; 83 | 84 | done: 85 | if (json) json_decref(json); 86 | return rc; 87 | } 88 | 89 | int main(int argc, char *argv[]) { 90 | 91 | zmq_rcvmore_t more; size_t more_sz = sizeof(more); 92 | char *exe = argv[0], *filter = ""; 93 | int part_num,opt,rc=-1; 94 | void *msg_data, *sp, *set=NULL; 95 | char **endpoint; 96 | UT_array *endpoints; 97 | size_t msg_len; 98 | zmq_msg_t part; 99 | 100 | utarray_new(endpoints,&ut_str_icd); 101 | 102 | while ( (opt = getopt(argc, argv, "sd:f:v+")) != -1) { 103 | switch (opt) { 104 | case 'v': verbose++; break; 105 | case 's': pull_mode++; break; 106 | case 'd': dir=strdup(optarg); break; 107 | case 'f': remotes_file=strdup(optarg); break; 108 | default: usage(exe); break; 109 | } 110 | } 111 | if (!dir) usage(exe); 112 | 113 | sp = kv_spoolwriter_new(dir); 114 | if (!sp) usage(exe); 115 | set = kv_set_new(); 116 | 117 | /* connect socket to each publisher. yes, zeromq lets you connect n times */ 118 | if ( !(context = zmq_init(1))) goto done; 119 | if ( !(socket = zmq_socket(context, pull_mode?ZMQ_PULL:ZMQ_SUB))) goto done; 120 | if (remotes_file) if (read_lines(remotes_file,endpoints)) goto done; 121 | while (optind < argc) utarray_push_back(endpoints,&argv[optind++]); 122 | endpoint=NULL; 123 | if (utarray_len(endpoints) == 0) usage(argv[0]); 124 | while ( (endpoint=(char**)utarray_next(endpoints,endpoint))) { 125 | if (verbose) fprintf(stderr,"connecting to %s\n", *endpoint); 126 | if (zmq_connect(socket, *endpoint)) goto done; 127 | } 128 | if (!pull_mode) { 129 | if (zmq_setsockopt(socket, ZMQ_SUBSCRIBE, filter, strlen(filter))) goto done; 130 | } 131 | 132 | while(1) { 133 | 134 | /* receive a multi-part message */ 135 | part_num=1; 136 | do { 137 | if ( (rc= zmq_msg_init(&part))) goto done; 138 | if ( ((rc= zmq_recvmsg(socket, &part, 0)) == -1) || 139 | ((rc= zmq_getsockopt(socket, ZMQ_RCVMORE, &more,&more_sz)) != 0)) { 140 | zmq_msg_close(&part); 141 | goto done; 142 | } 143 | 144 | msg_data = zmq_msg_data(&part); 145 | msg_len = zmq_msg_size(&part); 146 | 147 | switch(part_num) { /* part 1 has serialized frame */ 148 | case 1: if (json_to_frame(sp,set,msg_data,msg_len)) goto done; break; 149 | default: assert(0); 150 | } 151 | 152 | zmq_msg_close(&part); 153 | part_num++; 154 | } while(more); 155 | } 156 | rc = 0; /* not reached TODO under clean shutdown on signal */ 157 | 158 | 159 | done: 160 | if (rc) fprintf(stderr,"%s: %s\n", exe, zmq_strerror(errno)); 161 | if(socket) zmq_close(socket); 162 | if(context) zmq_term(context); 163 | kv_spoolwriter_free(sp); 164 | if (set) kv_set_free(set); 165 | utarray_free(endpoints); 166 | return rc; 167 | } 168 | 169 | -------------------------------------------------------------------------------- /utils/kvsp-tee.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include "kvspool_internal.h" 9 | #include "utarray.h" 10 | 11 | typedef struct { 12 | char *dir; 13 | void *sp; 14 | } ospool_t; 15 | 16 | /* plumbing for ospool_t array */ 17 | void ospool_ini(void *_osp) { 18 | ospool_t *osp = (ospool_t*)_osp; 19 | osp->dir = NULL; 20 | osp->sp = NULL; 21 | } 22 | void ospool_cpy(void *_dst, const void *_src) { 23 | ospool_t *dst = (ospool_t*)_dst; 24 | ospool_t *src = (ospool_t*)_src; 25 | dst->dir = src->dir ? strdup(src->dir) : NULL; 26 | dst->sp = src->sp; 27 | } 28 | void ospool_fin(void *_osp) { 29 | ospool_t *osp = (ospool_t*)_osp; 30 | if (osp->dir) free(osp->dir); 31 | } 32 | 33 | UT_icd ospool_icd = {sizeof(ospool_t),ospool_ini,ospool_cpy,ospool_fin}; 34 | 35 | void usage(char *prog) { 36 | fprintf(stderr, "usage: %s [-v] [-k key -r regex] -s spool ...\n", prog); 37 | exit(-1); 38 | } 39 | 40 | #define OVECSZ 30 /* must be multiple of 3 */ 41 | int keep_record(void *set, char *key, pcre *re) { 42 | int rc, ovec[OVECSZ]; 43 | if (!key) return 1; 44 | if (!re) return 1; 45 | 46 | /* does set contain key */ 47 | kv_t *kv = kv_get(set, key); 48 | if (kv == NULL) return 0; 49 | 50 | /* does value of key match regex */ 51 | rc = pcre_exec(re, NULL, kv->val, kv->vlen, 0, 0, ovec, OVECSZ); 52 | return (rc > 0) ? 1 : 0; 53 | } 54 | 55 | int main(int argc, char * argv[]) { 56 | char *key=NULL, *regex=NULL; 57 | pcre *re; 58 | int opt,verbose=0,raw=0; 59 | ospool_t *osp; 60 | void *set; 61 | /* input spool */ 62 | char *dir=NULL; 63 | void *sp; 64 | 65 | set = kv_set_new(); 66 | UT_array *ospoolv; 67 | utarray_new(ospoolv, &ospool_icd); 68 | 69 | while ( (opt = getopt(argc, argv, "v+s:k:r:W")) != -1) { 70 | switch (opt) { 71 | case 'v': verbose++; break; 72 | case 's': dir = strdup(optarg); break; 73 | case 'k': key = strdup(optarg); break; 74 | case 'r': regex = strdup(optarg); break; 75 | case 'W': raw = 1; break; 76 | default: usage(argv[0]); break; 77 | } 78 | } 79 | if (raw) fprintf(stderr, "-W (raw mode) is deprecated, using regular mode\n"); 80 | if (dir==NULL) usage(argv[0]); 81 | if (key && regex) { 82 | const char *err; 83 | int off; 84 | re = pcre_compile(regex, 0, &err, &off, NULL); 85 | if (!re) { 86 | fprintf(stderr, "error in regex\n"); 87 | goto done; 88 | } 89 | } 90 | sp = kv_spoolreader_new(dir); 91 | if (sp == NULL) { 92 | fprintf(stderr, "failed to open input spool %s\n", dir); 93 | goto done; 94 | } 95 | 96 | while (optind < argc) { 97 | utarray_extend_back(ospoolv); 98 | osp = (ospool_t*)utarray_back(ospoolv); 99 | osp->dir = strdup(argv[optind++]); 100 | } 101 | 102 | while (kv_spool_read(sp,set,1) == 1) { 103 | if (!keep_record(set,key,re)) continue; 104 | osp=NULL; 105 | while ( (osp=(ospool_t*)utarray_next(ospoolv,osp))) { 106 | if (osp->sp ==NULL) { /* do lazy open */ 107 | osp->sp = kv_spoolwriter_new(osp->dir); 108 | if (!osp->sp) { 109 | fprintf(stderr, "failed to open output spool %s\n",dir); 110 | goto done; 111 | } 112 | } 113 | kv_spool_write(osp->sp,set); 114 | } 115 | } 116 | 117 | done: 118 | kv_set_free(set); 119 | osp=NULL; 120 | /* free the spoolwriter handles */ 121 | while ( (osp=(ospool_t*)utarray_next(ospoolv,osp))) { 122 | kv_spoolwriter_free(osp->sp); 123 | } 124 | utarray_free(ospoolv); 125 | return 0; 126 | } 127 | 128 | -------------------------------------------------------------------------------- /utils/kvsp-tpub.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include "kvspool_internal.h" 17 | #include "kvsp-bconfig.h" 18 | #include "ringbuf.h" 19 | 20 | /* 21 | * publish spool over TCP in binary 22 | */ 23 | 24 | #define BATCH_FRAMES 10000 25 | #define OUTPUT_BUFSZ (10 * 1024 * 1024) 26 | #define OUTPUT_CUSHION (0.2 * OUTPUT_BUFSZ) 27 | 28 | struct { 29 | char *prog; 30 | enum {mode_pub } mode; 31 | int verbose; 32 | int epoll_fd; /* epoll descriptor */ 33 | uint32_t events; /* epoll event status */ 34 | int signal_fd; /* to receive signals */ 35 | int listen_fd; /* listening tcp socket */ 36 | int client_fd; /* connected tcp socket */ 37 | in_addr_t addr; /* IP address to listen on */ 38 | int port; /* TCP port to listen on */ 39 | char *spool; /* spool file name */ 40 | void *sp; /* spool handle */ 41 | int spool_fd; /* spool descriptor */ 42 | char *cast; /* cast file name */ 43 | void *set; /* kvspool set */ 44 | UT_string *tmp; /* scratch area */ 45 | ringbuf *rb; /* pending output */ 46 | void *setv[BATCH_FRAMES]; /* bulk set array */ 47 | } cfg = { 48 | .addr = INADDR_ANY, /* by default, listen on all local IP's */ 49 | .port = 1919, /* arbitrary */ 50 | .epoll_fd = -1, 51 | .signal_fd = -1, 52 | .listen_fd = -1, 53 | .spool_fd = -1, 54 | .client_fd = -1, 55 | }; 56 | 57 | /* signals that we'll accept via signalfd in epoll */ 58 | int sigs[] = {SIGHUP,SIGTERM,SIGINT,SIGQUIT,SIGALRM}; 59 | 60 | void usage() { 61 | fprintf(stderr,"usage: %s [options] \n", cfg.prog); 62 | fprintf(stderr,"options:\n" 63 | " -p (TCP port to listen on)\n" 64 | " -d (spool directory to read)\n" 65 | " -b (cast config file)\n" 66 | " -v (verbose)\n" 67 | " -h (this help)\n" 68 | "\n"); 69 | exit(-1); 70 | } 71 | 72 | int add_epoll(int events, int fd) { 73 | int rc; 74 | struct epoll_event ev; 75 | memset(&ev,0,sizeof(ev)); // placate valgrind 76 | ev.events = events; 77 | ev.data.fd= fd; 78 | if (cfg.verbose) fprintf(stderr,"adding fd %d events %d\n", fd, events); 79 | rc = epoll_ctl(cfg.epoll_fd, EPOLL_CTL_ADD, fd, &ev); 80 | if (rc == -1) { 81 | fprintf(stderr,"epoll_ctl: %s\n", strerror(errno)); 82 | } 83 | return rc; 84 | } 85 | 86 | int mod_epoll(int events, int fd) { 87 | int rc; 88 | struct epoll_event ev; 89 | memset(&ev,0,sizeof(ev)); // placate valgrind 90 | ev.events = events; 91 | ev.data.fd= fd; 92 | if (cfg.verbose) fprintf(stderr,"modding fd %d events %d\n", fd, events); 93 | rc = epoll_ctl(cfg.epoll_fd, EPOLL_CTL_MOD, fd, &ev); 94 | if (rc == -1) { 95 | fprintf(stderr,"epoll_ctl: %s\n", strerror(errno)); 96 | } 97 | return rc; 98 | } 99 | 100 | int del_epoll(int fd) { 101 | int rc; 102 | struct epoll_event ev; 103 | rc = epoll_ctl(cfg.epoll_fd, EPOLL_CTL_DEL, fd, &ev); 104 | if (rc == -1) { 105 | fprintf(stderr,"epoll_ctl: %s\n", strerror(errno)); 106 | } 107 | return rc; 108 | } 109 | 110 | /* work we do at 1hz */ 111 | int periodic_work(void) { 112 | int rc = -1; 113 | 114 | rc = 0; 115 | 116 | done: 117 | return rc; 118 | } 119 | 120 | int handle_signal(void) { 121 | int rc=-1; 122 | struct signalfd_siginfo info; 123 | 124 | if (read(cfg.signal_fd, &info, sizeof(info)) != sizeof(info)) { 125 | fprintf(stderr,"failed to read signal fd buffer\n"); 126 | goto done; 127 | } 128 | 129 | switch(info.ssi_signo) { 130 | case SIGALRM: 131 | if (periodic_work() < 0) goto done; 132 | alarm(1); 133 | break; 134 | default: 135 | fprintf(stderr,"got signal %d\n", info.ssi_signo); 136 | goto done; 137 | break; 138 | } 139 | 140 | rc = 0; 141 | 142 | done: 143 | return rc; 144 | } 145 | 146 | int setup_listener() { 147 | int rc = -1, one=1; 148 | 149 | int fd = socket(AF_INET, SOCK_STREAM, 0); 150 | if (fd == -1) { 151 | fprintf(stderr,"socket: %s\n", strerror(errno)); 152 | goto done; 153 | } 154 | 155 | /********************************************************** 156 | * internet socket address structure: our address and port 157 | *********************************************************/ 158 | struct sockaddr_in sin; 159 | sin.sin_family = AF_INET; 160 | sin.sin_addr.s_addr = cfg.addr; 161 | sin.sin_port = htons(cfg.port); 162 | 163 | /********************************************************** 164 | * bind socket to address and port 165 | *********************************************************/ 166 | setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one)); 167 | if (bind(fd, (struct sockaddr*)&sin, sizeof(sin)) == -1) { 168 | fprintf(stderr,"bind: %s\n", strerror(errno)); 169 | goto done; 170 | } 171 | 172 | /********************************************************** 173 | * put socket into listening state 174 | *********************************************************/ 175 | if (listen(fd,1) == -1) { 176 | fprintf(stderr,"listen: %s\n", strerror(errno)); 177 | goto done; 178 | } 179 | 180 | if (add_epoll(EPOLLIN, fd)) goto done; 181 | cfg.listen_fd = fd; 182 | rc=0; 183 | 184 | done: 185 | if ((rc < 0) && (fd != -1)) close(fd); 186 | return rc; 187 | } 188 | 189 | /* accept a new client connection to the listening socket */ 190 | int accept_client() { 191 | int fd=-1, rc=-1; 192 | struct sockaddr_in in; 193 | socklen_t sz = sizeof(in); 194 | 195 | fd = accept(cfg.listen_fd,(struct sockaddr*)&in, &sz); 196 | if (fd == -1) { 197 | fprintf(stderr,"accept: %s\n", strerror(errno)); 198 | goto done; 199 | } 200 | 201 | if (cfg.verbose && (sizeof(in)==sz)) { 202 | fprintf(stderr,"connection fd %d from %s:%d\n", fd, 203 | inet_ntoa(in.sin_addr), (int)ntohs(in.sin_port)); 204 | } 205 | 206 | if (cfg.client_fd != -1) { /* already have a client? */ 207 | fprintf(stderr,"refusing client\n"); 208 | close(fd); 209 | rc = 0; 210 | goto done; 211 | } 212 | 213 | cfg.client_fd = fd; 214 | 215 | /* epoll on both the spool and the client */ 216 | if (add_epoll(EPOLLIN, cfg.client_fd) < 0) goto done; 217 | mod_epoll(EPOLLIN, cfg.spool_fd); 218 | 219 | rc = 0; 220 | 221 | done: 222 | return rc; 223 | } 224 | 225 | int handle_spool(void) { 226 | int rc = -1, sc, i=0; 227 | char *buf; 228 | size_t len; 229 | int fl, nset; 230 | 231 | /* suspend spool reading if output buffer < 20% free */ 232 | if (ringbuf_get_freespace(cfg.rb) < OUTPUT_CUSHION) { 233 | mod_epoll(0, cfg.spool_fd); 234 | rc = 0; 235 | goto done; 236 | } 237 | 238 | nset = BATCH_FRAMES; 239 | 240 | sc = kv_spool_readN(cfg.sp, cfg.setv, &nset); 241 | if (sc < 0) { 242 | fprintf(stderr, "kv_spool_readN: error\n"); 243 | goto done; 244 | } 245 | 246 | if (cfg.verbose) fprintf(stderr,"%d sets\n", nset); 247 | 248 | for(i=0; i < nset; i++) { 249 | 250 | sc = set_to_binary(cfg.setv[i], cfg.tmp); 251 | if (sc < 0) goto done; 252 | 253 | buf = utstring_body(cfg.tmp); 254 | len = utstring_len(cfg.tmp); 255 | sc = ringbuf_put(cfg.rb, buf, len); 256 | if (sc < 0) { 257 | /* unexpected; we checked it was 20% free */ 258 | fprintf(stderr, "buffer exhausted\n"); 259 | goto done; 260 | } 261 | } 262 | 263 | fl = EPOLLIN | (ringbuf_get_pending_size(cfg.rb) ? EPOLLOUT : 0); 264 | mod_epoll(fl, cfg.client_fd); 265 | rc = 0; 266 | 267 | done: 268 | return rc; 269 | } 270 | 271 | void close_client(void) { 272 | ringbuf_clear(cfg.rb); 273 | mod_epoll(0,cfg.spool_fd); /* ignore spool til new client */ 274 | close(cfg.client_fd); /* close removes client epoll */ 275 | cfg.client_fd = -1; 276 | cfg.events = 0; 277 | } 278 | 279 | void drain_client(void) { 280 | char buf[1024]; 281 | ssize_t nr; 282 | 283 | nr = read(cfg.client_fd, buf, sizeof(buf)); 284 | if(nr > 0) { 285 | if (cfg.verbose) fprintf(stderr,"client: %lu bytes\n", (long unsigned)nr); 286 | return; 287 | } 288 | 289 | /* disconnect or socket error are handled the same - close it */ 290 | assert(nr <= 0); 291 | fprintf(stderr,"client: %s\n", nr ? strerror(errno) : "closed"); 292 | close_client(); 293 | } 294 | 295 | void send_client(void) { 296 | size_t nr, wr; 297 | char *buf; 298 | int fl; 299 | 300 | nr = ringbuf_get_next_chunk(cfg.rb, &buf); 301 | assert(nr > 0); 302 | 303 | wr = write(cfg.client_fd, buf, nr); 304 | if (wr < 0) { 305 | fprintf(stderr, "write: %s\n", strerror(errno)); 306 | close_client(); 307 | return; 308 | } 309 | 310 | ringbuf_mark_consumed(cfg.rb, wr); 311 | 312 | /* adjust epoll on client based on we have more output to send */ 313 | fl = EPOLLIN | (ringbuf_get_pending_size(cfg.rb) ? EPOLLOUT : 0); 314 | mod_epoll(fl, cfg.client_fd); 315 | 316 | /* reinstate/retain spool reads if output buffer is > 20% free */ 317 | if (ringbuf_get_freespace(cfg.rb) > OUTPUT_CUSHION) { 318 | mod_epoll(EPOLLIN, cfg.spool_fd); 319 | } 320 | } 321 | 322 | int handle_client() { 323 | int rc = -1; 324 | assert(cfg.client_fd != -1); 325 | 326 | if (cfg.events & EPOLLIN) drain_client(); 327 | if (cfg.events & EPOLLOUT) send_client(); 328 | 329 | rc = 0; 330 | return rc; 331 | } 332 | 333 | int main(int argc, char *argv[]) { 334 | int opt, rc=-1, n, ec, i; 335 | struct epoll_event ev; 336 | cfg.prog = argv[0]; 337 | char unit, *c, buf[100]; 338 | ssize_t nr; 339 | 340 | utarray_new(output_keys, &ut_str_icd); 341 | utarray_new(output_defaults, &ut_str_icd); 342 | utarray_new(output_types,&ut_int_icd); 343 | cfg.set = kv_set_new(); 344 | for(i=0; i < BATCH_FRAMES; i++) cfg.setv[i] = kv_set_new(); 345 | utstring_new(cfg.tmp); 346 | cfg.rb = ringbuf_new(OUTPUT_BUFSZ); 347 | if (cfg.rb == NULL) goto done; 348 | 349 | while ( (opt = getopt(argc,argv,"vhp:d:b:")) > 0) { 350 | switch(opt) { 351 | case 'v': cfg.verbose++; break; 352 | case 'h': default: usage(); break; 353 | case 'p': cfg.port = atoi(optarg); break; 354 | case 'd': cfg.spool = strdup(optarg); break; 355 | case 'b': cfg.cast = strdup(optarg); break; 356 | } 357 | } 358 | 359 | if (cfg.spool == NULL) usage(); 360 | if (cfg.cast == NULL) usage(); 361 | 362 | if (parse_config(cfg.cast) < 0) goto done; 363 | cfg.sp = kv_spoolreader_new_nb(cfg.spool, &cfg.spool_fd); 364 | if (cfg.sp == NULL) goto done; 365 | 366 | /* block all signals. we accept signals via signal_fd */ 367 | sigset_t all; 368 | sigfillset(&all); 369 | sigprocmask(SIG_SETMASK,&all,NULL); 370 | 371 | /* a few signals we'll accept via our signalfd */ 372 | sigset_t sw; 373 | sigemptyset(&sw); 374 | for(n=0; n < sizeof(sigs)/sizeof(*sigs); n++) sigaddset(&sw, sigs[n]); 375 | 376 | /* create the signalfd for receiving signals */ 377 | cfg.signal_fd = signalfd(-1, &sw, 0); 378 | if (cfg.signal_fd == -1) { 379 | fprintf(stderr,"signalfd: %s\n", strerror(errno)); 380 | goto done; 381 | } 382 | 383 | /* set up the epoll instance */ 384 | cfg.epoll_fd = epoll_create(1); 385 | if (cfg.epoll_fd == -1) { 386 | fprintf(stderr,"epoll: %s\n", strerror(errno)); 387 | goto done; 388 | } 389 | 390 | /* add descriptors of interest. idle spool til connect */ 391 | if (add_epoll(EPOLLIN, cfg.signal_fd)) goto done; 392 | if (add_epoll(0, cfg.spool_fd) < 0) goto done; 393 | 394 | if (setup_listener() < 0) goto done; 395 | 396 | alarm(1); 397 | 398 | while (1) { 399 | ec = epoll_wait(cfg.epoll_fd, &ev, 1, -1); 400 | if (ec < 0) { 401 | fprintf(stderr, "epoll: %s\n", strerror(errno)); 402 | goto done; 403 | } 404 | 405 | cfg.events = ev.events; 406 | 407 | if (ec == 0) { assert(0); goto done; } 408 | else if (ev.data.fd == cfg.signal_fd) { if (handle_signal() < 0) goto done; } 409 | else if (ev.data.fd == cfg.listen_fd) { if (accept_client() < 0) goto done; } 410 | else if (ev.data.fd == cfg.client_fd) { if (handle_client() < 0) goto done; } 411 | else if (ev.data.fd == cfg.spool_fd) { if (handle_spool() < 0) goto done; } 412 | else { assert(0); goto done; } 413 | } 414 | 415 | rc = 0; 416 | 417 | done: 418 | if (cfg.signal_fd != -1) close(cfg.signal_fd); 419 | if (cfg.epoll_fd != -1) close(cfg.epoll_fd); 420 | if (cfg.listen_fd != -1) close(cfg.listen_fd); 421 | if (cfg.client_fd != -1) close(cfg.client_fd); 422 | if (cfg.sp) kv_spoolreader_free(cfg.sp); 423 | kv_set_free(cfg.set); 424 | for(i=0; i < BATCH_FRAMES; i++) kv_set_free(cfg.setv[i]); 425 | utstring_free(cfg.tmp); 426 | if (cfg.rb) ringbuf_free(cfg.rb); 427 | return 0; 428 | } 429 | -------------------------------------------------------------------------------- /utils/kvsp-tsub.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include "shr.h" 18 | #include "utstring.h" 19 | #include "kvspool_internal.h" 20 | #include "kvsp-bconfig.h" 21 | 22 | /* 23 | * kvsp-tsub 24 | * 25 | * connect to TCP host 26 | * read binary frames 27 | * reverse to kv set 28 | * write to local spool 29 | * 30 | */ 31 | 32 | #define MAX_FRAME (1024*1024) 33 | #define BUFSZ (MAX_FRAME * 10) 34 | struct { 35 | char *prog; 36 | int verbose; 37 | int epoll_fd; /* epoll descriptor */ 38 | int signal_fd; /* to receive signals */ 39 | int client_fd; /* connected tcp socket */ 40 | char *host; /* host to connect to */ 41 | int port; /* TCP port to connect to */ 42 | char *cast; /* cast config file name */ 43 | char *spool; /* spool file name */ 44 | void *sp; /* spool handle */ 45 | void *set; /* set handle */ 46 | UT_string *tmp; /* temp buffer */ 47 | char buf[BUFSZ]; /* temp receive buffer */ 48 | size_t bsz; /* bytes ready in buf */ 49 | } cfg = { 50 | .host = "127.0.0.1", 51 | .epoll_fd = -1, 52 | .signal_fd = -1, 53 | .client_fd = -1, 54 | }; 55 | 56 | /* signals that we'll accept via signalfd in epoll */ 57 | int sigs[] = {SIGHUP,SIGTERM,SIGINT,SIGQUIT,SIGALRM}; 58 | 59 | void usage() { 60 | fprintf(stderr,"usage: %s [options]\n", cfg.prog); 61 | fprintf(stderr,"required flags:\n" 62 | " -s (hostname)\n" 63 | " -p (port)\n" 64 | " -b (cast config)\n" 65 | " -d (spool dir)\n" 66 | "other options:\n" 67 | " -v (verbose)\n" 68 | " -h (this help)\n" 69 | "\n"); 70 | exit(-1); 71 | } 72 | 73 | int connect_up(void) { 74 | int rc = -1, fd = -1; 75 | 76 | fd = socket(AF_INET, SOCK_STREAM, 0); 77 | if (fd == -1) { 78 | fprintf(stderr,"socket: %s\n", strerror(errno)); 79 | goto done; 80 | } 81 | 82 | struct hostent *h=gethostbyname(cfg.host); 83 | if (!h) { 84 | fprintf(stderr,"cannot resolve name: %s\n", hstrerror(h_errno)); 85 | goto done; 86 | } 87 | 88 | struct sockaddr_in sin; 89 | sin.sin_family = AF_INET; 90 | sin.sin_addr.s_addr = ((struct in_addr*)h->h_addr)->s_addr; 91 | sin.sin_port = htons(cfg.port); 92 | 93 | if (connect(fd, (struct sockaddr*)&sin, sizeof(sin)) == -1) { 94 | fprintf(stderr,"connect: %s\n", strerror(errno)); 95 | goto done; 96 | } 97 | 98 | cfg.client_fd = fd; 99 | rc = 0; 100 | 101 | done: 102 | if ((rc < 0) && (fd != -1)) close(fd); 103 | return rc; 104 | } 105 | 106 | int add_epoll(int events, int fd) { 107 | int rc; 108 | struct epoll_event ev; 109 | memset(&ev,0,sizeof(ev)); // placate valgrind 110 | ev.events = events; 111 | ev.data.fd= fd; 112 | if (cfg.verbose) fprintf(stderr,"adding fd %d to epoll\n", fd); 113 | rc = epoll_ctl(cfg.epoll_fd, EPOLL_CTL_ADD, fd, &ev); 114 | if (rc == -1) { 115 | fprintf(stderr,"epoll_ctl: %s\n", strerror(errno)); 116 | } 117 | return rc; 118 | } 119 | 120 | int del_epoll(int fd) { 121 | int rc; 122 | struct epoll_event ev; 123 | rc = epoll_ctl(cfg.epoll_fd, EPOLL_CTL_DEL, fd, &ev); 124 | if (rc == -1) { 125 | fprintf(stderr,"epoll_ctl: %s\n", strerror(errno)); 126 | } 127 | return rc; 128 | } 129 | 130 | /* work we do at 1hz */ 131 | int periodic_work(void) { 132 | int rc = -1; 133 | 134 | rc = 0; 135 | 136 | done: 137 | return rc; 138 | } 139 | 140 | int handle_signal(void) { 141 | int rc=-1; 142 | struct signalfd_siginfo info; 143 | 144 | if (read(cfg.signal_fd, &info, sizeof(info)) != sizeof(info)) { 145 | fprintf(stderr,"failed to read signal fd buffer\n"); 146 | goto done; 147 | } 148 | 149 | switch(info.ssi_signo) { 150 | case SIGALRM: 151 | if (periodic_work() < 0) goto done; 152 | alarm(1); 153 | break; 154 | default: 155 | fprintf(stderr,"got signal %d\n", info.ssi_signo); 156 | goto done; 157 | break; 158 | } 159 | 160 | rc = 0; 161 | 162 | done: 163 | return rc; 164 | } 165 | 166 | /* 167 | * given a buffer of N frames 168 | * with a possible partial final frame 169 | * decode them according to the cast config 170 | * saving the last frame prefix if partial 171 | */ 172 | int decode_frames(void) { 173 | char *c, *body, *eob; 174 | uint32_t blen; 175 | size_t remsz; 176 | int rc = -1; 177 | 178 | eob = cfg.buf + cfg.bsz; 179 | c = cfg.buf; 180 | while(1) { 181 | if (c + sizeof(uint32_t) > eob) break; 182 | memcpy(&blen, c, sizeof(uint32_t)); 183 | if (blen > MAX_FRAME) goto done; 184 | body = c + sizeof(uint32_t); 185 | if (body + blen > eob) break; 186 | if (binary_to_frame(cfg.sp, cfg.set, body, blen, cfg.tmp) < 0) goto done; 187 | c += sizeof(uint32_t) + blen; 188 | } 189 | 190 | /* if buffer ends with partial frame, save it */ 191 | if (c < eob) memmove(cfg.buf, c, eob - c); 192 | cfg.bsz = eob - c; 193 | 194 | rc = 0; 195 | 196 | done: 197 | if (rc < 0) fprintf(stderr, "frame parsing error\n"); 198 | return rc; 199 | } 200 | 201 | /* 202 | * read from the publisher 203 | * each frame is prefixed with a uint32 length 204 | */ 205 | int handle_io(void) { 206 | int rc = -1; 207 | size_t avail; 208 | ssize_t nr; 209 | 210 | /* the buffer must have some free space because 211 | * any time we read data we process it right here, 212 | * leaving at most a tiny fragment of a partial 213 | * frame to prepend the next read */ 214 | assert(cfg.bsz < BUFSZ); 215 | avail = BUFSZ - cfg.bsz; 216 | 217 | nr = read(cfg.client_fd, cfg.buf + cfg.bsz, avail); 218 | if (nr <= 0) { 219 | fprintf(stderr, "read: %s\n", nr ? strerror(errno) : "eof"); 220 | goto done; 221 | } 222 | 223 | cfg.bsz += nr; 224 | if (decode_frames() < 0) goto done; 225 | 226 | rc = 0; 227 | 228 | done: 229 | return rc; 230 | } 231 | 232 | int main(int argc, char *argv[]) { 233 | int opt, rc=-1, n, ec; 234 | struct epoll_event ev; 235 | cfg.prog = argv[0]; 236 | char unit, *c, buf[100]; 237 | struct shr_stat stat; 238 | ssize_t nr; 239 | 240 | utstring_new(cfg.tmp); 241 | cfg.set = kv_set_new(); 242 | 243 | utarray_new(output_keys, &ut_str_icd); 244 | utarray_new(output_defaults, &ut_str_icd); 245 | utarray_new(output_types,&ut_int_icd); 246 | 247 | while ( (opt = getopt(argc,argv,"vhs:p:d:b:")) > 0) { 248 | switch(opt) { 249 | case 'v': cfg.verbose++; break; 250 | case 'h': default: usage(); break; 251 | case 's': cfg.host = strdup(optarg); break; 252 | case 'p': cfg.port = atoi(optarg); break; 253 | case 'd': cfg.spool = strdup(optarg); break; 254 | case 'b': cfg.cast = strdup(optarg); break; 255 | } 256 | } 257 | 258 | if (cfg.spool == NULL) usage(); 259 | if (cfg.cast == NULL) usage(); 260 | if (cfg.host == NULL) usage(); 261 | if (cfg.port == 0) usage(); 262 | 263 | if (parse_config(cfg.cast) < 0) goto done; 264 | cfg.sp = kv_spoolwriter_new(cfg.spool); 265 | if (cfg.sp == NULL) goto done; 266 | 267 | if (connect_up() < 0) goto done; 268 | 269 | /* block all signals. we accept signals via signal_fd */ 270 | sigset_t all; 271 | sigfillset(&all); 272 | sigprocmask(SIG_SETMASK,&all,NULL); 273 | 274 | /* a few signals we'll accept via our signalfd */ 275 | sigset_t sw; 276 | sigemptyset(&sw); 277 | for(n=0; n < sizeof(sigs)/sizeof(*sigs); n++) sigaddset(&sw, sigs[n]); 278 | 279 | /* create the signalfd for receiving signals */ 280 | cfg.signal_fd = signalfd(-1, &sw, 0); 281 | if (cfg.signal_fd == -1) { 282 | fprintf(stderr,"signalfd: %s\n", strerror(errno)); 283 | goto done; 284 | } 285 | 286 | /* set up the epoll instance */ 287 | cfg.epoll_fd = epoll_create(1); 288 | if (cfg.epoll_fd == -1) { 289 | fprintf(stderr,"epoll: %s\n", strerror(errno)); 290 | goto done; 291 | } 292 | 293 | /* add descriptors of interest */ 294 | if (add_epoll(EPOLLIN, cfg.signal_fd)) goto done; 295 | if (add_epoll(EPOLLIN, cfg.client_fd)) goto done; 296 | 297 | alarm(1); 298 | 299 | while (1) { 300 | ec = epoll_wait(cfg.epoll_fd, &ev, 1, -1); 301 | if (ec < 0) { 302 | fprintf(stderr, "epoll: %s\n", strerror(errno)); 303 | goto done; 304 | } 305 | 306 | if (ec == 0) { assert(0); goto done; } 307 | else if (ev.data.fd == cfg.signal_fd) { if (handle_signal() < 0) goto done; } 308 | else if (ev.data.fd == cfg.client_fd) { if (handle_io() < 0) goto done; } 309 | else { assert(0); goto done; } 310 | } 311 | 312 | rc = 0; 313 | 314 | done: 315 | utarray_free(output_keys); 316 | utarray_free(output_defaults); 317 | utarray_free(output_types); 318 | utstring_free(cfg.tmp); 319 | if (cfg.sp) kv_spoolwriter_free(cfg.sp); 320 | if (cfg.set) kv_set_free(cfg.set); 321 | if (cfg.signal_fd != -1) close(cfg.signal_fd); 322 | if (cfg.epoll_fd != -1) close(cfg.epoll_fd); 323 | if (cfg.client_fd != -1) close(cfg.client_fd); 324 | return 0; 325 | } 326 | -------------------------------------------------------------------------------- /utils/kvsp-upub.c: -------------------------------------------------------------------------------- 1 | /* this utility sends spool frames out as UDP packets with JSON payloads */ 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include "utarray.h" 17 | #include "utstring.h" 18 | #include "kvspool.h" 19 | #include "kvsp-bconfig.h" 20 | #include 21 | 22 | int verbose; 23 | int port=5139; // arbitrary 24 | char *spool; 25 | UT_string *buf; 26 | UT_array *fds; 27 | 28 | void usage(char *prog) { 29 | fprintf(stderr, "usage: %s [-v] -d spool -p ... \n", prog); 30 | exit(-1); 31 | } 32 | 33 | void setup_udp(char *host) { 34 | if (verbose) fprintf(stderr,"connecting to %s\n",host); 35 | int fd; 36 | in_addr_t addr; 37 | struct hostent *h = gethostbyname(host); 38 | if (!h) {fprintf(stderr,"%s\n",hstrerror(h_errno)); exit(-1);} 39 | addr = ((struct in_addr*)h->h_addr)->s_addr; 40 | 41 | /********************************************************** 42 | * create an IPv4/UDP socket, not yet bound to any address 43 | *********************************************************/ 44 | fd = socket(AF_INET, SOCK_DGRAM, 0); 45 | if (fd == -1) { 46 | fprintf(stderr,"socket: %s\n", strerror(errno)); 47 | exit(-1); 48 | } 49 | 50 | /********************************************************** 51 | * internet socket address structure, for the remote side 52 | *********************************************************/ 53 | struct sockaddr_in sin; 54 | sin.sin_family = AF_INET; 55 | sin.sin_addr.s_addr = addr; 56 | sin.sin_port = htons(port); 57 | 58 | if (sin.sin_addr.s_addr == INADDR_NONE) { 59 | fprintf(stderr,"invalid remote IP %s\n", host); 60 | exit(-1); 61 | } 62 | 63 | /********************************************************** 64 | * UDP is connectionless; connect only sets dest for writes 65 | *********************************************************/ 66 | if (connect(fd, (struct sockaddr*)&sin, sizeof(sin)) == -1) { 67 | fprintf(stderr,"connect: %s\n", strerror(errno)); 68 | exit(-1); 69 | } 70 | utarray_push_back(fds,&fd); 71 | } 72 | 73 | int main(int argc, char *argv[]) { 74 | void *sp=NULL; 75 | void *set=NULL; 76 | int c, opt,rc=-1; 77 | size_t sz; void *b; 78 | json_t *o = NULL; 79 | char *config_file; 80 | set = kv_set_new(); 81 | utarray_new(fds,&ut_int_icd); 82 | utstring_new(buf); 83 | o = json_object(); 84 | 85 | signal(SIGPIPE,SIG_IGN); 86 | 87 | while ( (opt = getopt(argc, argv, "v+d:p:")) != -1) { 88 | switch (opt) { 89 | case 'v': verbose++; break; 90 | case 'd': spool=strdup(optarg); break; 91 | case 'p': port=atoi(optarg); break; 92 | default: usage(argv[0]); break; 93 | } 94 | } 95 | if (spool == NULL) usage(argv[0]); 96 | while(optind < argc) setup_udp(argv[optind++]); 97 | 98 | sp = kv_spoolreader_new(spool); 99 | if (!sp) goto done; 100 | 101 | while (kv_spool_read(sp,set,1) > 0) { /* read til interrupted by signal */ 102 | json_object_clear(o); 103 | kv_t *kv = NULL; 104 | while (kv = kv_next(set,kv)) { 105 | json_t *jval = json_string(kv->val); 106 | json_object_set_new(o, kv->key, jval); 107 | } 108 | if (verbose) json_dumpf(o, stderr, JSON_INDENT(1)); 109 | char *json = json_dumps(o, JSON_INDENT(1)); 110 | size_t len = strlen(json); 111 | int *fd=NULL; 112 | while ( (fd=(int*)utarray_next(fds,fd))) { 113 | if (write(*fd,json,len) == -1) { 114 | fprintf(stderr,"write: %s\n",strerror(errno)); 115 | goto done; 116 | } 117 | } 118 | free(json); 119 | } 120 | 121 | 122 | // close all the fds. 123 | // TODO 124 | 125 | rc = 0; 126 | 127 | done: 128 | if (sp) kv_spoolreader_free(sp); 129 | kv_set_free(set); 130 | utarray_free(fds); 131 | utstring_free(buf); 132 | if(o) json_decref(o); 133 | 134 | return 0; 135 | } 136 | -------------------------------------------------------------------------------- /utils/ramdisk.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include "utarray.h" 15 | #include 16 | 17 | #define TMPFS_MAGIC 0x01021994 18 | 19 | /****************************************************************************** 20 | * ramdisk Troy D. Hanson 21 | * 22 | * a utility with modes to: 23 | * - create a ramdisk, 24 | * - query a ramdisk (see its size and percent full) 25 | * - unmount a ramdisk 26 | * 27 | * The ramdisk used here is the 'tmpfs' filesystem which is not strictly a 28 | * pure RAM device; it can swap under the kernel's discretion. I have also 29 | * noticed that a large ramdisk (say, 6gb on a system with 8gb ram) might 30 | * exhibit 'no space left on device' even when only 50% full. A better 31 | * query mode would show the status (resident, paged, etc) of ramdisk pages. 32 | *****************************************************************************/ 33 | 34 | /* command line configuration parameters */ 35 | int verbose; 36 | enum {MODE_NONE,MODE_QUERY,MODE_CREATE,MODE_UNMOUNT} mode = MODE_NONE; 37 | char *sz="50%"; 38 | char *ramdisk; 39 | UT_array *dirs; 40 | 41 | void usage(char *prog) { 42 | fprintf(stderr, "This utility creates a tmpfs ramdisk on a given mountpount.\n"); 43 | fprintf(stderr, "It does nothing if a tmpfs is already mounted on that point.\n"); 44 | fprintf(stderr, "\n"); 45 | fprintf(stderr,"usage:\n\n"); 46 | fprintf(stderr, "-c (create mode):\n"); 47 | fprintf(stderr, " %s -c [-s ] [-d ] \n", prog); 48 | fprintf(stderr, " -s suffixed with k|m|g|%% [default: 50%%]\n"); 49 | fprintf(stderr, " -d directory to post-create inside ramdisk (repeatable)\n"); 50 | fprintf(stderr, "\n"); 51 | fprintf(stderr, "-q (query mode):\n"); 52 | fprintf(stderr, " %s -q \n", prog); 53 | fprintf(stderr, "\n"); 54 | fprintf(stderr, "-u (unmount mode):\n"); 55 | fprintf(stderr, " %s -u \n", prog); 56 | fprintf(stderr, "\n"); 57 | fprintf(stderr, "Examples of creating a ramdisk:\n"); 58 | fprintf(stderr, " %s -c -s 1g /mnt/ramdisk\n", prog); 59 | fprintf(stderr, " %s -c -s 1g -d /mnt/ramdisk/in -d /mnt/ramdisk/out /mnt/ramdisk\n", prog); 60 | fprintf(stderr, "\n"); 61 | fprintf(stderr, "Note: 'cat /proc/mounts' to see mounted tmpfs ramdisks.\n"); 62 | exit(-1); 63 | } 64 | 65 | /* Prevent a ramdisk from being mounted at the mount-point of an 66 | * existing ramdisk. This prevents people from accidently stacking tmpfs. 67 | * However it is OK to mount a ramdisk on a subdirectory of another ramdisk. */ 68 | int suitable_mountpoint(char *dir, struct stat *sb, struct statfs *sf) { 69 | size_t dlen = strlen(dir); 70 | char pdir[PATH_MAX]; 71 | struct stat psb; 72 | 73 | if (dlen+4 > PATH_MAX) { 74 | syslog(LOG_ERR, "path too long\n"); 75 | return -1; 76 | } 77 | 78 | if (stat(ramdisk, sb) == -1) { /* does mount point exist? */ 79 | syslog(LOG_ERR, "no mount point %s: %s\n", ramdisk, strerror(errno)); 80 | return -1; 81 | } 82 | if (S_ISDIR(sb->st_mode) == 0) { /* has to be a directory */ 83 | syslog(LOG_ERR, "mount point %s: not a directory\n", ramdisk); 84 | return -1; 85 | } 86 | if (statfs(ramdisk, sf) == -1) { /* what kinda filesystem is it on? */ 87 | syslog(LOG_ERR, "can't statfs %s: %s\n", ramdisk, strerror(errno)); 88 | return -1; 89 | } 90 | 91 | /* is it already a tmpfs mountpoint? */ 92 | memcpy(pdir,dir,dlen+1); strcat(pdir,"/.."); 93 | if (stat(pdir, &psb) == -1) { 94 | syslog(LOG_ERR, "can't stat %s: %s\n", pdir, strerror(errno)); 95 | return -1; 96 | } 97 | int is_mountpoint = (psb.st_dev == sb->st_dev) ? 0 : 1; 98 | int is_tmpfs = (sf->f_type == TMPFS_MAGIC); 99 | if (is_mountpoint && is_tmpfs) { 100 | //syslog(LOG_INFO, "already a tmpfs mountpoint: %s\n", dir, strerror(errno)); 101 | return -2; 102 | } 103 | 104 | return 0; 105 | } 106 | 107 | #define KB 1024L 108 | #define MB (1024*1024L) 109 | #define GB (1024*1024*1024L) 110 | int query_ramdisk(void) { 111 | struct stat sb; struct statfs sf; 112 | if (suitable_mountpoint(ramdisk, &sb, &sf) != -2) { 113 | printf("%s: not a ramdisk\n", ramdisk); 114 | return -1; 115 | } 116 | char szb[100]; 117 | long bytes = sf.f_bsize*sf.f_blocks; 118 | if (bytes < KB) snprintf(szb, sizeof(szb), "%ld bytes", bytes); 119 | else if (bytes < MB) snprintf(szb, sizeof(szb), "%ld kb", bytes/KB); 120 | else if (bytes < GB) snprintf(szb, sizeof(szb), "%ld mb", bytes/MB); 121 | else snprintf(szb, sizeof(szb), "%ld gb", bytes/GB); 122 | int used_pct = 100 - (sf.f_bfree * 100.0 / sf.f_blocks); 123 | printf("%s: ramdisk of size %s (%d%% used)\n", ramdisk, szb, used_pct); 124 | } 125 | 126 | int unmount_ramdisk(void) { 127 | struct stat sb; struct statfs sf; 128 | if (suitable_mountpoint(ramdisk, &sb, &sf) != -2) { 129 | syslog(LOG_ERR,"%s: not a ramdisk\n", ramdisk); 130 | return -1; 131 | } 132 | if (umount(ramdisk) == -1) { 133 | syslog(LOG_ERR,"%s: cannot unmount\n", ramdisk); 134 | return -1; 135 | } 136 | return 0; 137 | } 138 | 139 | int create_ramdisk(void) { 140 | int rc; 141 | char opts[100]; 142 | 143 | struct stat sb; struct statfs sf; 144 | rc = suitable_mountpoint(ramdisk, &sb, &sf); 145 | if (rc) return rc; 146 | 147 | /* ok, mount a ramdisk on this point */ 148 | snprintf(opts,sizeof(opts),"size=%s",sz); 149 | rc=mount("none", ramdisk, "tmpfs", MS_NOATIME|MS_NODEV, opts); 150 | if (rc) syslog(LOG_ERR, "can't make ramdisk %s: %s\n", ramdisk, strerror(errno)); 151 | return rc; 152 | } 153 | 154 | void make_dirs(UT_array *dirs) { 155 | char **d, *dir; 156 | d=NULL; 157 | while ( (d=(char**)utarray_next(dirs,d))) { 158 | dir = *d; 159 | /* fprintf(stderr,"dir is %s\n",dir); */ 160 | if (mkdir(dir, 0777) == -1) { 161 | fprintf(stderr,"failed to make %s: %s\n",dir,strerror(errno)); 162 | } 163 | } 164 | } 165 | 166 | int main(int argc, char * argv[]) { 167 | int opt, rc; 168 | utarray_new(dirs,&ut_str_icd); 169 | 170 | while ( (opt = getopt(argc, argv, "v+cqus:hd:")) != -1) { 171 | switch (opt) { 172 | case 'v': verbose++; break; 173 | case 'q': if (mode) usage(argv[0]); mode=MODE_QUERY; break; 174 | case 'c': if (mode) usage(argv[0]); mode=MODE_CREATE; break; 175 | case 'u': if (mode) usage(argv[0]); mode=MODE_UNMOUNT; break; 176 | case 's': sz=strdup(optarg); break; 177 | case 'd': utarray_push_back(dirs,&optarg); break; 178 | case 'h': default: usage(argv[0]); break; 179 | } 180 | } 181 | if (optind < argc) ramdisk=argv[optind++]; 182 | if (!ramdisk) usage(argv[0]); 183 | openlog("ramdisk", LOG_PID|LOG_PERROR, LOG_LOCAL0); 184 | 185 | switch(mode) { 186 | case MODE_CREATE: rc=create_ramdisk(); make_dirs(dirs); break; 187 | case MODE_UNMOUNT: rc=unmount_ramdisk(); break; 188 | case MODE_QUERY: rc=query_ramdisk(); break; 189 | default: usage(argv[0]); break; 190 | } 191 | utarray_free(dirs); 192 | return rc; 193 | } 194 | -------------------------------------------------------------------------------- /utils/ringbuf.c: -------------------------------------------------------------------------------- 1 | #include "ringbuf.h" 2 | 3 | ringbuf *ringbuf_new(size_t sz) { 4 | ringbuf *r = malloc(sizeof(*r) + sz); 5 | if (r == NULL) { 6 | fprintf(stderr,"out of memory\n"); 7 | goto done; 8 | } 9 | 10 | r->u = r->i = r->o = 0; 11 | r->n = sz; 12 | 13 | done: 14 | return r; 15 | } 16 | 17 | void ringbuf_free(ringbuf* r) { 18 | free(r); 19 | } 20 | 21 | /* ringbuf_take: alternative to ringbuf_new; caller 22 | * provides the buffer to use as the ringbuf. 23 | * buffer should be aligned e.g. from malloc/mmap. 24 | * note: do not ringbuf_free afterward. 25 | */ 26 | #define MIN_RINGBUF (sizeof(ringbuf) + 1) 27 | ringbuf *ringbuf_take(void *buf, size_t sz) { 28 | if (sz < MIN_RINGBUF) return NULL; 29 | ringbuf *r = (ringbuf*)buf; 30 | 31 | r->u = r->i = r->o = 0; 32 | r->n = sz - sizeof(*r); // alignment should be ok 33 | assert(r->n > 0); 34 | 35 | return r; 36 | } 37 | 38 | 39 | /* copy data in. fails if ringbuf has insuff space. */ 40 | #define MIN(a,b) (((a) < (b)) ? (a) : (b)) 41 | int ringbuf_put(ringbuf *r, const void *_data, size_t len) { 42 | char *data = (char*)_data; 43 | size_t a,b,c; 44 | if (r->i < r->o) { // available space is a contiguous buffer 45 | a = r->o - r->i; 46 | assert(a == r->n - r->u); 47 | if (len > a) return -1; 48 | memcpy(&r->d[r->i], data, len); 49 | } else { // available space wraps; it's two buffers 50 | b = r->n - r->i; // in-head to eob (receives leading input) 51 | c = r->o; // out-head to in-head (receives trailing input) 52 | a = b + c; // available space 53 | // the only ambiguous case is i==o, that's why u is needed 54 | if (r->i == r->o) a = r->n - r->u; 55 | assert(a == r->n - r->u); 56 | if (len > a) return -1; 57 | memcpy(&r->d[r->i], data, MIN(b, len)); 58 | if (len > b) memcpy(r->d, &data[b], len-b); 59 | } 60 | r->i = (r->i + len) % r->n; 61 | r->u += len; 62 | return 0; 63 | } 64 | 65 | size_t ringbuf_get_freespace(ringbuf *r) { 66 | return r->n - r->u; 67 | } 68 | 69 | size_t ringbuf_get_pending_size(ringbuf *r) { 70 | return r->u; 71 | } 72 | 73 | size_t ringbuf_get_next_chunk(ringbuf *r, char **data) { 74 | // in this case the next chunk is the whole pending buffer 75 | if (r->o < r->i) { 76 | assert(r->u == r->i - r->o); 77 | *data = &r->d[r->o]; 78 | return r->u; 79 | } 80 | // in this case (i==o) either the buffer is empty of full. 81 | // r->u tells distinguishes these cases. 82 | if ((r->o == r->i) && (r->u == 0)) { *data=NULL; return 0; } 83 | // if we're here, that means r->o > r->i. the pending 84 | // output is wrapped around the buffer. this function 85 | // returns the chunk prior to eob. the next call gets 86 | // the next chunk that's wrapped around the buffer. 87 | size_t b,c; 88 | b = r->n - r->o; // length of the part we're returning 89 | c = r->i; // wrapped part length- a sanity check 90 | assert(r->u == b + c); 91 | *data = &r->d[r->o]; 92 | return b; 93 | } 94 | 95 | void ringbuf_mark_consumed(ringbuf *r, size_t len) { 96 | assert(len <= r->u); 97 | r->o = (r->o + len ) % r->n; 98 | r->u -= len; 99 | } 100 | 101 | void ringbuf_clear(ringbuf *r) { 102 | r->u = r->i = r->o = 0; 103 | } 104 | -------------------------------------------------------------------------------- /utils/ringbuf.h: -------------------------------------------------------------------------------- 1 | #ifndef _RINGBUF_H_ 2 | #define _RINGBUF_H_ 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | /* simple ring buffer */ 9 | 10 | typedef struct _ringbuf { 11 | size_t n; /* allocd size */ 12 | size_t u; /* used space */ 13 | size_t i; /* input pos */ 14 | size_t o; /* output pos */ 15 | char d[]; /* C99 flexible array member */ 16 | } ringbuf; 17 | 18 | ringbuf *ringbuf_new(size_t sz); 19 | ringbuf *ringbuf_take(void *buf, size_t sz); 20 | int ringbuf_put(ringbuf *r, const void *data, size_t len); 21 | size_t ringbuf_get_pending_size(ringbuf *r); 22 | size_t ringbuf_get_next_chunk(ringbuf *r, char **data); 23 | void ringbuf_mark_consumed(ringbuf *r, size_t len); 24 | void ringbuf_free(ringbuf *r); 25 | void ringbuf_clear(ringbuf *r); 26 | size_t ringbuf_get_freespace(ringbuf *r); 27 | 28 | #endif /* _RINGBUF_H_ */ 29 | -------------------------------------------------------------------------------- /utils/spr-cast.cfg: -------------------------------------------------------------------------------- 1 | str from 2 | str when 3 | i32 iter 4 | i8 test_i8 5 | i16 test_i16 6 | i32 test_i32 7 | str test_str 8 | str8 test_str8 9 | d64 test_d64 10 | ipv4 test_ipv4 11 | ipv46 test1_ipv46 12 | ipv46 test2_ipv46 13 | ipv46 test3_ipv46 14 | mac test_mac 15 | -------------------------------------------------------------------------------- /utils/ts.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include "ts.h" 6 | 7 | static void ts_def_clear(char *data, size_t sz) { memset(data,0,sz); } 8 | static void ts_def_incr(int *cur, int *incr) { *cur += (incr ? (*incr) : 1); } 9 | static void ts_def_show(int *cur) { printf("%d\n",*cur); } 10 | 11 | ts_t *ts_new(unsigned num_buckets, unsigned secs_per_bucket, const ts_mm *mm) { 12 | int i; 13 | ts_t *t = calloc(1,sizeof(ts_t)); if (!t) return NULL; 14 | t->secs_per_bucket = secs_per_bucket; 15 | t->num_buckets = num_buckets; 16 | t->mm = *mm; /* struct copy */ 17 | // pad sz so sequential buckets' time_t are naturally aligned 18 | int pad = ((t->mm.sz % sizeof(ts_bucket)) > 0) ? 19 | (sizeof(ts_bucket) - (t->mm.sz % sizeof(ts_bucket))) : 20 | 0; 21 | t->mm.sz += pad; 22 | if (t->mm.ctor == NULL) t->mm.ctor = (ts_ctor_f*)ts_def_clear; 23 | if (t->mm.data == NULL) { 24 | if (mm->sz == sizeof(int)) t->mm.data = (ts_data_f*)ts_def_incr; 25 | else assert(0); 26 | } 27 | if (t->mm.show == NULL) { 28 | if (mm->sz == sizeof(int)) t->mm.show = (ts_show_f*)ts_def_show; 29 | } 30 | t->buckets = calloc(num_buckets,sizeof(ts_bucket)+t->mm.sz); 31 | if (t->buckets == NULL) { free(t); return NULL; } 32 | for(i=0; inum_buckets; i++) { 33 | //fprintf(stderr,"t->buckets %p bkt(t,%d) %p\n", t->buckets, i, bkt(t,i)); 34 | bkt(t,i)->start = i * t->secs_per_bucket; 35 | t->mm.ctor(bkt(t,i)->data,t->mm.sz); 36 | } 37 | return t; 38 | } 39 | 40 | void ts_add(ts_t *t, time_t when, void *data) { 41 | int i; 42 | if (bkt(t,0)->start > when) return; // too old 43 | /* figure out bucket it should go in */ 44 | unsigned idx = (when - bkt(t,0)->start) / t->secs_per_bucket; 45 | if (idx >= t->num_buckets) { // shift 46 | unsigned shift = (idx - t->num_buckets) + 1; 47 | if (shift > t->num_buckets) shift = t->num_buckets; 48 | if (shift <= t->num_buckets) { 49 | if (t->mm.dtor) { 50 | for(i=0; imm.dtor(bkt(t,i)->data); 51 | } 52 | } 53 | if (shift < t->num_buckets) { 54 | memmove(bkt(t,0),bkt(t,shift),(t->num_buckets-shift)*(sizeof(ts_bucket)+t->mm.sz)); 55 | } 56 | if (shift) { 57 | for(i=t->num_buckets-shift; inum_buckets; i++) { 58 | t->mm.ctor(bkt(t,i)->data,t->mm.sz); 59 | bkt(t,i)->start = (i > 0) ? (bkt(t,i-1)->start + t->secs_per_bucket) : when; 60 | } 61 | } 62 | idx = (when - bkt(t,0)->start) / t->secs_per_bucket; 63 | assert(idx < t->num_buckets); 64 | } 65 | void *cur = bkt(t,idx)->data; 66 | t->mm.data(cur,data); 67 | } 68 | 69 | void ts_free(ts_t *t) { 70 | int i; 71 | if (t->mm.dtor) { 72 | for(i=0; inum_buckets; i++) t->mm.dtor(bkt(t,i)->data); 73 | } 74 | free(t->buckets); 75 | free(t); 76 | } 77 | 78 | void ts_show(ts_t *t) { 79 | int i; 80 | for(i=0; inum_buckets; i++) { 81 | printf("#%d(%lu): ", i, (long)(bkt(t,i)->start)); 82 | if (t->mm.show) t->mm.show(bkt(t,i)->data); 83 | else printf("\n"); 84 | } 85 | printf("\n"); 86 | } 87 | -------------------------------------------------------------------------------- /utils/ts.h: -------------------------------------------------------------------------------- 1 | 2 | typedef void (ts_data_f)(char *cur, char *add); 3 | typedef void (ts_ctor_f)(void *elt, size_t sz); 4 | typedef void (ts_dtor_f)(void *elt); 5 | typedef void (ts_show_f)(void *elt); 6 | typedef struct { 7 | size_t sz; 8 | ts_data_f *data; 9 | ts_ctor_f *ctor; 10 | ts_dtor_f *dtor; 11 | ts_show_f *show; 12 | } ts_mm; 13 | 14 | typedef struct { 15 | time_t start; 16 | char data[]; /* C99 flexible array member */ 17 | } ts_bucket; 18 | 19 | #define bkt(t,i) ((ts_bucket*)((char*)((t)->buckets) + ((i)*(sizeof(ts_bucket)+(t)->mm.sz)))) 20 | typedef struct { 21 | ts_mm mm; 22 | unsigned secs_per_bucket; 23 | unsigned num_buckets; 24 | ts_bucket *buckets; 25 | } ts_t; 26 | 27 | ts_t *ts_new(unsigned num_buckets, unsigned secs_per_bucket, const ts_mm *mm); 28 | void ts_add(ts_t *t, time_t when, void *data); 29 | void ts_free(ts_t *t); 30 | void ts_show(ts_t *t); 31 | 32 | --------------------------------------------------------------------------------