├── hiredis ├── TODO ├── fmacros.h ├── COPYING ├── util.h ├── net.h ├── sds.h ├── async.h ├── net.c └── hiredis.h ├── sample-images ├── screenshot-interlaced.png └── screenshot-noninterlaced.png ├── .gitignore ├── png-db.xcodeproj └── project.xcworkspace │ └── contents.xcworkspacedata ├── DbDefBackend.h ├── kyotocabinet ├── kyotocabinet.pc.in ├── README ├── kcdbext.cc ├── kcplantdb.cc ├── kcstashdb.cc ├── kccachedb.cc ├── kcdirdb.cc ├── kchashdb.cc ├── kcmap.cc ├── kcpolydb.cc ├── kcprotodb.cc ├── kcdb.cc ├── kyotocabinet.def ├── kccompare.cc ├── kcregex.h ├── kccommon.h ├── kcregex.cc ├── myconf.h ├── kccompare.h ├── kcutil.cc ├── kyotocabinet.idl ├── cmdcommon.h └── kccompress.cc ├── Crc.h ├── DbFsBackend.h ├── StaticAssert.h ├── stats.sh ├── Sha1.h ├── Return.h ├── test-png-dumpchunks.cpp ├── StringUtils.h ├── DbRedisBackend.h ├── DbKyotoBackend.h ├── StringUtils.cpp ├── Mutex.h ├── DbFileBackend.h ├── Endianess.h ├── DbPng.h ├── db-extract-file.cpp ├── db-push.cpp ├── test-png-reader.cpp ├── db-list-dir.cpp ├── pnginfo.cpp ├── Utils.h ├── FileUtils.cpp ├── db-push-dir.cpp ├── compile.sh ├── FileUtils.h ├── Png.h ├── Db.cpp ├── Db.h ├── Crc.cpp ├── DbFsBackend.cpp ├── SmartPointer.h ├── DbKyotoBackend.cpp ├── README.md ├── DbRedisBackend.cpp ├── DbPng.cpp ├── db-fuse.cpp └── Sha1.cpp /hiredis/TODO: -------------------------------------------------------------------------------- 1 | - add redisCommandVector() 2 | - add support for pipelining 3 | -------------------------------------------------------------------------------- /sample-images/screenshot-interlaced.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/albertz/png-db/HEAD/sample-images/screenshot-interlaced.png -------------------------------------------------------------------------------- /sample-images/screenshot-noninterlaced.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/albertz/png-db/HEAD/sample-images/screenshot-noninterlaced.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | *.bin 3 | *.dSYM 4 | *.deps 5 | *.o 6 | bin 7 | build 8 | db 9 | db.kch 10 | db.pngdb 11 | xcuserdata 12 | -------------------------------------------------------------------------------- /png-db.xcodeproj/project.xcworkspace/contents.xcworkspacedata: -------------------------------------------------------------------------------- 1 | 2 | 4 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /DbDefBackend.h: -------------------------------------------------------------------------------- 1 | /* DB default backend 2 | * by Albert Zeyer, 2011 3 | * code under LGPL 4 | */ 5 | 6 | #ifndef __AZ_DBDEFBACKEND_H__ 7 | #define __AZ_DBDEFBACKEND_H__ 8 | 9 | #include "DbKyotoBackend.h" 10 | 11 | typedef DbKyotoBackend DbDefBackend; 12 | 13 | #endif 14 | -------------------------------------------------------------------------------- /hiredis/fmacros.h: -------------------------------------------------------------------------------- 1 | #ifndef _REDIS_FMACRO_H 2 | #define _REDIS_FMACRO_H 3 | 4 | #define _BSD_SOURCE 5 | 6 | #ifdef __linux__ 7 | #define _XOPEN_SOURCE 700 8 | #else 9 | #define _XOPEN_SOURCE 10 | #endif 11 | 12 | #define _LARGEFILE_SOURCE 13 | #define _FILE_OFFSET_BITS 64 14 | 15 | #endif 16 | -------------------------------------------------------------------------------- /kyotocabinet/kyotocabinet.pc.in: -------------------------------------------------------------------------------- 1 | prefix=@prefix@ 2 | exec_prefix=@exec_prefix@ 3 | datarootdir = @datarootdir@ 4 | bindir=@bindir@ 5 | libdir=@libdir@ 6 | libexecdir=@libexecdir@ 7 | includedir=@includedir@ 8 | datadir=@datadir@ 9 | 10 | Name: Kyoto Cabinet 11 | Description: a straightforward implementation of DBM 12 | Version: @PACKAGE_VERSION@ 13 | Libs: -L${libdir} -lkyotocabinet 14 | Libs.private: @LIBS@ 15 | Cflags: -I${includedir} 16 | -------------------------------------------------------------------------------- /Crc.h: -------------------------------------------------------------------------------- 1 | /* calculate CRC 2 | * based on http://www.w3.org/TR/PNG/#D-CRCAppendix 3 | * by Albert Zeyer, 2011 4 | * code public domain 5 | */ 6 | 7 | #ifndef __AZ__CRC_H__ 8 | #define __AZ__CRC_H__ 9 | 10 | #include 11 | 12 | uint32_t update_crc(uint32_t crc, const char *buf, size_t len); 13 | uint32_t calc_crc(const char *buf, size_t len); 14 | uint32_t calc_crc(const std::string& s); 15 | uint32_t calc_crc(const std::string& s1, const std::string& s2); 16 | 17 | #endif 18 | -------------------------------------------------------------------------------- /DbFsBackend.h: -------------------------------------------------------------------------------- 1 | /* filesystem DB backend 2 | * by Albert Zeyer, 2011 3 | * code under LGPL 4 | */ 5 | 6 | #ifndef __AZ__DBFSBACKEND_H__ 7 | #define __AZ__DBFSBACKEND_H__ 8 | 9 | #include "Db.h" 10 | 11 | struct DbFsBackend : DbIntf { 12 | std::string baseDir; 13 | 14 | DbFsBackend(const std::string& d = "db") : baseDir(d) {} 15 | Return init() { return true; } 16 | Return push(/*out*/ DbEntryId& id, const DbEntry& entry); 17 | Return get(/*out*/ DbEntry& entry, const DbEntryId& id); 18 | }; 19 | 20 | #endif 21 | -------------------------------------------------------------------------------- /StaticAssert.h: -------------------------------------------------------------------------------- 1 | /* 2 | * StaticAssert.h 3 | * OpenLieroX 4 | * 5 | * Created by Albert Zeyer on 30.03.09. 6 | * code under LGPL 7 | * 8 | */ 9 | 10 | #ifndef __OLX__STATICASSERT_H__ 11 | #define __OLX__STATICASSERT_H__ 12 | 13 | template class static_assert_failure; 14 | template <> class static_assert_failure { char foo; }; 15 | template class static_assert_test{}; 16 | 17 | #define static_assert(X, desc) \ 18 | typedef static_assert_test< (int)sizeof(static_assert_failure< (bool)(X) >) > static_assert_typedef_##desc; 19 | 20 | #endif 21 | -------------------------------------------------------------------------------- /stats.sh: -------------------------------------------------------------------------------- 1 | #!/bin/zsh 2 | 3 | DBFILE=db.kch 4 | SRCDIR="$HOME/screenies" 5 | [ "$1" != "" ] && SCRDIR="$1" 6 | 7 | zmodload zsh/stat 8 | sizesum=0 9 | numfiles=0 10 | 11 | ./bin/db-list-dir | while read l; do 12 | [[ "$l" =~ "^file: /(.*), .*, .*$" ]] && { 13 | fn="$match" 14 | size="$(stat -L +size $SRCDIR/$fn)" 15 | sizesum=$(($sizesum + $size)) 16 | numfiles=$(($numfiles + 1)) 17 | } 18 | done 19 | 20 | dbsize="$(stat -L +size $DBFILE)" 21 | 22 | echo "num files: $numfiles" 23 | echo "fs size: $(($sizesum / 1024 / 1024.0)) MB" 24 | echo "db size: $(($dbsize / 1024 / 1024.0)) MB" 25 | -------------------------------------------------------------------------------- /Sha1.h: -------------------------------------------------------------------------------- 1 | /* public api for steve reid's public domain SHA-1 implementation */ 2 | /* this file is in the public domain */ 3 | 4 | #ifndef __SHA1_H 5 | #define __SHA1_H 6 | 7 | #include 8 | #include 9 | 10 | #define SHA1_DIGEST_SIZE 20 11 | 12 | struct Sha1Context { 13 | uint32_t state[5]; 14 | uint32_t count[2]; 15 | uint8_t buffer[64]; 16 | 17 | Sha1Context(); 18 | void update(const char* data, size_t len); 19 | std::string final(); 20 | }; 21 | 22 | std::string calc_sha1(const char* data, size_t s); 23 | 24 | inline std::string calc_sha1(const std::string& data) { 25 | return calc_sha1(&data[0], data.size()); 26 | } 27 | 28 | #endif /* __SHA1_H */ 29 | -------------------------------------------------------------------------------- /Return.h: -------------------------------------------------------------------------------- 1 | /* Return value class 2 | * by Albert Zeyer, 2011 3 | * code under LGPL 4 | */ 5 | 6 | #ifndef __AZ__RETURN_H__ 7 | #define __AZ__RETURN_H__ 8 | 9 | #include 10 | 11 | struct Return { 12 | bool success; 13 | std::string errmsg; 14 | 15 | Return(bool s = true) : success(s) {} 16 | Return(const char* errm) : success(false), errmsg(errm) {} 17 | Return(const std::string& errm) : success(false), errmsg(errm) {} 18 | Return(const Return& r, const std::string& extmsg) : success(false) { 19 | if(r) errmsg = extmsg; 20 | else errmsg = extmsg + ": " + r.errmsg; 21 | } 22 | operator bool() const { return success; } 23 | }; 24 | 25 | #define ASSERT(x) { Return ___r = (x); if(!___r) return ___r; } 26 | #define ASSERT_EXT(x, msg) { Return ___r = (x); if(!___r) return Return(___r, msg); } 27 | 28 | #endif 29 | -------------------------------------------------------------------------------- /test-png-dumpchunks.cpp: -------------------------------------------------------------------------------- 1 | /* demo for PNG chunk reading 2 | * by Albert Zeyer, 2011 3 | * code under LGPL 4 | */ 5 | 6 | #include "Png.h" 7 | 8 | #include 9 | #include 10 | using namespace std; 11 | 12 | int main(int argc, char** argv) { 13 | if(argc <= 1) { 14 | cout << "please give me a filename" << endl; 15 | return 1; 16 | } 17 | 18 | FILE* f = fopen(argv[1], "rb"); 19 | if(Return r = png_read_sig(f)) 20 | cout << "* signature: OK" << endl; 21 | else { 22 | cout << "* signature: " << r.errmsg << endl; 23 | return 1; 24 | } 25 | 26 | while(!feof(f) && !ferror(f)) { 27 | PngChunk chunk; 28 | Return r = png_read_chunk(f, chunk); 29 | if(!r) { 30 | cout << "* chunk: " << r.errmsg << endl; 31 | return 1; 32 | } 33 | cout << "* chunk: " << chunk.type << ", len=" << chunk.data.size() << endl; 34 | } 35 | 36 | return 0; 37 | } 38 | -------------------------------------------------------------------------------- /StringUtils.h: -------------------------------------------------------------------------------- 1 | /* String utils 2 | * by Albert Zeyer, 2011 3 | * code under LGPL 4 | */ 5 | 6 | #ifndef __AZ__STRINGUTILS_H__ 7 | #define __AZ__STRINGUTILS_H__ 8 | 9 | #include 10 | #include 11 | #include "Endianess.h" 12 | 13 | std::string hexString(char c); 14 | std::string hexString(const std::string& rawData); 15 | inline static std::string hexString(const char* s) { return hexString(std::string(s)); } 16 | 17 | template 18 | std::string rawString(T val) { 19 | BEndianSwap(val); 20 | return std::string((char*)&val, sizeof(T)); 21 | } 22 | 23 | template 24 | std::string hexString(T val) { 25 | return hexString(rawString(val)); 26 | } 27 | 28 | template 29 | T valueFromRaw(const char* s) { 30 | T val = *(const T*)s; 31 | BEndianSwap(val); 32 | return val; 33 | } 34 | 35 | size_t findLastPathSep(const std::string& path); 36 | std::string baseFilename(const std::string& filename); 37 | std::string dirName(const std::string& filename); 38 | 39 | #endif 40 | -------------------------------------------------------------------------------- /DbRedisBackend.h: -------------------------------------------------------------------------------- 1 | /* Redis DB backend 2 | * by Albert Zeyer, 2011 3 | * code under LGPL 4 | */ 5 | 6 | #ifndef __AZ__DBREDISBACKEND_H__ 7 | #define __AZ__DBREDISBACKEND_H__ 8 | 9 | #include "Db.h" 10 | #include "hiredis/hiredis.h" 11 | 12 | struct DbRedisBackend : DbIntf { 13 | std::string prefix; 14 | std::string host; 15 | int port; 16 | redisContext* redis; 17 | 18 | DbRedisBackend(const std::string& _prefix = "db.", const std::string& _host = "127.0.0.1", int _port = 6379) 19 | : prefix(_prefix), host(_host), port(_port), redis(NULL) {} 20 | ~DbRedisBackend(); 21 | Return init(); 22 | Return push(/*out*/ DbEntryId& id, const DbEntry& entry); 23 | Return get(/*out*/ DbEntry& entry, const DbEntryId& id); 24 | Return pushToDir(const std::string& path, const DbDirEntry& dirEntry); 25 | Return getDir(/*out*/ std::list& dirList, const std::string& path); 26 | Return setFileRef(/*can be empty*/ const DbEntryId& id, const std::string& path); 27 | Return getFileRef(/*out (can be empty)*/ DbEntryId& id, const std::string& path); 28 | }; 29 | 30 | #endif 31 | -------------------------------------------------------------------------------- /DbKyotoBackend.h: -------------------------------------------------------------------------------- 1 | /* KyotoCabinet DB backend 2 | * by Albert Zeyer, 2011 3 | * code under LGPL 4 | */ 5 | 6 | #ifndef __AZ__DBKYOTOBACKEND_H__ 7 | #define __AZ__DBKYOTOBACKEND_H__ 8 | 9 | #include "Db.h" 10 | #include "kyotocabinet/kcpolydb.h" 11 | 12 | struct DbKyotoBackend : DbIntf { 13 | kyotocabinet::PolyDB db; 14 | std::string filename; 15 | bool readonly; 16 | 17 | DbKyotoBackend(const std::string& dbfilename = "db.kch", bool ro = false) : filename(dbfilename), readonly(ro) {} 18 | ~DbKyotoBackend(); 19 | Return setReadOnly(bool ro) { readonly = ro; return true; } 20 | Return init(); 21 | Return push(/*out*/ DbEntryId& id, const DbEntry& entry); 22 | Return get(/*out*/ DbEntry& entry, const DbEntryId& id); 23 | Return pushToDir(const std::string& path, const DbDirEntry& dirEntry); 24 | Return getDir(/*out*/ std::list& dirList, const std::string& path); 25 | Return setFileRef(/*can be empty*/ const DbEntryId& id, const std::string& path); 26 | Return getFileRef(/*out (can be empty)*/ DbEntryId& id, const std::string& path); 27 | }; 28 | 29 | #endif 30 | -------------------------------------------------------------------------------- /kyotocabinet/README: -------------------------------------------------------------------------------- 1 | ================================================================ 2 | Kyoto Cabinet: a straightforward implementation of DBM 3 | Copyright (C) 2009-2011 FAL Labs 4 | ================================================================ 5 | 6 | 7 | Please read the following documents with a WWW browser. 8 | How to install Kyoto Cabinet is explained in the specification. 9 | 10 | README - this file 11 | COPYING - license 12 | ChangeLog - history of enhancement 13 | doc/index.html - index of documents 14 | 15 | 16 | Contents of the directory tree is below. 17 | 18 | ./ - sources of Kyoto Cabinet 19 | ./doc/ - manuals and specifications 20 | ./man/ - manuals for nroff 21 | ./example/ - sample code for tutorial 22 | ./lab/ - for test and experiment 23 | 24 | 25 | Kyoto Cabinet is released under the terms of the GNU General Public 26 | License version 3. See the file `COPYING' for details. 27 | 28 | Kyoto Cabinet was written by FAL Labs. You can contact the author 29 | by e-mail to `info@fallabs.com'. 30 | 31 | 32 | Thanks. 33 | 34 | 35 | 36 | == END OF FILE == 37 | -------------------------------------------------------------------------------- /StringUtils.cpp: -------------------------------------------------------------------------------- 1 | /* String utils 2 | * by Albert Zeyer, 2011 3 | * code under LGPL 4 | */ 5 | 6 | #include "StringUtils.h" 7 | #include 8 | 9 | std::string hexString(char c) { 10 | char buf[3]; 11 | sprintf(buf, "%02X", (int)(unsigned char)c); 12 | return buf; 13 | } 14 | 15 | std::string hexString(const std::string& rawData) { 16 | std::string ret; 17 | for(size_t i = 0; i < rawData.size(); ++i) 18 | ret += hexString(rawData[i]); 19 | return ret; 20 | } 21 | 22 | size_t findLastPathSep(const std::string& path) { 23 | size_t slash = path.rfind('\\'); 24 | size_t slash2 = path.rfind('/'); 25 | if(slash == std::string::npos) 26 | slash = slash2; 27 | else if(slash2 != std::string::npos) 28 | slash = std::max(slash, slash2); 29 | return slash; 30 | } 31 | 32 | std::string baseFilename(const std::string& filename) { 33 | size_t p = findLastPathSep(filename); 34 | if(p == std::string::npos) return filename; 35 | return filename.substr(p+1); 36 | } 37 | 38 | std::string dirName(const std::string& filename) { 39 | size_t p = findLastPathSep(filename); 40 | if(p == std::string::npos) return ""; 41 | return filename.substr(0, p); 42 | } 43 | -------------------------------------------------------------------------------- /Mutex.h: -------------------------------------------------------------------------------- 1 | /* Mutex class - small wrapper around pthread 2 | * by Albert Zeyer, 2011 3 | * code under LGPL 4 | */ 5 | 6 | #ifndef __AZ__MUTEX_H__ 7 | #define __AZ__MUTEX_H__ 8 | 9 | #include "Utils.h" 10 | #include 11 | 12 | struct Mutex { 13 | pthread_mutex_t m; 14 | Mutex() { pthread_mutex_init(&m, NULL); } 15 | Mutex(const Mutex&) { pthread_mutex_init(&m, NULL); } // ignore copy constr 16 | Mutex& operator=(const Mutex&) {} // ignore assign 17 | ~Mutex() { pthread_mutex_destroy(&m); } 18 | void lock() { pthread_mutex_lock(&m); } 19 | bool trylock() { if(pthread_mutex_trylock(&m) == 0) return true; return false; } 20 | void unlock() { pthread_mutex_unlock(&m); } 21 | }; 22 | 23 | struct ScopedLock : DontCopyTag { 24 | Mutex& mutex; 25 | ScopedLock(Mutex& m) : mutex(m) { mutex.lock(); } 26 | ~ScopedLock() { mutex.unlock(); } 27 | }; 28 | 29 | struct ScopedTryLock : DontCopyTag { 30 | Mutex& mutex; 31 | bool islocked; 32 | ScopedTryLock(Mutex& m) : mutex(m), islocked(false) { islocked = mutex.trylock(); } 33 | ~ScopedTryLock() { if(islocked) { mutex.unlock(); islocked = false; } } 34 | operator bool() const { return islocked; } 35 | }; 36 | 37 | #endif 38 | -------------------------------------------------------------------------------- /DbFileBackend.h: -------------------------------------------------------------------------------- 1 | /* single raw file DB backend 2 | * by Albert Zeyer, 2011 3 | * code under LGPL 4 | */ 5 | 6 | #ifndef __AZ__DBFILEBACKEND_H__ 7 | #define __AZ__DBFILEBACKEND_H__ 8 | 9 | #include "Db.h" 10 | #include "Mutex.h" 11 | 12 | struct DbFile_TreeChunk; 13 | 14 | struct DbFileBackend : DbIntf { 15 | Mutex mutex; 16 | FILE* file; 17 | DbFile_TreeChunk* rootChunk; 18 | size_t fileSize; 19 | std::string filename; 20 | bool readonly; 21 | 22 | DbFileBackend(const std::string& dbfilename = "db.pngdb", bool ro = false) 23 | : file(NULL), rootChunk(NULL), fileSize(0), filename(dbfilename), readonly(ro) {} 24 | ~DbFileBackend() { reset(); } 25 | void reset(); 26 | Return setReadOnly(bool ro) { readonly = ro; return true; } 27 | Return init(); 28 | Return push(/*out*/ DbEntryId& id, const DbEntry& entry); 29 | Return get(/*out*/ DbEntry& entry, const DbEntryId& id); 30 | Return pushToDir(const std::string& path, const DbDirEntry& dirEntry); 31 | Return getDir(/*out*/ std::list& dirList, const std::string& path); 32 | Return setFileRef(/*can be empty*/ const DbEntryId& id, const std::string& path); 33 | Return getFileRef(/*out (can be empty)*/ DbEntryId& id, const std::string& path); 34 | }; 35 | 36 | #endif 37 | -------------------------------------------------------------------------------- /Endianess.h: -------------------------------------------------------------------------------- 1 | /* 2 | (from OpenLieroX) 3 | 4 | makros for endianess / swapping / conversions 5 | 6 | Code under LGPL 7 | by Albert Zeyer, Dark Charlie, 23-06-2007 8 | */ 9 | 10 | #ifndef __ENDIANSWAP_H__ 11 | #define __ENDIANSWAP_H__ 12 | 13 | #include 14 | 15 | #if !defined(BYTE_ORDER) 16 | # error BYTE_ORDER not defined 17 | #endif 18 | #if BYTE_ORDER == LITTLE_ENDIAN 19 | # define EndianSwap(x) ; 20 | # define BEndianSwap(x) ByteSwap5(x); 21 | # define GetEndianSwapped(x) (x) 22 | #elif BYTEORDER == BIG_ENDIAN 23 | # define EndianSwap(x) ByteSwap5(x); 24 | # define BEndianSwap(x) ; 25 | # define GetEndianSwapped(x) (GetByteSwapped(x)) 26 | #else 27 | # error unknown ENDIAN type 28 | #endif 29 | 30 | #include 31 | #include "StaticAssert.h" 32 | 33 | template 34 | void ByteSwap(unsigned char * b) { 35 | static_assert(n == 1 || n % 2 == 0, n_must_be_equal); 36 | for(int i = 0; i < n/2; ++i) { 37 | std::swap(b[i], b[n - i - 1]); 38 | } 39 | } 40 | 41 | template 42 | T GetByteSwapped(T b) { 43 | ByteSwap(&b); 44 | return b; 45 | } 46 | 47 | template 48 | void ByteSwap5(T& x) { 49 | ByteSwap((unsigned char*) &x); 50 | } 51 | 52 | #endif 53 | 54 | -------------------------------------------------------------------------------- /kyotocabinet/kcdbext.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Database extension 3 | * Copyright (C) 2009-2011 FAL Labs 4 | * This file is part of Kyoto Cabinet. 5 | * This program is free software: you can redistribute it and/or modify it under the terms of 6 | * the GNU General Public License as published by the Free Software Foundation, either version 7 | * 3 of the License, or any later version. 8 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 9 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 10 | * See the GNU General Public License for more details. 11 | * You should have received a copy of the GNU General Public License along with this program. 12 | * If not, see . 13 | *************************************************************************************************/ 14 | 15 | 16 | #include "kcdbext.h" 17 | #include "myconf.h" 18 | 19 | namespace kyotocabinet { // common namespace 20 | 21 | 22 | // There is no implementation now. 23 | 24 | 25 | } // common namespace 26 | 27 | // END OF FILE 28 | -------------------------------------------------------------------------------- /kyotocabinet/kcplantdb.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Plant database 3 | * Copyright (C) 2009-2011 FAL Labs 4 | * This file is part of Kyoto Cabinet. 5 | * This program is free software: you can redistribute it and/or modify it under the terms of 6 | * the GNU General Public License as published by the Free Software Foundation, either version 7 | * 3 of the License, or any later version. 8 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 9 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 10 | * See the GNU General Public License for more details. 11 | * You should have received a copy of the GNU General Public License along with this program. 12 | * If not, see . 13 | *************************************************************************************************/ 14 | 15 | 16 | #include "kcplantdb.h" 17 | #include "myconf.h" 18 | 19 | namespace kyotocabinet { // common namespace 20 | 21 | 22 | // There is no implementation now. 23 | 24 | 25 | } // common namespace 26 | 27 | // END OF FILE 28 | -------------------------------------------------------------------------------- /kyotocabinet/kcstashdb.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Stash database 3 | * Copyright (C) 2009-2011 FAL Labs 4 | * This file is part of Kyoto Cabinet. 5 | * This program is free software: you can redistribute it and/or modify it under the terms of 6 | * the GNU General Public License as published by the Free Software Foundation, either version 7 | * 3 of the License, or any later version. 8 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 9 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 10 | * See the GNU General Public License for more details. 11 | * You should have received a copy of the GNU General Public License along with this program. 12 | * If not, see . 13 | *************************************************************************************************/ 14 | 15 | 16 | #include "kcstashdb.h" 17 | #include "myconf.h" 18 | 19 | namespace kyotocabinet { // common namespace 20 | 21 | 22 | // There is no implementation now. 23 | 24 | 25 | } // common namespace 26 | 27 | // END OF FILE 28 | -------------------------------------------------------------------------------- /kyotocabinet/kccachedb.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Cache hash database 3 | * Copyright (C) 2009-2011 FAL Labs 4 | * This file is part of Kyoto Cabinet. 5 | * This program is free software: you can redistribute it and/or modify it under the terms of 6 | * the GNU General Public License as published by the Free Software Foundation, either version 7 | * 3 of the License, or any later version. 8 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 9 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 10 | * See the GNU General Public License for more details. 11 | * You should have received a copy of the GNU General Public License along with this program. 12 | * If not, see . 13 | *************************************************************************************************/ 14 | 15 | 16 | #include "kccachedb.h" 17 | #include "myconf.h" 18 | 19 | namespace kyotocabinet { // common namespace 20 | 21 | 22 | // There is no implementation now. 23 | 24 | 25 | } // common namespace 26 | 27 | // END OF FILE 28 | -------------------------------------------------------------------------------- /kyotocabinet/kcdirdb.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Directory hash database 3 | * Copyright (C) 2009-2011 FAL Labs 4 | * This file is part of Kyoto Cabinet. 5 | * This program is free software: you can redistribute it and/or modify it under the terms of 6 | * the GNU General Public License as published by the Free Software Foundation, either version 7 | * 3 of the License, or any later version. 8 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 9 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 10 | * See the GNU General Public License for more details. 11 | * You should have received a copy of the GNU General Public License along with this program. 12 | * If not, see . 13 | *************************************************************************************************/ 14 | 15 | 16 | #include "kcdirdb.h" 17 | #include "myconf.h" 18 | 19 | namespace kyotocabinet { // common namespace 20 | 21 | 22 | // There is no implementation now. 23 | 24 | 25 | } // common namespace 26 | 27 | // END OF FILE 28 | -------------------------------------------------------------------------------- /kyotocabinet/kchashdb.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * File hash database 3 | * Copyright (C) 2009-2011 FAL Labs 4 | * This file is part of Kyoto Cabinet. 5 | * This program is free software: you can redistribute it and/or modify it under the terms of 6 | * the GNU General Public License as published by the Free Software Foundation, either version 7 | * 3 of the License, or any later version. 8 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 9 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 10 | * See the GNU General Public License for more details. 11 | * You should have received a copy of the GNU General Public License along with this program. 12 | * If not, see . 13 | *************************************************************************************************/ 14 | 15 | 16 | #include "kchashdb.h" 17 | #include "myconf.h" 18 | 19 | namespace kyotocabinet { // common namespace 20 | 21 | 22 | // There is no implementation now. 23 | 24 | 25 | } // common namespace 26 | 27 | // END OF FILE 28 | -------------------------------------------------------------------------------- /kyotocabinet/kcmap.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Data mapping structures 3 | * Copyright (C) 2009-2011 FAL Labs 4 | * This file is part of Kyoto Cabinet. 5 | * This program is free software: you can redistribute it and/or modify it under the terms of 6 | * the GNU General Public License as published by the Free Software Foundation, either version 7 | * 3 of the License, or any later version. 8 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 9 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 10 | * See the GNU General Public License for more details. 11 | * You should have received a copy of the GNU General Public License along with this program. 12 | * If not, see . 13 | *************************************************************************************************/ 14 | 15 | 16 | #include "kcmap.h" 17 | #include "myconf.h" 18 | 19 | namespace kyotocabinet { // common namespace 20 | 21 | 22 | // There is no implementation now. 23 | 24 | 25 | } // common namespace 26 | 27 | // END OF FILE 28 | -------------------------------------------------------------------------------- /kyotocabinet/kcpolydb.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Polymorphic database 3 | * Copyright (C) 2009-2011 FAL Labs 4 | * This file is part of Kyoto Cabinet. 5 | * This program is free software: you can redistribute it and/or modify it under the terms of 6 | * the GNU General Public License as published by the Free Software Foundation, either version 7 | * 3 of the License, or any later version. 8 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 9 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 10 | * See the GNU General Public License for more details. 11 | * You should have received a copy of the GNU General Public License along with this program. 12 | * If not, see . 13 | *************************************************************************************************/ 14 | 15 | 16 | #include "kcpolydb.h" 17 | #include "myconf.h" 18 | 19 | namespace kyotocabinet { // common namespace 20 | 21 | 22 | // There is no implementation now. 23 | 24 | 25 | } // common namespace 26 | 27 | // END OF FILE 28 | -------------------------------------------------------------------------------- /kyotocabinet/kcprotodb.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Prototype database 3 | * Copyright (C) 2009-2011 FAL Labs 4 | * This file is part of Kyoto Cabinet. 5 | * This program is free software: you can redistribute it and/or modify it under the terms of 6 | * the GNU General Public License as published by the Free Software Foundation, either version 7 | * 3 of the License, or any later version. 8 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 9 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 10 | * See the GNU General Public License for more details. 11 | * You should have received a copy of the GNU General Public License along with this program. 12 | * If not, see . 13 | *************************************************************************************************/ 14 | 15 | 16 | #include "kcprotodb.h" 17 | #include "myconf.h" 18 | 19 | namespace kyotocabinet { // common namespace 20 | 21 | 22 | // There is no implementation now. 23 | 24 | 25 | } // common namespace 26 | 27 | // END OF FILE 28 | -------------------------------------------------------------------------------- /DbPng.h: -------------------------------------------------------------------------------- 1 | /* PNG data <-> DB interface 2 | * by Albert Zeyer, 2011 3 | * code under LGPL 4 | */ 5 | 6 | #ifndef __AZ__DBPNG_H__ 7 | #define __AZ__DBPNG_H__ 8 | 9 | #include "Png.h" 10 | #include "Db.h" 11 | #include "Utils.h" 12 | #include 13 | #include 14 | #include 15 | 16 | struct DbPngEntryWriter { 17 | PngReader reader; 18 | DbIntf* db; 19 | std::list contentChunkEntries; 20 | std::list contentDataEntries; 21 | DbEntryId contentId; 22 | 23 | DbPngEntryWriter(FILE* f, DbIntf* _db) : reader(f), db(_db) {} 24 | Return next(); 25 | operator bool() const { return !reader.hasFinishedReading; } 26 | }; 27 | 28 | struct DbPngEntryBlockList { 29 | uint8_t blockHeight; 30 | size_t scanlineWidth; 31 | std::list blocks; 32 | DbPngEntryBlockList() : scanlineWidth(0), blockHeight(0) {} 33 | }; 34 | 35 | struct DbPngEntryReader { 36 | PngWriter writer; 37 | DbIntf* db; 38 | DbEntryId contentId; 39 | std::list contentEntries; 40 | bool haveContentEntries; 41 | DbPngEntryBlockList blockList; 42 | 43 | DbPngEntryReader(WriteCallbackIntf* w, DbIntf* _db, const DbEntryId& _contentId) 44 | : writer(w), db(_db), contentId(_contentId), haveContentEntries(false) {} 45 | Return next(); 46 | operator bool() const { return !writer.hasFinishedWriting; } 47 | }; 48 | 49 | #endif 50 | -------------------------------------------------------------------------------- /kyotocabinet/kcdb.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Database interface 3 | * Copyright (C) 2009-2011 FAL Labs 4 | * This file is part of Kyoto Cabinet. 5 | * This program is free software: you can redistribute it and/or modify it under the terms of 6 | * the GNU General Public License as published by the Free Software Foundation, either version 7 | * 3 of the License, or any later version. 8 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 9 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 10 | * See the GNU General Public License for more details. 11 | * You should have received a copy of the GNU General Public License along with this program. 12 | * If not, see . 13 | *************************************************************************************************/ 14 | 15 | 16 | #include "kcdb.h" 17 | #include "myconf.h" 18 | 19 | namespace kyotocabinet { // common namespace 20 | 21 | 22 | /** Special pointer for no operation. */ 23 | const char* const DB::Visitor::NOP = (const char*)0; 24 | 25 | /** Special pointer to remove the record. */ 26 | const char* const DB::Visitor::REMOVE = (const char*)1; 27 | 28 | 29 | } // common namespace 30 | 31 | // END OF FILE 32 | -------------------------------------------------------------------------------- /hiredis/COPYING: -------------------------------------------------------------------------------- 1 | Copyright (c) 2006-2009, Salvatore Sanfilippo 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 5 | 6 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 7 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 8 | * Neither the name of Redis nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 9 | 10 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 11 | -------------------------------------------------------------------------------- /db-extract-file.cpp: -------------------------------------------------------------------------------- 1 | /* simple tool to extract a single file from the DB 2 | * by Albert Zeyer, 2011 3 | * code under LGPL 4 | */ 5 | 6 | #include "Png.h" 7 | #include "DbDefBackend.h" 8 | #include "DbPng.h" 9 | #include "StringUtils.h" 10 | #include "FileUtils.h" 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | using namespace std; 17 | 18 | Return _main(const std::string& filename) { 19 | DbDefBackend db; 20 | db.setReadOnly(true); 21 | ASSERT( db.init() ); 22 | 23 | DbEntryId fileEntryId; 24 | ASSERT( db.getFileRef(fileEntryId, filename) ); 25 | cout << "entry id: " << hexString(fileEntryId) << endl; 26 | 27 | std::string extract_fn = baseFilename(filename); 28 | FILE* f = fopen(extract_fn.c_str(), "wb"); 29 | if(f == NULL) 30 | return "cannot open " + extract_fn; 31 | 32 | if(!fileEntryId.empty()) { 33 | FileWriteCallback writer(f); 34 | DbPngEntryReader dbPngReader(&writer, &db, fileEntryId); 35 | while(dbPngReader) 36 | ASSERT( dbPngReader.next() ); 37 | } 38 | 39 | cout << "wrote " << ftell(f) << " bytes" << endl; 40 | fclose(f); 41 | 42 | return true; 43 | } 44 | 45 | int main(int argc, char** argv) { 46 | if(argc <= 1) { 47 | cerr << "please give me a filename" << endl; 48 | return 1; 49 | } 50 | 51 | std::string filename = argv[1]; 52 | srandom(time(NULL)); 53 | Return r = _main(filename); 54 | if(!r) { 55 | cerr << "error: " << r.errmsg << endl; 56 | return 1; 57 | } 58 | 59 | cout << "success" << endl; 60 | return 0; 61 | } 62 | -------------------------------------------------------------------------------- /db-push.cpp: -------------------------------------------------------------------------------- 1 | /* tool to push some entry to the DB 2 | * by Albert Zeyer, 2011 3 | * code under LGPL 4 | */ 5 | 6 | #include "Png.h" 7 | #include "DbDefBackend.h" 8 | #include "DbPng.h" 9 | #include "StringUtils.h" 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | using namespace std; 16 | 17 | Return _main(const std::string& filename) { 18 | FILE* f = fopen(filename.c_str(), "rb"); 19 | if(f == NULL) 20 | return "cannot open " + filename; 21 | 22 | DbDefBackend db; 23 | ASSERT( db.init() ); 24 | DbPngEntryWriter dbPngWriter(f, &db); 25 | while(dbPngWriter) 26 | ASSERT( dbPngWriter.next() ); 27 | 28 | ASSERT( db.pushToDir("", DbDirEntry::File(baseFilename(filename), ftell(f))) ); 29 | ASSERT( db.setFileRef(dbPngWriter.contentId, "/" + baseFilename(filename)) ); 30 | fclose(f); 31 | 32 | cout << "content id: " << hexString(dbPngWriter.contentId) << endl; 33 | cout << "num content entries: " << dbPngWriter.contentChunkEntries.size() + dbPngWriter.contentDataEntries.size() << endl; 34 | cout << "db stats: push new: " << db.stats.pushNew << endl; 35 | cout << "db stats: push reuse: " << db.stats.pushReuse << endl; 36 | 37 | return true; 38 | } 39 | 40 | int main(int argc, char** argv) { 41 | if(argc <= 1) { 42 | cerr << "please give me a filename" << endl; 43 | return 1; 44 | } 45 | 46 | std::string filename = argv[1]; 47 | srandom(time(NULL)); 48 | Return r = _main(filename); 49 | if(!r) { 50 | cerr << "error: " << r.errmsg << endl; 51 | return 1; 52 | } 53 | 54 | cout << "success" << endl; 55 | return 0; 56 | } 57 | -------------------------------------------------------------------------------- /test-png-reader.cpp: -------------------------------------------------------------------------------- 1 | /* demo for PNG reader 2 | * by Albert Zeyer, 2011 3 | * code under LGPL 4 | */ 5 | 6 | #include "Png.h" 7 | 8 | #include 9 | #include 10 | using namespace std; 11 | 12 | int main(int argc, char** argv) { 13 | if(argc <= 1) { 14 | cout << "please give me a filename" << endl; 15 | return 1; 16 | } 17 | 18 | FILE* f = fopen(argv[1], "rb"); 19 | PngReader reader(f); 20 | bool haveSeenHeader = false; 21 | while(!reader.hasFinishedReading) { 22 | Return r = reader.read(); 23 | if(!r) { 24 | cout << "error: " << r.errmsg << endl; 25 | return 1; 26 | } 27 | 28 | if(!haveSeenHeader && reader.gotHeader) { 29 | cout << "header: width=" << reader.header.width << ", height=" << reader.header.height << endl; 30 | cout << "header: bitDepth=" << (int)reader.header.bitDepth << endl; 31 | cout << "header: colorType=" << (int)reader.header.colourType << endl; 32 | cout << "header: bytesPerPixel=" << (int)reader.header.bytesPerPixel() << endl; 33 | cout << "header: compressionMethod=" << (int)reader.header.compressionMethod << endl; 34 | cout << "header: filterMethod=" << (int)reader.header.filterMethod << endl; 35 | cout << "header: interlaceMethod=" << (int)reader.header.interlaceMethod << endl; 36 | haveSeenHeader = true; 37 | } 38 | 39 | size_t s = 0; 40 | for(std::list::iterator i = reader.scanlines.begin(); i != reader.scanlines.end(); ++i) 41 | s += i->size(); 42 | cout << "at " << ftell(f) << ": got " << s << " uncompressed" << endl; 43 | } 44 | 45 | cout << "success" << endl; 46 | return 0; 47 | } 48 | -------------------------------------------------------------------------------- /db-list-dir.cpp: -------------------------------------------------------------------------------- 1 | /* tool to list a dir from the DB 2 | * by Albert Zeyer, 2011 3 | * code under LGPL 4 | */ 5 | 6 | #include "Png.h" 7 | #include "DbDefBackend.h" 8 | #include "DbPng.h" 9 | #include "StringUtils.h" 10 | #include "FileUtils.h" 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | using namespace std; 17 | 18 | DbDefBackend* db = NULL; 19 | 20 | static Return listDir(const std::string& path) { 21 | std::list dirList; 22 | ASSERT( db->getDir(dirList, path) ); 23 | 24 | for(std::list::iterator i = dirList.begin(); i != dirList.end(); ++i) { 25 | if(i->mode & S_IFREG) 26 | cout << "file: "; 27 | else if(i->mode & S_IFDIR) 28 | cout << "dir: "; 29 | else 30 | cout << hexString(i->mode) << ": "; 31 | cout << path << "/" << i->name; 32 | if(i->mode & S_IFREG) { 33 | cout << ", " << i->size << " bytes"; 34 | DbEntryId id; 35 | ASSERT( db->getFileRef(id, path + "/" + i->name) ); 36 | if(id.size() > 0) 37 | cout << ", ref=" << hexString(id); 38 | else 39 | cout << ", empty"; 40 | } 41 | else if(i->mode & S_IFDIR) 42 | cout << "/"; 43 | cout << endl; 44 | if(i->mode & S_IFDIR) 45 | ASSERT( listDir(path + "/" + i->name) ); 46 | } 47 | 48 | return true; 49 | } 50 | 51 | static Return _main() { 52 | DbDefBackend dbInst; 53 | db = &dbInst; 54 | db->setReadOnly(true); 55 | ASSERT( db->init() ); 56 | ASSERT( listDir("") ); 57 | return true; 58 | } 59 | 60 | int main(int argc, char** argv) { 61 | srandom(time(NULL)); 62 | 63 | Return r = _main(); 64 | if(!r) { 65 | cerr << "error: " << r.errmsg << endl; 66 | return 1; 67 | } 68 | 69 | cout << "success" << endl; 70 | return 0; 71 | } 72 | -------------------------------------------------------------------------------- /pnginfo.cpp: -------------------------------------------------------------------------------- 1 | /* print some PNG info/stats 2 | * by Albert Zeyer, 2011 3 | * code under LGPL 4 | */ 5 | 6 | #include "Png.h" 7 | 8 | #include 9 | #include 10 | using namespace std; 11 | 12 | int main(int argc, char** argv) { 13 | if(argc <= 1) { 14 | cerr << "please give me a filename" << endl; 15 | return 1; 16 | } 17 | 18 | FILE* f = fopen(argv[1], "rb"); 19 | if(f == NULL) { 20 | cerr << "error: cannot open " << argv[1] << endl; 21 | return 1; 22 | } 23 | PngReader reader(f); 24 | bool haveSeenHeader = false; 25 | while(!reader.hasFinishedReading) { 26 | Return r = reader.read(); 27 | if(!r) { 28 | cerr << "error: " << r.errmsg << endl; 29 | return 1; 30 | } 31 | 32 | if(!haveSeenHeader && reader.gotHeader) { 33 | cout << "header: width=" << reader.header.width << ", height=" << reader.header.height << endl; 34 | cout << "header: bitDepth=" << (int)reader.header.bitDepth << endl; 35 | cout << "header: colorType=" << (int)reader.header.colourType << endl; 36 | cout << "header: bytesPerPixel=" << (int)reader.header.bytesPerPixel() << endl; 37 | cout << "header: compressionMethod=" << (int)reader.header.compressionMethod << endl; 38 | cout << "header: filterMethod=" << (int)reader.header.filterMethod << endl; 39 | cout << "header: interlaceMethod=" << (int)reader.header.interlaceMethod << endl; 40 | haveSeenHeader = true; 41 | } 42 | } 43 | 44 | size_t s = 0; 45 | for(std::list::iterator i = reader.scanlines.begin(); i != reader.scanlines.end(); ++i) 46 | s += i->size(); 47 | cout << "uncompressed data stream size: " << s << endl; 48 | cout << "number scanlines: " << reader.scanlines.size() << endl; 49 | 50 | cout << "success" << endl; 51 | return 0; 52 | } 53 | -------------------------------------------------------------------------------- /Utils.h: -------------------------------------------------------------------------------- 1 | /* 2 | OpenLieroX 3 | 4 | various utilities 5 | 6 | code under LGPL 7 | created 01-05-2007 8 | by Albert Zeyer and Dark Charlie 9 | */ 10 | 11 | #ifndef __UTILS_H__ 12 | #define __UTILS_H__ 13 | 14 | #include // for size_t 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include "Return.h" 20 | 21 | template 22 | bool isSameType(const _src& obj1, const _dst& obj2) { 23 | if(sizeof(_dst) < sizeof(_src)) return isSameType(obj2, obj1); 24 | return dynamic_cast(&obj1) != NULL; 25 | } 26 | 27 | 28 | 29 | template void SafeAdvance(Iter& it, size_t count, const Iter& end) { 30 | for (size_t i=0; i < count && it != end; i++, it++) {} 31 | } 32 | 33 | 34 | template 35 | int highestBit(T d) { 36 | for(int i = sizeof(T) * 8 - 1; i >= 0; --i) 37 | if((1 << i) <= d) return i + 1; 38 | return 0; 39 | } 40 | 41 | 42 | class DontCopyTag { 43 | public: 44 | DontCopyTag() {} 45 | private: 46 | DontCopyTag(const DontCopyTag&) { assert(false); } 47 | DontCopyTag& operator=(const DontCopyTag&) { assert(false); return *this; } 48 | }; 49 | 50 | 51 | template 52 | std::vector ListAsVector(const std::list& l) { 53 | return std::vector(l.begin(), l.end()); 54 | } 55 | 56 | template std::set Set(T v1) { std::set ret; ret.insert(v1); return ret; } 57 | template std::set Set(T v1, T v2) { std::set ret; ret.insert(v1); ret.insert(v2); return ret; } 58 | template std::set Set(T v1, T v2, T v3) { std::set ret; ret.insert(v1); ret.insert(v2); ret.insert(v3); return ret; } 59 | 60 | 61 | struct WriteCallbackIntf { 62 | virtual Return write(const char* data, size_t s) = 0; 63 | Return write(const std::string& data) { return write(&data[0], data.size()); } 64 | }; 65 | 66 | 67 | 68 | #endif 69 | 70 | -------------------------------------------------------------------------------- /hiredis/util.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2009-2010, Salvatore Sanfilippo 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, 9 | * this list of conditions and the following disclaimer. 10 | * * Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * * Neither the name of Redis nor the names of its contributors may be used 14 | * to endorse or promote products derived from this software without 15 | * specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | * POSSIBILITY OF SUCH DAMAGE. 28 | */ 29 | 30 | #ifndef __UTIL_H 31 | #define __UTIL_H 32 | #include 33 | 34 | /* Abort on out of memory */ 35 | static void redisOOM(void) { 36 | fprintf(stderr,"Out of memory in hiredis"); 37 | exit(1); 38 | } 39 | 40 | #endif 41 | -------------------------------------------------------------------------------- /kyotocabinet/kyotocabinet.def: -------------------------------------------------------------------------------- 1 | EXPORTS 2 | KCVERSION = KCVERSION CONSTANT 3 | KCVISNOP = KCVISNOP CONSTANT 4 | KCVISREMOVE = KCVISREMOVE CONSTANT 5 | kcatof = kcatof 6 | kcatoi = kcatoi 7 | kcatoix = kcatoix 8 | kcchkinf = kcchkinf 9 | kcchknan = kcchknan 10 | kccuraccept = kccuraccept 11 | kccurdb = kccurdb 12 | kccurdel = kccurdel 13 | kccurecode = kccurecode 14 | kccuremsg = kccuremsg 15 | kccurget = kccurget 16 | kccurgetkey = kccurgetkey 17 | kccurgetvalue = kccurgetvalue 18 | kccurjump = kccurjump 19 | kccurjumpback = kccurjumpback 20 | kccurjumpbackkey = kccurjumpbackkey 21 | kccurjumpkey = kccurjumpkey 22 | kccurremove = kccurremove 23 | kccursetvalue = kccursetvalue 24 | kccurstep = kccurstep 25 | kccurstepback = kccurstepback 26 | kcdbaccept = kcdbaccept 27 | kcdbadd = kcdbadd 28 | kcdbappend = kcdbappend 29 | kcdbbegintran = kcdbbegintran 30 | kcdbbegintrantry = kcdbbegintrantry 31 | kcdbcas = kcdbcas 32 | kcdbclear = kcdbclear 33 | kcdbclose = kcdbclose 34 | kcdbcopy = kcdbcopy 35 | kcdbcount = kcdbcount 36 | kcdbcursor = kcdbcursor 37 | kcdbdel = kcdbdel 38 | kcdbdumpsnap = kcdbdumpsnap 39 | kcdbecode = kcdbecode 40 | kcdbemsg = kcdbemsg 41 | kcdbendtran = kcdbendtran 42 | kcdbget = kcdbget 43 | kcdbgetbuf = kcdbgetbuf 44 | kcdbincrdouble = kcdbincrdouble 45 | kcdbincrint = kcdbincrint 46 | kcdbiterate = kcdbiterate 47 | kcdbloadsnap = kcdbloadsnap 48 | kcdbmatchprefix = kcdbmatchprefix 49 | kcdbmatchregex = kcdbmatchregex 50 | kcdbmerge = kcdbmerge 51 | kcdbnew = kcdbnew 52 | kcdbopen = kcdbopen 53 | kcdbpath = kcdbpath 54 | kcdbremove = kcdbremove 55 | kcdbreplace = kcdbreplace 56 | kcdbset = kcdbset 57 | kcdbsize = kcdbsize 58 | kcdbstatus = kcdbstatus 59 | kcdbsync = kcdbsync 60 | kcecodename = kcecodename 61 | kcfree = kcfree 62 | kchashfnv = kchashfnv 63 | kchashmurmur = kchashmurmur 64 | kcinf = kcinf 65 | kcmalloc = kcmalloc 66 | kcnan = kcnan 67 | kctime = kctime 68 | -------------------------------------------------------------------------------- /kyotocabinet/kccompare.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Comparator functions 3 | * Copyright (C) 2009-2011 FAL Labs 4 | * This file is part of Kyoto Cabinet. 5 | * This program is free software: you can redistribute it and/or modify it under the terms of 6 | * the GNU General Public License as published by the Free Software Foundation, either version 7 | * 3 of the License, or any later version. 8 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 9 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 10 | * See the GNU General Public License for more details. 11 | * You should have received a copy of the GNU General Public License along with this program. 12 | * If not, see . 13 | *************************************************************************************************/ 14 | 15 | 16 | #include "kccompare.h" 17 | #include "myconf.h" 18 | 19 | namespace kyotocabinet { // common namespace 20 | 21 | 22 | /** 23 | * Prepared pointer of the comparator in the lexical order. 24 | */ 25 | LexicalComparator lexicalfunc; 26 | LexicalComparator* const LEXICALCOMP = &lexicalfunc; 27 | 28 | 29 | /** 30 | * Prepared pointer of the comparator in the lexical descending order. 31 | */ 32 | LexicalDescendingComparator lexicaldescfunc; 33 | LexicalDescendingComparator* const LEXICALDESCCOMP = &lexicaldescfunc; 34 | 35 | 36 | /** 37 | * Prepared pointer of the comparator in the decimal order. 38 | */ 39 | DecimalComparator decimalfunc; 40 | DecimalComparator* const DECIMALCOMP = &decimalfunc; 41 | 42 | 43 | /** 44 | * Prepared pointer of the comparator in the decimal descending order. 45 | */ 46 | DecimalDescendingComparator decimaldescfunc; 47 | DecimalDescendingComparator* const DECIMALDESCCOMP = &decimaldescfunc; 48 | 49 | 50 | } // common namespace 51 | 52 | // END OF FILE 53 | -------------------------------------------------------------------------------- /hiredis/net.h: -------------------------------------------------------------------------------- 1 | /* Extracted from anet.c to work properly with Hiredis error reporting. 2 | * 3 | * Copyright (c) 2006-2010, Salvatore Sanfilippo 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * * Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * * Neither the name of Redis nor the names of its contributors may be used 15 | * to endorse or promote products derived from this software without 16 | * specific prior written permission. 17 | * 18 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 22 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28 | * POSSIBILITY OF SUCH DAMAGE. 29 | */ 30 | 31 | #ifndef __NET_H 32 | #define __NET_H 33 | 34 | #include "hiredis.h" 35 | 36 | #if defined(__sun) 37 | #define AF_LOCAL AF_UNIX 38 | #endif 39 | 40 | int redisContextConnectTcp(redisContext *c, const char *addr, int port); 41 | int redisContextConnectUnix(redisContext *c, const char *path); 42 | 43 | #endif 44 | -------------------------------------------------------------------------------- /FileUtils.cpp: -------------------------------------------------------------------------------- 1 | /* File utils 2 | * by Albert Zeyer, 2011 3 | * code under LGPL 4 | */ 5 | 6 | #include "FileUtils.h" 7 | #include // mkdir 8 | #include 9 | #include 10 | 11 | #include 12 | using namespace std; 13 | 14 | Return fread_all(FILE* fp, std::string& out) { 15 | fseek(fp, 0, SEEK_END); 16 | size_t size = ftell(fp); 17 | fseek(fp, 0, SEEK_SET); 18 | 19 | out = std::string(size, '\0'); 20 | char* outP = &out[0]; 21 | size_t remaining = size; 22 | 23 | while(remaining > 0) { 24 | if(feof(fp)) 25 | return "end-of-file"; 26 | if(ferror(fp)) 27 | return "file-read-error"; 28 | size_t n = fread(outP, 1, remaining, fp); 29 | remaining -= n; 30 | outP += n; 31 | } 32 | 33 | return true; 34 | } 35 | 36 | Return fwrite_bytes(FILE* fp, const char* d, size_t size) { 37 | while(size > 0) { 38 | if(ferror(fp)) 39 | return "file-write-error"; 40 | size_t n = fwrite(d, 1, size, fp); 41 | d += n; 42 | size -= n; 43 | } 44 | 45 | return true; 46 | } 47 | 48 | Return fwrite_all(FILE* fp, const std::string& in) { 49 | return fwrite_bytes(fp, &in[0], in.size()); 50 | } 51 | 52 | DirIter::~DirIter() { 53 | if(dir != NULL) { 54 | closedir(dir); 55 | dir = NULL; 56 | } 57 | } 58 | 59 | void DirIter::next() { 60 | filename = ""; 61 | if(dir == NULL) return; 62 | dirent* entry = readdir(dir); 63 | if(entry == NULL) return; 64 | filename = std::string(entry->d_name, entry->d_namlen); 65 | } 66 | 67 | static Return __createDir(const std::string& dir, mode_t mode = 0777) { 68 | if(mkdir(dir.c_str(), mode) != 0) { 69 | if(errno == EEXIST) return true; // no error 70 | return std::string() + "cannot create dir '" + dir + "': " + strerror(errno); 71 | } 72 | return true; 73 | } 74 | 75 | Return createRecDir(const std::string& abs_filename, bool last_is_dir) { 76 | std::string tmp; 77 | std::string::const_iterator f = abs_filename.begin(); 78 | for(tmp = ""; f != abs_filename.end(); f++) { 79 | if(*f == '\\' || *f == '/') 80 | ASSERT( __createDir(tmp) ); 81 | tmp += *f; 82 | } 83 | if(last_is_dir) 84 | ASSERT( __createDir(tmp) ); 85 | return true; 86 | } 87 | -------------------------------------------------------------------------------- /db-push-dir.cpp: -------------------------------------------------------------------------------- 1 | /* tool to push a dir to the DB 2 | * by Albert Zeyer, 2011 3 | * code under LGPL 4 | */ 5 | 6 | #include "Png.h" 7 | #include "DbDefBackend.h" 8 | #include "DbPng.h" 9 | #include "StringUtils.h" 10 | #include "FileUtils.h" 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | using namespace std; 17 | 18 | Return _main(const std::string& dirname) { 19 | DbDefBackend db; 20 | ASSERT( db.init() ); 21 | 22 | DirIter dir(dirname); 23 | if(dir.dir == NULL) 24 | return "cannot open directory " + dirname; 25 | 26 | for(; dir; dir.next()) { 27 | if(dir.filename.size() <= 4) continue; 28 | if(dir.filename.substr(dir.filename.size()-4) != ".png") continue; 29 | std::string filename = dirname + "/" + dir.filename; 30 | 31 | DbEntryId ref; 32 | if(db.getFileRef(ref, "/" + baseFilename(filename))) 33 | // skip files we already have in DB 34 | continue; 35 | 36 | FILE* f = fopen(filename.c_str(), "rb"); 37 | if(f == NULL) { 38 | cerr << "error: " << dir.filename << ": cannot open file" << endl; 39 | continue; 40 | } 41 | 42 | DbPngEntryWriter dbPngWriter(f, &db); 43 | while(dbPngWriter) { 44 | Return r = dbPngWriter.next(); 45 | if(!r) { 46 | cerr << "error: " << dir.filename << ": " << r.errmsg << endl; 47 | break; 48 | } 49 | } 50 | db.pushToDir("", DbDirEntry::File(baseFilename(filename), ftell(f))); 51 | db.setFileRef(dbPngWriter.contentId, "/" + baseFilename(filename)); 52 | fclose(f); 53 | 54 | cout << dir.filename << ": " 55 | << (100.0f * float(db.stats.pushReuse) / (db.stats.pushNew + db.stats.pushReuse)) << "%, " 56 | << db.stats.pushReuse << " / " << db.stats.pushNew 57 | << endl; 58 | } 59 | 60 | return true; 61 | } 62 | 63 | int main(int argc, char** argv) { 64 | if(argc <= 1) { 65 | cerr << "please give me a dirname" << endl; 66 | return 1; 67 | } 68 | 69 | srandom(time(NULL)); 70 | 71 | std::string dirname = argv[1]; 72 | Return r = _main(dirname); 73 | if(!r) { 74 | cerr << "error: " << r.errmsg << endl; 75 | return 1; 76 | } 77 | 78 | cout << "success" << endl; 79 | return 0; 80 | } 81 | -------------------------------------------------------------------------------- /compile.sh: -------------------------------------------------------------------------------- 1 | #!/bin/zsh 2 | 3 | # simple compile.sh script 4 | # by Albert Zeyer, 2010 or so 5 | # code under zlib 6 | 7 | cd "$(dirname $0)" 8 | ROOTDIR="$(pwd)" 9 | BUILDDIR="build" 10 | 11 | # $1 - target 12 | # $... - deps 13 | function checkdeps() { 14 | local t="$1" 15 | [ ! -e "$t" ] && return -1 16 | for arg in $*; do 17 | [ "$arg" != "" ] && [ ! -e "$arg" ] && return -1 18 | [ "$arg" != "" ] && [ "$arg" -nt "$t" ] && return -1 19 | done 20 | return 0 21 | } 22 | 23 | # $1 - dep-file 24 | function listdeps() { 25 | sed -e "s/\\\\$//g" -e "s/.*\\.o://g" \ 26 | -e "s/^ *//g" -e "s/ *$//g" \ 27 | "$1" 28 | #-e "s/^/\\\"/g" -e "s/$/\\\"/g" 29 | } 30 | 31 | typeset -A C 32 | C[cpp]=g++ 33 | C[cc]=g++ 34 | C[c]=gcc 35 | 36 | typeset -A Cflags 37 | Cflags[.]="-I kyotocabinet" 38 | 39 | # $1 - c/cpp-file 40 | # will compile the o-file 41 | function srccompile() { 42 | local f="$1" 43 | local fext="${f##*.}" 44 | local o="$BUILDDIR/${f/.${fext}/.o}" 45 | local deps="$BUILDDIR/$f.deps" 46 | mkdir -p "$(dirname "$o")" 47 | [ -e $deps ] && checkdeps $o $f $(listdeps $deps) && echo "uptodate: $o" && return 0 48 | echo "compiling $o" 49 | $C[$fext] -c -MMD -MF $deps -o $o -iquote $(dirname $f) ${(z)Cflags[$f]} ${(z)Cflags[.]} -g $f || exit -1 50 | } 51 | 52 | typeset -A Lflags 53 | Lflags[.]="-lz" 54 | Lflags[db-fuse.cpp]="-lfuse" 55 | [ "$(uname)" = "Darwin" ] && Lflags[db-fuse.cpp]="-lfuse_ino64" 56 | 57 | # $1 - c/cpp-file 58 | # will link all the $OBJS together 59 | function srclink() { 60 | local f="$1" 61 | local fext="${f##*.}" 62 | local o="$BUILDDIR/${f/.${fext}/.o}" 63 | local b="bin/${f/.${fext}/}" 64 | checkdeps $b $OBJS $o && echo "uptodate: $b" && return 0 65 | echo "linking $b" 66 | $C[$fext] $OBJS $o -o $b ${(z)Lflags[$f]} ${(z)Lflags[.]} || exit -1 67 | } 68 | 69 | BINS=("test-png-dumpchunks.cpp" "test-png-reader.cpp" 70 | "pnginfo.cpp" 71 | "db-push.cpp" "db-push-dir.cpp" 72 | "db-list-dir.cpp" "db-extract-file.cpp" 73 | "db-fuse.cpp") 74 | 75 | # compile all sources 76 | OBJS=() 77 | for f in *.cpp; do 78 | srccompile "$f" 79 | [[ ${BINS[(i)$f]} -gt ${#BINS} ]] && \ 80 | OBJS=($OBJS "$BUILDDIR/${f/.cpp/.o}") 81 | done 82 | for f in hiredis/*.c; do 83 | srccompile "$f" 84 | OBJS=($OBJS "$BUILDDIR/${f/.c/.o}") 85 | done 86 | for f in kyotocabinet/*.cc; do 87 | srccompile "$f" 88 | OBJS=($OBJS "$BUILDDIR/${f/.cc/.o}") 89 | done 90 | 91 | mkdir -p bin 92 | for b in $BINS; do 93 | srclink $b 94 | done 95 | -------------------------------------------------------------------------------- /FileUtils.h: -------------------------------------------------------------------------------- 1 | /* File utils 2 | * by Albert Zeyer, 2011 3 | * code under LGPL 4 | */ 5 | 6 | #ifndef __AZ__FILEUTILS_H__ 7 | #define __AZ__FILEUTILS_H__ 8 | 9 | #include "Return.h" 10 | #include "Endianess.h" 11 | #include "Utils.h" 12 | 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | static inline Return fread_bytes(FILE* f, char* d, size_t s) { 20 | if(fread(d, s, 1, f) > 0) return true; 21 | while(s > 0) { 22 | while(fread(d, 1, 1, f) == 0) { 23 | if(feof(f)) 24 | return "end-of-file"; 25 | if(ferror(f)) 26 | return "file-read-error"; 27 | } 28 | ++d; --s; 29 | } 30 | return true; 31 | } 32 | 33 | Return fwrite_bytes(FILE* f, const char* d, size_t s); 34 | 35 | template 36 | static inline Return fread_bytes(FILE* f, T& d) { 37 | ASSERT( fread_bytes(f, &d[0], sizeof(T)/sizeof(d[0])) ); 38 | return true; 39 | } 40 | 41 | template 42 | static Return fread_litendian(FILE* stream, _D& d) { 43 | T data; 44 | ASSERT( fread_bytes(stream, (char*) &data, sizeof(T)) ); 45 | EndianSwap(data); 46 | d = (_D)data; 47 | return true; 48 | } 49 | 50 | template 51 | static Return fread_bigendian(FILE* stream, _D& d) { 52 | T data; 53 | ASSERT( fread_bytes(stream, (char*) &data, sizeof(T)) ); 54 | BEndianSwap(data); 55 | d = (_D)data; 56 | return true; 57 | } 58 | 59 | template 60 | static Return fwrite_litendian(FILE* stream, T data) { 61 | EndianSwap(data); 62 | ASSERT( fwrite_bytes(stream, (const char*) &data, sizeof(T)) ); 63 | return true; 64 | } 65 | 66 | template 67 | static Return fwrite_bigendian(FILE* stream, T data) { 68 | BEndianSwap(data); 69 | ASSERT( fwrite_bytes(stream, (const char*) &data, sizeof(T)) ); 70 | return true; 71 | } 72 | 73 | Return fread_all(FILE* fp, std::string& out); 74 | Return fwrite_all(FILE* fp, const std::string& in); 75 | 76 | struct DirIter : DontCopyTag { 77 | DIR* dir; 78 | std::string filename; 79 | 80 | DirIter(const std::string& dirname) : dir(opendir(dirname.c_str())) { next(); /* set first filename */ } 81 | ~DirIter(); 82 | void next(); 83 | operator bool() const { return dir != NULL && !filename.empty(); } 84 | }; 85 | 86 | Return createRecDir(const std::string& abs_filename, bool last_is_dir = true); 87 | 88 | struct FileWriteCallback : WriteCallbackIntf { 89 | FILE* file; 90 | FileWriteCallback(FILE* f) : file(f) {} 91 | Return write(const char* data, size_t s) { return fwrite_bytes(file, data, s); } 92 | }; 93 | 94 | #endif 95 | -------------------------------------------------------------------------------- /Png.h: -------------------------------------------------------------------------------- 1 | /* PNG parser 2 | * by Albert Zeyer, 2011 3 | * code under LGPL 4 | */ 5 | 6 | #ifndef __AZ__PNG_H__ 7 | #define __AZ__PNG_H__ 8 | 9 | #include "Return.h" 10 | #include "Utils.h" 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | struct PngChunk { 19 | std::string type; 20 | std::string data; 21 | }; 22 | 23 | Return png_read_sig(FILE* f); 24 | Return png_read_chunk(FILE* f, PngChunk& chunk); 25 | Return png_write_sig(WriteCallbackIntf* w); 26 | Return png_write_chunk(WriteCallbackIntf* w, const PngChunk& chunk); 27 | 28 | struct PngHeader { 29 | static const size_t SIZE = 13; 30 | uint32_t width, height; 31 | uint8_t bitDepth; 32 | uint8_t colourType; 33 | uint8_t compressionMethod; 34 | uint8_t filterMethod; 35 | uint8_t interlaceMethod; 36 | 37 | uint8_t samplesPerPixel() { 38 | switch(colourType) { 39 | case 0: return 1; // greyscale 40 | case 2: return 3; // truecolour 41 | case 3: return 1; // indexed 42 | case 4: return 2; // greyscale+alpha 43 | case 6: return 4; // truecolour+alpha 44 | } 45 | return 1; // error anyway; return 1 to avoid infinite loops 46 | } 47 | 48 | uint8_t bytesPerPixel() { 49 | return (samplesPerPixel() * bitDepth + 7) / 8; 50 | } 51 | 52 | uint32_t scanlineSize(uint32_t width) { 53 | return (width * samplesPerPixel() * bitDepth + 7) / 8 + /* filter type byte */ 1; 54 | } 55 | }; 56 | 57 | struct PngInterlacedPos { 58 | short pass; 59 | uint32_t row; 60 | PngInterlacedPos() : pass(0), row(0) {} 61 | void inc(uint32_t height); 62 | size_t scanlineWidth(uint32_t width); 63 | }; 64 | 65 | struct PngReader : DontCopyTag { 66 | FILE* file; 67 | z_stream stream; 68 | PngHeader header; 69 | bool hasInitialized, gotHeader, gotStreamEnd, gotEndChunk, hasFinishedReading; 70 | std::list chunks; 71 | 72 | PngInterlacedPos interlacedPos; 73 | std::string incompleteScanline; 74 | size_t incompleteScanlineOffset; 75 | std::list scanlines; 76 | 77 | PngReader(FILE* f = NULL); 78 | ~PngReader(); 79 | Return read(); 80 | }; 81 | 82 | struct PngWriter : DontCopyTag { 83 | WriteCallbackIntf* writer; 84 | z_stream stream; 85 | std::list chunks; 86 | std::list scanlines; 87 | std::list dataChunks; 88 | bool hasInitialized, hasFinishedWriting; // these are set from write() 89 | bool hasAllChunks, hasAllScanlines; // these are expected to be set from outside 90 | 91 | PngWriter(WriteCallbackIntf* w = NULL); 92 | ~PngWriter(); 93 | Return write(); 94 | }; 95 | 96 | #endif 97 | -------------------------------------------------------------------------------- /Db.cpp: -------------------------------------------------------------------------------- 1 | /* simple DB interface 2 | * by Albert Zeyer, 2011 3 | * code under LGPL 4 | */ 5 | 6 | #ifndef Z_CompressionLevel 7 | #define Z_CompressionLevel 9 8 | #endif 9 | 10 | #ifndef Z_BufSize 11 | #define Z_BufSize 1024*128 12 | #endif 13 | 14 | #include "Db.h" 15 | #include "Sha1.h" 16 | #include "StringUtils.h" 17 | #include "FileUtils.h" 18 | #include 19 | #include 20 | #include 21 | 22 | #include 23 | using namespace std; 24 | 25 | void DbEntry::calcSha1() { 26 | sha1 = calc_sha1(data); 27 | } 28 | 29 | void DbEntry::compress() { 30 | compressed = ""; 31 | 32 | z_stream stream; 33 | stream.zalloc = Z_NULL; 34 | stream.zfree = Z_NULL; 35 | stream.opaque = Z_NULL; 36 | deflateInit(&stream, Z_CompressionLevel); 37 | 38 | stream.avail_in = data.size(); 39 | stream.next_in = (unsigned char*) &data[0]; 40 | while(true) { 41 | char outputData[Z_BufSize]; 42 | stream.avail_out = sizeof(outputData); 43 | stream.next_out = (unsigned char*) outputData; 44 | int ret = deflate(&stream, Z_FINISH); 45 | switch(ret) { 46 | case Z_OK: break; 47 | case Z_STREAM_END: break; 48 | // these cases should not happen. but check anyway 49 | case Z_STREAM_ERROR: 50 | default: 51 | cerr << "error to deflate " << data.size() << " bytes" << endl; 52 | cerr << "remaining: " << stream.avail_in << " bytes" << endl; 53 | cerr << "deflate ret: " << ret << endl; 54 | assert(false); 55 | return; 56 | } 57 | size_t out_size = sizeof(outputData) - stream.avail_out; 58 | compressed += std::string(outputData, out_size); 59 | if(ret == Z_STREAM_END) break; 60 | } 61 | deflateEnd(&stream); 62 | } 63 | 64 | Return DbEntry::uncompress() { 65 | data = ""; 66 | 67 | z_stream stream; 68 | stream.zalloc = Z_NULL; 69 | stream.zfree = Z_NULL; 70 | stream.opaque = Z_NULL; 71 | inflateInit(&stream); 72 | 73 | bool gotStreamEnd = false; 74 | stream.avail_in = compressed.size(); 75 | stream.next_in = (unsigned char*) &compressed[0]; 76 | while(true) { 77 | char outputData[Z_BufSize]; 78 | stream.avail_out = sizeof(outputData); 79 | stream.next_out = (unsigned char*) outputData; 80 | int ret = inflate(&stream, Z_NO_FLUSH); 81 | switch(ret) { 82 | case Z_STREAM_ERROR: inflateEnd(&stream); return "zlib stream error / invalid compression level"; 83 | case Z_NEED_DICT: inflateEnd(&stream); return "zlib need dict error"; 84 | case Z_DATA_ERROR: inflateEnd(&stream); return "zlib data error"; 85 | case Z_MEM_ERROR: inflateEnd(&stream); return "zlib out-of-memory error"; 86 | case Z_STREAM_END: gotStreamEnd = true; 87 | } 88 | size_t out_size = sizeof(outputData) - stream.avail_out; 89 | if(out_size == 0) break; 90 | data += std::string(outputData, out_size); 91 | } 92 | inflateEnd(&stream); 93 | 94 | if(!gotStreamEnd) 95 | return "zlib stream incomplete"; 96 | return true; 97 | } 98 | -------------------------------------------------------------------------------- /hiredis/sds.h: -------------------------------------------------------------------------------- 1 | /* SDSLib, A C dynamic strings library 2 | * 3 | * Copyright (c) 2006-2010, Salvatore Sanfilippo 4 | * All rights reserved. 5 | * 6 | * Redistribution and use in source and binary forms, with or without 7 | * modification, are permitted provided that the following conditions are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright notice, 10 | * this list of conditions and the following disclaimer. 11 | * * Redistributions in binary form must reproduce the above copyright 12 | * notice, this list of conditions and the following disclaimer in the 13 | * documentation and/or other materials provided with the distribution. 14 | * * Neither the name of Redis nor the names of its contributors may be used 15 | * to endorse or promote products derived from this software without 16 | * specific prior written permission. 17 | * 18 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 22 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28 | * POSSIBILITY OF SUCH DAMAGE. 29 | */ 30 | 31 | #ifndef __SDS_H 32 | #define __SDS_H 33 | 34 | #include 35 | #include 36 | 37 | typedef char *sds; 38 | 39 | struct sdshdr { 40 | int len; 41 | int free; 42 | char buf[]; 43 | }; 44 | 45 | sds sdsnewlen(const void *init, size_t initlen); 46 | sds sdsnew(const char *init); 47 | sds sdsempty(); 48 | size_t sdslen(const sds s); 49 | sds sdsdup(const sds s); 50 | void sdsfree(sds s); 51 | size_t sdsavail(sds s); 52 | sds sdscatlen(sds s, const void *t, size_t len); 53 | sds sdscat(sds s, const char *t); 54 | sds sdscpylen(sds s, char *t, size_t len); 55 | sds sdscpy(sds s, char *t); 56 | 57 | sds sdscatvprintf(sds s, const char *fmt, va_list ap); 58 | #ifdef __GNUC__ 59 | sds sdscatprintf(sds s, const char *fmt, ...) 60 | __attribute__((format(printf, 2, 3))); 61 | #else 62 | sds sdscatprintf(sds s, const char *fmt, ...); 63 | #endif 64 | 65 | sds sdstrim(sds s, const char *cset); 66 | sds sdsrange(sds s, int start, int end); 67 | void sdsupdatelen(sds s); 68 | int sdscmp(sds s1, sds s2); 69 | sds *sdssplitlen(char *s, int len, char *sep, int seplen, int *count); 70 | void sdsfreesplitres(sds *tokens, int count); 71 | void sdstolower(sds s); 72 | void sdstoupper(sds s); 73 | sds sdsfromlonglong(long long value); 74 | sds sdscatrepr(sds s, char *p, size_t len); 75 | sds *sdssplitargs(char *line, int *argc); 76 | 77 | #endif 78 | -------------------------------------------------------------------------------- /Db.h: -------------------------------------------------------------------------------- 1 | /* simple DB interface 2 | * by Albert Zeyer, 2011 3 | * code under LGPL 4 | */ 5 | 6 | #ifndef __AZ__DB_H__ 7 | #define __AZ__DB_H__ 8 | 9 | #include "Return.h" 10 | #include "StringUtils.h" 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | #define DbEntryType_PngContentList 1 17 | #define DbEntryType_PngChunk 2 18 | #define DbEntryType_PngBlock 3 19 | 20 | struct DbEntry { 21 | std::string data; 22 | std::string sha1; 23 | std::string compressed; 24 | 25 | DbEntry() {} 26 | DbEntry(const std::string& d) : data(d) { prepare(); } 27 | bool haveSha1() const { return !sha1.empty(); } 28 | void calcSha1(); 29 | bool haveCompressed() const { return !compressed.empty(); } 30 | void compress(); 31 | Return uncompress(); 32 | void prepare() { calcSha1(); compress(); } 33 | 34 | bool operator==(const DbEntry& other) const { 35 | assert(haveSha1()); 36 | assert(haveCompressed()); 37 | assert(other.haveSha1()); 38 | assert(other.haveCompressed()); 39 | if(sha1 != other.sha1) return false; 40 | // Note: If we would ensure that the compression algorithm always works exactly 41 | // the same way, we could just restrict the check on the compressed data. 42 | // But as we don't want to restrict ourself to this, we can't. 43 | if(compressed == other.compressed) return true; 44 | return data == other.data; 45 | } 46 | }; 47 | 48 | typedef std::string DbEntryId; /* guaranteed to not contain \0 and to be not empty */ 49 | 50 | struct DbStats { 51 | size_t pushNew; 52 | size_t pushReuse; 53 | DbStats() : pushNew(0), pushReuse(0) {} 54 | }; 55 | 56 | struct DbDirEntry { 57 | mode_t mode; 58 | std::string name; 59 | size_t size; 60 | DbDirEntry(mode_t m = 0, const std::string& fn = "", size_t s = 0) : mode(m), name(fn), size(s) {} 61 | std::string serialized() const { return rawString(mode) + rawString(size) + name; } 62 | static DbDirEntry FromSerialized(const std::string& raw) { 63 | if(raw.size() <= 6) return DbDirEntry(); 64 | return DbDirEntry(valueFromRaw(&raw[0]), raw.substr(6), valueFromRaw(&raw[2])); 65 | } 66 | static DbDirEntry File(const std::string& fn, size_t s) { return DbDirEntry(S_IFREG | 0444, fn, s); } 67 | static DbDirEntry Dir(const std::string& fn) { return DbDirEntry(S_IFDIR | 0755, fn); } 68 | }; 69 | 70 | struct DbIntf { 71 | DbStats stats; 72 | virtual Return setReadOnly(bool ro) { return "Db::setReadOnly: not implemented"; } 73 | virtual Return init() = 0; 74 | virtual Return push(/*out*/ DbEntryId& id, const DbEntry& entry) = 0; 75 | virtual Return get(/*out*/ DbEntry& entry, const DbEntryId& id) = 0; 76 | virtual Return pushToDir(const std::string& path, const DbDirEntry& dirEntry) { 77 | return "Db::pushToDir: not implemented"; 78 | } 79 | virtual Return getDir(/*out*/ std::list& dirList, const std::string& path) { 80 | return "Db::getDir: not implemented"; 81 | } 82 | virtual Return setFileRef(/*can be empty*/ const DbEntryId& id, const std::string& path) { 83 | return "Db::setFileRef: not implemented"; 84 | } 85 | virtual Return getFileRef(/*out (can be empty)*/ DbEntryId& id, const std::string& path) { 86 | return "Db::getFileRef: not implemented"; 87 | } 88 | }; 89 | 90 | #endif 91 | -------------------------------------------------------------------------------- /Crc.cpp: -------------------------------------------------------------------------------- 1 | /* calculate CRC 2 | * based on http://www.w3.org/TR/PNG/#D-CRCAppendix 3 | * by Albert Zeyer, 2011 4 | * code public domain 5 | */ 6 | 7 | #include "Crc.h" 8 | 9 | /* Table of CRCs of all 8-bit messages. */ 10 | static const uint32_t crc_table[256] = { 11 | 0, 1996959894, 3993919788, 2567524794, 124634137, 1886057615, 3915621685, 2657392035, 249268274, 2044508324, 3772115230, 2547177864, 162941995, 2125561021, 3887607047, 2428444049, 498536548, 1789927666, 4089016648, 2227061214, 450548861, 1843258603, 4107580753, 2211677639, 325883990, 1684777152, 4251122042, 2321926636, 335633487, 1661365465, 4195302755, 2366115317, 997073096, 1281953886, 3579855332, 2724688242, 1006888145, 1258607687, 3524101629, 2768942443, 901097722, 1119000684, 3686517206, 2898065728, 853044451, 1172266101, 3705015759, 2882616665, 651767980, 1373503546, 12 | 3369554304, 3218104598, 565507253, 1454621731, 3485111705, 3099436303, 671266974, 1594198024, 3322730930, 2970347812, 795835527, 1483230225, 3244367275, 3060149565, 1994146192, 31158534, 2563907772, 4023717930, 1907459465, 112637215, 2680153253, 3904427059, 2013776290, 251722036, 2517215374, 3775830040, 2137656763, 141376813, 2439277719, 3865271297, 1802195444, 476864866, 2238001368, 4066508878, 1812370925, 453092731, 2181625025, 4111451223, 1706088902, 314042704, 2344532202, 4240017532, 1658658271, 366619977, 2362670323, 4224994405, 1303535960, 984961486, 2747007092, 3569037538, 13 | 1256170817, 1037604311, 2765210733, 3554079995, 1131014506, 879679996, 2909243462, 3663771856, 1141124467, 855842277, 2852801631, 3708648649, 1342533948, 654459306, 3188396048, 3373015174, 1466479909, 544179635, 3110523913, 3462522015, 1591671054, 702138776, 2966460450, 3352799412, 1504918807, 783551873, 3082640443, 3233442989, 3988292384, 2596254646, 62317068, 1957810842, 3939845945, 2647816111, 81470997, 1943803523, 3814918930, 2489596804, 225274430, 2053790376, 3826175755, 2466906013, 167816743, 2097651377, 4027552580, 2265490386, 503444072, 1762050814, 4150417245, 2154129355, 14 | 426522225, 1852507879, 4275313526, 2312317920, 282753626, 1742555852, 4189708143, 2394877945, 397917763, 1622183637, 3604390888, 2714866558, 953729732, 1340076626, 3518719985, 2797360999, 1068828381, 1219638859, 3624741850, 2936675148, 906185462, 1090812512, 3747672003, 2825379669, 829329135, 1181335161, 3412177804, 3160834842, 628085408, 1382605366, 3423369109, 3138078467, 570562233, 1426400815, 3317316542, 2998733608, 733239954, 1555261956, 3268935591, 3050360625, 752459403, 1541320221, 2607071920, 3965973030, 1969922972, 40735498, 2617837225, 3943577151, 1913087877, 83908371, 15 | 2512341634, 3803740692, 2075208622, 213261112, 2463272603, 3855990285, 2094854071, 198958881, 2262029012, 4057260610, 1759359992, 534414190, 2176718541, 4139329115, 1873836001, 414664567, 2282248934, 4279200368, 1711684554, 285281116, 2405801727, 4167216745, 1634467795, 376229701, 2685067896, 3608007406, 1308918612, 956543938, 2808555105, 3495958263, 1231636301, 1047427035, 2932959818, 3654703836, 1088359270, 936918000, 2847714899, 3736837829, 1202900863, 817233897, 3183342108, 3401237130, 1404277552, 615818150, 3134207493, 3453421203, 1423857449, 601450431, 3009837614, 3294710456, 16 | 1567103746, 711928724, 3020668471, 3272380065, 1510334235, 755167117 }; 17 | 18 | /* Update a running CRC with the bytes buf[0..len-1]--the CRC 19 | should be initialized to all 1's, and the transmitted value 20 | is the 1's complement of the final running CRC (see the 21 | crc() routine below). */ 22 | uint32_t update_crc(uint32_t crc, const char *buf, size_t len) { 23 | uint32_t c = crc; 24 | for (size_t n = 0; n < len; n++) 25 | c = crc_table[(c ^ (unsigned char)buf[n]) & 0xff] ^ (c >> 8); 26 | return c; 27 | } 28 | 29 | /* Return the CRC of the bytes buf[0..len-1]. */ 30 | uint32_t calc_crc(const char *buf, size_t len) { 31 | return update_crc(0xffffffffL, buf, len) ^ 0xffffffffL; 32 | } 33 | 34 | uint32_t calc_crc(const std::string& s) { 35 | return calc_crc(&s[0], s.size()); 36 | } 37 | 38 | uint32_t calc_crc(const std::string& s1, const std::string& s2) { 39 | uint32_t c = 0xffffffffL; 40 | c = update_crc(c, &s1[0], s1.size()); 41 | c = update_crc(c, &s2[0], s2.size()); 42 | return c ^ 0xffffffffL; 43 | } 44 | 45 | -------------------------------------------------------------------------------- /DbFsBackend.cpp: -------------------------------------------------------------------------------- 1 | /* filesystem DB backend 2 | * by Albert Zeyer, 2011 3 | * code under LGPL 4 | */ 5 | 6 | #include "DbFsBackend.h" 7 | #include "Sha1.h" 8 | #include "StringUtils.h" 9 | #include "FileUtils.h" 10 | #include 11 | #include 12 | #include 13 | 14 | #include 15 | using namespace std; 16 | 17 | static std::string __filenameForDbEntryId(const DbEntryId& id) { 18 | assert(!id.empty()); 19 | 20 | std::string ret = "data/"; 21 | for(size_t i = 0; i < id.size() - 1; ++i) 22 | ret += hexString(id[i]) + "/"; 23 | ret += hexString(id[id.size()-1]) + ".dat"; 24 | return ret; 25 | } 26 | 27 | static std::string __dirnameForSha1Ref(const std::string& sha1) { 28 | assert(sha1.size() == SHA1_DIGEST_SIZE); 29 | 30 | std::string ret = "sha1refs/"; 31 | for(size_t i = 0; i < sha1.size(); ++i) { 32 | if(i % 5 == 0 && i > 0) ret += "/"; 33 | ret += hexString(sha1[i]); 34 | } 35 | return ret; 36 | } 37 | 38 | static int __hexnumFromChar(char c) { 39 | if(c >= '0' && c <= '9') return c - '0'; 40 | if(c >= 'A' && c <= 'F') return 10 + c - 'A'; 41 | if(c >= 'a' && c <= 'f') return 10 + c - 'a'; 42 | return -1; 43 | } 44 | 45 | static DbEntryId __entryIdFromSha1RefFilename(const std::string& fn) { 46 | DbEntryId id; 47 | size_t i = 0; 48 | for(; i + 1 < fn.size(); i += 2) { 49 | if(fn.substr(i) == ".ref") return id; 50 | int n1 = __hexnumFromChar(fn[i]); 51 | int n2 = __hexnumFromChar(fn[i+1]); 52 | if(n1 < 0 || n2 < 0) return ""; 53 | id += (char)(unsigned char)(n1 * 16 + n2); 54 | } 55 | return ""; 56 | } 57 | 58 | static Return __openNewDbEntry(const std::string& baseDir, DbEntryId& id, FILE*& f) { 59 | bool createdDir = false; 60 | unsigned short triesNum = (id.size() <= 4) ? (2 << id.size()) : 64; 61 | for(unsigned short i = 0; i < triesNum; ++i) { 62 | DbEntryId newId = id; 63 | newId += (char)random(); 64 | std::string filename = baseDir + "/" + __filenameForDbEntryId(newId); 65 | if(!createdDir) { 66 | ASSERT( createRecDir(filename, false) ); 67 | createdDir = true; 68 | } 69 | f = fopen(filename.c_str(), "wbx"); 70 | if(f) { 71 | id = newId; 72 | return true; 73 | } 74 | } 75 | 76 | id += (char)random(); 77 | return __openNewDbEntry(baseDir, id, f); 78 | } 79 | 80 | Return DbFsBackend::push(/*out*/ DbEntryId& id, const DbEntry& entry) { 81 | if(!entry.haveSha1()) 82 | return "DB push: entry SHA1 not calculated"; 83 | if(!entry.haveCompressed()) 84 | return "DB push: entry compression not calculated"; 85 | 86 | // search for existing entry 87 | std::string sha1refdir = __dirnameForSha1Ref(entry.sha1); 88 | for(DirIter dir(baseDir + "/" + sha1refdir); dir; dir.next()) { 89 | DbEntryId otherId = __entryIdFromSha1RefFilename(dir.filename); 90 | if(otherId != "") { 91 | DbEntry otherEntry; 92 | if(get(otherEntry, otherId)) { 93 | if(entry == otherEntry) { 94 | // found 95 | id = otherId; 96 | stats.pushReuse++; 97 | return true; 98 | } 99 | } 100 | } 101 | } 102 | 103 | // write DB entry 104 | id = ""; 105 | FILE* f = NULL; 106 | ASSERT( __openNewDbEntry(baseDir, id, f) ); 107 | { 108 | Return r = fwrite_all(f, entry.compressed); 109 | fclose(f); 110 | if(!r) return r; 111 | } 112 | 113 | // create sha1 ref 114 | std::string sha1reffn = baseDir + "/" + sha1refdir + "/" + hexString(id) + ".ref"; 115 | ASSERT( createRecDir(sha1reffn, false) ); 116 | f = fopen(sha1reffn.c_str(), "w"); 117 | if(f == NULL) 118 | return "DB push: cannot create SHA1 ref: cannot create file '" + sha1reffn + "'"; 119 | fclose(f); 120 | 121 | stats.pushNew++; 122 | return true; 123 | } 124 | 125 | Return DbFsBackend::get(/*out*/ DbEntry& entry, const DbEntryId& id) { 126 | std::string filename = baseDir + "/" + __filenameForDbEntryId(id); 127 | FILE* f = fopen(filename.c_str(), "rb"); 128 | if(f == NULL) 129 | return "Db::get: cannot open file '" + filename + "'"; 130 | 131 | { 132 | Return r = fread_all(f, entry.compressed); 133 | fclose(f); 134 | if(!r) return r; 135 | } 136 | 137 | ASSERT( entry.uncompress() ); 138 | entry.calcSha1(); 139 | 140 | return true; 141 | } 142 | -------------------------------------------------------------------------------- /SmartPointer.h: -------------------------------------------------------------------------------- 1 | /* 2 | OpenLieroX 3 | code under LGPL 4 | 27-12-2007 Albert Zeyer 5 | */ 6 | 7 | #ifndef __SMARTPOINTER_H__ 8 | #define __SMARTPOINTER_H__ 9 | 10 | #include 11 | #include 12 | #include "Mutex.h" 13 | 14 | // Default de-initialization action is to call operator delete, for each object type. 15 | template < typename _Type > 16 | void SmartPointer_ObjectDeinit( _Type * obj ) { 17 | delete obj; 18 | } 19 | 20 | 21 | /* 22 | standard smartpointer based on simple refcounting 23 | 24 | The refcounting is multithreading safe in this class, 25 | you can have copies of this object in different threads. 26 | Though it's not designed to operate with the same 27 | object in different threads. Also there is absolutly no 28 | thread safty on the pointer itself, you have to care 29 | about this yourself. 30 | */ 31 | 32 | 33 | template < typename _Type > 34 | class SmartPointer { 35 | public: 36 | typedef _Type value_type; 37 | private: 38 | _Type* obj; 39 | int* refCount; 40 | Mutex* mutex; 41 | 42 | void init(_Type* newObj) { 43 | if( newObj == NULL ) 44 | return; 45 | if(!mutex) { 46 | mutex = new Mutex(); 47 | obj = newObj; 48 | refCount = new int(1); 49 | } 50 | } 51 | 52 | void reset() { 53 | if(mutex) { 54 | lock(); 55 | (*refCount)--; 56 | if(*refCount == 0) { 57 | SmartPointer_ObjectDeinit( obj ); 58 | delete refCount; // safe, because there is no other ref anymore 59 | obj = NULL; 60 | refCount = NULL; 61 | unlock(); 62 | delete mutex; // safe because there is no other ref anymore 63 | mutex = NULL; 64 | } else 65 | unlock(); 66 | } 67 | obj = NULL; 68 | refCount = NULL; 69 | mutex = NULL; 70 | } 71 | 72 | void incCounter() { 73 | assert(*refCount > 0 && *refCount < INT_MAX); 74 | (*refCount)++; 75 | } 76 | 77 | void lock() { mutex->lock(); } 78 | void unlock() { mutex->unlock(); } 79 | 80 | public: 81 | SmartPointer() : obj(NULL), refCount(NULL), mutex(NULL) {} 82 | ~SmartPointer() { reset(); } 83 | 84 | // Default copy constructor and operator= 85 | // If you specify any template<> params here these funcs will be silently ignored by compiler 86 | SmartPointer(const SmartPointer& pt) : obj(NULL), refCount(NULL), mutex(NULL) { operator=(pt); } 87 | SmartPointer& operator=(const SmartPointer& pt) { 88 | if(mutex == pt.mutex) return *this; // ignore this case 89 | reset(); 90 | mutex = pt.mutex; 91 | if(mutex) { 92 | lock(); 93 | obj = pt.obj; refCount = pt.refCount; 94 | incCounter(); 95 | unlock(); 96 | } else { obj = NULL; refCount = NULL; } 97 | return *this; 98 | } 99 | 100 | // WARNING: Be carefull, don't assing a pointer to different SmartPointer objects, 101 | // else they will get freed twice in the end. Always copy the SmartPointer itself. 102 | // In short: SmartPointer ptr(SomeObj); SmartPointer ptr1( ptr.get() ); // It's wrong, don't do that. 103 | SmartPointer(_Type* pt): obj(NULL), refCount(NULL), mutex(NULL) { operator=(pt); } 104 | SmartPointer& operator=(_Type* pt) { 105 | if(obj == pt) return *this; // ignore this case 106 | reset(); 107 | init(pt); 108 | return *this; 109 | } 110 | 111 | _Type* get() const { return obj; } // The smartpointer itself won't change when returning address of obj, so it's const. 112 | 113 | // HINT: no convenient cast functions in this class to avoid error-prone automatic casts 114 | // (which would lead to collisions!) 115 | // This operator is safe though. 116 | _Type * operator -> () const { return obj; }; 117 | 118 | // refcount may be changed from another thread, though if refcount==1 or 0 it won't change 119 | int getRefCount() { 120 | int ret = 0; 121 | if(mutex) { 122 | lock(); 123 | ret = *refCount; 124 | unlock(); // Here the other thread may change refcount, that's why it's approximate 125 | } 126 | return ret; 127 | } 128 | 129 | // Returns true only if the data is deleted (no other smartpointer used it), sets pointer to NULL then 130 | bool tryDeleteData() { 131 | if(mutex) { 132 | lock(); 133 | if( *refCount == 1 ) 134 | { 135 | unlock(); // Locks mutex again inside reset(), since we're only ones using data refcount cannot change from other thread 136 | reset(); 137 | return true; // Data deleted 138 | } 139 | unlock(); 140 | return false; // Data not deleted 141 | } 142 | return true; // Data was already deleted 143 | } 144 | 145 | }; 146 | 147 | #endif 148 | 149 | -------------------------------------------------------------------------------- /DbKyotoBackend.cpp: -------------------------------------------------------------------------------- 1 | /* KyotoCabinet DB backend 2 | * by Albert Zeyer, 2011 3 | * code under LGPL 4 | */ 5 | 6 | #include "DbKyotoBackend.h" 7 | #include "StringUtils.h" 8 | #include "Utils.h" 9 | #include 10 | 11 | #include 12 | using namespace std; 13 | using namespace kyotocabinet; 14 | 15 | DbKyotoBackend::~DbKyotoBackend() { 16 | db.close(); 17 | } 18 | 19 | Return DbKyotoBackend::init() { 20 | if(!db.open(filename, readonly ? PolyDB::OREADER : (PolyDB::OWRITER | PolyDB::OCREATE))) 21 | return std::string() + "failed to open KyotoCabinet DB: " + db.error().name(); 22 | return true; 23 | } 24 | 25 | typedef PolyDB KyotoDB; 26 | 27 | static Return __addEntryToList(KyotoDB& db, const std::string& key, const std::string& entry) { 28 | if(entry.size() > 255) 29 | return "cannot add entries with size>255 to list"; 30 | if(!db.append(key, rawString(entry.size()) + entry)) 31 | return std::string() + "error adding entry to list: " + db.error().name(); 32 | return true; 33 | } 34 | 35 | static Return __getEntryList(KyotoDB& db, const std::string& key, std::list& entries) { 36 | std::string value; 37 | if(!db.get(key, &value)) 38 | return std::string() + "error getting entry list: " + db.error().name(); 39 | 40 | size_t i = 0; 41 | while(i < value.size()) { 42 | uint8_t size = value[i]; 43 | ++i; 44 | if(i + size > value.size()) 45 | return "entry list data is inconsistent"; 46 | entries.push_back( value.substr(i, size) ); 47 | i += size; 48 | } 49 | 50 | return true; 51 | } 52 | 53 | static Return __saveNewDbEntry(KyotoDB& db, DbEntryId& id, const std::string& content) { 54 | unsigned short triesNum = (id.size() <= 4) ? (2 << id.size()) : 64; 55 | for(unsigned short i = 0; i < triesNum; ++i) { 56 | DbEntryId newId = id; 57 | newId += (char)random(); 58 | std::string key = "data." + newId; 59 | if(db.add(key, content)) { 60 | id = newId; 61 | return true; 62 | } 63 | } 64 | 65 | id += (char)random(); 66 | return __saveNewDbEntry(db, id, content); 67 | } 68 | 69 | Return DbKyotoBackend::push(/*out*/ DbEntryId& id, const DbEntry& entry) { 70 | if(!entry.haveSha1()) 71 | return "DB push: entry SHA1 not calculated"; 72 | if(!entry.haveCompressed()) 73 | return "DB push: entry compression not calculated"; 74 | 75 | // search for existing entry 76 | std::string sha1refkey = "sha1ref." + entry.sha1; 77 | std::list sha1refs; 78 | if(__getEntryList(db, sha1refkey, sha1refs)) 79 | for(std::list::iterator i = sha1refs.begin(); i != sha1refs.end(); ++i) { 80 | DbEntryId otherId = *i; 81 | DbEntry otherEntry; 82 | if(get(otherEntry, otherId)) { 83 | if(entry == otherEntry) { 84 | // found 85 | id = otherId; 86 | stats.pushReuse++; 87 | return true; 88 | } 89 | } 90 | } 91 | 92 | // write DB entry 93 | id = ""; 94 | ASSERT( __saveNewDbEntry(db, id, entry.compressed) ); 95 | 96 | // create sha1 ref 97 | ASSERT( __addEntryToList(db, sha1refkey, id) ); 98 | 99 | stats.pushNew++; 100 | return true; 101 | } 102 | 103 | Return DbKyotoBackend::get(/*out*/ DbEntry& entry, const DbEntryId& id) { 104 | std::string key = "data." + id; 105 | if(!db.get(key, &entry.compressed)) 106 | return std::string() + "DB get: error getting entry: " + db.error().name(); 107 | 108 | ASSERT( entry.uncompress() ); 109 | entry.calcSha1(); 110 | 111 | return true; 112 | } 113 | 114 | Return DbKyotoBackend::pushToDir(const std::string& path, const DbDirEntry& dirEntry) { 115 | std::string key = "fs." + path; 116 | std::string dirEntryRaw = dirEntry.serialized(); 117 | ASSERT( __addEntryToList(db, key, dirEntryRaw) ); 118 | return true; 119 | } 120 | 121 | Return DbKyotoBackend::getDir(/*out*/ std::list& dirList, const std::string& path) { 122 | std::string key = "fs." + path; 123 | std::list entries; 124 | ASSERT( __getEntryList(db, key, entries) ); 125 | for(std::list::iterator i = entries.begin(); i != entries.end(); ++i) 126 | dirList.push_back( DbDirEntry::FromSerialized(*i) ); 127 | 128 | return true; 129 | } 130 | 131 | Return DbKyotoBackend::setFileRef(/*can be empty*/ const DbEntryId& id, const std::string& path) { 132 | std::string key = "fs." + path; 133 | if(!db.set(key, id)) 134 | return std::string() + "DB setFileRef: error setting entry: " + db.error().name(); 135 | return true; 136 | } 137 | 138 | Return DbKyotoBackend::getFileRef(/*out (can be empty)*/ DbEntryId& id, const std::string& path) { 139 | std::string key = "fs." + path; 140 | if(!db.get(key, &id)) 141 | return std::string() + "DB getFileRef: error getting entry: " + db.error().name(); 142 | return true; 143 | } 144 | 145 | -------------------------------------------------------------------------------- /kyotocabinet/kcregex.h: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Regular expression 3 | * Copyright (C) 2009-2011 FAL Labs 4 | * This file is part of Kyoto Cabinet. 5 | * This program is free software: you can redistribute it and/or modify it under the terms of 6 | * the GNU General Public License as published by the Free Software Foundation, either version 7 | * 3 of the License, or any later version. 8 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 9 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 10 | * See the GNU General Public License for more details. 11 | * You should have received a copy of the GNU General Public License along with this program. 12 | * If not, see . 13 | *************************************************************************************************/ 14 | 15 | 16 | #ifndef _KCREGEX_H // duplication check 17 | #define _KCREGEX_H 18 | 19 | #include 20 | #include 21 | 22 | namespace kyotocabinet { // common namespace 23 | 24 | 25 | /** 26 | * Regular expression. 27 | */ 28 | class Regex { 29 | public: 30 | /** 31 | * Options. 32 | */ 33 | enum Option { 34 | IGNCASE = 1 << 0, ///< case-insensitive 35 | MATCHONLY = 1 << 1, ///< matching only 36 | }; 37 | /** 38 | * Default constructor. 39 | */ 40 | explicit Regex(); 41 | /** 42 | * Destructor. 43 | */ 44 | ~Regex(); 45 | /** 46 | * Compile a string of regular expression. 47 | * @param regex the string of regular expression. 48 | * @param opts the optional features by bitwise-or: Regex::IGNCASE for case-insensitive 49 | * matching, Regex::MATCHONLY for matching only usage. 50 | */ 51 | bool compile(const std::string& regex, uint32_t opts = 0); 52 | /** 53 | * Check whether a string matches the regular expression. 54 | * @param str the string. 55 | * @return true if the string matches, or false if not. 56 | */ 57 | bool match(const std::string& str); 58 | /** 59 | * Check whether a string matches the regular expression. 60 | * @param str the string. 61 | * @param alt the alternative string with which each substring is replaced. Each "$" in the 62 | * string escapes the following character. Special escapes "$1" through "$9" refer to partial 63 | * substrings corresponding to sub-expressions in the regular expression. "$0" and "$&" refer 64 | * to the whole matching substring. 65 | * @return the result string. 66 | */ 67 | std::string replace(const std::string& str, const std::string& alt); 68 | /** 69 | * Check whether a string matches a regular expression. 70 | * @param str the string. 71 | * @param pattern the matching pattern. 72 | * @param opts the optional features by bitwise-or: Regex::IGNCASE for case-insensitive 73 | * matching, Regex::MATCHONLY for matching only usage. 74 | * @return true if the string matches, or false if not. 75 | */ 76 | static bool match(const std::string& str, const std::string& pattern, uint32_t opts = 0) { 77 | Regex regex; 78 | if (!regex.compile(pattern, opts)) return false; 79 | return regex.match(str); 80 | } 81 | /** 82 | * Check whether a string matches the regular expression. 83 | * @param str the string. 84 | * @param pattern the matching pattern. 85 | * @param alt the alternative string with which each substring is replaced. Each "$" in the 86 | * string escapes the following character. Special escapes "$1" through "$9" refer to partial 87 | * substrings corresponding to sub-expressions in the regular expression. "$0" and "$&" refer 88 | * to the whole matching substring. 89 | * @param opts the optional features by bitwise-or: Regex::IGNCASE for case-insensitive 90 | * matching, Regex::MATCHONLY for matching only usage. 91 | * @return the result string. 92 | */ 93 | static std::string replace(const std::string& str, const std::string& pattern, 94 | const std::string& alt, uint32_t opts = 0) { 95 | Regex regex; 96 | if (!regex.compile(pattern, opts)) return str; 97 | return regex.replace(str, alt); 98 | } 99 | private: 100 | /** Dummy constructor to forbid the use. */ 101 | Regex(const Regex&); 102 | /** Dummy Operator to forbid the use. */ 103 | Regex& operator =(const Regex&); 104 | /** Opaque pointer. */ 105 | void* opq_; 106 | }; 107 | 108 | 109 | } // common namespace 110 | 111 | #endif // duplication check 112 | 113 | // END OF FILE 114 | -------------------------------------------------------------------------------- /hiredis/async.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2009-2010, Salvatore Sanfilippo 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions are met: 7 | * 8 | * * Redistributions of source code must retain the above copyright notice, 9 | * this list of conditions and the following disclaimer. 10 | * * Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * * Neither the name of Redis nor the names of its contributors may be used 14 | * to endorse or promote products derived from this software without 15 | * specific prior written permission. 16 | * 17 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 25 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27 | * POSSIBILITY OF SUCH DAMAGE. 28 | */ 29 | 30 | #ifndef __HIREDIS_ASYNC_H 31 | #define __HIREDIS_ASYNC_H 32 | #include "hiredis.h" 33 | 34 | #ifdef __cplusplus 35 | extern "C" { 36 | #endif 37 | 38 | struct redisAsyncContext; /* need forward declaration of redisAsyncContext */ 39 | 40 | /* Reply callback prototype and container */ 41 | typedef void (redisCallbackFn)(struct redisAsyncContext*, void*, void*); 42 | typedef struct redisCallback { 43 | struct redisCallback *next; /* simple singly linked list */ 44 | redisCallbackFn *fn; 45 | void *privdata; 46 | } redisCallback; 47 | 48 | /* List of callbacks for either regular replies or pub/sub */ 49 | typedef struct redisCallbackList { 50 | redisCallback *head, *tail; 51 | } redisCallbackList; 52 | 53 | /* Connection callback prototypes */ 54 | typedef void (redisDisconnectCallback)(const struct redisAsyncContext*, int status); 55 | typedef void (redisConnectCallback)(const struct redisAsyncContext*); 56 | 57 | /* Context for an async connection to Redis */ 58 | typedef struct redisAsyncContext { 59 | /* Hold the regular context, so it can be realloc'ed. */ 60 | redisContext c; 61 | 62 | /* Setup error flags so they can be used directly. */ 63 | int err; 64 | char *errstr; 65 | 66 | /* Not used by hiredis */ 67 | void *data; 68 | 69 | /* Used by the different event lib adapters to store their private data */ 70 | void *_adapter_data; 71 | 72 | /* Called when the library expects to start reading/writing. 73 | * The supplied functions should be idempotent. */ 74 | void (*evAddRead)(void *privdata); 75 | void (*evDelRead)(void *privdata); 76 | void (*evAddWrite)(void *privdata); 77 | void (*evDelWrite)(void *privdata); 78 | void (*evCleanup)(void *privdata); 79 | 80 | /* Called when either the connection is terminated due to an error or per 81 | * user request. The status is set accordingly (REDIS_OK, REDIS_ERR). */ 82 | redisDisconnectCallback *onDisconnect; 83 | 84 | /* Called when the first write event was received. */ 85 | redisConnectCallback *onConnect; 86 | 87 | /* Reply callbacks */ 88 | redisCallbackList replies; 89 | } redisAsyncContext; 90 | 91 | /* Functions that proxy to hiredis */ 92 | redisAsyncContext *redisAsyncConnect(const char *ip, int port); 93 | int redisAsyncSetReplyObjectFunctions(redisAsyncContext *ac, redisReplyObjectFunctions *fn); 94 | int redisAsyncSetConnectCallback(redisAsyncContext *ac, redisConnectCallback *fn); 95 | int redisAsyncSetDisconnectCallback(redisAsyncContext *ac, redisDisconnectCallback *fn); 96 | void redisAsyncDisconnect(redisAsyncContext *ac); 97 | 98 | /* Handle read/write events */ 99 | void redisAsyncHandleRead(redisAsyncContext *ac); 100 | void redisAsyncHandleWrite(redisAsyncContext *ac); 101 | 102 | /* Command functions for an async context. Write the command to the 103 | * output buffer and register the provided callback. */ 104 | int redisvAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *format, va_list ap); 105 | int redisAsyncCommand(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, const char *format, ...); 106 | int redisAsyncCommandArgv(redisAsyncContext *ac, redisCallbackFn *fn, void *privdata, int argc, const char **argv, const size_t *argvlen); 107 | 108 | #ifdef __cplusplus 109 | } 110 | #endif 111 | 112 | #endif 113 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | DB optimized for a bunch of PNG images. 2 | 3 | Idea 4 | ==== 5 | 6 | The idea is to split PNG images into many blocks and have each block stored in a DB. 7 | If there are several equal blocks, it is only stored once. 8 | Via a hash table, the lookup for such blocks is made fast. 9 | 10 | Use case 11 | ======== 12 | 13 | I am collecting screenshots (for several reasons; one is to play around with machine learning / computer vision; 14 | one example is here: ). A lot of them. :) 15 | 16 | Right now, I have about 88k screenshots with about 77GB. And as many of them have a lot of repetitive areas 17 | (on some days, I were making a screenshot every 10 seconds, even when not using the computer at all, 18 | so the only changing part was the time display), I didn't wanted to waste so much space on so much repetitive data. 19 | 20 | With this PNG DB, I have a compression rate of about 400-500% 21 | (for the first 1k screenshots or so; probably the rate will even be higher for all of them). 22 | 23 | This example with the screenshots is probably an extreme case (where this applies extremely well). 24 | But I guess in many other cases where you are collecting a huge amount of PNG images 25 | (with computer-generated content; real-world images would not work that well), you can safe some space by it. 26 | 27 | And if this gets optimized as far as possible, it may be even faster than normal filesystem access 28 | (because of less disk IO). 29 | 30 | Technical details 31 | ================= 32 | 33 | To make things easier on the PNG side, it just parses down until it gets a scanline serialization. 34 | Multiple directly following scanline (of same width) serializations parts build up a block 35 | (so it actually really matches a block in the real picture). But I don't do any of the PNG filtering. 36 | PNG spec: 37 | 38 | The general DB layout is as follows: 39 | 40 | - ("data." unique id -> zlib compressed data) data pairs 41 | - ("sha1refs." SHA1 -> set of ids) data pairs 42 | - ("fs." filename -> id) data pairs 43 | 44 | Such data value (uncompressed) starts with a data-type-byte. Only 3 types are there currently: 45 | 46 | - PNG file summary 47 | - PNG chunk (all non-data PNG chunks) 48 | - PNG block 49 | 50 | There are multiple DB backend implementations: 51 | 52 | - The filesystem itself. But creates a lot of files! 53 | - [Redis](http://redis.io/). Via [hiredis](https://github.com/antirez/hiredis). As everything is in memory, you are a bit limited. 54 | - [KyotoCabinet](http://fallabs.com/kyotocabinet/). (Currently the default.) 55 | 56 | Comparison with other compression methods / deduplicators 57 | ========================================================= 58 | 59 | In the beginning, I thought about using some generic image library to be able to handle just any 60 | image type and then operate just on the raw data. This would even give me some slight better compression rate 61 | because now, I am operating on PNGs scanline serializations and there are 5 different ways (filters) in PNG 62 | to represent a scanline. 63 | 64 | However, because I am storing all the data as PNG raw data in the DB, the reconstruction of the PNG should 65 | be much faster. In the more generic case, I would have to recompress/reencode the PNG. Now I only have 66 | to (roughly) collect and glew the parts together and run the PNG zlib over it. 67 | 68 | Using a general deduplicator / compressor on the raw data (uncompressed PNG, TGA or BMP): 69 | It would be based on connected chunks of data; i.e., in the image, it would mean one or many following scanlines. 70 | But what I am doing is based on rectangular blocks in the image. So I am able to get much bigger 71 | chunks of data which is repetitive. 72 | 73 | Something like what is done in video compressions methods like H264: This might actually be a very good idea. 74 | And it should be possible to just add it now to my current method. 75 | 76 | Tools 77 | ===== 78 | 79 | It comes with several tools. Some of them: 80 | 81 | - db-push: Pushes a single PNG into the DB. 82 | - db-push-dir: Pushes all PNGs in a given directory into the DB. 83 | - db-extract-file: Extracts a single PNG from the DB. 84 | - db-fuse: Simple FUSE interface to the DB. (Slow though because it is not very optimized!) 85 | 86 | Compilation 87 | =========== 88 | 89 | Just run `./compile.sh`. 90 | 91 | For Mac: If you haven't MacFUSE installed, install it from here: 92 | 93 | TODOs 94 | ===== 95 | 96 | - Many parts could be optimized a lot. 97 | - Try with other DB backend implementations. 98 | Maybe [mongoDB](http://www.mongodb.org/) or [Basho Riak](http://www.basho.com/products_riak_overview.php). 99 | Or improve the filesystem implementation (which is incomplete anyway currently). 100 | - To make the FUSE interface faster, the caching must be improved. 101 | Also, there should be a way to get the filesize in advance and maybe also to seek in constant time. 102 | Probably, to make this possible, we need to have a fixed compression algorithm and 103 | the file summary must contain some offset information. 104 | - We could also store other image formats in a similar way. And also general files. 105 | There should be also a standard fallback. 106 | - The FUSE interface could also support writing. 107 | 108 | 109 | -Albert Zeyer, 110 | -------------------------------------------------------------------------------- /kyotocabinet/kccommon.h: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Common symbols for the library 3 | * Copyright (C) 2009-2011 FAL Labs 4 | * This file is part of Kyoto Cabinet. 5 | * This program is free software: you can redistribute it and/or modify it under the terms of 6 | * the GNU General Public License as published by the Free Software Foundation, either version 7 | * 3 of the License, or any later version. 8 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 9 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 10 | * See the GNU General Public License for more details. 11 | * You should have received a copy of the GNU General Public License along with this program. 12 | * If not, see . 13 | *************************************************************************************************/ 14 | 15 | 16 | #ifndef _KCCOMMON_H // duplication check 17 | #define _KCCOMMON_H 18 | 19 | extern "C" { 20 | #include 21 | } 22 | 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include 36 | #include 37 | #include 38 | #include 39 | 40 | #include 41 | #include 42 | #include 43 | #include 44 | #include 45 | 46 | #include 47 | #include 48 | #include 49 | #include 50 | #include 51 | #include 52 | 53 | #include 54 | #include 55 | #include 56 | #include 57 | #include 58 | #include 59 | 60 | #include 61 | #include 62 | #include 63 | #include 64 | #include 65 | 66 | #if defined(_MSC_VER) 67 | #define snprintf _snprintf 68 | #endif 69 | 70 | #if defined(__CYGWIN__) 71 | inline long double modfl(long double val, long double* iptr) { 72 | double integ; 73 | double fract = std::modf(val, &integ); 74 | *iptr = integ; 75 | return fract; 76 | } 77 | #endif 78 | 79 | namespace std { 80 | using ::modfl; 81 | using ::snprintf; 82 | } 83 | 84 | #if __cplusplus > 199711L || defined(__GXX_EXPERIMENTAL_CXX0X__) || defined(_MSC_VER) 85 | 86 | #include 87 | #include 88 | 89 | #else 90 | 91 | #include 92 | #include 93 | 94 | namespace std { 95 | using tr1::hash; 96 | using tr1::unordered_map; 97 | using tr1::unordered_set; 98 | } 99 | 100 | #endif 101 | 102 | #undef VERSION 103 | #undef LIBVER 104 | #undef LIBREV 105 | #undef SYSNAME 106 | #undef BIGEND 107 | #undef CLOCKTICK 108 | #undef PAGESIZE 109 | #undef FEATURES 110 | #undef NUMBUFSIZ 111 | #undef MEMMAXSIZ 112 | 113 | #undef IOBUFSIZ 114 | #undef SUCCESS 115 | #undef NOIMPL 116 | #undef INVALID 117 | #undef NOREPOS 118 | #undef NOPERM 119 | #undef BROKEN 120 | #undef DUPREC 121 | #undef NOREC 122 | #undef LOGIC 123 | #undef SYSTEM 124 | #undef MISC 125 | 126 | #undef DEBUG 127 | #undef INFO 128 | #undef WARN 129 | #undef ERROR 130 | #undef OPEN 131 | #undef CLOSE 132 | #undef CLEAR 133 | #undef ITERATE 134 | #undef SYNCHRONIZE 135 | #undef OCCUPY 136 | #undef BEGINTRAN 137 | #undef COMMITTRAN 138 | #undef ABORTTRAN 139 | 140 | #undef INT8MAX 141 | #undef INT16MAX 142 | #undef INT32MAX 143 | #undef INT64MAX 144 | #undef INT8MIN 145 | #undef INT16MIN 146 | #undef INT32MIN 147 | #undef INT64MIN 148 | #undef UINT8MAX 149 | #undef UINT16MAX 150 | #undef UINT32MAX 151 | #undef UINT64MAX 152 | #undef SIZEMAX 153 | #undef FLTMAX 154 | #undef DBLMAX 155 | 156 | #if defined(_KCUYIELD) 157 | #if defined(_MSC_VER) 158 | #include 159 | #define _yield_() ::Sleep(0) 160 | #else 161 | #include 162 | #define _yield_() ::sched_yield() 163 | #endif 164 | #define _testyield_() \ 165 | do { \ 166 | static uint32_t _KC_seed = 725; \ 167 | _KC_seed = _KC_seed * 123456761 + 211; \ 168 | if (_KC_seed % 0x100 == 0) _yield_(); \ 169 | } while(false) 170 | #define _assert_(KC_a) \ 171 | do { \ 172 | _testyield_(); \ 173 | assert(KC_a); \ 174 | } while(false) 175 | #elif defined(_KCDEBUG) 176 | #define _yield_() 177 | #define _testyield_() 178 | #define _assert_(KC_a) assert(KC_a) 179 | #else 180 | #define _yield_() ///< for debugging 181 | #define _testyield_() ///< for debugging 182 | #define _assert_(KC_a) ///< for debugging 183 | #endif 184 | 185 | #if defined(__GNUC__) 186 | #define __KCFUNC__ __func__ ///< for debugging 187 | #elif defined(_MSC_VER) 188 | #define __KCFUNC__ __FUNCTION__ ///< for debugging 189 | #else 190 | #define __KCFUNC__ "-" ///< for debugging 191 | #endif 192 | #define _KCCODELINE_ __FILE__, __LINE__, __KCFUNC__ ///< for debugging 193 | 194 | /** 195 | * All symbols of Kyoto Cabinet. 196 | */ 197 | namespace kyotocabinet {} 198 | 199 | 200 | #endif // duplication check 201 | 202 | // END OF FILE 203 | -------------------------------------------------------------------------------- /DbRedisBackend.cpp: -------------------------------------------------------------------------------- 1 | /* Redis DB backend 2 | * by Albert Zeyer, 2011 3 | * code under LGPL 4 | */ 5 | 6 | #include "DbRedisBackend.h" 7 | #include "StringUtils.h" 8 | #include "Utils.h" 9 | #include 10 | 11 | #include 12 | using namespace std; 13 | 14 | Return DbRedisBackend::init() { 15 | redis = redisConnect(host.c_str(), port); 16 | if(redis == NULL) 17 | return "failed to init Redis"; 18 | if(!(redis->flags & REDIS_CONNECTED)) 19 | return "failed to connect to Redis server"; 20 | return true; 21 | } 22 | 23 | DbRedisBackend::~DbRedisBackend() { 24 | if(redis != NULL) { 25 | redisFree(redis); 26 | redis = NULL; 27 | } 28 | } 29 | 30 | struct RedisReplyWrapper : DontCopyTag { 31 | redisReply* reply; 32 | RedisReplyWrapper(void* r = NULL) : reply(NULL) { (*this) = r; } 33 | ~RedisReplyWrapper() { clear(); } 34 | RedisReplyWrapper& operator=(void* r) { clear(); reply = (redisReply*)r; } 35 | void clear() { 36 | if(reply != NULL) { 37 | freeReplyObject(reply); 38 | reply = NULL; 39 | } 40 | } 41 | operator Return() const { 42 | if(reply == NULL) return "Redis: no reply"; 43 | if(reply->type == REDIS_REPLY_ERROR) return std::string() + "Redis: " + reply->str; 44 | return true; 45 | } 46 | }; 47 | 48 | static Return __saveNewDbEntry(redisContext* redis, const std::string& prefix, DbEntryId& id, const std::string& content) { 49 | unsigned short triesNum = (id.size() <= 4) ? (2 << id.size()) : 64; 50 | for(unsigned short i = 0; i < triesNum; ++i) { 51 | DbEntryId newId = id; 52 | newId += (char)random(); 53 | std::string key = prefix + "data." + newId; 54 | RedisReplyWrapper reply( redisCommand(redis, "SETNX %b %b", &key[0], key.size(), &content[0], content.size()) ); 55 | ASSERT( reply ); 56 | if(reply.reply->type == REDIS_REPLY_INTEGER && reply.reply->integer == 1) { 57 | id = newId; 58 | return true; 59 | } 60 | } 61 | 62 | id += (char)random(); 63 | return __saveNewDbEntry(redis, prefix, id, content); 64 | } 65 | 66 | Return DbRedisBackend::push(/*out*/ DbEntryId& id, const DbEntry& entry) { 67 | if(!entry.haveSha1()) 68 | return "DB push: entry SHA1 not calculated"; 69 | if(!entry.haveCompressed()) 70 | return "DB push: entry compression not calculated"; 71 | if(redis == NULL) 72 | return "DB push: Redis connection not initialized"; 73 | if(!(redis->flags & REDIS_CONNECTED)) 74 | return "DB push: Redis not connected"; 75 | 76 | // search for existing entry 77 | std::string sha1refkey = prefix + "sha1ref." + entry.sha1; 78 | RedisReplyWrapper reply( redisCommand(redis, "SMEMBERS %b", &sha1refkey[0], sha1refkey.size()) ); 79 | ASSERT( reply ); 80 | if(reply.reply->type == REDIS_REPLY_ARRAY) 81 | for(int i = 0; i < reply.reply->elements; ++i) { 82 | DbEntryId otherId = std::string(reply.reply->element[i]->str, reply.reply->element[i]->len); 83 | DbEntry otherEntry; 84 | if(get(otherEntry, otherId)) { 85 | if(entry == otherEntry) { 86 | // found 87 | id = otherId; 88 | stats.pushReuse++; 89 | return true; 90 | } 91 | } 92 | } 93 | 94 | // write DB entry 95 | id = ""; 96 | ASSERT( __saveNewDbEntry(redis, prefix, id, entry.compressed) ); 97 | 98 | // create sha1 ref 99 | reply = redisCommand(redis, "SADD %b %b", &sha1refkey[0], sha1refkey.size(), &id[0], id.size()); 100 | ASSERT( reply ); 101 | 102 | stats.pushNew++; 103 | return true; 104 | } 105 | 106 | Return DbRedisBackend::get(/*out*/ DbEntry& entry, const DbEntryId& id) { 107 | if(redis == NULL) 108 | return "DB get: Redis connection not initialized"; 109 | if(!(redis->flags & REDIS_CONNECTED)) 110 | return "DB get: Redis not connected"; 111 | 112 | std::string key = prefix + "data." + id; 113 | RedisReplyWrapper reply( redisCommand(redis, "GET %b", &key[0], key.size()) ); 114 | ASSERT(reply); 115 | if(reply.reply->type == REDIS_REPLY_NIL) 116 | return "DB get: entry not found"; 117 | if(reply.reply->type != REDIS_REPLY_STRING) 118 | return "DB get: Redis: invalid GET reply"; 119 | entry.compressed = std::string(reply.reply->str, reply.reply->len); 120 | 121 | ASSERT( entry.uncompress() ); 122 | entry.calcSha1(); 123 | 124 | return true; 125 | } 126 | 127 | Return DbRedisBackend::pushToDir(const std::string& path, const DbDirEntry& dirEntry) { 128 | std::string key = prefix + "fs." + path; 129 | std::string dirEntryRaw = dirEntry.serialized(); 130 | RedisReplyWrapper reply( redisCommand(redis, "SADD %b %b", &key[0], key.size(), &dirEntryRaw[0], dirEntryRaw.size()) ); 131 | ASSERT( reply ); 132 | return true; 133 | } 134 | 135 | Return DbRedisBackend::getDir(/*out*/ std::list& dirList, const std::string& path) { 136 | std::string key = prefix + "fs." + path; 137 | RedisReplyWrapper reply( redisCommand(redis, "SMEMBERS %b", &key[0], key.size()) ); 138 | ASSERT( reply ); 139 | if(reply.reply->type != REDIS_REPLY_ARRAY) 140 | return "DB getDir: invalid SMEMBERS reply"; 141 | 142 | for(int i = 0; i < reply.reply->elements; ++i) { 143 | std::string dirEntryRaw(reply.reply->element[i]->str, reply.reply->element[i]->len); 144 | dirList.push_back( DbDirEntry::FromSerialized(dirEntryRaw) ); 145 | } 146 | 147 | return true; 148 | } 149 | 150 | Return DbRedisBackend::setFileRef(/*can be empty*/ const DbEntryId& id, const std::string& path) { 151 | std::string key = prefix + "fs." + path; 152 | RedisReplyWrapper reply( redisCommand(redis, "SET %b %b", &key[0], key.size(), &id[0], id.size()) ); 153 | ASSERT( reply ); 154 | return true; 155 | } 156 | 157 | Return DbRedisBackend::getFileRef(/*out (can be empty)*/ DbEntryId& id, const std::string& path) { 158 | std::string key = prefix + "fs." + path; 159 | RedisReplyWrapper reply( redisCommand(redis, "GET %b", &key[0], key.size()) ); 160 | ASSERT( reply ); 161 | if(reply.reply->type != REDIS_REPLY_STRING) 162 | return "DB getFileRef: invalid GET reply"; 163 | id = std::string(reply.reply->str, reply.reply->len); 164 | return true; 165 | } 166 | 167 | -------------------------------------------------------------------------------- /kyotocabinet/kcregex.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Regular expression 3 | * Copyright (C) 2009-2011 FAL Labs 4 | * This file is part of Kyoto Cabinet. 5 | * This program is free software: you can redistribute it and/or modify it under the terms of 6 | * the GNU General Public License as published by the Free Software Foundation, either version 7 | * 3 of the License, or any later version. 8 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 9 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 10 | * See the GNU General Public License for more details. 11 | * You should have received a copy of the GNU General Public License along with this program. 12 | * If not, see . 13 | *************************************************************************************************/ 14 | 15 | 16 | #include "kcregex.h" 17 | #include "myconf.h" 18 | 19 | #if _KC_PXREGEX 20 | extern "C" { 21 | #include 22 | } 23 | #else 24 | #include 25 | #endif 26 | 27 | 28 | namespace kyotocabinet { // common namespace 29 | 30 | 31 | /** 32 | * Regex internal. 33 | */ 34 | struct RegexCore { 35 | #if _KC_PXREGEX 36 | ::regex_t rbuf; 37 | bool alive; 38 | bool nosub; 39 | #else 40 | std::regex* rbuf; 41 | #endif 42 | }; 43 | 44 | 45 | /** 46 | * Default constructor. 47 | */ 48 | Regex::Regex() : opq_(NULL) { 49 | #if _KC_PXREGEX 50 | _assert_(true); 51 | RegexCore* core = new RegexCore; 52 | core->alive = false; 53 | core->nosub = false; 54 | opq_ = (void*)core; 55 | #else 56 | _assert_(true); 57 | RegexCore* core = new RegexCore; 58 | core->rbuf = NULL; 59 | opq_ = (void*)core; 60 | #endif 61 | } 62 | 63 | 64 | /** 65 | * Destructor. 66 | */ 67 | Regex::~Regex() { 68 | #if _KC_PXREGEX 69 | _assert_(true); 70 | RegexCore* core = (RegexCore*)opq_; 71 | if (core->alive) ::regfree(&core->rbuf); 72 | delete core; 73 | #else 74 | _assert_(true); 75 | RegexCore* core = (RegexCore*)opq_; 76 | delete core->rbuf; 77 | delete core; 78 | #endif 79 | } 80 | 81 | 82 | /** 83 | * Compile a string of regular expression. 84 | */ 85 | bool Regex::compile(const std::string& regex, uint32_t opts) { 86 | #if _KC_PXREGEX 87 | _assert_(true); 88 | RegexCore* core = (RegexCore*)opq_; 89 | if (core->alive) { 90 | ::regfree(&core->rbuf); 91 | core->alive = false; 92 | } 93 | int32_t cflags = REG_EXTENDED; 94 | if (opts & IGNCASE) cflags |= REG_ICASE; 95 | if ((opts & MATCHONLY) || regex.empty()) { 96 | cflags |= REG_NOSUB; 97 | core->nosub = true; 98 | } 99 | if (::regcomp(&core->rbuf, regex.c_str(), cflags) != 0) return false; 100 | core->alive = true; 101 | return true; 102 | #else 103 | _assert_(true); 104 | RegexCore* core = (RegexCore*)opq_; 105 | if (core->rbuf) { 106 | delete core->rbuf; 107 | core->rbuf = NULL; 108 | } 109 | int32_t cflags = std::regex::ECMAScript; 110 | if (opts & IGNCASE) cflags |= std::regex::icase; 111 | if ((opts & MATCHONLY) || regex.empty()) cflags |= std::regex::nosubs; 112 | try { 113 | core->rbuf = new std::regex(regex, (std::regex::flag_type)cflags); 114 | } catch (...) { 115 | core->rbuf = NULL; 116 | return false; 117 | } 118 | return true; 119 | #endif 120 | } 121 | 122 | 123 | /** 124 | * Check whether a string matches the regular expression. 125 | */ 126 | bool Regex::match(const std::string& str) { 127 | #if _KC_PXREGEX 128 | _assert_(true); 129 | RegexCore* core = (RegexCore*)opq_; 130 | if (!core->alive) return false; 131 | if (core->nosub) return ::regexec(&core->rbuf, str.c_str(), 0, NULL, 0) == 0; 132 | ::regmatch_t subs[1]; 133 | return ::regexec(&core->rbuf, str.c_str(), 1, subs, 0) == 0; 134 | #else 135 | _assert_(true); 136 | RegexCore* core = (RegexCore*)opq_; 137 | if (!core->rbuf) return false; 138 | std::smatch res; 139 | return std::regex_search(str, res, *core->rbuf); 140 | #endif 141 | } 142 | 143 | 144 | /** 145 | * Check whether a string matches the regular expression. 146 | */ 147 | std::string Regex::replace(const std::string& str, const std::string& alt) { 148 | #if _KC_PXREGEX 149 | _assert_(true); 150 | RegexCore* core = (RegexCore*)opq_; 151 | if (!core->alive || core->nosub) return str; 152 | regmatch_t subs[256]; 153 | if (::regexec(&core->rbuf, str.c_str(), sizeof(subs) / sizeof(*subs), subs, 0) != 0) 154 | return str; 155 | const char* sp = str.c_str(); 156 | std::string xstr; 157 | bool first = true; 158 | while (sp[0] != '\0' && ::regexec(&core->rbuf, sp, 10, subs, first ? 0 : REG_NOTBOL) == 0) { 159 | first = false; 160 | if (subs[0].rm_so == -1) break; 161 | xstr.append(sp, subs[0].rm_so); 162 | for (const char* rp = alt.c_str(); *rp != '\0'; rp++) { 163 | if (*rp == '$') { 164 | if (rp[1] >= '0' && rp[1] <= '9') { 165 | int32_t num = rp[1] - '0'; 166 | if (subs[num].rm_so != -1 && subs[num].rm_eo != -1) 167 | xstr.append(sp + subs[num].rm_so, subs[num].rm_eo - subs[num].rm_so); 168 | ++rp; 169 | } else if (rp[1] == '&') { 170 | xstr.append(sp + subs[0].rm_so, subs[0].rm_eo - subs[0].rm_so); 171 | ++rp; 172 | } else if (rp[1] != '\0') { 173 | xstr.append(++rp, 1); 174 | } 175 | } else { 176 | xstr.append(rp, 1); 177 | } 178 | } 179 | sp += subs[0].rm_eo; 180 | if (subs[0].rm_eo < 1) break; 181 | } 182 | xstr.append(sp); 183 | return xstr; 184 | #else 185 | _assert_(true); 186 | RegexCore* core = (RegexCore*)opq_; 187 | if (!core->rbuf) return str; 188 | return std::regex_replace(str, *core->rbuf, alt); 189 | #endif 190 | } 191 | 192 | 193 | } // common namespace 194 | 195 | // END OF FILE 196 | -------------------------------------------------------------------------------- /hiredis/net.c: -------------------------------------------------------------------------------- 1 | /* Extracted from anet.c to work properly with Hiredis error reporting. 2 | * 3 | * Copyright (c) 2006-2010, Salvatore Sanfilippo 4 | * Copyright (c) 2010, Pieter Noordhuis 5 | * 6 | * All rights reserved. 7 | * Redistribution and use in source and binary forms, with or without 8 | * modification, are permitted provided that the following conditions are met: 9 | * 10 | * * Redistributions of source code must retain the above copyright notice, 11 | * this list of conditions and the following disclaimer. 12 | * * Redistributions in binary form must reproduce the above copyright 13 | * notice, this list of conditions and the following disclaimer in the 14 | * documentation and/or other materials provided with the distribution. 15 | * * Neither the name of Redis nor the names of its contributors may be used 16 | * to endorse or promote products derived from this software without 17 | * specific prior written permission. 18 | * 19 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 23 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 | * POSSIBILITY OF SUCH DAMAGE. 30 | */ 31 | 32 | #include "fmacros.h" 33 | #include 34 | #include 35 | #include 36 | #include 37 | #include 38 | #include 39 | #include 40 | #include 41 | #include 42 | #include 43 | #include 44 | #include 45 | #include 46 | 47 | #include "net.h" 48 | #include "sds.h" 49 | 50 | /* Forward declaration */ 51 | void __redisSetError(redisContext *c, int type, sds err); 52 | 53 | static int redisCreateSocket(redisContext *c, int type) { 54 | int s, on = 1; 55 | if ((s = socket(type, SOCK_STREAM, 0)) == -1) { 56 | __redisSetError(c,REDIS_ERR_IO,NULL); 57 | return REDIS_ERR; 58 | } 59 | if (type == AF_INET) { 60 | if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)) == -1) { 61 | __redisSetError(c,REDIS_ERR_IO,NULL); 62 | close(s); 63 | return REDIS_ERR; 64 | } 65 | } 66 | return s; 67 | } 68 | 69 | static int redisSetNonBlock(redisContext *c, int fd) { 70 | int flags; 71 | 72 | /* Set the socket nonblocking. 73 | * Note that fcntl(2) for F_GETFL and F_SETFL can't be 74 | * interrupted by a signal. */ 75 | if ((flags = fcntl(fd, F_GETFL)) == -1) { 76 | __redisSetError(c,REDIS_ERR_IO, 77 | sdscatprintf(sdsempty(), "fcntl(F_GETFL): %s", strerror(errno))); 78 | close(fd); 79 | return REDIS_ERR; 80 | } 81 | if (fcntl(fd, F_SETFL, flags | O_NONBLOCK) == -1) { 82 | __redisSetError(c,REDIS_ERR_IO, 83 | sdscatprintf(sdsempty(), "fcntl(F_SETFL,O_NONBLOCK): %s", strerror(errno))); 84 | close(fd); 85 | return REDIS_ERR; 86 | } 87 | return REDIS_OK; 88 | } 89 | 90 | static int redisSetTcpNoDelay(redisContext *c, int fd) { 91 | int yes = 1; 92 | if (setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &yes, sizeof(yes)) == -1) { 93 | __redisSetError(c,REDIS_ERR_IO, 94 | sdscatprintf(sdsempty(), "setsockopt(TCP_NODELAY): %s", strerror(errno))); 95 | return REDIS_ERR; 96 | } 97 | return REDIS_OK; 98 | } 99 | 100 | int redisContextConnectTcp(redisContext *c, const char *addr, int port) { 101 | int s; 102 | int blocking = (c->flags & REDIS_BLOCK); 103 | struct sockaddr_in sa; 104 | 105 | if ((s = redisCreateSocket(c,AF_INET)) == REDIS_ERR) 106 | return REDIS_ERR; 107 | if (!blocking && redisSetNonBlock(c,s) == REDIS_ERR) 108 | return REDIS_ERR; 109 | 110 | sa.sin_family = AF_INET; 111 | sa.sin_port = htons(port); 112 | if (inet_aton(addr, &sa.sin_addr) == 0) { 113 | struct hostent *he; 114 | 115 | he = gethostbyname(addr); 116 | if (he == NULL) { 117 | __redisSetError(c,REDIS_ERR_OTHER, 118 | sdscatprintf(sdsempty(),"Can't resolve: %s",addr)); 119 | close(s); 120 | return REDIS_ERR; 121 | } 122 | memcpy(&sa.sin_addr, he->h_addr, sizeof(struct in_addr)); 123 | } 124 | 125 | if (connect(s, (struct sockaddr*)&sa, sizeof(sa)) == -1) { 126 | if (errno == EINPROGRESS && !blocking) { 127 | /* This is ok. */ 128 | } else { 129 | __redisSetError(c,REDIS_ERR_IO,NULL); 130 | close(s); 131 | return REDIS_ERR; 132 | } 133 | } 134 | 135 | if (redisSetTcpNoDelay(c,s) != REDIS_OK) { 136 | close(s); 137 | return REDIS_ERR; 138 | } 139 | 140 | c->fd = s; 141 | c->flags |= REDIS_CONNECTED; 142 | return REDIS_OK; 143 | } 144 | 145 | int redisContextConnectUnix(redisContext *c, const char *path) { 146 | int s; 147 | int blocking = (c->flags & REDIS_BLOCK); 148 | struct sockaddr_un sa; 149 | 150 | if ((s = redisCreateSocket(c,AF_LOCAL)) == REDIS_ERR) 151 | return REDIS_ERR; 152 | if (!blocking && redisSetNonBlock(c,s) != REDIS_OK) 153 | return REDIS_ERR; 154 | 155 | sa.sun_family = AF_LOCAL; 156 | strncpy(sa.sun_path,path,sizeof(sa.sun_path)-1); 157 | if (connect(s, (struct sockaddr*)&sa, sizeof(sa)) == -1) { 158 | if (errno == EINPROGRESS && !blocking) { 159 | /* This is ok. */ 160 | } else { 161 | __redisSetError(c,REDIS_ERR_IO,NULL); 162 | close(s); 163 | return REDIS_ERR; 164 | } 165 | } 166 | 167 | c->fd = s; 168 | c->flags |= REDIS_CONNECTED; 169 | return REDIS_OK; 170 | } 171 | -------------------------------------------------------------------------------- /kyotocabinet/myconf.h: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * System-dependent configurations 3 | * Copyright (C) 2009-2011 FAL Labs 4 | * This file is part of Kyoto Cabinet. 5 | * This program is free software: you can redistribute it and/or modify it under the terms of 6 | * the GNU General Public License as published by the Free Software Foundation, either version 7 | * 3 of the License, or any later version. 8 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 9 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 10 | * See the GNU General Public License for more details. 11 | * You should have received a copy of the GNU General Public License along with this program. 12 | * If not, see . 13 | *************************************************************************************************/ 14 | 15 | 16 | #ifndef _MYCONF_H // duplication check 17 | #define _MYCONF_H 18 | 19 | 20 | 21 | /************************************************************************************************* 22 | * system discrimination 23 | *************************************************************************************************/ 24 | 25 | 26 | #if defined(__linux__) 27 | 28 | #define _SYS_LINUX_ 29 | #define _KC_SYSNAME "Linux" 30 | 31 | #elif defined(__FreeBSD__) 32 | 33 | #define _SYS_FREEBSD_ 34 | #define _KC_SYSNAME "FreeBSD" 35 | 36 | #elif defined(__NetBSD__) 37 | 38 | #define _SYS_NETBSD_ 39 | #define _KC_SYSNAME "NetBSD" 40 | 41 | #elif defined(__OpenBSD__) 42 | 43 | #define _SYS_OPENBSD_ 44 | #define _KC_SYSNAME "OpenBSD" 45 | 46 | #elif defined(__sun__) || defined(__sun) 47 | 48 | #define _SYS_SUNOS_ 49 | #define _KC_SYSNAME "SunOS" 50 | 51 | #elif defined(__hpux) 52 | 53 | #define _SYS_HPUX_ 54 | #define _KC_SYSNAME "HP-UX" 55 | 56 | #elif defined(__osf) 57 | 58 | #define _SYS_TRU64_ 59 | #define _KC_SYSNAME "Tru64" 60 | 61 | #elif defined(_AIX) 62 | 63 | #define _SYS_AIX_ 64 | #define _KC_SYSNAME "AIX" 65 | 66 | #elif defined(__APPLE__) && defined(__MACH__) 67 | 68 | #define _SYS_MACOSX_ 69 | #define _KC_SYSNAME "Mac OS X" 70 | 71 | #elif defined(_MSC_VER) 72 | 73 | #define _SYS_MSVC_ 74 | #define _KC_SYSNAME "Windows (VC++)" 75 | 76 | #elif defined(_WIN32) 77 | 78 | #define _SYS_MINGW_ 79 | #define _KC_SYSNAME "Windows (MinGW)" 80 | 81 | #elif defined(__CYGWIN__) 82 | 83 | #define _SYS_CYGWIN_ 84 | #define _KC_SYSNAME "Windows (Cygwin)" 85 | 86 | #else 87 | 88 | #define _SYS_GENERIC_ 89 | #define _KC_SYSNAME "Generic" 90 | 91 | #endif 92 | 93 | #define _KC_VERSION "1.2.48" 94 | #define _KC_LIBVER 9 95 | #define _KC_LIBREV 9 96 | #define _KC_FMTVER 5 97 | 98 | #if defined(_MYBIGEND) 99 | #define _KC_BIGEND 1 100 | #else 101 | #define _KC_BIGEND 0 102 | #endif 103 | 104 | #if defined(_MYGCCATOMIC) 105 | #define _KC_GCCATOMIC 1 106 | #else 107 | #define _KC_GCCATOMIC 0 108 | #endif 109 | 110 | #if defined(_MYZLIB) 111 | #define _KC_ZLIB 1 112 | #else 113 | #define _KC_ZLIB 0 114 | #endif 115 | 116 | #if defined(_MYLZO) 117 | #define _KC_LZO 1 118 | #else 119 | #define _KC_LZO 0 120 | #endif 121 | 122 | #if defined(_MYLZMA) 123 | #define _KC_LZMA 1 124 | #else 125 | #define _KC_LZMA 0 126 | #endif 127 | 128 | #if defined(_SYS_MSVC_) 129 | #define _KC_PXREGEX 0 130 | #else 131 | #define _KC_PXREGEX 1 132 | #endif 133 | 134 | 135 | 136 | /************************************************************************************************* 137 | * notation of the file system 138 | *************************************************************************************************/ 139 | 140 | 141 | #if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) 142 | 143 | #define MYPATHCHR '\\' 144 | #define MYPATHSTR "\\" 145 | #define MYEXTCHR '.' 146 | #define MYEXTSTR "." 147 | #define MYCDIRSTR "." 148 | #define MYPDIRSTR ".." 149 | 150 | #else 151 | 152 | #define MYPATHCHR '/' 153 | #define MYPATHSTR "/" 154 | #define MYEXTCHR '.' 155 | #define MYEXTSTR "." 156 | #define MYCDIRSTR "." 157 | #define MYPDIRSTR ".." 158 | 159 | #endif 160 | 161 | 162 | 163 | /************************************************************************************************* 164 | * general headers 165 | *************************************************************************************************/ 166 | 167 | 168 | extern "C" { 169 | #include 170 | #include 171 | #include 172 | #include 173 | #include 174 | #include 175 | #include 176 | #include 177 | #include 178 | #include 179 | #include 180 | #include 181 | #include 182 | #include 183 | #include 184 | } 185 | 186 | extern "C" { 187 | #include 188 | } 189 | 190 | #if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) 191 | 192 | #include 193 | #include 194 | #include 195 | #include 196 | #include 197 | #include 198 | #include 199 | 200 | #else 201 | 202 | extern "C" { 203 | #include 204 | #include 205 | #include 206 | #include 207 | #include 208 | #include 209 | #include 210 | #include 211 | #include 212 | #include 213 | #include 214 | } 215 | 216 | extern "C" { 217 | #include 218 | #include 219 | } 220 | 221 | #endif 222 | 223 | #if defined(_SYS_FREEBSD_) || defined(_SYS_OPENBSD_) || defined(_SYS_NETBSD_) || \ 224 | defined(_SYS_MACOSX_) 225 | #define pthread_spinlock_t pthread_mutex_t 226 | #define pthread_spin_init(KC_a, KC_b) \ 227 | pthread_mutex_init(KC_a, NULL) 228 | #define pthread_spin_destroy(KC_a) \ 229 | pthread_mutex_destroy(KC_a) 230 | #define pthread_spin_lock(KC_a) \ 231 | pthread_mutex_lock(KC_a) 232 | #define pthread_spin_trylock(KC_a) \ 233 | pthread_mutex_trylock(KC_a) 234 | #define pthread_spin_unlock(KC_a) \ 235 | pthread_mutex_unlock(KC_a) 236 | #endif 237 | 238 | 239 | #endif // duplication check 240 | 241 | 242 | // END OF FILE 243 | -------------------------------------------------------------------------------- /DbPng.cpp: -------------------------------------------------------------------------------- 1 | /* PNG data <-> DB interface 2 | * by Albert Zeyer, 2011 3 | * code under LGPL 4 | */ 5 | 6 | #include "Db.h" 7 | #include "DbPng.h" 8 | #include "StringUtils.h" 9 | #include "Sha1.h" 10 | #include 11 | #include 12 | using namespace std; 13 | 14 | #define PngBlockSize 64 15 | 16 | Return DbPngEntryWriter::next() { 17 | ASSERT( reader.read() ); 18 | 19 | while(true) { 20 | bool isFinal = false; 21 | if(reader.chunks.size() > 0) { 22 | DbEntry entry; 23 | entry.data += (char)DbEntryType_PngChunk; 24 | entry.data += reader.chunks.front().type; 25 | entry.data += reader.chunks.front().data; 26 | reader.chunks.pop_front(); 27 | entry.prepare(); 28 | 29 | DbEntryId id; 30 | ASSERT( db->push(id, entry) ); 31 | contentChunkEntries.push_back(id); 32 | continue; 33 | } 34 | else if(reader.scanlines.size() > 0) { 35 | if(reader.scanlines.size() >= PngBlockSize || reader.hasFinishedReading) { 36 | size_t scanlineWidth = reader.scanlines.front().size(); 37 | uint8_t blockHeight = 0; 38 | for(std::list::iterator it = reader.scanlines.begin(); 39 | it != reader.scanlines.end() && blockHeight < PngBlockSize; 40 | ++it) { 41 | if(it->size() != scanlineWidth) break; 42 | ++blockHeight; 43 | } 44 | 45 | for(size_t x = 0; x < scanlineWidth; x += PngBlockSize) { 46 | DbEntry entry; 47 | entry.data += (char)DbEntryType_PngBlock; 48 | entry.data += rawString( x / PngBlockSize ); 49 | if(x == 0) entry.data += rawString( blockHeight ); 50 | size_t i = 0; 51 | for(std::list::iterator it = reader.scanlines.begin(); i < blockHeight; ++it, ++i) 52 | entry.data += it->substr(x, PngBlockSize); 53 | entry.prepare(); 54 | 55 | DbEntryId id; 56 | ASSERT( db->push(id, entry) ); 57 | contentDataEntries.push_back(id); 58 | } 59 | 60 | for(size_t i = 0; i < blockHeight; ++i) 61 | reader.scanlines.pop_front(); 62 | continue; 63 | } 64 | else 65 | break; 66 | } 67 | else if(reader.hasFinishedReading) { 68 | DbEntry entry; 69 | entry.data += (char)DbEntryType_PngContentList; 70 | for(std::list::iterator i = contentChunkEntries.begin(); i != contentChunkEntries.end(); ++i) { 71 | if(i->size() > 255) 72 | return "we have an ID with len > 255"; 73 | entry.data += rawString(i->size()); 74 | entry.data += *i; 75 | } 76 | for(std::list::iterator i = contentDataEntries.begin(); i != contentDataEntries.end(); ++i) { 77 | if(i->size() > 255) 78 | return "we have an ID with len > 255"; 79 | entry.data += rawString(i->size()); 80 | entry.data += *i; 81 | } 82 | entry.prepare(); 83 | DbEntryId id; 84 | ASSERT( db->push(id, entry) ); 85 | contentId = id; 86 | break; 87 | } 88 | 89 | break; 90 | } 91 | 92 | return true; 93 | } 94 | 95 | static Return __readPngChunk(PngChunk& chunk, const std::string& data) { 96 | if(data.size() < 5) 97 | return "chunk entry too small"; 98 | chunk.type = data.substr(1,4); 99 | chunk.data = data.substr(5); 100 | return true; 101 | } 102 | 103 | static Return __finishBlock(DbPngEntryReader& png) { 104 | if(png.blockList.blockHeight == 0 && png.blockList.blocks.size() == 0) 105 | // there was no previos unfinished block. but this is ok 106 | return true; 107 | 108 | std::vector scanlines(png.blockList.blockHeight); 109 | size_t x = 0; 110 | for(std::list::iterator block = png.blockList.blocks.begin(); block != png.blockList.blocks.end(); ++block) { 111 | size_t blockWidth = block->size() / png.blockList.blockHeight; 112 | for(short y = 0; y < png.blockList.blockHeight; ++y) { 113 | scanlines[y] += block->substr(y * blockWidth, blockWidth); 114 | } 115 | x += blockWidth; 116 | } 117 | 118 | for(short y = 0; y < png.blockList.blockHeight; ++y) 119 | png.writer.scanlines.push_back(scanlines[y]); 120 | 121 | png.blockList = DbPngEntryBlockList(); 122 | return true; 123 | } 124 | 125 | static Return __readBlock(DbPngEntryReader& png, const std::string& data) { 126 | size_t offset = 1; // first was the DbEntry type 127 | if(data.size() < offset + sizeof(uint16_t)) 128 | return "block entry too small (before reading x-offset)"; 129 | uint32_t x = valueFromRaw(&data[offset]) * PngBlockSize; 130 | offset += sizeof(uint16_t); 131 | 132 | if(x == 0) { 133 | // new block row start 134 | ASSERT( __finishBlock(png) ); 135 | if(data.size() < offset + sizeof(uint8_t)) 136 | return "block entry too small (before reading block height)"; 137 | png.blockList.blockHeight = valueFromRaw(&data[offset]); 138 | offset += sizeof(uint8_t); 139 | if(png.blockList.blockHeight == 0) 140 | return "block height invalid"; 141 | } 142 | 143 | if(((data.size() - offset) % png.blockList.blockHeight) != 0) 144 | return "block entry raw size is not a multiple of blockHeight"; 145 | size_t blockWidth = (data.size() - offset) / png.blockList.blockHeight; 146 | 147 | png.blockList.scanlineWidth += blockWidth; 148 | png.blockList.blocks.push_back( data.substr(offset) ); 149 | 150 | return true; 151 | } 152 | 153 | Return DbPngEntryReader::next() { 154 | if(!haveContentEntries) { 155 | DbEntry entry; 156 | ASSERT( db->get(entry, contentId) ); 157 | if(entry.data.size() == 0) 158 | return "content entry list data is empty"; 159 | if(entry.data[0] != DbEntryType_PngContentList) 160 | return "content entry list data is invalid"; 161 | size_t i = 1; 162 | while(i < entry.data.size()) { 163 | uint8_t size = entry.data[i]; 164 | ++i; 165 | if(i + size > entry.data.size()) 166 | return "content entry list data is inconsistent"; 167 | contentEntries.push_back( DbEntryId(entry.data.substr(i, size)) ); 168 | i += size; 169 | } 170 | haveContentEntries = true; 171 | return true; 172 | } 173 | 174 | if(contentEntries.size() > 0) { 175 | DbEntry entry; 176 | ASSERT( db->get(entry, contentEntries.front()) ); 177 | contentEntries.pop_front(); 178 | if(entry.data.size() == 0) 179 | return "content entry data is empty"; 180 | switch(entry.data[0]) { 181 | case DbEntryType_PngChunk: { 182 | PngChunk chunk; 183 | ASSERT( __readPngChunk(chunk, entry.data) ); 184 | writer.chunks.push_back(chunk); 185 | break; 186 | } 187 | case DbEntryType_PngBlock: { 188 | writer.hasAllChunks = true; // there wont be any more PngChunk entries 189 | ASSERT( __readBlock(*this, entry.data) ); 190 | break; 191 | } 192 | default: 193 | return "content entry data is invalid"; 194 | } 195 | } 196 | if(contentEntries.size() == 0) { 197 | ASSERT( __finishBlock(*this) ); 198 | writer.hasAllChunks = true; 199 | writer.hasAllScanlines = true; 200 | } 201 | 202 | ASSERT( writer.write() ); 203 | 204 | return true; 205 | } 206 | -------------------------------------------------------------------------------- /kyotocabinet/kccompare.h: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Comparator functions 3 | * Copyright (C) 2009-2011 FAL Labs 4 | * This file is part of Kyoto Cabinet. 5 | * This program is free software: you can redistribute it and/or modify it under the terms of 6 | * the GNU General Public License as published by the Free Software Foundation, either version 7 | * 3 of the License, or any later version. 8 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 9 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 10 | * See the GNU General Public License for more details. 11 | * You should have received a copy of the GNU General Public License along with this program. 12 | * If not, see . 13 | *************************************************************************************************/ 14 | 15 | 16 | #ifndef _KCCOMPARE_H // duplication check 17 | #define _KCCOMPARE_H 18 | 19 | #include 20 | #include 21 | 22 | namespace kyotocabinet { // common namespace 23 | 24 | 25 | /** 26 | * Interfrace of comparator of record keys. 27 | */ 28 | class Comparator { 29 | public: 30 | /** 31 | * Destructor. 32 | */ 33 | virtual ~Comparator() {} 34 | /** 35 | * Compare two keys. 36 | * @param akbuf the pointer to the region of one key. 37 | * @param aksiz the size of the region of one key. 38 | * @param bkbuf the pointer to the region of the other key. 39 | * @param bksiz the size of the region of the other key. 40 | * @return positive if the former is big, negative if the latter is big, 0 if both are 41 | * equivalent. 42 | */ 43 | virtual int32_t compare(const char* akbuf, size_t aksiz, const char* bkbuf, size_t bksiz) = 0; 44 | }; 45 | 46 | 47 | /** 48 | * Comparator in the lexical order. 49 | */ 50 | class LexicalComparator : public Comparator { 51 | public: 52 | explicit LexicalComparator() {} 53 | int32_t compare(const char* akbuf, size_t aksiz, const char* bkbuf, size_t bksiz) { 54 | _assert_(akbuf && bkbuf); 55 | size_t msiz = aksiz < bksiz ? aksiz : bksiz; 56 | for (size_t i = 0; i < msiz; i++) { 57 | if (((uint8_t*)akbuf)[i] != ((uint8_t*)bkbuf)[i]) 58 | return ((uint8_t*)akbuf)[i] - ((uint8_t*)bkbuf)[i]; 59 | } 60 | return (int32_t)aksiz - (int32_t)bksiz; 61 | } 62 | }; 63 | 64 | 65 | /** 66 | * Comparator in the lexical descending order. 67 | */ 68 | class LexicalDescendingComparator : public Comparator { 69 | public: 70 | explicit LexicalDescendingComparator() : comp_() {} 71 | int32_t compare(const char* akbuf, size_t aksiz, const char* bkbuf, size_t bksiz) { 72 | return -comp_.compare(akbuf, aksiz, bkbuf, bksiz); 73 | } 74 | private: 75 | LexicalComparator comp_; 76 | }; 77 | 78 | 79 | /** 80 | * Comparator in the decimal order. 81 | */ 82 | class DecimalComparator : public Comparator { 83 | public: 84 | explicit DecimalComparator() {} 85 | int32_t compare(const char* akbuf, size_t aksiz, const char* bkbuf, size_t bksiz) { 86 | _assert_(akbuf && bkbuf); 87 | const int32_t LDBLCOLMAX = 16; 88 | const unsigned char* arp = (unsigned char*)akbuf; 89 | int32_t alen = aksiz; 90 | while (alen > 0 && (*arp <= ' ' || *arp == 0x7f)) { 91 | arp++; 92 | alen--; 93 | } 94 | int64_t anum = 0; 95 | int32_t asign = 1; 96 | if (alen > 0 && *arp == '-') { 97 | arp++; 98 | alen--; 99 | asign = -1; 100 | } 101 | while (alen > 0) { 102 | int32_t c = *arp; 103 | if (c < '0' || c > '9') break; 104 | anum = anum * 10 + c - '0'; 105 | arp++; 106 | alen--; 107 | } 108 | anum *= asign; 109 | const unsigned char* brp = (unsigned char*)bkbuf; 110 | int32_t blen = bksiz; 111 | while (blen > 0 && (*brp <= ' ' || *brp == 0x7f)) { 112 | brp++; 113 | blen--; 114 | } 115 | int64_t bnum = 0; 116 | int32_t bsign = 1; 117 | if (blen > 0 && *brp == '-') { 118 | brp++; 119 | blen--; 120 | bsign = -1; 121 | } 122 | while (blen > 0) { 123 | int32_t c = *brp; 124 | if (c < '0' || c > '9') break; 125 | bnum = bnum * 10 + c - '0'; 126 | brp++; 127 | blen--; 128 | } 129 | bnum *= bsign; 130 | if (anum < bnum) return -1; 131 | if (anum > bnum) return 1; 132 | if ((alen > 1 && *arp == '.') || (blen > 1 && *brp == '.')) { 133 | long double aflt = 0; 134 | if (alen > 1 && *arp == '.') { 135 | arp++; 136 | alen--; 137 | if (alen > LDBLCOLMAX) alen = LDBLCOLMAX; 138 | long double base = 10; 139 | while (alen > 0) { 140 | if (*arp < '0' || *arp > '9') break; 141 | aflt += (*arp - '0') / base; 142 | arp++; 143 | alen--; 144 | base *= 10; 145 | } 146 | aflt *= asign; 147 | } 148 | long double bflt = 0; 149 | if (blen > 1 && *brp == '.') { 150 | brp++; 151 | blen--; 152 | if (blen > LDBLCOLMAX) blen = LDBLCOLMAX; 153 | long double base = 10; 154 | while (blen > 0) { 155 | if (*brp < '0' || *brp > '9') break; 156 | bflt += (*brp - '0') / base; 157 | brp++; 158 | blen--; 159 | base *= 10; 160 | } 161 | bflt *= bsign; 162 | } 163 | if (aflt < bflt) return -1; 164 | if (aflt > bflt) return 1; 165 | } 166 | LexicalComparator lexcomp; 167 | int32_t rv = lexcomp.compare(akbuf, aksiz, bkbuf, bksiz); 168 | return rv; 169 | } 170 | }; 171 | 172 | 173 | /** 174 | * Comparator in the decimal descending order. 175 | */ 176 | class DecimalDescendingComparator : public Comparator { 177 | public: 178 | explicit DecimalDescendingComparator() : comp_() {} 179 | int32_t compare(const char* akbuf, size_t aksiz, const char* bkbuf, size_t bksiz) { 180 | return -comp_.compare(akbuf, aksiz, bkbuf, bksiz); 181 | } 182 | private: 183 | DecimalComparator comp_; 184 | }; 185 | 186 | 187 | /** 188 | * Prepared pointer of the comparator in the lexical order. 189 | */ 190 | extern LexicalComparator* const LEXICALCOMP; 191 | 192 | 193 | /** 194 | * Prepared pointer of the comparator in the lexical descending order. 195 | */ 196 | extern LexicalDescendingComparator* const LEXICALDESCCOMP; 197 | 198 | 199 | /** 200 | * Prepared pointer of the comparator in the decimal order. 201 | */ 202 | extern DecimalComparator* const DECIMALCOMP; 203 | 204 | 205 | /** 206 | * Prepared pointer of the comparator in the decimal descending order. 207 | */ 208 | extern DecimalDescendingComparator* const DECIMALDESCCOMP; 209 | 210 | 211 | } // common namespace 212 | 213 | #endif // duplication check 214 | 215 | // END OF FILE 216 | -------------------------------------------------------------------------------- /db-fuse.cpp: -------------------------------------------------------------------------------- 1 | /* FUSE access to DB 2 | * by Albert Zeyer, 2011 3 | * code under LGPL 4 | */ 5 | 6 | #include "Db.h" 7 | #include "DbDefBackend.h" 8 | #include "Utils.h" 9 | #include "DbPng.h" 10 | #include "Mutex.h" 11 | #include "SmartPointer.h" 12 | 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | #define _FILE_OFFSET_BITS 64 19 | #define FUSE_USE_VERSION 26 20 | #include 21 | 22 | #include 23 | using namespace std; 24 | 25 | static DbIntf* db = NULL; 26 | 27 | //#define DEBUG 28 | 29 | #ifdef DEBUG 30 | #define debugPrint(out, msg) { out << "**** " << (std::string() + msg) << " ****" << endl; } 31 | #else 32 | #define debugPrint(out, msg) {} 33 | #endif 34 | 35 | #define CHECK_RET(x, err_ret, err_msg) \ 36 | { Return ___r = (x); if(!___r) { debugPrint(cerr, err_msg + ": " + ___r.errmsg); return err_ret; } } 37 | 38 | // Currently, the size we return here is not exactly correct (it is the size 39 | // of the files when we pushed them to the DB but they will likely be somewhat 40 | // different when you get them out). 41 | // If this size is smaller then what we would return here, we get into trouble 42 | // that many readers would skip the rest of the data and thus the PNG seems 43 | // incomplete. 44 | // So just add some random value to make sure we return a size which is bigger 45 | // than the actual file we would return here. 46 | #define SAFETY_ADDED_SIZE 1000000 47 | 48 | static int db_getattr(const char *path, struct stat *stbuf) { 49 | memset(stbuf, 0, sizeof(struct stat)); 50 | debugPrint(cout, "db_getattr: " + path); 51 | 52 | if(*path == '/') ++path; // skip '/' at the beginning 53 | if(*path == '\0') { 54 | stbuf->st_mode = S_IFDIR | 0755; 55 | stbuf->st_nlink = 2; 56 | } 57 | else { 58 | std::string filename = path; 59 | std::string basename = baseFilename(filename); 60 | std::string dirname = dirName(filename); 61 | 62 | std::list dirList; 63 | CHECK_RET(db->getDir(dirList, dirname), -ENOENT, "db_getattr: getDir failed"); 64 | 65 | for(std::list::iterator i = dirList.begin(); i != dirList.end(); ++i) 66 | if(i->name == basename) { 67 | stbuf->st_mode = i->mode; 68 | stbuf->st_nlink = 1; 69 | stbuf->st_size = i->size + /* to be sure */ SAFETY_ADDED_SIZE; 70 | break; 71 | } 72 | } 73 | 74 | CHECK_RET(stbuf->st_mode != 0, -ENOENT, "db_getattr: not found"); 75 | return 0; 76 | } 77 | 78 | static int db_open(const char *path, struct fuse_file_info *fi) { 79 | if(*path == '/') ++path; // skip '/' at the beginning 80 | std::string filename = path; 81 | filename = dirName(filename) + "/" + baseFilename(filename); 82 | 83 | debugPrint(cout, "db_open: " + filename); 84 | 85 | DbEntryId id; 86 | CHECK_RET(db->getFileRef(id, filename), -ENOENT, "db_open: fileref not found"); 87 | 88 | CHECK_RET((fi->flags & O_ACCMODE) == O_RDONLY, -EACCES, "db_open: only reading allowed"); 89 | 90 | return 0; 91 | } 92 | 93 | static int db_readdir(const char *path, void *buf, fuse_fill_dir_t filler, 94 | off_t offset, struct fuse_file_info *fi) { 95 | if(*path == '/') ++path; // skip '/' at the beginning 96 | 97 | std::list dirList; 98 | CHECK_RET(db->getDir(dirList, path), -ENOENT, "db_readdir: getDir failed"); 99 | 100 | filler(buf, ".", NULL, 0); /* Current directory (.) */ 101 | filler(buf, "..", NULL, 0); /* Parent directory (..) */ 102 | 103 | for(std::list::iterator i = dirList.begin(); i != dirList.end(); ++i) { 104 | struct stat stbuf; 105 | memset(&stbuf, 0, sizeof(struct stat)); 106 | stbuf.st_mode = i->mode; 107 | stbuf.st_nlink = 1; 108 | stbuf.st_size = i->size + /* to be sure */ SAFETY_ADDED_SIZE; 109 | filler(buf, i->name.c_str(), &stbuf, 0); 110 | } 111 | 112 | return 0; 113 | } 114 | 115 | 116 | struct FileContent : WriteCallbackIntf { 117 | Mutex mutex; 118 | DbEntryId fileEntryId; 119 | std::string data; 120 | bool finished; 121 | DbPngEntryReader dbPngReader; 122 | 123 | FileContent(const std::string& _fileEntryId) 124 | : fileEntryId(_fileEntryId), finished(false), dbPngReader(this, db, fileEntryId) {} 125 | 126 | Return write(const char* d, size_t s) { 127 | // we already locked the mutex here 128 | data.append(d, s); 129 | return true; 130 | } 131 | 132 | int read(char* buf, size_t size, off_t offset) { 133 | ScopedLock lock(mutex); 134 | 135 | while(!finished && offset + size > data.size()) { 136 | if(dbPngReader) { 137 | CHECK_RET(dbPngReader.next(), -EIO, "db_read: reading failed"); 138 | } else 139 | finished = true; 140 | } 141 | 142 | if(offset >= data.size()) // Reading behind the content. 143 | return 0; 144 | 145 | if (offset + size > data.size()) // Trim the read to the file size. 146 | size = data.size() - offset; 147 | 148 | memcpy(buf, &data[offset], size); // Provide the content. 149 | return size; 150 | } 151 | }; 152 | 153 | struct FileCache { 154 | static const size_t NUM_ENTRIES = 20; 155 | Mutex mutex; 156 | typedef std::list< SmartPointer > CacheList; 157 | CacheList cache; 158 | 159 | SmartPointer getContent(const DbEntryId& fileEntryId) { 160 | ScopedLock lock(mutex); 161 | for(CacheList::iterator i = cache.begin(); i != cache.end(); ++i) { 162 | // i->fileEntryId can be accessed safely because we never write it again 163 | if((*i)->fileEntryId == fileEntryId) { 164 | // found it 165 | // now push to front of cacheList 166 | SmartPointer content = *i; 167 | cache.erase(i); 168 | cache.push_front(content); 169 | return content; 170 | } 171 | } 172 | 173 | // not found -> create new 174 | SmartPointer content = new FileContent(fileEntryId); 175 | cache.push_front(content); 176 | // pop old entries 177 | while(cache.size() > NUM_ENTRIES) 178 | cache.pop_back(); 179 | return content; 180 | } 181 | 182 | int read(const DbEntryId& fileEntryId, char* buf, size_t size, off_t offset) { 183 | return getContent(fileEntryId)->read(buf, size, offset); 184 | } 185 | }; 186 | 187 | static int db_read(const char *path, char *buf, size_t size, off_t offset, 188 | struct fuse_file_info *fi) { 189 | if(*path == '/') ++path; // skip '/' at the beginning 190 | if(*path == '\0') return -EISDIR; // this is the root-dir, not a file 191 | 192 | std::string filename = path; 193 | filename = dirName(filename) + "/" + baseFilename(filename); 194 | 195 | DbEntryId fileEntryId; 196 | CHECK_RET(db->getFileRef(fileEntryId, filename), -ENOENT, "db_read: getFileRef failed"); 197 | 198 | // fileEntryId.empty() is allowed and means we have an empty file 199 | if(fileEntryId.empty()) return 0; 200 | 201 | static FileCache fileCache; 202 | return fileCache.read(fileEntryId, buf, size, offset); 203 | } 204 | 205 | int main(int argc, char **argv) { 206 | DbDefBackend dbInst; 207 | db = &dbInst; 208 | db->setReadOnly(true); 209 | Return r = db->init(); 210 | if(!r) { 211 | cerr << "error: failed to init DB: " << r.errmsg << endl; 212 | return 1; 213 | } 214 | 215 | struct fuse_operations ops; 216 | memset(&ops, 0, sizeof(ops)); 217 | ops.getattr = db_getattr; /* To provide size, permissions, etc. */ 218 | ops.open = db_open; /* To enforce read-only access. */ 219 | ops.read = db_read; /* To provide file content. */ 220 | ops.readdir = db_readdir; /* To provide directory listing. */ 221 | return fuse_main(argc, argv, &ops, NULL); 222 | } 223 | -------------------------------------------------------------------------------- /hiredis/hiredis.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2009-2010, Salvatore Sanfilippo 3 | * Copyright (c) 2010, Pieter Noordhuis 4 | * 5 | * All rights reserved. 6 | * 7 | * Redistribution and use in source and binary forms, with or without 8 | * modification, are permitted provided that the following conditions are met: 9 | * 10 | * * Redistributions of source code must retain the above copyright notice, 11 | * this list of conditions and the following disclaimer. 12 | * * Redistributions in binary form must reproduce the above copyright 13 | * notice, this list of conditions and the following disclaimer in the 14 | * documentation and/or other materials provided with the distribution. 15 | * * Neither the name of Redis nor the names of its contributors may be used 16 | * to endorse or promote products derived from this software without 17 | * specific prior written permission. 18 | * 19 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 | * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 | * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 23 | * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 | * POSSIBILITY OF SUCH DAMAGE. 30 | */ 31 | 32 | #ifndef __HIREDIS_H 33 | #define __HIREDIS_H 34 | #include /* for size_t */ 35 | #include /* for va_list */ 36 | 37 | #define HIREDIS_MAJOR 0 38 | #define HIREDIS_MINOR 9 39 | #define HIREDIS_PATCH 2 40 | 41 | #define REDIS_ERR -1 42 | #define REDIS_OK 0 43 | 44 | /* When an error occurs, the err flag in a context is set to hold the type of 45 | * error that occured. REDIS_ERR_IO means there was an I/O error and you 46 | * should use the "errno" variable to find out what is wrong. 47 | * For other values, the "errstr" field will hold a description. */ 48 | #define REDIS_ERR_IO 1 /* error in read or write */ 49 | #define REDIS_ERR_EOF 3 /* eof */ 50 | #define REDIS_ERR_PROTOCOL 4 /* protocol error */ 51 | #define REDIS_ERR_OTHER 2 /* something else */ 52 | 53 | /* Connection type can be blocking or non-blocking and is set in the 54 | * least significant bit of the flags field in redisContext. */ 55 | #define REDIS_BLOCK 0x1 56 | 57 | /* Connection may be disconnected before being free'd. The second bit 58 | * in the flags field is set when the context is connected. */ 59 | #define REDIS_CONNECTED 0x2 60 | 61 | /* The async API might try to disconnect cleanly and flush the output 62 | * buffer and read all subsequent replies before disconnecting. 63 | * This flag means no new commands can come in and the connection 64 | * should be terminated once all replies have been read. */ 65 | #define REDIS_DISCONNECTING 0x4 66 | 67 | #define REDIS_REPLY_STRING 1 68 | #define REDIS_REPLY_ARRAY 2 69 | #define REDIS_REPLY_INTEGER 3 70 | #define REDIS_REPLY_NIL 4 71 | #define REDIS_REPLY_STATUS 5 72 | #define REDIS_REPLY_ERROR 6 73 | 74 | #ifdef __cplusplus 75 | extern "C" { 76 | #endif 77 | 78 | /* This is the reply object returned by redisCommand() */ 79 | typedef struct redisReply { 80 | int type; /* REDIS_REPLY_* */ 81 | long long integer; /* The integer when type is REDIS_REPLY_INTEGER */ 82 | int len; /* Length of string */ 83 | char *str; /* Used for both REDIS_REPLY_ERROR and REDIS_REPLY_STRING */ 84 | size_t elements; /* number of elements, for REDIS_REPLY_ARRAY */ 85 | struct redisReply **element; /* elements vector for REDIS_REPLY_ARRAY */ 86 | } redisReply; 87 | 88 | typedef struct redisReadTask { 89 | int type; 90 | int elements; /* number of elements in multibulk container */ 91 | int idx; /* index in parent (array) object */ 92 | void *obj; /* holds user-generated value for a read task */ 93 | struct redisReadTask *parent; /* parent task */ 94 | void *privdata; /* user-settable arbitrary field */ 95 | } redisReadTask; 96 | 97 | typedef struct redisReplyObjectFunctions { 98 | void *(*createString)(const redisReadTask*, char*, size_t); 99 | void *(*createArray)(const redisReadTask*, int); 100 | void *(*createInteger)(const redisReadTask*, long long); 101 | void *(*createNil)(const redisReadTask*); 102 | void (*freeObject)(void*); 103 | } redisReplyObjectFunctions; 104 | 105 | struct redisContext; /* need forward declaration of redisContext */ 106 | 107 | /* Context for a connection to Redis */ 108 | typedef struct redisContext { 109 | int fd; 110 | int flags; 111 | char *obuf; /* Write buffer */ 112 | int err; /* Error flags, 0 when there is no error */ 113 | char *errstr; /* String representation of error when applicable */ 114 | 115 | /* Function set for reply buildup and reply reader */ 116 | redisReplyObjectFunctions *fn; 117 | void *reader; 118 | } redisContext; 119 | 120 | void freeReplyObject(void *reply); 121 | void *redisReplyReaderCreate(); 122 | int redisReplyReaderSetReplyObjectFunctions(void *reader, redisReplyObjectFunctions *fn); 123 | int redisReplyReaderSetPrivdata(void *reader, void *privdata); 124 | void *redisReplyReaderGetObject(void *reader); 125 | char *redisReplyReaderGetError(void *reader); 126 | void redisReplyReaderFree(void *ptr); 127 | void redisReplyReaderFeed(void *reader, char *buf, size_t len); 128 | int redisReplyReaderGetReply(void *reader, void **reply); 129 | 130 | /* Functions to format a command according to the protocol. */ 131 | int redisvFormatCommand(char **target, const char *format, va_list ap); 132 | int redisFormatCommand(char **target, const char *format, ...); 133 | int redisFormatCommandArgv(char **target, int argc, const char **argv, const size_t *argvlen); 134 | 135 | redisContext *redisConnect(const char *ip, int port); 136 | redisContext *redisConnectNonBlock(const char *ip, int port); 137 | redisContext *redisConnectUnix(const char *path); 138 | redisContext *redisConnectUnixNonBlock(const char *path); 139 | int redisSetReplyObjectFunctions(redisContext *c, redisReplyObjectFunctions *fn); 140 | void redisFree(redisContext *c); 141 | int redisBufferRead(redisContext *c); 142 | int redisBufferWrite(redisContext *c, int *done); 143 | 144 | /* In a blocking context, this function first checks if there are unconsumed 145 | * replies to return and returns one if so. Otherwise, it flushes the output 146 | * buffer to the socket and reads until it has a reply. In a non-blocking 147 | * context, it will return unconsumed replies until there are no more. */ 148 | int redisGetReply(redisContext *c, void **reply); 149 | int redisGetReplyFromReader(redisContext *c, void **reply); 150 | 151 | /* Write a command to the output buffer. Use these functions in blocking mode 152 | * to get a pipeline of commands. */ 153 | void redisvAppendCommand(redisContext *c, const char *format, va_list ap); 154 | void redisAppendCommand(redisContext *c, const char *format, ...); 155 | void redisAppendCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen); 156 | 157 | /* Issue a command to Redis. In a blocking context, it is identical to calling 158 | * redisAppendCommand, followed by redisGetReply. The function will return 159 | * NULL if there was an error in performing the request, otherwise it will 160 | * return the reply. In a non-blocking context, it is identical to calling 161 | * only redisAppendCommand and will always return NULL. */ 162 | void *redisvCommand(redisContext *c, const char *format, va_list ap); 163 | void *redisCommand(redisContext *c, const char *format, ...); 164 | void *redisCommandArgv(redisContext *c, int argc, const char **argv, const size_t *argvlen); 165 | 166 | #ifdef __cplusplus 167 | } 168 | #endif 169 | 170 | #endif 171 | -------------------------------------------------------------------------------- /Sha1.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | SHA-1 in C 3 | By Steve Reid 4 | 100% Public Domain 5 | 6 | ----------------- 7 | Modified 7/98 8 | By James H. Brown 9 | Still 100% Public Domain 10 | 11 | Corrected a problem which generated improper hash values on 16 bit machines 12 | Routine SHA1Update changed from 13 | void SHA1Update(SHA1_CTX* context, unsigned char* data, unsigned int 14 | len) 15 | to 16 | void SHA1Update(SHA1_CTX* context, unsigned char* data, unsigned 17 | long len) 18 | 19 | The 'len' parameter was declared an int which works fine on 32 bit machines. 20 | However, on 16 bit machines an int is too small for the shifts being done 21 | against 22 | it. This caused the hash function to generate incorrect values if len was 23 | greater than 8191 (8K - 1) due to the 'len << 3' on line 3 of SHA1Update(). 24 | 25 | Since the file IO in main() reads 16K at a time, any file 8K or larger would 26 | be guaranteed to generate the wrong hash (e.g. Test Vector #3, a million 27 | "a"s). 28 | 29 | I also changed the declaration of variables i & j in SHA1Update to 30 | unsigned long from unsigned int for the same reason. 31 | 32 | These changes should make no difference to any 32 bit implementations since 33 | an 34 | int and a long are the same size in those environments. 35 | 36 | -- 37 | I also corrected a few compiler warnings generated by Borland C. 38 | 1. Added #include for exit() prototype 39 | 2. Removed unused variable 'j' in SHA1Final 40 | 3. Changed exit(0) to return(0) at end of main. 41 | 42 | ALL changes I made can be located by searching for comments containing 'JHB' 43 | ----------------- 44 | Modified 8/98 45 | By Steve Reid 46 | Still 100% public domain 47 | 48 | 1- Removed #include and used return() instead of exit() 49 | 2- Fixed overwriting of finalcount in SHA1Final() (discovered by Chris Hall) 50 | 3- Changed email address from steve@edmweb.com to sreid@sea-to-sky.net 51 | 52 | ----------------- 53 | Modified 4/01 54 | By Saul Kravitz 55 | Still 100% PD 56 | Modified to run on Compaq Alpha hardware. 57 | 58 | ----------------- 59 | Modified 07/2002 60 | By Ralph Giles 61 | Still 100% public domain 62 | modified for use with stdint types, autoconf 63 | code cleanup, removed attribution comments 64 | switched SHA1Final() argument order for consistency 65 | use SHA1_ prefix for public api 66 | move public api to sha1.h 67 | */ 68 | 69 | /* 70 | Test Vectors (from FIPS PUB 180-1) 71 | "abc" 72 | A9993E36 4706816A BA3E2571 7850C26C 9CD0D89D 73 | "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq" 74 | 84983E44 1C3BD26E BAAE4AA1 F95129E5 E54670F1 75 | A million repetitions of "a" 76 | 34AA973C D4C4DAA4 F61EEB2B DBAD2731 6534016F 77 | */ 78 | 79 | #define SHA1HANDSOFF 80 | 81 | #include 82 | #include 83 | 84 | #include "Sha1.h" 85 | 86 | static void SHA1_Transform(uint32_t state[5], const uint8_t buffer[64]); 87 | 88 | #define rol(value, bits) (((value) << (bits)) | ((value) >> (32 - (bits)))) 89 | 90 | /* blk0() and blk() perform the initial expand. */ 91 | /* I got the idea of expanding during the round function from SSLeay */ 92 | /* FIXME: can we do this in an endian-proof way? */ 93 | #ifdef WORDS_BIGENDIAN 94 | #define blk0(i) block->l[i] 95 | #else 96 | #define blk0(i) (block->l[i] = (rol(block->l[i],24)&0xFF00FF00) \ 97 | |(rol(block->l[i],8)&0x00FF00FF)) 98 | #endif 99 | #define blk(i) (block->l[i&15] = rol(block->l[(i+13)&15]^block->l[(i+8)&15] \ 100 | ^block->l[(i+2)&15]^block->l[i&15],1)) 101 | 102 | /* (R0+R1), R2, R3, R4 are the different operations used in SHA1 */ 103 | #define R0(v,w,x,y,z,i) z+=((w&(x^y))^y)+blk0(i)+0x5A827999+rol(v,5);w=rol(w,30); 104 | #define R1(v,w,x,y,z,i) z+=((w&(x^y))^y)+blk(i)+0x5A827999+rol(v,5);w=rol(w,30); 105 | #define R2(v,w,x,y,z,i) z+=(w^x^y)+blk(i)+0x6ED9EBA1+rol(v,5);w=rol(w,30); 106 | #define R3(v,w,x,y,z,i) z+=(((w|x)&y)|(w&x))+blk(i)+0x8F1BBCDC+rol(v,5);w=rol(w,30); 107 | #define R4(v,w,x,y,z,i) z+=(w^x^y)+blk(i)+0xCA62C1D6+rol(v,5);w=rol(w,30); 108 | 109 | 110 | #ifdef VERBOSE /* SAK */ 111 | void SHAPrintContext(Sha1Context *context, char *msg){ 112 | printf("%s (%d,%d) %x %x %x %x %x\n", 113 | msg, 114 | context->count[0], context->count[1], 115 | context->state[0], 116 | context->state[1], 117 | context->state[2], 118 | context->state[3], 119 | context->state[4]); 120 | } 121 | #endif /* VERBOSE */ 122 | 123 | /* Hash a single 512-bit block. This is the core of the algorithm. */ 124 | static void SHA1_Transform(uint32_t state[5], const uint8_t buffer[64]) 125 | { 126 | uint32_t a, b, c, d, e; 127 | typedef union { 128 | uint8_t c[64]; 129 | uint32_t l[16]; 130 | } CHAR64LONG16; 131 | CHAR64LONG16* block; 132 | 133 | #ifdef SHA1HANDSOFF 134 | static uint8_t workspace[64]; 135 | block = (CHAR64LONG16*)workspace; 136 | memcpy(block, buffer, 64); 137 | #else 138 | block = (CHAR64LONG16*)buffer; 139 | #endif 140 | 141 | /* Copy context->state[] to working vars */ 142 | a = state[0]; 143 | b = state[1]; 144 | c = state[2]; 145 | d = state[3]; 146 | e = state[4]; 147 | 148 | /* 4 rounds of 20 operations each. Loop unrolled. */ 149 | R0(a,b,c,d,e, 0); R0(e,a,b,c,d, 1); R0(d,e,a,b,c, 2); R0(c,d,e,a,b, 3); 150 | R0(b,c,d,e,a, 4); R0(a,b,c,d,e, 5); R0(e,a,b,c,d, 6); R0(d,e,a,b,c, 7); 151 | R0(c,d,e,a,b, 8); R0(b,c,d,e,a, 9); R0(a,b,c,d,e,10); R0(e,a,b,c,d,11); 152 | R0(d,e,a,b,c,12); R0(c,d,e,a,b,13); R0(b,c,d,e,a,14); R0(a,b,c,d,e,15); 153 | R1(e,a,b,c,d,16); R1(d,e,a,b,c,17); R1(c,d,e,a,b,18); R1(b,c,d,e,a,19); 154 | R2(a,b,c,d,e,20); R2(e,a,b,c,d,21); R2(d,e,a,b,c,22); R2(c,d,e,a,b,23); 155 | R2(b,c,d,e,a,24); R2(a,b,c,d,e,25); R2(e,a,b,c,d,26); R2(d,e,a,b,c,27); 156 | R2(c,d,e,a,b,28); R2(b,c,d,e,a,29); R2(a,b,c,d,e,30); R2(e,a,b,c,d,31); 157 | R2(d,e,a,b,c,32); R2(c,d,e,a,b,33); R2(b,c,d,e,a,34); R2(a,b,c,d,e,35); 158 | R2(e,a,b,c,d,36); R2(d,e,a,b,c,37); R2(c,d,e,a,b,38); R2(b,c,d,e,a,39); 159 | R3(a,b,c,d,e,40); R3(e,a,b,c,d,41); R3(d,e,a,b,c,42); R3(c,d,e,a,b,43); 160 | R3(b,c,d,e,a,44); R3(a,b,c,d,e,45); R3(e,a,b,c,d,46); R3(d,e,a,b,c,47); 161 | R3(c,d,e,a,b,48); R3(b,c,d,e,a,49); R3(a,b,c,d,e,50); R3(e,a,b,c,d,51); 162 | R3(d,e,a,b,c,52); R3(c,d,e,a,b,53); R3(b,c,d,e,a,54); R3(a,b,c,d,e,55); 163 | R3(e,a,b,c,d,56); R3(d,e,a,b,c,57); R3(c,d,e,a,b,58); R3(b,c,d,e,a,59); 164 | R4(a,b,c,d,e,60); R4(e,a,b,c,d,61); R4(d,e,a,b,c,62); R4(c,d,e,a,b,63); 165 | R4(b,c,d,e,a,64); R4(a,b,c,d,e,65); R4(e,a,b,c,d,66); R4(d,e,a,b,c,67); 166 | R4(c,d,e,a,b,68); R4(b,c,d,e,a,69); R4(a,b,c,d,e,70); R4(e,a,b,c,d,71); 167 | R4(d,e,a,b,c,72); R4(c,d,e,a,b,73); R4(b,c,d,e,a,74); R4(a,b,c,d,e,75); 168 | R4(e,a,b,c,d,76); R4(d,e,a,b,c,77); R4(c,d,e,a,b,78); R4(b,c,d,e,a,79); 169 | 170 | /* Add the working vars back into context.state[] */ 171 | state[0] += a; 172 | state[1] += b; 173 | state[2] += c; 174 | state[3] += d; 175 | state[4] += e; 176 | 177 | /* Wipe variables */ 178 | a = b = c = d = e = 0; 179 | } 180 | 181 | 182 | /* SHA1Init - Initialize new context */ 183 | Sha1Context::Sha1Context() 184 | { 185 | /* SHA1 initialization constants */ 186 | state[0] = 0x67452301; 187 | state[1] = 0xEFCDAB89; 188 | state[2] = 0x98BADCFE; 189 | state[3] = 0x10325476; 190 | state[4] = 0xC3D2E1F0; 191 | count[0] = count[1] = 0; 192 | } 193 | 194 | 195 | /* Run your data through this. */ 196 | void Sha1Context::update(const char* d, size_t len) 197 | { 198 | Sha1Context* context = this; 199 | const uint8_t* data = (uint8_t*)d; 200 | size_t i, j; 201 | 202 | #ifdef VERBOSE 203 | SHAPrintContext(context, "before"); 204 | #endif 205 | 206 | j = (context->count[0] >> 3) & 63; 207 | if ((context->count[0] += len << 3) < (len << 3)) context->count[1]++; 208 | context->count[1] += (len >> 29); 209 | if ((j + len) > 63) { 210 | memcpy(&context->buffer[j], data, (i = 64-j)); 211 | SHA1_Transform(context->state, context->buffer); 212 | for ( ; i + 63 < len; i += 64) { 213 | SHA1_Transform(context->state, data + i); 214 | } 215 | j = 0; 216 | } 217 | else i = 0; 218 | memcpy(&context->buffer[j], &data[i], len - i); 219 | 220 | #ifdef VERBOSE 221 | SHAPrintContext(context, "after "); 222 | #endif 223 | } 224 | 225 | 226 | /* Add padding and return the message digest. */ 227 | std::string Sha1Context::final() 228 | { 229 | uint8_t digest[SHA1_DIGEST_SIZE]; 230 | Sha1Context* context = this; 231 | uint32_t i; 232 | uint8_t finalcount[8]; 233 | 234 | for (i = 0; i < 8; i++) { 235 | finalcount[i] = (unsigned char)((context->count[(i >= 4 ? 0 : 1)] 236 | >> ((3-(i & 3)) * 8) ) & 255); /* Endian independent */ 237 | } 238 | update("\200", 1); 239 | while ((context->count[0] & 504) != 448) { 240 | update("\0", 1); 241 | } 242 | update((char*)finalcount, 8); /* Should cause a SHA1_Transform() */ 243 | for (i = 0; i < SHA1_DIGEST_SIZE; i++) { 244 | digest[i] = (uint8_t) 245 | ((context->state[i>>2] >> ((3-(i & 3)) * 8) ) & 255); 246 | } 247 | 248 | /* Wipe variables */ 249 | i = 0; 250 | memset(context->buffer, 0, 64); 251 | memset(context->state, 0, 20); 252 | memset(context->count, 0, 8); 253 | memset(finalcount, 0, 8); /* SWR */ 254 | 255 | #ifdef SHA1HANDSOFF /* make SHA1Transform overwrite its own static vars */ 256 | SHA1_Transform(context->state, context->buffer); 257 | #endif 258 | 259 | return std::string((char*)digest, SHA1_DIGEST_SIZE); 260 | } 261 | 262 | /* 263 | // OpenSSL code 264 | 265 | #include 266 | 267 | std::string calc_sha1(const char* data, size_t size) { 268 | std::string s(SHA_DIGEST_LENGTH, 0); 269 | SHA1((const unsigned char*) data, size, (unsigned char*) &s[0]); 270 | return s; 271 | } 272 | */ 273 | 274 | std::string calc_sha1(const char* data, size_t size) { 275 | Sha1Context c; 276 | c.update(data, size); 277 | return c.final(); 278 | } 279 | -------------------------------------------------------------------------------- /kyotocabinet/kcutil.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Utility functions 3 | * Copyright (C) 2009-2011 FAL Labs 4 | * This file is part of Kyoto Cabinet. 5 | * This program is free software: you can redistribute it and/or modify it under the terms of 6 | * the GNU General Public License as published by the Free Software Foundation, either version 7 | * 3 of the License, or any later version. 8 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 9 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 10 | * See the GNU General Public License for more details. 11 | * You should have received a copy of the GNU General Public License along with this program. 12 | * If not, see . 13 | *************************************************************************************************/ 14 | 15 | 16 | #include "kcutil.h" 17 | #include "myconf.h" 18 | 19 | namespace kyotocabinet { // common namespace 20 | 21 | 22 | /** The package version. */ 23 | const char* const VERSION = _KC_VERSION; 24 | 25 | 26 | /** The library version. */ 27 | const int32_t LIBVER = _KC_LIBVER; 28 | 29 | 30 | /** The library revision. */ 31 | const int32_t LIBREV = _KC_LIBREV; 32 | 33 | 34 | /** The database format version. */ 35 | const int32_t FMTVER = _KC_FMTVER; 36 | 37 | 38 | /** The system name. */ 39 | const char* const SYSNAME = _KC_SYSNAME; 40 | 41 | 42 | /** The flag for big endian environments. */ 43 | const bool BIGEND = _KC_BIGEND ? true : false; 44 | 45 | 46 | /** The clock tick of interruption. */ 47 | #if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) 48 | const int32_t CLOCKTICK = 100; 49 | #else 50 | const int32_t CLOCKTICK = sysconf(_SC_CLK_TCK); 51 | #endif 52 | 53 | 54 | /** The size of a page. */ 55 | #if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) 56 | static int32_t win_getpagesize() { 57 | ::SYSTEM_INFO ibuf; 58 | ::GetSystemInfo(&ibuf); 59 | return ibuf.dwPageSize; 60 | } 61 | const int32_t PAGESIZE = win_getpagesize(); 62 | #else 63 | const int32_t PAGESIZE = sysconf(_SC_PAGESIZE); 64 | #endif 65 | 66 | 67 | /** The extra feature list. */ 68 | const char* const FEATURES = "" 69 | #if _KC_GCCATOMIC 70 | "(atomic)" 71 | #endif 72 | #if _KC_ZLIB 73 | "(zlib)" 74 | #endif 75 | #if _KC_LZO 76 | "(lzo)" 77 | #endif 78 | #if _KC_LZMA 79 | "(lzma)" 80 | #endif 81 | ; 82 | 83 | 84 | /** 85 | * Allocate a nullified region on memory. 86 | */ 87 | void* mapalloc(size_t size) { 88 | #if defined(_SYS_LINUX_) 89 | _assert_(size > 0 && size <= MEMMAXSIZ); 90 | void* ptr = ::mmap(0, sizeof(size) + size, 91 | PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); 92 | if (ptr == MAP_FAILED) throw std::bad_alloc(); 93 | *(size_t*)ptr = size; 94 | return (char*)ptr + sizeof(size); 95 | #else 96 | _assert_(size > 0 && size <= MEMMAXSIZ); 97 | void* ptr = std::calloc(size, 1); 98 | if (!ptr) throw std::bad_alloc(); 99 | return ptr; 100 | #endif 101 | } 102 | 103 | 104 | /** 105 | * Free a region on memory. 106 | */ 107 | void mapfree(void* ptr) { 108 | #if defined(_SYS_LINUX_) 109 | _assert_(ptr); 110 | size_t size = *((size_t*)ptr - 1); 111 | ::munmap((char*)ptr - sizeof(size), sizeof(size) + size); 112 | #else 113 | _assert_(ptr); 114 | std::free(ptr); 115 | #endif 116 | } 117 | 118 | 119 | /** 120 | * Get the time of day in seconds. 121 | * @return the time of day in seconds. The accuracy is in microseconds. 122 | */ 123 | double time() { 124 | #if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) 125 | _assert_(true); 126 | ::FILETIME ft; 127 | ::GetSystemTimeAsFileTime(&ft); 128 | ::LARGE_INTEGER li; 129 | li.LowPart = ft.dwLowDateTime; 130 | li.HighPart = ft.dwHighDateTime; 131 | return li.QuadPart / 10000000.0; 132 | #else 133 | _assert_(true); 134 | struct ::timeval tv; 135 | if (::gettimeofday(&tv, NULL) != 0) return 0.0; 136 | return tv.tv_sec + tv.tv_usec / 1000000.0; 137 | #endif 138 | } 139 | 140 | 141 | /** 142 | * Get the process ID. 143 | */ 144 | int64_t getpid() { 145 | #if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) 146 | _assert_(true); 147 | return ::GetCurrentProcessId(); 148 | #else 149 | _assert_(true); 150 | return ::getpid(); 151 | #endif 152 | } 153 | 154 | 155 | /** 156 | * Get the value of an environment variable. 157 | */ 158 | const char* getenv(const char* name) { 159 | _assert_(name); 160 | return ::getenv(name); 161 | } 162 | 163 | 164 | /** 165 | * Get system information of the environment. 166 | */ 167 | void getsysinfo(std::map* strmap) { 168 | #if defined(_SYS_LINUX_) 169 | _assert_(strmap); 170 | struct ::rusage rbuf; 171 | std::memset(&rbuf, 0, sizeof(rbuf)); 172 | if (::getrusage(RUSAGE_SELF, &rbuf) == 0) { 173 | (*strmap)["ru_utime"] = strprintf("%0.6f", 174 | rbuf.ru_utime.tv_sec + rbuf.ru_utime.tv_usec / 1000000.0); 175 | (*strmap)["ru_stime"] = strprintf("%0.6f", 176 | rbuf.ru_stime.tv_sec + rbuf.ru_stime.tv_usec / 1000000.0); 177 | if (rbuf.ru_maxrss > 0) { 178 | int64_t size = rbuf.ru_maxrss * 1024LL; 179 | (*strmap)["mem_peak"] = strprintf("%lld", (long long)size); 180 | (*strmap)["mem_size"] = strprintf("%lld", (long long)size); 181 | (*strmap)["mem_rss"] = strprintf("%lld", (long long)size); 182 | } 183 | } 184 | std::ifstream ifs; 185 | ifs.open("/proc/self/status", std::ios_base::in | std::ios_base::binary); 186 | if (ifs) { 187 | std::string line; 188 | while (getline(ifs, line)) { 189 | size_t idx = line.find(':'); 190 | if (idx != std::string::npos) { 191 | const std::string& name = line.substr(0, idx); 192 | idx++; 193 | while (idx < line.size() && line[idx] >= '\0' && line[idx] <= ' ') { 194 | idx++; 195 | } 196 | const std::string& value = line.substr(idx); 197 | if (name == "VmPeak") { 198 | int64_t size = atoix(value.c_str()); 199 | if (size > 0) (*strmap)["mem_peak"] = strprintf("%lld", (long long)size); 200 | } else if (name == "VmSize") { 201 | int64_t size = atoix(value.c_str()); 202 | if (size > 0) (*strmap)["mem_size"] = strprintf("%lld", (long long)size); 203 | } else if (name == "VmRSS") { 204 | int64_t size = atoix(value.c_str()); 205 | if (size > 0) (*strmap)["mem_rss"] = strprintf("%lld", (long long)size); 206 | } 207 | } 208 | } 209 | ifs.close(); 210 | } 211 | ifs.open("/proc/meminfo", std::ios_base::in | std::ios_base::binary); 212 | if (ifs) { 213 | std::string line; 214 | while (getline(ifs, line)) { 215 | size_t idx = line.find(':'); 216 | if (idx != std::string::npos) { 217 | const std::string& name = line.substr(0, idx); 218 | idx++; 219 | while (idx < line.size() && line[idx] >= '\0' && line[idx] <= ' ') { 220 | idx++; 221 | } 222 | const std::string& value = line.substr(idx); 223 | if (name == "MemTotal") { 224 | int64_t size = atoix(value.c_str()); 225 | if (size > 0) (*strmap)["mem_total"] = strprintf("%lld", (long long)size); 226 | } else if (name == "MemFree") { 227 | int64_t size = atoix(value.c_str()); 228 | if (size > 0) (*strmap)["mem_free"] = strprintf("%lld", (long long)size); 229 | } else if (name == "Cached") { 230 | int64_t size = atoix(value.c_str()); 231 | if (size > 0) (*strmap)["mem_cached"] = strprintf("%lld", (long long)size); 232 | } 233 | } 234 | } 235 | ifs.close(); 236 | } 237 | #elif defined(_SYS_MACOSX_) 238 | _assert_(strmap); 239 | struct ::rusage rbuf; 240 | std::memset(&rbuf, 0, sizeof(rbuf)); 241 | if (::getrusage(RUSAGE_SELF, &rbuf) == 0) { 242 | (*strmap)["ru_utime"] = strprintf("%0.6f", 243 | rbuf.ru_utime.tv_sec + rbuf.ru_utime.tv_usec / 1000000.0); 244 | (*strmap)["ru_stime"] = strprintf("%0.6f", 245 | rbuf.ru_stime.tv_sec + rbuf.ru_stime.tv_usec / 1000000.0); 246 | if (rbuf.ru_maxrss > 0) { 247 | int64_t size = rbuf.ru_maxrss; 248 | (*strmap)["mem_peak"] = strprintf("%lld", (long long)size); 249 | (*strmap)["mem_size"] = strprintf("%lld", (long long)size); 250 | (*strmap)["mem_rss"] = strprintf("%lld", (long long)size); 251 | } 252 | } 253 | #elif defined(_SYS_FREEBSD_) || defined(_SYS_SUNOS_) 254 | _assert_(strmap); 255 | struct ::rusage rbuf; 256 | std::memset(&rbuf, 0, sizeof(rbuf)); 257 | if (::getrusage(RUSAGE_SELF, &rbuf) == 0) { 258 | (*strmap)["ru_utime"] = strprintf("%0.6f", 259 | rbuf.ru_utime.tv_sec + rbuf.ru_utime.tv_usec / 1000000.0); 260 | (*strmap)["ru_stime"] = strprintf("%0.6f", 261 | rbuf.ru_stime.tv_sec + rbuf.ru_stime.tv_usec / 1000000.0); 262 | if (rbuf.ru_maxrss > 0) { 263 | int64_t size = rbuf.ru_maxrss * 1024LL; 264 | (*strmap)["mem_peak"] = strprintf("%lld", (long long)size); 265 | (*strmap)["mem_size"] = strprintf("%lld", (long long)size); 266 | (*strmap)["mem_rss"] = strprintf("%lld", (long long)size); 267 | } 268 | } 269 | #elif defined(_SYS_MSVC_) || defined(_SYS_MINGW_) 270 | _assert_(strmap); 271 | ::DWORD pid = ::GetCurrentProcessId(); 272 | ::HANDLE ph = ::OpenProcess(PROCESS_QUERY_INFORMATION, false, pid); 273 | if (ph) { 274 | ::FILETIME ct, et, kt, ut; 275 | if (::GetProcessTimes(ph, &ct, &et, &kt, &ut)) { 276 | ::LARGE_INTEGER li; 277 | li.LowPart = ut.dwLowDateTime; 278 | li.HighPart = ut.dwHighDateTime; 279 | (*strmap)["ru_utime"] = strprintf("%0.6f", li.QuadPart / 10000000.0); 280 | li.LowPart = kt.dwLowDateTime; 281 | li.HighPart = kt.dwHighDateTime; 282 | (*strmap)["ru_stime"] = strprintf("%0.6f", li.QuadPart / 10000000.0); 283 | } 284 | ::CloseHandle(ph); 285 | } 286 | ::MEMORYSTATUSEX msbuf; 287 | msbuf.dwLength = sizeof(msbuf); 288 | ::GlobalMemoryStatusEx(&msbuf); 289 | (*strmap)["mem_total"] = strprintf("%lld", (long long)msbuf.ullTotalPhys); 290 | (*strmap)["mem_free"] = strprintf("%lld", (long long)msbuf.ullAvailPhys); 291 | int64_t cached = msbuf.ullTotalPhys - msbuf.ullAvailPhys; 292 | (*strmap)["mem_cached"] = strprintf("%lld", (long long)cached); 293 | #else 294 | _assert_(strmap); 295 | #endif 296 | } 297 | 298 | 299 | /** 300 | * Set the standard streams into the binary mode. 301 | */ 302 | void setstdiobin() { 303 | #if defined(_SYS_MSVC_) || defined(_SYS_MINGW_) 304 | _assert_(true); 305 | _setmode(_fileno(stdin), O_BINARY); 306 | _setmode(_fileno(stdout), O_BINARY); 307 | _setmode(_fileno(stderr), O_BINARY); 308 | #else 309 | _assert_(true); 310 | #endif 311 | } 312 | 313 | 314 | } // common namespace 315 | 316 | // END OF FILE 317 | -------------------------------------------------------------------------------- /kyotocabinet/kyotocabinet.idl: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * IDL for bindings of scripting languages 3 | * Copyright (C) 2009-2011 FAL Labs 4 | * This file is part of Kyoto Cabinet. 5 | * This program is free software: you can redistribute it and/or modify it under the terms of 6 | * the GNU General Public License as published by the Free Software Foundation, either version 7 | * 3 of the License, or any later version. 8 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 9 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 10 | * See the GNU General Public License for more details. 11 | * You should have received a copy of the GNU General Public License along with this program. 12 | * If not, see . 13 | *************************************************************************************************/ 14 | 15 | 16 | /** 17 | * namespace of Kyoto Cabinet 18 | */ 19 | module kyotocabinet { 20 | //---------------------------------------------------------------- 21 | // prediction 22 | //---------------------------------------------------------------- 23 | interface List; 24 | interface Map; 25 | interface Error; 26 | interface Visitor; 27 | interface FileProcessor; 28 | interface Logger; 29 | interface Cursor; 30 | interface DB; 31 | //---------------------------------------------------------------- 32 | // list of strings (substituted by the native mechanism) 33 | //---------------------------------------------------------------- 34 | interface List { 35 | string get(in long index); 36 | }; 37 | //---------------------------------------------------------------- 38 | // map of strings (substituted by the native mechanism) 39 | //---------------------------------------------------------------- 40 | interface Map { 41 | string get(in string key); 42 | }; 43 | //---------------------------------------------------------------- 44 | // error information 45 | //---------------------------------------------------------------- 46 | interface Error { 47 | const long SUCCESS = 0; 48 | const long NOIMPL = 1; 49 | const long INVALID = 2; 50 | const long NOREPOS = 3; 51 | const long NOPERM = 4; 52 | const long BROKEN = 5; 53 | const long DUPREC = 6; 54 | const long NOREC = 7; 55 | const long LOGIC = 8; 56 | const long SYSTEM = 9; 57 | const long MISC = 15; 58 | long code(); 59 | string name(); 60 | string message(); 61 | }; 62 | //---------------------------------------------------------------- 63 | // record visitor 64 | //---------------------------------------------------------------- 65 | interface Visitor { 66 | const string NOP = ""; 67 | const string REMOVE = ""; 68 | string visit_full(in string key, in string value); 69 | string visit_empty(in string key); 70 | }; 71 | //---------------------------------------------------------------- 72 | // file processor 73 | //---------------------------------------------------------------- 74 | interface FileProcessor { 75 | boolean process(in string path, in long long count, in long long size); 76 | }; 77 | //---------------------------------------------------------------- 78 | // event logger 79 | //---------------------------------------------------------------- 80 | interface Logger { 81 | const long INFO = 0; 82 | const long WARN = 1; 83 | const long ERROR = 2; 84 | void log(in string file, in long line, in string func, in long kind, in string message); 85 | }; 86 | //---------------------------------------------------------------- 87 | // meta operation trigger 88 | //---------------------------------------------------------------- 89 | interface MetaTrigger { 90 | const long OPEN = 0; 91 | const long CLOSE = 1; 92 | const long CLEAR = 2; 93 | const long ITERATE = 3; 94 | const long SYNCHRONIZE = 4; 95 | const long OCCUPY = 5; 96 | const long BEGINTRAN = 6; 97 | const long COMMITTRAN = 7; 98 | const long ABORTTRAN = 8; 99 | const long MISC = 15; 100 | void trigger(in long kind, in string message); 101 | }; 102 | //---------------------------------------------------------------- 103 | // cursor 104 | //---------------------------------------------------------------- 105 | interface Cursor { 106 | boolean accept(inout Visitor visitor, in boolean writable, in boolean step); 107 | boolean set_value(in string value, in boolean step); 108 | boolean remove(); 109 | string get_key(in boolean step); 110 | string get_value(in boolean step); 111 | boolean jump(); 112 | boolean jump_(in string key); 113 | boolean jump_back(); 114 | boolean jump_back_(in string key); 115 | boolean step(); 116 | boolean step_back(); 117 | DB db(); 118 | Error error(); 119 | }; 120 | //---------------------------------------------------------------- 121 | // common database operations 122 | //---------------------------------------------------------------- 123 | interface DB { 124 | const long OREADER = 1 << 0; 125 | const long OWRITER = 1 << 1; 126 | const long OCREATE = 1 << 2; 127 | const long OTRUNCATE = 1 << 3; 128 | const long OAUTOTRAN = 1 << 4; 129 | const long OAUTOSYNC = 1 << 5; 130 | const long ONOLOCK = 1 << 6; 131 | const long OTRYLOCK = 1 << 7; 132 | const long ONOREPAIR = 1 << 8; 133 | Error error(); 134 | boolean open(in string path, in long mode); 135 | boolean close(); 136 | boolean accept(in string key, inout Visitor visitor, in boolean writable); 137 | boolean accept_bulk(in List keys, inout Visitor visitor, in boolean writable); 138 | boolean iterate(inout Visitor visitor, in boolean writable); 139 | boolean set(in string key, in string value); 140 | boolean add(in string key, in string value); 141 | boolean replace(in string key, in string value); 142 | boolean append(in string key, in string value); 143 | long long increment(in string key, in long long num); 144 | double increment_double(in string key, in double num); 145 | boolean cas(in string key, in string oval, in string nval); 146 | boolean remove(in string key); 147 | string get(in string key); 148 | long long set_bulk(in Map recs); 149 | long long remove_bulk(in List keys); 150 | Map get_bulk(in List keys); 151 | boolean clear(); 152 | boolean synchronize(in boolean hard, inout FileProcessor proc); 153 | boolean occupy(in boolean writable, inout FileProcessor proc); 154 | boolean copy(in string dest); 155 | boolean begin_transaction(in boolean hard); 156 | boolean end_transaction(in boolean commit); 157 | boolean dump_snapshot(in string dest); 158 | boolean load_snapshot(in string src); 159 | long long count(); 160 | long long size(); 161 | string path(); 162 | Map status(); 163 | Cursor cursor(); 164 | boolean tune_logger(inout Logger logger); 165 | boolean tune_meta_trigger(inout MetaTrigger trigger); 166 | }; 167 | //---------------------------------------------------------------- 168 | // prototype hash database 169 | //---------------------------------------------------------------- 170 | interface ProtoHashDB :DB { 171 | }; 172 | //---------------------------------------------------------------- 173 | // prototype tree database 174 | //---------------------------------------------------------------- 175 | interface ProtoTreeDB :DB { 176 | }; 177 | //---------------------------------------------------------------- 178 | // stash database 179 | //---------------------------------------------------------------- 180 | interface StashDB :DB { 181 | boolean tune_buckets(in long long bnum); 182 | }; 183 | //---------------------------------------------------------------- 184 | // cache hash database 185 | //---------------------------------------------------------------- 186 | interface CacheDB :DB { 187 | boolean tune_options(in long opts); 188 | boolean tune_buckets(in long long bnum); 189 | boolean cap_count(in long long count); 190 | boolean cap_size(in long long size); 191 | }; 192 | //---------------------------------------------------------------- 193 | // cache tree database 194 | //---------------------------------------------------------------- 195 | interface GrassDB :DB { 196 | boolean tune_options(in long opts); 197 | boolean tune_buckets(in long long bnum); 198 | boolean tune_page(in long psiz); 199 | boolean tune_page_cache(in long long pccap); 200 | }; 201 | //---------------------------------------------------------------- 202 | // file hash database 203 | //---------------------------------------------------------------- 204 | interface HashDB :DB { 205 | const long TSMALL = 1 << 0; 206 | const long TLINEAR = 1 << 1; 207 | const long TCOMPRESS = 1 << 2; 208 | boolean tune_alignment(in long apow); 209 | boolean tune_fbp(in long fpow); 210 | boolean tune_options(in long opts); 211 | boolean tune_buckets(in long long bnum); 212 | boolean tune_map(in long long msiz); 213 | boolean tune_defrag(in long dfunit); 214 | }; 215 | //---------------------------------------------------------------- 216 | // file tree database 217 | //---------------------------------------------------------------- 218 | interface TreeDB :DB { 219 | const long TSMALL = 1 << 0; 220 | const long TLINEAR = 1 << 1; 221 | const long TCOMPRESS = 1 << 2; 222 | boolean tune_alignment(in long apow); 223 | boolean tune_fbp(in long fpow); 224 | boolean tune_options(in long opts); 225 | boolean tune_buckets(in long long bnum); 226 | boolean tune_page(in long psiz); 227 | boolean tune_map(in long long msiz); 228 | boolean tune_defrag(in long dfunit); 229 | boolean tune_page_cache(in long long pccap); 230 | }; 231 | //---------------------------------------------------------------- 232 | // directory hash database 233 | //---------------------------------------------------------------- 234 | interface DirDB :DB { 235 | const long TCOMPRESS = 1 << 2; 236 | boolean tune_options(in long opts); 237 | }; 238 | //---------------------------------------------------------------- 239 | // directory tree database 240 | //---------------------------------------------------------------- 241 | interface ForestDB :DB { 242 | const long TCOMPRESS = 1 << 2; 243 | boolean tune_options(in long opts); 244 | boolean tune_buckets(in long long bnum); 245 | boolean tune_page(in long psiz); 246 | boolean tune_page_cache(in long long pccap); 247 | }; 248 | //---------------------------------------------------------------- 249 | // polymorphic database 250 | //---------------------------------------------------------------- 251 | interface PolyDB :DB { 252 | List match_prefix(in string prefix, in long long max); 253 | List match_regex(in string regex, in long long max); 254 | }; 255 | }; 256 | 257 | 258 | 259 | /* END OF FILE */ 260 | -------------------------------------------------------------------------------- /kyotocabinet/cmdcommon.h: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Common symbols for command line utilities 3 | * Copyright (C) 2009-2011 FAL Labs 4 | * This file is part of Kyoto Cabinet. 5 | * This program is free software: you can redistribute it and/or modify it under the terms of 6 | * the GNU General Public License as published by the Free Software Foundation, either version 7 | * 3 of the License, or any later version. 8 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 9 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 10 | * See the GNU General Public License for more details. 11 | * You should have received a copy of the GNU General Public License along with this program. 12 | * If not, see . 13 | *************************************************************************************************/ 14 | 15 | 16 | #ifndef _CMDCOMMON_H // duplication check 17 | #define _CMDCOMMON_H 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | 30 | #if !defined(_KC_PREFIX) 31 | #define _KC_PREFIX "*" 32 | #endif 33 | #if !defined(_KC_INCLUDEDIR) 34 | #define _KC_INCLUDEDIR "*" 35 | #endif 36 | #if !defined(_KC_LIBDIR) 37 | #define _KC_LIBDIR "*" 38 | #endif 39 | #if !defined(_KC_BINDIR) 40 | #define _KC_BINDIR "*" 41 | #endif 42 | #if !defined(_KC_LIBEXECDIR) 43 | #define _KC_LIBEXECDIR "*" 44 | #endif 45 | #if !defined(_KC_APPINC) 46 | #define _KC_APPINC "*" 47 | #endif 48 | #if !defined(_KC_APPLIBS) 49 | #define _KC_APPLIBS "*" 50 | #endif 51 | 52 | namespace kc = kyotocabinet; 53 | 54 | 55 | // constants 56 | const int32_t THREADMAX = 64; // maximum number of threads 57 | const size_t RECBUFSIZ = 64; // buffer size for a record 58 | const size_t RECBUFSIZL = 1024; // buffer size for a long record 59 | 60 | 61 | // global variables 62 | uint64_t g_rnd_x = 123456789; 63 | uint64_t g_rnd_y = 362436069; 64 | uint64_t g_rnd_z = 521288629; 65 | uint64_t g_rnd_w = 88675123; 66 | 67 | 68 | // function prototypes 69 | void mysrand(int64_t seed); 70 | int64_t myrand(int64_t range); 71 | int64_t memusage(); 72 | void oprintf(const char* format, ...); 73 | void oputchar(char c); 74 | void eprintf(const char* format, ...); 75 | void printversion(); 76 | void printdata(const char* buf, int32_t size, bool px); 77 | bool mygetline(std::istream* is, std::string* str); 78 | std::string unitnumstr(int64_t num); 79 | std::string unitnumstrbyte(int64_t num); 80 | kc::BasicDB::ProgressChecker* stdchecker(const char* prefix, std::ostream* strm); 81 | kc::BasicDB::Logger* stdlogger(const char* progname, std::ostream* strm); 82 | void printdb(kc::BasicDB* db, bool px = false); 83 | 84 | 85 | // checker to show progress by printing dots 86 | class DotChecker : public kc::BasicDB::ProgressChecker { 87 | public: 88 | explicit DotChecker(std::ostream* strm, int64_t freq) : strm_(strm), freq_(freq), cnt_(0) {} 89 | int64_t count() { 90 | return cnt_; 91 | } 92 | private: 93 | bool check(const char* name, const char* message, int64_t curcnt, int64_t allcnt) { 94 | if (std::strcmp(message, "processing") || freq_ == 0) return true; 95 | if (freq_ < 0) { 96 | cnt_++; 97 | if (cnt_ % -freq_ == 0) { 98 | oputchar('.'); 99 | if (cnt_ % (-freq_ * 50) == 0) oprintf(" (%lld)\n", (long long)cnt_); 100 | } 101 | } else { 102 | if (curcnt > cnt_) { 103 | cnt_ = curcnt; 104 | if (cnt_ % freq_ == 0) { 105 | oputchar('.'); 106 | if (cnt_ % (freq_ * 50) == 0) oprintf(" (%lld)\n", (long long)cnt_); 107 | } 108 | } 109 | } 110 | return true; 111 | } 112 | std::ostream* strm_; 113 | int64_t freq_; 114 | int64_t cnt_; 115 | }; 116 | 117 | 118 | // get the random seed 119 | inline void mysrand(int64_t seed) { 120 | g_rnd_x = seed; 121 | for (int32_t i = 0; i < 16; i++) { 122 | myrand(1); 123 | } 124 | } 125 | 126 | 127 | // get a random number 128 | inline int64_t myrand(int64_t range) { 129 | uint64_t t = g_rnd_x ^ (g_rnd_x << 11); 130 | g_rnd_x = g_rnd_y; 131 | g_rnd_y = g_rnd_z; 132 | g_rnd_z = g_rnd_w; 133 | g_rnd_w = (g_rnd_w ^ (g_rnd_w >> 19)) ^ (t ^ (t >> 8)); 134 | return (g_rnd_w & kc::INT64MAX) % range; 135 | } 136 | 137 | 138 | // get the current memory usage 139 | inline int64_t memusage() { 140 | std::map info; 141 | kc::getsysinfo(&info); 142 | return kc::atoi(info["mem_rss"].c_str()); 143 | } 144 | 145 | 146 | // print formatted information string and flush the buffer 147 | inline void oprintf(const char* format, ...) { 148 | std::string msg; 149 | va_list ap; 150 | va_start(ap, format); 151 | kc::vstrprintf(&msg, format, ap); 152 | va_end(ap); 153 | std::cout << msg; 154 | std::cout.flush(); 155 | } 156 | 157 | 158 | // print a character and flush the buffer 159 | inline void oputchar(char c) { 160 | std::cout << c; 161 | std::cout.flush(); 162 | } 163 | 164 | 165 | // print formatted error string and flush the buffer 166 | inline void eprintf(const char* format, ...) { 167 | std::string msg; 168 | va_list ap; 169 | va_start(ap, format); 170 | kc::vstrprintf(&msg, format, ap); 171 | va_end(ap); 172 | std::cerr << msg; 173 | std::cerr.flush(); 174 | } 175 | 176 | 177 | // print the versin information 178 | inline void printversion() { 179 | oprintf("Kyoto Cabinet %s (%d.%d:%d) on %s\n", 180 | kc::VERSION, kc::LIBVER, kc::LIBREV, kc::FMTVER, kc::SYSNAME); 181 | } 182 | 183 | 184 | // print record data 185 | inline void printdata(const char* buf, int32_t size, bool px) { 186 | size_t cnt = 0; 187 | char numbuf[kc::NUMBUFSIZ]; 188 | while (size-- > 0) { 189 | if (px) { 190 | if (cnt++ > 0) putchar(' '); 191 | std::sprintf(numbuf, "%02X", *(unsigned char*)buf); 192 | std::cout << numbuf; 193 | } else { 194 | std::cout << *buf; 195 | } 196 | buf++; 197 | } 198 | } 199 | 200 | 201 | // read a line from a file descriptor 202 | inline bool mygetline(std::istream* is, std::string* str) { 203 | str->clear(); 204 | bool hit = false; 205 | char c; 206 | while (is->get(c)) { 207 | hit = true; 208 | if (c == '\0' || c == '\r') continue; 209 | if (c == '\n') break; 210 | str->append(1, c); 211 | } 212 | return hit; 213 | } 214 | 215 | 216 | // convert a number into the string with the decimal unit 217 | inline std::string unitnumstr(int64_t num) { 218 | if (num >= std::pow(1000.0, 6)) { 219 | return kc::strprintf("%.3Lf quintillion", (long double)num / std::pow(1000.0, 6)); 220 | } else if (num >= std::pow(1000.0, 5)) { 221 | return kc::strprintf("%.3Lf quadrillion", (long double)num / std::pow(1000.0, 5)); 222 | } else if (num >= std::pow(1000.0, 4)) { 223 | return kc::strprintf("%.3Lf trillion", (long double)num / std::pow(1000.0, 4)); 224 | } else if (num >= std::pow(1000.0, 3)) { 225 | return kc::strprintf("%.3Lf billion", (long double)num / std::pow(1000.0, 3)); 226 | } else if (num >= std::pow(1000.0, 2)) { 227 | return kc::strprintf("%.3Lf million", (long double)num / std::pow(1000.0, 2)); 228 | } else if (num >= std::pow(1000.0, 1)) { 229 | return kc::strprintf("%.3Lf thousand", (long double)num / std::pow(1000.0, 1)); 230 | } 231 | return kc::strprintf("%lld", (long long)num); 232 | } 233 | 234 | 235 | // convert a number into the string with the byte unit 236 | inline std::string unitnumstrbyte(int64_t num) { 237 | if ((unsigned long long)num >= 1ULL << 60) { 238 | return kc::strprintf("%.3Lf EiB", (long double)num / (1ULL << 60)); 239 | } else if ((unsigned long long)num >= 1ULL << 50) { 240 | return kc::strprintf("%.3Lf PiB", (long double)num / (1ULL << 50)); 241 | } else if ((unsigned long long)num >= 1ULL << 40) { 242 | return kc::strprintf("%.3Lf TiB", (long double)num / (1ULL << 40)); 243 | } else if ((unsigned long long)num >= 1ULL << 30) { 244 | return kc::strprintf("%.3Lf GiB", (long double)num / (1ULL << 30)); 245 | } else if ((unsigned long long)num >= 1ULL << 20) { 246 | return kc::strprintf("%.3Lf MiB", (long double)num / (1ULL << 20)); 247 | } else if ((unsigned long long)num >= 1ULL << 10) { 248 | return kc::strprintf("%.3Lf KiB", (long double)num / (1ULL << 10)); 249 | } 250 | return kc::strprintf("%lld B", (long long)num); 251 | } 252 | 253 | 254 | // get the progress checker to print the parameters 255 | inline kc::BasicDB::ProgressChecker* stdchecker(const char* prefix, std::ostream* strm) { 256 | class CheckerImpl : public kc::BasicDB::ProgressChecker { 257 | public: 258 | explicit CheckerImpl(std::ostream* strm, const char* prefix) : 259 | strm_(strm), prefix_(prefix) {} 260 | bool check(const char* name, const char* message, int64_t curcnt, int64_t allcnt) { 261 | *strm_ << prefix_ << ": " << name << ": " << message << ": " << 262 | curcnt << "/" << allcnt << std::endl; 263 | return true; 264 | } 265 | private: 266 | std::ostream* strm_; 267 | const char* prefix_; 268 | }; 269 | static CheckerImpl checker(strm, prefix); 270 | return &checker; 271 | } 272 | 273 | 274 | // get the logger into the standard stream 275 | inline kc::BasicDB::Logger* stdlogger(const char* prefix, std::ostream* strm) { 276 | class LoggerImpl : public kc::BasicDB::Logger { 277 | public: 278 | explicit LoggerImpl(std::ostream* strm, const char* prefix) : 279 | strm_(strm), prefix_(prefix) {} 280 | void log(const char* file, int32_t line, const char* func, Kind kind, 281 | const char* message) { 282 | const char* kstr = "MISC"; 283 | switch (kind) { 284 | case kc::BasicDB::Logger::DEBUG: kstr = "DEBUG"; break; 285 | case kc::BasicDB::Logger::INFO: kstr = "INFO"; break; 286 | case kc::BasicDB::Logger::WARN: kstr = "WARN"; break; 287 | case kc::BasicDB::Logger::ERROR: kstr = "ERROR"; break; 288 | } 289 | *strm_ << prefix_ << ": [" << kstr << "]: " << 290 | file << ": " << line << ": " << func << ": " << message << std::endl; 291 | } 292 | private: 293 | std::ostream* strm_; 294 | const char* prefix_; 295 | }; 296 | static LoggerImpl logger(strm, prefix); 297 | return &logger; 298 | } 299 | 300 | 301 | // print all record of a database 302 | inline void printdb(kc::BasicDB* db, bool px) { 303 | class Printer : public kc::DB::Visitor { 304 | public: 305 | explicit Printer(bool px) : px_(px) {} 306 | private: 307 | const char* visit_full(const char* kbuf, size_t ksiz, 308 | const char* vbuf, size_t vsiz, size_t* sp) { 309 | printdata(kbuf, ksiz, px_); 310 | oputchar('\t'); 311 | printdata(vbuf, vsiz, px_); 312 | oputchar('\n'); 313 | return NOP; 314 | } 315 | bool px_; 316 | } printer(px); 317 | db->iterate(&printer, false); 318 | } 319 | 320 | 321 | #endif // duplication check 322 | 323 | // END OF FILE 324 | -------------------------------------------------------------------------------- /kyotocabinet/kccompress.cc: -------------------------------------------------------------------------------- 1 | /************************************************************************************************* 2 | * Data compressor and decompressor 3 | * Copyright (C) 2009-2011 FAL Labs 4 | * This file is part of Kyoto Cabinet. 5 | * This program is free software: you can redistribute it and/or modify it under the terms of 6 | * the GNU General Public License as published by the Free Software Foundation, either version 7 | * 3 of the License, or any later version. 8 | * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 9 | * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 10 | * See the GNU General Public License for more details. 11 | * You should have received a copy of the GNU General Public License along with this program. 12 | * If not, see . 13 | *************************************************************************************************/ 14 | 15 | 16 | #include "kccompress.h" 17 | #include "myconf.h" 18 | 19 | #if _KC_ZLIB 20 | extern "C" { 21 | #include 22 | } 23 | #endif 24 | 25 | #if _KC_LZO 26 | extern "C" { 27 | #include 28 | } 29 | #endif 30 | 31 | #if _KC_LZMA 32 | extern "C" { 33 | #include 34 | } 35 | #endif 36 | 37 | namespace kyotocabinet { // common namespace 38 | 39 | 40 | /** 41 | * Compress a serial data. 42 | */ 43 | char* ZLIB::compress(const void* buf, size_t size, size_t* sp, Mode mode) { 44 | #if _KC_ZLIB 45 | _assert_(buf && size <= MEMMAXSIZ && sp); 46 | z_stream zs; 47 | zs.zalloc = Z_NULL; 48 | zs.zfree = Z_NULL; 49 | zs.opaque = Z_NULL; 50 | switch (mode) { 51 | default: { 52 | if (deflateInit2(&zs, 6, Z_DEFLATED, -15, 9, Z_DEFAULT_STRATEGY) != Z_OK) return NULL; 53 | break; 54 | } 55 | case DEFLATE: { 56 | if (deflateInit2(&zs, 6, Z_DEFLATED, 15, 9, Z_DEFAULT_STRATEGY) != Z_OK) return NULL; 57 | break; 58 | } 59 | case GZIP: { 60 | if (deflateInit2(&zs, 6, Z_DEFLATED, 15 + 16, 9, Z_DEFAULT_STRATEGY) != Z_OK) return NULL; 61 | break; 62 | } 63 | } 64 | const char* rp = (const char*)buf; 65 | size_t zsiz = size + size / 8 + 32; 66 | char* zbuf = new char[zsiz+1]; 67 | char* wp = zbuf; 68 | zs.next_in = (Bytef*)rp; 69 | zs.avail_in = size; 70 | zs.next_out = (Bytef*)wp; 71 | zs.avail_out = zsiz; 72 | if (deflate(&zs, Z_FINISH) != Z_STREAM_END) { 73 | delete[] zbuf; 74 | deflateEnd(&zs); 75 | return NULL; 76 | } 77 | deflateEnd(&zs); 78 | zsiz -= zs.avail_out; 79 | zbuf[zsiz] = '\0'; 80 | if (mode == RAW) zsiz++; 81 | *sp = zsiz; 82 | return zbuf; 83 | #else 84 | _assert_(buf && size <= MEMMAXSIZ && sp); 85 | char* zbuf = new char[size+2]; 86 | char* wp = zbuf; 87 | *(wp++) = 'z'; 88 | *(wp++) = (uint8_t)mode; 89 | std::memcpy(wp, buf, size); 90 | *sp = size + 2; 91 | return zbuf; 92 | #endif 93 | } 94 | 95 | 96 | /** 97 | * Decompress a serial data. 98 | */ 99 | char* ZLIB::decompress(const void* buf, size_t size, size_t* sp, Mode mode) { 100 | #if _KC_ZLIB 101 | _assert_(buf && size <= MEMMAXSIZ && sp); 102 | size_t zsiz = size * 8 + 32; 103 | while (true) { 104 | z_stream zs; 105 | zs.zalloc = Z_NULL; 106 | zs.zfree = Z_NULL; 107 | zs.opaque = Z_NULL; 108 | switch (mode) { 109 | default: { 110 | if (inflateInit2(&zs, -15) != Z_OK) return NULL; 111 | break; 112 | } 113 | case DEFLATE: { 114 | if (inflateInit2(&zs, 15) != Z_OK) return NULL; 115 | break; 116 | } 117 | case GZIP: { 118 | if (inflateInit2(&zs, 15 + 16) != Z_OK) return NULL; 119 | break; 120 | } 121 | } 122 | char* zbuf = new char[zsiz+1]; 123 | zs.next_in = (Bytef*)buf; 124 | zs.avail_in = size; 125 | zs.next_out = (Bytef*)zbuf; 126 | zs.avail_out = zsiz; 127 | int32_t rv = inflate(&zs, Z_FINISH); 128 | inflateEnd(&zs); 129 | if (rv == Z_STREAM_END) { 130 | zsiz -= zs.avail_out; 131 | zbuf[zsiz] = '\0'; 132 | *sp = zsiz; 133 | return zbuf; 134 | } else if (rv == Z_BUF_ERROR) { 135 | delete[] zbuf; 136 | zsiz *= 2; 137 | } else { 138 | delete[] zbuf; 139 | break; 140 | } 141 | } 142 | return NULL; 143 | #else 144 | _assert_(buf && size <= MEMMAXSIZ && sp); 145 | if (size < 2 || ((char*)buf)[0] != 'z' || ((char*)buf)[1] != (uint8_t)mode) return NULL; 146 | buf = (char*)buf + 2; 147 | size -= 2; 148 | char* zbuf = new char[size+1]; 149 | std::memcpy(zbuf, buf, size); 150 | zbuf[size] = '\0'; 151 | *sp = size; 152 | return zbuf; 153 | #endif 154 | } 155 | 156 | 157 | /** 158 | * Calculate the CRC32 checksum of a serial data. 159 | */ 160 | uint32_t ZLIB::calculate_crc(const void* buf, size_t size, uint32_t seed) { 161 | #if _KC_ZLIB 162 | _assert_(buf && size <= MEMMAXSIZ); 163 | return crc32(seed, (unsigned char*)buf, size); 164 | #else 165 | _assert_(buf && size <= MEMMAXSIZ); 166 | return 0; 167 | #endif 168 | } 169 | 170 | 171 | /** 172 | * Hidden resources for LZO. 173 | */ 174 | #if _KC_LZO 175 | int32_t lzo_init_func() { 176 | if (lzo_init() != LZO_E_OK) throw std::runtime_error("lzo_init"); 177 | return 0; 178 | } 179 | int32_t lzo_init_var = lzo_init_func(); 180 | #endif 181 | 182 | 183 | /** 184 | * Compress a serial data. 185 | */ 186 | char* LZO::compress(const void* buf, size_t size, size_t* sp, Mode mode) { 187 | #if _KC_LZO 188 | _assert_(buf && size <= MEMMAXSIZ && sp); 189 | char* zbuf = new char[size+size/16+80]; 190 | lzo_uint zsiz; 191 | char wrkmem[LZO1X_1_MEM_COMPRESS]; 192 | if (lzo1x_1_compress((lzo_bytep)buf, size, (lzo_bytep)zbuf, &zsiz, wrkmem) != LZO_E_OK) { 193 | delete[] zbuf; 194 | return NULL; 195 | } 196 | if (mode == CRC) { 197 | uint32_t hash = lzo_crc32(0, (const lzo_bytep)zbuf, zsiz); 198 | writefixnum(zbuf + zsiz, hash, sizeof(hash)); 199 | zsiz += sizeof(hash); 200 | } 201 | zbuf[zsiz] = '\0'; 202 | *sp = zsiz; 203 | return (char*)zbuf; 204 | #else 205 | _assert_(buf && size <= MEMMAXSIZ && sp); 206 | char* zbuf = new char[size+2]; 207 | char* wp = zbuf; 208 | *(wp++) = 'o'; 209 | *(wp++) = mode; 210 | std::memcpy(wp, buf, size); 211 | *sp = size + 2; 212 | return zbuf; 213 | #endif 214 | } 215 | 216 | 217 | /** 218 | * Decompress a serial data. 219 | */ 220 | char* LZO::decompress(const void* buf, size_t size, size_t* sp, Mode mode) { 221 | #if _KC_LZO 222 | _assert_(buf && size <= MEMMAXSIZ && sp); 223 | if (mode == CRC) { 224 | if (size < sizeof(uint32_t)) return NULL; 225 | uint32_t hash = readfixnum((const char*)buf + size - sizeof(hash), sizeof(hash)); 226 | size -= sizeof(hash); 227 | if (lzo_crc32(0, (const lzo_bytep)buf, size) != hash) return NULL; 228 | } 229 | char* zbuf; 230 | lzo_uint zsiz; 231 | int32_t rat = 6; 232 | while (true) { 233 | zsiz = (size + 256) * rat + 3; 234 | zbuf = new char[zsiz+1]; 235 | int32_t rv; 236 | if (mode == RAW) { 237 | rv = lzo1x_decompress_safe((lzo_bytep)buf, size, (lzo_bytep)zbuf, &zsiz, NULL); 238 | } else { 239 | rv = lzo1x_decompress((lzo_bytep)buf, size, (lzo_bytep)zbuf, &zsiz, NULL); 240 | } 241 | if (rv == LZO_E_OK) { 242 | break; 243 | } else if (rv == LZO_E_OUTPUT_OVERRUN) { 244 | delete[] zbuf; 245 | rat *= 2; 246 | } else { 247 | delete[] zbuf; 248 | return NULL; 249 | } 250 | } 251 | zbuf[zsiz] = '\0'; 252 | if (sp) *sp = zsiz; 253 | return (char*)zbuf; 254 | #else 255 | _assert_(buf && size <= MEMMAXSIZ && sp); 256 | if (size < 2 || ((char*)buf)[0] != 'o' || ((char*)buf)[1] != mode) return NULL; 257 | buf = (char*)buf + 2; 258 | size -= 2; 259 | char* zbuf = new char[size+1]; 260 | std::memcpy(zbuf, buf, size); 261 | zbuf[size] = '\0'; 262 | *sp = size; 263 | return zbuf; 264 | #endif 265 | } 266 | 267 | 268 | /** 269 | * Calculate the CRC32 checksum of a serial data. 270 | */ 271 | uint32_t LZO::calculate_crc(const void* buf, size_t size, uint32_t seed) { 272 | #if _KC_LZO 273 | _assert_(buf && size <= MEMMAXSIZ); 274 | return lzo_crc32(seed, (const lzo_bytep)buf, size); 275 | #else 276 | _assert_(buf && size <= MEMMAXSIZ); 277 | return 0; 278 | #endif 279 | } 280 | 281 | 282 | /** 283 | * Compress a serial data. 284 | */ 285 | char* LZMA::compress(const void* buf, size_t size, size_t* sp, Mode mode) { 286 | #if _KC_LZMA 287 | _assert_(buf && size <= MEMMAXSIZ && sp); 288 | lzma_stream zs = LZMA_STREAM_INIT; 289 | const char* rp = (const char*)buf; 290 | size_t zsiz = size + 1024; 291 | char* zbuf = new char[zsiz+1]; 292 | char* wp = zbuf; 293 | zs.next_in = (const uint8_t*)rp; 294 | zs.avail_in = size; 295 | zs.next_out = (uint8_t*)wp; 296 | zs.avail_out = zsiz; 297 | switch (mode) { 298 | default: { 299 | if (lzma_easy_encoder(&zs, 6, LZMA_CHECK_NONE) != LZMA_OK) return NULL; 300 | break; 301 | } 302 | case CRC: { 303 | if (lzma_easy_encoder(&zs, 6, LZMA_CHECK_CRC32) != LZMA_OK) return NULL; 304 | break; 305 | } 306 | case SHA: { 307 | if (lzma_easy_encoder(&zs, 6, LZMA_CHECK_SHA256) != LZMA_OK) return NULL; 308 | break; 309 | } 310 | } 311 | if (lzma_code(&zs, LZMA_FINISH) != LZMA_STREAM_END) { 312 | delete[] zbuf; 313 | lzma_end(&zs); 314 | return NULL; 315 | } 316 | lzma_end(&zs); 317 | zsiz -= zs.avail_out; 318 | *sp = zsiz; 319 | return zbuf; 320 | #else 321 | _assert_(buf && size <= MEMMAXSIZ && sp); 322 | char* zbuf = new char[size+2]; 323 | char* wp = zbuf; 324 | *(wp++) = 'x'; 325 | *(wp++) = mode; 326 | std::memcpy(wp, buf, size); 327 | *sp = size + 2; 328 | return zbuf; 329 | #endif 330 | } 331 | 332 | 333 | /** 334 | * Decompress a serial data. 335 | */ 336 | char* LZMA::decompress(const void* buf, size_t size, size_t* sp, Mode mode) { 337 | #if _KC_LZMA 338 | _assert_(buf && size <= MEMMAXSIZ && sp); 339 | size_t zsiz = size * 8 + 32; 340 | while (true) { 341 | lzma_stream zs = LZMA_STREAM_INIT; 342 | const char* rp = (const char*)buf; 343 | char* zbuf = new char[zsiz+1]; 344 | char* wp = zbuf; 345 | zs.next_in = (const uint8_t*)rp; 346 | zs.avail_in = size; 347 | zs.next_out = (uint8_t*)wp; 348 | zs.avail_out = zsiz; 349 | if (lzma_auto_decoder(&zs, 1ULL << 30, 0) != LZMA_OK) return NULL; 350 | int32_t rv = lzma_code(&zs, LZMA_FINISH); 351 | lzma_end(&zs); 352 | if (rv == LZMA_STREAM_END) { 353 | zsiz -= zs.avail_out; 354 | zbuf[zsiz] = '\0'; 355 | *sp = zsiz; 356 | return zbuf; 357 | } else if (rv == LZMA_OK) { 358 | delete[] zbuf; 359 | zsiz *= 2; 360 | } else { 361 | delete[] zbuf; 362 | break; 363 | } 364 | } 365 | return NULL; 366 | #else 367 | _assert_(buf && size <= MEMMAXSIZ && sp); 368 | if (size < 2 || ((char*)buf)[0] != 'x' || ((char*)buf)[1] != mode) return NULL; 369 | buf = (char*)buf + 2; 370 | size -= 2; 371 | char* zbuf = new char[size+1]; 372 | std::memcpy(zbuf, buf, size); 373 | zbuf[size] = '\0'; 374 | *sp = size; 375 | return zbuf; 376 | #endif 377 | } 378 | 379 | 380 | /** 381 | * Calculate the CRC32 checksum of a serial data. 382 | */ 383 | uint32_t LZMA::calculate_crc(const void* buf, size_t size, uint32_t seed) { 384 | #if _KC_LZMA 385 | _assert_(buf && size <= MEMMAXSIZ); 386 | return lzma_crc32((const uint8_t*)buf, size, seed); 387 | #else 388 | _assert_(buf && size <= MEMMAXSIZ); 389 | return 0; 390 | #endif 391 | } 392 | 393 | 394 | /** 395 | * Prepared pointer of the ZLIB raw mode. 396 | */ 397 | ZLIBCompressor zlibrawfunc; 398 | ZLIBCompressor* const ZLIBRAWCOMP = &zlibrawfunc; 399 | 400 | 401 | } // common namespace 402 | 403 | // END OF FILE 404 | --------------------------------------------------------------------------------