├── hibase ├── AUTHORS ├── COPYING ├── ChangeLog ├── INSTALL ├── Makefile.am ├── Makefile.in ├── NEWS ├── README ├── acinclude.m4 ├── aclocal.m4 ├── compile ├── config.h.in ├── configure ├── configure.in ├── control ├── debian.sh ├── depcomp ├── doc │ ├── Makefile.am │ ├── Makefile.in │ ├── base64.php │ ├── hibase.spec │ ├── hidocd.init │ ├── himasterd.init │ ├── hindexd.init │ ├── hiservice.sh │ ├── rc.dict.txt │ ├── rc.hidocd │ ├── rc.hidocd.html │ ├── rc.hidocd.ini │ ├── rc.himasterd │ ├── rc.himasterd.html │ ├── rc.himasterd.ini │ ├── rc.hindexd │ ├── rc.hindexd.html │ ├── rc.hindexd.ini │ └── sinit ├── install-sh ├── missing └── src │ ├── Makefile.am │ ├── Makefile.in │ ├── base64docdhtml.h │ ├── base64indexdhtml.h │ ├── base64masterdhtml.h │ ├── hidoc.c │ ├── hidoc.h │ ├── hidocd.c │ ├── hidump.c │ ├── himasterd.c │ ├── hindexd.c │ ├── html │ ├── charcode.h │ ├── html.c │ ├── html.h │ └── htmltags.h │ ├── iniparser.c │ ├── iniparser.h │ ├── itask.h │ ├── mmdb.c │ ├── mmdb.h │ └── utils │ ├── base64.c │ ├── base64.h │ ├── db.c │ ├── db.h │ ├── http.c │ ├── http.h │ ├── immx.c │ ├── immx.h │ ├── iqueue.c │ ├── iqueue.h │ ├── kvmap.c │ ├── kvmap.h │ ├── logger.c │ ├── logger.h │ ├── md5.c │ ├── md5.h │ ├── mime.c │ ├── mime.h │ ├── mmqueue.c │ ├── mmqueue.h │ ├── mmtree.c │ ├── mmtree.h │ ├── mmtree64.c │ ├── mmtree64.h │ ├── mmtrie.c │ ├── mmtrie.h │ ├── mtree.c │ ├── mtree.h │ ├── mtree64.c │ ├── mtree64.h │ ├── mtrie.c │ ├── mtrie.h │ ├── mutex.h │ ├── qsmap.c │ ├── qsmap.h │ ├── rwlock.h │ ├── stime.c │ ├── stime.h │ ├── timer.h │ ├── xmm.c │ ├── xmm.h │ ├── zvbcode.c │ └── zvbcode.h ├── ispider ├── AUTHORS ├── COPYING ├── ChangeLog ├── INSTALL ├── Makefile.am ├── Makefile.in ├── NEWS ├── README ├── aclocal.m4 ├── compile ├── config.h.in ├── configure ├── configure.in ├── control ├── debian.sh ├── depcomp ├── doc │ ├── Makefile.am │ ├── Makefile.in │ ├── base64.php │ ├── iextractor.init │ ├── imonitord.init │ ├── ispider.init │ ├── ispider.spec │ ├── rc.iextractor.ini │ ├── rc.iextractor.init │ ├── rc.imonitord.html │ ├── rc.imonitord.ini │ ├── rc.imonitord.init │ ├── rc.ispider.ini │ └── rc.ispider.init ├── install-sh ├── missing └── src │ ├── Makefile.am │ ├── Makefile.in │ ├── base64monitordhtml.h │ ├── iextractor.c │ ├── imonitord.c │ ├── iniparser.c │ ├── iniparser.h │ ├── ispider.c │ ├── utils │ ├── base64.c │ ├── base64.h │ ├── charcode.h │ ├── crc32.c │ ├── crc32.h │ ├── db.c │ ├── db.h │ ├── evdns.c │ ├── evdns.h │ ├── html.c │ ├── html.h │ ├── htmltags.h │ ├── http.c │ ├── http.h │ ├── iqueue.c │ ├── iqueue.h │ ├── kmap.c │ ├── kmap.h │ ├── logger.c │ ├── logger.h │ ├── md5.c │ ├── md5.h │ ├── mime.c │ ├── mime.h │ ├── mmqueue.c │ ├── mmqueue.h │ ├── mmtrie.c │ ├── mmtrie.h │ ├── mtree.c │ ├── mtree.h │ ├── mtrie.c │ ├── mtrie.h │ ├── mutex.h │ ├── qchardet.c │ ├── qchardet.h │ ├── qmap.c │ ├── qmap.h │ ├── rwlock.h │ ├── stime.c │ ├── stime.h │ ├── timer.h │ ├── url.h │ ├── vmap.c │ ├── vmap.h │ ├── xmm.c │ ├── xmm.h │ ├── zstream.c │ └── zstream.h │ ├── xtask.c │ └── xtask.h ├── ithunder ├── AUTHORS ├── COPYING ├── ChangeLog ├── INSTALL ├── Makefile.am ├── Makefile.in ├── NEWS ├── README ├── aclocal.m4 ├── compile ├── config.h.in ├── configure ├── configure.in ├── control ├── debian.sh ├── depcomp ├── doc │ ├── Makefile.am │ ├── Makefile.in │ ├── base64.php │ ├── ithunder.spec │ ├── rc.idispatchd.html │ ├── rc.idispatchd.ini │ ├── rc.idispatchd.init │ ├── rc.ispider.ini │ ├── rc.ithinkd.html │ ├── rc.ithinkd.ini │ └── rc.ithinkd.init ├── install-sh ├── missing └── src │ ├── Makefile.am │ ├── Makefile.in │ ├── base64dispatchdhtml.h │ ├── base64monitordhtml.h │ ├── base64thinkdhtml.h │ ├── dbtask.c │ ├── idispatchd.c │ ├── iextractor.c │ ├── imonitord.c │ ├── iniparser.c │ ├── iniparser.h │ ├── ispider.c │ ├── ithinkd.c │ ├── kindex.c │ ├── kindex.h │ ├── qindex.c │ ├── qindex.h │ ├── utils │ ├── base64.c │ ├── base64.h │ ├── charcode.h │ ├── crc32.c │ ├── crc32.h │ ├── db.c │ ├── db.h │ ├── evdns.c │ ├── evdns.h │ ├── html.c │ ├── html.h │ ├── htmltags.h │ ├── http.c │ ├── http.h │ ├── iqueue.c │ ├── iqueue.h │ ├── kmap.c │ ├── kmap.h │ ├── kvmap.c │ ├── kvmap.h │ ├── logger.c │ ├── logger.h │ ├── md5.c │ ├── md5.h │ ├── mime.c │ ├── mime.h │ ├── mmqueue.c │ ├── mmqueue.h │ ├── mmtree.c │ ├── mmtree.h │ ├── mmtree64.c │ ├── mmtree64.h │ ├── mmtrie.c │ ├── mmtrie.h │ ├── mtree.c │ ├── mtree.h │ ├── mtrie.c │ ├── mtrie.h │ ├── mutex.h │ ├── stime.c │ ├── stime.h │ ├── timer.h │ ├── trie.c │ ├── trie.h │ ├── url.h │ ├── vmap.c │ ├── vmap.h │ ├── xmm.c │ ├── xmm.h │ ├── zstream.c │ ├── zstream.h │ ├── zvbcode.c │ └── zvbcode.h │ ├── xtask.c │ └── xtask.h ├── libchardet ├── AUTHORS ├── COPYING ├── ChangeLog ├── INSTALL ├── Makefile.am ├── Makefile.in ├── NEWS ├── README ├── aclocal.m4 ├── config.guess ├── config.h.in ├── config.sub ├── configure ├── configure.in ├── control ├── debian.sh ├── depcomp ├── install-sh ├── ltmain.sh ├── missing └── src │ ├── CharDistribution.cpp │ ├── CharDistribution.h │ ├── JpCntx.cpp │ ├── JpCntx.h │ ├── LangBulgarianModel.cpp │ ├── LangCyrillicModel.cpp │ ├── LangGreekModel.cpp │ ├── LangHebrewModel.cpp │ ├── LangHungarianModel.cpp │ ├── LangThaiModel.cpp │ ├── Makefile.am │ ├── Makefile.in │ ├── chardet.h │ ├── chardetor.cpp │ ├── entry │ └── impl.cpp │ ├── nsBig5Prober.cpp │ ├── nsBig5Prober.h │ ├── nsCharSetProber.cpp │ ├── nsCharSetProber.h │ ├── nsCodingStateMachine.h │ ├── nsEUCJPProber.cpp │ ├── nsEUCJPProber.h │ ├── nsEUCKRProber.cpp │ ├── nsEUCKRProber.h │ ├── nsEUCTWProber.cpp │ ├── nsEUCTWProber.h │ ├── nsEscCharsetProber.cpp │ ├── nsEscCharsetProber.h │ ├── nsEscSM.cpp │ ├── nsGB2312Prober.cpp │ ├── nsGB2312Prober.h │ ├── nsHebrewProber.cpp │ ├── nsHebrewProber.h │ ├── nsLatin1Prober.cpp │ ├── nsLatin1Prober.h │ ├── nsMBCSGroupProber.cpp │ ├── nsMBCSGroupProber.h │ ├── nsMBCSSM.cpp │ ├── nsPkgInt.h │ ├── nsSBCSGroupProber.cpp │ ├── nsSBCSGroupProber.h │ ├── nsSBCharSetProber.cpp │ ├── nsSBCharSetProber.h │ ├── nsSJISProber.cpp │ ├── nsSJISProber.h │ ├── nsUTF8Prober.cpp │ ├── nsUTF8Prober.h │ ├── nsUniversalDetector.cpp │ ├── nsUniversalDetector.h │ ├── nscore.h │ ├── prmem.h │ └── tables │ ├── Big5Freq.tab │ ├── EUCKRFreq.tab │ ├── EUCTWFreq.tab │ ├── GB2312Freq.tab │ └── JISFreq.tab ├── libibase ├── AUTHORS ├── COPYING ├── ChangeLog ├── INSTALL ├── Makefile.am ├── Makefile.in ├── NEWS ├── README ├── acinclude.m4 ├── aclocal.m4 ├── compile ├── config.guess ├── config.h.in ├── config.sub ├── configure ├── configure.in ├── control ├── debian.sh ├── depcomp ├── doc │ ├── Makefile.am │ ├── Makefile.in │ ├── README.txt │ └── libibase.spec ├── install-sh ├── ltmain.sh ├── missing └── src │ ├── Makefile.am │ ├── Makefile.in │ ├── bquery.c │ ├── dbcache.c │ ├── hquery.c │ ├── ibase.c │ ├── ibase.h │ ├── iheaders.h │ ├── index.c │ ├── query.c │ ├── summary.c │ ├── utils │ ├── bmap.c │ ├── bmap.h │ ├── cdb.c │ ├── cdb.h │ ├── db.c │ ├── db.h │ ├── dkv.c │ ├── dkv.h │ ├── dmap.c │ ├── dmap.h │ ├── dtree.c │ ├── dtree.h │ ├── gen.sh │ ├── ikv.c │ ├── ikv.h │ ├── imap.c │ ├── imap.h │ ├── immx.c │ ├── immx.h │ ├── itree.c │ ├── itree.h │ ├── kvmap.c │ ├── kvmap.h │ ├── lkv.c │ ├── lkv.h │ ├── lmap.c │ ├── lmap.h │ ├── logger.c │ ├── logger.h │ ├── ltree.c │ ├── ltree.h │ ├── mdb.c │ ├── mdb.h │ ├── mmtrie.c │ ├── mmtrie.h │ ├── mtree.c │ ├── mtree.h │ ├── mtree64.c │ ├── mtree64.h │ ├── mutex.h │ ├── qsmap.c │ ├── qsmap.h │ ├── queue.h │ ├── rbmap.c │ ├── rbmap.h │ ├── rwlock.h │ ├── timer.h │ ├── xmm.c │ ├── xmm.h │ ├── zvbcode.c │ └── zvbcode.h │ └── xquery.c ├── libscws ├── COPYING ├── ChangeLog ├── Makefile.am ├── Makefile.in ├── README ├── aclocal.m4 ├── cli │ ├── Makefile.am │ ├── Makefile.in │ ├── gen_dict.c │ └── scws_cmd.c ├── config.guess ├── config.h.in ├── config.sub ├── configure ├── configure.in ├── debian.sh ├── depcomp ├── dict │ ├── Makefile.am │ ├── Makefile.in │ ├── dict.gbk.xdb │ ├── dict.utf8.xdb │ └── dict_cht.utf8.xdb ├── etc │ ├── Makefile.am │ ├── Makefile.in │ ├── rules.ini │ ├── rules.utf8.ini │ └── rules_cht.utf8.ini ├── install-sh ├── libscws │ ├── Makefile.am │ ├── Makefile.in │ ├── charset.c │ ├── charset.h │ ├── config_win32.h │ ├── crc32.c │ ├── crc32.h │ ├── darray.c │ ├── darray.h │ ├── lock.c │ ├── lock.h │ ├── pool.c │ ├── pool.h │ ├── rule.c │ ├── rule.h │ ├── scws.c │ ├── scws.h │ ├── version.h │ ├── version.h.in │ ├── xdb.c │ ├── xdb.h │ ├── xdict.c │ ├── xdict.h │ ├── xtree.c │ └── xtree.h ├── ltmain.sh ├── missing ├── mkinstalldirs ├── phpext │ ├── CREDITS │ ├── config.m4 │ ├── php_scws.c │ ├── php_scws.h │ ├── php_scws_manual.txt │ ├── scws.php │ ├── scws_test.php │ └── win32 │ │ ├── Readme.txt │ │ └── php_scws-1.1.8-win32.zip └── win32 │ ├── libscws.dsp │ ├── libscws.vcproj │ ├── readme.txt │ ├── scws.dsp │ ├── scws.dsw │ ├── scws.sln │ └── scws.vcproj ├── qmtask ├── AUTHORS ├── COPYING ├── ChangeLog ├── INSTALL ├── Makefile.am ├── Makefile.in ├── NEWS ├── README ├── aclocal.m4 ├── compile ├── config.guess ├── config.h.in ├── config.sub ├── configure ├── configure.in ├── control ├── debian.sh ├── depcomp ├── doc │ ├── Makefile.am │ ├── Makefile.in │ ├── base64.php │ ├── qmtask.spec │ ├── rc.qtaskd.html │ ├── rc.qtaskd.ini │ └── rc.qtaskd.init ├── install-sh ├── ltmain.sh ├── missing └── src │ ├── Makefile.am │ ├── Makefile.in │ ├── a.sh │ ├── base64qtaskdhtml.h │ ├── btask.c │ ├── iniparser.c │ ├── iniparser.h │ ├── mtask.c │ ├── mtask.h │ ├── qtask.c │ ├── qtask.h │ ├── qtaskd.c │ └── utils │ ├── base64.c │ ├── base64.h │ ├── db.c │ ├── db.h │ ├── http.c │ ├── http.h │ ├── logger.c │ ├── logger.h │ ├── mime.c │ ├── mime.h │ ├── mmqueue.c │ ├── mmqueue.h │ ├── mmqueue64.c │ ├── mmqueue64.h │ ├── mmtree.c │ ├── mmtree.h │ ├── mmtree64.c │ ├── mmtree64.h │ ├── mmtrie.c │ ├── mmtrie.h │ ├── mtrie.c │ ├── mtrie.h │ ├── mutex.h │ ├── stime.c │ ├── stime.h │ ├── timer.h │ ├── xmm.c │ └── xmm.h ├── shell ├── build.sh ├── install.sh ├── remote.sh ├── rename.sh ├── reset.sh ├── sodo.sh ├── sowork.sh └── tarball.sh └── utils ├── db.c ├── db.h ├── imap.c ├── imap.h ├── logger.c ├── logger.h ├── mmlist.c ├── mmlist.h ├── mutex.h ├── rwlock.h ├── timer.h ├── xmm.c ├── xmm.h ├── xxlist.c └── xxlist.h /hibase/AUTHORS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sounos/ithunder/241d17ce44ed90bc3d8ce4f0b8d5ae52498db270/hibase/AUTHORS -------------------------------------------------------------------------------- /hibase/ChangeLog: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sounos/ithunder/241d17ce44ed90bc3d8ce4f0b8d5ae52498db270/hibase/ChangeLog -------------------------------------------------------------------------------- /hibase/Makefile.am: -------------------------------------------------------------------------------- 1 | SUBDIRS = src doc 2 | EXTRA_DIST = debian.sh control 3 | -------------------------------------------------------------------------------- /hibase/NEWS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sounos/ithunder/241d17ce44ed90bc3d8ce4f0b8d5ae52498db270/hibase/NEWS -------------------------------------------------------------------------------- /hibase/README: -------------------------------------------------------------------------------- 1 | required libibase >= 0.1.1 2 | -------------------------------------------------------------------------------- /hibase/acinclude.m4: -------------------------------------------------------------------------------- 1 | AC_DEFUN([AC_CHECK_EXTRA_OPTIONS],[ 2 | AC_MSG_CHECKING(for debugging) 3 | AC_ARG_ENABLE(debug, [ --enable-debug compile for debugging]) 4 | if test -z "$enable_debug" ; then 5 | enable_debug="no" 6 | elif test $enable_debug = "yes" ; then 7 | CPPFLAGS="${CPPFLAGS} -g -D_DEBUG" 8 | fi 9 | AC_MSG_RESULT([$enable_debug]) 10 | 11 | AC_MSG_CHECKING(for pthread_mutex) 12 | AC_ARG_ENABLE(pthread_mutex, [ --enable-pthread_mutex compile for enable pthread mutex]) 13 | if test -z "$enable_pthread_mutex" ; then 14 | enable_pthread_mutex="no" 15 | elif test $enable_pthread_mutex = "yes" ; then 16 | CPPFLAGS="${CPPFLAGS} -DUSE_PTHREAD_MUTEX" 17 | fi 18 | AC_MSG_RESULT([$enable_pthread_mutex]) 19 | ]) 20 | 21 | -------------------------------------------------------------------------------- /hibase/configure.in: -------------------------------------------------------------------------------- 1 | # -*- Autoconf -*- 2 | # Process this file with autoconf to produce a configure script. 3 | 4 | AC_PREREQ(2.59) 5 | AC_INIT(hibase, [0.5.0], [SounOS@gmail.com]) 6 | AC_CONFIG_SRCDIR([src/hidoc.h]) 7 | AC_CONFIG_HEADER([config.h]) 8 | AM_INIT_AUTOMAKE 9 | 10 | # Checks for programs. 11 | AC_PROG_CC 12 | AM_PROG_CC_C_O 13 | AC_PROG_INSTALL 14 | 15 | # Checks for libraries. 16 | # FIXME: Replace `main' with a function in `-libase': 17 | AC_CHECK_LIB([ibase], [main]) 18 | # FIXME: Replace `main' with a function in `-lsbase': 19 | AC_CHECK_LIB([sbase], [main]) 20 | # FIXME: Replace `main' with a function in `-lz': 21 | AC_CHECK_LIB([z], [main]) 22 | 23 | # Checks for header files. 24 | AC_HEADER_DIRENT 25 | AC_HEADER_STDC 26 | AC_CHECK_HEADERS([arpa/inet.h fcntl.h locale.h netdb.h stdint.h stdlib.h string.h sys/file.h sys/time.h unistd.h]) 27 | 28 | # Checks for typedefs, structures, and compiler characteristics. 29 | AC_C_CONST 30 | AC_TYPE_OFF_T 31 | AC_TYPE_PID_T 32 | AC_TYPE_SIZE_T 33 | AC_HEADER_TIME 34 | AC_STRUCT_TM 35 | 36 | # Checks for library functions. 37 | AC_FUNC_FORK 38 | AC_FUNC_LSTAT 39 | AC_FUNC_LSTAT_FOLLOWS_SLASHED_SYMLINK 40 | AC_FUNC_MALLOC 41 | AC_FUNC_MEMCMP 42 | AC_FUNC_MKTIME 43 | AC_FUNC_MMAP 44 | AC_FUNC_REALLOC 45 | AC_TYPE_SIGNAL 46 | AC_FUNC_STAT 47 | AC_CHECK_FUNCS([ftruncate gettimeofday memset mkdir munmap setlocale strcasecmp strchr strdup strerror strncasecmp]) 48 | 49 | #pthread 50 | havepthread=no 51 | AC_CHECK_LIB(pthread,pthread_create,havepthread=yes) 52 | if test "$havepthread" = "yes" ; then 53 | CPPFLAGS="${CPPFLAGS} -DHAVE_PTHREAD" 54 | LDFLAGS="${LDFLAGS} -lpthread" 55 | LIBS="${LIBS} -lpthread" 56 | fi 57 | #scws 58 | havescws=no 59 | AC_CHECK_LIB(scws,scws_get_result,havescws=yes) 60 | if test "$havescws" = "yes" ; then 61 | CPPFLAGS="${CPPFLAGS} -DHAVE_SCWS" 62 | LDFLAGS="${LDFLAGS} -lscws" 63 | LIBS="${LIBS} -lscws" 64 | fi 65 | 66 | #semaphore 67 | #havesemaphore=no 68 | #AC_CHECK_LIB(c, sem_timedwait, havesemaphore=yes) 69 | #if test "$havesemaphore" = "yes" ; then 70 | # CPPFLAGS="${CPPFLAGS} -DHAVE_SEMAPHORE" 71 | #fi 72 | 73 | #mmap 74 | havemmap=no 75 | AC_CHECK_LIB(c, munmap, havemmap=yes) 76 | if test "$havemmap" = "yes" ; then 77 | CPPFLAGS="${CPPFLAGS} -DHAVE_MMAP" 78 | fi 79 | 80 | 81 | AC_CONFIG_FILES([Makefile 82 | doc/Makefile 83 | src/Makefile]) 84 | AC_OUTPUT 85 | -------------------------------------------------------------------------------- /hibase/control: -------------------------------------------------------------------------------- 1 | Source: 2 | Section: libs 3 | Priority: extra 4 | Maintainer: SounOS 5 | Build-Depends: debhelper (>= 7.0.50~), autotools-dev 6 | Standards-Version: 7 | Homepage: http://sbase.googlecode.com/ 8 | #Vcs-Git: git://git.debian.org/collab-maint/libevbase.git 9 | #Vcs-Browser: http://git.debian.org/?p=collab-maint/libevbase.git;a=summary 10 | 11 | Package: 12 | Architecture: any 13 | Depends: ${shlibs:Depends}, ${misc:Depends}, libevbase, libsbase, libibase 14 | Description: 15 | 16 | -------------------------------------------------------------------------------- /hibase/debian.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | rm -rf debian 3 | pkg=`grep -E "^PACKAGE_NAME=" configure |sed -e "s/PACKAGE_NAME='\(.*\)'/\1/"` 4 | ver=`grep -E "^PACKAGE_VERSION=" configure |sed -e "s/PACKAGE_VERSION='\(.*\)'/\1/"` 5 | dh_make -y -n -s -c bsd -e SounOS@gmail.com -p "${pkg}_${ver}" 6 | cp control debian/control 7 | sed -i "s/^Package:.*/Package: $pkg/" debian/control 8 | sed -i "s/^Source:.*/Source: $pkg/" debian/control 9 | sed -i "s/^License:.*/License: $pkg/" debian/control 10 | sed -i "s/^Standards-Version:.*/Standards-Version: $ver/" debian/control 11 | sed -i "s/root /SounOS /" debian/* 12 | #export CFLAGS='-ggdb -O0' 13 | #export CXXFLAGS='-ggdb -O0' 14 | #export FFLAGS='-ggdb -O0' 15 | dpkg-buildpackage -rfakeroot -eSounOS -mSounOS 16 | -------------------------------------------------------------------------------- /hibase/doc/base64.php: -------------------------------------------------------------------------------- 1 | #!/usr/bin/php 2 | 22 | -------------------------------------------------------------------------------- /hibase/doc/hibase.spec: -------------------------------------------------------------------------------- 1 | # Authority: SounOS.org 2 | 3 | Summary: High speed/performence Search Engine Base Tools 4 | Name: hibase 5 | Version: 0.4.19 6 | Release: 66%{?dist} 7 | License: BSD 8 | Group: System Environment/Libraries 9 | URL: http://code.google.com/p/libibase/ 10 | 11 | Source: http://code.google.com/p/libibase/download/%{name}-%{version}.tar.gz 12 | Packager: SounOS 13 | Vendor: SounOS 14 | BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root 15 | 16 | BuildRequires: libevbase >= 1.0.0 libsbase >= 1.0.0 libibase >= 0.5.14 17 | Requires: libevbase >= 1.0.0 libsbase >= 1.0.0 libibase >= 0.5.14 18 | 19 | %description 20 | Hibase is some tools set for search engine. 21 | 22 | %prep 23 | %setup 24 | 25 | %build 26 | %configure 27 | %{__make} 28 | 29 | %install 30 | %{__rm} -rf %{buildroot} 31 | %{__make} install DESTDIR="%{buildroot}" 32 | 33 | %clean 34 | %{__rm} -rf %{buildroot} 35 | 36 | %post 37 | 38 | /sbin/chkconfig --level 345 hindexd off 39 | /sbin/chkconfig --level 345 hidocd off 40 | /sbin/chkconfig --level 345 hiqdocd off 41 | /sbin/chkconfig --level 345 himasterd off 42 | /sbin/chkconfig --level 345 hiqparserd off 43 | 44 | %preun 45 | [ "`pstree|grep himasterd|wc -l`" -gt "0" ] && /sbin/service himasterd stop 46 | [ "`pstree|grep hindexd|wc -l`" -gt "0" ] && /sbin/service hindexd stop 47 | [ "`pstree|grep hidocd|wc -l`" -gt "0" ] && /sbin/service hidocd stop 48 | [ "`pstree|grep hiqdocd|wc -l`" -gt "0" ] && /sbin/service hiqdocd stop 49 | [ "`pstree|grep hiqparserd|wc -l`" -gt "0" ] && /sbin/service hiqparserd stop 50 | /sbin/chkconfig --del hindexd 51 | /sbin/chkconfig --del hidocd 52 | /sbin/chkconfig --del hiqdocd 53 | /sbin/chkconfig --del himasterd 54 | /sbin/chkconfig --del hiqparserd 55 | 56 | %files 57 | %defattr(-, root, root, 0755) 58 | %{_sbindir}/* 59 | %{_localstatedir}/* 60 | %{_sysconfdir}/rc.d/* 61 | %config(noreplace) %{_sysconfdir}/*.ini 62 | 63 | %changelog 64 | * Thu Apr 22 2010 11:37:25 CST SounOS 65 | - fixed hidoc_set_basedir()::iqueue_push(taskid) 66 | * Tue Mar 09 2010 15:02:39 CST SounOS 67 | - added hidoc_set_ccompress_status(); 68 | - added hidoc_set_phrase_status(); 69 | - added hidoc_add_task(); 70 | - added hidoc_del_task(); 71 | - added hidoc_pop_task(); 72 | - added hidoc_push_task(); 73 | - added hidoc_over_task(); 74 | -------------------------------------------------------------------------------- /hibase/doc/rc.hidocd: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # HIDOCD: 4 | # 5 | # chkconfig: 23 70 40 6 | # description: HIDOCD 7 | # 8 | # 9 | 10 | # Source function library. 11 | . /etc/rc.d/init.d/functions 12 | 13 | #[ -r /etc/sysconfig/hidocd ] && . /etc/sysconfig/hidocd 14 | HIDOCDARGS=" -d -c /usr/local/etc/hidocd.ini > /dev/null 2>&1 " 15 | 16 | start() 17 | { 18 | echo -n $"Starting hidocd: " 19 | daemon /usr/local/sbin/hidocd $HIDOCDARGS 20 | echo 21 | } 22 | 23 | stop() 24 | { 25 | echo -n $"Shutting down hidocd: " 26 | killproc hidocd 27 | 28 | echo 29 | } 30 | 31 | [ -f /usr/local/sbin/hidocd ] || exit 0 32 | 33 | # See how we were called. 34 | case "$1" in 35 | start) 36 | start 37 | ;; 38 | stop) 39 | stop 40 | ;; 41 | restart|reload) 42 | stop 43 | start 44 | ;; 45 | *) 46 | echo $"Usage: $0 {start|stop|restart|reload}" 47 | exit 1 48 | esac 49 | 50 | exit 0 51 | -------------------------------------------------------------------------------- /hibase/doc/rc.himasterd: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # HIMASTERD: 4 | # 5 | # chkconfig: 23 90 55 6 | # description: HIMASTERD 7 | # 8 | # 9 | 10 | # Source function library. 11 | . /etc/rc.d/init.d/functions 12 | 13 | #[ -r /etc/sysconfig/himasterd ] && . /etc/sysconfig/himasterd 14 | HIMASTERDARGS=" -d -c /usr/local/etc/himasterd.ini > /dev/null 2>&1 " 15 | 16 | start() 17 | { 18 | echo -n $"Starting himasterd: " 19 | daemon /usr/local/sbin/himasterd $HIMASTERDARGS 20 | echo 21 | } 22 | 23 | stop() 24 | { 25 | echo -n $"Shutting down himasterd: " 26 | killproc himasterd 27 | echo 28 | } 29 | 30 | [ -f /usr/local/sbin/himasterd ] || exit 0 31 | 32 | # See how we were called. 33 | case "$1" in 34 | start) 35 | start 36 | ;; 37 | stop) 38 | stop 39 | ;; 40 | restart|reload) 41 | stop 42 | start 43 | ;; 44 | *) 45 | echo $"Usage: $0 {start|stop|restart|reload}" 46 | exit 1 47 | esac 48 | 49 | exit 0 50 | -------------------------------------------------------------------------------- /hibase/doc/rc.hindexd: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # HINDEXD: 4 | # 5 | # chkconfig: 23 90 60 6 | # description: HINDEXD 7 | # 8 | # 9 | 10 | # Source function library. 11 | . /etc/rc.d/init.d/functions 12 | 13 | #[ -r /etc/sysconfig/hindexd ] && . /etc/sysconfig/hindexd 14 | HINDEXDARGS=" -d -c /usr/local/etc/hindexd.ini > /dev/null 2>&1 " 15 | 16 | start() 17 | { 18 | echo -n $"Starting hindexd: " 19 | daemon /usr/local/sbin/hindexd $HINDEXDARGS 20 | echo 21 | } 22 | 23 | stop() 24 | { 25 | echo -n $"Shutting down hindexd: " 26 | killproc hindexd 27 | echo 28 | } 29 | 30 | [ -f /usr/local/sbin/hindexd ] || exit 0 31 | 32 | # See how we were called. 33 | case "$1" in 34 | start) 35 | start 36 | ;; 37 | stop) 38 | stop 39 | ;; 40 | restart|reload) 41 | stop 42 | start 43 | ;; 44 | *) 45 | echo $"Usage: $0 {start|stop|restart|reload}" 46 | exit 1 47 | esac 48 | 49 | exit 0 50 | -------------------------------------------------------------------------------- /hibase/doc/sinit: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | nod=$1; 3 | update-rc.d -f $nod start 21 2 3 4 5 . stop 21 0 1 6 . 4 | update-rc.d $nod enable 5 | -------------------------------------------------------------------------------- /hibase/src/Makefile.am: -------------------------------------------------------------------------------- 1 | #automake 2 | sbin_PROGRAMS = himasterd hindexd hidocd 3 | himasterd_SOURCES = himasterd.c utils/http.c utils/http.h iniparser.c iniparser.h utils/rwlock.h\ 4 | base64masterdhtml.h mmdb.h mmdb.c utils/mmtrie.h utils/mmtrie.c utils/db.c utils/db.h \ 5 | utils/logger.h utils/logger.c utils/mutex.h utils/timer.h utils/mtrie.h utils/mtrie.c \ 6 | utils/stime.h utils/stime.c utils/base64.h utils/base64.c utils/md5.h utils/md5.c \ 7 | utils/iqueue.h utils/iqueue.c utils/xmm.h utils/xmm.c utils/mtree64.h utils/mtree64.c \ 8 | utils/immx.h utils/immx.c 9 | himasterd_CPPFLAGS = -I utils/ -D_FILE_OFFSET_BITS=64 -Wall 10 | himasterd_LDFLAGS = -lz -lsbase 11 | 12 | hindexd_SOURCES = hindexd.c utils/http.c utils/http.h iniparser.c iniparser.h base64indexdhtml.h \ 13 | utils/logger.h utils/mutex.h utils/timer.h utils/mtrie.h utils/mtrie.c utils/logger.c \ 14 | utils/stime.h utils/stime.c utils/base64.h utils/base64.c utils/xmm.h utils/xmm.c \ 15 | utils/mtree64.h utils/mtree64.c utils/immx.h utils/immx.c 16 | hindexd_CPPFLAGS = -I utils/ -D_FILE_OFFSET_BITS=64 -Wall 17 | hindexd_LDFLAGS = -lz -lsbase -libase 18 | 19 | hidocd_SOURCES = hidocd.c utils/http.c utils/http.h iniparser.c iniparser.h utils/mtrie.h \ 20 | utils/logger.h utils/logger.c utils/mutex.h utils/timer.h html/charcode.h html/html.c html/html.h \ 21 | html/htmltags.h hidoc.h hidoc.c utils/zvbcode.c utils/zvbcode.h utils/mtrie.c utils/mtrie.h \ 22 | utils/mtree.h utils/mtree.c utils/iqueue.h utils/iqueue.c utils/stime.h utils/stime.c \ 23 | utils/base64.h utils/base64.c base64docdhtml.h utils/mmtrie.h utils/mmtrie.c utils/rwlock.h \ 24 | utils/mmqueue.h utils/mmqueue.c utils/db.h utils/db.c utils/xmm.h utils/xmm.c \ 25 | utils/mmtree64.h utils/mmtree64.c 26 | 27 | hidocd_CPPFLAGS = -I utils -I html -D_FILE_OFFSET_BITS=64 -Wall 28 | hidocd_LDFLAGS = -lz -lsbase 29 | 30 | bin_PROGRAMS = hidump 31 | hidump_SOURCES = hidump.c 32 | hidump_CPPFLAGS = -I utils -D_FILE_OFFSET_BITS=64 -Wall 33 | hidump_LDFLAGS = -libase 34 | -------------------------------------------------------------------------------- /hibase/src/html/html.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #ifndef _HTML_H 5 | #define _HTML_H 6 | #define HTML_TAG_NMAX 2000000 7 | #define HTML_FIELD_MAX 256 8 | #define LEVEL_MAX 100000 9 | #define HTML_CONTENT_MAX 16777216 10 | #define HTML_TITLE_MAX 1048576 11 | #define HTML_LINK_SCALE 0.50f 12 | typedef struct _HBLOCK 13 | { 14 | char *start; 15 | int nbytes; 16 | int nlinks; 17 | struct _HBLOCK *parent; 18 | int npairs; 19 | char *last; 20 | int id; 21 | }HBLOCK; 22 | typedef struct _HFIELD 23 | { 24 | int from; 25 | int len; 26 | }HFIELD; 27 | typedef struct _HTML 28 | { 29 | char *block; 30 | int nblock; 31 | char *content; 32 | int ncontent; 33 | char *title; 34 | int ntitle; 35 | int titleoff; 36 | HBLOCK blocklist[HTML_TAG_NMAX]; 37 | int nblocks; 38 | HFIELD fieldlist[HTML_FIELD_MAX]; 39 | int nfields; 40 | void *table; 41 | 42 | int (*get_content)(struct _HTML *html, char *content, size_t len, 43 | int filter, int new_field); 44 | int (*add_field)(struct _HTML *html, char *text, int ntext); 45 | void (*reset)(struct _HTML *html); 46 | void (*clean)(struct _HTML **html); 47 | }HTML; 48 | /* Initialize HTML */ 49 | HTML *html_init(); 50 | /* clean html */ 51 | void html_clean(HTML **); 52 | #endif 53 | -------------------------------------------------------------------------------- /hibase/src/iniparser.h: -------------------------------------------------------------------------------- 1 | /* 2 | Based upon libiniparser, by Nicolas Devillard 3 | Hacked into 1 file (m-iniparser) by Freek/2005 4 | Original terms following: 5 | 6 | -- - 7 | 8 | Copyright (c) 2000 by Nicolas Devillard (ndevilla AT free DOT fr). 9 | 10 | Written by Nicolas Devillard. Not derived from licensed software. 11 | 12 | Permission is granted to anyone to use this software for any 13 | purpose on any computer system, and to redistribute it freely, 14 | subject to the following restrictions: 15 | 16 | 1. The author is not responsible for the consequences of use of 17 | this software, no matter how awful, even if they arise 18 | from defects in it. 19 | 20 | 2. The origin of this software must not be misrepresented, either 21 | by explicit claim or by omission. 22 | 23 | 3. Altered versions must be plainly marked as such, and must not 24 | be misrepresented as being the original software. 25 | 26 | 4. This notice may not be removed or altered. 27 | 28 | */ 29 | 30 | 31 | #ifndef _INIPARSER_H_ 32 | #define _INIPARSER_H_ 33 | #include 34 | #include 35 | #include 36 | #include 37 | #include 38 | 39 | #ifdef __cplusplus 40 | extern "C" { 41 | #endif 42 | 43 | 44 | typedef struct _dictionary_ { 45 | /** Number of entries in dictionary */ 46 | int n; 47 | /** Storage size */ 48 | int size; 49 | /** List of string values */ 50 | char **val; 51 | /** List of string keys */ 52 | char **key ; 53 | /** List of hash values for keys */ 54 | unsigned *hash; 55 | } dictionary ; 56 | 57 | 58 | /* generated by genproto */ 59 | 60 | dictionary * iniparser_new(char *ininame); 61 | void iniparser_free(dictionary * d); 62 | 63 | int iniparser_getnsec(dictionary * d); 64 | char * iniparser_getsecname(dictionary * d, int n); 65 | void iniparser_dump(dictionary * d, FILE * f); 66 | void iniparser_dump_ini(dictionary * d, FILE * f); 67 | char * iniparser_getkey(dictionary *d, char *section, char *key); 68 | char * iniparser_getstr(dictionary * d, char * key); 69 | char * iniparser_getstring(dictionary * d, char * key, char * def); 70 | int iniparser_getint(dictionary * d, char * key, int notfound); 71 | double iniparser_getdouble(dictionary * d, char * key, double notfound); 72 | int iniparser_getboolean(dictionary * d, char * key, int notfound); 73 | int iniparser_find_entry(dictionary * ini, char * entry); 74 | int iniparser_setstr(dictionary * ini, char * entry, char * val); 75 | void iniparser_unset(dictionary * ini, char * entry); 76 | 77 | #ifdef __cplusplus 78 | } 79 | #endif 80 | 81 | #endif 82 | 83 | -------------------------------------------------------------------------------- /hibase/src/itask.h: -------------------------------------------------------------------------------- 1 | #ifdef __ITASK__H__ 2 | #define __ITASK__H__ 3 | #define QT_PATH_MAX 256 4 | typedef struct _QHEAD 5 | { 6 | int cmd; 7 | int flag; 8 | int req_queue_no; 9 | int over_queue_no; 10 | int64_t taskid; 11 | }QHEAD; 12 | typedef struct _QTASK 13 | { 14 | int ntasks; 15 | void *qhead; 16 | void *queue; 17 | void *waitover; 18 | }QTASK; 19 | typedef struct _QTSTATE 20 | { 21 | int nqueues; 22 | 23 | }QTSTATE; 24 | typedef struct _ITASK 25 | { 26 | char basedir[QT_PATH_MAX]; 27 | void *mutex; 28 | void *logger; 29 | }ITASK; 30 | ITASK *itask_init(char *basedir); 31 | int itask_push_task(ITASK *itask, int queue_no, int flag, int64_t id); 32 | int64_t itask_pop_task(ITASK *itask, int queue_no); 33 | void itask_clean(ITASK *itask); 34 | #endif 35 | -------------------------------------------------------------------------------- /hibase/src/utils/base64.h: -------------------------------------------------------------------------------- 1 | #ifndef _BASE64_H 2 | #define _BASE64_H 3 | #define BASE64_LEN(x) ((x + 2) / 3 * 4 + 1) 4 | int base64_encode(char *out, const unsigned char *in, int inlen); 5 | int base64_decode(unsigned char *out, const char *in, int inlen); 6 | #endif 7 | -------------------------------------------------------------------------------- /hibase/src/utils/iqueue.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include "mutex.h" 5 | #ifndef _QUEUE_H 6 | #define _QUEUE_H 7 | #ifdef __cplusplus 8 | extern "C" { 9 | #endif 10 | typedef struct _QNODE 11 | { 12 | //void *ptr; 13 | int data; 14 | struct _QNODE *next; 15 | }QNODE; 16 | #define QNODE_LINE_MAX 10240 17 | #define QNODE_LINE_NUM 1024 18 | typedef struct _QUEUE 19 | { 20 | int total; 21 | int nlist; 22 | QNODE *list[QNODE_LINE_MAX]; 23 | QNODE *left; 24 | QNODE *first; 25 | QNODE *last; 26 | MUTEX *mutex; 27 | }QUEUE; 28 | void *iqueue_init(); 29 | void iqueue_push(void *q, int data); 30 | void iqueue_pop(void *q, int *data); 31 | void iqueue_head(void *q, int *data); 32 | void iqueue_clean(void *q); 33 | #define QTOTAL(q) (((QUEUE *)q)->total) 34 | #ifdef __cplusplus 35 | } 36 | #endif 37 | #endif 38 | -------------------------------------------------------------------------------- /hibase/src/utils/md5.h: -------------------------------------------------------------------------------- 1 | #include 2 | #ifndef _MD5_H 3 | #define _MD5_H 4 | #ifdef __cplusplus 5 | extern "C" { 6 | #endif 7 | #define MD5_LEN 16 8 | #define _MD5_BLOCK_N 64 9 | #define _MD5_SET_N 56 10 | #define _MD5_BITS_N 8 11 | #ifndef _TYPEDEF_MD5_CTX 12 | #define _TYPEDEF_MD5_CTX 13 | typedef struct _MD5_CTX 14 | { 15 | u_int32_t state[4]; 16 | u_int32_t total[2]; 17 | unsigned char digest[MD5_LEN]; 18 | unsigned char buf[_MD5_BLOCK_N]; 19 | }MD5_CTX; 20 | /* Initialize */ 21 | void md5_init(MD5_CTX *context); 22 | /* Update */ 23 | void md5_update(MD5_CTX *context, unsigned char *data, u_int32_t ndata); 24 | /* Final */ 25 | void md5_final(MD5_CTX *context); 26 | void md5(unsigned char *data, u_int32_t ndata, unsigned char *digest); 27 | /* Cacalute FILE md5 */ 28 | int md5_file(const char *file, unsigned char *digest); 29 | #endif 30 | #define MD5(_data, _ndata, md) { \ 31 | MD5_CTX ctx; \ 32 | md5_init(&ctx); \ 33 | md5_update(&ctx, _data, _ndata); \ 34 | md5_final(&ctx); \ 35 | memcpy(md, ctx.digest, MD5_LEN); \ 36 | } 37 | #define MD5OUT(md, out) \ 38 | { \ 39 | int i = 0; \ 40 | do{ \ 41 | fprintf(out, "%02x", md[i++]); \ 42 | }while(i < MD5_LEN); \ 43 | } 44 | #ifdef __cplusplus 45 | } 46 | #endif 47 | #endif 48 | -------------------------------------------------------------------------------- /hibase/src/utils/mime.h: -------------------------------------------------------------------------------- 1 | #ifndef _MIME_H 2 | #define _MIME_H 3 | #define MIME_NUM_MAX 1024 4 | typedef struct _MIME_MAP 5 | { 6 | void *map; 7 | int num; 8 | int bits; 9 | }MIME_MAP; 10 | /* initialize mime */ 11 | int mime_map_init(MIME_MAP *mime_map); 12 | /* add mime */ 13 | int mime_add(MIME_MAP *mime_map, char *mime, int len); 14 | /* add mime line */ 15 | int mime_add_line(MIME_MAP *mime_map, char *p, char *end); 16 | /* return mime id*/ 17 | int mime_id(MIME_MAP *mime_map, char *mime, int len); 18 | /* clean mime map*/ 19 | void mime_map_clean(MIME_MAP *mime_map); 20 | #endif 21 | -------------------------------------------------------------------------------- /hibase/src/utils/mmqueue.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include "mutex.h" 5 | #ifndef _MMQUEUE64_H 6 | #define _MMQUEUE64_H 7 | #ifdef __cplusplus 8 | extern "C" { 9 | #endif 10 | typedef struct _MMQODE 11 | { 12 | //void *ptr; 13 | unsigned int data; 14 | unsigned int next; 15 | }MMQNODE; 16 | typedef struct _MMQROOT 17 | { 18 | int status; 19 | int total; 20 | int first; 21 | int last; 22 | }MMQROOT; 23 | #define MMQ_INCRE_NUM 1000000 24 | #define MMQ_NODE_MAX 100000000 25 | #define MMQ_ROOT_MAX 8192 26 | typedef struct _MMQSTATE 27 | { 28 | int qtotal; 29 | int qleft; 30 | int nroots; 31 | MMQROOT roots[MMQ_ROOT_MAX]; 32 | }MMQSTATE; 33 | typedef struct _MMQUEUE 34 | { 35 | int fd; 36 | int bits; 37 | off_t end; 38 | off_t size; 39 | off_t old; 40 | void *map; 41 | MUTEX *mutex; 42 | MMQSTATE *state; 43 | MMQNODE *nodes; 44 | }MMQUEUE; 45 | MMQUEUE *mmqueue_init(char *qfile); 46 | int mmqueue_new(MMQUEUE *mmq); 47 | int mmqueue_total(MMQUEUE *mmq, int rootid); 48 | int mmqueue_close(MMQUEUE *mmq, int rootid); 49 | int mmqueue_push(MMQUEUE *mmq, int rootid, int data); 50 | int mmqueue_pop(MMQUEUE *mmq, int rootid, int *data); 51 | int mmqueue_head(MMQUEUE *mmq, int rootid, int *data); 52 | void mmqueue_clean(MMQUEUE *mmq); 53 | #define MMQ(x) ((MMQUEUE*)x) 54 | #ifdef __cplusplus 55 | } 56 | #endif 57 | #endif 58 | -------------------------------------------------------------------------------- /hibase/src/utils/mmtree.h: -------------------------------------------------------------------------------- 1 | #ifndef _MMTREE_H 2 | #define _MMTREE_H 3 | #include "mutex.h" 4 | #define MMTREE_INCRE_NUM 10000000 5 | #define MMTREE_NODES_MAX 2000000000 6 | #define MMTREE_ROOT_MAX 10240 7 | typedef struct _MTNODE 8 | { 9 | unsigned int left; 10 | unsigned int right; 11 | unsigned int parent; 12 | unsigned int color; 13 | int data; 14 | int key; 15 | }MTNODE; 16 | typedef struct _MMROOT 17 | { 18 | int status; 19 | int bits; 20 | unsigned int total; 21 | unsigned int rootid; 22 | }MMROOT; 23 | typedef struct _MTSTATE 24 | { 25 | int kmax; 26 | int kmin; 27 | unsigned int nmax; 28 | unsigned int nmin; 29 | unsigned int count; 30 | unsigned int left; 31 | unsigned int current; 32 | unsigned int total; 33 | unsigned int qleft; 34 | unsigned int qfirst; 35 | unsigned int qlast; 36 | unsigned int nroots; 37 | MMROOT roots[MMTREE_ROOT_MAX]; 38 | }MTSTATE; 39 | typedef struct _MMTREE 40 | { 41 | int fd; 42 | int status; 43 | off_t size; 44 | off_t end; 45 | off_t old; 46 | void *start; 47 | MTSTATE *state; 48 | MTNODE *map; 49 | MUTEX *mutex; 50 | }MMTREE; 51 | void *mmtree_init(char *file); 52 | int mmtree_new_tree(void *mmtree); 53 | unsigned int mmtree_total(void *mmtree, int rootid); 54 | unsigned int mmtree_try_insert(void *mmtree, int rootid, int key, int data, int *old); 55 | unsigned int mmtree_insert(void *mmtree, int rootid, int key, int data, int *old); 56 | unsigned int mmtree_get(void *mmtree, unsigned int nodeid, int *key, int *data); 57 | unsigned int mmtree_find(void *mmtree, int rootid, int key, int *data); 58 | unsigned int mmtree_min(void *mmtree, int rootid, int *key, int *data); 59 | unsigned int mmtree_max(void *mmtree, int rootid, int *key, int *data); 60 | unsigned int mmtree_next(void *mmtree, int rootid, unsigned int nodeid, int *key, int *data); 61 | unsigned int mmtree_prev(void *mmtree, int rootid, unsigned int nodeid, int *key, int *data); 62 | int mmtree_set_data(void *mmtree, unsigned int nodeid, int data); 63 | void mmtree_view_tree(void *mmtree, int rootid, FILE *fp); 64 | void mmtree_remove(void *mmtree, int rootid, unsigned int nodeid, int *key, int *data); 65 | void mmtree_remove_tree(void *mmtree, int rootid); 66 | void mmtree_close(void *mmtree); 67 | #endif 68 | -------------------------------------------------------------------------------- /hibase/src/utils/mmtree64.h: -------------------------------------------------------------------------------- 1 | #ifndef _MMTREE64_H 2 | #define _MMTREE64_H 3 | #define MMTREE64_INCRE_NUM 10000000 4 | #define MMTREE64_NODES_MAX 2000000000 5 | #define MMTREE64_ROOT_MAX 10240 6 | #include "mutex.h" 7 | typedef struct _MTNODE 8 | { 9 | unsigned int left; 10 | unsigned int right; 11 | unsigned int parent; 12 | unsigned int color; 13 | int data; 14 | int bit; 15 | int64_t key; 16 | }MTNODE; 17 | typedef struct _MMROOT 18 | { 19 | int status; 20 | int bits; 21 | unsigned int total; 22 | unsigned int rootid; 23 | }MMROOT; 24 | typedef struct _MTSTATE 25 | { 26 | int64_t kmax; 27 | int64_t kmin; 28 | unsigned int nmax; 29 | unsigned int nmin; 30 | unsigned int count; 31 | unsigned int left; 32 | unsigned int current; 33 | unsigned int total; 34 | unsigned int qleft; 35 | unsigned int qfirst; 36 | unsigned int qlast; 37 | unsigned int nroots; 38 | MMROOT roots[MMTREE64_ROOT_MAX]; 39 | }MTSTATE; 40 | typedef struct _MMTREE64 41 | { 42 | int fd; 43 | int status; 44 | off_t size; 45 | off_t end; 46 | off_t old; 47 | char *start; 48 | MTSTATE *state; 49 | MTNODE *map; 50 | MUTEX *mutex; 51 | }MMTREE64; 52 | void *mmtree64_init(char *file); 53 | int mmtree64_new_tree(void *mmtree); 54 | unsigned int mmtree64_total(void *mmtree, int rootid); 55 | unsigned int mmtree64_try_insert(void *mmtree, int rootid, int64_t key, int data, int *old); 56 | unsigned int mmtree64_insert(void *mmtree, int rootid, int64_t key, int data, int *old); 57 | unsigned int mmtree64_get(void *mmtree, unsigned int nodeid, int64_t *key, int *data); 58 | unsigned int mmtree64_find(void *mmtree, int rootid, int64_t key, int *data); 59 | unsigned int mmtree64_min(void *mmtree, int rootid, int64_t *key, int *data); 60 | unsigned int mmtree64_max(void *mmtree, int rootid, int64_t *key, int *data); 61 | unsigned int mmtree64_next(void *mmtree, int rootid, unsigned int nodeid, int64_t *key, int *data); 62 | unsigned int mmtree64_prev(void *mmtree, int rootid, unsigned int nodeid, int64_t *key, int *data); 63 | int mmtree64_set_data(void *mmtree, unsigned int nodeid, int data); 64 | void mmtree64_view_tree(void *mmtree, int rootid, FILE *fp); 65 | void mmtree64_remove(void *mmtree, int rootid, unsigned int nodeid, int64_t *key, int *data); 66 | void mmtree64_remove_tree(void *mmtree, int rootid); 67 | void mmtree64_close(void *mmtree); 68 | #endif 69 | -------------------------------------------------------------------------------- /hibase/src/utils/rwlock.h: -------------------------------------------------------------------------------- 1 | #ifndef __RWLOCK__H 2 | #define __RWLOCK__H 3 | #ifdef HAVE_PTHREAD 4 | #define RWLOCK_INIT(x) pthread_rwlock_init(&(x), NULL) 5 | #define RWLOCK_RDLOCK(x) pthread_rwlock_rdlock(&(x)) 6 | #define RWLOCK_WRLOCK(x) pthread_rwlock_wrlock(&(x)) 7 | #define RWLOCK_UNLOCK(x) pthread_rwlock_unlock(&(x)) 8 | #define RWLOCK_DESTROY(x) pthread_rwlock_destroy(&(x)) 9 | #else 10 | #define RWLOCK_INIT(x) 11 | #define RWLOCK_RDLOCK(x) 12 | #define RWLOCK_WRLOCK(x) 13 | #define RWLOCK_UNLOCK(x) 14 | #define RWLOCK_DESTROY(x) 15 | #endif 16 | #endif 17 | -------------------------------------------------------------------------------- /hibase/src/utils/stime.h: -------------------------------------------------------------------------------- 1 | #ifndef _STIME_H 2 | #define _STIME_H 3 | #include 4 | #include 5 | //convert str datetime to time 6 | time_t str2time(char *datestr); 7 | /* time to GMT */ 8 | int GMTstrdate(time_t time, char *date); 9 | /* strdate */ 10 | int strdate(time_t time, char *date); 11 | /* local date time */ 12 | int datetime(time_t times, char *date); 13 | /* timetospec */ 14 | void timetospec(struct timespec *ts, int usecs); 15 | #endif 16 | -------------------------------------------------------------------------------- /hibase/src/utils/xmm.h: -------------------------------------------------------------------------------- 1 | #ifndef _XMM_H_ 2 | #define _XMM_H_ 3 | void* xmm_mnew(size_t size); 4 | void* xmm_new(size_t size); 5 | void* xmm_mrenew(void *old, size_t old_size, size_t new_size); 6 | void* xmm_renew(void *old, size_t old_size, size_t new_size); 7 | void* xmm_mresize(void *old, size_t old_size, size_t new_size); 8 | void* xmm_resize(void *old, size_t old_size, size_t new_size); 9 | void xmm_free(void *m, size_t size); 10 | #endif 11 | -------------------------------------------------------------------------------- /hibase/src/utils/zvbcode.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include "zvbcode.h" 5 | #define VB_NUM_MAX 100000000 6 | #ifdef _DEBUG_ZVBCODE 7 | #include "timer.h" 8 | #define __ZINT_OUT__ 0 9 | #define __ZINT__ "%d\n" 10 | typedef int ZINT; 11 | int main(int argc, char **argv) 12 | { 13 | int i = 0, n = 0, x = 0; 14 | ZINT rand = 0, *dp = NULL, *np = NULL; 15 | char *p = NULL, *ps = NULL, *ep = NULL; 16 | FILE *fp = NULL, *dfp = NULL; 17 | void *timer = NULL; 18 | 19 | if((ps = (char *)calloc(VB_NUM_MAX*2, sizeof(ZINT)))) 20 | { 21 | TIMER_INIT(timer); 22 | n = VB_NUM_MAX * sizeof(ZINT); 23 | if((fp = fopen("/tmp/double.txt", "a+"))) 24 | { 25 | p = ps; 26 | np = &rand; 27 | for(i = 0; i < VB_NUM_MAX; i++) 28 | { 29 | rand = (ZINT)random()%100000000; 30 | if(__ZINT_OUT__)fprintf(fp, __ZINT__, rand); 31 | ZVBCODE(np, p); 32 | } 33 | TIMER_SAMPLE(timer); 34 | fflush(fp); 35 | fprintf(stdout, "compressed(sizeof(zint):%d) %d to %d time:%lld avg:%f\n", 36 | sizeof(ZINT), n, (p - ps), PT_LU_USEC(timer), 37 | (double)PT_LU_USEC(timer)/(double)VB_NUM_MAX); 38 | fclose(fp); 39 | } 40 | if((dfp = fopen("/tmp/double.text", "a+"))) 41 | { 42 | ep = p; 43 | p = ps; 44 | np = &rand; 45 | TIMER_SAMPLE(timer); 46 | i = 0; 47 | while(p < ep) 48 | { 49 | rand = 0; 50 | UZVBCODE(p, x, np); 51 | i++; 52 | if(__ZINT_OUT__) fprintf(dfp, __ZINT__, rand); 53 | } 54 | TIMER_SAMPLE(timer); 55 | fflush(fp); 56 | fprintf(stdout, "decompressed(typesizeof(ZINT):%d) %d to %d time:%lld avg:%f\n", 57 | sizeof(ZINT), (p - ps), i * sizeof(ZINT), PT_LU_USEC(timer), 58 | (double)PT_LU_USEC(timer)/(double)i); 59 | fclose(dfp); 60 | } 61 | free(ps); 62 | ps = NULL; 63 | } 64 | } 65 | #endif 66 | -------------------------------------------------------------------------------- /ispider/AUTHORS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sounos/ithunder/241d17ce44ed90bc3d8ce4f0b8d5ae52498db270/ispider/AUTHORS -------------------------------------------------------------------------------- /ispider/ChangeLog: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sounos/ithunder/241d17ce44ed90bc3d8ce4f0b8d5ae52498db270/ispider/ChangeLog -------------------------------------------------------------------------------- /ispider/Makefile.am: -------------------------------------------------------------------------------- 1 | SUBDIRS = src doc 2 | EXTRA_DIST = debian.sh control 3 | -------------------------------------------------------------------------------- /ispider/NEWS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sounos/ithunder/241d17ce44ed90bc3d8ce4f0b8d5ae52498db270/ispider/NEWS -------------------------------------------------------------------------------- /ispider/README: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sounos/ithunder/241d17ce44ed90bc3d8ce4f0b8d5ae52498db270/ispider/README -------------------------------------------------------------------------------- /ispider/configure.in: -------------------------------------------------------------------------------- 1 | # -*- Autoconf -*- 2 | # Process this file with autoconf to produce a configure script. 3 | 4 | AC_PREREQ(2.59) 5 | AC_INIT(ispider, [0.0.4], [SounOS@gmail.com]) 6 | AC_CONFIG_SRCDIR([src/iniparser.c]) 7 | AC_CONFIG_HEADER([config.h]) 8 | AM_INIT_AUTOMAKE 9 | 10 | # Checks for programs. 11 | AC_PROG_CC 12 | AM_PROG_CC_C_O 13 | AC_PROG_INSTALL 14 | 15 | # Checks for libraries. 16 | # FIXME: Replace `main' with a function in `-lchardet': 17 | AC_CHECK_LIB([chardet], [main]) 18 | # FIXME: Replace `main' with a function in `-libase': 19 | AC_CHECK_LIB([ibase], [main]) 20 | # FIXME: Replace `main' with a function in `-lpcre': 21 | AC_CHECK_LIB([pcre], [main]) 22 | # FIXME: Replace `main' with a function in `-lsbase': 23 | AC_CHECK_LIB([sbase], [main]) 24 | # FIXME: Replace `main' with a function in `-lz': 25 | AC_CHECK_LIB([z], [main]) 26 | 27 | #pthread 28 | havepthread=no 29 | AC_CHECK_LIB(pthread,pthread_create,havepthread=yes) 30 | if test "$havepthread" = "yes" ; then 31 | CPPFLAGS="${CPPFLAGS} -DHAVE_PTHREAD" 32 | LDFLAGS="${LDFLAGS} -lpthread" 33 | LIBS="${LIBS} -lpthread" 34 | fi 35 | 36 | #mmap 37 | havemmap=no 38 | AC_CHECK_LIB(c, munmap, havemmap=yes) 39 | if test "$havemmap" = "yes" ; then 40 | CPPFLAGS="${CPPFLAGS} -DHAVE_MMAP" 41 | fi 42 | 43 | #iconv 44 | haveiconv=no 45 | AC_CHECK_LIB(iconv, iconv_open, haveiconv=yes) 46 | if test "$haveiconv" = "yes" ; then 47 | CPPFLAGS="${CPPFLAGS} -DHAVE_ICONV" 48 | LDFLAGS="${LDFLAGS} -liconv" 49 | LIBS="${LIBS} -liconv" 50 | fi 51 | 52 | # Checks for header files. 53 | AC_HEADER_DIRENT 54 | AC_HEADER_STDC 55 | AC_CHECK_HEADERS([arpa/inet.h fcntl.h locale.h netdb.h netinet/in.h stdint.h stdlib.h string.h sys/file.h sys/socket.h sys/time.h unistd.h]) 56 | 57 | # Checks for typedefs, structures, and compiler characteristics. 58 | AC_C_CONST 59 | AC_TYPE_OFF_T 60 | AC_TYPE_PID_T 61 | AC_TYPE_SIZE_T 62 | AC_HEADER_TIME 63 | AC_STRUCT_TM 64 | 65 | # Checks for library functions. 66 | AC_FUNC_FORK 67 | AC_FUNC_LSTAT 68 | AC_FUNC_LSTAT_FOLLOWS_SLASHED_SYMLINK 69 | AC_FUNC_MALLOC 70 | AC_FUNC_MEMCMP 71 | AC_FUNC_MKTIME 72 | AC_FUNC_MMAP 73 | AC_FUNC_REALLOC 74 | AC_TYPE_SIGNAL 75 | AC_FUNC_STAT 76 | AC_CHECK_FUNCS([ftruncate gethostbyname getpagesize gettimeofday inet_ntoa memset mkdir munmap setlocale strcasecmp strchr strdup strerror strncasecmp strstr]) 77 | 78 | AC_CONFIG_FILES([Makefile 79 | doc/Makefile 80 | src/Makefile]) 81 | AC_OUTPUT 82 | -------------------------------------------------------------------------------- /ispider/control: -------------------------------------------------------------------------------- 1 | Source: 2 | Section: libs 3 | Priority: extra 4 | Maintainer: SounOS 5 | Build-Depends: debhelper (>= 7.0.50~), autotools-dev 6 | Standards-Version: 7 | Homepage: http://sbase.googlecode.com/ 8 | #Vcs-Git: git://git.debian.org/collab-maint/libevbase.git 9 | #Vcs-Browser: http://git.debian.org/?p=collab-maint/libevbase.git;a=summary 10 | 11 | Package: 12 | Architecture: any 13 | Depends: ${shlibs:Depends}, ${misc:Depends}, libevbase, libsbase, libibase 14 | Description: 15 | 16 | -------------------------------------------------------------------------------- /ispider/debian.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | rm -rf debian 3 | pkg=`grep -E "^PACKAGE_NAME=" configure |sed -e "s/PACKAGE_NAME='\(.*\)'/\1/"` 4 | ver=`grep -E "^PACKAGE_VERSION=" configure |sed -e "s/PACKAGE_VERSION='\(.*\)'/\1/"` 5 | dh_make -y -n -s -c bsd -e SounOS@gmail.com -p "${pkg}_${ver}" 6 | cp control debian/control 7 | sed -i "s/^Package:.*/Package: $pkg/" debian/control 8 | sed -i "s/^Source:.*/Source: $pkg/" debian/control 9 | sed -i "s/^License:.*/License: $pkg/" debian/control 10 | sed -i "s/^Standards-Version:.*/Standards-Version: $ver/" debian/control 11 | sed -i "s/root /SounOS /" debian/* 12 | #export CFLAGS='-ggdb -O0' 13 | #export CXXFLAGS='-ggdb -O0' 14 | #export FFLAGS='-ggdb -O0' 15 | dpkg-buildpackage -rfakeroot -eSounOS -mSounOS 16 | -------------------------------------------------------------------------------- /ispider/doc/base64.php: -------------------------------------------------------------------------------- 1 | #!/usr/bin/php 2 | 22 | -------------------------------------------------------------------------------- /ispider/doc/rc.iextractor.init: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # IEXTRACTOR: 4 | # 5 | # chkconfig: 2345 90 70 6 | # description: IEXTRACTOR 7 | # 8 | # 9 | 10 | # Source function library. 11 | . /etc/rc.d/init.d/functions 12 | 13 | #[ -r /etc/sysconfig/iextractor ] && . /etc/sysconfig/iextractor 14 | IEXTRACTORARGS=" -d -c /usr/local/etc/iextractor.ini > /dev/null 2>&1 " 15 | 16 | start() 17 | { 18 | echo -n $"Starting iextractor: " 19 | daemon /usr/local/bin/iextractor $IEXTRACTORARGS 20 | 21 | echo 22 | } 23 | 24 | stop() 25 | { 26 | echo -n $"Shutting down iextractor: " 27 | killproc iextractor 28 | 29 | echo 30 | } 31 | 32 | [ -f /usr/local/bin/iextractor ] || exit 0 33 | 34 | # See how we were called. 35 | case "$1" in 36 | start) 37 | start 38 | ;; 39 | stop) 40 | stop 41 | ;; 42 | restart|reload) 43 | stop 44 | start 45 | ;; 46 | *) 47 | echo $"Usage: $0 {start|stop|restart|reload}" 48 | exit 1 49 | esac 50 | 51 | exit 0 52 | -------------------------------------------------------------------------------- /ispider/doc/rc.imonitord.init: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # IMONITORD: 4 | # 5 | # chkconfig: 2345 70 40 6 | # description: IMONITORD 7 | # 8 | # 9 | 10 | # Source function library. 11 | . /etc/rc.d/init.d/functions 12 | 13 | #[ -r /etc/sysconfig/imonitord ] && . /etc/sysconfig/imonitord 14 | IMONITORDARGS=" -d -c /usr/local/etc/imonitord.ini > /dev/null 2>&1 " 15 | 16 | start() 17 | { 18 | echo -n $"Starting imonitord: " 19 | daemon /usr/local/sbin/imonitord $IMONITORDARGS 20 | 21 | echo 22 | } 23 | 24 | stop() 25 | { 26 | echo -n $"Shutting down imonitord: " 27 | killproc imonitord 28 | 29 | echo 30 | } 31 | 32 | [ -f /usr/local/sbin/imonitord ] || exit 0 33 | 34 | # See how we were called. 35 | case "$1" in 36 | start) 37 | start 38 | ;; 39 | stop) 40 | stop 41 | ;; 42 | restart|reload) 43 | stop 44 | start 45 | ;; 46 | *) 47 | echo $"Usage: $0 {start|stop|restart|reload}" 48 | exit 1 49 | esac 50 | 51 | exit 0 52 | -------------------------------------------------------------------------------- /ispider/doc/rc.ispider.init: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # ISPIDER: 4 | # 5 | # chkconfig: 2345 90 70 6 | # description: ISPIDER 7 | # 8 | # 9 | 10 | # Source function library. 11 | . /etc/rc.d/init.d/functions 12 | 13 | #[ -r /etc/sysconfig/ispider ] && . /etc/sysconfig/ispider 14 | ISPIDERARGS=" -d -c /usr/local/etc/ispider.ini > /dev/null 2>&1 " 15 | 16 | start() 17 | { 18 | echo -n $"Starting ispider: " 19 | daemon /usr/local/bin/ispider $ISPIDERARGS 20 | 21 | echo 22 | } 23 | 24 | stop() 25 | { 26 | echo -n $"Shutting down ispider: " 27 | killproc ispider 28 | 29 | echo 30 | } 31 | 32 | [ -f /usr/local/bin/ispider ] || exit 0 33 | 34 | # See how we were called. 35 | case "$1" in 36 | start) 37 | start 38 | ;; 39 | stop) 40 | stop 41 | ;; 42 | restart|reload) 43 | stop 44 | start 45 | ;; 46 | *) 47 | echo $"Usage: $0 {start|stop|restart|reload}" 48 | exit 1 49 | esac 50 | 51 | exit 0 52 | -------------------------------------------------------------------------------- /ispider/src/Makefile.am: -------------------------------------------------------------------------------- 1 | #automake 2 | sbin_PROGRAMS = imonitord 3 | imonitord_SOURCES = xtask.h xtask.c imonitord.c iniparser.c iniparser.h base64monitordhtml.h \ 4 | utils/http.h utils/http.c utils/base64.c utils/base64.h utils/mutex.h utils/db.h utils/db.c \ 5 | utils/mmqueue.c utils/mmqueue.h utils/mmtrie.c utils/mmtrie.h utils/vmap.h utils/vmap.c \ 6 | utils/logger.h utils/logger.c utils/stime.c utils/stime.h utils/mtrie.c utils/mtrie.h \ 7 | utils/xmm.c utils/xmm.h utils/iqueue.h utils/iqueue.c utils/kmap.h utils/kmap.c \ 8 | utils/timer.h utils/evdns.h utils/evdns.c utils/zstream.h utils/zstream.c utils/rwlock.h\ 9 | utils/mtree.h utils/mtree.c utils/md5.h utils/md5.c utils/qmap.h utils/qmap.c 10 | imonitord_CPPFLAGS = -I utils/ -D_FILE_OFFSET_BITS=64 -D _HTTP_CHARSET_CONVERT -Wall 11 | imonitord_LDFLAGS = -lz -lsbase -lchardet 12 | 13 | 14 | bin_PROGRAMS = ispider iextractor 15 | 16 | ispider_SOURCES = ispider.c iniparser.h iniparser.c utils/md5.h utils/md5.c \ 17 | utils/http.h utils/http.c utils/mime.h utils/mime.c utils/zstream.h utils/zstream.c \ 18 | utils/base64.h utils/base64.c utils/mtrie.h utils/mtrie.c utils/iqueue.h utils/iqueue.c \ 19 | utils/qchardet.h utils/qchardet.c 20 | 21 | ispider_CPPFLAGS = -I utils/ -D_FILE_OFFSET_BITS=64 -Wall 22 | ispider_LDFLAGS = -lz -lsbase -libase -lchardet 23 | 24 | iextractor_SOURCES = iextractor.c iniparser.h iniparser.c utils/http.h utils/http.c \ 25 | utils/zstream.h utils/zstream.c utils/base64.h utils/base64.c utils/mtrie.h utils/mtrie.c \ 26 | utils/iqueue.h utils/iqueue.c utils/html.h utils/html.c utils/htmltags.h \ 27 | utils/charcode.h utils/url.h 28 | 29 | iextractor_CPPFLAGS = -I utils/ -D_FILE_OFFSET_BITS=64 -Wall 30 | iextractor_LDFLAGS = -lz -lsbase -libase -lpcre 31 | -------------------------------------------------------------------------------- /ispider/src/iniparser.h: -------------------------------------------------------------------------------- 1 | /* 2 | Based upon libiniparser, by Nicolas Devillard 3 | Hacked into 1 file (m-iniparser) by Freek/2005 4 | Original terms following: 5 | 6 | -- - 7 | 8 | Copyright (c) 2000 by Nicolas Devillard (ndevilla AT free DOT fr). 9 | 10 | Written by Nicolas Devillard. Not derived from licensed software. 11 | 12 | Permission is granted to anyone to use this software for any 13 | purpose on any computer system, and to redistribute it freely, 14 | subject to the following restrictions: 15 | 16 | 1. The author is not responsible for the consequences of use of 17 | this software, no matter how awful, even if they arise 18 | from defects in it. 19 | 20 | 2. The origin of this software must not be misrepresented, either 21 | by explicit claim or by omission. 22 | 23 | 3. Altered versions must be plainly marked as such, and must not 24 | be misrepresented as being the original software. 25 | 26 | 4. This notice may not be removed or altered. 27 | 28 | */ 29 | 30 | 31 | #ifndef _INIPARSER_H_ 32 | #define _INIPARSER_H_ 33 | #include 34 | #include 35 | #include 36 | #include 37 | #include 38 | 39 | #ifdef __cplusplus 40 | extern "C" { 41 | #endif 42 | 43 | 44 | typedef struct _dictionary_ { 45 | /** Number of entries in dictionary */ 46 | int n; 47 | /** Storage size */ 48 | int size; 49 | /** List of string values */ 50 | char **val; 51 | /** List of string keys */ 52 | char **key ; 53 | /** List of hash values for keys */ 54 | unsigned *hash; 55 | } dictionary ; 56 | 57 | 58 | /* generated by genproto */ 59 | 60 | dictionary * iniparser_new(char *ininame); 61 | void iniparser_free(dictionary * d); 62 | 63 | int iniparser_getnsec(dictionary * d); 64 | char * iniparser_getsecname(dictionary * d, int n); 65 | void iniparser_dump(dictionary * d, FILE * f); 66 | void iniparser_dump_ini(dictionary * d, FILE * f); 67 | char * iniparser_getkey(dictionary *d, char *section, char *key); 68 | char * iniparser_getstr(dictionary * d, char * key); 69 | char * iniparser_getstring(dictionary * d, char * key, char * def); 70 | int iniparser_getint(dictionary * d, char * key, int notfound); 71 | double iniparser_getdouble(dictionary * d, char * key, double notfound); 72 | int iniparser_getboolean(dictionary * d, char * key, int notfound); 73 | int iniparser_find_entry(dictionary * ini, char * entry); 74 | int iniparser_setstr(dictionary * ini, char * entry, char * val); 75 | void iniparser_unset(dictionary * ini, char * entry); 76 | 77 | #ifdef __cplusplus 78 | } 79 | #endif 80 | 81 | #endif 82 | 83 | -------------------------------------------------------------------------------- /ispider/src/utils/base64.h: -------------------------------------------------------------------------------- 1 | #ifndef _BASE64_H 2 | #define _BASE64_H 3 | #define BASE64_LEN(x) ((x + 2) / 3 * 4 + 1) 4 | int base64_encode(char *out, const unsigned char *in, int inlen); 5 | int base64_decode(unsigned char *out, const char *in, int inlen); 6 | #endif 7 | -------------------------------------------------------------------------------- /ispider/src/utils/crc32.h: -------------------------------------------------------------------------------- 1 | #ifndef CRC32_H 2 | #define CRC32_H 3 | unsigned int crc32(char *buf, int len); 4 | #endif 5 | -------------------------------------------------------------------------------- /ispider/src/utils/html.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #ifndef _HTML_H 5 | #define _HTML_H 6 | #define HTML_TAG_NMAX 200000 7 | #define HTML_FIELD_MAX 256 8 | #define LEVEL_MAX 10000 9 | //#define HTML_CONTENT_MAX 16777216 10 | //#define HTML_CONTENT_MAX 2097152 11 | //#define HTML_CONTENT_MAX 4194304 12 | //#define HTML_TITLE_MAX 1048576 13 | #define HTML_MBLOCK_MAX 1048576 14 | #define HTML_MTEXT_MAX 1048576 15 | #define HTML_MLINK_MAX 1048576 16 | #define HTML_TITLE_MAX 65536 17 | #define HTML_LINK_SCALE 0.50f 18 | #define HTML_LINK_FILTER 50 19 | typedef struct _HBLOCK 20 | { 21 | int npairs; 22 | int nbytes; 23 | int nlinks; 24 | int id; 25 | char *start; 26 | char *last; 27 | struct _HBLOCK *parent; 28 | }HBLOCK; 29 | typedef struct _HFIELD 30 | { 31 | int from; 32 | int len; 33 | }HFIELD; 34 | #define HTML_LINK_IMG 0x01 35 | #define HTML_LINK_URL 0x02 36 | #define HTML_LINKS_MAX 8192 37 | typedef struct _HLINK 38 | { 39 | int off; 40 | short ntitle; 41 | short nauthor; 42 | short nlink; 43 | short flag; 44 | }HLINK; 45 | typedef struct _HTML 46 | { 47 | char block[HTML_MBLOCK_MAX]; 48 | char content[HTML_MTEXT_MAX]; 49 | char link[HTML_MLINK_MAX]; 50 | char *plink; 51 | int nblock; 52 | int ncontent; 53 | int ntitle; 54 | int titleoff; 55 | int nblocks; 56 | int nfields; 57 | int nlinks; 58 | HBLOCK blocklist[HTML_TAG_NMAX]; 59 | HFIELD fieldlist[HTML_FIELD_MAX]; 60 | HLINK linklist[HTML_LINKS_MAX]; 61 | void *table; 62 | }HTML; 63 | /* Initialize HTML */ 64 | HTML *html_init(); 65 | /* get content */ 66 | int html_get_content(HTML *html, char *content, size_t len, int filter, int new_filed); 67 | /* add field */ 68 | int html_add_field(HTML *html, char *content, int ncontent); 69 | /* reset HTML */ 70 | void html_reset(HTML *); 71 | /* clean html */ 72 | void html_clean(HTML *); 73 | #endif 74 | -------------------------------------------------------------------------------- /ispider/src/utils/iqueue.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include "mutex.h" 5 | #ifndef _QUEUE_H 6 | #define _QUEUE_H 7 | #ifdef __cplusplus 8 | extern "C" { 9 | #endif 10 | typedef struct _QNODE 11 | { 12 | //void *ptr; 13 | int data; 14 | struct _QNODE *next; 15 | }QNODE; 16 | #define QNODE_LINE_MAX 10240 17 | #define QNODE_LINE_NUM 1024 18 | typedef struct _QUEUE 19 | { 20 | int total; 21 | int nlist; 22 | QNODE *list[QNODE_LINE_MAX]; 23 | QNODE *left; 24 | QNODE *first; 25 | QNODE *last; 26 | MUTEX *mutex; 27 | }QUEUE; 28 | void *iqueue_init(); 29 | void iqueue_push(void *q, int data); 30 | void iqueue_pop(void *q, int *data); 31 | void iqueue_head(void *q, int *data); 32 | void iqueue_clean(void *q); 33 | #define QTOTAL(q) (((QUEUE *)q)->total) 34 | #ifdef __cplusplus 35 | } 36 | #endif 37 | #endif 38 | -------------------------------------------------------------------------------- /ispider/src/utils/kmap.h: -------------------------------------------------------------------------------- 1 | #ifndef _KMAP_H 2 | #define _KMAP_H 3 | #include "mutex.h" 4 | #define KMAP_INCRE_NUM 100000 5 | #define KMAP_NODES_MAX 2000000000 6 | #define KMROOT_INCRE_NUM 10000 7 | #define KMAP_ROOT_MAX 2000000000 8 | typedef struct _KMNODE 9 | { 10 | unsigned int left; 11 | unsigned int right; 12 | unsigned int parent; 13 | unsigned int color; 14 | int data; 15 | int bits; 16 | int64_t key; 17 | }KMNODE; 18 | typedef struct _KMROOT 19 | { 20 | int status; 21 | unsigned int total; 22 | unsigned int rootid; 23 | }KMROOT; 24 | typedef struct _KMSTATE 25 | { 26 | int64_t kmax; 27 | int64_t kmin; 28 | unsigned int nmax; 29 | unsigned int nmin; 30 | unsigned int count; 31 | unsigned int left; 32 | unsigned int current; 33 | unsigned int total; 34 | unsigned int qleft; 35 | unsigned int qfirst; 36 | unsigned int rleft; 37 | unsigned int rcurrent; 38 | unsigned int rtotal; 39 | unsigned int qlast; 40 | unsigned int qrleft; 41 | unsigned int qrfirst; 42 | unsigned int qrlast; 43 | int bits; 44 | }KMSTATE; 45 | typedef struct _KMAP 46 | { 47 | int fd; 48 | int rfd; 49 | off_t size; 50 | off_t end; 51 | off_t old; 52 | off_t rsize; 53 | off_t rend; 54 | off_t rold; 55 | void *start; 56 | KMSTATE *state; 57 | KMNODE *map; 58 | KMROOT *roots; 59 | MUTEX *mutex; 60 | }KMAP; 61 | void *kmap_init(char *basedir); 62 | int kmap_new_tree(void *kmap); 63 | unsigned int kmap_total(void *kmap, int rootid); 64 | unsigned int kmap_try_insert(void *kmap, int rootid, int64_t key, int data, int *old); 65 | unsigned int kmap_insert(void *kmap, int rootid, int64_t key, int data, int *old); 66 | unsigned int kmap_get(void *kmap, unsigned int nodeid, int64_t *key, int *data); 67 | unsigned int kmap_find(void *kmap, int rootid, int64_t key, int *data); 68 | unsigned int kmap_min(void *kmap, int rootid, int64_t *key, int *data); 69 | unsigned int kmap_max(void *kmap, int rootid, int64_t *key, int *data); 70 | unsigned int kmap_next(void *kmap, int rootid, unsigned int nodeid, int64_t *key, int *data); 71 | unsigned int kmap_prev(void *kmap, int rootid, unsigned int nodeid, int64_t *key, int *data); 72 | int kmap_set_data(void *kmap, unsigned int nodeid, int data); 73 | void kmap_view_tree(void *kmap, int rootid, FILE *fp); 74 | void kmap_remove(void *kmap, int rootid, unsigned int nodeid, int64_t *key, int *data); 75 | void kmap_remove_tree(void *kmap, int rootid); 76 | void kmap_close(void *kmap); 77 | #endif 78 | -------------------------------------------------------------------------------- /ispider/src/utils/md5.h: -------------------------------------------------------------------------------- 1 | #include 2 | #ifndef _MD5_H 3 | #define _MD5_H 4 | #ifdef __cplusplus 5 | extern "C" { 6 | #endif 7 | #define MD5_LEN 16 8 | #define _MD5_BLOCK_N 64 9 | #define _MD5_SET_N 56 10 | #define _MD5_BITS_N 8 11 | #ifndef _TYPEDEF_MD5_CTX 12 | #define _TYPEDEF_MD5_CTX 13 | typedef struct _MD5_CTX 14 | { 15 | u_int32_t state[4]; 16 | u_int32_t total[2]; 17 | unsigned char digest[MD5_LEN]; 18 | unsigned char buf[_MD5_BLOCK_N]; 19 | }MD5_CTX; 20 | /* Initialize */ 21 | void md5_init(MD5_CTX *context); 22 | /* Update */ 23 | void md5_update(MD5_CTX *context, unsigned char *data, u_int32_t ndata); 24 | /* Final */ 25 | void md5_final(MD5_CTX *context); 26 | void md5(unsigned char *data, u_int32_t ndata, unsigned char *digest); 27 | /* Cacalute FILE md5 */ 28 | int md5_file(const char *file, unsigned char *digest); 29 | #endif 30 | #define MD5(_data, _ndata, md) { \ 31 | MD5_CTX ctx; \ 32 | md5_init(&ctx); \ 33 | md5_update(&ctx, _data, _ndata); \ 34 | md5_final(&ctx); \ 35 | memcpy(md, ctx.digest, MD5_LEN); \ 36 | } 37 | #define MD5OUT(md, out) \ 38 | { \ 39 | int i = 0; \ 40 | do{ \ 41 | fprintf(out, "%02x", md[i++]); \ 42 | }while(i < MD5_LEN); \ 43 | } 44 | #ifdef __cplusplus 45 | } 46 | #endif 47 | #endif 48 | -------------------------------------------------------------------------------- /ispider/src/utils/mime.h: -------------------------------------------------------------------------------- 1 | #ifndef _MIME_H 2 | #define _MIME_H 3 | #define MIME_NUM_MAX 1024 4 | typedef struct _MIME_MAP 5 | { 6 | void *map; 7 | int num; 8 | int bits; 9 | }MIME_MAP; 10 | /* initialize mime */ 11 | int mime_map_init(MIME_MAP *mime_map); 12 | /* add mime */ 13 | int mime_add(MIME_MAP *mime_map, char *mime, int len); 14 | /* add mime line */ 15 | int mime_add_line(MIME_MAP *mime_map, char *p, char *end); 16 | /* return mime id*/ 17 | int mime_id(MIME_MAP *mime_map, char *mime, int len); 18 | /* clean mime map*/ 19 | void mime_map_clean(MIME_MAP *mime_map); 20 | #endif 21 | -------------------------------------------------------------------------------- /ispider/src/utils/mmqueue.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include "mutex.h" 5 | #ifndef _MMQUEUE64_H 6 | #define _MMQUEUE64_H 7 | #ifdef __cplusplus 8 | extern "C" { 9 | #endif 10 | typedef struct _MMQODE 11 | { 12 | //void *ptr; 13 | unsigned int data; 14 | unsigned int next; 15 | }MMQNODE; 16 | typedef struct _MMQROOT 17 | { 18 | int status; 19 | int total; 20 | int first; 21 | int last; 22 | }MMQROOT; 23 | #define MMQ_INCRE_NUM 1000000 24 | #define MMQ_NODE_MAX 100000000 25 | #define MMQ_ROOT_MAX 5000000 26 | typedef struct _MMQSTATE 27 | { 28 | int qtotal; 29 | int qleft; 30 | int nroots; 31 | MMQROOT roots[MMQ_ROOT_MAX]; 32 | }MMQSTATE; 33 | typedef struct _MMQUEUE 34 | { 35 | int fd; 36 | int bits; 37 | off_t end; 38 | off_t size; 39 | off_t old; 40 | void *map; 41 | MUTEX *mutex; 42 | MMQSTATE *state; 43 | MMQNODE *nodes; 44 | }MMQUEUE; 45 | MMQUEUE *mmqueue_init(char *qfile); 46 | int mmqueue_new(MMQUEUE *mmq); 47 | int mmqueue_total(MMQUEUE *mmq, int rootid); 48 | int mmqueue_close(MMQUEUE *mmq, int rootid); 49 | int mmqueue_push(MMQUEUE *mmq, int rootid, int data); 50 | int mmqueue_pop(MMQUEUE *mmq, int rootid, int *data); 51 | int mmqueue_head(MMQUEUE *mmq, int rootid, int *data); 52 | void mmqueue_clean(MMQUEUE *mmq); 53 | #define MMQ(x) ((MMQUEUE*)x) 54 | #ifdef __cplusplus 55 | } 56 | #endif 57 | #endif 58 | -------------------------------------------------------------------------------- /ispider/src/utils/qchardet.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include "qchardet.h" 5 | #include "mutex.h" 6 | void qchardet_init(QCHARDET *q) 7 | { 8 | if(q) 9 | { 10 | memset(q, 0, sizeof(QCHARDET)); 11 | MUTEX_INIT(q->mutex); 12 | } 13 | return ; 14 | } 15 | 16 | chardet_t qchardet_pop(QCHARDET *q) 17 | { 18 | chardet_t pdet = NULL; 19 | 20 | if(q) 21 | { 22 | MUTEX_LOCK(q->mutex); 23 | if(q->nqpool > 0) 24 | { 25 | pdet = q->qpool[--(q->nqpool)]; 26 | //fprintf(stdout, "%s::%d pdet:%p npool:%d\r\n", __FILE__, __LINE__, pdet, q->nqpool); 27 | } 28 | else 29 | { 30 | chardet_create(&pdet); 31 | //fprintf(stdout, "%s::%d pdet:%p\r\n", __FILE__, __LINE__, pdet); 32 | } 33 | MUTEX_UNLOCK(q->mutex); 34 | } 35 | return pdet; 36 | } 37 | 38 | void qchardet_push(QCHARDET *q, chardet_t pdet) 39 | { 40 | if(q) 41 | { 42 | MUTEX_LOCK(q->mutex); 43 | if(q->nqpool < Q_CHARDET_MAX) 44 | { 45 | chardet_reset(pdet); 46 | q->qpool[(q->nqpool)++] = pdet; 47 | } 48 | else 49 | { 50 | chardet_destroy(pdet); 51 | } 52 | MUTEX_UNLOCK(q->mutex); 53 | } 54 | return ; 55 | } 56 | 57 | void qchardet_close(QCHARDET *q) 58 | { 59 | int i = 0; 60 | 61 | if(q) 62 | { 63 | MUTEX_LOCK(q->mutex); 64 | for(i = 0; i < q->nqpool; i++) 65 | { 66 | chardet_destroy(q->qpool[i]); 67 | } 68 | MUTEX_UNLOCK(q->mutex); 69 | MUTEX_DESTROY(q->mutex); 70 | } 71 | return ; 72 | } 73 | -------------------------------------------------------------------------------- /ispider/src/utils/qchardet.h: -------------------------------------------------------------------------------- 1 | #include 2 | #ifndef __QCHARDET_H__ 3 | #define __QCHARDET_H__ 4 | #define Q_CHARDET_MAX 4096 5 | typedef struct _QCHARDET 6 | { 7 | void *mutex; 8 | chardet_t qpool[Q_CHARDET_MAX]; 9 | int nqpool; 10 | int bits; 11 | }QCHARDET; 12 | void qchardet_init(QCHARDET *q); 13 | chardet_t qchardet_pop(QCHARDET *q); 14 | void qchardet_push(QCHARDET *q, chardet_t pdet); 15 | void qchardet_close(QCHARDET *q); 16 | #endif 17 | -------------------------------------------------------------------------------- /ispider/src/utils/qmap.h: -------------------------------------------------------------------------------- 1 | #ifndef _QMAP_H 2 | #define _QMAP_H 3 | #include "mutex.h" 4 | #define QMAP_INCRE_NUM 100000 5 | #define QMAP_NODES_MAX 2000000000 6 | #define QMROOT_INCRE_NUM 10000 7 | #define QMAP_ROOT_MAX 2000000000 8 | typedef struct _QMNODE 9 | { 10 | unsigned int left; 11 | unsigned int right; 12 | unsigned int parent; 13 | unsigned int color; 14 | int data; 15 | int key; 16 | }QMNODE; 17 | typedef struct _QMROOT 18 | { 19 | int status; 20 | unsigned int total; 21 | unsigned int rootid; 22 | }QMROOT; 23 | typedef struct _QMSTATE 24 | { 25 | int kmax; 26 | int kmin; 27 | unsigned int nmax; 28 | unsigned int nmin; 29 | unsigned int count; 30 | unsigned int left; 31 | unsigned int current; 32 | unsigned int total; 33 | unsigned int qleft; 34 | unsigned int qfirst; 35 | unsigned int rleft; 36 | unsigned int rcurrent; 37 | unsigned int rtotal; 38 | unsigned int qlast; 39 | unsigned int qrleft; 40 | unsigned int qrfirst; 41 | unsigned int qrlast; 42 | int bits; 43 | }QMSTATE; 44 | typedef struct _QMAP 45 | { 46 | int fd; 47 | int rfd; 48 | off_t size; 49 | off_t end; 50 | off_t old; 51 | off_t rsize; 52 | off_t rend; 53 | off_t rold; 54 | void *start; 55 | QMSTATE *state; 56 | QMNODE *map; 57 | QMROOT *roots; 58 | MUTEX *mutex; 59 | }QMAP; 60 | void *qmap_init(char *basedir); 61 | int qmap_new_tree(void *qmap); 62 | unsigned int qmap_total(void *qmap, int rootid); 63 | unsigned int qmap_try_insert(void *qmap, int rootid, int key, int data, int *old); 64 | unsigned int qmap_insert(void *qmap, int rootid, int key, int data, int *old); 65 | unsigned int qmap_get(void *qmap, unsigned int nodeid, int *key, int *data); 66 | unsigned int qmap_find(void *qmap, int rootid, int key, int *data); 67 | unsigned int qmap_min(void *qmap, int rootid, int *key, int *data); 68 | unsigned int qmap_max(void *qmap, int rootid, int *key, int *data); 69 | unsigned int qmap_next(void *qmap, int rootid, unsigned int nodeid, int *key, int *data); 70 | unsigned int qmap_prev(void *qmap, int rootid, unsigned int nodeid, int *key, int *data); 71 | int qmap_set_data(void *qmap, unsigned int nodeid, int data); 72 | void qmap_view_tree(void *qmap, int rootid, FILE *fp); 73 | void qmap_remove(void *qmap, int rootid, unsigned int nodeid, int *key, int *data); 74 | void qmap_remove_tree(void *qmap, int rootid); 75 | void qmap_close(void *qmap); 76 | #endif 77 | -------------------------------------------------------------------------------- /ispider/src/utils/rwlock.h: -------------------------------------------------------------------------------- 1 | #ifndef __RWLOCK__H 2 | #define __RWLOCK__H 3 | #ifdef HAVE_PTHREAD 4 | #define RWLOCK_INIT(x) pthread_rwlock_init(&(x), NULL) 5 | #define RWLOCK_RDLOCK(x) pthread_rwlock_rdlock(&(x)) 6 | #define RWLOCK_WRLOCK(x) pthread_rwlock_wrlock(&(x)) 7 | #define RWLOCK_UNLOCK(x) pthread_rwlock_unlock(&(x)) 8 | #define RWLOCK_DESTROY(x) pthread_rwlock_destroy(&(x)) 9 | #else 10 | #define RWLOCK_INIT(x) 11 | #define RWLOCK_RDLOCK(x) 12 | #define RWLOCK_WRLOCK(x) 13 | #define RWLOCK_UNLOCK(x) 14 | #define RWLOCK_DESTROY(x) 15 | #endif 16 | #endif 17 | -------------------------------------------------------------------------------- /ispider/src/utils/stime.h: -------------------------------------------------------------------------------- 1 | #ifndef _STIME_H 2 | #define _STIME_H 3 | #include 4 | #include 5 | //convert str datetime to time 6 | time_t str2time(char *datestr); 7 | /* time to GMT */ 8 | int GMTstrdate(time_t time, char *date); 9 | /* strdate */ 10 | int strdate(time_t time, char *date); 11 | /* local date time */ 12 | int datetime(time_t times, char *date); 13 | /* timetospec */ 14 | void timetospec(struct timespec *ts, int usecs); 15 | #endif 16 | -------------------------------------------------------------------------------- /ispider/src/utils/vmap.h: -------------------------------------------------------------------------------- 1 | #ifndef _VMAP_H 2 | #define _VMAP_H 3 | #include "mutex.h" 4 | #define VMAP_INCRE_NUM 100000 5 | #define VMAP_NODES_MAX 2000000000 6 | #define VMROOT_INCRE_NUM 10000 7 | #define VMAP_ROOT_MAX 2000000000 8 | typedef struct _VMNODE 9 | { 10 | unsigned int left; 11 | unsigned int right; 12 | unsigned int parent; 13 | unsigned int color; 14 | int data; 15 | int key; 16 | }VMNODE; 17 | typedef struct _VMROOT 18 | { 19 | int status; 20 | unsigned int total; 21 | unsigned int rootid; 22 | }VMROOT; 23 | typedef struct _VMSTATE 24 | { 25 | int kmax; 26 | int kmin; 27 | unsigned int nmax; 28 | unsigned int nmin; 29 | unsigned int count; 30 | unsigned int left; 31 | unsigned int current; 32 | unsigned int total; 33 | unsigned int qleft; 34 | unsigned int qfirst; 35 | unsigned int rleft; 36 | unsigned int rcurrent; 37 | unsigned int rtotal; 38 | unsigned int qlast; 39 | unsigned int qrleft; 40 | unsigned int qrfirst; 41 | unsigned int qrlast; 42 | int bits; 43 | }VMSTATE; 44 | typedef struct _VMAP 45 | { 46 | int fd; 47 | int rfd; 48 | off_t size; 49 | off_t end; 50 | off_t old; 51 | off_t rsize; 52 | off_t rend; 53 | off_t rold; 54 | void *start; 55 | VMSTATE *state; 56 | VMNODE *map; 57 | VMROOT *roots; 58 | MUTEX *mutex; 59 | }VMAP; 60 | void *vmap_init(char *basedir); 61 | int vmap_new_tree(void *vmap); 62 | unsigned int vmap_total(void *vmap, int rootid); 63 | unsigned int vmap_try_insert(void *vmap, int rootid, int key, int data, int *old); 64 | unsigned int vmap_insert(void *vmap, int rootid, int key, int data, int *old); 65 | unsigned int vmap_get(void *vmap, unsigned int nodeid, int *key, int *data); 66 | unsigned int vmap_find(void *vmap, int rootid, int key, int *data); 67 | unsigned int vmap_min(void *vmap, int rootid, int *key, int *data); 68 | unsigned int vmap_max(void *vmap, int rootid, int *key, int *data); 69 | unsigned int vmap_next(void *vmap, int rootid, unsigned int nodeid, int *key, int *data); 70 | unsigned int vmap_prev(void *vmap, int rootid, unsigned int nodeid, int *key, int *data); 71 | int vmap_set_data(void *vmap, unsigned int nodeid, int data); 72 | void vmap_view_tree(void *vmap, int rootid, FILE *fp); 73 | void vmap_remove(void *vmap, int rootid, unsigned int nodeid, int *key, int *data); 74 | void vmap_remove_tree(void *vmap, int rootid); 75 | void vmap_close(void *vmap); 76 | #endif 77 | -------------------------------------------------------------------------------- /ispider/src/utils/xmm.h: -------------------------------------------------------------------------------- 1 | #ifndef _XMM_H_ 2 | #define _XMM_H_ 3 | void* xmm_mnew(size_t size); 4 | void* xmm_new(size_t size); 5 | void* xmm_mrenew(void *old, size_t old_size, size_t new_size); 6 | void* xmm_renew(void *old, size_t old_size, size_t new_size); 7 | void* xmm_mresize(void *old, size_t old_size, size_t new_size); 8 | void* xmm_resize(void *old, size_t old_size, size_t new_size); 9 | void xmm_free(void *m, size_t size); 10 | #endif 11 | -------------------------------------------------------------------------------- /ispider/src/utils/zstream.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #ifndef _ZSTREAM_H 6 | #define _ZSTREAM_H 7 | #define Z_HEADER_SIZE 12 8 | /* Compress data */ 9 | int zcompress(Bytef *data, uLong ndata, 10 | Bytef *zdata, uLong *nzdata); 11 | /* Uncompress data */ 12 | int zdecompress(Bytef *zdata, uLong nzdata, 13 | Bytef *data, uLong *ndata); 14 | /* gzip Compress data */ 15 | int gzcompress(Bytef *data, uLong ndata, 16 | Bytef *zdata, uLong *nzdata); 17 | /* gzip Uncompress data */ 18 | int gzdecompress(Bytef *zdata, uLong nzdata, 19 | Bytef *data, uLong *ndata); 20 | /* http gzip Uncompress data */ 21 | int httpgzdecompress(Bytef *zdata, uLong nzdata, 22 | Bytef *data, uLong *ndata); 23 | 24 | #endif 25 | -------------------------------------------------------------------------------- /ithunder/AUTHORS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sounos/ithunder/241d17ce44ed90bc3d8ce4f0b8d5ae52498db270/ithunder/AUTHORS -------------------------------------------------------------------------------- /ithunder/ChangeLog: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sounos/ithunder/241d17ce44ed90bc3d8ce4f0b8d5ae52498db270/ithunder/ChangeLog -------------------------------------------------------------------------------- /ithunder/Makefile.am: -------------------------------------------------------------------------------- 1 | SUBDIRS = doc src 2 | EXTRA_DIST = debian.sh control 3 | -------------------------------------------------------------------------------- /ithunder/NEWS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sounos/ithunder/241d17ce44ed90bc3d8ce4f0b8d5ae52498db270/ithunder/NEWS -------------------------------------------------------------------------------- /ithunder/README: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sounos/ithunder/241d17ce44ed90bc3d8ce4f0b8d5ae52498db270/ithunder/README -------------------------------------------------------------------------------- /ithunder/configure.in: -------------------------------------------------------------------------------- 1 | # -*- Autoconf -*- 2 | # Process this file with autoconf to produce a configure script. 3 | 4 | AC_PREREQ(2.59) 5 | AC_INIT(ithunder, 0.0.4, [SounOS@gmail.com]) 6 | AC_CONFIG_SRCDIR([config.h.in]) 7 | AC_CONFIG_HEADER([config.h]) 8 | AM_INIT_AUTOMAKE 9 | 10 | # Checks for programs. 11 | AC_PROG_CC 12 | AM_PROG_CC_C_O 13 | AC_PROG_INSTALL 14 | 15 | # Checks for libraries. 16 | # FIXME: Replace `main' with a function in `-ldbase': 17 | AC_CHECK_LIB([dbase], [dbase_int]) 18 | # FIXME: Replace `main' with a function in `-lmtask': 19 | AC_CHECK_LIB([mtask], [mtask_init]) 20 | # FIXME: Replace `main' with a function in `-lsbase': 21 | AC_CHECK_LIB([sbase], [sbase_init]) 22 | # FIXME: Replace `main' with a function in `-lz': 23 | AC_CHECK_LIB([z], [deflateInit2]) 24 | 25 | # Checks for header files. 26 | AC_HEADER_DIRENT 27 | AC_HEADER_STDC 28 | AC_CHECK_HEADERS([arpa/inet.h fcntl.h locale.h netdb.h stdint.h stdlib.h string.h sys/file.h sys/time.h unistd.h]) 29 | 30 | # Checks for typedefs, structures, and compiler characteristics. 31 | AC_C_CONST 32 | AC_TYPE_OFF_T 33 | AC_TYPE_PID_T 34 | AC_TYPE_SIZE_T 35 | AC_HEADER_TIME 36 | AC_STRUCT_TM 37 | 38 | #pthread 39 | havepthread=no 40 | AC_CHECK_LIB(pthread,pthread_create,havepthread=yes) 41 | if test "$havepthread" = "yes" ; then 42 | CPPFLAGS="${CPPFLAGS} -DHAVE_PTHREAD" 43 | LDFLAGS="${LDFLAGS} -lpthread" 44 | LIBS="${LIBS} -lpthread" 45 | fi 46 | #semaphore 47 | #havesemaphore=no 48 | #AC_CHECK_LIB(c, sem_timedwait, havesemaphore=yes) 49 | #if test "$havesemaphore" = "yes" ; then 50 | # CPPFLAGS="${CPPFLAGS} -DHAVE_SEMAPHORE" 51 | #fi 52 | 53 | #scws 54 | havescws=no 55 | AC_CHECK_LIB(scws,scws_get_result,havescws=yes) 56 | if test "$havescws" = "yes" ; then 57 | CPPFLAGS="${CPPFLAGS} -DHAVE_SCWS" 58 | LDFLAGS="${LDFLAGS} -lscws" 59 | LIBS="${LIBS} -lscws" 60 | fi 61 | #mmap 62 | havemmap=no 63 | AC_CHECK_LIB(c, munmap, havemmap=yes) 64 | if test "$havemmap" = "yes" ; then 65 | CPPFLAGS="${CPPFLAGS} -DHAVE_MMAP" 66 | fi 67 | 68 | 69 | # Checks for library functions. 70 | AC_FUNC_FORK 71 | AC_FUNC_LSTAT 72 | AC_FUNC_LSTAT_FOLLOWS_SLASHED_SYMLINK 73 | AC_FUNC_MALLOC 74 | AC_FUNC_MEMCMP 75 | AC_FUNC_MKTIME 76 | AC_FUNC_MMAP 77 | AC_FUNC_REALLOC 78 | AC_TYPE_SIGNAL 79 | AC_FUNC_STAT 80 | AC_CHECK_FUNCS([ftruncate gettimeofday memset mkdir munmap setlocale strcasecmp strchr strdup strerror strncasecmp]) 81 | 82 | AC_CONFIG_FILES([Makefile 83 | doc/Makefile 84 | src/Makefile]) 85 | AC_OUTPUT 86 | -------------------------------------------------------------------------------- /ithunder/control: -------------------------------------------------------------------------------- 1 | Source: 2 | Section: libs 3 | Priority: extra 4 | Maintainer: SounOS 5 | Build-Depends: debhelper (>= 7.0.50~), autotools-dev 6 | Standards-Version: 7 | Homepage: http://sbase.googlecode.com/ 8 | #Vcs-Git: git://git.debian.org/collab-maint/libevbase.git 9 | #Vcs-Browser: http://git.debian.org/?p=collab-maint/libevbase.git;a=summary 10 | 11 | Package: 12 | Architecture: any 13 | Depends: ${shlibs:Depends}, ${misc:Depends}, libevbase, libsbase, libibase, hidbase, qmtask 14 | Description: 15 | 16 | -------------------------------------------------------------------------------- /ithunder/debian.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | rm -rf debian 3 | pkg=`grep -E "^PACKAGE_NAME=" configure |sed -e "s/PACKAGE_NAME='\(.*\)'/\1/"` 4 | ver=`grep -E "^PACKAGE_VERSION=" configure |sed -e "s/PACKAGE_VERSION='\(.*\)'/\1/"` 5 | dh_make -y -n -s -c bsd -e SounOS@gmail.com -p "${pkg}_${ver}" 6 | cp control debian/control 7 | sed -i "s/^Package:.*/Package: $pkg/" debian/control 8 | sed -i "s/^Source:.*/Source: $pkg/" debian/control 9 | sed -i "s/^License:.*/License: $pkg/" debian/control 10 | sed -i "s/^Standards-Version:.*/Standards-Version: $ver/" debian/control 11 | sed -i "s/root /SounOS /" debian/* 12 | #export CFLAGS='-ggdb -O0' 13 | #export CXXFLAGS='-ggdb -O0' 14 | #export FFLAGS='-ggdb -O0' 15 | dpkg-buildpackage -rfakeroot -eSounOS -mSounOS 16 | -------------------------------------------------------------------------------- /ithunder/doc/Makefile.am: -------------------------------------------------------------------------------- 1 | #automake 2 | EXTRA_DIST = rc.ithinkd.ini rc.ithinkd.html rc.ithinkd.init base64.php \ 3 | rc.idispatchd.ini rc.idispatchd.html rc.idispatchd.init ithunder.spec 4 | 5 | install: install-ini 6 | 7 | conftodir = "`echo $(sysconfdir)|sed -e 's|/usr/etc|/etc|'`" 8 | 9 | initdir = "$(DESTDIR)$(conftodir)/init.d" 10 | 11 | sysconftodir = "$(DESTDIR)$(conftodir)" 12 | 13 | htmldir = "`echo @prefix@"/var/html"|sed -e 's|/usr/var|/var|g'`" 14 | htmltodir = "$(DESTDIR)$(htmldir)" 15 | 16 | install-ini: 17 | @echo install *.html to $(htmldir) 18 | @test -d $(htmltodir)/think || mkdir -p $(htmltodir)/think 19 | @cp -p rc.ithinkd.html $(htmltodir)/think/index.html 20 | @test -d $(htmltodir)/dispatch || mkdir -p $(htmltodir)/dispatch 21 | @cp -p rc.idispatchd.html $(htmltodir)/dispatch/index.html 22 | @echo install *.ini to $(sysconftodir) 23 | @test -d $(sysconftodir) || mkdir -p $(sysconftodir) 24 | @test -f $(sysconftodir)/ithinkd.ini \ 25 | || cp -p rc.ithinkd.ini $(sysconftodir)/ithinkd.ini 26 | @test -f $(sysconftodir)/idispatchd.ini \ 27 | || cp -p rc.idispatchd.ini $(sysconftodir)/idispatchd.ini 28 | @perl -i -p -e "s|\"/var/html|\"$(htmldir)|g" \ 29 | $(sysconftodir)/ithinkd.ini \ 30 | $(sysconftodir)/idispatchd.ini 31 | @echo install *.init to $(initdir) 32 | @test -d $(initdir) || mkdir -p $(initdir) 33 | @test -f $(initdir)/ithinkd \ 34 | || cp -p rc.ithinkd.init $(initdir)/ithinkd 35 | @test -f $(initdir)/idispatchd \ 36 | || cp -p rc.idispatchd.init $(initdir)/idispatchd 37 | @perl -i -p -e "s@/usr/local/sbin@$(prefix)/sbin@g" $(initdir)/ithinkd \ 38 | $(initdir)/idispatchd 39 | @perl -i -p -e "s@/usr/local/etc@$(conftodir)@g" $(initdir)/ithinkd \ 40 | $(initdir)/idispatchd 41 | -------------------------------------------------------------------------------- /ithunder/doc/base64.php: -------------------------------------------------------------------------------- 1 | #!/usr/bin/php 2 | 22 | -------------------------------------------------------------------------------- /ithunder/doc/ithunder.spec: -------------------------------------------------------------------------------- 1 | # Authority: SounOS.org 2 | 3 | Summary: IndexWorker based-on qmtask & libibase 4 | Name: ithunder 5 | Version: 0.0.4 6 | Release: 6%{?dist} 7 | License: BSD 8 | Group: System Environment/Libraries 9 | URL: http://code.google.com/p/libibase/ 10 | 11 | Source: http://code.google.com/p/libibase/download/%{name}-%{version}.tar.gz 12 | Packager: SounOS 13 | Vendor: SounOS 14 | BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root 15 | 16 | BuildRequires: libevbase >= 1.0.0 libsbase >= 1.0.0 libibase >= 0.5.15 libscws >= 1.1.1 libdbase >= 0.0.3 libmtask >= 0.0.5 17 | Requires: libevbase >= 1.0.0 libsbase >= 1.0.0 libibase >= 0.5.15 libscws >= 1.1.1 libdbase >= 0.0.3 libmtask >= 0.0.5 18 | 19 | %description 20 | IndexWorker based-on qmtask & libibase 21 | 22 | %prep 23 | %setup 24 | 25 | %build 26 | %configure 27 | %{__make} 28 | 29 | %install 30 | %{__rm} -rf %{buildroot} 31 | %{__make} install DESTDIR="%{buildroot}" 32 | 33 | %clean 34 | %{__rm} -rf %{buildroot} 35 | 36 | %post 37 | 38 | /sbin/chkconfig --level 345 ithinkd off 39 | /sbin/chkconfig --level 345 idispatchd off 40 | 41 | %preun 42 | [ "`pstree|grep ithinkd|wc -l`" -gt "0" ] && /sbin/service ithinkd stop 43 | [ "`pstree|grep idispatchd|wc -l`" -gt "0" ] && /sbin/service idispatchd stop 44 | /sbin/chkconfig --del ithinkd 45 | /sbin/chkconfig --del idispatchd 46 | 47 | %files 48 | %defattr(-, root, root, 0755) 49 | %{_bindir}/* 50 | %{_sbindir}/* 51 | %{_localstatedir}/* 52 | %{_sysconfdir}/rc.d/* 53 | %config(noreplace) %{_sysconfdir}/*.ini 54 | 55 | %changelog 56 | * Wed Jun 8 2011 16:53:13 CST SounOS 57 | - added kvmap.* zvbcode.* 58 | - added rename packet name to iworker 59 | -------------------------------------------------------------------------------- /ithunder/doc/rc.idispatchd.init: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # IDISPATCHD: 4 | # 5 | # chkconfig: 2345 70 40 6 | # description: IDISPATCHD 7 | # 8 | # 9 | 10 | # Source function library. 11 | . /etc/rc.d/init.d/functions 12 | 13 | #[ -r /etc/sysconfig/idispatchd ] && . /etc/sysconfig/idispatchd 14 | IDISPATCHDARGS=" -d -c /usr/local/etc/idispatchd.ini > /dev/null 2>&1 " 15 | 16 | start() 17 | { 18 | echo -n $"Starting idispatchd: " 19 | daemon /usr/local/sbin/idispatchd $IDISPATCHDARGS 20 | 21 | echo 22 | } 23 | 24 | stop() 25 | { 26 | echo -n $"Shutting down idispatchd: " 27 | killproc idispatchd 28 | 29 | echo 30 | } 31 | 32 | [ -f /usr/local/sbin/idispatchd ] || exit 0 33 | 34 | # See how we were called. 35 | case "$1" in 36 | start) 37 | start 38 | ;; 39 | stop) 40 | stop 41 | ;; 42 | restart|reload) 43 | stop 44 | start 45 | ;; 46 | *) 47 | echo $"Usage: $0 {start|stop|restart|reload}" 48 | exit 1 49 | esac 50 | 51 | exit 0 52 | -------------------------------------------------------------------------------- /ithunder/doc/rc.ithinkd.init: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # ITHINKD: 4 | # 5 | # chkconfig: 2345 70 40 6 | # description: ITHINKD 7 | # 8 | # 9 | 10 | # Source function library. 11 | . /etc/rc.d/init.d/functions 12 | 13 | #[ -r /etc/sysconfig/ithinkd ] && . /etc/sysconfig/ithinkd 14 | ITHINKDARGS=" -d -c /usr/local/etc/ithinkd.ini > /dev/null 2>&1 " 15 | 16 | start() 17 | { 18 | echo -n $"Starting ithinkd: " 19 | daemon /usr/local/sbin/ithinkd $ITHINKDARGS 20 | 21 | echo 22 | } 23 | 24 | stop() 25 | { 26 | echo -n $"Shutting down ithinkd: " 27 | killproc ithinkd 28 | 29 | echo 30 | } 31 | 32 | [ -f /usr/local/sbin/ithinkd ] || exit 0 33 | 34 | # See how we were called. 35 | case "$1" in 36 | start) 37 | start 38 | ;; 39 | stop) 40 | stop 41 | ;; 42 | restart|reload) 43 | stop 44 | start 45 | echo $"Usage: $0 {start|stop|restart|reload}" 46 | exit 1 47 | esac 48 | 49 | exit 0 50 | -------------------------------------------------------------------------------- /ithunder/src/Makefile.am: -------------------------------------------------------------------------------- 1 | #automake 2 | sbin_PROGRAMS = ithinkd idispatchd 3 | ithinkd_SOURCES = kindex.h kindex.c ithinkd.c iniparser.c iniparser.h base64thinkdhtml.h \ 4 | utils/http.h utils/http.c utils/base64.c utils/base64.h utils/logger.h utils/logger.c \ 5 | utils/db.h utils/db.c utils/mmqueue.c utils/mmqueue.h utils/mmtrie.c utils/mmtrie.h \ 6 | utils/mmtree.h utils/mmtree.c utils/stime.c utils/stime.h utils/mtrie.c utils/mtrie.h \ 7 | utils/xmm.c utils/xmm.h utils/mtree.h utils/mtree.c utils/zvbcode.h utils/zvbcode.c \ 8 | utils/iqueue.h utils/iqueue.c utils/timer.h utils/mutex.h 9 | ithinkd_CPPFLAGS = -I utils/ -D_FILE_OFFSET_BITS=64 -Wall 10 | ithinkd_LDFLAGS = -lz -lsbase -lmtask -ldbase 11 | 12 | idispatchd_SOURCES = qindex.h qindex.c idispatchd.c iniparser.c iniparser.h base64dispatchdhtml.h \ 13 | utils/http.h utils/http.c utils/base64.c utils/base64.h utils/mutex.h utils/db.h utils/db.c \ 14 | utils/mmqueue.c utils/mmqueue.h utils/mmtrie.c utils/mmtrie.h utils/mmtree.h utils/mmtree.c \ 15 | utils/logger.h utils/logger.c utils/stime.c utils/stime.h utils/mtrie.c utils/mtrie.h \ 16 | utils/xmm.c utils/xmm.h utils/iqueue.h utils/iqueue.c utils/mmtree64.h utils/mmtree64.c \ 17 | utils/timer.h 18 | idispatchd_CPPFLAGS = -I utils/ -D_FILE_OFFSET_BITS=64 -Wall 19 | idispatchd_LDFLAGS = -lz -lsbase -lmtask -ldbase 20 | 21 | bin_PROGRAMS = dbtask 22 | dbtask_SOURCES = dbtask.c utils/md5.h utils/md5.c utils/crc32.h utils/crc32.c 23 | dbtask_CPPFLAGS = -I utils/ -D_FILE_OFFSET_BITS=64 -Wall 24 | dbtask_LDFLAGS = -lz -lsbase -lmtask -ldbase 25 | LIBS += -lz -lsbase -lmtask -ldbase 26 | -------------------------------------------------------------------------------- /ithunder/src/utils/base64.h: -------------------------------------------------------------------------------- 1 | #ifndef _BASE64_H 2 | #define _BASE64_H 3 | #define BASE64_LEN(x) ((x + 2) / 3 * 4 + 1) 4 | int base64_encode(char *out, const unsigned char *in, int inlen); 5 | int base64_decode(unsigned char *out, const char *in, int inlen); 6 | #endif 7 | -------------------------------------------------------------------------------- /ithunder/src/utils/crc32.h: -------------------------------------------------------------------------------- 1 | #ifndef CRC32_H 2 | #define CRC32_H 3 | unsigned int crc32(char *buf, int len); 4 | #endif 5 | -------------------------------------------------------------------------------- /ithunder/src/utils/html.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #ifndef _HTML_H 5 | #define _HTML_H 6 | #define HTML_TAG_NMAX 200000 7 | #define HTML_FIELD_MAX 256 8 | #define LEVEL_MAX 10000 9 | //#define HTML_CONTENT_MAX 16777216 10 | //#define HTML_CONTENT_MAX 2097152 11 | //#define HTML_CONTENT_MAX 4194304 12 | //#define HTML_TITLE_MAX 1048576 13 | #define HTML_MBLOCK_MAX 1048576 14 | #define HTML_MTEXT_MAX 1048576 15 | #define HTML_MLINK_MAX 1048576 16 | #define HTML_TITLE_MAX 65536 17 | #define HTML_LINK_SCALE 0.50f 18 | #define HTML_LINK_FILTER 50 19 | typedef struct _HBLOCK 20 | { 21 | int npairs; 22 | int nbytes; 23 | int nlinks; 24 | int id; 25 | char *start; 26 | char *last; 27 | struct _HBLOCK *parent; 28 | }HBLOCK; 29 | typedef struct _HFIELD 30 | { 31 | int from; 32 | int len; 33 | }HFIELD; 34 | #define HTML_LINK_IMG 0x01 35 | #define HTML_LINK_URL 0x02 36 | #define HTML_LINKS_MAX 8192 37 | typedef struct _HLINK 38 | { 39 | int off; 40 | short ntitle; 41 | short nauthor; 42 | short nlink; 43 | short flag; 44 | }HLINK; 45 | typedef struct _HTML 46 | { 47 | char block[HTML_MBLOCK_MAX]; 48 | char content[HTML_MTEXT_MAX]; 49 | char link[HTML_MLINK_MAX]; 50 | char *plink; 51 | int nblock; 52 | int ncontent; 53 | int ntitle; 54 | int titleoff; 55 | int nblocks; 56 | int nfields; 57 | int nlinks; 58 | HBLOCK blocklist[HTML_TAG_NMAX]; 59 | HFIELD fieldlist[HTML_FIELD_MAX]; 60 | HLINK linklist[HTML_LINKS_MAX]; 61 | void *table; 62 | }HTML; 63 | /* Initialize HTML */ 64 | HTML *html_init(); 65 | /* get content */ 66 | int html_get_content(HTML *html, char *content, size_t len, int filter, int new_filed); 67 | /* add field */ 68 | int html_add_field(HTML *html, char *content, int ncontent); 69 | /* reset HTML */ 70 | void html_reset(HTML *); 71 | /* clean html */ 72 | void html_clean(HTML *); 73 | #endif 74 | -------------------------------------------------------------------------------- /ithunder/src/utils/iqueue.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include "mutex.h" 5 | #ifndef _QUEUE_H 6 | #define _QUEUE_H 7 | #ifdef __cplusplus 8 | extern "C" { 9 | #endif 10 | typedef struct _QNODE 11 | { 12 | //void *ptr; 13 | int data; 14 | struct _QNODE *next; 15 | }QNODE; 16 | #define QNODE_LINE_MAX 10240 17 | #define QNODE_LINE_NUM 1024 18 | typedef struct _QUEUE 19 | { 20 | int total; 21 | int nlist; 22 | QNODE *list[QNODE_LINE_MAX]; 23 | QNODE *left; 24 | QNODE *first; 25 | QNODE *last; 26 | MUTEX *mutex; 27 | }QUEUE; 28 | void *iqueue_init(); 29 | void iqueue_push(void *q, int data); 30 | void iqueue_pop(void *q, int *data); 31 | void iqueue_head(void *q, int *data); 32 | void iqueue_clean(void *q); 33 | #define QTOTAL(q) (((QUEUE *)q)->total) 34 | #ifdef __cplusplus 35 | } 36 | #endif 37 | #endif 38 | -------------------------------------------------------------------------------- /ithunder/src/utils/kmap.h: -------------------------------------------------------------------------------- 1 | #ifndef _KMAP_H 2 | #define _KMAP_H 3 | #include "mutex.h" 4 | #define KMAP_INCRE_NUM 100000 5 | #define KMAP_NODES_MAX 2000000000 6 | #define KMROOT_INCRE_NUM 10000 7 | #define KMAP_ROOT_MAX 2000000000 8 | typedef struct _KMNODE 9 | { 10 | unsigned int left; 11 | unsigned int right; 12 | unsigned int parent; 13 | unsigned int color; 14 | int data; 15 | int bits; 16 | int64_t key; 17 | }KMNODE; 18 | typedef struct _KMROOT 19 | { 20 | int status; 21 | unsigned int total; 22 | unsigned int rootid; 23 | }KMROOT; 24 | typedef struct _KMSTATE 25 | { 26 | int64_t kmax; 27 | int64_t kmin; 28 | unsigned int nmax; 29 | unsigned int nmin; 30 | unsigned int count; 31 | unsigned int left; 32 | unsigned int current; 33 | unsigned int total; 34 | unsigned int qleft; 35 | unsigned int qfirst; 36 | unsigned int rleft; 37 | unsigned int rcurrent; 38 | unsigned int rtotal; 39 | unsigned int qlast; 40 | unsigned int qrleft; 41 | unsigned int qrfirst; 42 | unsigned int qrlast; 43 | int bits; 44 | }KMSTATE; 45 | typedef struct _KMAP 46 | { 47 | int fd; 48 | int rfd; 49 | off_t size; 50 | off_t end; 51 | off_t old; 52 | off_t rsize; 53 | off_t rend; 54 | off_t rold; 55 | void *start; 56 | KMSTATE *state; 57 | KMNODE *map; 58 | KMROOT *roots; 59 | MUTEX *mutex; 60 | }KMAP; 61 | void *kmap_init(char *basedir); 62 | int kmap_new_tree(void *kmap); 63 | unsigned int kmap_total(void *kmap, int rootid); 64 | unsigned int kmap_try_insert(void *kmap, int rootid, int64_t key, int data, int *old); 65 | unsigned int kmap_insert(void *kmap, int rootid, int64_t key, int data, int *old); 66 | unsigned int kmap_get(void *kmap, unsigned int nodeid, int64_t *key, int *data); 67 | unsigned int kmap_find(void *kmap, int rootid, int64_t key, int *data); 68 | unsigned int kmap_min(void *kmap, int rootid, int64_t *key, int *data); 69 | unsigned int kmap_max(void *kmap, int rootid, int64_t *key, int *data); 70 | unsigned int kmap_next(void *kmap, int rootid, unsigned int nodeid, int64_t *key, int *data); 71 | unsigned int kmap_prev(void *kmap, int rootid, unsigned int nodeid, int64_t *key, int *data); 72 | int kmap_set_data(void *kmap, unsigned int nodeid, int data); 73 | void kmap_view_tree(void *kmap, int rootid, FILE *fp); 74 | void kmap_remove(void *kmap, int rootid, unsigned int nodeid, int64_t *key, int *data); 75 | void kmap_remove_tree(void *kmap, int rootid); 76 | void kmap_close(void *kmap); 77 | #endif 78 | -------------------------------------------------------------------------------- /ithunder/src/utils/md5.h: -------------------------------------------------------------------------------- 1 | #include 2 | #ifndef _MD5_H 3 | #define _MD5_H 4 | #ifdef __cplusplus 5 | extern "C" { 6 | #endif 7 | #define MD5_LEN 16 8 | #define _MD5_BLOCK_N 64 9 | #define _MD5_SET_N 56 10 | #define _MD5_BITS_N 8 11 | #ifndef _TYPEDEF_MD5_CTX 12 | #define _TYPEDEF_MD5_CTX 13 | typedef struct _MD5_CTX 14 | { 15 | u_int32_t state[4]; 16 | u_int32_t total[2]; 17 | unsigned char digest[MD5_LEN]; 18 | unsigned char buf[_MD5_BLOCK_N]; 19 | }MD5_CTX; 20 | /* Initialize */ 21 | void md5_init(MD5_CTX *context); 22 | /* Update */ 23 | void md5_update(MD5_CTX *context, unsigned char *data, u_int32_t ndata); 24 | /* Final */ 25 | void md5_final(MD5_CTX *context); 26 | void md5(unsigned char *data, u_int32_t ndata, unsigned char *digest); 27 | /* Cacalute FILE md5 */ 28 | int md5_file(const char *file, unsigned char *digest); 29 | #endif 30 | #define MD5(_data, _ndata, md) { \ 31 | MD5_CTX ctx; \ 32 | md5_init(&ctx); \ 33 | md5_update(&ctx, _data, _ndata); \ 34 | md5_final(&ctx); \ 35 | memcpy(md, ctx.digest, MD5_LEN); \ 36 | } 37 | #define MD5OUT(md, out) \ 38 | { \ 39 | int i = 0; \ 40 | do{ \ 41 | fprintf(out, "%02x", md[i++]); \ 42 | }while(i < MD5_LEN); \ 43 | } 44 | #ifdef __cplusplus 45 | } 46 | #endif 47 | #endif 48 | -------------------------------------------------------------------------------- /ithunder/src/utils/mime.h: -------------------------------------------------------------------------------- 1 | #ifndef _MIME_H 2 | #define _MIME_H 3 | #define MIME_NUM_MAX 1024 4 | typedef struct _MIME_MAP 5 | { 6 | void *map; 7 | int num; 8 | int bits; 9 | }MIME_MAP; 10 | /* initialize mime */ 11 | int mime_map_init(MIME_MAP *mime_map); 12 | /* add mime */ 13 | int mime_add(MIME_MAP *mime_map, char *mime, int len); 14 | /* add mime line */ 15 | int mime_add_line(MIME_MAP *mime_map, char *p, char *end); 16 | /* return mime id*/ 17 | int mime_id(MIME_MAP *mime_map, char *mime, int len); 18 | /* clean mime map*/ 19 | void mime_map_clean(MIME_MAP *mime_map); 20 | #endif 21 | -------------------------------------------------------------------------------- /ithunder/src/utils/mmqueue.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include "mutex.h" 5 | #ifndef _MMQUEUE64_H 6 | #define _MMQUEUE64_H 7 | #ifdef __cplusplus 8 | extern "C" { 9 | #endif 10 | typedef struct _MMQODE 11 | { 12 | //void *ptr; 13 | unsigned int data; 14 | unsigned int next; 15 | }MMQNODE; 16 | typedef struct _MMQROOT 17 | { 18 | int status; 19 | int total; 20 | int first; 21 | int last; 22 | }MMQROOT; 23 | #define MMQ_INCRE_NUM 1000000 24 | #define MMQ_NODE_MAX 100000000 25 | #define MMQ_ROOT_MAX 5000000 26 | typedef struct _MMQSTATE 27 | { 28 | int qtotal; 29 | int qleft; 30 | int nroots; 31 | MMQROOT roots[MMQ_ROOT_MAX]; 32 | }MMQSTATE; 33 | typedef struct _MMQUEUE 34 | { 35 | int fd; 36 | int bits; 37 | off_t end; 38 | off_t size; 39 | off_t old; 40 | void *map; 41 | MUTEX *mutex; 42 | MMQSTATE *state; 43 | MMQNODE *nodes; 44 | }MMQUEUE; 45 | MMQUEUE *mmqueue_init(char *qfile); 46 | int mmqueue_new(MMQUEUE *mmq); 47 | int mmqueue_total(MMQUEUE *mmq, int rootid); 48 | int mmqueue_close(MMQUEUE *mmq, int rootid); 49 | int mmqueue_push(MMQUEUE *mmq, int rootid, int data); 50 | int mmqueue_pop(MMQUEUE *mmq, int rootid, int *data); 51 | int mmqueue_head(MMQUEUE *mmq, int rootid, int *data); 52 | void mmqueue_clean(MMQUEUE *mmq); 53 | #define MMQ(x) ((MMQUEUE*)x) 54 | #ifdef __cplusplus 55 | } 56 | #endif 57 | #endif 58 | -------------------------------------------------------------------------------- /ithunder/src/utils/mmtree.h: -------------------------------------------------------------------------------- 1 | #ifndef _MMTREE_H 2 | #define _MMTREE_H 3 | #include "mutex.h" 4 | #define MMTREE_INCRE_NUM 1000000 5 | #define MMTREE_NODES_MAX 2000000000 6 | #define MMTREE_ROOT_MAX 10240 7 | typedef struct _MTNODE 8 | { 9 | unsigned int left; 10 | unsigned int right; 11 | unsigned int parent; 12 | unsigned int color; 13 | int data; 14 | int key; 15 | }MTNODE; 16 | typedef struct _MMROOT 17 | { 18 | int status; 19 | int bits; 20 | unsigned int total; 21 | unsigned int rootid; 22 | }MMROOT; 23 | typedef struct _MTSTATE 24 | { 25 | int kmax; 26 | int kmin; 27 | unsigned int nmax; 28 | unsigned int nmin; 29 | unsigned int count; 30 | unsigned int left; 31 | unsigned int current; 32 | unsigned int total; 33 | unsigned int qleft; 34 | unsigned int qfirst; 35 | unsigned int qlast; 36 | unsigned int nroots; 37 | MMROOT roots[MMTREE_ROOT_MAX]; 38 | }MTSTATE; 39 | typedef struct _MMTREE 40 | { 41 | int fd; 42 | int status; 43 | off_t size; 44 | off_t end; 45 | off_t old; 46 | void *start; 47 | MTSTATE *state; 48 | MTNODE *map; 49 | MUTEX *mutex; 50 | }MMTREE; 51 | void *mmtree_init(char *file); 52 | int mmtree_new_tree(void *mmtree); 53 | unsigned int mmtree_total(void *mmtree, int rootid); 54 | unsigned int mmtree_try_insert(void *mmtree, int rootid, int key, int data, int *old); 55 | unsigned int mmtree_insert(void *mmtree, int rootid, int key, int data, int *old); 56 | unsigned int mmtree_get(void *mmtree, unsigned int nodeid, int *key, int *data); 57 | unsigned int mmtree_find(void *mmtree, int rootid, int key, int *data); 58 | unsigned int mmtree_min(void *mmtree, int rootid, int *key, int *data); 59 | unsigned int mmtree_max(void *mmtree, int rootid, int *key, int *data); 60 | unsigned int mmtree_next(void *mmtree, int rootid, unsigned int nodeid, int *key, int *data); 61 | unsigned int mmtree_prev(void *mmtree, int rootid, unsigned int nodeid, int *key, int *data); 62 | int mmtree_set_data(void *mmtree, unsigned int nodeid, int data); 63 | void mmtree_view_tree(void *mmtree, int rootid, FILE *fp); 64 | void mmtree_remove(void *mmtree, int rootid, unsigned int nodeid, int *key, int *data); 65 | void mmtree_remove_tree(void *mmtree, int rootid); 66 | void mmtree_close(void *mmtree); 67 | #endif 68 | -------------------------------------------------------------------------------- /ithunder/src/utils/mmtree64.h: -------------------------------------------------------------------------------- 1 | #ifndef _MMTREE64_H 2 | #define _MMTREE64_H 3 | #define MMTREE64_INCRE_NUM 1000000 4 | #define MMTREE64_NODES_MAX 2000000000 5 | #define MMTREE64_ROOT_MAX 10240 6 | #include "mutex.h" 7 | typedef struct _MTNODE 8 | { 9 | unsigned int left; 10 | unsigned int right; 11 | unsigned int parent; 12 | unsigned int color; 13 | int data; 14 | int bit; 15 | int64_t key; 16 | }MTNODE; 17 | typedef struct _MMROOT 18 | { 19 | int status; 20 | int bits; 21 | unsigned int total; 22 | unsigned int rootid; 23 | }MMROOT; 24 | typedef struct _MTSTATE 25 | { 26 | int64_t kmax; 27 | int64_t kmin; 28 | unsigned int nmax; 29 | unsigned int nmin; 30 | unsigned int count; 31 | unsigned int left; 32 | unsigned int current; 33 | unsigned int total; 34 | unsigned int qleft; 35 | unsigned int qfirst; 36 | unsigned int qlast; 37 | unsigned int nroots; 38 | MMROOT roots[MMTREE64_ROOT_MAX]; 39 | }MTSTATE; 40 | typedef struct _MMTREE64 41 | { 42 | int fd; 43 | int status; 44 | off_t size; 45 | off_t end; 46 | off_t old; 47 | char *start; 48 | MTSTATE *state; 49 | MTNODE *map; 50 | MUTEX *mutex; 51 | }MMTREE64; 52 | void *mmtree64_init(char *file); 53 | int mmtree64_new_tree(void *mmtree); 54 | unsigned int mmtree64_total(void *mmtree, int rootid); 55 | unsigned int mmtree64_try_insert(void *mmtree, int rootid, int64_t key, int data, int *old); 56 | unsigned int mmtree64_insert(void *mmtree, int rootid, int64_t key, int data, int *old); 57 | unsigned int mmtree64_get(void *mmtree, unsigned int nodeid, int64_t *key, int *data); 58 | unsigned int mmtree64_find(void *mmtree, int rootid, int64_t key, int *data); 59 | unsigned int mmtree64_min(void *mmtree, int rootid, int64_t *key, int *data); 60 | unsigned int mmtree64_max(void *mmtree, int rootid, int64_t *key, int *data); 61 | unsigned int mmtree64_next(void *mmtree, int rootid, unsigned int nodeid, int64_t *key, int *data); 62 | unsigned int mmtree64_prev(void *mmtree, int rootid, unsigned int nodeid, int64_t *key, int *data); 63 | int mmtree64_set_data(void *mmtree, unsigned int nodeid, int data); 64 | void mmtree64_view_tree(void *mmtree, int rootid, FILE *fp); 65 | void mmtree64_remove(void *mmtree, int rootid, unsigned int nodeid, int64_t *key, int *data); 66 | void mmtree64_remove_tree(void *mmtree, int rootid); 67 | void mmtree64_close(void *mmtree); 68 | #endif 69 | -------------------------------------------------------------------------------- /ithunder/src/utils/stime.h: -------------------------------------------------------------------------------- 1 | #ifndef _STIME_H 2 | #define _STIME_H 3 | #include 4 | #include 5 | //convert str datetime to time 6 | time_t str2time(char *datestr); 7 | /* time to GMT */ 8 | int GMTstrdate(time_t time, char *date); 9 | /* strdate */ 10 | int strdate(time_t time, char *date); 11 | /* local date time */ 12 | int datetime(time_t times, char *date); 13 | /* timetospec */ 14 | void timetospec(struct timespec *ts, int usecs); 15 | #endif 16 | -------------------------------------------------------------------------------- /ithunder/src/utils/vmap.h: -------------------------------------------------------------------------------- 1 | #ifndef _VMAP_H 2 | #define _VMAP_H 3 | #include "mutex.h" 4 | #define VMAP_INCRE_NUM 100000 5 | #define VMAP_NODES_MAX 2000000000 6 | #define VMROOT_INCRE_NUM 10000 7 | #define VMAP_ROOT_MAX 2000000000 8 | typedef struct _VMNODE 9 | { 10 | unsigned int left; 11 | unsigned int right; 12 | unsigned int parent; 13 | unsigned int color; 14 | int data; 15 | int key; 16 | }VMNODE; 17 | typedef struct _VMROOT 18 | { 19 | int status; 20 | unsigned int total; 21 | unsigned int rootid; 22 | }VMROOT; 23 | typedef struct _VMSTATE 24 | { 25 | int kmax; 26 | int kmin; 27 | unsigned int nmax; 28 | unsigned int nmin; 29 | unsigned int count; 30 | unsigned int left; 31 | unsigned int current; 32 | unsigned int total; 33 | unsigned int qleft; 34 | unsigned int qfirst; 35 | unsigned int rleft; 36 | unsigned int rcurrent; 37 | unsigned int rtotal; 38 | unsigned int qlast; 39 | unsigned int qrleft; 40 | unsigned int qrfirst; 41 | unsigned int qrlast; 42 | int bits; 43 | }VMSTATE; 44 | typedef struct _VMAP 45 | { 46 | int fd; 47 | int rfd; 48 | off_t size; 49 | off_t end; 50 | off_t old; 51 | off_t rsize; 52 | off_t rend; 53 | off_t rold; 54 | void *start; 55 | VMSTATE *state; 56 | VMNODE *map; 57 | VMROOT *roots; 58 | MUTEX *mutex; 59 | }VMAP; 60 | void *vmap_init(char *basedir); 61 | int vmap_new_tree(void *vmap); 62 | unsigned int vmap_total(void *vmap, int rootid); 63 | unsigned int vmap_try_insert(void *vmap, int rootid, int key, int data, int *old); 64 | unsigned int vmap_insert(void *vmap, int rootid, int key, int data, int *old); 65 | unsigned int vmap_get(void *vmap, unsigned int nodeid, int *key, int *data); 66 | unsigned int vmap_find(void *vmap, int rootid, int key, int *data); 67 | unsigned int vmap_min(void *vmap, int rootid, int *key, int *data); 68 | unsigned int vmap_max(void *vmap, int rootid, int *key, int *data); 69 | unsigned int vmap_next(void *vmap, int rootid, unsigned int nodeid, int *key, int *data); 70 | unsigned int vmap_prev(void *vmap, int rootid, unsigned int nodeid, int *key, int *data); 71 | int vmap_set_data(void *vmap, unsigned int nodeid, int data); 72 | void vmap_view_tree(void *vmap, int rootid, FILE *fp); 73 | void vmap_remove(void *vmap, int rootid, unsigned int nodeid, int *key, int *data); 74 | void vmap_remove_tree(void *vmap, int rootid); 75 | void vmap_close(void *vmap); 76 | #endif 77 | -------------------------------------------------------------------------------- /ithunder/src/utils/xmm.h: -------------------------------------------------------------------------------- 1 | #ifndef _XMM_H_ 2 | #define _XMM_H_ 3 | void* xmm_mnew(size_t size); 4 | void* xmm_new(size_t size); 5 | void* xmm_mrenew(void *old, size_t old_size, size_t new_size); 6 | void* xmm_renew(void *old, size_t old_size, size_t new_size); 7 | void* xmm_mresize(void *old, size_t old_size, size_t new_size); 8 | void* xmm_resize(void *old, size_t old_size, size_t new_size); 9 | void xmm_free(void *m, size_t size); 10 | #endif 11 | -------------------------------------------------------------------------------- /ithunder/src/utils/zstream.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #ifndef _ZSTREAM_H 6 | #define _ZSTREAM_H 7 | #define Z_HEADER_SIZE 12 8 | /* Compress data */ 9 | int zcompress(Bytef *data, uLong ndata, 10 | Bytef *zdata, uLong *nzdata); 11 | /* Uncompress data */ 12 | int zdecompress(Bytef *zdata, uLong nzdata, 13 | Bytef *data, uLong *ndata); 14 | /* gzip Compress data */ 15 | int gzcompress(Bytef *data, uLong ndata, 16 | Bytef *zdata, uLong *nzdata); 17 | /* gzip Uncompress data */ 18 | int gzdecompress(Bytef *zdata, uLong nzdata, 19 | Bytef *data, uLong *ndata); 20 | /* http gzip Uncompress data */ 21 | int httpgzdecompress(Bytef *zdata, uLong nzdata, 22 | Bytef *data, uLong *ndata); 23 | 24 | #endif 25 | -------------------------------------------------------------------------------- /ithunder/src/utils/zvbcode.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include "zvbcode.h" 5 | #define VB_NUM_MAX 100000000 6 | #ifdef _DEBUG_ZVBCODE 7 | #include "timer.h" 8 | #define __ZINT_OUT__ 0 9 | #define __ZINT__ "%d\n" 10 | typedef int ZINT; 11 | int main(int argc, char **argv) 12 | { 13 | int i = 0, n = 0, x = 0; 14 | ZINT rand = 0, *dp = NULL, *np = NULL; 15 | char *p = NULL, *ps = NULL, *ep = NULL; 16 | FILE *fp = NULL, *dfp = NULL; 17 | void *timer = NULL; 18 | 19 | if((ps = (char *)calloc(VB_NUM_MAX*2, sizeof(ZINT)))) 20 | { 21 | TIMER_INIT(timer); 22 | n = VB_NUM_MAX * sizeof(ZINT); 23 | if((fp = fopen("/tmp/double.txt", "a+"))) 24 | { 25 | p = ps; 26 | np = &rand; 27 | for(i = 0; i < VB_NUM_MAX; i++) 28 | { 29 | rand = (ZINT)random()%100000000; 30 | if(__ZINT_OUT__)fprintf(fp, __ZINT__, rand); 31 | ZVBCODE(np, p); 32 | } 33 | TIMER_SAMPLE(timer); 34 | fflush(fp); 35 | fprintf(stdout, "compressed(sizeof(zint):%d) %d to %d time:%lld avg:%f\n", 36 | sizeof(ZINT), n, (p - ps), PT_LU_USEC(timer), 37 | (double)PT_LU_USEC(timer)/(double)VB_NUM_MAX); 38 | fclose(fp); 39 | } 40 | if((dfp = fopen("/tmp/double.text", "a+"))) 41 | { 42 | ep = p; 43 | p = ps; 44 | np = &rand; 45 | TIMER_SAMPLE(timer); 46 | i = 0; 47 | while(p < ep) 48 | { 49 | rand = 0; 50 | UZVBCODE(p, x, np); 51 | i++; 52 | if(__ZINT_OUT__) fprintf(dfp, __ZINT__, rand); 53 | } 54 | TIMER_SAMPLE(timer); 55 | fflush(fp); 56 | fprintf(stdout, "decompressed(typesizeof(ZINT):%d) %d to %d time:%lld avg:%f\n", 57 | sizeof(ZINT), (p - ps), i * sizeof(ZINT), PT_LU_USEC(timer), 58 | (double)PT_LU_USEC(timer)/(double)i); 59 | fclose(dfp); 60 | } 61 | free(ps); 62 | ps = NULL; 63 | } 64 | } 65 | #endif 66 | -------------------------------------------------------------------------------- /libchardet/AUTHORS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sounos/ithunder/241d17ce44ed90bc3d8ce4f0b8d5ae52498db270/libchardet/AUTHORS -------------------------------------------------------------------------------- /libchardet/ChangeLog: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sounos/ithunder/241d17ce44ed90bc3d8ce4f0b8d5ae52498db270/libchardet/ChangeLog -------------------------------------------------------------------------------- /libchardet/Makefile.am: -------------------------------------------------------------------------------- 1 | SUBDIRS = src 2 | EXTRA_DIST = debian.sh control control control 3 | -------------------------------------------------------------------------------- /libchardet/NEWS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sounos/ithunder/241d17ce44ed90bc3d8ce4f0b8d5ae52498db270/libchardet/NEWS -------------------------------------------------------------------------------- /libchardet/README: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sounos/ithunder/241d17ce44ed90bc3d8ce4f0b8d5ae52498db270/libchardet/README -------------------------------------------------------------------------------- /libchardet/configure.in: -------------------------------------------------------------------------------- 1 | # -*- Autoconf -*- 2 | # Process this file with autoconf to produce a configure script. 3 | 4 | AC_PREREQ(2.59) 5 | AC_INIT(libchardet, 0.0.4, [SounOS@gmail.com]) 6 | AM_INIT_AUTOMAKE 7 | AC_CONFIG_SRCDIR([src/CharDistribution.cpp]) 8 | AC_CONFIG_HEADER([config.h]) 9 | 10 | # Checks for programs. 11 | AC_PROG_CXX 12 | AC_PROG_CC 13 | AC_PROG_LIBTOOL 14 | 15 | # Checks for libraries. 16 | 17 | # Checks for header files. 18 | AC_HEADER_STDC 19 | AC_CHECK_HEADERS([stddef.h stdlib.h string.h]) 20 | 21 | # Checks for typedefs, structures, and compiler characteristics. 22 | AC_HEADER_STDBOOL 23 | AC_C_CONST 24 | AC_C_INLINE 25 | AC_TYPE_SIZE_T 26 | 27 | # Checks for library functions. 28 | AC_FUNC_MALLOC 29 | AC_CHECK_FUNCS([strdup]) 30 | 31 | AC_CONFIG_FILES([Makefile 32 | src/Makefile]) 33 | AC_OUTPUT 34 | -------------------------------------------------------------------------------- /libchardet/control: -------------------------------------------------------------------------------- 1 | Source: 2 | Section: libs 3 | Priority: extra 4 | Maintainer: SounOS 5 | Build-Depends: debhelper (>= 7.0.50~), autotools-dev 6 | Standards-Version: 7 | Homepage: http://sbase.googlecode.com/ 8 | #Vcs-Git: git://git.debian.org/collab-maint/libevbase.git 9 | #Vcs-Browser: http://git.debian.org/?p=collab-maint/libevbase.git;a=summary 10 | 11 | Package: 12 | Architecture: any 13 | Depends: ${shlibs:Depends}, ${misc:Depends} 14 | Description: 15 | 16 | -------------------------------------------------------------------------------- /libchardet/debian.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | rm -rf debian 3 | pkg=`grep -E "^PACKAGE_NAME=" configure |sed -e "s/PACKAGE_NAME='\(.*\)'/\1/"` 4 | ver=`grep -E "^PACKAGE_VERSION=" configure |sed -e "s/PACKAGE_VERSION='\(.*\)'/\1/"` 5 | dh_make -y -n -s -c bsd -e SounOS@gmail.com -p "${pkg}_${ver}" 6 | cp control debian/control 7 | sed -i "s/^Package:.*/Package: $pkg/" debian/control 8 | sed -i "s/^Source:.*/Source: $pkg/" debian/control 9 | sed -i "s/^License:.*/License: $pkg/" debian/control 10 | sed -i "s/^Standards-Version:.*/Standards-Version: $ver/" debian/control 11 | sed -i "s/root /SounOS /" debian/* 12 | #export CFLAGS='-ggdb -O0' 13 | #export CXXFLAGS='-ggdb -O0' 14 | #export FFLAGS='-ggdb -O0' 15 | dpkg-buildpackage -rfakeroot -eSounOS -mSounOS 16 | -------------------------------------------------------------------------------- /libchardet/src/Makefile.am: -------------------------------------------------------------------------------- 1 | #Makefile.am 2 | lib_LTLIBRARIES = libchardet.la 3 | libchardet_la_SOURCES = CharDistribution.cpp \ 4 | nsEUCJPProber.h \ 5 | nsEscCharsetProber.cpp \ 6 | nsHebrewProber.h \ 7 | prmem.h \ 8 | nsCharSetProber.cpp \ 9 | nsSBCSGroupProber.cpp \ 10 | entry/impl.cpp \ 11 | JpCntx.cpp \ 12 | LangThaiModel.cpp \ 13 | nsMBCSGroupProber.cpp \ 14 | nsEscCharsetProber.h \ 15 | nsBig5Prober.h \ 16 | nsBig5Prober.cpp \ 17 | nsLatin1Prober.h \ 18 | nsPkgInt.h \ 19 | nsUniversalDetector.h \ 20 | nsEUCTWProber.cpp \ 21 | nsEUCKRProber.h \ 22 | nsEscSM.cpp \ 23 | nsEUCKRProber.cpp \ 24 | nscore.h \ 25 | LangGreekModel.cpp \ 26 | nsCodingStateMachine.h \ 27 | nsUniversalDetector.cpp \ 28 | CharDistribution.h \ 29 | nsSBCharSetProber.h \ 30 | nsEUCTWProber.h \ 31 | Makefile.am \ 32 | chardet.h \ 33 | nsSBCSGroupProber.h \ 34 | nsCharSetProber.h \ 35 | LangBulgarianModel.cpp \ 36 | nsMBCSSM.cpp \ 37 | nsGB2312Prober.h \ 38 | nsSBCharSetProber.cpp \ 39 | JpCntx.h \ 40 | LangHungarianModel.cpp \ 41 | nsUTF8Prober.h \ 42 | nsMBCSGroupProber.h \ 43 | nsGB2312Prober.cpp \ 44 | nsEUCJPProber.cpp \ 45 | nsHebrewProber.cpp \ 46 | LangHebrewModel.cpp \ 47 | nsLatin1Prober.cpp \ 48 | nsSJISProber.h \ 49 | tables/JISFreq.tab \ 50 | tables/Big5Freq.tab \ 51 | tables/EUCTWFreq.tab \ 52 | tables/GB2312Freq.tab \ 53 | tables/EUCKRFreq.tab \ 54 | nsUTF8Prober.cpp \ 55 | nsSJISProber.cpp \ 56 | LangCyrillicModel.cpp 57 | 58 | AM_CXXFLAGS = -Wall -Wno-non-virtual-dtor -nostdinc++ -fno-rtti -fno-exceptions 59 | include_HEADERS = chardet.h 60 | libchardet_la_CPPFLAGS = -nostdinc++ -fno-rtti -fno-exceptions -I tables 61 | 62 | bin_PROGRAMS = chardetor 63 | chardetor_SOURCES = chardetor.cpp 64 | chardetor_LDADD = libchardet.la 65 | chardetor_LDFLAGS = -static 66 | 67 | -------------------------------------------------------------------------------- /libchardet/src/chardetor.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include "chardet.h" 8 | #define CHARSET_MAX 1024 9 | 10 | int main(int argc, char **argv) 11 | { 12 | int i = 0, fd = 0; 13 | struct stat st; 14 | char *s = NULL, charset[CHARSET_MAX]; 15 | chardet_t pdet = NULL; 16 | 17 | if(argc < 2) 18 | { 19 | fprintf(stderr, "Usage:%s file ...\n", argv[0]); 20 | _exit(-1); 21 | } 22 | 23 | if(chardet_create(&pdet) == 0) 24 | { 25 | for(i = 1; i < argc; i++) 26 | { 27 | if((fd = open(argv[i], O_RDONLY, 0644)) > 0) 28 | { 29 | if(fstat(fd, &st) == 0 && st.st_size > 0 30 | && (s = (char *)calloc(1, st.st_size + 1))) 31 | { 32 | if(read(fd, s, st.st_size) > 0 33 | && chardet_handle_data(pdet, s, st.st_size) == 0) 34 | { 35 | chardet_data_end(pdet); 36 | chardet_get_charset(pdet, charset, CHARSET_MAX); 37 | fprintf(stdout, "charset of %s:%s\n", argv[i], charset); 38 | chardet_reset(pdet); 39 | } 40 | free(s); 41 | s = NULL; 42 | } 43 | close(fd); 44 | } 45 | } 46 | chardet_destroy(pdet); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /libchardet/src/nscore.h: -------------------------------------------------------------------------------- 1 | /* ***** BEGIN LICENSE BLOCK ***** 2 | * Version: MPL 1.1/GPL 2.0/LGPL 2.1 3 | * 4 | * The contents of this file are subject to the Mozilla Public License Version 5 | * 1.1 (the "License"); you may not use this file except in compliance with 6 | * the License. You may obtain a copy of the License at 7 | * http://www.mozilla.org/MPL/ 8 | * 9 | * Software distributed under the License is distributed on an "AS IS" basis, 10 | * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License 11 | * for the specific language governing rights and limitations under the 12 | * License. 13 | * 14 | * The Original Code is Mozilla Universal charset detector code. 15 | * 16 | * The Initial Developer of the Original Code is 17 | * Netscape Communications Corporation. 18 | * Portions created by the Initial Developer are Copyright (C) 2001 19 | * the Initial Developer. All Rights Reserved. 20 | * 21 | * Contributor(s): 22 | * Kohei TAKETA 23 | * 24 | * Alternatively, the contents of this file may be used under the terms of 25 | * either the GNU General Public License Version 2 or later (the "GPL"), or 26 | * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), 27 | * in which case the provisions of the GPL or the LGPL are applicable instead 28 | * of those above. If you wish to allow use of your version of this file only 29 | * under the terms of either the GPL or the LGPL, and not to allow others to 30 | * use your version of this file under the terms of the MPL, indicate your 31 | * decision by deleting the provisions above and replace them with the notice 32 | * and other provisions required by the GPL or the LGPL. If you do not delete 33 | * the provisions above, a recipient may use your version of this file under 34 | * the terms of any one of the MPL, the GPL or the LGPL. 35 | * 36 | * ***** END LICENSE BLOCK ***** */ 37 | #ifndef nsDummyCore_h__ 38 | #define nsDummyCore_h__ 39 | 40 | typedef bool PRBool; 41 | typedef int PRInt32; 42 | typedef unsigned int PRUint32; 43 | typedef short PRInt16; 44 | typedef unsigned short PRUint16; 45 | 46 | #define PR_FALSE false 47 | #define PR_TRUE true 48 | #define nsnull 0 49 | 50 | 51 | enum nsresult 52 | { 53 | NS_OK, 54 | NS_ERROR_OUT_OF_MEMORY 55 | }; 56 | 57 | #endif 58 | -------------------------------------------------------------------------------- /libchardet/src/prmem.h: -------------------------------------------------------------------------------- 1 | /* ***** BEGIN LICENSE BLOCK ***** 2 | * Version: MPL 1.1/GPL 2.0/LGPL 2.1 3 | * 4 | * The contents of this file are subject to the Mozilla Public License Version 5 | * 1.1 (the "License"); you may not use this file except in compliance with 6 | * the License. You may obtain a copy of the License at 7 | * http://www.mozilla.org/MPL/ 8 | * 9 | * Software distributed under the License is distributed on an "AS IS" basis, 10 | * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License 11 | * for the specific language governing rights and limitations under the 12 | * License. 13 | * 14 | * The Original Code is Mozilla Universal charset detector code. 15 | * 16 | * The Initial Developer of the Original Code is 17 | * Netscape Communications Corporation. 18 | * Portions created by the Initial Developer are Copyright (C) 2001 19 | * the Initial Developer. All Rights Reserved. 20 | * 21 | * Contributor(s): 22 | * Kohei TAKETA 23 | * 24 | * Alternatively, the contents of this file may be used under the terms of 25 | * either the GNU General Public License Version 2 or later (the "GPL"), or 26 | * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"), 27 | * in which case the provisions of the GPL or the LGPL are applicable instead 28 | * of those above. If you wish to allow use of your version of this file only 29 | * under the terms of either the GPL or the LGPL, and not to allow others to 30 | * use your version of this file under the terms of the MPL, indicate your 31 | * decision by deleting the provisions above and replace them with the notice 32 | * and other provisions required by the GPL or the LGPL. If you do not delete 33 | * the provisions above, a recipient may use your version of this file under 34 | * the terms of any one of the MPL, the GPL or the LGPL. 35 | * 36 | * ***** END LICENSE BLOCK ***** */ 37 | #ifndef nsDummyPrmem_h__ 38 | #define nsDummyPrmem_h__ 39 | 40 | #include 41 | 42 | inline void* PR_Malloc(size_t len) 43 | { 44 | return malloc(len); 45 | } 46 | 47 | #define PR_FREEIF(p) do { if (p) delete p; } while(0) 48 | 49 | #endif 50 | -------------------------------------------------------------------------------- /libibase/AUTHORS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sounos/ithunder/241d17ce44ed90bc3d8ce4f0b8d5ae52498db270/libibase/AUTHORS -------------------------------------------------------------------------------- /libibase/ChangeLog: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sounos/ithunder/241d17ce44ed90bc3d8ce4f0b8d5ae52498db270/libibase/ChangeLog -------------------------------------------------------------------------------- /libibase/Makefile.am: -------------------------------------------------------------------------------- 1 | SUBDIRS = src doc 2 | EXTRA_DIST = debian.sh control 3 | -------------------------------------------------------------------------------- /libibase/NEWS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sounos/ithunder/241d17ce44ed90bc3d8ce4f0b8d5ae52498db270/libibase/NEWS -------------------------------------------------------------------------------- /libibase/README: -------------------------------------------------------------------------------- 1 | #compile and install 2 | ./configure --enable-debug 3 | make && make install 4 | 5 | #INDEX 6 | ./hibase --basedir=/tmp/index --add --dict=/data/dict/dict.txt --doc=/data/html/index.html --url="http://www.sina.com.cn/" --charset="gbk" --date="Thu, 03 Jul 2008 03:15:52 GMT" 7 | 8 | #QUERY 9 | ./hibase --basedir=/tmp/index --query --request="贵州瓮安死者第3次尸检公开进行 结果数天后公布" --topN=1000 10 | 11 | #INDEX AND QUERY 12 | ./hibase --basedir=/tmp/index --dict=/data/dict/dict.txt --add --doc=/data/html/index.html --url="http://www.sina.com.cn/" --charset="gbk" --date="Thu, 03 Jul 2008 03:15:52 GMT" --query --request="贵州瓮安死者第3次尸检公开进行 结果数天后公布" --topN=1000 13 | 14 | -------------------------------------------------------------------------------- /libibase/acinclude.m4: -------------------------------------------------------------------------------- 1 | AC_DEFUN([AC_CHECK_EXTRA_OPTIONS],[ 2 | AC_MSG_CHECKING(for debugging) 3 | AC_ARG_ENABLE(debug, [ --enable-debug compile for debugging]) 4 | if test -z "$enable_debug" ; then 5 | enable_debug="no" 6 | elif test $enable_debug = "yes" ; then 7 | CPPFLAGS="${CPPFLAGS} -g -D_DEBUG" 8 | fi 9 | AC_MSG_RESULT([$enable_debug]) 10 | 11 | AC_MSG_CHECKING(for pthread_mutex) 12 | AC_ARG_ENABLE(pthread_mutex, [ --enable-pthread_mutex compile for enable pthread mutex]) 13 | if test -z "$enable_pthread_mutex" ; then 14 | enable_pthread_mutex="no" 15 | elif test $enable_pthread_mutex = "yes" ; then 16 | CPPFLAGS="${CPPFLAGS} -DUSE_PTHREAD_MUTEX" 17 | fi 18 | AC_MSG_RESULT([$enable_pthread_mutex]) 19 | 20 | AC_MSG_CHECKING(for index_compress) 21 | AC_ARG_ENABLE(compress, [ --disable-compress compile for disable index compress], [ disable_compress="yes"], [ disable_compress="no" ]) 22 | if test "$disable_compress" = "yes" ; then 23 | CPPFLAGS="${CPPFLAGS} -D_DISABLE_COMPRESS" 24 | fi 25 | AC_MSG_RESULT([$enable_compress]) 26 | 27 | AC_MSG_CHECKING(whether to disable memindex) 28 | AC_ARG_ENABLE(memindex, [ --disable-memindex compile for disable memindex], [ disable_memindex="yes" ], [ disable_memindex="no" ]) 29 | if test "$disable_memindex" = "yes" ; then 30 | CPPFLAGS="${CPPFLAGS} -D_DISABLE_MEMINDEX" 31 | fi 32 | AC_MSG_RESULT([$enable_memindex]) 33 | ]) 34 | 35 | -------------------------------------------------------------------------------- /libibase/configure.in: -------------------------------------------------------------------------------- 1 | # -*- Autoconf -*- 2 | # Process this file with autoconf to produce a configure script. 3 | 4 | AC_PREREQ(2.59) 5 | AC_INIT(libibase, [0.6.0], [SounOS@gmail.com]) 6 | AC_CONFIG_SRCDIR([src/ibase.h]) 7 | AC_CONFIG_HEADER([config.h]) 8 | AM_INIT_AUTOMAKE 9 | 10 | # Checks for programs. 11 | AC_PROG_CC 12 | AC_PROG_LIBTOOL 13 | 14 | # Checks for libraries. 15 | # FIXME: Replace `main' with a function in `-lm': 16 | AC_CHECK_LIB([m], [main]) 17 | # FIXME: Replace `main' with a function in `-lz': 18 | AC_CHECK_LIB([z], [main]) 19 | 20 | # Checks for header files. 21 | AC_HEADER_STDC 22 | AC_CHECK_HEADERS([fcntl.h stdint.h stdlib.h string.h sys/time.h unistd.h]) 23 | 24 | # Checks for typedefs, structures, and compiler characteristics. 25 | AC_TYPE_OFF_T 26 | AC_TYPE_SIZE_T 27 | AC_HEADER_TIME 28 | AC_STRUCT_TM 29 | 30 | # Checks for library functions. 31 | AC_FUNC_MMAP 32 | AC_FUNC_STAT 33 | AC_CHECK_FUNCS([ftruncate gettimeofday memset mkdir munmap strerror]) 34 | 35 | #pthread 36 | havepthread=no 37 | AC_CHECK_LIB(pthread,pthread_create,havepthread=yes) 38 | if test "$havepthread" = "yes" ; then 39 | CPPFLAGS="${CPPFLAGS} -DHAVE_PTHREAD" 40 | LDFLAGS="${LDFLAGS} -lpthread" 41 | LIBS="${LIBS} -lpthread" 42 | fi 43 | #semaphore 44 | #havesemaphore=no 45 | #AC_CHECK_LIB(c, sem_timedwait, havesemaphore=yes) 46 | #if test "$havesemaphore" = "yes" ; then 47 | # CPPFLAGS="${CPPFLAGS} -DHAVE_SEMAPHORE" 48 | #fi 49 | 50 | #check mmap 51 | havemmap=no 52 | AC_CHECK_LIB(c, munmap, havemmap=yes) 53 | if test "$havemmap" = "yes" ; then 54 | CPPFLAGS="${CPPFLAGS} -DHAVE_MMAP" 55 | fi 56 | 57 | #scws 58 | havescws=no 59 | AC_CHECK_LIB(scws,scws_get_result,havescws=yes) 60 | if test "$havescws" = "yes" ; then 61 | CPPFLAGS="${CPPFLAGS} -DHAVE_SCWS" 62 | LDFLAGS="${LDFLAGS} -lscws" 63 | LIBS="${LIBS} -lscws" 64 | fi 65 | 66 | 67 | AC_CHECK_EXTRA_OPTIONS 68 | 69 | AC_CONFIG_FILES([Makefile 70 | doc/Makefile 71 | src/Makefile]) 72 | AC_OUTPUT 73 | -------------------------------------------------------------------------------- /libibase/control: -------------------------------------------------------------------------------- 1 | Source: 2 | Section: libs 3 | Priority: extra 4 | Maintainer: SounOS 5 | Build-Depends: debhelper (>= 7.0.50~), autotools-dev 6 | Standards-Version: 7 | Homepage: http://sbase.googlecode.com/ 8 | #Vcs-Git: git://git.debian.org/collab-maint/libevbase.git 9 | #Vcs-Browser: http://git.debian.org/?p=collab-maint/libevbase.git;a=summary 10 | 11 | Package: 12 | Architecture: any 13 | Depends: ${shlibs:Depends}, ${misc:Depends} 14 | Description: 15 | 16 | -------------------------------------------------------------------------------- /libibase/debian.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | rm -rf debian 3 | pkg=`grep -E "^PACKAGE_NAME=" configure |sed -e "s/PACKAGE_NAME='\(.*\)'/\1/"` 4 | ver=`grep -E "^PACKAGE_VERSION=" configure |sed -e "s/PACKAGE_VERSION='\(.*\)'/\1/"` 5 | dh_make -y -n -s -c bsd -e SounOS@gmail.com -p "${pkg}_${ver}" 6 | cp control debian/control 7 | sed -i "s/^Package:.*/Package: $pkg/" debian/control 8 | sed -i "s/^Source:.*/Source: $pkg/" debian/control 9 | sed -i "s/^License:.*/License: $pkg/" debian/control 10 | sed -i "s/^Standards-Version:.*/Standards-Version: $ver/" debian/control 11 | sed -i "s/root /SounOS /" debian/* 12 | #export CFLAGS='-ggdb -O0' 13 | #export CXXFLAGS='-ggdb -O0' 14 | #export FFLAGS='-ggdb -O0' 15 | dpkg-buildpackage -rfakeroot -eSounOS -mSounOS 16 | -------------------------------------------------------------------------------- /libibase/doc/Makefile.am: -------------------------------------------------------------------------------- 1 | EXTRA_DIST = README.txt libibase.spec 2 | 3 | -------------------------------------------------------------------------------- /libibase/doc/README.txt: -------------------------------------------------------------------------------- 1 | ibase 使用手册 2 | 3 | -------------------------------------------------------------------------------- /libibase/doc/libibase.spec: -------------------------------------------------------------------------------- 1 | # Authority: SounOS.org 2 | 3 | Summary: Instant and Incremental Full-Text Search Engine 4 | Name: libibase 5 | Version: 0.3.8 6 | Release: 1%{?dist} 7 | License: BSD 8 | Group: System Environment/Libraries 9 | URL: http://code.google.com/p/libibase/ 10 | 11 | Source: http://code.google.com/p/libibase/download/%{name}-%{version}.tar.gz 12 | Packager: SounOS 13 | Vendor: SounOS 14 | BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root 15 | 16 | %description 17 | Libibase is a library for Search Engine which is Instant and Incremental. 18 | 19 | %prep 20 | %setup 21 | 22 | %build 23 | %configure 24 | %{__make} 25 | 26 | %install 27 | %{__rm} -rf %{buildroot} 28 | %{__make} install DESTDIR="%{buildroot}" 29 | 30 | %clean 31 | %{__rm} -rf %{buildroot} 32 | 33 | %post 34 | /sbin/ldconfig 35 | 36 | %postun 37 | /sbin/ldconfig 38 | 39 | %files 40 | %defattr(-, root, root, 0755) 41 | %{_includedir}/* 42 | %{_libdir}/* 43 | 44 | %changelog 45 | * Thu Apr 22 2010 11:38:18 CST SounOS 46 | - updated idb.c added memory index 47 | * Tue Mar 09 2010 14:48:22 CST SounOS 48 | - added ibase_set_phrase_status(); 49 | - added ibase_set_index_status(); 50 | -------------------------------------------------------------------------------- /libibase/src/Makefile.am: -------------------------------------------------------------------------------- 1 | EXTRA_DIST = iheaders.h utils/gen.sh 2 | lib_LTLIBRARIES = libibase.la 3 | libibase_la_SOURCES = ibase.c \ 4 | ibase.h \ 5 | index.c \ 6 | bquery.c \ 7 | hquery.c \ 8 | query.c \ 9 | summary.c \ 10 | utils/logger.h \ 11 | utils/logger.c \ 12 | utils/mutex.h \ 13 | utils/timer.h \ 14 | utils/mmtrie.c \ 15 | utils/mmtrie.h \ 16 | utils/db.h \ 17 | utils/db.c \ 18 | utils/mdb.h \ 19 | utils/mdb.c \ 20 | utils/rwlock.h \ 21 | utils/imap.h \ 22 | utils/imap.c \ 23 | utils/lmap.h \ 24 | utils/lmap.c \ 25 | utils/dmap.h \ 26 | utils/dmap.c \ 27 | utils/immx.h \ 28 | utils/immx.c \ 29 | utils/xmm.h \ 30 | utils/xmm.c \ 31 | utils/bmap.h \ 32 | utils/bmap.c \ 33 | utils/mtree64.h \ 34 | utils/mtree64.c \ 35 | utils/zvbcode.c \ 36 | utils/zvbcode.h 37 | 38 | libibase_la_CPPFLAGS = -I utils -I html -D_FILE_OFFSET_BITS=64 -Wall 39 | libibase_la_LDFLAGS = -lz -lm 40 | include_HEADERS = ibase.h iheaders.h 41 | -------------------------------------------------------------------------------- /libibase/src/utils/bmap.h: -------------------------------------------------------------------------------- 1 | #ifndef __BMAP_H__ 2 | #define __BMAP_H__ 3 | #ifdef __cplusplus 4 | extern "C" { 5 | #endif 6 | #include 7 | #define BMAP_ID_MAX 2000000000 8 | #define BMAP_BASE_NUM 1000000 9 | typedef struct _BMAP 10 | { 11 | int fd; 12 | int id_max; 13 | int bytes; 14 | int bit32; 15 | char *mbits; 16 | char *bits; 17 | pthread_rwlock_t mutex; 18 | }BMAP; 19 | void *bmap_init(char *file); 20 | int bmap_set(void *p, int id); 21 | int bmap_unset(void *p, int id); 22 | int bmap_check(void *p, int id); 23 | void bmap_clean(void *bmap); 24 | #ifdef __cplusplus 25 | } 26 | #endif 27 | #endif 28 | -------------------------------------------------------------------------------- /libibase/src/utils/dkv.h: -------------------------------------------------------------------------------- 1 | #ifndef __DKV__H 2 | #define __DKV__H 3 | #include 4 | #include 5 | #define DVV_SLOT_MAX 524288 6 | #define DVV_SLOT_INC 2000 7 | #define DVV_SLOT_NUM 1024 8 | #define DVV_SLOT2_NUM 512 /* half of DVV_SLOT_NUM */ 9 | #define DVV_DVVV_INC 1000000 10 | #define DVV_NODES_MAX 4294967296 11 | //#define __DKV_USE_IDX__ 1 12 | #pragma pack(push, 4) 13 | #ifndef U32_T 14 | #define U32_T 15 | typedef uint32_t u32_t; 16 | #endif 17 | typedef struct DVVKV 18 | { 19 | double key; /* 数值 */ 20 | u32_t val; /* 文档序号 */ 21 | }DVVKV; 22 | typedef struct _DVVV 23 | { 24 | double val; /* 数值 */ 25 | #ifdef __DKV_USE_IDX__ 26 | int off; /* 节点偏移 */ 27 | #endif 28 | }DVVV; 29 | typedef struct _DVVSLOT 30 | { 31 | double min; /* 当前桶的最小值 */ 32 | double max; /* 当前桶的最大值 */ 33 | int nodeid; /* 桶偏移地址 */ 34 | int count; /* 桶元素个数 */ 35 | }DVVSLOT; 36 | typedef struct _DVVSTATE 37 | { 38 | DVVSLOT slots[DVV_SLOT_MAX]; /* 槽 */ 39 | u32_t qleft[DVV_SLOT_MAX]; /* 空闲的桶 */ 40 | u32_t roots[DVV_SLOT_MAX]; /* 桶和槽的衍射 */ 41 | u32_t nleft; /* 空闲桶个数 */ 42 | u32_t count; /* 槽的个数 */ 43 | }DVVSTATE; 44 | typedef struct _DKV 45 | { 46 | DVVSTATE *state; 47 | DVVKV *map; 48 | DVVV *vmap; 49 | DVVSLOT *slots; 50 | u32_t *roots; 51 | int fd; 52 | int vfd; 53 | off_t size; 54 | off_t msize; 55 | off_t vsize; 56 | off_t vmsize; 57 | pthread_rwlock_t rwlock; 58 | }DKV; 59 | DKV *dkv_init(char *file); 60 | int dkv_set(DKV *dkv, u32_t no, double key); 61 | int dkv_get(DKV *dkv, u32_t no, u32_t *val); 62 | void dkv_close(DKV *dkv); 63 | #ifdef __DKV_USE_IDX__ 64 | /* return number of the hits */ 65 | int dkv_del(DKV *dkv, u32_t no); 66 | int dkv_range(DKV *dkv, double from, double to, u32_t *list); 67 | int dkv_rangefrom(DKV *dkv, double key, u32_t *list); /* key = from */ 68 | int dkv_rangeto(DKV *dkv, double key, u32_t *list); /* key = to */ 69 | int dkv_in(DKV *dkv, double key, u32_t *list); 70 | int dkv_ins(DKV *dkv, double *keys, int nkeys, u32_t *list); 71 | /* set list[] if (list != NULL) */ 72 | #define DKV_DEL(x, no) dkv_del(((DKV *)x), no) 73 | #else 74 | #define DKV_DEL(x, no) 75 | #endif 76 | #define DKV_GET(x, no) ((DKV *)x)->vmap[no].val 77 | #define DKV_SET(x, no, key) dkv_set(((DKV *)x), no, key) 78 | #pragma pack(pop) 79 | #endif 80 | -------------------------------------------------------------------------------- /libibase/src/utils/dmap.h: -------------------------------------------------------------------------------- 1 | #ifndef __DMAP__H 2 | #define __DMAP__H 3 | #include 4 | #include 5 | #define DMM_SLOT_MAX 524288 6 | #define DMM_SLOT_INC 2000 7 | #define DMM_SLOT_NUM 1024 8 | #define DMM_SLOT2_NUM 512 /* half of DMM_SLOT_NUM */ 9 | #define DMM_DMMV_INC 1000000 10 | #define DMM_NODES_MAX 4294967296 11 | #define __DMAP_USE_IDX__ 1 12 | #pragma pack(push, 4) 13 | #ifndef U32_T 14 | #define U32_T 15 | typedef uint32_t u32_t; 16 | #endif 17 | typedef struct DMMKV 18 | { 19 | double key; /* 数值 */ 20 | u32_t val; /* 文档序号 */ 21 | }DMMKV; 22 | typedef struct _DMMV 23 | { 24 | double val; /* 数值 */ 25 | #ifdef __DMAP_USE_IDX__ 26 | int off; /* 节点偏移 */ 27 | #endif 28 | }DMMV; 29 | typedef struct _DMMSLOT 30 | { 31 | double min; /* 当前桶的最小值 */ 32 | double max; /* 当前桶的最大值 */ 33 | int nodeid; /* 桶偏移地址 */ 34 | int count; /* 桶元素个数 */ 35 | }DMMSLOT; 36 | typedef struct _DMMSTATE 37 | { 38 | DMMSLOT slots[DMM_SLOT_MAX]; /* 槽 */ 39 | u32_t qleft[DMM_SLOT_MAX]; /* 空闲的桶 */ 40 | u32_t roots[DMM_SLOT_MAX]; /* 桶和槽的衍射 */ 41 | u32_t nleft; /* 空闲桶个数 */ 42 | u32_t count; /* 槽的个数 */ 43 | }DMMSTATE; 44 | typedef struct _DMAP 45 | { 46 | DMMSTATE *state; 47 | DMMKV *map; 48 | DMMV *vmap; 49 | DMMSLOT *slots; 50 | u32_t *roots; 51 | int fd; 52 | int vfd; 53 | off_t size; 54 | off_t msize; 55 | off_t vsize; 56 | off_t vmsize; 57 | pthread_rwlock_t rwlock; 58 | }DMAP; 59 | DMAP *dmap_init(char *file); 60 | int dmap_set(DMAP *dmap, u32_t no, double key); 61 | int dmap_get(DMAP *dmap, u32_t no, u32_t *val); 62 | void dmap_close(DMAP *dmap); 63 | #ifdef __DMAP_USE_IDX__ 64 | /* return number of the hits */ 65 | int dmap_del(DMAP *dmap, u32_t no); 66 | int dmap_range(DMAP *dmap, double from, double to, u32_t *list); 67 | int dmap_rangefrom(DMAP *dmap, double key, u32_t *list); /* key = from */ 68 | int dmap_rangeto(DMAP *dmap, double key, u32_t *list); /* key = to */ 69 | int dmap_in(DMAP *dmap, double key, u32_t *list); 70 | int dmap_ins(DMAP *dmap, double *keys, int nkeys, u32_t *list); 71 | /* set list[] if (list != NULL) */ 72 | #define DMAP_DEL(x, no) dmap_del(((DMAP *)x), no) 73 | #else 74 | #define DMAP_DEL(x, no) 75 | #endif 76 | #define DMAP_GET(x, no) ((DMAP *)x)->vmap[no].val 77 | #define DMAP_SET(x, no, key) dmap_set(((DMAP *)x), no, key) 78 | #pragma pack(pop) 79 | #endif 80 | -------------------------------------------------------------------------------- /libibase/src/utils/dtree.h: -------------------------------------------------------------------------------- 1 | #ifndef _DTREE_H 2 | #define _DTREE_H 3 | #define DTREE_INCRE_NUM 1000000 4 | #define DTREE_NODES_MAX 2000000000 5 | #define DTREE_MUTEX_MAX 256 6 | #define DTREE_ROOT_MAX 1000000 7 | #include 8 | #include "mutex.h" 9 | typedef struct _DTNODE 10 | { 11 | uint32_t left; 12 | uint32_t right; 13 | uint32_t parent; 14 | uint32_t color; 15 | int data; 16 | int bit; 17 | double key; 18 | }DTNODE; 19 | typedef struct _DTROOT 20 | { 21 | int16_t status; 22 | int16_t flag; 23 | uint32_t total; 24 | uint32_t rootid; 25 | }DTROOT; 26 | typedef struct _DTSTATE 27 | { 28 | double kmax; 29 | double kmin; 30 | uint32_t nmax; 31 | uint32_t nmin; 32 | uint32_t count; 33 | uint32_t left; 34 | uint32_t current; 35 | uint32_t total; 36 | uint32_t qleft; 37 | uint32_t qfirst; 38 | uint32_t qlast; 39 | uint32_t nroots; 40 | DTROOT roots[DTREE_ROOT_MAX]; 41 | }DTSTATE; 42 | typedef struct _DTREE 43 | { 44 | int fd; 45 | int status; 46 | off_t size; 47 | off_t end; 48 | off_t old; 49 | char *start; 50 | DTSTATE *state; 51 | DTNODE *map; 52 | MUTEX *mutex; 53 | #ifdef HAVE_PTHREAD 54 | pthread_mutex_t mutexs[DTREE_MUTEX_MAX]; 55 | #endif 56 | }DTREE; 57 | void *dtree_init(char *file); 58 | int dtree_new_tree(void *dtree); 59 | uint32_t dtree_total(void *dtree, int rootid); 60 | uint32_t dtree_try_insert(void *dtree, int rootid, double key, int data, int *old); 61 | uint32_t dtree_insert(void *dtree, int rootid, double key, int data, int *old); 62 | uint32_t dtree_get(void *dtree, uint32_t nodeid, double *key, int *data); 63 | uint32_t dtree_find(void *dtree, int rootid, double key, int *data); 64 | uint32_t dtree_min(void *dtree, int rootid, double *key, int *data); 65 | uint32_t dtree_max(void *dtree, int rootid, double *key, int *data); 66 | uint32_t dtree_next(void *dtree, int rootid, uint32_t nodeid, double *key, int *data); 67 | uint32_t dtree_prev(void *dtree, int rootid, uint32_t nodeid, double *key, int *data); 68 | int dtree_set_data(void *dtree, uint32_t nodeid, int data); 69 | void dtree_view_tree(void *dtree, int rootid, FILE *fp); 70 | void dtree_remove(void *dtree, int rootid, uint32_t nodeid, double *key, int *data); 71 | void dtree_remove_tree(void *dtree, int rootid); 72 | void dtree_close(void *dtree); 73 | #endif 74 | -------------------------------------------------------------------------------- /libibase/src/utils/gen.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #int 3 | cp -f imap.h ikv.h 4 | cp -f imap.c ikv.c 5 | perl -i -p -e "s/IMM/IVV/g" ikv.h ikv.c 6 | perl -i -p -e "s/IMAP/IKV/g" ikv.h ikv.c 7 | perl -i -p -e "s/imap/ikv/g" ikv.h ikv.c 8 | perl -i -p -e "s@#define __IKV_USE_IDX__@//#define __IKV_USE_IDX__@g" ikv.h ikv.c 9 | #long map 10 | cp -f imap.h lmap.h 11 | cp -f imap.c lmap.c 12 | perl -i -p -e "s/int32_t/int64_t/g" lmap.h lmap.c 13 | perl -i -p -e "s/IMM/LMM/g" lmap.h lmap.c 14 | perl -i -p -e "s/IMAP/LMAP/g" lmap.h lmap.c 15 | perl -i -p -e "s/imap/lmap/g" lmap.h lmap.c 16 | perl -i -p -e "s/typedef uint64_t u32_t/typedef uint32_t u32_t/g" lmap.h lmap.c 17 | #long kv 18 | cp -f ikv.h lkv.h 19 | cp -f ikv.c lkv.c 20 | perl -i -p -e "s/int32_t/int64_t/g" lkv.h lkv.c 21 | perl -i -p -e "s/IVV/LVV/g" lkv.h lkv.c 22 | perl -i -p -e "s/IKV/LKV/g" lkv.h lkv.c 23 | perl -i -p -e "s/ikv/lkv/g" lkv.h lkv.c 24 | perl -i -p -e "s/typedef uint64_t u32_t/typedef uint32_t u32_t/g" lkv.h lkv.c 25 | #double map 26 | cp -f imap.h dmap.h 27 | cp -f imap.c dmap.c 28 | perl -i -p -e "s/int32_t/double/g" dmap.h dmap.c 29 | perl -i -p -e "s/IMM/DMM/g" dmap.h dmap.c 30 | perl -i -p -e "s/IMAP/DMAP/g" dmap.h dmap.c 31 | perl -i -p -e "s/imap/dmap/g" dmap.h dmap.c 32 | perl -i -p -e "s/typedef udouble u32_t/typedef uint32_t u32_t/g" dmap.h dmap.c 33 | #double kv 34 | cp -f ikv.h dkv.h 35 | cp -f ikv.c dkv.c 36 | perl -i -p -e "s/int32_t/double/g" dkv.h dkv.c 37 | perl -i -p -e "s/IVV/DVV/g" dkv.h dkv.c 38 | perl -i -p -e "s/IKV/DKV/g" dkv.h dkv.c 39 | perl -i -p -e "s/ikv/dkv/g" dkv.h dkv.c 40 | perl -i -p -e "s/typedef udouble u32_t/typedef uint32_t u32_t/g" dkv.h dkv.c 41 | #cdb.c 42 | cp -f db.h cdb.h 43 | cp -f db.c cdb.c 44 | perl -i -p -e "s/DB/CDB/g" cdb.h cdb.c 45 | perl -i -p -e "s/db_/cdb_/g" cdb.h cdb.c 46 | perl -i -p -e "s/XIO/XCIO/g" cdb.h cdb.c 47 | perl -i -p -e "s/XBLOCK/XCBLOCK/g" cdb.h cdb.c 48 | perl -i -p -e "s/XSTATE/XCSTATE/g" cdb.h cdb.c 49 | perl -i -p -e 's|"db.h"|"cdb.h"|g' cdb.c 50 | perl -i -p -e "s/CDB_BASE_SIZE 64/CDB_BASE_SIZE 4096/g" cdb.h 51 | #mdb.c 52 | cp -f db.h mdb.h 53 | cp -f db.c mdb.c 54 | perl -i -p -e "s/DB/MDB/g" mdb.h mdb.c 55 | perl -i -p -e "s/db_/mdb_/g" mdb.h mdb.c 56 | perl -i -p -e "s/XIO/XMIO/g" mdb.h mdb.c 57 | perl -i -p -e "s/XBLOCK/XMBLOCK/g" mdb.h mdb.c 58 | perl -i -p -e "s/XSTATE/XMSTATE/g" mdb.h mdb.c 59 | perl -i -p -e 's|"db.h"|"mdb.h"|g' mdb.c 60 | perl -i -p -e "s|//#define __USE_X_TAG__|#define __USE_X_TAG__|g" mdb.c 61 | 62 | -------------------------------------------------------------------------------- /libibase/src/utils/ikv.h: -------------------------------------------------------------------------------- 1 | #ifndef __IKV__H 2 | #define __IKV__H 3 | #include 4 | #include 5 | #define IVV_SLOT_MAX 524288 6 | #define IVV_SLOT_INC 2000 7 | #define IVV_SLOT_NUM 1024 8 | #define IVV_SLOT2_NUM 512 /* half of IVV_SLOT_NUM */ 9 | #define IVV_IVVV_INC 1000000 10 | #define IVV_NODES_MAX 4294967296 11 | //#define __IKV_USE_IDX__ 1 12 | #pragma pack(push, 4) 13 | #ifndef U32_T 14 | #define U32_T 15 | typedef uint32_t u32_t; 16 | #endif 17 | typedef struct IVVKV 18 | { 19 | int32_t key; /* 数值 */ 20 | u32_t val; /* 文档序号 */ 21 | }IVVKV; 22 | typedef struct _IVVV 23 | { 24 | int32_t val; /* 数值 */ 25 | #ifdef __IKV_USE_IDX__ 26 | int off; /* 节点偏移 */ 27 | #endif 28 | }IVVV; 29 | typedef struct _IVVSLOT 30 | { 31 | int32_t min; /* 当前桶的最小值 */ 32 | int32_t max; /* 当前桶的最大值 */ 33 | int nodeid; /* 桶偏移地址 */ 34 | int count; /* 桶元素个数 */ 35 | }IVVSLOT; 36 | typedef struct _IVVSTATE 37 | { 38 | IVVSLOT slots[IVV_SLOT_MAX]; /* 槽 */ 39 | u32_t qleft[IVV_SLOT_MAX]; /* 空闲的桶 */ 40 | u32_t roots[IVV_SLOT_MAX]; /* 桶和槽的衍射 */ 41 | u32_t nleft; /* 空闲桶个数 */ 42 | u32_t count; /* 槽的个数 */ 43 | }IVVSTATE; 44 | typedef struct _IKV 45 | { 46 | IVVSTATE *state; 47 | IVVKV *map; 48 | IVVV *vmap; 49 | IVVSLOT *slots; 50 | u32_t *roots; 51 | int fd; 52 | int vfd; 53 | off_t size; 54 | off_t msize; 55 | off_t vsize; 56 | off_t vmsize; 57 | pthread_rwlock_t rwlock; 58 | }IKV; 59 | IKV *ikv_init(char *file); 60 | int ikv_set(IKV *ikv, u32_t no, int32_t key); 61 | int ikv_get(IKV *ikv, u32_t no, u32_t *val); 62 | void ikv_close(IKV *ikv); 63 | #ifdef __IKV_USE_IDX__ 64 | /* return number of the hits */ 65 | int ikv_del(IKV *ikv, u32_t no); 66 | int ikv_range(IKV *ikv, int32_t from, int32_t to, u32_t *list); 67 | int ikv_rangefrom(IKV *ikv, int32_t key, u32_t *list); /* key = from */ 68 | int ikv_rangeto(IKV *ikv, int32_t key, u32_t *list); /* key = to */ 69 | int ikv_in(IKV *ikv, int32_t key, u32_t *list); 70 | int ikv_ins(IKV *ikv, int32_t *keys, int nkeys, u32_t *list); 71 | /* set list[] if (list != NULL) */ 72 | #define IKV_DEL(x, no) ikv_del(((IKV *)x), no) 73 | #else 74 | #define IKV_DEL(x, no) 75 | #endif 76 | #define IKV_GET(x, no) ((IKV *)x)->vmap[no].val 77 | #define IKV_SET(x, no, key) ikv_set(((IKV *)x), no, key) 78 | #pragma pack(pop) 79 | #endif 80 | -------------------------------------------------------------------------------- /libibase/src/utils/imap.h: -------------------------------------------------------------------------------- 1 | #ifndef __IMAP__H 2 | #define __IMAP__H 3 | #include 4 | #include 5 | #define IMM_SLOT_MAX 524288 6 | #define IMM_SLOT_INC 2000 7 | #define IMM_SLOT_NUM 1024 8 | #define IMM_SLOT2_NUM 512 /* half of IMM_SLOT_NUM */ 9 | #define IMM_IMMV_INC 1000000 10 | #define IMM_NODES_MAX 4294967296 11 | #define __IMAP_USE_IDX__ 1 12 | #pragma pack(push, 4) 13 | #ifndef U32_T 14 | #define U32_T 15 | typedef uint32_t u32_t; 16 | #endif 17 | typedef struct IMMKV 18 | { 19 | int32_t key; /* 数值 */ 20 | u32_t val; /* 文档序号 */ 21 | }IMMKV; 22 | typedef struct _IMMV 23 | { 24 | int32_t val; /* 数值 */ 25 | #ifdef __IMAP_USE_IDX__ 26 | int off; /* 节点偏移 */ 27 | #endif 28 | }IMMV; 29 | typedef struct _IMMSLOT 30 | { 31 | int32_t min; /* 当前桶的最小值 */ 32 | int32_t max; /* 当前桶的最大值 */ 33 | int nodeid; /* 桶偏移地址 */ 34 | int count; /* 桶元素个数 */ 35 | }IMMSLOT; 36 | typedef struct _IMMSTATE 37 | { 38 | IMMSLOT slots[IMM_SLOT_MAX]; /* 槽 */ 39 | u32_t qleft[IMM_SLOT_MAX]; /* 空闲的桶 */ 40 | u32_t roots[IMM_SLOT_MAX]; /* 桶和槽的衍射 */ 41 | u32_t nleft; /* 空闲桶个数 */ 42 | u32_t count; /* 槽的个数 */ 43 | }IMMSTATE; 44 | typedef struct _IMAP 45 | { 46 | IMMSTATE *state; 47 | IMMKV *map; 48 | IMMV *vmap; 49 | IMMSLOT *slots; 50 | u32_t *roots; 51 | int fd; 52 | int vfd; 53 | off_t size; 54 | off_t msize; 55 | off_t vsize; 56 | off_t vmsize; 57 | pthread_rwlock_t rwlock; 58 | }IMAP; 59 | IMAP *imap_init(char *file); 60 | int imap_set(IMAP *imap, u32_t no, int32_t key); 61 | int imap_get(IMAP *imap, u32_t no, u32_t *val); 62 | void imap_close(IMAP *imap); 63 | #ifdef __IMAP_USE_IDX__ 64 | /* return number of the hits */ 65 | int imap_del(IMAP *imap, u32_t no); 66 | int imap_range(IMAP *imap, int32_t from, int32_t to, u32_t *list); 67 | int imap_rangefrom(IMAP *imap, int32_t key, u32_t *list); /* key = from */ 68 | int imap_rangeto(IMAP *imap, int32_t key, u32_t *list); /* key = to */ 69 | int imap_in(IMAP *imap, int32_t key, u32_t *list); 70 | int imap_ins(IMAP *imap, int32_t *keys, int nkeys, u32_t *list); 71 | /* set list[] if (list != NULL) */ 72 | #define IMAP_DEL(x, no) imap_del(((IMAP *)x), no) 73 | #else 74 | #define IMAP_DEL(x, no) 75 | #endif 76 | #define IMAP_GET(x, no) ((IMAP *)x)->vmap[no].val 77 | #define IMAP_SET(x, no, key) imap_set(((IMAP *)x), no, key) 78 | #pragma pack(pop) 79 | #endif 80 | -------------------------------------------------------------------------------- /libibase/src/utils/itree.h: -------------------------------------------------------------------------------- 1 | #ifndef _ITREE_H 2 | #define _ITREE_H 3 | #include 4 | #include "mutex.h" 5 | #define ITREE_INCRE_NUM 1000000 6 | #define ITREE_NODES_MAX 2000000000 7 | #define ITREE_ROOT_MAX 1000000 8 | typedef struct _ITNODE 9 | { 10 | uint32_t left; 11 | uint32_t right; 12 | uint32_t parent; 13 | uint32_t color; 14 | int data; 15 | int key; 16 | }ITNODE; 17 | typedef struct _ITROOT 18 | { 19 | uint16_t status; 20 | uint16_t flag; 21 | uint32_t total; 22 | uint32_t rootid; 23 | }ITROOT; 24 | typedef struct _ITSTATE 25 | { 26 | int kmax; 27 | int kmin; 28 | uint32_t nmax; 29 | uint32_t nmin; 30 | uint32_t count; 31 | uint32_t left; 32 | uint32_t current; 33 | uint32_t total; 34 | uint32_t qleft; 35 | uint32_t qfirst; 36 | uint32_t qlast; 37 | uint32_t nroots; 38 | ITROOT roots[ITREE_ROOT_MAX]; 39 | }ITSTATE; 40 | typedef struct _ITREE 41 | { 42 | int fd; 43 | int status; 44 | off_t size; 45 | off_t end; 46 | off_t old; 47 | void *start; 48 | ITSTATE *state; 49 | ITNODE *map; 50 | MUTEX *mutex; 51 | }ITREE; 52 | void *itree_init(char *file); 53 | int itree_new_tree(void *itree); 54 | uint32_t itree_total(void *itree, int rootid); 55 | uint32_t itree_try_insert(void *itree, int rootid, int key, int data, int *old); 56 | uint32_t itree_insert(void *itree, int rootid, int key, int data, int *old); 57 | uint32_t itree_get(void *itree, uint32_t nodeid, int *key, int *data); 58 | uint32_t itree_find(void *itree, int rootid, int key, int *data); 59 | uint32_t itree_min(void *itree, int rootid, int *key, int *data); 60 | uint32_t itree_max(void *itree, int rootid, int *key, int *data); 61 | uint32_t itree_next(void *itree, int rootid, uint32_t nodeid, int *key, int *data); 62 | uint32_t itree_prev(void *itree, int rootid, uint32_t nodeid, int *key, int *data); 63 | int itree_set_data(void *itree, uint32_t nodeid, int data); 64 | void itree_view_tree(void *itree, int rootid, FILE *fp); 65 | void itree_remove(void *itree, int rootid, uint32_t nodeid, int *key, int *data); 66 | void itree_remove_tree(void *itree, int rootid); 67 | void itree_close(void *itree); 68 | #endif 69 | -------------------------------------------------------------------------------- /libibase/src/utils/lkv.h: -------------------------------------------------------------------------------- 1 | #ifndef __LKV__H 2 | #define __LKV__H 3 | #include 4 | #include 5 | #define LVV_SLOT_MAX 524288 6 | #define LVV_SLOT_INC 2000 7 | #define LVV_SLOT_NUM 1024 8 | #define LVV_SLOT2_NUM 512 /* half of LVV_SLOT_NUM */ 9 | #define LVV_LVVV_INC 1000000 10 | #define LVV_NODES_MAX 4294967296 11 | //#define __LKV_USE_IDX__ 1 12 | #pragma pack(push, 4) 13 | #ifndef U32_T 14 | #define U32_T 15 | typedef uint32_t u32_t; 16 | #endif 17 | typedef struct LVVKV 18 | { 19 | int64_t key; /* 数值 */ 20 | u32_t val; /* 文档序号 */ 21 | }LVVKV; 22 | typedef struct _LVVV 23 | { 24 | int64_t val; /* 数值 */ 25 | #ifdef __LKV_USE_IDX__ 26 | int off; /* 节点偏移 */ 27 | #endif 28 | }LVVV; 29 | typedef struct _LVVSLOT 30 | { 31 | int64_t min; /* 当前桶的最小值 */ 32 | int64_t max; /* 当前桶的最大值 */ 33 | int nodeid; /* 桶偏移地址 */ 34 | int count; /* 桶元素个数 */ 35 | }LVVSLOT; 36 | typedef struct _LVVSTATE 37 | { 38 | LVVSLOT slots[LVV_SLOT_MAX]; /* 槽 */ 39 | u32_t qleft[LVV_SLOT_MAX]; /* 空闲的桶 */ 40 | u32_t roots[LVV_SLOT_MAX]; /* 桶和槽的衍射 */ 41 | u32_t nleft; /* 空闲桶个数 */ 42 | u32_t count; /* 槽的个数 */ 43 | }LVVSTATE; 44 | typedef struct _LKV 45 | { 46 | LVVSTATE *state; 47 | LVVKV *map; 48 | LVVV *vmap; 49 | LVVSLOT *slots; 50 | u32_t *roots; 51 | int fd; 52 | int vfd; 53 | off_t size; 54 | off_t msize; 55 | off_t vsize; 56 | off_t vmsize; 57 | pthread_rwlock_t rwlock; 58 | }LKV; 59 | LKV *lkv_init(char *file); 60 | int lkv_set(LKV *lkv, u32_t no, int64_t key); 61 | int lkv_get(LKV *lkv, u32_t no, u32_t *val); 62 | void lkv_close(LKV *lkv); 63 | #ifdef __LKV_USE_IDX__ 64 | /* return number of the hits */ 65 | int lkv_del(LKV *lkv, u32_t no); 66 | int lkv_range(LKV *lkv, int64_t from, int64_t to, u32_t *list); 67 | int lkv_rangefrom(LKV *lkv, int64_t key, u32_t *list); /* key = from */ 68 | int lkv_rangeto(LKV *lkv, int64_t key, u32_t *list); /* key = to */ 69 | int lkv_in(LKV *lkv, int64_t key, u32_t *list); 70 | int lkv_ins(LKV *lkv, int64_t *keys, int nkeys, u32_t *list); 71 | /* set list[] if (list != NULL) */ 72 | #define LKV_DEL(x, no) lkv_del(((LKV *)x), no) 73 | #else 74 | #define LKV_DEL(x, no) 75 | #endif 76 | #define LKV_GET(x, no) ((LKV *)x)->vmap[no].val 77 | #define LKV_SET(x, no, key) lkv_set(((LKV *)x), no, key) 78 | #pragma pack(pop) 79 | #endif 80 | -------------------------------------------------------------------------------- /libibase/src/utils/lmap.h: -------------------------------------------------------------------------------- 1 | #ifndef __LMAP__H 2 | #define __LMAP__H 3 | #include 4 | #include 5 | #define LMM_SLOT_MAX 524288 6 | #define LMM_SLOT_INC 2000 7 | #define LMM_SLOT_NUM 1024 8 | #define LMM_SLOT2_NUM 512 /* half of LMM_SLOT_NUM */ 9 | #define LMM_LMMV_INC 1000000 10 | #define LMM_NODES_MAX 4294967296 11 | #define __LMAP_USE_IDX__ 1 12 | #pragma pack(push, 4) 13 | #ifndef U32_T 14 | #define U32_T 15 | typedef uint32_t u32_t; 16 | #endif 17 | typedef struct LMMKV 18 | { 19 | int64_t key; /* 数值 */ 20 | u32_t val; /* 文档序号 */ 21 | }LMMKV; 22 | typedef struct _LMMV 23 | { 24 | int64_t val; /* 数值 */ 25 | #ifdef __LMAP_USE_IDX__ 26 | int off; /* 节点偏移 */ 27 | #endif 28 | }LMMV; 29 | typedef struct _LMMSLOT 30 | { 31 | int64_t min; /* 当前桶的最小值 */ 32 | int64_t max; /* 当前桶的最大值 */ 33 | int nodeid; /* 桶偏移地址 */ 34 | int count; /* 桶元素个数 */ 35 | }LMMSLOT; 36 | typedef struct _LMMSTATE 37 | { 38 | LMMSLOT slots[LMM_SLOT_MAX]; /* 槽 */ 39 | u32_t qleft[LMM_SLOT_MAX]; /* 空闲的桶 */ 40 | u32_t roots[LMM_SLOT_MAX]; /* 桶和槽的衍射 */ 41 | u32_t nleft; /* 空闲桶个数 */ 42 | u32_t count; /* 槽的个数 */ 43 | }LMMSTATE; 44 | typedef struct _LMAP 45 | { 46 | LMMSTATE *state; 47 | LMMKV *map; 48 | LMMV *vmap; 49 | LMMSLOT *slots; 50 | u32_t *roots; 51 | int fd; 52 | int vfd; 53 | off_t size; 54 | off_t msize; 55 | off_t vsize; 56 | off_t vmsize; 57 | pthread_rwlock_t rwlock; 58 | }LMAP; 59 | LMAP *lmap_init(char *file); 60 | int lmap_set(LMAP *lmap, u32_t no, int64_t key); 61 | int lmap_get(LMAP *lmap, u32_t no, u32_t *val); 62 | void lmap_close(LMAP *lmap); 63 | #ifdef __LMAP_USE_IDX__ 64 | /* return number of the hits */ 65 | int lmap_del(LMAP *lmap, u32_t no); 66 | int lmap_range(LMAP *lmap, int64_t from, int64_t to, u32_t *list); 67 | int lmap_rangefrom(LMAP *lmap, int64_t key, u32_t *list); /* key = from */ 68 | int lmap_rangeto(LMAP *lmap, int64_t key, u32_t *list); /* key = to */ 69 | int lmap_in(LMAP *lmap, int64_t key, u32_t *list); 70 | int lmap_ins(LMAP *lmap, int64_t *keys, int nkeys, u32_t *list); 71 | /* set list[] if (list != NULL) */ 72 | #define LMAP_DEL(x, no) lmap_del(((LMAP *)x), no) 73 | #else 74 | #define LMAP_DEL(x, no) 75 | #endif 76 | #define LMAP_GET(x, no) ((LMAP *)x)->vmap[no].val 77 | #define LMAP_SET(x, no, key) lmap_set(((LMAP *)x), no, key) 78 | #pragma pack(pop) 79 | #endif 80 | -------------------------------------------------------------------------------- /libibase/src/utils/ltree.h: -------------------------------------------------------------------------------- 1 | #ifndef _LTREE_H 2 | #define _LTREE_H 3 | #define LTREE_INCRE_NUM 1000000 4 | #define LTREE_NODES_MAX 2000000000 5 | #define LTREE_MUTEX_MAX 256 6 | #define LTREE_ROOT_MAX 1000000 7 | #include 8 | #include "mutex.h" 9 | typedef struct _LTNODE 10 | { 11 | uint32_t left; 12 | uint32_t right; 13 | uint32_t parent; 14 | uint32_t color; 15 | int data; 16 | int bit; 17 | int64_t key; 18 | }LTNODE; 19 | typedef struct _LTROOT 20 | { 21 | int16_t status; 22 | int16_t flag; 23 | uint32_t total; 24 | uint32_t rootid; 25 | }LTROOT; 26 | typedef struct _LTSTATE 27 | { 28 | int64_t kmax; 29 | int64_t kmin; 30 | uint32_t nmax; 31 | uint32_t nmin; 32 | uint32_t count; 33 | uint32_t left; 34 | uint32_t current; 35 | uint32_t total; 36 | uint32_t qleft; 37 | uint32_t qfirst; 38 | uint32_t qlast; 39 | uint32_t nroots; 40 | LTROOT roots[LTREE_ROOT_MAX]; 41 | }LTSTATE; 42 | typedef struct _LTREE 43 | { 44 | int fd; 45 | int status; 46 | off_t size; 47 | off_t end; 48 | off_t old; 49 | char *start; 50 | LTSTATE *state; 51 | LTNODE *map; 52 | MUTEX *mutex; 53 | #ifdef HAVE_PTHREAD 54 | pthread_mutex_t mutexs[LTREE_MUTEX_MAX]; 55 | #endif 56 | }LTREE; 57 | void *ltree_init(char *file); 58 | int ltree_new_tree(void *ltree); 59 | uint32_t ltree_total(void *ltree, int rootid); 60 | uint32_t ltree_try_insert(void *ltree, int rootid, int64_t key, int data, int *old); 61 | uint32_t ltree_insert(void *ltree, int rootid, int64_t key, int data, int *old); 62 | uint32_t ltree_get(void *ltree, uint32_t nodeid, int64_t *key, int *data); 63 | uint32_t ltree_find(void *ltree, int rootid, int64_t key, int *data); 64 | uint32_t ltree_min(void *ltree, int rootid, int64_t *key, int *data); 65 | uint32_t ltree_max(void *ltree, int rootid, int64_t *key, int *data); 66 | uint32_t ltree_next(void *ltree, int rootid, uint32_t nodeid, int64_t *key, int *data); 67 | uint32_t ltree_prev(void *ltree, int rootid, uint32_t nodeid, int64_t *key, int *data); 68 | int ltree_set_data(void *ltree, uint32_t nodeid, int data); 69 | void ltree_view_tree(void *ltree, int rootid, FILE *fp); 70 | void ltree_remove(void *ltree, int rootid, uint32_t nodeid, int64_t *key, int *data); 71 | void ltree_remove_tree(void *ltree, int rootid); 72 | void ltree_close(void *ltree); 73 | #endif 74 | -------------------------------------------------------------------------------- /libibase/src/utils/rwlock.h: -------------------------------------------------------------------------------- 1 | #ifndef __RWLOCK__H 2 | #define __RWLOCK__H 3 | #ifdef HAVE_PTHREAD 4 | #define RWLOCK_INIT(x) pthread_rwlock_init(&(x), NULL) 5 | #define RWLOCK_RDLOCK(x) pthread_rwlock_rdlock(&(x)) 6 | #define RWLOCK_WRLOCK(x) pthread_rwlock_wrlock(&(x)) 7 | #define RWLOCK_UNLOCK(x) pthread_rwlock_unlock(&(x)) 8 | #define RWLOCK_DESTROY(x) pthread_rwlock_destroy(&(x)) 9 | #else 10 | #define RWLOCK_INIT(x) 11 | #define RWLOCK_RDLOCK(x) 12 | #define RWLOCK_WRLOCK(x) 13 | #define RWLOCK_UNLOCK(x) 14 | #define RWLOCK_DESTROY(x) 15 | #endif 16 | #endif 17 | -------------------------------------------------------------------------------- /libibase/src/utils/xmm.h: -------------------------------------------------------------------------------- 1 | #ifndef _XMM_H_ 2 | #define _XMM_H_ 3 | void* xmm_mnew(size_t size); 4 | void* xmm_new(size_t size); 5 | void* xmm_mrenew(void *old, size_t old_size, size_t new_size); 6 | void* xmm_renew(void *old, size_t old_size, size_t new_size); 7 | void* xmm_mresize(void *old, size_t old_size, size_t new_size); 8 | void* xmm_resize(void *old, size_t old_size, size_t new_size); 9 | void xmm_free(void *m, size_t size); 10 | #endif 11 | -------------------------------------------------------------------------------- /libibase/src/utils/zvbcode.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include "zvbcode.h" 5 | #define VB_NUM_MAX 100000000 6 | #ifdef _DEBUG_ZVBCODE 7 | #include "timer.h" 8 | #define __ZINT_OUT__ 0 9 | #define __ZINT__ "%d\n" 10 | typedef int ZINT; 11 | int main(int argc, char **argv) 12 | { 13 | int i = 0, n = 0, x = 0; 14 | ZINT rand = 0, *dp = NULL, *np = NULL; 15 | char *p = NULL, *ps = NULL, *ep = NULL; 16 | FILE *fp = NULL, *dfp = NULL; 17 | void *timer = NULL; 18 | 19 | if((ps = (char *)calloc(VB_NUM_MAX*2, sizeof(ZINT)))) 20 | { 21 | TIMER_INIT(timer); 22 | n = VB_NUM_MAX * sizeof(ZINT); 23 | if((fp = fopen("/tmp/double.txt", "a+"))) 24 | { 25 | p = ps; 26 | np = &rand; 27 | for(i = 0; i < VB_NUM_MAX; i++) 28 | { 29 | rand = (ZINT)random()%100000000; 30 | if(__ZINT_OUT__)fprintf(fp, __ZINT__, rand); 31 | ZVBCODE(np, p); 32 | } 33 | TIMER_SAMPLE(timer); 34 | fflush(fp); 35 | fprintf(stdout, "compressed(sizeof(zint):%d) %d to %d time:%lld avg:%f\n", 36 | sizeof(ZINT), n, (p - ps), PT_LU_USEC(timer), 37 | (double)PT_LU_USEC(timer)/(double)VB_NUM_MAX); 38 | fclose(fp); 39 | } 40 | if((dfp = fopen("/tmp/double.text", "a+"))) 41 | { 42 | ep = p; 43 | p = ps; 44 | np = &rand; 45 | TIMER_SAMPLE(timer); 46 | i = 0; 47 | while(p < ep) 48 | { 49 | rand = 0; 50 | UZVBCODE(p, x, np); 51 | i++; 52 | if(__ZINT_OUT__) fprintf(dfp, __ZINT__, rand); 53 | } 54 | TIMER_SAMPLE(timer); 55 | fflush(fp); 56 | fprintf(stdout, "decompressed(typesizeof(ZINT):%d) %d to %d time:%lld avg:%f\n", 57 | sizeof(ZINT), (p - ps), i * sizeof(ZINT), PT_LU_USEC(timer), 58 | (double)PT_LU_USEC(timer)/(double)i); 59 | fclose(dfp); 60 | } 61 | free(ps); 62 | ps = NULL; 63 | } 64 | } 65 | #endif 66 | -------------------------------------------------------------------------------- /libscws/COPYING: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sounos/ithunder/241d17ce44ed90bc3d8ce4f0b8d5ae52498db270/libscws/COPYING -------------------------------------------------------------------------------- /libscws/ChangeLog: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sounos/ithunder/241d17ce44ed90bc3d8ce4f0b8d5ae52498db270/libscws/ChangeLog -------------------------------------------------------------------------------- /libscws/Makefile.am: -------------------------------------------------------------------------------- 1 | # $Id: Makefile.am,v 1.5 2009/05/26 02:44:07 hightman Exp $ 2 | # automake file by hightman for SCWS-0.1.x 3 | # 4 | 5 | AUTOMAKE_OPTIONS = foreign 6 | 7 | SUBDIRS = . libscws cli etc dict 8 | 9 | DIST_SUBDIRS = libscws cli etc dict 10 | 11 | EXTRA_DIST = COPYING README ChangeLog phpext win32 debian.sh 12 | 13 | dist-hook: 14 | -rm -rf $(distdir)/phpext/CVS $(distdir)/phpext/win32/CVS $(distdir)/win32/CVS 15 | -------------------------------------------------------------------------------- /libscws/README: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sounos/ithunder/241d17ce44ed90bc3d8ce4f0b8d5ae52498db270/libscws/README -------------------------------------------------------------------------------- /libscws/cli/Makefile.am: -------------------------------------------------------------------------------- 1 | # $Id: Makefile.am,v 1.2 2010/07/10 06:58:12 hightman Exp $ 2 | # 3 | 4 | #LIBTOOL = @LIBTOOL@ $(QUIET:y=--quiet) 5 | 6 | libscwsdir = $(top_srcdir)/libscws 7 | 8 | INCLUDES = -I$(top_srcdir) -I$(libscwsdir) 9 | 10 | bin_PROGRAMS = scws gen-scws-dict 11 | 12 | scws_SOURCES = scws_cmd.c 13 | scws_LDADD = $(libscwsdir)/libscws.la 14 | 15 | gen_scws_dict_SOURCES = gen_dict.c 16 | gen_scws_dict_LDADD = $(libscwsdir)/libscws.la 17 | 18 | -------------------------------------------------------------------------------- /libscws/cli/scws_cmd.c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sounos/ithunder/241d17ce44ed90bc3d8ce4f0b8d5ae52498db270/libscws/cli/scws_cmd.c -------------------------------------------------------------------------------- /libscws/configure.in: -------------------------------------------------------------------------------- 1 | # -*- Autoconf -*- 2 | # Process this file with autoconf to produce a configure script. 3 | 4 | AC_PREREQ([2.59]) 5 | AC_INIT(libscws, [1.2.2], [http://www.ftphp.com/scws]) 6 | AC_CONFIG_SRCDIR([config.h.in]) 7 | AC_CONFIG_HEADERS([config.h]) 8 | AM_INIT_AUTOMAKE 9 | 10 | # Checks for programs. 11 | AC_PROG_CXX 12 | AC_PROG_AWK 13 | AC_PROG_CC 14 | AC_PROG_CPP 15 | AC_PROG_INSTALL 16 | AC_PROG_LN_S 17 | AC_PROG_MAKE_SET 18 | AC_PROG_LIBTOOL 19 | 20 | # libtool 21 | LIBTOOL="$LIBTOOL --preserve-dup-deps" 22 | AC_SUBST(LIBTOOL) 23 | 24 | # libtool compile 25 | SHARED_LIB_VERSION=2:0:1 26 | AC_SUBST(SHARED_LIB_VERSION) 27 | 28 | # Checks for libraries. 29 | # FIXME: Replace `main' with a function in `-lm': 30 | AC_CHECK_LIB([m], [main]) 31 | 32 | # Checks for header files. 33 | AC_CHECK_HEADERS([fcntl.h netinet/in.h stdlib.h string.h sys/file.h sys/param.h sys/time.h unistd.h]) 34 | 35 | # Checks for typedefs, structures, and compiler characteristics. 36 | AC_C_INLINE 37 | 38 | # Checks for library functions. 39 | AC_FUNC_LSTAT_FOLLOWS_SLASHED_SYMLINK 40 | AC_FUNC_MALLOC 41 | AC_FUNC_MMAP 42 | AC_CHECK_FUNCS([getcwd gettimeofday memset munmap pow strcasecmp strchr strdup strrchr]) 43 | 44 | AC_CONFIG_FILES([Makefile 45 | cli/Makefile 46 | dict/Makefile 47 | etc/Makefile 48 | libscws/Makefile 49 | libscws/version.h]) 50 | AC_OUTPUT 51 | -------------------------------------------------------------------------------- /libscws/debian.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | rm -rf debian 3 | pkg=`grep -E "^PACKAGE_NAME=" configure |sed -e "s/PACKAGE_NAME='\(.*\)'/\1/"` 4 | ver=`grep -E "^PACKAGE_VERSION=" configure |sed -e "s/PACKAGE_VERSION='\(.*\)'/\1/"` 5 | dh_make -y -n -s -c bsd -e SounOS@gmail.com -p "${pkg}_${ver}" 6 | cp control debian/control 7 | sed -i "s/^Package:.*/Package: $pkg/" debian/control 8 | sed -i "s/^Source:.*/Source: $pkg/" debian/control 9 | sed -i "s/^License:.*/License: $pkg/" debian/control 10 | sed -i "s/^Standards-Version:.*/Standards-Version: $ver/" debian/control 11 | sed -i "s/root /SounOS /" debian/* 12 | #export CFLAGS='-ggdb -O0' 13 | #export CXXFLAGS='-ggdb -O0' 14 | #export FFLAGS='-ggdb -O0' 15 | dpkg-buildpackage -rfakeroot -eSounOS -mSounOS 16 | -------------------------------------------------------------------------------- /libscws/dict/Makefile.am: -------------------------------------------------------------------------------- 1 | EXTRA_DIST = dict_cht.utf8.xdb dict.utf8.xdb dict.gbk.xdb 2 | dist = @prefix@/var/dict 3 | dictdir = "`echo "$(dist)"|sed -e 's|/usr/var|/var|g'`" 4 | dict_DATA = dict_cht.utf8.xdb dict.utf8.xdb dict.gbk.xdb 5 | -------------------------------------------------------------------------------- /libscws/dict/dict.gbk.xdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sounos/ithunder/241d17ce44ed90bc3d8ce4f0b8d5ae52498db270/libscws/dict/dict.gbk.xdb -------------------------------------------------------------------------------- /libscws/dict/dict.utf8.xdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sounos/ithunder/241d17ce44ed90bc3d8ce4f0b8d5ae52498db270/libscws/dict/dict.utf8.xdb -------------------------------------------------------------------------------- /libscws/dict/dict_cht.utf8.xdb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sounos/ithunder/241d17ce44ed90bc3d8ce4f0b8d5ae52498db270/libscws/dict/dict_cht.utf8.xdb -------------------------------------------------------------------------------- /libscws/etc/Makefile.am: -------------------------------------------------------------------------------- 1 | # $Id: Makefile.am,v 1.3 2008/12/21 05:01:15 hightman Exp $ 2 | # 3 | 4 | EXTRA_DIST = rules.ini rules.utf8.ini rules_cht.utf8.ini 5 | 6 | sysconf_DATA = rules.ini rules.utf8.ini rules_cht.utf8.ini 7 | -------------------------------------------------------------------------------- /libscws/etc/rules.ini: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sounos/ithunder/241d17ce44ed90bc3d8ce4f0b8d5ae52498db270/libscws/etc/rules.ini -------------------------------------------------------------------------------- /libscws/libscws/Makefile.am: -------------------------------------------------------------------------------- 1 | # $Id$ 2 | 3 | LIBTOOL = @LIBTOOL@ $(QUIET:y=--quiet) 4 | 5 | libscwsincludedir = @prefix@/include/scws 6 | 7 | libscwsinclude_HEADERS = charset.h crc32.h pool.h scws.h xdict.h darray.h rule.h xdb.h xtree.h version.h 8 | 9 | lib_LTLIBRARIES = libscws.la 10 | 11 | libscws_la_SOURCES = charset.c crc32.c pool.c scws.c xdict.c darray.c rule.c lock.c xdb.c xtree.c 12 | 13 | libscws_la_LDFLAGS = @LDFLAGS@ -no-undefined -version-info @SHARED_LIB_VERSION@ 14 | 15 | EXTRA_DIST = lock.h config_win32.h 16 | -------------------------------------------------------------------------------- /libscws/libscws/charset.h: -------------------------------------------------------------------------------- 1 | /** 2 | * @file charset.h 3 | * @author Hightman Mar 4 | * @editor set number ; syntax on ; set autoindent ; set tabstop=4 (vim) 5 | * $Id$ 6 | */ 7 | 8 | #ifndef _SCWS_CHARSET_20070528_H_ 9 | #define _SCWS_CHARSET_20070528_H_ 10 | 11 | /* api used to change the charset assocted */ 12 | unsigned char *charset_table_get(const char *cs); 13 | 14 | #endif 15 | -------------------------------------------------------------------------------- /libscws/libscws/config_win32.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFIG_W32_H 2 | #define CONFIG_W32_H 3 | 4 | #include 5 | #include 6 | 7 | #ifndef inline 8 | # define inline __inline 9 | #endif 10 | 11 | #define strcasecmp(s1, s2) _stricmp(s1, s2) 12 | #define strncasecmp(s1, s2, n) strnicmp(s1, s2, n) 13 | 14 | #ifndef S_ISREG 15 | #define S_ISREG(m) (((m) & S_IFMT) == S_IFREG) 16 | #endif 17 | 18 | #ifndef logf 19 | #define logf(x) ((float)log((double)(x))) 20 | #endif 21 | 22 | #endif 23 | -------------------------------------------------------------------------------- /libscws/libscws/crc32.h: -------------------------------------------------------------------------------- 1 | /** 2 | * @file crc32.h 3 | * @author Hightman Mar 4 | * @editor set number ; syntax on ; set autoindent ; set tabstop=4 (vim) 5 | * $Id$ 6 | */ 7 | 8 | #ifndef _SCWS_CRC32_20100128_H_ 9 | #define _SCWS_CRC32_20100128_H_ 10 | 11 | unsigned int scws_crc32(const char *str); 12 | 13 | #endif 14 | -------------------------------------------------------------------------------- /libscws/libscws/darray.c: -------------------------------------------------------------------------------- 1 | /** 2 | * @file xdict.c (dictionary query) 3 | * @author Hightman Mar 4 | * @editor set number ; syntax on ; set autoindent ; set tabstop=4 (vim) 5 | * $Id$ 6 | */ 7 | 8 | #include "darray.h" 9 | #include 10 | #include 11 | #include 12 | 13 | void **darray_new(int row, int col, int size) 14 | { 15 | void **arr; 16 | 17 | arr = (void **) malloc(sizeof(void *) * row + size * row * col); 18 | if (arr != NULL) 19 | { 20 | void *head; 21 | 22 | head = (void *) ((char *)arr + sizeof(void *) * row); 23 | memset(arr, 0, sizeof(void *) * row + size * row * col); 24 | while (row--) 25 | arr[row] = (void *)((char *)head + size * row * col); 26 | } 27 | return arr; 28 | } 29 | 30 | void darray_free(void **arr) 31 | { 32 | if (arr != NULL) 33 | free(arr); 34 | } 35 | 36 | -------------------------------------------------------------------------------- /libscws/libscws/darray.h: -------------------------------------------------------------------------------- 1 | /** 2 | * @file darray.h (double array) 3 | * @author Hightman Mar 4 | * @editor set number ; syntax on ; set autoindent ; set tabstop=4 (vim) 5 | * $Id$ 6 | */ 7 | 8 | #ifndef _SCWS_DARRAY_20070525_H_ 9 | #define _SCWS_DARRAY_20070525_H_ 10 | 11 | #ifdef __cplusplus 12 | extern "C" { 13 | #endif 14 | 15 | void **darray_new(int row, int col, int size); 16 | void darray_free(void **arr); 17 | 18 | #ifdef __cplusplus 19 | } 20 | #endif 21 | 22 | #endif 23 | -------------------------------------------------------------------------------- /libscws/libscws/lock.h: -------------------------------------------------------------------------------- 1 | /* 2 | +----------------------------------------------------------------------+ 3 | | PHP Version 5 | 4 | +----------------------------------------------------------------------+ 5 | | Copyright (c) 1997-2007 The PHP Group | 6 | +----------------------------------------------------------------------+ 7 | | This source file is subject to version 3.01 of the PHP license, | 8 | | that is bundled with this package in the file LICENSE, and is | 9 | | available through the world-wide-web at the following url: | 10 | | http://www.php.net/license/3_01.txt | 11 | | If you did not receive a copy of the PHP license and are unable to | 12 | | obtain it through the world-wide-web, please send a note to | 13 | | license@php.net so we can mail you a copy immediately. | 14 | +----------------------------------------------------------------------+ 15 | | Author: Sascha Schumann | 16 | +----------------------------------------------------------------------+ 17 | */ 18 | 19 | /* $Id$ */ 20 | 21 | #ifndef LOCK_H 22 | #define LOCK_H 23 | 24 | /* php_flock internally uses fcntl whether or not flock is available 25 | * This way our php_flock even works on NFS files. 26 | * More info: /usr/src/linux/Documentation 27 | */ 28 | int _xdb_flock(int fd, int operation); 29 | 30 | #ifndef HAVE_FLOCK 31 | # define LOCK_SH 1 32 | # define LOCK_EX 2 33 | # define LOCK_NB 4 34 | # define LOCK_UN 8 35 | #endif 36 | 37 | #ifdef WIN32 38 | # define EWOULDBLOCK WSAEWOULDBLOCK 39 | # define fsync _commit 40 | # define ftruncate(a, b) chsize(a, b) 41 | # include 42 | #endif /* defined(WIN32) */ 43 | 44 | #endif /* LOCK_H */ 45 | -------------------------------------------------------------------------------- /libscws/libscws/pool.h: -------------------------------------------------------------------------------- 1 | /** 2 | * @file pool.h 3 | * @author Hightman Mar 4 | * @editor set number ; syntax on ; set autoindent ; set tabstop=4 (vim) 5 | * $Id$ 6 | */ 7 | 8 | #ifndef _SCWS_POOL_20070525_H_ 9 | #define _SCWS_POOL_20070525_H_ 10 | 11 | #ifdef __cplusplus 12 | extern "C" { 13 | #endif 14 | 15 | /* block size for pool */ 16 | #define POOL_BLK_SIZ 4096 17 | 18 | /* data structure for pool */ 19 | struct pheap 20 | { 21 | int size; 22 | int used; 23 | char block[0]; 24 | }; 25 | 26 | struct pclean 27 | { 28 | void *obj; 29 | struct pclean *nxt; 30 | }; 31 | 32 | typedef struct 33 | { 34 | int size; /* total allocated */ 35 | int dirty; /* total wasted */ 36 | struct pheap *heap; 37 | struct pclean *clean; 38 | } pool_st, *pool_t; 39 | 40 | /* pool: api */ 41 | pool_t pool_new(); /* create a new memory pool with an initial heap size */ 42 | void pool_free(pool_t p); /* frees all the data on the pool & delete the pool itself */ 43 | void *pmalloc(pool_t p, int size); /* wrapper around malloc, takes from the pool */ 44 | void *pmalloc_x(pool_t p, int size, char c); /* wrapper around pmalloc which prefills buffer with c */ 45 | void *pmalloc_z(pool_t p, int size); /* wrapper around pmalloc, which prefills by zero */ 46 | char *pstrdup(pool_t p, const char *s); /* wrapper around strdup, gains the mem from the pool */ 47 | char *pstrndup(pool_t p, const char *s, int l); 48 | 49 | #ifdef __cplusplus 50 | } 51 | #endif 52 | 53 | #endif 54 | -------------------------------------------------------------------------------- /libscws/libscws/rule.h: -------------------------------------------------------------------------------- 1 | /** 2 | * @file rule.h 3 | * @author Hightman Mar 4 | * @editor set number ; syntax on ; set autoindent ; set tabstop=4 (vim) 5 | * $Id$ 6 | */ 7 | 8 | #ifndef _SCWS_RULE_20070525_H_ 9 | #define _SCWS_RULE_20070525_H_ 10 | 11 | /* xtree required */ 12 | #include "xtree.h" 13 | 14 | #define SCWS_RULE_MAX 32 15 | #define SCWS_RULE_SPECIAL 0x80000000 16 | #define SCWS_RULE_NOSTATS 0x40000000 17 | 18 | /* flag: 0x00 ~ 0x4000 */ 19 | #define SCWS_ZRULE_NONE 0x00 20 | #define SCWS_ZRULE_PREFIX 0x01 21 | #define SCWS_ZRULE_SUFFIX 0x02 22 | #define SCWS_ZRULE_INCLUDE 0x04 /* with include */ 23 | #define SCWS_ZRULE_EXCLUDE 0x08 /* with exclude */ 24 | #define SCWS_ZRULE_RANGE 0x10 /* with znum range */ 25 | 26 | /* data structure */ 27 | typedef struct scws_rule_item 28 | { 29 | short flag; 30 | char zmin; 31 | char zmax; 32 | char name[17]; 33 | char attr[3]; 34 | float tf; 35 | float idf; 36 | unsigned int bit; /* my bit */ 37 | unsigned int inc; /* include */ 38 | unsigned int exc; /* exclude */ 39 | } *rule_item_t; 40 | 41 | /* special attrs ratio list(single chain, 12bytes) */ 42 | typedef struct scws_rule_attr *rule_attr_t; 43 | struct scws_rule_attr 44 | { 45 | char attr1[2]; 46 | char attr2[2]; 47 | unsigned char npath[2]; 48 | short ratio; 49 | rule_attr_t next; 50 | }; 51 | 52 | typedef struct scws_rule 53 | { 54 | xtree_t tree; 55 | rule_attr_t attr; 56 | struct scws_rule_item items[SCWS_RULE_MAX]; 57 | int ref; // hightman.20130110: refcount (zero to really free/close) 58 | } rule_st, *rule_t; 59 | 60 | /* scws ruleset: api */ 61 | 62 | /* create & load ruleset, by fpath & charset */ 63 | rule_t scws_rule_new(const char *fpath, unsigned char *mblen); 64 | 65 | /* fork ruleset */ 66 | rule_t scws_rule_fork(rule_t r); 67 | 68 | /* free the memory & resource for ruleset */ 69 | void scws_rule_free(rule_t r); 70 | 71 | /* get the rule tree record by str */ 72 | rule_item_t scws_rule_get(rule_t r, const char *str, int len); 73 | 74 | /* check bit */ 75 | int scws_rule_checkbit(rule_t r, const char *str, int len, unsigned int bit); 76 | 77 | /* get rule attr x */ 78 | int scws_rule_attr_ratio(rule_t r, const char *attr1, const char *attr2, const unsigned char *npath); 79 | 80 | /* check exclude or include */ 81 | int scws_rule_check(rule_t r, rule_item_t cr, const char *str, int len); 82 | 83 | #endif 84 | -------------------------------------------------------------------------------- /libscws/libscws/version.h: -------------------------------------------------------------------------------- 1 | /* version.h.in. input file for configure */ 2 | #define SCWS_VERSION "1.2.2" 3 | #define SCWS_BUGREPORT "http://www.ftphp.com/scws" 4 | 5 | -------------------------------------------------------------------------------- /libscws/libscws/version.h.in: -------------------------------------------------------------------------------- 1 | /* version.h.in. input file for configure */ 2 | #define SCWS_VERSION "@VERSION@" 3 | #define SCWS_BUGREPORT "@PACKAGE_BUGREPORT@" 4 | 5 | -------------------------------------------------------------------------------- /libscws/libscws/xdb.h: -------------------------------------------------------------------------------- 1 | /** 2 | * @file xdb.h (read only) 3 | * @author Hightman Mar 4 | * @editor set number ; syntax on ; set autoindent ; set tabstop=4 (vim) 5 | * $Id$ 6 | */ 7 | 8 | #ifndef _SCWS_XDB_20070525_H_ 9 | #define _SCWS_XDB_20070525_H_ 10 | 11 | #ifdef __cplusplus 12 | extern "C" { 13 | #endif 14 | 15 | /* constant var define */ 16 | #define XDB_FLOAT_CHECK (3.14) 17 | #define XDB_TAGNAME "XDB" 18 | #define XDB_MAXKLEN 0xf0 19 | #define XDB_VERSION 34 /* version: 3bit+5bit */ 20 | 21 | #include "xtree.h" 22 | 23 | /* data structure for [Record] */ 24 | typedef struct xdb_pointer 25 | { 26 | unsigned int off; 27 | unsigned int len; 28 | } xptr_st, *xptr_t; 29 | 30 | typedef struct xdb_record 31 | { 32 | unsigned int poff; 33 | xptr_st me; 34 | xptr_st value; 35 | } xrec_st, *xrec_t; 36 | 37 | /* header struct */ 38 | struct xdb_header 39 | { 40 | char tag[3]; 41 | unsigned char ver; 42 | int base; 43 | int prime; 44 | unsigned int fsize; 45 | float check; 46 | char unused[12]; 47 | }; 48 | 49 | typedef struct 50 | { 51 | int fd; /* file descriptoin */ 52 | int base; /* basenum for hash count */ 53 | int prime; /* base prime for hash mod */ 54 | unsigned int fsize; /* total filesize */ 55 | int version; /* version: low 4bytes */ 56 | char *fmap; /* file content image by mmap (read only) */ 57 | int mode; /* xdb_open for write or read-only */ 58 | } xdb_st, *xdb_t; 59 | 60 | /* xdb: open the db, mode = r|w|n */ 61 | xdb_t xdb_open(const char *fpath, int mode); 62 | xdb_t xdb_create(const char *fpath, int base, int prime); 63 | 64 | /* read mode */ 65 | void *xdb_nget(xdb_t x, const char *key, int len, unsigned int *vlen); 66 | void *xdb_get(xdb_t x, const char *key, unsigned int *vlen); 67 | 68 | #ifdef DEBUG 69 | void xdb_draw(xdb_t x); 70 | #endif 71 | 72 | /* return the xtree pointer */ 73 | xtree_t xdb_to_xtree(xdb_t x, xtree_t xt); 74 | 75 | /* write mode */ 76 | void xdb_nput(xdb_t x, void *value, unsigned int vlen, const char *key, int len); 77 | void xdb_put(xdb_t x, const char *value, const char *key); 78 | void xdb_optimize(xdb_t x); 79 | 80 | /* xdb: close the db */ 81 | void xdb_close(xdb_t x); 82 | void xdb_version(xdb_t x); 83 | 84 | #ifdef __cplusplus 85 | } 86 | #endif 87 | 88 | #endif 89 | -------------------------------------------------------------------------------- /libscws/libscws/xdict.h: -------------------------------------------------------------------------------- 1 | /** 2 | * @file xdict (dictionary) 3 | * @author Hightman Mar 4 | * @editor set number ; syntax on ; set autoindent ; set tabstop=4 (vim) 5 | * $Id$ 6 | */ 7 | 8 | #ifndef _SCWS_XDICT_20070528_H_ 9 | #define _SCWS_XDICT_20070528_H_ 10 | 11 | #ifdef __cplusplus 12 | extern "C" { 13 | #endif 14 | 15 | /* constant var define */ 16 | #define SCWS_WORD_FULL 0x01 // 多字: 整词 17 | #define SCWS_WORD_PART 0x02 // 多字: 前词段 18 | #define SCWS_WORD_USED 0x04 // 多字: 已使用 19 | #define SCWS_WORD_RULE 0x08 // 多字: 自动识别的 20 | #define SCWS_WORD_LONG 0x10 // 多字: 短词组成的长词 21 | 22 | #define SCWS_WORD_MALLOCED 0x80 // xdict_query 结果必须调用 free 23 | 24 | #define SCWS_ZFLAG_PUT 0x02 // 单字: 已使用 25 | #define SCWS_ZFLAG_N2 0x04 // 单字: 双字名词头 26 | #define SCWS_ZFLAG_NR2 0x08 // 单字: 词头且为双字人名 27 | #define SCWS_ZFLAG_WHEAD 0x10 // 单字: 词头 28 | #define SCWS_ZFLAG_WPART 0x20 // 单字: 词尾或词中 29 | #define SCWS_ZFLAG_ENGLISH 0x40 // 单字: 夹在中间的英文 30 | #define SCWS_ZFLAG_SYMBOL 0x80 // 单字: 符号系列 31 | #define SCWS_XDICT_PRIME 0x3ffd // 词典结构树数：16381 32 | 33 | /* xdict open mode */ 34 | #define SCWS_XDICT_XDB 1 35 | #define SCWS_XDICT_MEM 2 36 | #define SCWS_XDICT_TXT 4 // ... 37 | #define SCWS_XDICT_SET 4096 // set flag. 38 | 39 | /* data structure for word(12bytes) */ 40 | typedef struct scws_word 41 | { 42 | float tf; 43 | float idf; 44 | unsigned char flag; 45 | char attr[3]; 46 | } word_st, *word_t; 47 | 48 | typedef struct scws_xdict 49 | { 50 | void *xdict; 51 | int xmode; 52 | int ref; // hightman.20130110: refcount (zero to really free/close) 53 | struct scws_xdict *next; 54 | } xdict_st, *xdict_t; 55 | 56 | /* pub function (api) */ 57 | xdict_t xdict_open(const char *fpath, int mode); 58 | void xdict_close(xdict_t xd); 59 | 60 | /* fork xdict */ 61 | xdict_t xdict_fork(xdict_t xd); 62 | 63 | /* add a new dict file into xd, succ: 0, error: -1, Mblen only used for XDICT_TXT */ 64 | xdict_t xdict_add(xdict_t xd, const char *fpath, int mode, unsigned char *ml); 65 | 66 | /* NOW this is ThreadSafe function */ 67 | word_t xdict_query(xdict_t xd, const char *key, int len); 68 | 69 | #ifdef __cplusplus 70 | } 71 | #endif 72 | 73 | #endif 74 | -------------------------------------------------------------------------------- /libscws/libscws/xtree.h: -------------------------------------------------------------------------------- 1 | /** 2 | * @file xtree.h 3 | * @author Hightman Mar 4 | * @editor set number ; syntax on ; set autoindent ; set tabstop=4 (vim) 5 | * $Id$ 6 | */ 7 | 8 | #ifndef _SCWS_XTREE_20070525_H_ 9 | #define _SCWS_XTREE_20070525_H_ 10 | 11 | #ifdef __cplusplus 12 | extern "C" { 13 | #endif 14 | 15 | /* pool required */ 16 | #include "pool.h" 17 | 18 | /* data structure for Hash+Tree */ 19 | typedef struct tree_node node_st, *node_t; 20 | struct tree_node 21 | { 22 | char *key; 23 | void *value; 24 | int vlen; 25 | node_t left; 26 | node_t right; 27 | }; 28 | 29 | typedef struct 30 | { 31 | pool_t p; /* pool for memory manager */ 32 | int base; /* base number for hasher (prime number recommend) */ 33 | int prime; /* good prime number for hasher */ 34 | int count; /* total nodes */ 35 | node_t *trees; /* trees [total=prime+1] */ 36 | } xtree_st, *xtree_t; 37 | 38 | /* xtree: api */ 39 | int xtree_hasher(xtree_t xt, const char *key, int len); 40 | xtree_t xtree_new(int base, int prime); /* create a new hasxtree */ 41 | void xtree_free(xtree_t xt); /* delete & free xthe xtree */ 42 | 43 | void xtree_put(xtree_t xt, const char *value, const char *key); 44 | void xtree_nput(xtree_t xt, void *value, int vlen, const char *key, int len); 45 | 46 | void *xtree_get(xtree_t xt, const char *key, int *vlen); 47 | void *xtree_nget(xtree_t xt, const char *key, int len, int *vlen); 48 | 49 | /* 50 | void xtree_del(xtree_t xt, const char *key); 51 | void xtree_ndel(xtree_t xt, const char *key, int len); 52 | */ 53 | 54 | #ifdef DEBUG 55 | void xtree_draw(xtree_t xt); 56 | #endif 57 | 58 | void xtree_optimize(xtree_t xt); 59 | void xtree_to_xdb(xtree_t xt, const char *fpath); 60 | 61 | #ifdef __cplusplus 62 | } 63 | #endif 64 | 65 | #endif 66 | -------------------------------------------------------------------------------- /libscws/mkinstalldirs: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | # mkinstalldirs --- make directory hierarchy 3 | # Author: Noah Friedman 4 | # Created: 1993-05-16 5 | # Public domain 6 | 7 | errstatus=0 8 | dirmode="" 9 | 10 | usage="\ 11 | Usage: mkinstalldirs [-h] [--help] [-m mode] dir ..." 12 | 13 | # process command line arguments 14 | while test $# -gt 0 ; do 15 | case "${1}" in 16 | -h | --help | --h* ) # -h for help 17 | echo "${usage}" 1>&2; exit 0 ;; 18 | -m ) # -m PERM arg 19 | shift 20 | test $# -eq 0 && { echo "${usage}" 1>&2; exit 1; } 21 | dirmode="${1}" 22 | shift ;; 23 | -- ) shift; break ;; # stop option processing 24 | -* ) echo "${usage}" 1>&2; exit 1 ;; # unknown option 25 | * ) break ;; # first non-opt arg 26 | esac 27 | done 28 | 29 | for file 30 | do 31 | if test -d "$file"; then 32 | shift 33 | else 34 | break 35 | fi 36 | done 37 | 38 | case $# in 39 | 0) exit 0 ;; 40 | esac 41 | 42 | case $dirmode in 43 | '') 44 | if mkdir -p -- . 2>/dev/null; then 45 | echo "mkdir -p -- $*" 46 | exec mkdir -p -- "$@" 47 | fi ;; 48 | *) 49 | if mkdir -m "$dirmode" -p -- . 2>/dev/null; then 50 | echo "mkdir -m $dirmode -p -- $*" 51 | exec mkdir -m "$dirmode" -p -- "$@" 52 | fi ;; 53 | esac 54 | 55 | for file 56 | do 57 | set fnord `echo ":$file" | sed -ne 's/^:\//#/;s/^://;s/\// /g;s/^#/\//;p'` 58 | shift 59 | 60 | pathcomp= 61 | for d 62 | do 63 | pathcomp="$pathcomp$d" 64 | case "$pathcomp" in 65 | -* ) pathcomp=./$pathcomp ;; 66 | esac 67 | 68 | if test ! -d "$pathcomp"; then 69 | echo "mkdir $pathcomp" 70 | 71 | mkdir "$pathcomp" || lasterr=$? 72 | 73 | if test ! -d "$pathcomp"; then 74 | errstatus=$lasterr 75 | else 76 | if test ! -z "$dirmode"; then 77 | echo "chmod $dirmode $pathcomp" 78 | 79 | lasterr="" 80 | chmod "$dirmode" "$pathcomp" || lasterr=$? 81 | 82 | if test ! -z "$lasterr"; then 83 | errstatus=$lasterr 84 | fi 85 | fi 86 | fi 87 | fi 88 | 89 | pathcomp="$pathcomp/" 90 | done 91 | done 92 | 93 | exit $errstatus 94 | 95 | # Local Variables: 96 | # mode: shell-script 97 | # sh-indentation: 3 98 | # End: 99 | # mkinstalldirs ends here 100 | -------------------------------------------------------------------------------- /libscws/phpext/CREDITS: -------------------------------------------------------------------------------- 1 | hightman Mar QQ: 16139558 QQ-Group: 17708754 2 | http://www.hightman.cn 3 | http://www.twomice.net 4 | -------------------------------------------------------------------------------- /libscws/phpext/php_scws_manual.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sounos/ithunder/241d17ce44ed90bc3d8ce4f0b8d5ae52498db270/libscws/phpext/php_scws_manual.txt -------------------------------------------------------------------------------- /libscws/phpext/scws.php: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sounos/ithunder/241d17ce44ed90bc3d8ce4f0b8d5ae52498db270/libscws/phpext/scws.php -------------------------------------------------------------------------------- /libscws/phpext/win32/Readme.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sounos/ithunder/241d17ce44ed90bc3d8ce4f0b8d5ae52498db270/libscws/phpext/win32/Readme.txt -------------------------------------------------------------------------------- /libscws/phpext/win32/php_scws-1.1.8-win32.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sounos/ithunder/241d17ce44ed90bc3d8ce4f0b8d5ae52498db270/libscws/phpext/win32/php_scws-1.1.8-win32.zip -------------------------------------------------------------------------------- /libscws/win32/readme.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sounos/ithunder/241d17ce44ed90bc3d8ce4f0b8d5ae52498db270/libscws/win32/readme.txt -------------------------------------------------------------------------------- /libscws/win32/scws.dsw: -------------------------------------------------------------------------------- 1 | Microsoft Developer Studio Workspace File, Format Version 6.00 2 | # WARNING: DO NOT EDIT OR DELETE THIS WORKSPACE FILE! 3 | 4 | ############################################################################### 5 | 6 | Project: "libscws"=.\libscws.dsp - Package Owner=<4> 7 | 8 | Package=<5> 9 | {{{ 10 | }}} 11 | 12 | Package=<4> 13 | {{{ 14 | }}} 15 | 16 | ############################################################################### 17 | 18 | Project: "scws"=.\scws.dsp - Package Owner=<4> 19 | 20 | Package=<5> 21 | {{{ 22 | }}} 23 | 24 | Package=<4> 25 | {{{ 26 | Begin Project Dependency 27 | Project_Dep_Name libscws 28 | End Project Dependency 29 | }}} 30 | 31 | ############################################################################### 32 | 33 | Global: 34 | 35 | Package=<5> 36 | {{{ 37 | }}} 38 | 39 | Package=<3> 40 | {{{ 41 | }}} 42 | 43 | ############################################################################### 44 | 45 | -------------------------------------------------------------------------------- /libscws/win32/scws.sln: -------------------------------------------------------------------------------- 1 |  2 | Microsoft Visual Studio Solution File, Format Version 10.00 3 | # Visual Studio 2008 4 | Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libscws", "libscws.vcproj", "{893E9BEE-1010-4CDD-BD3B-F9B33DCC02EF}" 5 | EndProject 6 | Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "scws", "scws.vcproj", "{5A97F474-271E-4899-809F-D76CD20B8E44}" 7 | ProjectSection(ProjectDependencies) = postProject 8 | {893E9BEE-1010-4CDD-BD3B-F9B33DCC02EF} = {893E9BEE-1010-4CDD-BD3B-F9B33DCC02EF} 9 | EndProjectSection 10 | EndProject 11 | Global 12 | GlobalSection(SolutionConfigurationPlatforms) = preSolution 13 | Debug_PHP53|Win32 = Debug_PHP53|Win32 14 | Debug|Win32 = Debug|Win32 15 | Release_PHP53|Win32 = Release_PHP53|Win32 16 | Release|Win32 = Release|Win32 17 | EndGlobalSection 18 | GlobalSection(ProjectConfigurationPlatforms) = postSolution 19 | {893E9BEE-1010-4CDD-BD3B-F9B33DCC02EF}.Debug_PHP53|Win32.ActiveCfg = Debug|Win32 20 | {893E9BEE-1010-4CDD-BD3B-F9B33DCC02EF}.Debug_PHP53|Win32.Build.0 = Debug|Win32 21 | {893E9BEE-1010-4CDD-BD3B-F9B33DCC02EF}.Debug|Win32.ActiveCfg = Debug|Win32 22 | {893E9BEE-1010-4CDD-BD3B-F9B33DCC02EF}.Debug|Win32.Build.0 = Debug|Win32 23 | {893E9BEE-1010-4CDD-BD3B-F9B33DCC02EF}.Release_PHP53|Win32.ActiveCfg = Release|Win32 24 | {893E9BEE-1010-4CDD-BD3B-F9B33DCC02EF}.Release_PHP53|Win32.Build.0 = Release|Win32 25 | {893E9BEE-1010-4CDD-BD3B-F9B33DCC02EF}.Release|Win32.ActiveCfg = Release|Win32 26 | {893E9BEE-1010-4CDD-BD3B-F9B33DCC02EF}.Release|Win32.Build.0 = Release|Win32 27 | {5A97F474-271E-4899-809F-D76CD20B8E44}.Debug_PHP53|Win32.ActiveCfg = Debug_PHP53|Win32 28 | {5A97F474-271E-4899-809F-D76CD20B8E44}.Debug_PHP53|Win32.Build.0 = Debug_PHP53|Win32 29 | {5A97F474-271E-4899-809F-D76CD20B8E44}.Debug|Win32.ActiveCfg = Debug_PHP53|Win32 30 | {5A97F474-271E-4899-809F-D76CD20B8E44}.Debug|Win32.Build.0 = Debug_PHP53|Win32 31 | {5A97F474-271E-4899-809F-D76CD20B8E44}.Release_PHP53|Win32.ActiveCfg = Release_PHP53|Win32 32 | {5A97F474-271E-4899-809F-D76CD20B8E44}.Release_PHP53|Win32.Build.0 = Release_PHP53|Win32 33 | {5A97F474-271E-4899-809F-D76CD20B8E44}.Release|Win32.ActiveCfg = Release_PHP53|Win32 34 | {5A97F474-271E-4899-809F-D76CD20B8E44}.Release|Win32.Build.0 = Release_PHP53|Win32 35 | EndGlobalSection 36 | GlobalSection(SolutionProperties) = preSolution 37 | HideSolutionNode = FALSE 38 | EndGlobalSection 39 | EndGlobal 40 | -------------------------------------------------------------------------------- /qmtask/AUTHORS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sounos/ithunder/241d17ce44ed90bc3d8ce4f0b8d5ae52498db270/qmtask/AUTHORS -------------------------------------------------------------------------------- /qmtask/ChangeLog: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sounos/ithunder/241d17ce44ed90bc3d8ce4f0b8d5ae52498db270/qmtask/ChangeLog -------------------------------------------------------------------------------- /qmtask/Makefile.am: -------------------------------------------------------------------------------- 1 | SUBDIRS = doc src 2 | EXTRA_DIST = debian.sh control control 3 | -------------------------------------------------------------------------------- /qmtask/NEWS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sounos/ithunder/241d17ce44ed90bc3d8ce4f0b8d5ae52498db270/qmtask/NEWS -------------------------------------------------------------------------------- /qmtask/README: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sounos/ithunder/241d17ce44ed90bc3d8ce4f0b8d5ae52498db270/qmtask/README -------------------------------------------------------------------------------- /qmtask/configure.in: -------------------------------------------------------------------------------- 1 | # -*- Autoconf -*- 2 | # Process this file with autoconf to produce a configure script. 3 | 4 | AC_PREREQ(2.59) 5 | AC_INIT(qmtask, [0.0.5], [SounOS@gmail.com]) 6 | AC_CONFIG_SRCDIR([src/qtaskd.c]) 7 | AC_CONFIG_HEADER([config.h]) 8 | AM_INIT_AUTOMAKE 9 | 10 | # Checks for programs. 11 | AC_PROG_CC 12 | AM_PROG_CC_C_O 13 | AC_PROG_LIBTOOL 14 | 15 | # Checks for libraries. 16 | 17 | # Checks for header files. 18 | AC_HEADER_DIRENT 19 | AC_HEADER_STDC 20 | AC_CHECK_HEADERS([arpa/inet.h fcntl.h locale.h netdb.h stdint.h stdlib.h string.h sys/file.h sys/time.h unistd.h]) 21 | 22 | # Checks for typedefs, structures, and compiler characteristics. 23 | AC_C_CONST 24 | AC_TYPE_OFF_T 25 | AC_TYPE_PID_T 26 | AC_TYPE_SIZE_T 27 | AC_HEADER_TIME 28 | AC_STRUCT_TM 29 | 30 | #check headers 31 | #pthread 32 | havepthread=no 33 | AC_CHECK_LIB(pthread,pthread_create,havepthread=yes) 34 | if test "$havepthread" = "yes" ; then 35 | CPPFLAGS="${CPPFLAGS} -DHAVE_PTHREAD" 36 | LDFLAGS="${LDFLAGS} -lpthread" 37 | LIBS="${LIBS} -lpthread" 38 | fi 39 | #semaphore 40 | #havesemaphore=no 41 | #AC_CHECK_LIB(c, sem_timedwait, havesemaphore=yes) 42 | #if test "$havesemaphore" = "yes" ; then 43 | # CPPFLAGS="${CPPFLAGS} -DHAVE_SEMAPHORE" 44 | #fi 45 | 46 | #mmap 47 | havemmap=no 48 | AC_CHECK_LIB(c, munmap, havemmap=yes) 49 | if test "$havemmap" = "yes" ; then 50 | CPPFLAGS="${CPPFLAGS} -DHAVE_MMAP" 51 | fi 52 | 53 | #AC_CHECK_EXTRA_OPTIONS 54 | 55 | # Checks for library functions. 56 | AC_FUNC_FORK 57 | AC_FUNC_LSTAT 58 | AC_FUNC_LSTAT_FOLLOWS_SLASHED_SYMLINK 59 | AC_FUNC_MALLOC 60 | AC_FUNC_MEMCMP 61 | AC_FUNC_MKTIME 62 | AC_FUNC_MMAP 63 | AC_FUNC_REALLOC 64 | AC_TYPE_SIGNAL 65 | AC_FUNC_STAT 66 | AC_CHECK_FUNCS([ftruncate gettimeofday memset mkdir munmap setlocale socket strcasecmp strchr strdup strerror strncasecmp]) 67 | 68 | AC_CONFIG_FILES([Makefile 69 | doc/Makefile 70 | src/Makefile]) 71 | AC_OUTPUT 72 | -------------------------------------------------------------------------------- /qmtask/control: -------------------------------------------------------------------------------- 1 | Source: 2 | Section: libs 3 | Priority: extra 4 | Maintainer: SounOS 5 | Build-Depends: debhelper (>= 7.0.50~), autotools-dev 6 | Standards-Version: 7 | Homepage: http://sbase.googlecode.com/ 8 | #Vcs-Git: git://git.debian.org/collab-maint/libevbase.git 9 | #Vcs-Browser: http://git.debian.org/?p=collab-maint/libevbase.git;a=summary 10 | 11 | Package: 12 | Architecture: any 13 | Depends: ${shlibs:Depends}, ${misc:Depends} 14 | Description: 15 | 16 | -------------------------------------------------------------------------------- /qmtask/debian.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | rm -rf debian 3 | pkg=`grep -E "^PACKAGE_NAME=" configure |sed -e "s/PACKAGE_NAME='\(.*\)'/\1/"` 4 | ver=`grep -E "^PACKAGE_VERSION=" configure |sed -e "s/PACKAGE_VERSION='\(.*\)'/\1/"` 5 | dh_make -y -n -s -c bsd -e SounOS@gmail.com -p "${pkg}_${ver}" 6 | cp control debian/control 7 | sed -i "s/^Package:.*/Package: $pkg/" debian/control 8 | sed -i "s/^Source:.*/Source: $pkg/" debian/control 9 | sed -i "s/^License:.*/License: $pkg/" debian/control 10 | sed -i "s/^Standards-Version:.*/Standards-Version: $ver/" debian/control 11 | sed -i "s/root /SounOS /" debian/* 12 | #export CFLAGS='-ggdb -O0' 13 | #export CXXFLAGS='-ggdb -O0' 14 | #export FFLAGS='-ggdb -O0' 15 | dpkg-buildpackage -rfakeroot -eSounOS -mSounOS 16 | -------------------------------------------------------------------------------- /qmtask/doc/Makefile.am: -------------------------------------------------------------------------------- 1 | #automake 2 | EXTRA_DIST = rc.qtaskd.ini rc.qtaskd.html rc.qtaskd.init base64.php qmtask.spec 3 | 4 | install: install-ini 5 | 6 | conftodir = "`echo $(sysconfdir)|sed -e 's|/usr/etc|/etc|'`" 7 | 8 | initdir = "$(DESTDIR)$(conftodir)/init.d" 9 | 10 | sysconftodir = "$(DESTDIR)$(conftodir)" 11 | 12 | htmldir = "`echo @prefix@"/var/html"|sed -e 's|/usr/var|/var|g'`" 13 | htmltodir = "$(DESTDIR)$(htmldir)" 14 | 15 | install-ini: 16 | @echo install *.html to $(htmldir) 17 | @test -d $(htmltodir)/qtask || mkdir -p $(htmltodir)/qtask 18 | @cp -p rc.qtaskd.html $(htmltodir)/qtask/index.html 19 | @echo install qtaskd.ini to $(sysconftodir) 20 | @test -d $(sysconftodir) || mkdir -p $(sysconftodir) 21 | @test -f $(sysconftodir)/qtaskd.ini \ 22 | || cp -p rc.qtaskd.ini $(sysconftodir)/qtaskd.ini 23 | @perl -i -p -e "s|\"/var/html|\"$(htmldir)|g" $(sysconftodir)/qtaskd.ini 24 | @echo install qtaskd.init to $(initdir) 25 | @test -d $(initdir) || mkdir -p $(initdir) 26 | @test -f $(initdir)/qtaskd \ 27 | || cp -p rc.qtaskd.init $(initdir)/qtaskd 28 | @perl -i -p -e "s@/usr/local/sbin@$(prefix)/sbin@g" $(initdir)/qtaskd 29 | @perl -i -p -e "s@/usr/local/etc@$(conftodir)@g" $(initdir)/qtaskd 30 | 31 | 32 | -------------------------------------------------------------------------------- /qmtask/doc/base64.php: -------------------------------------------------------------------------------- 1 | #!/usr/bin/php 2 | 22 | -------------------------------------------------------------------------------- /qmtask/doc/qmtask.spec: -------------------------------------------------------------------------------- 1 | # Authority: SounOS.org 2 | Summary: Queue Monitor for Tasks used for distrubuted computing 3 | Name: qmtask 4 | Version: 0.0.5 5 | Release: 39%{?dist} 6 | License: BSD 7 | Group: System Environment/Libraries 8 | URL: http://code.google.com/p/libibase/ 9 | 10 | Source: http://code.google.com/p/libibase/download/%{name}-%{version}.tar.gz 11 | Packager: SounOS 12 | Vendor: SounOS 13 | BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root 14 | BuildRequires: libevbase >= 1.0.0 libsbase >= 1.0.0 15 | Requires: libevbase >= 1.0.0 libsbase >= 1.0.0 16 | 17 | %description 18 | Queue Monitor for Tasks used for distrubuted computing. 19 | 20 | %package -n libmtask 21 | Group: Development/Libraries 22 | Summary: Development tools for the qmtask server. 23 | 24 | %description -n libmtask 25 | The hidbase-devel package contains the API files libmtask and mtask.h 26 | 27 | %prep 28 | %setup 29 | 30 | %build 31 | %configure 32 | %{__make} 33 | 34 | %install 35 | %{__rm} -rf %{buildroot} 36 | %{__make} install DESTDIR="%{buildroot}" 37 | 38 | %clean 39 | %{__rm} -rf %{buildroot} 40 | 41 | %post 42 | 43 | /sbin/chkconfig --add qtaskd 44 | 45 | %preun 46 | [ "`pstree|grep qtaskd|wc -l`" -gt "0" ] && /sbin/service qtaskd stop 47 | /sbin/chkconfig --del qtaskd 48 | 49 | %files -n libmtask 50 | %defattr(-, root, root) 51 | %{_libdir}/* 52 | %{_includedir}/* 53 | 54 | %files 55 | %defattr(-, root, root, 0755) 56 | %{_sbindir}/* 57 | %{_bindir}/* 58 | %{_localstatedir}/* 59 | %{_sysconfdir}/rc.d/* 60 | %config(noreplace) %{_sysconfdir}/*.ini 61 | 62 | %changelog 63 | * Thu Jun 2 2011 11:27:49 CST SounOS 64 | - qmtask-0.0.3.tar.gz (qmtask　0.0.3 分布式计算任务调度) file uploaded by sounos@gmail.com 65 | - r1902 ( fixed mmtree.c::mmtree_insert()/mmtree_insert()::mmtree...) committed by sounos.@gmail.com 66 | - fixed mmtree.c::mmtree_insert()/mmtree_insert()::mmtree->state->roots[rootid].total; 67 | - r1901 ( updated qtask.c added XTASK{total,over}; ) committed by sounos@gmail.com 68 | - updated qtask.c added XTASK{total,over}; 69 | - r1900 ( updated qtaskd_request_handler(); ) committed by sounos@gmail.com 70 | - updated qtaskd_request_handler(); 71 | -------------------------------------------------------------------------------- /qmtask/doc/rc.qtaskd.init: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # QTASKD: 4 | # 5 | # chkconfig: 2345 70 40 6 | # description: QTASKD 7 | # 8 | # 9 | 10 | # Source function library. 11 | . /etc/rc.d/init.d/functions 12 | 13 | #[ -r /etc/sysconfig/qtaskd ] && . /etc/sysconfig/qtaskd 14 | QTASKDARGS=" -d -c /usr/local/etc/qtaskd.ini > /dev/null 2>&1 " 15 | 16 | start() 17 | { 18 | echo -n $"Starting qtaskd: " 19 | daemon /usr/local/sbin/qtaskd $QTASKDARGS 20 | echo 21 | } 22 | 23 | stop() 24 | { 25 | echo -n $"Shutting down qtaskd: " 26 | killproc qtaskd 27 | 28 | echo 29 | } 30 | 31 | [ -f /usr/local/sbin/qtaskd ] || exit 0 32 | 33 | # See how we were called. 34 | case "$1" in 35 | start) 36 | start 37 | ;; 38 | stop) 39 | stop 40 | ;; 41 | restart|reload) 42 | stop 43 | start 44 | ;; 45 | *) 46 | echo $"Usage: $0 {start|stop|restart|reload}" 47 | exit 1 48 | esac 49 | 50 | exit 0 51 | -------------------------------------------------------------------------------- /qmtask/src/Makefile.am: -------------------------------------------------------------------------------- 1 | lib_LTLIBRARIES = libmtask.la 2 | libmtask_la_SOURCES = mtask.c mtask.h utils/xmm.h utils/xmm.c 3 | 4 | libmtask_la_CPPFLAGS = -D_FILE_OFFSET_BITS=64 -Wall -I utils 5 | include_HEADERS = mtask.h 6 | 7 | sbin_PROGRAMS = qtaskd 8 | 9 | qtaskd_SOURCES = qtaskd.c qtask.h qtask.c utils/http.h utils/http.c iniparser.h iniparser.c \ 10 | utils/mtrie.h utils/mtrie.c utils/stime.h utils/stime.c utils/logger.h \ 11 | utils/mutex.h utils/mmqueue.h utils/mmqueue.c utils/mmtrie.h utils/mmtrie.c \ 12 | utils/mmtree.h utils/mmtree.c utils/xmm.h utils/xmm.c utils/base64.h \ 13 | utils/base64.c base64qtaskdhtml.h utils/timer.h utils/db.h utils/db.c utils/logger.c 14 | qtaskd_CPPFLAGS = -I utils -D_FILE_OFFSET_BITS=64 -Wall 15 | qtaskd_LDADD = libmtask.la 16 | qtaskd_LDFLAGS = -static -levbase -lsbase 17 | 18 | bin_PROGRAMS = btask 19 | btask_SOURCES = btask.c utils/xmm.h utils/xmm.c 20 | btask_CPPFLAGS = -D_FILE_OFFSET_BITS=64 -Wall -I utils 21 | btask_LDADD = libmtask.la 22 | btask_LDFLAGS = -static 23 | 24 | -------------------------------------------------------------------------------- /qmtask/src/a.sh: -------------------------------------------------------------------------------- 1 | x=0; 2 | while [ $x -lt 256 ]; 3 | do 4 | if [ "$x" -lt 96 ];then 5 | ./btask -h127.0.0.1 -p2066 -m0 -q1 2 >&1 > 01.txt & 6 | fi 7 | #./btask -h127.0.0.1 -p2066 -m0 -q1 2 >&1 > 01.txt & 8 | ./btask -h127.0.0.1 -p2066 -m1 -q2 2 >&1 > 12.txt & 9 | ./btask -h127.0.0.1 -p2066 -m2 -q3 2 >&1 > 23.txt & 10 | ./btask -h127.0.0.1 -p2066 -m3 -q0 2 >&1 > 30.txt & 11 | ((x++)); 12 | done; 13 | -------------------------------------------------------------------------------- /qmtask/src/iniparser.h: -------------------------------------------------------------------------------- 1 | /* 2 | Based upon libiniparser, by Nicolas Devillard 3 | Hacked into 1 file (m-iniparser) by Freek/2005 4 | Original terms following: 5 | 6 | -- - 7 | 8 | Copyright (c) 2000 by Nicolas Devillard (ndevilla AT free DOT fr). 9 | 10 | Written by Nicolas Devillard. Not derived from licensed software. 11 | 12 | Permission is granted to anyone to use this software for any 13 | purpose on any computer system, and to redistribute it freely, 14 | subject to the following restrictions: 15 | 16 | 1. The author is not responsible for the consequences of use of 17 | this software, no matter how awful, even if they arise 18 | from defects in it. 19 | 20 | 2. The origin of this software must not be misrepresented, either 21 | by explicit claim or by omission. 22 | 23 | 3. Altered versions must be plainly marked as such, and must not 24 | be misrepresented as being the original software. 25 | 26 | 4. This notice may not be removed or altered. 27 | 28 | */ 29 | 30 | 31 | #ifndef _INIPARSER_H_ 32 | #define _INIPARSER_H_ 33 | #include 34 | #include 35 | #include 36 | #include 37 | #include 38 | 39 | #ifdef __cplusplus 40 | extern "C" { 41 | #endif 42 | 43 | 44 | typedef struct _dictionary_ { 45 | /** Number of entries in dictionary */ 46 | int n; 47 | /** Storage size */ 48 | int size; 49 | /** List of string values */ 50 | char **val; 51 | /** List of string keys */ 52 | char **key ; 53 | /** List of hash values for keys */ 54 | unsigned *hash; 55 | } dictionary ; 56 | 57 | 58 | /* generated by genproto */ 59 | 60 | dictionary * iniparser_new(char *ininame); 61 | void iniparser_free(dictionary * d); 62 | 63 | int iniparser_getnsec(dictionary * d); 64 | char * iniparser_getsecname(dictionary * d, int n); 65 | void iniparser_dump(dictionary * d, FILE * f); 66 | void iniparser_dump_ini(dictionary * d, FILE * f); 67 | char * iniparser_getkey(dictionary *d, char *section, char *key); 68 | char * iniparser_getstr(dictionary * d, char * key); 69 | char * iniparser_getstring(dictionary * d, char * key, char * def); 70 | int iniparser_getint(dictionary * d, char * key, int notfound); 71 | double iniparser_getdouble(dictionary * d, char * key, double notfound); 72 | int iniparser_getboolean(dictionary * d, char * key, int notfound); 73 | int iniparser_find_entry(dictionary * ini, char * entry); 74 | int iniparser_setstr(dictionary * ini, char * entry, char * val); 75 | void iniparser_unset(dictionary * ini, char * entry); 76 | 77 | #ifdef __cplusplus 78 | } 79 | #endif 80 | 81 | #endif 82 | 83 | -------------------------------------------------------------------------------- /qmtask/src/mtask.h: -------------------------------------------------------------------------------- 1 | #ifndef __MTASK__H__ 2 | #define __MTASK__H__ 3 | #ifdef __cplusplus 4 | extern "C" { 5 | #endif 6 | #include 7 | #include 8 | #include 9 | #define MTASK_CMD_PUSH 0x01 10 | #define MTASK_CMD_POP 0x02 11 | #define MTASK_CMD_FINISH 0x04 12 | #define MTASK_CMD_OVER 0x08 13 | #define MTASK_TO_QHEAD 0x01 14 | #define MTASK_PACKET_MAX 8192 15 | #define MTASK_MM_BASE 4096 16 | typedef struct MHEAD 17 | { 18 | int cmd; 19 | int flag; 20 | int mtaskid; 21 | int qtaskid; 22 | int packetid; 23 | int length; 24 | }MHEAD; 25 | typedef struct _MTASK 26 | { 27 | int fd; 28 | int mtaskid; 29 | int qtaskid; 30 | int mm_size; 31 | int length; 32 | int packetid; 33 | off_t mtotal; 34 | off_t qtotal; 35 | char *packet; 36 | struct sockaddr_in rsa; 37 | }MTASK; 38 | /* set message task */ 39 | int mtask_set(MTASK *mtask, char *ip, int port, int mtaskid, int qtaskid); 40 | /* connect to qtask 41 | * -1 mtask is NULL 42 | * -2 socket() failed 43 | * -3 connect() failed 44 | * */ 45 | int mtask_connect(MTASK *mtask); 46 | 47 | /* get new task count 48 | * -1 mtask is NULL 49 | * -2 mtask->fd <= 0 and mtask_connect failed 50 | * -3 write() task header failed 51 | * -4 write() task list[] failed 52 | * -5 read task header failed 53 | * -6 malloc failed 54 | * -7 read task list[] failed 55 | * ret >= 0 mhead.packetid 56 | * */ 57 | int mtask_commit(MTASK *mtask, int flag, char *packet, int packet_len); 58 | 59 | /* push packet 60 | * return value 61 | * -1 mtask is NULL 62 | * -2 connection is bad 63 | * -3 Invalid packet data 64 | * -4 write() header failed 65 | * -5 write() packet failed 66 | * -6 recv() header failed 67 | * */ 68 | int mtask_push(MTASK *mtask, int flag, char *packet, int packet_len); 69 | 70 | /* pop packet 71 | * return value 72 | * -1 mtask is NULL 73 | * -2 connection is bad 74 | * -3 Invalid commitid 75 | * -4 write() header failed 76 | * -5 recv() header failed 77 | * -6 malloc() for packet failed 78 | * -7 recv() packet failed 79 | */ 80 | int mtask_pop(MTASK *mtask); 81 | 82 | /* over packet 83 | * return value 84 | * -1 mtask is NULL 85 | * -2 connection is bad 86 | * -3 Invalid packetid and commitid 87 | * -4 write() header failed 88 | * -5 recv() header failed 89 | * */ 90 | int mtask_finish(MTASK *mtask, int flag); 91 | 92 | /* close message task */ 93 | void mtask_close(MTASK *mtask); 94 | #ifdef __cplusplus 95 | } 96 | #endif 97 | #endif 98 | -------------------------------------------------------------------------------- /qmtask/src/qtask.h: -------------------------------------------------------------------------------- 1 | #include "mutex.h" 2 | #ifndef __QTASK__H__ 3 | #define __QTASK__H__ 4 | #define QT_PATH_MAX 256 5 | #define QT_TASKS_MAX 256 6 | #define QT_NAME_MAX 128 7 | #define QT_TASK_TOP 0x01 8 | #define QT_LINE_MAX 256 9 | typedef struct _XTASK 10 | { 11 | int status; 12 | int bits; 13 | int qhead; 14 | int queue; 15 | int wait; 16 | int workers; 17 | off_t total; 18 | off_t over; 19 | char name[QT_NAME_MAX]; 20 | }XTASK; 21 | /* FILE IO */ 22 | typedef struct _QTIO 23 | { 24 | int fd; 25 | int bits; 26 | void *map; 27 | off_t old; 28 | off_t end; 29 | off_t size; 30 | }QTIO; 31 | typedef struct _QTSTATE 32 | { 33 | int qtasks; 34 | int id_max; 35 | int old; 36 | int bits; 37 | XTASK tasks[QT_TASKS_MAX]; 38 | }QTSTATE; 39 | typedef struct _QTASK 40 | { 41 | int log_access; 42 | int bits; 43 | MUTEX *mutex; 44 | void *logger; 45 | void *map; 46 | void *qhead; 47 | void *queue; 48 | void *wait; 49 | void *old; 50 | void *db; 51 | QTSTATE *state; 52 | QTIO stateio; 53 | char basedir[QT_PATH_MAX]; 54 | }QTASK; 55 | QTASK *qtask_init(char *basedir); 56 | int qtask_add_task(QTASK *qtask, char *name); 57 | int qtask_rename_task(QTASK *qtask, int taskid, char *name); 58 | int qtask_del_task(QTASK *qtask, int taskid); 59 | int qtask_list_tasks(QTASK *qtask, char *out, char *end); 60 | int qtask_update_workers(QTASK *qtask, int taskid, int num); 61 | int qtask_gen_packet(QTASK *qtask, char *packet, int length); 62 | int qtask_push_packet(QTASK *qtask, int taskid, int flag, int packetid); 63 | int qtask_over_packet(QTASK *qtask, int taskid, int packetid); 64 | int qtask_pop_packet(QTASK *qtask, int taskid, int *packet_len); 65 | int qtask_read_packet(QTASK *qtask, int taskid, int packetid, char *packet); 66 | int qtask_remove_packet(QTASK *qtask, int packetid); 67 | int qtask_repacket(QTASK *qtask, int taskid, int packetid); 68 | void qtask_clean(QTASK *qtask); 69 | #endif 70 | -------------------------------------------------------------------------------- /qmtask/src/utils/base64.h: -------------------------------------------------------------------------------- 1 | #ifndef _BASE64_H 2 | #define _BASE64_H 3 | #define BASE64_LEN(x) ((x + 2) / 3 * 4 + 1) 4 | int base64_encode(char *out, const unsigned char *in, int inlen); 5 | int base64_decode(unsigned char *out, const char *in, int inlen); 6 | #endif 7 | -------------------------------------------------------------------------------- /qmtask/src/utils/mime.h: -------------------------------------------------------------------------------- 1 | #ifndef _MIME_H 2 | #define _MIME_H 3 | #define MIME_NUM_MAX 1024 4 | typedef struct _MIME_MAP 5 | { 6 | void *map; 7 | int num; 8 | int bits; 9 | }MIME_MAP; 10 | /* initialize mime */ 11 | int mime_map_init(MIME_MAP *mime_map); 12 | /* add mime */ 13 | int mime_add(MIME_MAP *mime_map, char *mime, int len); 14 | /* add mime line */ 15 | int mime_add_line(MIME_MAP *mime_map, char *p, char *end); 16 | /* return mime id*/ 17 | int mime_id(MIME_MAP *mime_map, char *mime, int len); 18 | /* clean mime map*/ 19 | void mime_map_clean(MIME_MAP *mime_map); 20 | #endif 21 | -------------------------------------------------------------------------------- /qmtask/src/utils/mmqueue.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include "mutex.h" 5 | #ifndef _MMQUEUE64_H 6 | #define _MMQUEUE64_H 7 | #ifdef __cplusplus 8 | extern "C" { 9 | #endif 10 | typedef struct _MMQODE 11 | { 12 | //void *ptr; 13 | unsigned int data; 14 | unsigned int next; 15 | }MMQNODE; 16 | typedef struct _MMQROOT 17 | { 18 | int status; 19 | int total; 20 | int first; 21 | int last; 22 | }MMQROOT; 23 | #define MMQ_INCRE_NUM 1000000 24 | #define MMQ_NODE_MAX 100000000 25 | #define MMQ_ROOT_MAX 8192 26 | typedef struct _MMQSTATE 27 | { 28 | int qtotal; 29 | int qleft; 30 | int nroots; 31 | MMQROOT roots[MMQ_ROOT_MAX]; 32 | }MMQSTATE; 33 | typedef struct _MMQUEUE 34 | { 35 | int fd; 36 | int bits; 37 | off_t end; 38 | off_t size; 39 | off_t old; 40 | void *map; 41 | MUTEX *mutex; 42 | MMQSTATE *state; 43 | MMQNODE *nodes; 44 | }MMQUEUE; 45 | MMQUEUE *mmqueue_init(char *qfile); 46 | int mmqueue_new(MMQUEUE *mmq); 47 | int mmqueue_total(MMQUEUE *mmq, int rootid); 48 | int mmqueue_close(MMQUEUE *mmq, int rootid); 49 | int mmqueue_push(MMQUEUE *mmq, int rootid, int data); 50 | int mmqueue_pop(MMQUEUE *mmq, int rootid, int *data); 51 | int mmqueue_head(MMQUEUE *mmq, int rootid, int *data); 52 | void mmqueue_clean(MMQUEUE *mmq); 53 | #define MMQ(x) ((MMQUEUE*)x) 54 | #ifdef __cplusplus 55 | } 56 | #endif 57 | #endif 58 | -------------------------------------------------------------------------------- /qmtask/src/utils/mmqueue64.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include "mutex.h" 5 | #ifndef _MMQUEUE64_H 6 | #define _MMQUEUE64_H 7 | #ifdef __cplusplus 8 | extern "C" { 9 | #endif 10 | typedef struct _MMQODE 11 | { 12 | //void *ptr; 13 | int64_t data; 14 | int next; 15 | int prev; 16 | }MMQNODE; 17 | typedef struct _MMQROOT 18 | { 19 | int status; 20 | int total; 21 | int first; 22 | int last; 23 | }MMQROOT; 24 | #define MMQ_INCRE_NUM 1000000 25 | #define MMQ_NODE_MAX 100000000 26 | #define MMQ_ROOT_MAX 8192 27 | typedef struct _MMQSTATE 28 | { 29 | int qtotal; 30 | int qleft; 31 | int nroots; 32 | MMQROOT roots[MMQ_ROOT_MAX]; 33 | }MMQSTATE; 34 | typedef struct _MMQUEUE 35 | { 36 | int fd; 37 | int bits; 38 | off_t end; 39 | off_t size; 40 | void *map; 41 | MUTEX mutex; 42 | MMQSTATE *state; 43 | MMQNODE *nodes; 44 | }MMQUEUE; 45 | MMQUEUE *mmqueue_init(char *qfile); 46 | int mmqueue_new(MMQUEUE *mmq); 47 | int mmqueue_total(MMQUEUE *mmq, int rootid); 48 | int mmqueue_close(MMQUEUE *mmq, int rootid); 49 | int mmqueue_push(MMQUEUE *mmq, int rootid, int64_t data); 50 | int mmqueue_pop(MMQUEUE *mmq, int rootid, int64_t *data); 51 | int mmqueue_head(MMQUEUE *mmq, int rootid, int64_t *data); 52 | void mmqueue_clean(MMQUEUE *mmq); 53 | #define MMQ(x) ((MMQUEUE*)x) 54 | #ifdef __cplusplus 55 | } 56 | #endif 57 | #endif 58 | -------------------------------------------------------------------------------- /qmtask/src/utils/mmtree.h: -------------------------------------------------------------------------------- 1 | #ifndef _MMTREE_H 2 | #define _MMTREE_H 3 | #include "mutex.h" 4 | #define MMTREE_INCRE_NUM 10000000 5 | #define MMTREE_NODES_MAX 2000000000 6 | #define MMTREE_ROOT_MAX 10240 7 | typedef struct _MTNODE 8 | { 9 | unsigned int left; 10 | unsigned int right; 11 | unsigned int parent; 12 | unsigned int color; 13 | int data; 14 | int key; 15 | }MTNODE; 16 | typedef struct _MMROOT 17 | { 18 | int status; 19 | int bits; 20 | unsigned int total; 21 | unsigned int rootid; 22 | }MMROOT; 23 | typedef struct _MTSTATE 24 | { 25 | int kmax; 26 | int kmin; 27 | unsigned int nmax; 28 | unsigned int nmin; 29 | unsigned int count; 30 | unsigned int left; 31 | unsigned int current; 32 | unsigned int total; 33 | unsigned int qleft; 34 | unsigned int qfirst; 35 | unsigned int qlast; 36 | unsigned int nroots; 37 | MMROOT roots[MMTREE_ROOT_MAX]; 38 | }MTSTATE; 39 | typedef struct _MMTREE 40 | { 41 | int fd; 42 | int status; 43 | off_t size; 44 | off_t end; 45 | off_t old; 46 | void *start; 47 | MTSTATE *state; 48 | MTNODE *map; 49 | MUTEX *mutex; 50 | }MMTREE; 51 | void *mmtree_init(char *file); 52 | int mmtree_new_tree(void *mmtree); 53 | unsigned int mmtree_total(void *mmtree, int rootid); 54 | unsigned int mmtree_try_insert(void *mmtree, int rootid, int key, int data, int *old); 55 | unsigned int mmtree_insert(void *mmtree, int rootid, int key, int data, int *old); 56 | unsigned int mmtree_get(void *mmtree, unsigned int nodeid, int *key, int *data); 57 | unsigned int mmtree_find(void *mmtree, int rootid, int key, int *data); 58 | unsigned int mmtree_min(void *mmtree, int rootid, int *key, int *data); 59 | unsigned int mmtree_max(void *mmtree, int rootid, int *key, int *data); 60 | unsigned int mmtree_next(void *mmtree, int rootid, unsigned int nodeid, int *key, int *data); 61 | unsigned int mmtree_prev(void *mmtree, int rootid, unsigned int nodeid, int *key, int *data); 62 | int mmtree_set_data(void *mmtree, unsigned int nodeid, int data); 63 | void mmtree_view_tree(void *mmtree, int rootid, FILE *fp); 64 | void mmtree_remove(void *mmtree, int rootid, unsigned int nodeid, int *key, int *data); 65 | void mmtree_remove_tree(void *mmtree, int rootid); 66 | void mmtree_close(void *mmtree); 67 | #endif 68 | -------------------------------------------------------------------------------- /qmtask/src/utils/mmtree64.h: -------------------------------------------------------------------------------- 1 | #ifndef _MMTREE64_H 2 | #define _MMTREE64_H 3 | #define MMTREE64_INCRE_NUM 10000000 4 | #define MMTREE64_NODES_MAX 2000000000 5 | #define MMTREE64_ROOT_MAX 10240 6 | #include "mutex.h" 7 | typedef struct _MTNODE 8 | { 9 | unsigned int left; 10 | unsigned int right; 11 | unsigned int parent; 12 | unsigned int color; 13 | int data; 14 | int bit; 15 | int64_t key; 16 | }MTNODE; 17 | typedef struct _MMROOT 18 | { 19 | int status; 20 | int bits; 21 | unsigned int total; 22 | unsigned int rootid; 23 | }MMROOT; 24 | typedef struct _MTSTATE 25 | { 26 | int64_t kmax; 27 | int64_t kmin; 28 | unsigned int nmax; 29 | unsigned int nmin; 30 | unsigned int count; 31 | unsigned int left; 32 | unsigned int current; 33 | unsigned int total; 34 | unsigned int qleft; 35 | unsigned int qfirst; 36 | unsigned int qlast; 37 | unsigned int nroots; 38 | MMROOT roots[MMTREE64_ROOT_MAX]; 39 | }MTSTATE; 40 | typedef struct _MMTREE64 41 | { 42 | int fd; 43 | int status; 44 | off_t size; 45 | off_t end; 46 | off_t old; 47 | char *start; 48 | MTSTATE *state; 49 | MTNODE *map; 50 | MUTEX *mutex; 51 | }MMTREE64; 52 | void *mmtree64_init(char *file); 53 | int mmtree64_new_tree(void *mmtree); 54 | unsigned int mmtree64_total(void *mmtree, int rootid); 55 | unsigned int mmtree64_try_insert(void *mmtree, int rootid, int64_t key, int data, int *old); 56 | unsigned int mmtree64_insert(void *mmtree, int rootid, int64_t key, int data, int *old); 57 | unsigned int mmtree64_get(void *mmtree, unsigned int nodeid, int64_t *key, int *data); 58 | unsigned int mmtree64_find(void *mmtree, int rootid, int64_t key, int *data); 59 | unsigned int mmtree64_min(void *mmtree, int rootid, int64_t *key, int *data); 60 | unsigned int mmtree64_max(void *mmtree, int rootid, int64_t *key, int *data); 61 | unsigned int mmtree64_next(void *mmtree, int rootid, unsigned int nodeid, int64_t *key, int *data); 62 | unsigned int mmtree64_prev(void *mmtree, int rootid, unsigned int nodeid, int64_t *key, int *data); 63 | int mmtree64_set_data(void *mmtree, unsigned int nodeid, int data); 64 | void mmtree64_view_tree(void *mmtree, int rootid, FILE *fp); 65 | void mmtree64_remove(void *mmtree, int rootid, unsigned int nodeid, int64_t *key, int *data); 66 | void mmtree64_remove_tree(void *mmtree, int rootid); 67 | void mmtree64_close(void *mmtree); 68 | #endif 69 | -------------------------------------------------------------------------------- /qmtask/src/utils/stime.h: -------------------------------------------------------------------------------- 1 | #ifndef _STIME_H 2 | #define _STIME_H 3 | #include 4 | #include 5 | //convert str datetime to time 6 | time_t str2time(char *datestr); 7 | /* time to GMT */ 8 | int GMTstrdate(time_t time, char *date); 9 | /* strdate */ 10 | int strdate(time_t time, char *date); 11 | /* local date time */ 12 | int datetime(time_t times, char *date); 13 | /* timetospec */ 14 | void timetospec(struct timespec *ts, int usecs); 15 | #endif 16 | -------------------------------------------------------------------------------- /qmtask/src/utils/xmm.h: -------------------------------------------------------------------------------- 1 | #ifndef _XMM_H_ 2 | #define _XMM_H_ 3 | void* xmm_mnew(size_t size); 4 | void* xmm_new(size_t size); 5 | void* xmm_mrenew(void *old, size_t old_size, size_t new_size); 6 | void* xmm_renew(void *old, size_t old_size, size_t new_size); 7 | void* xmm_mresize(void *old, size_t old_size, size_t new_size); 8 | void* xmm_resize(void *old, size_t old_size, size_t new_size); 9 | void xmm_free(void *m, size_t size); 10 | #endif 11 | -------------------------------------------------------------------------------- /shell/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | rpm -e libsbase --nodeps; 3 | rpm -e libevbase --nodeps; 4 | rpm -e libdbase --nodeps; 5 | rpm -e hidbase --nodeps; 6 | cd /data/build; 7 | rm -f rpms/*; 8 | tar -zxf rpms.tar.gz && rm -f rpms/*debuginfo* \ 9 | && rpm -Uvh rpms/libevbase* rpms/libsbase-* rpms/hidbase-* rpms/libdbase-* \ 10 | && ./reset.sh 11 | -------------------------------------------------------------------------------- /shell/remote.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | packet=$1 3 | #scp $packet root@s11:/data/software/rpms.tar.gz 4 | #scp reinstall.sh root@s4:/data/software/ 5 | #ssh root@s11 "nohup /data/software/reinstall.sh &" 6 | scp $packet root@worker1:/data/build/rpms.tar.gz 7 | scp install.sh root@worker1:/data/build/ 8 | scp reset.sh root@worker1:/data/build/ 9 | ssh root@worker1 "nohup /data/build/install.sh &" 10 | 11 | scp $packet root@worker2:/data/build/rpms.tar.gz 12 | scp install.sh root@worker2:/data/build/ 13 | scp reset.sh root@worker2:/data/build/ 14 | ssh root@worker2 "nohup /data/build/install.sh &" 15 | 16 | scp $packet root@worker3:/data/build/rpms.tar.gz 17 | scp install.sh root@worker3:/data/build/ 18 | scp reset.sh root@worker3:/data/build/ 19 | ssh root@worker3 "nohup /data/build/install.sh &" 20 | -------------------------------------------------------------------------------- /shell/reset.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | service hichunkd stop; 3 | #rm -rf /data/tmp/hichunk; 4 | #rm -rf /tmp/hichunk 5 | #rm -rf /db*/.db 6 | /bin/sed -e '88,88s/0/1/g' /etc/hitrackerd.ini > hitrackerd.ini ; 7 | /bin/mv -f hitrackerd.ini /etc/hitrackerd.ini 8 | /bin/sed -e '88,88s/0/1/g' -e '232,232s/0/2/g' /etc/hichunkd.ini > hichunkd.ini 9 | /bin/mv -f hichunkd.ini /etc/hichunkd.ini 10 | /sbin/chkconfig --level 3 hichunkd on; 11 | /sbin/service hichunkd start; 12 | while true; 13 | do 14 | if [ "`/bin/netstat -anp|grep LISTEN|grep 2480`" ]; then 15 | echo "starting set dbase and multicast"; 16 | x=0; 17 | for i in 1 2 3 4 5 6 ; 18 | do 19 | /usr/bin/curl -o "req.txt" -d "op=1&path=/db${i}/.db/mm&port=240${i}&limit=0&mode=0" "http://127.0.0.1:2480/q"; 20 | diskid=`expr $i - 1`; 21 | for m in 0 1 2 3; 22 | do 23 | mask="234.8.8.$x"; 24 | /usr/bin/curl -o "req.txt" -d "op=6&diskid=$diskid&mask=$mask" "http://127.0.0.1:2480/q"; 25 | ((x++)); 26 | done 27 | done; 28 | break; 29 | fi; 30 | sleep 1; 31 | done; 32 | -------------------------------------------------------------------------------- /shell/sodo.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | arch=`uname -p` 3 | mkdir -p build; pushd build;rm -rf debuginfo/ rpms/ test/ /usr/src/redhat/SRPMS/* /usr/src/redhat/RPMS/${arch}/* /usr/src/redhat/BUILD/* ;popd; 4 | rpm -Uvh srpms/*; 5 | basedir="`pwd`"; 6 | pushd /usr/src/redhat/SPECS; 7 | rpm -e sodo libsobase --nodeps; rm -f /etc/so*.ini* 8 | rpm -e libscws --nodeps 9 | rpm -e libsbase libevbase --nodeps; 10 | rm -rf /usr/src/redhat/BUILD/* /usr/src/redhat/SRPMS/* /usr/src/redhat/RPMS/x86_64/* ${basedir}/debuginfo/* ${basedir}/test/* ${basedir}/src/* ${basedir}/rpms/* ${basedir}/srpms/* /tmp/*; 11 | buildrpm() 12 | { 13 | name=$1;ver=$2;rel=$3;isinstall=$4;isdebug=$5;arch=`uname -p`; 14 | perl -i -p -e "s/^Version: .*/Version: ${ver}/" ${name}.spec \ 15 | && perl -i -p -e "s/^Release: .*%/Release: ${rel}%/" ${name}.spec \ 16 | && rpmbuild -ba ${name}.spec 17 | [ "$?" != 0 ] && exit 18 | if [ "$isinstall" == "yes" ];then 19 | rpm -Uvh /usr/src/redhat/RPMS/${arch}/${name}-${ver}-${rel}.${arch}.rpm --force 20 | [ "$?" != 0 ] && exit 21 | fi 22 | if [ "$isdebug" ]; 23 | then 24 | cp ${name}.spec ${name}_4DB.spec \ 25 | && perl -i -p -e "s/^%configure/%configure CFLAGS=\"-O0 -g\" CPPFLAGS=\"-O0 -g\" CXXFLAGS=\"-O0 -g\"/" ${name}_4DB.spec \ 26 | && perl -i -p -e "s/^Release: .*%/Release: 4DB${rel}%/" ${name}_4DB.spec \ 27 | && rpmbuild -ba ${name}_4DB.spec && rm -f ${name}_4DB.spec 28 | [ "$?" != 0 ] && exit 29 | fi 30 | } 31 | #libevbase 32 | buildrpm libevbase 1.0.2 1 yes 33 | #libsbase 34 | buildrpm libsbase 1.0.4 5 yes 35 | #libscws 36 | buildrpm libscws 1.1.8 1 yes 37 | #libsobase 38 | buildrpm libsobase 1.5.21 4 yes 39 | #sodo 40 | buildrpm sodo 1.4.21 3 yes 41 | popd 42 | perl -i -p -e "s@/tmp@/index/tmp@g" /etc/sodocd.ini; 43 | service sodocd restart 44 | rm -rf /tmp/* 45 | #tarball 46 | mkdir -p ${basedir}/{debuginfo,test,srpms,rpms,src} 47 | cp -f /usr/src/redhat/SOURCES/*.tar.gz ${basedir}/src/ 48 | mv -f /usr/src/redhat/SRPMS/* ${basedir}/srpms/ 49 | mv -f /usr/src/redhat/RPMS/x86_64/*debuginfo* ${basedir}/debuginfo/ 50 | #mv -f /usr/src/redhat/RPMS/x86_64/*debug* ${basedir}/test/ 51 | mv -f /usr/src/redhat/RPMS/x86_64/* ${basedir}/rpms/ 52 | pushd ${basedir}/; 53 | mkdir -p backup sodo; datetime=`date +%Y%m%d%H%M%S`;tar -zcf backup/rpms-${datetime}.tar.gz rpms/*;tar -zcf backup/srpms-${datetime}.tar.gz srpms/libevbase* srpms/* sodo.sh 54 | popd; 55 | -------------------------------------------------------------------------------- /shell/tarball.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | mkdir -p backup ithunder; datetime=`date +%Y%m%d%H%M%S`;tar -zcf backup/all-${datetime}.tar.gz rpms src srpms/ *.sh; tar -zcf backup/rpms-${datetime}.tar.gz rpms; tar -zcf backup/source-${datetime}.tar.gz src srpms *.sh; 3 | #sodo rpms 4 | #tar -zcf sodo/rpms-${datetime}.tar.gz rpms/libevbase* rpms/libsbase* rpms/libscws* rpms/libsobase* rpms/sodo* debuginfo/libevbase* debuginfo/libsbase* debuginfo/libscws* debuginfo/libsobase* debuginfo/sodo*; 5 | #sodo srpms 6 | #tar -zcf sodo/srpms-${datetime}.tar.gz srpms/libevbase* srpms/libsbase* srpms/libscws* srpms/libsobase* srpms/sodo* sodo.sh; 7 | #ithunder rpms 8 | tar -zcf ithunder/rpms-${datetime}.tar.gz rpms/libevbase* rpms/libsbase* rpms/libscws* rpms/libibase* rpms/libchardet* rpms/hibase* rpms/libdbase* rpms/libmtask* rpms/qmtask* rpms/hidbase* rpms/ithunder* 9 | #debuginfo/libevbase* debuginfo/libsbase* debuginfo/libscws* debuginfo/libibase* debuginfo/libchardet* debuginfo/hibase* debuginfo/qmtask* debuginfo/hidbase* debuginfo/ithunder* ; 10 | #ithunder srpms 11 | tar -zcf ithunder/srpms-${datetime}.tar.gz srpms/libevbase* srpms/libsbase* srpms/libscws* srpms/libibase* srpms/libchardet* srpms/hibase* srpms/qmtask* srpms/hidbase* srpms/ithunder* build.sh 12 | #sowork rpms 13 | #tar -zcf sowork/rpms-${datetime}.tar.gz rpms/libevbase* rpms/libsbase* rpms/libscws* rpms/libsobase* rpms/sodo* rpms/libdbase* rpms/libmtask* rpms/qmtask* rpms/hidbase* rpms/soworker* debuginfo/libevbase* debuginfo/libsbase* debuginfo/libscws* debuginfo/libsobase* debuginfo/sodo* debuginfo/qmtask* debuginfo/hidbase* debuginfo/soworker* ; 14 | #sowork srpms 15 | #tar -zcf sowork/srpms-${datetime}.tar.gz srpms/libevbase* srpms/libsbase* srpms/libscws* srpms/libsobase* srpms/sodo* srpms/qmtask* srpms/hidbase* srpms/soworker* sowork.sh 16 | #./remote.sh backup/rpms-${datetime}.tar.gz 17 | -------------------------------------------------------------------------------- /utils/imap.h: -------------------------------------------------------------------------------- 1 | #ifndef __IMAP__H 2 | #define __IMAP__H 3 | #include 4 | #include 5 | #define IMM_SLOT_MAX 524288 6 | #define IMM_SLOT_INC 2000 7 | #define IMM_SLOT_NUM 1024 8 | #define IMM_SLOT2_NUM 512 /* half of IMM_SLOT_NUM */ 9 | #define IMM_IMMV_INC 1000000 10 | #define IMM_NODES_MAX 4294967296 11 | #define __IMAP_USE_IDX__ 1 12 | #pragma pack(push, 4) 13 | #ifndef U32_T 14 | #define U32_T 15 | typedef uint32_t u32_t; 16 | #endif 17 | typedef struct IMMKV 18 | { 19 | int32_t key; /* 数值 */ 20 | u32_t val; /* 文档序号 */ 21 | }IMMKV; 22 | typedef struct _IMMV 23 | { 24 | int32_t val; /* 数值 */ 25 | #ifdef __IMAP_USE_IDX__ 26 | int off; /* 节点偏移 */ 27 | #endif 28 | }IMMV; 29 | typedef struct _IMMSLOT 30 | { 31 | int32_t min; /* 当前桶的最小值 */ 32 | int32_t max; /* 当前桶的最大值 */ 33 | int nodeid; /* 桶偏移地址 */ 34 | int count; /* 桶元素个数 */ 35 | }IMMSLOT; 36 | typedef struct _IMMSTATE 37 | { 38 | IMMSLOT slots[IMM_SLOT_MAX]; /* 槽 */ 39 | u32_t qleft[IMM_SLOT_MAX]; /* 空闲的桶 */ 40 | u32_t roots[IMM_SLOT_MAX]; /* 桶和槽的衍射 */ 41 | u32_t nleft; /* 空闲桶个数 */ 42 | u32_t count; /* 槽的个数 */ 43 | }IMMSTATE; 44 | typedef struct _IMAP 45 | { 46 | IMMSTATE *state; 47 | IMMKV *map; 48 | IMMV *vmap; 49 | IMMSLOT *slots; 50 | u32_t *roots; 51 | int fd; 52 | int vfd; 53 | off_t size; 54 | off_t msize; 55 | off_t vsize; 56 | off_t vmsize; 57 | pthread_rwlock_t rwlock; 58 | }IMAP; 59 | IMAP *imap_init(char *file); 60 | int imap_set(IMAP *imap, u32_t no, int32_t key); 61 | int imap_get(IMAP *imap, u32_t no, u32_t *val); 62 | void imap_close(IMAP *imap); 63 | #ifdef __IMAP_USE_IDX__ 64 | /* return number of the hits */ 65 | int imap_del(IMAP *imap, u32_t no); 66 | int imap_range(IMAP *imap, int32_t from, int32_t to, u32_t *list); 67 | int imap_rangefrom(IMAP *imap, int32_t key, u32_t *list); /* key = from */ 68 | int imap_rangeto(IMAP *imap, int32_t key, u32_t *list); /* key = to */ 69 | int imap_in(IMAP *imap, int32_t key, u32_t *list); 70 | int imap_ins(IMAP *imap, int32_t *keys, int nkeys, u32_t *list); 71 | /* set list[] if (list != NULL) */ 72 | #define IMAP_DEL(x, no) imap_del(((IMAP *)x), no) 73 | #else 74 | #define IMAP_DEL(x, no) 75 | #endif 76 | #define IMAP_GET(x, no) ((IMAP *)x)->vmap[no].val 77 | #define IMAP_SET(x, no, key) imap_set(((IMAP *)x), no, key) 78 | #pragma pack(pop) 79 | #endif 80 | -------------------------------------------------------------------------------- /utils/mmlist.h: -------------------------------------------------------------------------------- 1 | #ifndef __MMLIST__H 2 | #define __MMLIST__H 3 | #include 4 | #define MM_SLOT_MAX 524288 5 | #define MM_SLOT_NUM 1024 6 | #define MM_SLOT2_NUM 512 /* half of MM_SLOT_NUM */ 7 | #define MM_VNODE_INC 1000000 8 | #define MM_NODES_MAX 4294967296 9 | typedef struct MMKV 10 | { 11 | int32_t key; /* 数值 */ 12 | uint32_t val; /* 文档序号 */ 13 | }MMKV; 14 | typedef struct _VNODE 15 | { 16 | int32_t val; /* 数值 */ 17 | int32_t off; /* 节点偏移 */ 18 | }VNODE; 19 | typedef struct _MMSLOT 20 | { 21 | int32_t min; /* 当前桶的最小值 */ 22 | int32_t max; /* 当前桶的最大值 */ 23 | uint32_t nodeid; /* 桶偏移地址 */ 24 | int count; /* 桶元素个数 */ 25 | }MMSLOT; 26 | typedef struct _MMSTATE 27 | { 28 | MMSLOT slots[MM_SLOT_MAX]; /* 槽 */ 29 | uint32_t qleft[MM_SLOT_MAX]; /* 空闲的桶 */ 30 | uint32_t roots[MM_SLOT_MAX]; /* 桶和槽的衍射 */ 31 | uint32_t nleft; /* 空闲桶个数 */ 32 | uint32_t count; /* 槽的个数 */ 33 | }MMSTATE; 34 | typedef struct _MMLIST 35 | { 36 | MMSTATE *state; 37 | MMKV *map; 38 | VNODE *vmap; 39 | MMSLOT *slots; 40 | uint32_t *roots; 41 | int fd; 42 | int vfd; 43 | off_t size; 44 | off_t msize; 45 | off_t vsize; 46 | off_t vmsize; 47 | pthread_rwlock_t rwlock; 48 | }MMLIST; 49 | MMLIST *mmlist_init(char *file); 50 | int mmlist_set(MMLIST *mmlist, int no, int32_t key); 51 | int mmlist_del(MMLIST *mmlist, int no); 52 | /* return number of the hits */ 53 | int mmlist_range(MMLIST *mmlist, int32_t from, int32_t to, int32_t *list); 54 | int mmlist_rangefrom(MMLIST *mmlist, int32_t key, int32_t *list); /* key = from */ 55 | int mmlist_rangeto(MMLIST *mmlist, int32_t key, int32_t *list); /* key = to */ 56 | int mmlist_in(MMLIST *mlist, int32_t key, int32_t *list); 57 | int mmlist_ins(MMLIST *mmlist, int32_t *keys, int nkeys, int32_t *list); 58 | /* set list[] if (list != NULL) */ 59 | void mmlist_close(MMLIST *mmlist); 60 | #define MMLIST_GET(mlist, no) ((MMLIST *)mlist)->vmap[no].val 61 | #define MMLIST_SET(mlist, no, key) mmlist_set(((MMLIST *)mlist), no, key) 62 | #define MMLIST_DEL(mlist, no, key) mmlist_del(((MMLIST *)mlist), no) 63 | #endif 64 | -------------------------------------------------------------------------------- /utils/rwlock.h: -------------------------------------------------------------------------------- 1 | #ifndef __RWLOCK__H 2 | #define __RWLOCK__H 3 | #ifdef HAVE_PTHREAD 4 | #define RWLOCK_INIT(x) pthread_rwlock_init((x), NULL) 5 | #define RWLOCK_RDLOCK(x) pthread_rwlock_rdlock((x)) 6 | #define RWLOCK_WRLOCK(x) pthread_rwlock_wrlock((x)) 7 | #define RWLOCK_UNLOCK(x) pthread_rwlock_unlock((x)) 8 | #define RWLOCK_DESTROY(x) pthread_rwlock_destroy((x)) 9 | #else 10 | #define RWLOCK_INIT(x) 11 | #define RWLOCK_RDLOCK(x) 12 | #define RWLOCK_WRLOCK(x) 13 | #define RWLOCK_UNLOCK(x) 14 | #define RWLOCK_DESTROY(x) 15 | #endif 16 | #endif 17 | -------------------------------------------------------------------------------- /utils/xmm.h: -------------------------------------------------------------------------------- 1 | #ifndef _XMM_H_ 2 | #define _XMM_H_ 3 | void* xmm_mnew(size_t size); 4 | void* xmm_new(size_t size); 5 | void* xmm_mrenew(void *old, size_t old_size, size_t new_size); 6 | void* xmm_renew(void *old, size_t old_size, size_t new_size); 7 | void* xmm_mresize(void *old, size_t old_size, size_t new_size); 8 | void* xmm_resize(void *old, size_t old_size, size_t new_size); 9 | void xmm_free(void *m, size_t size); 10 | #endif 11 | -------------------------------------------------------------------------------- /utils/xxlist.h: -------------------------------------------------------------------------------- 1 | #ifndef __MMLIST__H 2 | #define __MMLIST__H 3 | #include 4 | #define MM_SLOT_MAX 524288 5 | #define MM_KV_NUM 1024 6 | #define MM_KV2_NUM 512 /* half of MM_KV_NUM */ 7 | #define MM_VNODE_INC 1000000 8 | #define MM_NODES_MAX 4294967296 9 | typedef struct MMKV 10 | { 11 | int32_t key; 12 | int32_t val; 13 | }MMKV; 14 | typedef struct _VNODE 15 | { 16 | int32_t val; 17 | ushort slot; 18 | ushort off; 19 | }VNODE; 20 | typedef struct _MMSLOT 21 | { 22 | int32_t min; 23 | int32_t max; 24 | int32_t count; 25 | int32_t bits; 26 | MMKV kvs[MM_KV_NUM]; 27 | }MMSLOT; 28 | typedef struct _MMSTATE 29 | { 30 | int32_t roots[MM_SLOT_MAX]; 31 | uint32_t left; 32 | int count; 33 | }MMSTATE; 34 | typedef struct _MMLIST 35 | { 36 | MMSTATE *state; 37 | VNODE *vmap; 38 | MMSLOT *slots; 39 | int fd; 40 | int vfd; 41 | off_t size; 42 | off_t msize; 43 | off_t vsize; 44 | off_t vmsize; 45 | }MMLIST; 46 | MMLIST *mmlist_init(char *file); 47 | int mmlist_insert(MMLIST *mmlist, int no, int32_t key); 48 | //int mmlist_remove(MMLIST *mmlist, int no, int32_t nodeid); 49 | int mmlist_update(MMLIST *mmlist, int no, int32_t nodeid); 50 | int mmlist_range(MMLIST *mmlist, int32_t from, int32_t to); 51 | int mmlist_range1(MMLIST *mmlist, int32_t from); 52 | int mmlist_range2(MMLIST *mmlist, int32_t from); 53 | void mmlist_close(MMLIST *mmlist); 54 | #endif 55 | --------------------------------------------------------------------------------