├── stamp.h.in ├── AUTHORS ├── fts-xapian-config.h.in ├── Makefile.am ├── contrib ├── systemd │ ├── dovecot-fts-optimize.timer.in │ └── dovecot-fts-optimize.service.in └── conf.d │ └── 90-fts.conf ├── src ├── Makefile.am ├── fts-xapian-settings.c ├── fts-xapian-plugin.h ├── fts-backend-xapian.h ├── fts-xapian-plugin.c ├── fts-backend-xapian.cpp └── fts-backend-xapian-functions.cpp ├── .gitignore ├── run-test.sh.in ├── configure.ac ├── README.md ├── m4 └── dovecot.m4 └── COPYING /stamp.h.in: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | Joan Moreau via dovecot 2 | 3 | -------------------------------------------------------------------------------- /fts-xapian-config.h.in: -------------------------------------------------------------------------------- 1 | #define FTS_XAPIAN_NAME "Dovecot FTS Xapian" 2 | #define FTS_XAPIAN_VERSION "1.9.3" 3 | -------------------------------------------------------------------------------- /Makefile.am: -------------------------------------------------------------------------------- 1 | SUBDIRS = src 2 | 3 | PACKAGE_VERSION = "1.9.3" 4 | VERSION = "1.9.3" 5 | 6 | ACLOCAL_AMFLAGS = -I m4 7 | -------------------------------------------------------------------------------- /contrib/systemd/dovecot-fts-optimize.timer.in: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Optimize Dovecot FTS Index 3 | 4 | [Timer] 5 | OnCalendar=@@index_frequency@@ 6 | Persistent=true 7 | 8 | [Install] 9 | WantedBy=timers.target 10 | -------------------------------------------------------------------------------- /contrib/systemd/dovecot-fts-optimize.service.in: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Optimize Dovecot FTS Index 3 | Requires=dovecot.service 4 | After=dovecot.service 5 | 6 | [Service] 7 | Type=oneshot 8 | ExecStart=@@prefix@@/doveadm fts optimize -A 9 | 10 | [Install] 11 | WantedBy=multi-user.target 12 | -------------------------------------------------------------------------------- /contrib/conf.d/90-fts.conf: -------------------------------------------------------------------------------- 1 | mail_plugins = $mail_plugins fts fts_xapian 2 | 3 | plugin { 4 | fts = xapian 5 | fts_xapian = partial=3 full=20 verbose=0 6 | fts_autoindex = yes 7 | fts_enforced = yes 8 | fts_autoindex_exclude = \Trash 9 | fts_decoder = decode2text 10 | } 11 | 12 | service indexer-worker { 13 | # Increase vsz_limit to 2GB or above. 14 | # Or 0 if you have rather large memory usable on your server, which is preferred for performance) 15 | vsz_limit = 2G 16 | } 17 | 18 | service decode2text { 19 | executable = script /usr/libexec/dovecot/decode2text.sh 20 | user = dovecot 21 | unix_listener decode2text { 22 | mode = 0666 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/Makefile.am: -------------------------------------------------------------------------------- 1 | AM_CPPFLAGS = -O2 \ 2 | $(LIBDOVECOT_INCLUDE) \ 3 | $(LIBDOVECOT_STORAGE_INCLUDE) \ 4 | $(LIBDOVECOT_DOVEADM_INCLUDE) \ 5 | $(LIBDOVECOT_LIBFTS_INCLUDE) \ 6 | $(LIBDOVECOT_FTS_INCLUDE) \ 7 | $(XAPIAN_INCLUDE) \ 8 | $(USERINIT_ARGS) \ 9 | $(ICU_CFLAGS) \ 10 | $(SQLITE_CFLAGS) 11 | 12 | AM_CXXFLAGS = -std=gnu++20 -Wall 13 | 14 | settingsdir = $(dovecot_moduledir)/settings 15 | settings_LTLIBRARIES = lib21_fts_xapian_settings.la 16 | 17 | lib21_fts_xapian_plugin_la_LDFLAGS = -module -avoid-version 18 | 19 | dovecot_module_LTLIBRARIES = lib21_fts_xapian_plugin.la 20 | 21 | lib21_fts_xapian_plugin_la_LIBADD = $(XAPIAN_LIBS) $(ICU_LIBS) $(SQLITE_LIBS) 22 | 23 | lib21_fts_xapian_settings_la_SOURCES = \ 24 | fts-xapian-settings.c 25 | 26 | lib21_fts_xapian_plugin_la_SOURCES = \ 27 | fts-xapian-plugin.c \ 28 | fts-backend-xapian.cpp \ 29 | fts-xapian-settings.c 30 | 31 | EXTRA_DIST = \ 32 | fts-backend-xapian-functions.cpp 33 | 34 | noinst_HEADERS = fts-xapian-plugin.h 35 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # http://www.gnu.org/software/automake 2 | 3 | Makefile.in 4 | /ar-lib 5 | /mdate-sh 6 | /py-compile 7 | /test-driver 8 | /ylwrap 9 | .deps/ 10 | .dirstamp 11 | 12 | # http://www.gnu.org/software/autoconf 13 | 14 | autom4te.cache 15 | /autoscan.log 16 | /autoscan-*.log 17 | /aclocal.m4 18 | /compile 19 | /config.guess 20 | /config.h.in 21 | /config.log 22 | /config.status 23 | /config.sub 24 | /configure 25 | /configure.scan 26 | /depcomp 27 | /install-sh 28 | /missing 29 | /stamp-h1 30 | /stamp-h2 31 | /stamp.h 32 | 33 | # https://www.gnu.org/software/libtool/ 34 | 35 | /ltmain.sh 36 | /libtool 37 | 38 | # http://www.gnu.org/software/texinfo 39 | 40 | /texinfo.tex 41 | 42 | # http://www.gnu.org/software/m4/ 43 | 44 | m4/libtool.m4 45 | m4/ltoptions.m4 46 | m4/ltsugar.m4 47 | m4/ltversion.m4 48 | m4/lt~obsolete.m4 49 | 50 | # Generated Makefile 51 | # (meta build system like autotools, 52 | # can automatically generate from config.status script 53 | # (which is called by configure script)) 54 | Makefile 55 | 56 | configure~ 57 | 58 | /dummy-config.h 59 | /dummy-config.h.in 60 | /dummy-config.h.in~ 61 | /fts-xapian-config.h 62 | /run-test.sh 63 | /PACKAGES/RPM/fedora 64 | 65 | src/*.o 66 | src/*.lo 67 | src/*.la 68 | 69 | src/.libs/** 70 | -------------------------------------------------------------------------------- /src/fts-xapian-settings.c: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2019 Joan Moreau , see the included COPYING file */ 2 | 3 | #ifdef FTS_DOVECOT24 4 | 5 | #include "lib.h" 6 | #include "settings.h" 7 | #include "settings-parser.h" 8 | #include "fts-xapian-plugin.h" 9 | 10 | #undef DEF 11 | #define DEF(type, name) SETTING_DEFINE_STRUCT_##type(XAPIAN_LABEL"_"#name, name, struct fts_xapian_settings) 12 | 13 | static const struct setting_define fts_xapian_setting_defines[] = 14 | { 15 | DEF(UINT, verbose), 16 | DEF(UINT, lowmemory), 17 | DEF(UINT, partial), 18 | DEF(UINT, maxthreads), 19 | SETTING_DEFINE_LIST_END 20 | }; 21 | 22 | static const struct fts_xapian_settings fts_xapian_default_settings = 23 | { 24 | .verbose = 0, 25 | .lowmemory = XAPIAN_MIN_RAM, 26 | .partial = XAPIAN_DEFAULT_PARTIAL, 27 | .maxthreads = 0, 28 | }; 29 | 30 | const struct setting_parser_info fts_xapian_setting_parser_info = 31 | { 32 | .name = XAPIAN_LABEL, 33 | 34 | .defines = fts_xapian_setting_defines, 35 | .defaults = &fts_xapian_default_settings, 36 | 37 | .struct_size = sizeof(struct fts_xapian_settings), 38 | .pool_offset1 = 1 + offsetof(struct fts_xapian_settings, pool), 39 | }; 40 | 41 | const char *fts_xapian_settings_version = DOVECOT_ABI_VERSION; 42 | 43 | const struct setting_parser_info *fts_xapian_settings_set_infos[] = 44 | { 45 | &fts_xapian_setting_parser_info, 46 | NULL, 47 | }; 48 | 49 | #endif 50 | 51 | -------------------------------------------------------------------------------- /run-test.sh.in: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | ## serial 2 4 | 5 | set -eu 6 | 7 | top_srcdir="@abs_top_srcdir@" 8 | VALGRIND="@VALGRIND@" 9 | 10 | if test $# -eq 0 || test "$1" = ""; then 11 | echo "Missing target binary" >&2 12 | exit 1 13 | fi 14 | 15 | if test "${NOUNDEF:-}" != ""; then 16 | noundef="--undef-value-errors=no" 17 | else 18 | noundef="" 19 | fi 20 | 21 | if test "${NOCHILDREN:-}" != ""; then 22 | trace_children="--trace-children=no" 23 | else 24 | trace_children="--trace-children=yes" 25 | fi 26 | 27 | skip_path="$top_srcdir/run-test-valgrind.exclude" 28 | if test -r "$skip_path" && grep -w -q "$(basename $[1])" "$skip_path"; then 29 | NOVALGRIND=true 30 | fi 31 | 32 | if test "${NOVALGRIND:-}" != ""; then 33 | "$@" 34 | ret=$? 35 | else 36 | test_out="test.out~$$" 37 | trap "rm -f $test_out" 0 1 2 3 15 38 | supp_path="$top_srcdir/run-test-valgrind.supp" 39 | ret=0 40 | if test -r "$supp_path"; then 41 | $VALGRIND -q $trace_children --error-exitcode=213 --leak-check=full --gen-suppressions=all --suppressions="$supp_path" --log-file=$test_out $noundef "$@" || ret=$? 42 | else 43 | $VALGRIND -q $trace_children --error-exitcode=213 --leak-check=full --gen-suppressions=all --log-file=$test_out $noundef "$@" || ret=$? 44 | fi 45 | if test -s $test_out; then 46 | cat $test_out 47 | ret=1 48 | fi 49 | fi 50 | if test $ret != 0; then 51 | echo "Failed to run: $@" >&2 52 | fi 53 | 54 | exit $ret 55 | -------------------------------------------------------------------------------- /src/fts-xapian-plugin.h: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2019 Joan Moreau , see the included COPYING file */ 2 | 3 | 4 | #ifndef FTS_XAPIAN_PLUGIN_H 5 | #define FTS_XAPIAN_PLUGIN_H 6 | 7 | #include "config.h" 8 | #include "lib.h" 9 | #include "mail-user.h" 10 | #include "fts-api.h" 11 | #include "fts-user.h" 12 | #include "mail-search.h" 13 | #include "mail-storage-private.h" 14 | #include "restrict-process-size.h" 15 | #include "mail-storage-hooks.h" 16 | #include "module-context.h" 17 | #include "fts-api-private.h" 18 | #include "master-service.h" 19 | #ifdef FTS_DOVECOT24 20 | #include "fts-settings.h" 21 | #include "settings-parser.h" 22 | #include "settings.h" 23 | #endif 24 | 25 | #if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__APPLE__) 26 | #include 27 | #include 28 | #endif 29 | 30 | #include 31 | 32 | #define XAPIAN_PLUGIN_VERSION "1.9.3" 33 | #define XAPIAN_LABEL "fts_xapian" 34 | 35 | // Main parameters 36 | #define XAPIAN_FILE_PREFIX "xapian-indexes" // Locations of indexes 37 | #define XAPIAN_MIN_RAM 300L // MB 38 | #define XAPIAN_DEFAULT_PARTIAL 3L 39 | 40 | struct fts_xapian_settings 41 | { 42 | #ifdef FTS_DOVECOT24 43 | pool_t pool; 44 | #endif 45 | unsigned int verbose; 46 | unsigned int lowmemory; 47 | unsigned int partial; 48 | unsigned int maxthreads; 49 | }; 50 | 51 | struct fts_xapian_user { 52 | union mail_user_module_context module_ctx; 53 | #ifdef FTS_DOVECOT24 54 | struct fts_xapian_settings *set; 55 | #else 56 | struct fts_xapian_settings set; 57 | #endif 58 | }; 59 | 60 | #define FTS_XAPIAN_USER_CONTEXT(obj) (struct fts_xapian_user *)MODULE_CONTEXT(obj, fts_xapian_user_module) 61 | #if ((DOVECOT_VERSION_MINOR > 2) || (DOVECOT_VERSION_MAJOR > 2) || (FTS_DOVECOT24 > 0)) 62 | #define FTS_XAPIAN_USER_CONTEXT_REQUIRE(obj) MODULE_CONTEXT_REQUIRE(obj, fts_xapian_user_module) 63 | #endif 64 | 65 | extern const char *fts_xapian_plugin_dependencies[]; 66 | extern MODULE_CONTEXT_DEFINE(fts_xapian_user_module, &mail_user_module_register); 67 | extern struct fts_backend fts_backend_xapian; 68 | 69 | #ifdef FTS_DOVECOT24 70 | 71 | int fts_xapian_mail_user_get(struct mail_user *user, struct event *event, 72 | struct fts_xapian_user **fuser_r, 73 | const char **error_r); 74 | 75 | extern const struct setting_parser_info fts_xapian_setting_parser_info; 76 | 77 | #endif 78 | 79 | void fts_xapian_plugin_init(struct module *module); 80 | void fts_xapian_plugin_deinit(void); 81 | 82 | #endif 83 | -------------------------------------------------------------------------------- /src/fts-backend-xapian.h: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2019 Joan Moreau , see the included COPYING file */ 2 | 3 | #ifndef FTS_XAPIAN_BACKEND_H 4 | #define FTS_XAPIAN_BACKEND_H 5 | 6 | #define XAPIAN_SLEEP std::chrono::milliseconds(200) 7 | 8 | // Ressources limits 9 | #define XAPIAN_TERM_SIZELIMIT 245L // Hard limit of Xapian library 10 | #define XAPIAN_MAXTERMS_PERDOC 50000L // Nb of keywords max per email 11 | #define XAPIAN_WRITING_CACHE 5000L // Max nb of emails processed in cache 12 | #define XAPIAN_DICT_MAX 60000L // Max nb of terms in the dict 13 | #define XAPIAN_MAX_ERRORS 1024L 14 | #define XAPIAN_MAX_SEC_WAIT 15L 15 | 16 | #define HDRS_NB 11 17 | static const char * hdrs_emails[HDRS_NB] = { "uid", "subject", "from", "to", "cc", "bcc", "messageid", "listid", "body", "contenttype", "" }; 18 | static const char * hdrs_xapian[HDRS_NB] = { "Q", "S", "A", "XTO", "XCC", "XBCC", "XMID", "XLIST", "XBDY", "XCT", "XBDY" }; 19 | static const char * hdrs_query[HDRS_NB] = { "a", "b", "c", "d", "d", "e", "f", "g", "h", "i", "h" }; 20 | #define HDR_BODY 8L 21 | 22 | static const char * createExpTable = "CREATE TABLE IF NOT EXISTS expunges(ID INTEGER PRIMARY KEY NOT NULL);"; 23 | static const char * selectExpUIDs = "select ID from expunges;"; 24 | static const char * replaceExpUID = "replace into expunges values (%d);"; 25 | static const char * deleteExpUID = "delete from expunges where ID=%d;"; 26 | static const char * suffixExp = "_exp.db"; 27 | 28 | static const char * createDictTable = "CREATE TABLE IF NOT EXISTS dict (keyword TEXT COLLATE NOCASE, header INTEGER, len INTEGER, UNIQUE(keyword,header));"; 29 | static const char * createDictIndexes = "CREATE INDEX IF NOT EXISTS dict_len ON dict (len); CREATE INDEX IF NOT EXISTS dict_h ON dict(header); CREATE INDEX IF NOT EXISTS dict_t ON dict(keyword);"; 30 | static const char * createTmpTable = "ATTACH DATABASE ':memory:' AS work; CREATE TABLE work.dict (keyword TEXT COLLATE NOCASE, header INTEGER, len INTEGER, UNIQUE(keyword,header) ); CREATE INDEX IF NOT EXISTS work.dict_h ON dict(header)"; 31 | static const char * replaceTmpWord ="INSERT OR IGNORE INTO work.dict VALUES('"; 32 | static const char * flushTmpWords = "BEGIN TRANSACTION; INSERT OR IGNORE INTO main.dict SELECT keyword, header, len FROM work.dict; DELETE FROM work.dict; COMMIT;"; 33 | static const char * searchDict1 = "SELECT keyword FROM dict WHERE keyword like '%"; 34 | static const char * searchDict2 = " ORDER BY len LIMIT 100"; 35 | static const char * suffixDict = "_dict.db"; 36 | 37 | #define CHAR_KEY "_" 38 | #define CHAR_SPACE " " 39 | 40 | #define CHARS_PB 21 41 | static const char * chars_pb[] = { "<", ">", ".", "-", "@", "&", "%", "*", "|", "`", "#", "^", "\\", "'", "/", "~", "[", "]", "{", "}", "-" }; 42 | 43 | #define CHARS_SEP 16 44 | static const char * chars_sep[] = { "\"", "\r", "\n", "\t", ",", ":", ";", "(", ")", "?", "!", "¿", "¡", "\u00A0", "‘", "“" }; 45 | 46 | #endif 47 | -------------------------------------------------------------------------------- /configure.ac: -------------------------------------------------------------------------------- 1 | AC_INIT([Dovecot FTS Xapian],[1.9.3],[jom@grosjo.net],[dovecot-fts-xapian]) 2 | AC_CONFIG_AUX_DIR([.]) 3 | AC_CONFIG_SRCDIR([src]) 4 | AC_CONFIG_MACRO_DIR([m4]) 5 | 6 | # Autoheader is not needed and does more harm than good for this package 7 | # However, it is tightly integrated in autoconf/automake and therefore it is 8 | # difficult not to use it. As a workaround we give autoheader a dummy config 9 | # header to chew on and we handle the real config header ourselves. 10 | AC_CONFIG_HEADERS([dummy-config.h fts-xapian-config.h]) 11 | 12 | AM_INIT_AUTOMAKE([no-define foreign tar-ustar]) 13 | AM_MAINTAINER_MODE 14 | AC_PROG_CC 15 | AC_PROG_CPP 16 | AC_PROG_CXX 17 | LT_INIT 18 | 19 | PKG_PROG_PKG_CONFIG 20 | 21 | DC_DOVECOT 22 | DC_DOVECOT_MODULEDIR 23 | LIBDOVECOT_INCLUDE="$LIBDOVECOT_INCLUDE" 24 | CFLAGS="$CFLAGS $EXTRA_CFLAGS $DOVECOT_FLAGS -O2" 25 | LIBS="$DOVECOT_LIBS" 26 | BINARY_LDFLAGS="$PIE_LDFLAGS $RELRO_LDFLAGS" 27 | BINARY_CFLAGS="$PIE_CFLAGS" 28 | AC_SUBST(LIBDOVECOT_INCLUDE) 29 | 30 | PKG_PROG_PKG_CONFIG() # check and set $PKG_CONFIG 31 | PKG_CHECK_MODULES([ICU_UC], [icu-uc >= 50], [have_icu_uc=true], [have_icu_uc=false]) 32 | PKG_CHECK_MODULES([ICU_I18N], [icu-i18n >= 50], [have_icu_i18n=true], [have_icu_i18n=false]) 33 | 34 | if !($have_icu_uc && $have_icu_i18n); then 35 | AC_MSG_ERROR([icu 50 or higher is required, but was not found.]) 36 | fi 37 | 38 | ICU_LDFLAGS=$($PKG_CONFIG --libs-only-L icu-uc | sed 's/-L//g') 39 | ICU_LIBS=$($PKG_CONFIG --libs icu-uc icu-io icu-i18n) 40 | ICU_CFLAGS=$($PKG_CONFIG --cflags-only-I icu-uc icu-io icu-i18n) 41 | ICU_INCLUDE=$($PKG_CONFIG --variable=includedir icu-uc) 42 | 43 | AC_MSG_CHECKING([for fts_mail_user_init]) 44 | 45 | AC_COMPILE_IFELSE( 46 | [AC_LANG_PROGRAM([[ 47 | #include "$dovecot_pkgincludedir/config.h" 48 | #include "$dovecot_pkgincludedir/lib.h" 49 | #include "$dovecot_pkgincludedir/mail-user.h" 50 | #include "$dovecot_pkgincludedir/fts-user.h" 51 | int f() 52 | { 53 | void *ptr = &fts_mail_user_init; 54 | (void)ptr; 55 | return 0; 56 | } 57 | ]])], 58 | [ 59 | AC_MSG_RESULT([yes]) 60 | ],[ 61 | AC_MSG_RESULT([no]) 62 | AC_MSG_ERROR([fts_mail_user_init not found 63 | dovecot headers 64 | are not installed correctly]) 65 | ]) 66 | 67 | AC_MSG_CHECKING([whether fts_mail_user_init has 2, 3 or 4 args]) 68 | 69 | AC_COMPILE_IFELSE( 70 | [AC_LANG_PROGRAM([[ 71 | #include "$dovecot_pkgincludedir/config.h" 72 | #include "$dovecot_pkgincludedir/lib.h" 73 | #include "$dovecot_pkgincludedir/mail-user.h" 74 | #include "$dovecot_pkgincludedir/fts-user.h" 75 | void f(struct mail_user *s, const char **error) { 76 | fts_mail_user_init(s, NULL, 0, error); 77 | } 78 | ]])], 79 | [ 80 | AC_MSG_RESULT([4]) 81 | USERINIT_ARGS="-DFTS_DOVECOT24=1" 82 | ],[ 83 | AC_COMPILE_IFELSE( 84 | [AC_LANG_PROGRAM([[ 85 | #include "$dovecot_pkgincludedir/config.h" 86 | #include "$dovecot_pkgincludedir/lib.h" 87 | #include "$dovecot_pkgincludedir/mail-user.h" 88 | #include "$dovecot_pkgincludedir/fts-user.h" 89 | void f(struct mail_user *s, const char **error) { 90 | fts_mail_user_init(s, 0, error); 91 | } 92 | ]])], 93 | [ 94 | AC_MSG_RESULT([3]) 95 | USERINIT_ARGS="-DFTS_MAIL_USER_INIT_THREE_ARGS=1" 96 | ],[ 97 | AC_MSG_RESULT([no]) 98 | USERINIT_ARGS="" 99 | ]) 100 | ]) 101 | 102 | AC_SUBST(ICU_CFLAGS) 103 | AC_SUBST(ICU_LIBS) 104 | AC_SUBST(ICU_INCLUDE) 105 | AC_SUBST(ICU_LDFLAGS) 106 | AC_SUBST(USERINIT_ARGS) 107 | 108 | PKG_CHECK_MODULES([SQLITE], [sqlite3], [have_sqlite=true], [have_sqlite=false]); 109 | if !($have_sqlite); then 110 | AC_MSG_ERROR([SQLite is required, but was not found.]) 111 | fi 112 | 113 | AC_CHECK_PROG(XAPIAN_CONFIG,xapian-config,xapian-config,no) 114 | AM_CONDITIONAL(HAVE_XAPIAN,test "x$XAPIAN_CONFIG" != "xno") 115 | AS_IF([test "x$XAPIAN_CONFIG" = "xno"],[ 116 | AC_MSG_ERROR([ 117 | *** xapian could not be found; please install it 118 | *** e.g., in debian/ubuntu the package would be 'libxapian-dev' 119 | *** If you compiled it yourself, you should ensure that xapian-config 120 | *** is in your PATH.])], 121 | [XAPIAN_VERSION=$($XAPIAN_CONFIG --version | sed -e 's/.* //')]) 122 | 123 | XAPIAN_CXXFLAGS="$($XAPIAN_CONFIG --cxxflags)" 124 | XAPIAN_LIBS="$($XAPIAN_CONFIG --libs)" 125 | 126 | AC_SUBST(XAPIAN_CXXFLAGS) 127 | AC_SUBST(XAPIAN_LIBS) 128 | 129 | if test "$DOVECOT_INSTALLED" != "yes"; then 130 | DOVECOT_FTS_PLUGIN="$abs_dovecotdir/src/plugins/fts/lib20_fts_plugin.la" 131 | else 132 | DOVECOT_FTS_PLUGIN="$dovecot_installed_moduledir/lib20_fts_plugin.la" 133 | fi 134 | AC_SUBST(DOVECOT_FTS_PLUGIN) 135 | 136 | AC_CONFIG_FILES([ 137 | Makefile 138 | src/Makefile 139 | stamp.h]) 140 | 141 | AC_OUTPUT 142 | -------------------------------------------------------------------------------- /src/fts-xapian-plugin.c: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2019 Joan Moreau , see the included COPYING file */ 2 | 3 | #include "fts-xapian-plugin.h" 4 | 5 | const char *fts_xapian_plugin_version = DOVECOT_ABI_VERSION; 6 | 7 | struct fts_xapian_user_module fts_xapian_user_module = MODULE_CONTEXT_INIT(&mail_user_module_register); 8 | 9 | static void fts_xapian_mail_user_deinit(struct mail_user *user) 10 | { 11 | #if ((DOVECOT_VERSION_MINOR > 2) || (DOVECOT_VERSION_MAJOR > 2) || (FTS_DOVECOT24 > 0)) 12 | struct fts_xapian_user *fuser = FTS_XAPIAN_USER_CONTEXT_REQUIRE(user); 13 | #else 14 | struct fts_xapian_user *fuser = FTS_XAPIAN_USER_CONTEXT(user); 15 | #endif 16 | 17 | #ifdef FTS_DOVECOT24 18 | settings_free(fuser->set); 19 | #else 20 | fts_mail_user_deinit(user); 21 | #endif 22 | fuser->module_ctx.super.deinit(user); 23 | } 24 | 25 | #ifdef FTS_DOVECOT24 26 | 27 | int fts_xapian_mail_user_get(struct mail_user *user, struct event *event, 28 | struct fts_xapian_user **fuser_r, const char **error_r) 29 | { 30 | struct fts_xapian_user *fuser = FTS_XAPIAN_USER_CONTEXT_REQUIRE(user); 31 | struct fts_xapian_settings *set; 32 | 33 | if (settings_get(event, &fts_xapian_setting_parser_info, 0, &set, error_r) < 0) 34 | { 35 | return -1; 36 | } 37 | 38 | /* Reference the user even when fuser is already initialized */ 39 | if (fts_mail_user_init(user, event, FALSE, error_r) < 0) 40 | { 41 | settings_free(set); 42 | return -1; 43 | } 44 | if (fuser->set == NULL) 45 | fuser->set = set; 46 | else 47 | settings_free(set); 48 | 49 | *fuser_r = fuser; 50 | return 0; 51 | } 52 | 53 | static void fts_xapian_mail_user_created(struct mail_user *user) 54 | { 55 | struct fts_xapian_user *fuser; 56 | struct mail_user_vfuncs *v = user->vlast; 57 | 58 | fuser = p_new(user->pool, struct fts_xapian_user, 1); 59 | fuser->module_ctx.super = *v; 60 | user->vlast = &fuser->module_ctx.super; 61 | v->deinit = fts_xapian_mail_user_deinit; 62 | MODULE_CONTEXT_SET(user, fts_xapian_user_module, fuser); 63 | } 64 | 65 | #else 66 | 67 | static void fts_xapian_mail_user_created(struct mail_user *user) 68 | { 69 | const char *error; 70 | 71 | struct mail_user_vfuncs *v = user->vlast; 72 | struct fts_xapian_user *fuser; 73 | 74 | fuser = p_new(user->pool, struct fts_xapian_user, 1); 75 | 76 | fuser->set.verbose = 0; 77 | fuser->set.lowmemory = XAPIAN_MIN_RAM; 78 | fuser->set.partial = XAPIAN_DEFAULT_PARTIAL; 79 | fuser->set.maxthreads = 0; 80 | 81 | const char * env = mail_user_plugin_getenv(user, XAPIAN_LABEL); 82 | if (env == NULL) 83 | { 84 | i_warning("FTS Xapian: missing configuration - Using default values"); 85 | } 86 | else 87 | { 88 | long len; 89 | const char *const *tmp; 90 | 91 | for (tmp = t_strsplit_spaces(env, " "); *tmp != NULL; tmp++) 92 | { 93 | if (strncmp(*tmp, "partial=",8)==0) 94 | { 95 | len=atol(*tmp + 8); 96 | if(len<3) 97 | { 98 | i_error("FTS Xapian: 'partial' parameter is incorrect (%ld). Try 'partial=%ld'",len,XAPIAN_DEFAULT_PARTIAL); 99 | len=XAPIAN_DEFAULT_PARTIAL; 100 | } 101 | fuser->set.partial = len; 102 | } 103 | else if (strncmp(*tmp,"verbose=",8)==0) 104 | { 105 | len=atol(*tmp + 8); 106 | if(len>0) { fuser->set.verbose = len; } 107 | } 108 | else if (strncmp(*tmp,"lowmemory=",10)==0) 109 | { 110 | len=atol(*tmp + 10); 111 | if(len>0) { fuser->set.lowmemory = len; } 112 | } 113 | else if (strncmp(*tmp,"maxthreads=",11)==0) 114 | { 115 | len=atol(*tmp + 11); 116 | if(len>0) { fuser->set.maxthreads = len; } 117 | } 118 | else if (strncmp(*tmp,"attachments=",12)==0) 119 | { 120 | // Legacy 121 | } 122 | else if (strncmp(*tmp,"full=",5)==0) 123 | { 124 | // Legacy 125 | } 126 | else if (strncmp(*tmp,"detach=",7)==0) 127 | { 128 | // Legacy 129 | } 130 | else 131 | { 132 | i_error("FTS Xapian: Invalid setting: %s", *tmp); 133 | } 134 | } 135 | } 136 | 137 | #ifdef FTS_MAIL_USER_INIT_THREE_ARGS 138 | if (fts_mail_user_init(user, FALSE, &error) < 0) 139 | #else 140 | if (fts_mail_user_init(user, &error) < 0) 141 | #endif 142 | 143 | { 144 | if ( fuser->set.verbose > 0 ) i_warning("FTS Xapian: %s", error); 145 | } 146 | 147 | fuser->module_ctx.super = *v; 148 | user->vlast = &fuser->module_ctx.super; 149 | v->deinit = fts_xapian_mail_user_deinit; 150 | 151 | MODULE_CONTEXT_SET(user, fts_xapian_user_module, fuser); 152 | } 153 | #endif 154 | 155 | static struct mail_storage_hooks fts_xapian_mail_storage_hooks = 156 | { 157 | .mail_user_created = fts_xapian_mail_user_created 158 | }; 159 | 160 | void fts_xapian_plugin_init(struct module *module ATTR_UNUSED) 161 | { 162 | fts_backend_register(&fts_backend_xapian); 163 | mail_storage_hooks_add(module, &fts_xapian_mail_storage_hooks); 164 | } 165 | 166 | void fts_xapian_plugin_deinit(void) 167 | { 168 | fts_backend_unregister(fts_backend_xapian.name); 169 | mail_storage_hooks_remove(&fts_xapian_mail_storage_hooks); 170 | } 171 | 172 | const char *fts_xapian_plugin_dependencies[] = { "fts", NULL }; 173 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | FTS Xapian plugin for Dovecot 2 | ============================= 3 | 4 | What is this? 5 | ------------- 6 | 7 | This project intends to provide a straightforward, simple and maintenance free, way to configure FTS plugin for [Dovecot](https://github.com/dovecot/), leveraging the efforts by the [Xapian.org](https://xapian.org/) team. 8 | 9 | This effort came after Dovecot team decided to deprecate "fts_squat" included in the dovecot core, and due to the complexity of the Solr plugin capabilitles, unneeded for most users. 10 | 11 | If you feel donating, kindly use Paypal : moreaujoan@gmail.com 12 | 13 | 14 | Debugging/Support 15 | ----------------- 16 | Please submit requests/bugs via the [GitHub issue tracker](https://github.com/grosjo/fts-xapian/issues). 17 | A Matrix Room exists also at : #xapian-dovecot:matrix.grosjo.net 18 | 19 | 20 | Availability in major distributions 21 | ----------------------------------- 22 | 23 | THis plugin is readly available in major distributions under the name "dovecot-fts-xapian" 24 | - Archlinux : https://archlinux.org/packages/?q=dovecot-fts-xapian 25 | - Debian : https://packages.debian.org/bookworm/dovecot-fts-xapian 26 | - Fedora : https://src.fedoraproject.org/rpms/dovecot-fts-xapian 27 | 28 | 29 | Configuration - dovecot.conf file 30 | --------------------------------- 31 | 32 | You need to setup LMTP properly with your SMTP server. Kindly refer to: 33 | - For Postfix : https://doc.dovecot.org/2.3/configuration_manual/howto/postfix_dovecot_lmtp/ 34 | - For Exim : https://doc.dovecot.org/2.3/configuration_manual/howto/dovecot_lmtp_exim/ 35 | 36 | 37 | Update your dovecot.conf file with something similar to: 38 | 39 | *VERSION 2.3.x* 40 | 41 | ``` 42 | (...) 43 | protocols = imap pop3 sieve lmtp 44 | 45 | mail_plugins = (...) fts fts_xapian 46 | 47 | plugin { 48 | fts = xapian 49 | fts_xapian = verbose=0 50 | 51 | fts_autoindex = yes 52 | fts_enforced = yes 53 | 54 | (...) 55 | } 56 | 57 | service indexer-worker { 58 | # Increase vsz_limit to 2GB or above. 59 | # Or 0 if you have rather large memory usable on your server, which is preferred for performance) 60 | vsz_limit = 2G 61 | # This one must be 0 62 | process_limit = 0 63 | } 64 | (...) 65 | 66 | ``` 67 | 68 | *VERSION 2.4.x* 69 | 70 | ``` 71 | (...) 72 | protocols = imap pop3 sieve lmtp 73 | 74 | mail_plugins = (...) fts fts_xapian 75 | 76 | fts_autoindex = yes 77 | 78 | language "en" { 79 | default = yes 80 | } 81 | // Note : the 'language' settings is set mandatory by dovecot 82 | // but has totally NO impact on FTS Xapian module 83 | 84 | fts xapian { 85 | // Note : All variables are optional 86 | verbose = 0 87 | maxthreads = 4 88 | lowmemory = 500 89 | partial = 3 90 | } 91 | (...) 92 | 93 | ``` 94 | 95 | Configuration options 96 | -------------------------------- 97 | 98 | | Option | Optional | Description | Possible values | Default value | 99 | |----------------|----------|---------------------------------|-----------------------------------------------------|---------------| 100 | | partial | yes | Minimum size of search keyword | 3 or above | 3 | 101 | | verbose | yes | Logs verbosity | 0 (silent), 1 (verbose) or 2 (debug) | 0 | 102 | | lowmemory | yes | Memory limit before disk commit | 0 (default, meaning 300MB), or set value (in MB) | 0 | 103 | | maxthreads | yes | Maximum number of threads | 0 (default, hardware limit), or value above 2 | 0 | 104 | 105 | 106 | 107 | Index updating 108 | ------------------------------ 109 | 110 | Just restart Dovecot: 111 | 112 | ```sh 113 | sudo service restart dovecot 114 | ``` 115 | 116 | You shall put in a cron the following command (daily for instance) to cleanup indexes : 117 | 118 | ```sh 119 | doveadm fts optimize -A 120 | ``` 121 | 122 | If this is not a fresh install of dovecot, you need to re-index your mailboxes: 123 | 124 | ```sh 125 | doveadm index -A -q \* 126 | ``` 127 | With argument -A, it will re-index all mailboxes, therefore may take a while. 128 | With argument -q, doveadm queues the indexing to be run by indexer process. Remove -q if you want to index immediately. 129 | 130 | 131 | Building yourself - Prerequisites 132 | ---------------------------------- 133 | 134 | You are going to need the following things to get this going: 135 | 136 | ``` 137 | * Dovecot 2.2.x (or above) 138 | * Xapian 1.2.x (or above) 139 | * ICU 50.x (or above) 140 | * SQlite 3.x 141 | ``` 142 | 143 | You will need to configure properly [Users Home Directories](https://doc.dovecot.org/2.3/configuration_manual/home_directories_for_virtual_users/) in dovecot configuration 144 | 145 | 146 | 147 | Building yourself - Installing the Dovecot plugin 148 | ----------------------------- 149 | 150 | First install the following packages, or equivalent for your operating system. 151 | 152 | ``` 153 | Ubuntu: 154 | apt-get build-dep dovecot-core 155 | apt-get install dovecot-dev git libxapian-dev libicu-dev libsqlite3-dev autoconf automake libtool pkg-config 156 | 157 | Archlinux: 158 | pacman -S dovecot 159 | pacman -S xapian-core icu git sqlite 160 | 161 | FreeBSD: 162 | pkg install xapian-core 163 | pkg install xapian-bindings 164 | pkg install icu 165 | pkg install git 166 | 167 | Fedora: 168 | dnf install sqlite-devel libicu-devel xapian-core-devel 169 | dnf install dovecot-devel git 170 | ``` 171 | 172 | Clone this project: 173 | 174 | ``` 175 | git clone https://github.com/grosjo/fts-xapian 176 | cd fts-xapian 177 | ``` 178 | 179 | Compile and install the plugin. 180 | 181 | ``` 182 | autoupdate 183 | autoreconf -vi 184 | ./configure --with-dovecot=/path/to/dovecot 185 | make 186 | sudo make install 187 | ``` 188 | 189 | Note: if your system is quite old, you may change gnu++20 by gnu++11 in src/Makefile.in 190 | 191 | Replace /path/to/dovecot by the actual path to 'dovecot-config'. 192 | Type 'locate dovecot-config' in a shell to figure this out. On ArchLinux , it is /usr/lib/dovecot. 193 | 194 | For specific configuration, you may have to 'export PKG_CONFIG_PATH=...'. To check that, type 'pkg-config --cflags-only-I icu-uc icu-io icu-i18n', it shall return no error. 195 | 196 | The module will be placed into the module directory of your dovecot configuration 197 | 198 | 199 | 200 | ------ 201 | 202 | 203 | Thanks to Aki Tuomi , Stephan Bosch , Paul Hecker 204 | -------------------------------------------------------------------------------- /m4/dovecot.m4: -------------------------------------------------------------------------------- 1 | dnl dovecot.m4 - Check presence of dovecot -*-Autoconf-*- 2 | dnl 3 | dnl Copyright (C) 2010 Dennis Schridde 4 | dnl 5 | dnl This file is free software; the authors give 6 | dnl unlimited permission to copy and/or distribute it, with or without 7 | dnl modifications, as long as this notice is preserved. 8 | 9 | # serial 41 10 | 11 | dnl 12 | dnl Check for support for D_FORTIFY_SOURCE=2 13 | dnl 14 | 15 | AC_DEFUN([AC_CC_D_FORTIFY_SOURCE],[ 16 | AC_REQUIRE([gl_UNKNOWN_WARNINGS_ARE_ERRORS]) 17 | AS_IF([test "$enable_hardening" = yes], [ 18 | case "$host" in 19 | *) 20 | gl_COMPILER_OPTION_IF([-O2 -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2], [ 21 | CFLAGS="$CFLAGS -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2" 22 | ], 23 | [], 24 | [AC_LANG_PROGRAM()] 25 | ) 26 | ;; 27 | esac 28 | ]) 29 | ]) 30 | 31 | dnl * gcc specific options 32 | AC_DEFUN([DC_DOVECOT_CFLAGS],[ 33 | m4_version_prereq(2.70, [AC_PROG_CC], [AC_PROG_CC_C99]) 34 | 35 | AS_IF([test "$ac_prog_cc_stdc" = "c89" || test "$ac_prog_cc_std" = "no" || test "$ac_cv_prog_cc_c99" = "no"], [ 36 | AC_MSG_ERROR(C99 capable compiler required) 37 | ]) 38 | 39 | AC_MSG_CHECKING([Which $CC -std flag to use]) 40 | old_cflags=$CFLAGS 41 | std= 42 | for mystd in gnu11 gnu99 c11 c99; do 43 | CFLAGS="-std=$mystd" 44 | AC_COMPILE_IFELSE([AC_LANG_PROGRAM() 45 | ], [ 46 | CFLAGS="$CFLAGS $old_cflags" 47 | std=$mystd 48 | break 49 | ], [ 50 | CFLAGS="$old_cflags" 51 | ]) 52 | done 53 | AC_MSG_RESULT($std) 54 | 55 | AS_IF([test "x$ac_cv_c_compiler_gnu" = "xyes"], [ 56 | dnl -Wcast-qual -Wcast-align -Wconversion -Wunreachable-code # too many warnings 57 | dnl -Wstrict-prototypes -Wredundant-decls # may give warnings in some systems 58 | dnl -Wmissing-format-attribute -Wmissing-noreturn -Wwrite-strings # a couple of warnings 59 | CFLAGS="$CFLAGS -Wall -W -Wmissing-prototypes -Wmissing-declarations -Wpointer-arith -Wchar-subscripts -Wformat=2 -Wbad-function-cast" 60 | 61 | AS_IF([test "$have_clang" = "yes"], [ 62 | AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ 63 | #if __clang_major__ > 3 || (__clang_major__ == 3 && __clang_minor__ >= 3) 64 | # error new clang 65 | #endif 66 | ]], [[]])],[],[ 67 | dnl clang 3.3+ unfortunately this gives warnings with hash.h 68 | CFLAGS="$CFLAGS -Wno-duplicate-decl-specifier" 69 | ]) 70 | ], [ 71 | dnl This is simply to avoid warning when building strftime() wrappers.. 72 | CFLAGS="$CFLAGS -fno-builtin-strftime" 73 | ]) 74 | 75 | AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ 76 | #if __GNUC__ < 4 77 | # error old gcc 78 | #endif 79 | ]], [[]])],[ 80 | dnl gcc4 81 | CFLAGS="$CFLAGS -Wstrict-aliasing=2" 82 | ],[]) 83 | ]) 84 | ]) 85 | 86 | AC_DEFUN([AC_LD_WHOLE_ARCHIVE], [ 87 | LD_WHOLE_ARCHIVE= 88 | LD_NO_WHOLE_ARCHIVE= 89 | AC_MSG_CHECKING([for linker option to include whole archive]) 90 | ld_help="`$CC -Wl,-help 2>&1`" 91 | case "$ld_help" in 92 | *"--whole-archive"*) 93 | LD_WHOLE_ARCHIVE="--whole-archive" 94 | LD_NO_WHOLE_ARCHIVE="--no-whole-archive" 95 | ;; 96 | esac 97 | AS_IF([test "x$LD_WHOLE_ARCHIVE" != "x"], 98 | [AC_MSG_RESULT([-Wl,$LD_WHOLE_ARCHIVE])], 99 | [AC_MSG_RESULT([not supported])] 100 | ) 101 | AC_SUBST([LD_WHOLE_ARCHIVE]) 102 | AC_SUBST([LD_NO_WHOLE_ARCHIVE]) 103 | AM_CONDITIONAL([HAVE_WHOLE_ARCHIVE], [test "x$LD_WHOLE_ARCHIVE" != "x"]) 104 | ]) 105 | 106 | dnl 107 | dnl Check for -z now and -z relro linker flags 108 | dnl 109 | dnl Copyright (C) 2013 Red Hat, Inc. 110 | dnl 111 | dnl This library is free software; you can redistribute it and/or 112 | dnl modify it under the terms of the GNU Lesser General Public 113 | dnl License as published by the Free Software Foundation; either 114 | dnl version 2.1 of the License, or (at your option) any later version. 115 | dnl 116 | dnl This library is distributed in the hope that it will be useful, 117 | dnl but WITHOUT ANY WARRANTY; without even the implied warranty of 118 | dnl MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 119 | dnl Lesser General Public License for more details. 120 | dnl 121 | dnl You should have received a copy of the GNU Lesser General Public 122 | dnl License along with this library. If not, see 123 | dnl . 124 | dnl 125 | 126 | AC_DEFUN([AC_LD_RELRO],[ 127 | RELRO_LDFLAGS= 128 | AS_IF([test "$enable_hardening" = yes], [ 129 | AC_MSG_CHECKING([for how to force completely read-only GOT table]) 130 | ld_help=`$CC -Wl,-help 2>&1` 131 | case $ld_help in 132 | *"-z relro"*) RELRO_LDFLAGS="-Wl,-z -Wl,relro" ;; 133 | esac 134 | case $ld_help in 135 | *"-z now"*) RELRO_LDFLAGS="$RELRO_LDFLAGS -Wl,-z -Wl,now" ;; 136 | esac 137 | AS_IF([test "x$RELRO_LDFLAGS" != "x"], 138 | [AC_MSG_RESULT([$RELRO_LDFLAGS])], 139 | [AC_MSG_RESULT([unknown])] 140 | ) 141 | ]) 142 | AC_SUBST([RELRO_LDFLAGS]) 143 | ]) 144 | 145 | dnl 146 | dnl Check for support for position independent executables 147 | dnl 148 | dnl Copyright (C) 2013 Red Hat, Inc. 149 | dnl 150 | dnl This library is free software; you can redistribute it and/or 151 | dnl modify it under the terms of the GNU Lesser General Public 152 | dnl License as published by the Free Software Foundation; either 153 | dnl version 2.1 of the License, or (at your option) any later version. 154 | dnl 155 | dnl This library is distributed in the hope that it will be useful, 156 | dnl but WITHOUT ANY WARRANTY; without even the implied warranty of 157 | dnl MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 158 | dnl Lesser General Public License for more details. 159 | dnl 160 | dnl You should have received a copy of the GNU Lesser General Public 161 | dnl License along with this library. If not, see 162 | dnl . 163 | dnl 164 | 165 | AC_DEFUN([AC_CC_PIE],[ 166 | AC_REQUIRE([gl_UNKNOWN_WARNINGS_ARE_ERRORS]) 167 | PIE_CFLAGS= 168 | PIE_LDFLAGS= 169 | 170 | AS_IF([test "$enable_hardening" = yes], [ 171 | OLD_CFLAGS=$CFLAGS 172 | case "$host" in 173 | *-*-mingw* | *-*-msvc* | *-*-cygwin* ) 174 | ;; dnl All code is position independent on Win32 target 175 | *) 176 | CFLAGS="-fPIE -DPIE" 177 | gl_COMPILER_OPTION_IF([-pie], [ 178 | PIE_CFLAGS="-fPIE -DPIE" 179 | PIE_LDFLAGS="-pie" 180 | ], [ 181 | dnl some versions of clang require -Wl,-pie instead of -pie 182 | gl_COMPILER_OPTION_IF([[-Wl,-pie]], [ 183 | PIE_CFLAGS="-fPIE -DPIE" 184 | PIE_LDFLAGS="-Wl,-pie" 185 | ], [AC_MSG_RESULT([not supported])], 186 | [AC_LANG_PROGRAM()] 187 | ) 188 | ], 189 | [AC_LANG_PROGRAM()] 190 | ) 191 | esac 192 | CFLAGS=$OLD_CFLAGS 193 | ]) 194 | AC_SUBST([PIE_CFLAGS]) 195 | AC_SUBST([PIE_LDFLAGS]) 196 | ]) 197 | 198 | dnl 199 | dnl Check for support for Retpoline 200 | dnl 201 | 202 | AC_DEFUN([AC_CC_RETPOLINE],[ 203 | AC_ARG_WITH(retpoline, 204 | AS_HELP_STRING([--with-retpoline=], [Retpoline mitigation choice (default: keep)]), 205 | with_retpoline=$withval, 206 | with_retpoline=keep) 207 | 208 | AC_REQUIRE([gl_UNKNOWN_WARNINGS_ARE_ERRORS]) 209 | AS_IF([test "$enable_hardening" = yes], [ 210 | case "$host" in 211 | *) 212 | gl_COMPILER_OPTION_IF([-mfunction-return=$with_retpoline], 213 | [CFLAGS="$CFLAGS -mfunction-return=$with_retpoline"], 214 | [], 215 | [AC_LANG_PROGRAM()] 216 | ) 217 | gl_COMPILER_OPTION_IF([-mindirect-branch=$with_retpoline], [ 218 | CFLAGS="$CFLAGS -mindirect-branch=$with_retpoline" 219 | ], 220 | [], 221 | [AC_LANG_PROGRAM()] 222 | ) 223 | esac 224 | ]) 225 | ]) 226 | 227 | dnl 228 | dnl Check for support for -fstack-protector or -strong 229 | dnl 230 | 231 | AC_DEFUN([AC_CC_F_STACK_PROTECTOR],[ 232 | AC_REQUIRE([gl_UNKNOWN_WARNINGS_ARE_ERRORS]) 233 | AS_IF([test "$enable_hardening" = yes], [ 234 | case "$host" in 235 | *) 236 | gl_COMPILER_OPTION_IF([-fstack-protector-strong], [ 237 | CFLAGS="$CFLAGS -fstack-protector-strong" 238 | ], 239 | [ 240 | gl_COMPILER_OPTION_IF([-fstack-protector], [ 241 | CFLAGS="$CFLAGS -fstack-protector" 242 | ], [], [AC_LANG_PROGRAM()]) 243 | ], 244 | [AC_LANG_PROGRAM()] 245 | ) 246 | esac 247 | ]) 248 | ]) 249 | 250 | AC_DEFUN([DC_DOVECOT_MODULEDIR],[ 251 | AC_ARG_WITH(moduledir, 252 | [ --with-moduledir=DIR Base directory for dynamically loadable modules], 253 | [moduledir="$withval"], 254 | [moduledir="\$(libdir)/dovecot"] 255 | ) 256 | AC_SUBST(moduledir) 257 | ]) 258 | 259 | AC_DEFUN([DC_PLUGIN_DEPS],[ 260 | _plugin_deps=yes 261 | AC_MSG_CHECKING([whether OS supports plugin dependencies]) 262 | case "$host_os" in 263 | darwin*) 264 | dnl OSX loads the plugins twice, which breaks stuff 265 | _plugin_deps=no 266 | ;; 267 | esac 268 | AC_MSG_RESULT([$_plugin_deps]) 269 | AM_CONDITIONAL([DOVECOT_PLUGIN_DEPS], [test "x$_plugin_deps" = "xyes"]) 270 | unset _plugin_deps 271 | ]) 272 | 273 | AC_DEFUN([DC_DOVECOT_TEST_WRAPPER],[ 274 | AC_REQUIRE_AUX_FILE([run-test.sh.in]) 275 | AC_ARG_VAR([VALGRIND], [Path to valgrind]) 276 | AC_PATH_PROG(VALGRIND, valgrind, reject) 277 | AS_IF([test "$VALGRIND" != reject], [ 278 | RUN_TEST='$(LIBTOOL) execute $(SHELL) $(top_builddir)/build-aux/run-test.sh' 279 | ], [ 280 | RUN_TEST='' 281 | ]) 282 | AC_SUBST(RUN_TEST) 283 | ]) 284 | 285 | dnl Substitute every var in the given comma separated list 286 | AC_DEFUN([AX_SUBST_L],[ 287 | m4_foreach([__var__], [$@], [AC_SUBST(__var__)]) 288 | ]) 289 | 290 | AC_DEFUN([DC_DOVECOT_HARDENING],[ 291 | AC_ARG_ENABLE(hardening, 292 | AS_HELP_STRING([--enable-hardening=yes], [Enable various hardenings (default: yes)]), 293 | enable_hardening=$enableval, 294 | enable_hardening=yes) 295 | 296 | AC_MSG_CHECKING([Whether to enable hardening]) 297 | AC_MSG_RESULT([$enable_hardening]) 298 | 299 | AC_CC_PIE 300 | AC_CC_F_STACK_PROTECTOR 301 | AC_CC_D_FORTIFY_SOURCE 302 | AC_CC_RETPOLINE 303 | AC_LD_RELRO 304 | DOVECOT_WANT_UBSAN 305 | ]) 306 | 307 | AC_DEFUN([DC_DOVECOT_FUZZER],[ 308 | AC_ARG_WITH(fuzzer, 309 | AS_HELP_STRING([--with-fuzzer=clang], [Build with clang fuzzer (default: no)]), 310 | with_fuzzer=$withval, 311 | with_fuzzer=no) 312 | AS_IF([test x$with_fuzzer = xclang], [ 313 | CFLAGS="$CFLAGS -fsanitize=fuzzer-no-link" 314 | # use $LIB_FUZZING_ENGINE for linking if it exists 315 | FUZZER_LDFLAGS=${LIB_FUZZING_ENGINE--fsanitize=fuzzer} 316 | # May need to use CXXLINK for linking, which wants sources to 317 | # be compiled with -fPIE 318 | FUZZER_CPPFLAGS='$(AM_CPPFLAGS) -fPIE -DPIE' 319 | ], [test x$with_fuzzer != xno], [ 320 | AC_MSG_ERROR([Unknown fuzzer $with_fuzzer]) 321 | ]) 322 | AC_SUBST([FUZZER_CPPFLAGS]) 323 | AC_SUBST([FUZZER_LDFLAGS]) 324 | AM_CONDITIONAL([USE_FUZZER], [test "x$with_fuzzer" != "xno"]) 325 | 326 | ]) 327 | 328 | AC_DEFUN([DC_DOVECOT],[ 329 | AC_ARG_WITH(dovecot, 330 | [ --with-dovecot=DIR Dovecot base directory], 331 | [ dovecotdir="$withval" ], [ 332 | dc_prefix=$prefix 333 | test "x$dc_prefix" = xNONE && dc_prefix=$ac_default_prefix 334 | dovecotdir="$dc_prefix/lib/dovecot" 335 | ] 336 | ) 337 | 338 | AC_ARG_WITH(dovecot-install-dirs, 339 | [AS_HELP_STRING([--with-dovecot-install-dirs], 340 | [Use install directories configured for Dovecot (default)])], 341 | AS_IF([test x$withval = xno], [ 342 | use_install_dirs=no 343 | ], [ 344 | use_install_dirs=yes 345 | ]), 346 | use_install_dirs=yes) 347 | 348 | AC_MSG_CHECKING([for "$dovecotdir/dovecot-config"]) 349 | AS_IF([test -f "$dovecotdir/dovecot-config"], [ 350 | AC_MSG_RESULT([$dovecotdir/dovecot-config]) 351 | ], [ 352 | AC_MSG_RESULT([not found]) 353 | AC_MSG_NOTICE([]) 354 | AC_MSG_NOTICE([Use --with-dovecot=DIR to provide the path to the dovecot-config file.]) 355 | AC_MSG_ERROR([dovecot-config not found]) 356 | ]) 357 | 358 | old=`pwd` 359 | cd $dovecotdir 360 | abs_dovecotdir=`pwd` 361 | cd $old 362 | DISTCHECK_CONFIGURE_FLAGS="--with-dovecot=$abs_dovecotdir --without-dovecot-install-dirs" 363 | 364 | dnl Make sure dovecot-config doesn't accidentically override flags 365 | ORIG_CFLAGS="$CFLAGS" 366 | ORIG_LDFLAGS="$LDFLAGS" 367 | ORIG_BINARY_CFLAGS="$BINARY_CFLAGS" 368 | ORIG_BINARY_LDFLAGS="$BINARY_LDFLAGS" 369 | 370 | eval `$GREP -i '^dovecot_[[a-z_]]*=' "$dovecotdir"/dovecot-config` 371 | eval `$GREP '^LIBDOVECOT[[A-Z0-9_]]*=' "$dovecotdir"/dovecot-config` 372 | 373 | CFLAGS="$ORIG_CFLAGS" 374 | LDFLAGS="$ORIG_LDFLAGS" 375 | BINARY_CFLAGS="$ORIG_BINARY_CFLAGS" 376 | BINARY_LDFLAGS="$ORIG_BINARY_LDFLAGS" 377 | 378 | dovecot_installed_moduledir="$dovecot_moduledir" 379 | 380 | AS_IF([test "$use_install_dirs" = "no"], [ 381 | dnl the main purpose of these is to fix make distcheck for plugins 382 | dnl other than that, they don't really make much sense 383 | dovecot_pkgincludedir='$(pkgincludedir)' 384 | dovecot_pkglibdir='$(pkglibdir)' 385 | dovecot_pkglibexecdir='$(libexecdir)/dovecot' 386 | dovecot_docdir='$(docdir)' 387 | dovecot_moduledir='$(moduledir)' 388 | dovecot_statedir='$(statedir)' 389 | ]) 390 | 391 | CC_CLANG 392 | CC_STRICT_BOOL 393 | DC_DOVECOT_CFLAGS 394 | DC_DOVECOT_HARDENING 395 | 396 | AX_SUBST_L([DISTCHECK_CONFIGURE_FLAGS], [dovecotdir], [dovecot_moduledir], [dovecot_installed_moduledir], [dovecot_pkgincludedir], [dovecot_pkglibexecdir], [dovecot_pkglibdir], [dovecot_docdir], [dovecot_statedir]) 397 | AX_SUBST_L([DOVECOT_INSTALLED], [DOVECOT_CFLAGS], [DOVECOT_LIBS], [DOVECOT_SSL_LIBS], [DOVECOT_SQL_LIBS], [DOVECOT_LDAP_LIBS], [DOVECOT_COMPRESS_LIBS], [DOVECOT_BINARY_CFLAGS], [DOVECOT_BINARY_LDFLAGS]) 398 | AX_SUBST_L([LIBDOVECOT], [LIBDOVECOT_LOGIN], [LIBDOVECOT_SQL], [LIBDOVECOT_LDAP], [LIBDOVECOT_OPENSSL], [LIBDOVECOT_COMPRESS], [LIBDOVECOT_LDA], [LIBDOVECOT_STORAGE], [LIBDOVECOT_DSYNC], [LIBDOVECOT_LIBLANG]) 399 | AX_SUBST_L([LIBDOVECOT_DEPS], [LIBDOVECOT_LOGIN_DEPS], [LIBDOVECOT_SQL_DEPS], [LIBDOVECOT_LDAP_DEPS], [LIBDOVECOT_OPENSSL_DEPS], [LIBDOVECOT_COMPRESS_DEPS], [LIBDOVECOT_LDA_DEPS], [LIBDOVECOT_STORAGE_DEPS], [LIBDOVECOT_DSYNC_DEPS], [LIBDOVECOT_LIBLANG_DEPS]) 400 | AX_SUBST_L([LIBDOVECOT_INCLUDE], [LIBDOVECOT_LDA_INCLUDE], [LIBDOVECOT_AUTH_INCLUDE], [LIBDOVECOT_DOVEADM_INCLUDE], [LIBDOVECOT_SERVICE_INCLUDE], [LIBDOVECOT_STORAGE_INCLUDE], [LIBDOVECOT_LOGIN_INCLUDE], [LIBDOVECOT_SQL_INCLUDE], [LIBDOVECOT_LDAP_INCLUDE]) 401 | AX_SUBST_L([LIBDOVECOT_IMAP_LOGIN_INCLUDE], [LIBDOVECOT_CONFIG_INCLUDE], [LIBDOVECOT_IMAP_INCLUDE], [LIBDOVECOT_POP3_INCLUDE], [LIBDOVECOT_SUBMISSION_INCLUDE], [LIBDOVECOT_LMTP_INCLUDE], [LIBDOVECOT_DSYNC_INCLUDE], [LIBDOVECOT_IMAPC_INCLUDE], [LIBDOVECOT_LANG_INCLUDE]) 402 | AX_SUBST_L([LIBDOVECOT_NOTIFY_INCLUDE], [LIBDOVECOT_PUSH_NOTIFICATION_INCLUDE], [LIBDOVECOT_ACL_INCLUDE], [LIBDOVECOT_LIBLANG_INCLUDE], [LIBDOVECOT_LUA_INCLUDE]) 403 | AX_SUBST_L([DOVECOT_LUA_LIBS], [DOVECOT_LUA_CFLAGS], [LIBDOVECOT_LUA], [LIBDOVECOT_LUA_DEPS]) 404 | 405 | AS_IF([test x$DOVECOT_HAVE_MAIL_UTF8 = xyes], [ 406 | AC_DEFINE([DOVECOT_HAVE_MAIL_UTF8],,"Define if Dovecot has mail UTF-8 support") 407 | ]) 408 | AM_CONDITIONAL(DOVECOT_INSTALLED, test "$DOVECOT_INSTALLED" = "yes") 409 | 410 | DC_PLUGIN_DEPS 411 | DC_DOVECOT_TEST_WRAPPER 412 | ]) 413 | 414 | AC_DEFUN([DC_CC_WRAPPER],[ 415 | AS_IF([test "$want_shared_libs" != "yes"], [ 416 | dnl want_shared_libs=no is for internal use. the liblib.la check is for plugins 417 | AS_IF([test "$want_shared_libs" = "no" || echo "$LIBDOVECOT" | $GREP "/liblib.la" > /dev/null], [ 418 | AS_IF([test "$with_gnu_ld" = yes], [ 419 | dnl libtool can't handle using whole-archive flags, so we need to do this 420 | dnl with a CC wrapper.. shouldn't be much of a problem, since most people 421 | dnl are building with shared libs. 422 | cat > cc-wrapper.sh <<_DC_EOF 423 | #!/bin/sh 424 | 425 | if echo "\$[*]" | $GREP -- -export-dynamic > /dev/null; then 426 | # the binary uses plugins. make sure we include everything from .a libs 427 | exec $CC -Wl,--whole-archive \$[*] -Wl,--no-whole-archive 428 | else 429 | exec $CC \$[*] 430 | fi 431 | _DC_EOF 432 | chmod +x cc-wrapper.sh 433 | CC=`pwd`/cc-wrapper.sh 434 | ]) 435 | ]) 436 | ]) 437 | ]) 438 | 439 | # warnings.m4 serial 11 440 | dnl Copyright (C) 2008-2015 Free Software Foundation, Inc. 441 | dnl This file is free software; the Free Software Foundation 442 | dnl gives unlimited permission to copy and/or distribute it, 443 | dnl with or without modifications, as long as this notice is preserved. 444 | 445 | dnl From Simon Josefsson 446 | 447 | # gl_AS_VAR_APPEND(VAR, VALUE) 448 | # ---------------------------- 449 | # Provide the functionality of AS_VAR_APPEND if Autoconf does not have it. 450 | m4_ifdef([AS_VAR_APPEND], 451 | [m4_copy([AS_VAR_APPEND], [gl_AS_VAR_APPEND])], 452 | [m4_define([gl_AS_VAR_APPEND], 453 | [AS_VAR_SET([$1], [AS_VAR_GET([$1])$2])])]) 454 | 455 | 456 | # gl_COMPILER_OPTION_IF(OPTION, [IF-SUPPORTED], [IF-NOT-SUPPORTED], 457 | # [PROGRAM = AC_LANG_PROGRAM()]) 458 | # ----------------------------------------------------------------- 459 | # Check if the compiler supports OPTION when compiling PROGRAM. 460 | # 461 | # FIXME: gl_Warn must be used unquoted until we can assume Autoconf 462 | # 2.64 or newer. 463 | AC_DEFUN([gl_COMPILER_OPTION_IF], 464 | [AS_VAR_PUSHDEF([gl_Warn], [gl_cv_warn_[]_AC_LANG_ABBREV[]_$1])dnl 465 | AS_VAR_PUSHDEF([gl_Flags], [_AC_LANG_PREFIX[]FLAGS])dnl 466 | AS_LITERAL_IF([$1], 467 | [m4_pushdef([gl_Positive], m4_bpatsubst([$1], [^-Wno-], [-W]))], 468 | [gl_positive="$1" 469 | case $gl_positive in 470 | -Wno-*) gl_positive=-W`expr "X$gl_positive" : 'X-Wno-\(.*\)'` ;; 471 | esac 472 | m4_pushdef([gl_Positive], [$gl_positive])])dnl 473 | AC_CACHE_CHECK([whether _AC_LANG compiler handles $1], m4_defn([gl_Warn]), [ 474 | gl_save_compiler_FLAGS="$gl_Flags" 475 | gl_AS_VAR_APPEND(m4_defn([gl_Flags]), 476 | [" $gl_unknown_warnings_are_errors ]m4_defn([gl_Positive])["]) 477 | AC_LINK_IFELSE([m4_default([$4], [AC_LANG_PROGRAM([])])], 478 | [AS_VAR_SET(gl_Warn, [yes])], 479 | [AS_VAR_SET(gl_Warn, [no])]) 480 | gl_Flags="$gl_save_compiler_FLAGS" 481 | ]) 482 | AS_VAR_IF(gl_Warn, [yes], [$2], [$3]) 483 | m4_popdef([gl_Positive])dnl 484 | AS_VAR_POPDEF([gl_Flags])dnl 485 | AS_VAR_POPDEF([gl_Warn])dnl 486 | ]) 487 | 488 | # gl_UNKNOWN_WARNINGS_ARE_ERRORS 489 | # ------------------------------ 490 | # Clang doesn't complain about unknown warning options unless one also 491 | # specifies -Wunknown-warning-option -Werror. Detect this. 492 | AC_DEFUN([gl_UNKNOWN_WARNINGS_ARE_ERRORS], 493 | [gl_COMPILER_OPTION_IF([-Werror -Wunknown-warning-option], 494 | [gl_unknown_warnings_are_errors='-Wunknown-warning-option -Werror'], 495 | [gl_unknown_warnings_are_errors=])]) 496 | 497 | # gl_WARN_ADD(OPTION, [VARIABLE = WARN_CFLAGS], 498 | # [PROGRAM = AC_LANG_PROGRAM()]) 499 | # --------------------------------------------- 500 | # Adds parameter to WARN_CFLAGS if the compiler supports it when 501 | # compiling PROGRAM. For example, gl_WARN_ADD([-Wparentheses]). 502 | # 503 | # If VARIABLE is a variable name, AC_SUBST it. 504 | AC_DEFUN([gl_WARN_ADD], 505 | [AC_REQUIRE([gl_UNKNOWN_WARNINGS_ARE_ERRORS]) 506 | gl_COMPILER_OPTION_IF([$1], 507 | [gl_AS_VAR_APPEND(m4_if([$2], [], [[WARN_CFLAGS]], [[$2]]), [" $1"])], 508 | [], 509 | [$3]) 510 | m4_ifval([$2], 511 | [AS_LITERAL_IF([$2], [AC_SUBST([$2])])], 512 | [AC_SUBST([WARN_CFLAGS])])dnl 513 | ]) 514 | 515 | # Local Variables: 516 | # mode: autoconf 517 | # End: 518 | dnl * clang check 519 | AC_DEFUN([CC_CLANG],[ 520 | AC_MSG_CHECKING([whether $CC is clang 3.3+]) 521 | AS_IF([$CC -dM -E -x c /dev/null | $GREP __clang__ > /dev/null 2>&1], [ 522 | AS_VAR_SET([have_clang], [yes]) 523 | ], [ 524 | AS_VAR_SET([have_clang], [no]) 525 | ]) 526 | AC_MSG_RESULT([$have_clang]) 527 | ]) 528 | 529 | AC_DEFUN([CC_STRICT_BOOL], [ 530 | AS_IF([test $have_clang = yes], [ 531 | AC_REQUIRE([gl_UNKNOWN_WARNINGS_ARE_ERRORS]) 532 | gl_COMPILER_OPTION_IF([-Wstrict-bool], [ 533 | AC_DEFINE(HAVE_STRICT_BOOL,, [we have strict bool]) 534 | ]) 535 | ]) 536 | ]) 537 | 538 | AC_DEFUN([DOVECOT_WANT_UBSAN], [ 539 | AC_ARG_ENABLE(ubsan, 540 | AS_HELP_STRING([--enable-ubsan], [Enable undefined behaviour sanitizes (default=no)]), 541 | [want_ubsan=yes], [want_ubsan=no]) 542 | AC_MSG_CHECKING([whether we want undefined behaviour sanitizer]) 543 | AC_MSG_RESULT([$want_ubsan]) 544 | AS_IF([test x$want_ubsan = xyes], [ 545 | san_flags="" 546 | gl_COMPILER_OPTION_IF([-fsanitize=undefined], [ 547 | san_flags="$san_flags -fsanitize=undefined -fno-sanitize=function,vptr" 548 | AC_DEFINE([HAVE_FSANITIZE_UNDEFINED], [1], [Define if your compiler has -fsanitize=undefined]) 549 | ]) 550 | gl_COMPILER_OPTION_IF([-fno-sanitize=nonnull-attribute], [ 551 | san_flags="$san_flags -fno-sanitize=nonnull-attribute" 552 | AC_DEFINE([HAVE_FNO_SANITIZE_NONNULL_ATTRIBUTE], [1], [Define if your compiler has -fno-sanitize=nonnull-attribute]) 553 | ]) 554 | gl_COMPILER_OPTION_IF([-fsanitize=implicit-integer-truncation], [ 555 | san_flags="$san_flags -fsanitize=implicit-integer-truncation" 556 | AC_DEFINE([HAVE_FSANITIZE_IMPLICIT_INTEGER_TRUNCATION], [1], [Define if your compiler has -fsanitize=implicit-integer-truncation]) 557 | ]) 558 | gl_COMPILER_OPTION_IF([-fsanitize=local-bounds], [ 559 | san_flags="$san_flags -fsanitize=local-bounds" 560 | AC_DEFINE([HAVE_FSANITIZE_LOCAL_BOUNDS], [1], [Define if your compiler has -fsanitize=local-bounds]) 561 | ]) 562 | gl_COMPILER_OPTION_IF([-fsanitize=integer], [ 563 | san_flags="$san_flags -fsanitize=integer" 564 | AC_DEFINE([HAVE_FSANITIZE_INTEGER], [1], [Define if your compiler has -fsanitize=integer]) 565 | ]) 566 | gl_COMPILER_OPTION_IF([-fsanitize=nullability], [ 567 | san_flags="$san_flags -fsanitize=nullability" 568 | AC_DEFINE([HAVE_FSANITIZE_NULLABILITY], [1], [Define if your compiler has -fsanitize=nullability]) 569 | ]) 570 | AS_IF([test "$san_flags" != "" ], [ 571 | EXTRA_CFLAGS="$EXTRA_CFLAGS $san_flags -U_FORTIFY_SOURCE -g -ggdb3 -O0 -fno-omit-frame-pointer" 572 | AC_DEFINE([HAVE_UNDEFINED_SANITIZER], [1], [Define if your compiler supports undefined sanitizers]) 573 | ], [ 574 | AC_MSG_ERROR([No undefined sanitizer support in your compiler]) 575 | ]) 576 | san_flags="" 577 | ]) 578 | ]) 579 | -------------------------------------------------------------------------------- /src/fts-backend-xapian.cpp: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2019 Joan Moreau , see the included COPYING file */ 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | extern "C" { 12 | #include "fts-xapian-plugin.h" 13 | } 14 | #include "fts-backend-xapian.h" 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #if defined(__FreeBSD__) || defined(__NetBSD__) 22 | #include 23 | #endif 24 | 25 | #include 26 | #include 27 | #include 28 | 29 | #include 30 | 31 | class XDoc; 32 | class XDocsWriter; 33 | 34 | struct xapian_fts_backend 35 | { 36 | struct fts_backend backend; 37 | char * path; 38 | 39 | char * guid; 40 | char * boxname; 41 | 42 | char * xap_db; 43 | char * exp_db; 44 | char * version_file; 45 | char * dict_db; 46 | long dict_nb; 47 | 48 | sqlite3 * ddb; 49 | Xapian::WritableDatabase * dbw; 50 | long pending; 51 | 52 | char * old_guid; 53 | char * old_boxname; 54 | 55 | std::vector docs; 56 | std::vector threads; 57 | std::timed_mutex mutex; 58 | std::unique_lock * mutex_t; 59 | unsigned int max_threads; 60 | 61 | #ifdef FTS_DOVECOT24 62 | struct event *event; 63 | #endif 64 | 65 | long lastuid; 66 | long total_docs; 67 | long start_time; 68 | }; 69 | 70 | struct xapian_fts_backend_update_context 71 | { 72 | struct fts_backend_update_context ctx; 73 | char * tbi_field=NULL; 74 | bool isattachment=false; 75 | bool tbi_isfield; 76 | uint32_t tbi_uid=0; 77 | }; 78 | 79 | static struct fts_xapian_settings fts_xapian_settings; 80 | 81 | struct event_category event_category_fts_xapian = { 82 | .parent = &event_category_fts, 83 | .name = XAPIAN_LABEL 84 | }; 85 | 86 | #include "fts-backend-xapian-functions.cpp" 87 | 88 | 89 | static struct fts_backend *fts_backend_xapian_alloc(void) 90 | { 91 | struct xapian_fts_backend *backend; 92 | 93 | backend = i_new(struct xapian_fts_backend, 1); 94 | backend->backend = fts_backend_xapian; 95 | return &backend->backend; 96 | } 97 | 98 | static int fts_backend_xapian_init(struct fts_backend *_backend, const char **error_r) 99 | { 100 | (void)error_r; 101 | 102 | struct xapian_fts_backend *backend = (struct xapian_fts_backend *)_backend; 103 | 104 | backend->xap_db = NULL; 105 | backend->exp_db = NULL; 106 | backend->dict_db = NULL; 107 | 108 | backend->docs.clear(); 109 | backend->threads.clear(); 110 | backend->total_docs =0; 111 | 112 | backend->lastuid = -1; 113 | 114 | backend->dbw = NULL; 115 | backend->ddb = NULL; 116 | backend->guid = NULL; 117 | backend->path = NULL; 118 | backend->old_guid = NULL; 119 | backend->old_boxname = NULL; 120 | 121 | struct fts_xapian_user *fuser = FTS_XAPIAN_USER_CONTEXT(_backend->ns->user); 122 | 123 | #ifdef FTS_DOVECOT24 124 | backend->event = event_create(_backend->event); 125 | event_add_category(backend->event, &event_category_fts_xapian); 126 | 127 | if (fts_xapian_mail_user_get(_backend->ns->user, backend->event, &fuser, error_r) < 0) 128 | { 129 | event_unref(&backend->event); 130 | return -1; 131 | } 132 | fts_xapian_settings.verbose = fuser->set->verbose; 133 | fts_xapian_settings.maxthreads = fuser->set->maxthreads; 134 | fts_xapian_settings.partial = fuser->set->partial; 135 | fts_xapian_settings.lowmemory = fuser->set->lowmemory; 136 | #else 137 | fts_xapian_settings = fuser->set; 138 | #endif 139 | 140 | if(fts_xapian_settings.maxthreads>0) 141 | { 142 | backend->max_threads=fts_xapian_settings.maxthreads; 143 | } 144 | else 145 | { 146 | backend->max_threads = std::thread::hardware_concurrency()-1; 147 | } 148 | if(backend->max_threads<2) backend->max_threads = 2; 149 | 150 | if(fts_backend_xapian_set_path(backend)<0) return -1; 151 | 152 | openlog("xapian-docswriter",0,LOG_MAIL); 153 | 154 | if(fts_xapian_settings.verbose>0) i_info("FTS Xapian: Starting version %s with partial=%d verbose=%d max_threads=%u lowmemory=%d MB", XAPIAN_PLUGIN_VERSION, fts_xapian_settings.partial,fts_xapian_settings.verbose,backend->max_threads,fts_xapian_settings.lowmemory); 155 | 156 | return 0; 157 | } 158 | 159 | static void fts_backend_xapian_deinit(struct fts_backend *_backend) 160 | { 161 | struct xapian_fts_backend *backend = (struct xapian_fts_backend *)_backend; 162 | 163 | if(fts_xapian_settings.verbose>1) i_info("FTS Xapian: Deinit %s)",backend->path); 164 | 165 | if(backend->guid != NULL) fts_backend_xapian_unset_box(backend); 166 | 167 | if(backend->old_guid != NULL) i_free(backend->old_guid); 168 | backend->old_guid = NULL; 169 | 170 | if(backend->old_boxname != NULL) i_free(backend->old_boxname); 171 | backend->old_boxname = NULL; 172 | 173 | if(backend->path != NULL) i_free(backend->path); 174 | backend->path = NULL; 175 | 176 | #ifdef FTS_DOVECOT24 177 | event_unref(&backend->event); 178 | #endif 179 | 180 | i_free(backend); 181 | 182 | closelog(); 183 | } 184 | 185 | 186 | static int fts_backend_xapian_get_last_uid(struct fts_backend *_backend, struct mailbox *box, uint32_t *last_uid_r) 187 | { 188 | if(fts_xapian_settings.verbose>1) i_info("FTS Xapian: fts_backend_xapian_get_last_uid"); 189 | 190 | struct xapian_fts_backend *backend = (struct xapian_fts_backend *)_backend; 191 | 192 | *last_uid_r = 0; 193 | 194 | if(fts_backend_xapian_set_box(backend, box) < 0) 195 | { 196 | i_error("FTS Xapian: get_last_uid: Can not select mailbox '%s'",box->name); 197 | return -1; 198 | } 199 | 200 | Xapian::Database * dbr; 201 | if(!fts_backend_xapian_open_readonly(backend, &dbr)) 202 | { 203 | i_error("FTS Xapian: GetLastUID: Can not open db RO (%s)",backend->xap_db); 204 | return 0; 205 | } 206 | 207 | try 208 | { 209 | *last_uid_r = Xapian::sortable_unserialise(dbr->get_value_upper_bound(1)); 210 | } 211 | catch(Xapian::Error e) 212 | { 213 | i_warning("FTS Xapian: fts_backend_xapian_get_last_uid for '%s' (%s)",backend->boxname,backend->guid); 214 | i_warning("FTS Xapian: %s",e.get_msg().c_str()); 215 | } 216 | 217 | dbr->close(); 218 | delete(dbr); 219 | if(fts_xapian_settings.verbose>1) i_info("FTS Xapian: Get last UID of %s (%s) = %d",backend->boxname,backend->guid,*last_uid_r); 220 | 221 | return 0; 222 | } 223 | 224 | 225 | static struct fts_backend_update_context * fts_backend_xapian_update_init(struct fts_backend *_backend) 226 | { 227 | if(fts_xapian_settings.verbose>1) i_info("FTS Xapian: fts_backend_update_context"); 228 | 229 | struct xapian_fts_backend_update_context *ctx; 230 | 231 | ctx = i_new(struct xapian_fts_backend_update_context, 1); 232 | ctx->ctx.backend = _backend; 233 | return &ctx->ctx; 234 | } 235 | 236 | static int fts_backend_xapian_update_deinit(struct fts_backend_update_context *_ctx) 237 | { 238 | struct xapian_fts_backend_update_context *ctx = (struct xapian_fts_backend_update_context *)_ctx; 239 | struct xapian_fts_backend *backend = (struct xapian_fts_backend *)ctx->ctx.backend; 240 | 241 | if(fts_xapian_settings.verbose>1) i_info("FTS Xapian: fts_backend_xapian_update_deinit (%s)",backend->path); 242 | 243 | i_free(ctx); 244 | 245 | return 0; 246 | } 247 | 248 | static void fts_backend_xapian_update_set_mailbox(struct fts_backend_update_context *_ctx, struct mailbox *box) 249 | { 250 | if(fts_xapian_settings.verbose>1) i_info("FTS Xapian: fts_backend_xapian_update_set_mailbox"); 251 | 252 | struct xapian_fts_backend_update_context *ctx = (struct xapian_fts_backend_update_context *)_ctx; 253 | struct xapian_fts_backend *backend = (struct xapian_fts_backend *)ctx->ctx.backend; 254 | 255 | fts_backend_xapian_set_box(backend, box); 256 | } 257 | 258 | static void fts_backend_xapian_update_expunge(struct fts_backend_update_context *_ctx, uint32_t uid) 259 | { 260 | if(fts_xapian_settings.verbose>1) i_info("FTS Xapian: fts_backend_xapian_update_expunge"); 261 | 262 | struct xapian_fts_backend_update_context *ctx = (struct xapian_fts_backend_update_context *)_ctx; 263 | struct xapian_fts_backend *backend = (struct xapian_fts_backend *)ctx->ctx.backend; 264 | 265 | sqlite3 * expdb = NULL; 266 | if(fts_xapian_settings.verbose>0) i_info("FTS Xapian: Opening expunge DB(%s)",backend->exp_db); 267 | 268 | if(sqlite3_open_v2(backend->exp_db,&expdb,SQLITE_OPEN_FULLMUTEX | SQLITE_OPEN_READWRITE,NULL)) 269 | { 270 | i_error("FTS Xapian: Expunging UID=%d : Can not open %s",uid,backend->exp_db); 271 | return; 272 | } 273 | char *zErrMsg = 0; 274 | char * u = i_strdup_printf(replaceExpUID,uid); 275 | if(sqlite3_exec(expdb,u,NULL,0,&zErrMsg) != SQLITE_OK) 276 | { 277 | i_error("FTS Xapian: Expunging (3) UID=%d : Can not add UID : %s",uid,zErrMsg); 278 | if(zErrMsg!=NULL) sqlite3_free(zErrMsg); 279 | } 280 | i_free(u); 281 | sqlite3_close(expdb); 282 | if(fts_xapian_settings.verbose>0) i_info("FTS Xapian : Expunge done"); 283 | } 284 | 285 | static bool fts_backend_xapian_update_set_build_key(struct fts_backend_update_context *_ctx, const struct fts_backend_build_key *key) 286 | { 287 | if(fts_xapian_settings.verbose>1) i_info("FTS Xapian: fts_backend_xapian_update_set_build_key"); 288 | 289 | struct xapian_fts_backend_update_context *ctx = (struct xapian_fts_backend_update_context *)_ctx; 290 | 291 | struct xapian_fts_backend *backend = (struct xapian_fts_backend *)ctx->ctx.backend; 292 | 293 | ctx->tbi_isfield=false; 294 | ctx->tbi_uid=0; 295 | ctx->tbi_field=NULL; 296 | 297 | if(backend->guid == NULL) 298 | { 299 | if(fts_xapian_settings.verbose>0) i_warning("FTS Xapian: Build key %s with no mailbox",key->hdr_name); 300 | return FALSE; 301 | } 302 | 303 | if((backend->old_guid == NULL) || (strcmp(backend->old_guid,backend->guid)!=0)) 304 | { 305 | fts_backend_xapian_oldbox(backend); 306 | backend->old_guid = i_strdup(backend->guid); 307 | backend->old_boxname = i_strdup(backend->boxname); 308 | } 309 | 310 | const char * type = key->body_content_type; 311 | const char * disposition = key->body_content_disposition; 312 | 313 | if(fts_xapian_settings.verbose>1) i_info("FTS Xapian: New part (Header=%s,Type=%s,Disposition=%s)",key->hdr_name,type,disposition); 314 | 315 | // Verify content-type 316 | if(key->type == FTS_BACKEND_BUILD_KEY_BODY_PART_BINARY) 317 | { 318 | if(fts_xapian_settings.verbose>1) i_info("FTS Xapian: Skipping binary part of type '%s'",type); 319 | return FALSE; 320 | } 321 | 322 | if((type != NULL) && (strncmp(type,"text",4)!=0) && ((disposition==NULL) || ((strstr(disposition,"filename=")==NULL) && (strstr(disposition,"attachment")==NULL)))) 323 | { 324 | if(fts_xapian_settings.verbose>1) i_info("FTS Xapian: Non-binary & non-text part of type '%s'",type); 325 | return FALSE; 326 | } 327 | 328 | if(!fts_backend_xapian_sqlite3_dict_open(backend)) return FALSE; 329 | 330 | // Verify content-disposition 331 | ctx->isattachment=false; 332 | if((disposition != NULL) && ((strstr(disposition,"filename=")!=NULL) || (strstr(disposition,"attachment")!=NULL))) 333 | { 334 | if(fts_xapian_settings.verbose>1) i_info("FTS Xapian: Found part as attachment of type '%s' and disposition '%s'",type,disposition); 335 | ctx->isattachment=true; 336 | } 337 | 338 | long field = HDR_BODY; 339 | if(key->hdr_name!=NULL) 340 | { 341 | field = fts_backend_xapian_clean_header(key->hdr_name); 342 | if(field<0) 343 | { 344 | if(fts_xapian_settings.verbose>1) i_info("FTS Xapian: Unknown header '%s' of part",key->hdr_name); 345 | return FALSE; 346 | } 347 | if(field<1) field=HDR_BODY; 348 | } 349 | 350 | switch (key->type) 351 | { 352 | case FTS_BACKEND_BUILD_KEY_HDR: 353 | case FTS_BACKEND_BUILD_KEY_MIME_HDR: 354 | ctx->tbi_isfield=true; 355 | ctx->tbi_uid=key->uid; 356 | break; 357 | case FTS_BACKEND_BUILD_KEY_BODY_PART: 358 | ctx->tbi_isfield=false; 359 | ctx->tbi_uid=key->uid; 360 | break; 361 | case FTS_BACKEND_BUILD_KEY_BODY_PART_BINARY: 362 | default: 363 | return FALSE; 364 | } 365 | 366 | long n = backend->threads.size(); 367 | while(n>0) 368 | { 369 | n--; 370 | if(backend->threads.at(n)->err) return FALSE; 371 | if(!(backend->threads.at(n)->started)) 372 | { 373 | backend->threads[n]->launch("Relaunch post error"); 374 | } 375 | } 376 | 377 | ctx->tbi_field = i_strdup_printf("%ld",field); 378 | 379 | if((ctx->tbi_uid>0) && (ctx->tbi_uid != backend->lastuid)) 380 | { 381 | std::string s("FTS Xapian: New doc incoming (#"); 382 | s.append(std::to_string(ctx->tbi_uid)+")"); 383 | 384 | if(fts_xapian_settings.verbose>0) i_info("%s",s.c_str()); 385 | 386 | if(backend->threads.size() < backend->max_threads ) 387 | { 388 | XDocsWriter * x = new XDocsWriter(backend,backend->threads.size()+1); 389 | x->launch(s.c_str()); 390 | backend->threads.push_back(x); 391 | } 392 | 393 | fts_backend_xapian_get_lock(backend, fts_xapian_settings.verbose, s.c_str()); 394 | { 395 | if(backend->lastuid>0) 396 | { 397 | if(fts_xapian_settings.verbose>0) i_info("FTS Xapian: Previous doc ready to index (#%ld)",backend->lastuid); 398 | backend->docs.front()->status=1; 399 | } 400 | backend->lastuid = ctx->tbi_uid; 401 | backend->docs.insert(backend->docs.begin(),new XDoc(backend)); 402 | 403 | if(fts_xapian_settings.verbose>0) i_info("FTS Xapian: Start indexing #%ld (%s) : Queue size = %ld",backend->lastuid, backend->boxname,backend->docs.size()); 404 | } 405 | fts_backend_xapian_release_lock(backend, fts_xapian_settings.verbose, s.c_str()); 406 | 407 | if(backend->docs.size() > (XAPIAN_WRITING_CACHE * 2) ) 408 | { 409 | n=0; 410 | while (backend->docs.size() > XAPIAN_WRITING_CACHE) 411 | { 412 | for(auto & xwr : backend->threads) 413 | { 414 | if(xwr->err) return FALSE; 415 | } 416 | n++; 417 | if(n>50) 418 | { 419 | if (fts_xapian_settings.verbose>0) i_info("FTS Xapian: Waiting for queue to be absorbed (pending=%ld)",backend->docs.size()); 420 | n=0; 421 | } 422 | std::this_thread::sleep_for(XAPIAN_SLEEP); 423 | } 424 | } 425 | } 426 | 427 | return TRUE; 428 | } 429 | 430 | static void fts_backend_xapian_update_unset_build_key(struct fts_backend_update_context *_ctx) 431 | { 432 | struct xapian_fts_backend_update_context *ctx = (struct xapian_fts_backend_update_context *)_ctx; 433 | 434 | if(fts_xapian_settings.verbose>1) i_info("FTS Xapian: fts_backend_xapian_update_unset_build_key"); 435 | 436 | if(ctx->tbi_field!=NULL) 437 | { 438 | i_free(ctx->tbi_field); 439 | } 440 | ctx->tbi_uid=0; 441 | ctx->tbi_field=NULL; 442 | } 443 | 444 | static int fts_backend_xapian_refresh(struct fts_backend * _backend) 445 | { 446 | if(fts_xapian_settings.verbose>1) i_info("FTS Xapian: fts_backend_xapian_refresh"); 447 | 448 | return 0; 449 | } 450 | 451 | static int fts_backend_xapian_update_build_more(struct fts_backend_update_context *_ctx, const unsigned char *data, size_t size) 452 | { 453 | struct xapian_fts_backend_update_context *ctx = (struct xapian_fts_backend_update_context *)_ctx; 454 | struct xapian_fts_backend *backend = (struct xapian_fts_backend *) ctx->ctx.backend; 455 | 456 | if(ctx->tbi_uid<1) return 0; 457 | if(strlen(ctx->tbi_field)<1) return 0; 458 | if(data == NULL) return 0; 459 | 460 | const char * d = (const char *) data; 461 | if(strlen(d)<(unsigned long)fts_xapian_settings.partial) return 0; 462 | 463 | long h = atol(ctx->tbi_field); 464 | 465 | if(backend->docs.size()>0) backend->docs.front()->raw_load(h,d,size,fts_xapian_settings.verbose,"fts_backend_xapian_index"); 466 | 467 | return 0; 468 | } 469 | 470 | static int fts_backend_xapian_optimize(struct fts_backend *_backend) 471 | { 472 | struct xapian_fts_backend *backend = (struct xapian_fts_backend *) _backend; 473 | 474 | if(fts_xapian_settings.verbose>1) i_info("FTS Xapian: fts_backend_xapian_optimize '%s'",backend->path); 475 | 476 | struct stat sb; 477 | if(!( (stat(backend->path, &sb)==0) && S_ISDIR(sb.st_mode))) 478 | { 479 | i_error("FTS Xapian: Optimize(0) Index folder inexistent"); 480 | return -1; 481 | } 482 | 483 | Xapian::WritableDatabase * db = NULL; 484 | sqlite3 * expdb = NULL; 485 | DIR* dirp = opendir(backend->path); 486 | struct dirent * dp; 487 | std::string s; 488 | uint32_t uid; 489 | int ret=0; 490 | std::vector uids; uids.clear(); 491 | char *zErrMsg = 0; 492 | XResultSet * result = NULL; 493 | Xapian::docid docid =0; 494 | while ((dp = readdir(dirp)) != NULL) 495 | { 496 | s = dp->d_name; 497 | if((dp->d_type == DT_REG) && s.starts_with("db_") && s.ends_with(suffixExp) ) 498 | { 499 | uids.clear(); 500 | i_info("FTS Xapian: Optimize (1) : Checking expunges from %s",dp->d_name); 501 | if(sqlite3_open_v2(dp->d_name,&expdb,SQLITE_OPEN_FULLMUTEX | SQLITE_OPEN_READWRITE,NULL) == SQLITE_OK) 502 | { 503 | if(fts_xapian_settings.verbose>0) i_info("FTS Xapian: Optimize (2) : Executing %s",selectExpUIDs); 504 | if(sqlite3_exec(expdb,selectExpUIDs,fts_backend_xapian_sqlite3_vector_int,&uids,&zErrMsg) != SQLITE_OK) 505 | { 506 | i_error("FTS Xapian: Optimize (3) : Can not select IDs (%s) : %s",selectExpUIDs,zErrMsg); 507 | if(zErrMsg!=NULL) sqlite3_free(zErrMsg); 508 | ret =-1; 509 | } 510 | s = s.substr(0,s.length()-strlen(suffixExp)); 511 | if(fts_xapian_settings.verbose>0) i_info("FTS Xapian: Optimize (4) : Opening Xapian DB (%s)",s.c_str()); 512 | try 513 | { 514 | bool ok=false; 515 | while(!ok) 516 | { 517 | try 518 | { 519 | db = new Xapian::WritableDatabase(s.c_str(),Xapian::DB_CREATE_OR_OPEN | Xapian::DB_BACKEND_GLASS); 520 | ok=true; 521 | } 522 | catch(Xapian::Error e) 523 | { 524 | i_warning("FTS Xapian: Retrying opening DB : %s - %s %s",e.get_type(),e.get_msg().c_str(),e.get_error_string()); 525 | std::this_thread::sleep_for(XAPIAN_SLEEP); 526 | } 527 | } 528 | long c=0; 529 | for(uint32_t n=0;n0) i_info("FTS Xapian: Optimize (5) Removing DOC UID=%d",uid); 533 | XQuerySet * xq = new XQuerySet(); 534 | xq->add(uid); 535 | result=fts_backend_xapian_query(db,xq,1); 536 | docid=0; 537 | if((result!=NULL) && (result->size>0)) 538 | { 539 | try 540 | { 541 | docid = result->data[0]; 542 | if(fts_xapian_settings.verbose>0) i_info("FTS Xapian: Optimize (5) Removing DOC UID=%d DOCID=%d",uid,docid); 543 | db->delete_document(docid); 544 | c++; 545 | if(c>XAPIAN_WRITING_CACHE) 546 | { 547 | i_info("FTS Xapian: Flushing changes on disk"); 548 | db->commit(); 549 | c=0; 550 | } 551 | } 552 | catch(Xapian::Error e) 553 | { 554 | i_error("FTS Xapian: Optimize (6) %s",e.get_msg().c_str()); 555 | } 556 | } 557 | else 558 | { 559 | if(fts_xapian_settings.verbose>0) i_info("FTS Xapian: Optimize UID=%d (DOCID=%d) inexistent",uid,docid); 560 | } 561 | if(result!=NULL) { delete(result); result=NULL; } 562 | delete(xq); 563 | char * u = i_strdup_printf(deleteExpUID,uid); 564 | if (sqlite3_exec(expdb,u,NULL,0,&zErrMsg) != SQLITE_OK ) 565 | { 566 | i_error("FTS Xapian : Optimize Sqlite error: %s",zErrMsg); 567 | if(zErrMsg!=NULL) sqlite3_free(zErrMsg); 568 | } 569 | i_free(u); 570 | } 571 | if(fts_xapian_settings.verbose>0) i_info("FTS Xapian: Optimize - Closing DB %s",s.c_str()); 572 | fts_backend_xapian_close_db(db,s.c_str(),"fts_optimize",fts_xapian_settings.verbose); 573 | } 574 | catch(Xapian::Error e) 575 | { 576 | i_error("FTS Xapian: Optimize (7) %s",e.get_msg().c_str()); 577 | } 578 | sqlite3_close(expdb); 579 | } 580 | } 581 | } 582 | closedir(dirp); 583 | return ret; 584 | } 585 | 586 | static int fts_backend_xapian_rescan(struct fts_backend *_backend) 587 | { 588 | if(fts_xapian_settings.verbose>1) i_info("FTS Xapian: fts_backend_xapian_rescan"); 589 | 590 | struct xapian_fts_backend *backend = (struct xapian_fts_backend *) _backend; 591 | 592 | struct stat sb; 593 | if(!( (stat(backend->path, &sb)==0) && S_ISDIR(sb.st_mode))) 594 | { 595 | i_error("FTS Xapian: Index folder (%s) inexistent",backend->path); 596 | return -1; 597 | } 598 | 599 | std::error_code errorCode; 600 | if(fts_xapian_settings.verbose>0) i_info("FTS Xapian: Rescan by deleting %s",backend->path); 601 | std::filesystem::remove_all(backend->path,errorCode); 602 | 603 | return 0; 604 | } 605 | 606 | static int fts_backend_xapian_lookup(struct fts_backend *_backend, struct mailbox *box, struct mail_search_arg *args, enum fts_lookup_flags flags, struct fts_result *result) 607 | { 608 | if(fts_xapian_settings.verbose>1) i_info("FTS Xapian: fts_backend_xapian_lookup"); 609 | 610 | struct xapian_fts_backend *backend = (struct xapian_fts_backend *) _backend; 611 | 612 | if(fts_backend_xapian_set_box(backend, box)<0) return -1; 613 | 614 | long current_time = fts_backend_xapian_current_time(); 615 | 616 | Xapian::Database * dbr; 617 | 618 | i_array_init(&(result->maybe_uids),0); 619 | i_array_init(&(result->scores),0); 620 | 621 | if(!fts_backend_xapian_open_readonly(backend, &dbr)) 622 | { 623 | i_array_init(&(result->definite_uids),0); 624 | return 0; 625 | } 626 | 627 | XQuerySet * qs; 628 | 629 | if((flags & FTS_LOOKUP_FLAG_AND_ARGS) != 0) 630 | { 631 | if(fts_xapian_settings.verbose>1) i_info("FTS Xapian: FLAG=AND"); 632 | qs = new XQuerySet(Xapian::Query::OP_AND,fts_xapian_settings.partial); 633 | } 634 | else 635 | { 636 | if(fts_xapian_settings.verbose>1) i_info("FTS Xapian: FLAG=OR"); 637 | qs = new XQuerySet(Xapian::Query::OP_OR,fts_xapian_settings.partial); 638 | } 639 | 640 | fts_backend_xapian_build_qs(qs,args,backend->dict_db); 641 | 642 | XResultSet * r=fts_backend_xapian_query(dbr,qs); 643 | 644 | long n=r->size; 645 | if(fts_xapian_settings.verbose>0) { i_info("FTS Xapian: Query '%s' -> %ld results",qs->get_string().c_str(),n); } 646 | 647 | i_array_init(&(result->definite_uids),r->size); 648 | 649 | uint32_t uid; 650 | for(long i=0;iget_document(r->data[i]).get_value(1)); 655 | seq_range_array_add(&result->definite_uids, uid); 656 | } 657 | catch(Xapian::Error e) 658 | { 659 | i_error("FTS Xapian: %s",e.get_msg().c_str()); 660 | } 661 | } 662 | delete(r); 663 | delete(qs); 664 | 665 | dbr->close(); 666 | delete(dbr); 667 | 668 | /* Performance calc */ 669 | if(fts_xapian_settings.verbose>0) i_info("FTS Xapian: %ld results in %ld ms",n,fts_backend_xapian_current_time() - current_time); 670 | 671 | return 0; 672 | } 673 | 674 | static int fts_backend_xapian_lookup_multi(struct fts_backend *_backend, struct mailbox *const boxes[], struct mail_search_arg *args, enum fts_lookup_flags flags, struct fts_multi_result *result) 675 | { 676 | if(fts_xapian_settings.verbose>1) i_info("FTS Xapian: fts_backend_xapian_lookup_multi"); 677 | 678 | ARRAY(struct fts_result) box_results; 679 | 680 | struct fts_result *box_result; 681 | int i; 682 | 683 | p_array_init(&box_results, result->pool, 0); 684 | for (i = 0; boxes[i] != NULL; i++) 685 | { 686 | box_result = array_append_space(&box_results); 687 | box_result->box = boxes[i]; 688 | if(fts_backend_xapian_lookup(_backend, boxes[i], args, flags, box_result)<0) 689 | { 690 | void* p=&box_results; 691 | p_free(result->pool, p); 692 | return -1; 693 | } 694 | } 695 | 696 | array_append_zero(&box_results); 697 | result->box_results = array_idx_modifiable(&box_results, 0); 698 | 699 | return 0; 700 | } 701 | 702 | struct fts_backend fts_backend_xapian = 703 | { 704 | .name = "xapian", 705 | .flags = FTS_BACKEND_FLAG_BUILD_FULL_WORDS, 706 | .v = { 707 | .alloc = fts_backend_xapian_alloc, 708 | .init = fts_backend_xapian_init, 709 | .deinit = fts_backend_xapian_deinit, 710 | .get_last_uid = fts_backend_xapian_get_last_uid, 711 | .update_init = fts_backend_xapian_update_init, 712 | .update_deinit = fts_backend_xapian_update_deinit, 713 | .update_set_mailbox = fts_backend_xapian_update_set_mailbox, 714 | .update_expunge = fts_backend_xapian_update_expunge, 715 | .update_set_build_key = fts_backend_xapian_update_set_build_key, 716 | .update_unset_build_key = fts_backend_xapian_update_unset_build_key, 717 | .update_build_more = fts_backend_xapian_update_build_more, 718 | .refresh = fts_backend_xapian_refresh, 719 | .rescan = fts_backend_xapian_rescan, 720 | .optimize = fts_backend_xapian_optimize, 721 | .can_lookup = fts_backend_default_can_lookup, 722 | .lookup = fts_backend_xapian_lookup, 723 | .lookup_multi = fts_backend_xapian_lookup_multi, 724 | } 725 | }; 726 | -------------------------------------------------------------------------------- /COPYING: -------------------------------------------------------------------------------- 1 | GNU LESSER GENERAL PUBLIC LICENSE 2 | Version 2.1, February 1999 3 | 4 | Copyright (C) 1991, 1999 Free Software Foundation, Inc. 5 | 6 | Everyone is permitted to copy and distribute verbatim copies 7 | of this license document, but changing it is not allowed. 8 | 9 | [This is the first released version of the Lesser GPL. It also counts 10 | as the successor of the GNU Library Public License, version 2, hence 11 | the version number 2.1.] 12 | 13 | Preamble 14 | 15 | The licenses for most software are designed to take away your 16 | freedom to share and change it. By contrast, the GNU General Public 17 | Licenses are intended to guarantee your freedom to share and change 18 | free software--to make sure the software is free for all its users. 19 | 20 | This license, the Lesser General Public License, applies to some 21 | specially designated software packages--typically libraries--of the 22 | Free Software Foundation and other authors who decide to use it. You 23 | can use it too, but we suggest you first think carefully about whether 24 | this license or the ordinary General Public License is the better 25 | strategy to use in any particular case, based on the explanations below. 26 | 27 | When we speak of free software, we are referring to freedom of use, 28 | not price. Our General Public Licenses are designed to make sure that 29 | you have the freedom to distribute copies of free software (and charge 30 | for this service if you wish); that you receive source code or can get 31 | it if you want it; that you can change the software and use pieces of 32 | it in new free programs; and that you are informed that you can do 33 | these things. 34 | 35 | To protect your rights, we need to make restrictions that forbid 36 | distributors to deny you these rights or to ask you to surrender these 37 | rights. These restrictions translate to certain responsibilities for 38 | you if you distribute copies of the library or if you modify it. 39 | 40 | For example, if you distribute copies of the library, whether gratis 41 | or for a fee, you must give the recipients all the rights that we gave 42 | you. You must make sure that they, too, receive or can get the source 43 | code. If you link other code with the library, you must provide 44 | complete object files to the recipients, so that they can relink them 45 | with the library after making changes to the library and recompiling 46 | it. And you must show them these terms so they know their rights. 47 | 48 | We protect your rights with a two-step method: (1) we copyright the 49 | library, and (2) we offer you this license, which gives you legal 50 | permission to copy, distribute and/or modify the library. 51 | 52 | To protect each distributor, we want to make it very clear that 53 | there is no warranty for the free library. Also, if the library is 54 | modified by someone else and passed on, the recipients should know 55 | that what they have is not the original version, so that the original 56 | author's reputation will not be affected by problems that might be 57 | introduced by others. 58 | 59 | Finally, software patents pose a constant threat to the existence of 60 | any free program. We wish to make sure that a company cannot 61 | effectively restrict the users of a free program by obtaining a 62 | restrictive license from a patent holder. Therefore, we insist that 63 | any patent license obtained for a version of the library must be 64 | consistent with the full freedom of use specified in this license. 65 | 66 | Most GNU software, including some libraries, is covered by the 67 | ordinary GNU General Public License. This license, the GNU Lesser 68 | General Public License, applies to certain designated libraries, and 69 | is quite different from the ordinary General Public License. We use 70 | this license for certain libraries in order to permit linking those 71 | libraries into non-free programs. 72 | 73 | When a program is linked with a library, whether statically or using 74 | a shared library, the combination of the two is legally speaking a 75 | combined work, a derivative of the original library. The ordinary 76 | General Public License therefore permits such linking only if the 77 | entire combination fits its criteria of freedom. The Lesser General 78 | Public License permits more lax criteria for linking other code with 79 | the library. 80 | 81 | We call this license the "Lesser" General Public License because it 82 | does Less to protect the user's freedom than the ordinary General 83 | Public License. It also provides other free software developers Less 84 | of an advantage over competing non-free programs. These disadvantages 85 | are the reason we use the ordinary General Public License for many 86 | libraries. However, the Lesser license provides advantages in certain 87 | special circumstances. 88 | 89 | For example, on rare occasions, there may be a special need to 90 | encourage the widest possible use of a certain library, so that it becomes 91 | a de-facto standard. To achieve this, non-free programs must be 92 | allowed to use the library. A more frequent case is that a free 93 | library does the same job as widely used non-free libraries. In this 94 | case, there is little to gain by limiting the free library to free 95 | software only, so we use the Lesser General Public License. 96 | 97 | In other cases, permission to use a particular library in non-free 98 | programs enables a greater number of people to use a large body of 99 | free software. For example, permission to use the GNU C Library in 100 | non-free programs enables many more people to use the whole GNU 101 | operating system, as well as its variant, the GNU/Linux operating 102 | system. 103 | 104 | Although the Lesser General Public License is Less protective of the 105 | users' freedom, it does ensure that the user of a program that is 106 | linked with the Library has the freedom and the wherewithal to run 107 | that program using a modified version of the Library. 108 | 109 | The precise terms and conditions for copying, distribution and 110 | modification follow. Pay close attention to the difference between a 111 | "work based on the library" and a "work that uses the library". The 112 | former contains code derived from the library, whereas the latter must 113 | be combined with the library in order to run. 114 | 115 | GNU LESSER GENERAL PUBLIC LICENSE 116 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 117 | 118 | 0. This License Agreement applies to any software library or other 119 | program which contains a notice placed by the copyright holder or 120 | other authorized party saying it may be distributed under the terms of 121 | this Lesser General Public License (also called "this License"). 122 | Each licensee is addressed as "you". 123 | 124 | A "library" means a collection of software functions and/or data 125 | prepared so as to be conveniently linked with application programs 126 | (which use some of those functions and data) to form executables. 127 | 128 | The "Library", below, refers to any such software library or work 129 | which has been distributed under these terms. A "work based on the 130 | Library" means either the Library or any derivative work under 131 | copyright law: that is to say, a work containing the Library or a 132 | portion of it, either verbatim or with modifications and/or translated 133 | straightforwardly into another language. (Hereinafter, translation is 134 | included without limitation in the term "modification".) 135 | 136 | "Source code" for a work means the preferred form of the work for 137 | making modifications to it. For a library, complete source code means 138 | all the source code for all modules it contains, plus any associated 139 | interface definition files, plus the scripts used to control compilation 140 | and installation of the library. 141 | 142 | Activities other than copying, distribution and modification are not 143 | covered by this License; they are outside its scope. The act of 144 | running a program using the Library is not restricted, and output from 145 | such a program is covered only if its contents constitute a work based 146 | on the Library (independent of the use of the Library in a tool for 147 | writing it). Whether that is true depends on what the Library does 148 | and what the program that uses the Library does. 149 | 150 | 1. You may copy and distribute verbatim copies of the Library's 151 | complete source code as you receive it, in any medium, provided that 152 | you conspicuously and appropriately publish on each copy an 153 | appropriate copyright notice and disclaimer of warranty; keep intact 154 | all the notices that refer to this License and to the absence of any 155 | warranty; and distribute a copy of this License along with the 156 | Library. 157 | 158 | You may charge a fee for the physical act of transferring a copy, 159 | and you may at your option offer warranty protection in exchange for a 160 | fee. 161 | 162 | 2. You may modify your copy or copies of the Library or any portion 163 | of it, thus forming a work based on the Library, and copy and 164 | distribute such modifications or work under the terms of Section 1 165 | above, provided that you also meet all of these conditions: 166 | 167 | a) The modified work must itself be a software library. 168 | 169 | b) You must cause the files modified to carry prominent notices 170 | stating that you changed the files and the date of any change. 171 | 172 | c) You must cause the whole of the work to be licensed at no 173 | charge to all third parties under the terms of this License. 174 | 175 | d) If a facility in the modified Library refers to a function or a 176 | table of data to be supplied by an application program that uses 177 | the facility, other than as an argument passed when the facility 178 | is invoked, then you must make a good faith effort to ensure that, 179 | in the event an application does not supply such function or 180 | table, the facility still operates, and performs whatever part of 181 | its purpose remains meaningful. 182 | 183 | (For example, a function in a library to compute square roots has 184 | a purpose that is entirely well-defined independent of the 185 | application. Therefore, Subsection 2d requires that any 186 | application-supplied function or table used by this function must 187 | be optional: if the application does not supply it, the square 188 | root function must still compute square roots.) 189 | 190 | These requirements apply to the modified work as a whole. If 191 | identifiable sections of that work are not derived from the Library, 192 | and can be reasonably considered independent and separate works in 193 | themselves, then this License, and its terms, do not apply to those 194 | sections when you distribute them as separate works. But when you 195 | distribute the same sections as part of a whole which is a work based 196 | on the Library, the distribution of the whole must be on the terms of 197 | this License, whose permissions for other licensees extend to the 198 | entire whole, and thus to each and every part regardless of who wrote 199 | it. 200 | 201 | Thus, it is not the intent of this section to claim rights or contest 202 | your rights to work written entirely by you; rather, the intent is to 203 | exercise the right to control the distribution of derivative or 204 | collective works based on the Library. 205 | 206 | In addition, mere aggregation of another work not based on the Library 207 | with the Library (or with a work based on the Library) on a volume of 208 | a storage or distribution medium does not bring the other work under 209 | the scope of this License. 210 | 211 | 3. You may opt to apply the terms of the ordinary GNU General Public 212 | License instead of this License to a given copy of the Library. To do 213 | this, you must alter all the notices that refer to this License, so 214 | that they refer to the ordinary GNU General Public License, version 2, 215 | instead of to this License. (If a newer version than version 2 of the 216 | ordinary GNU General Public License has appeared, then you can specify 217 | that version instead if you wish.) Do not make any other change in 218 | these notices. 219 | 220 | Once this change is made in a given copy, it is irreversible for 221 | that copy, so the ordinary GNU General Public License applies to all 222 | subsequent copies and derivative works made from that copy. 223 | 224 | This option is useful when you wish to copy part of the code of 225 | the Library into a program that is not a library. 226 | 227 | 4. You may copy and distribute the Library (or a portion or 228 | derivative of it, under Section 2) in object code or executable form 229 | under the terms of Sections 1 and 2 above provided that you accompany 230 | it with the complete corresponding machine-readable source code, which 231 | must be distributed under the terms of Sections 1 and 2 above on a 232 | medium customarily used for software interchange. 233 | 234 | If distribution of object code is made by offering access to copy 235 | from a designated place, then offering equivalent access to copy the 236 | source code from the same place satisfies the requirement to 237 | distribute the source code, even though third parties are not 238 | compelled to copy the source along with the object code. 239 | 240 | 5. A program that contains no derivative of any portion of the 241 | Library, but is designed to work with the Library by being compiled or 242 | linked with it, is called a "work that uses the Library". Such a 243 | work, in isolation, is not a derivative work of the Library, and 244 | therefore falls outside the scope of this License. 245 | 246 | However, linking a "work that uses the Library" with the Library 247 | creates an executable that is a derivative of the Library (because it 248 | contains portions of the Library), rather than a "work that uses the 249 | library". The executable is therefore covered by this License. 250 | Section 6 states terms for distribution of such executables. 251 | 252 | When a "work that uses the Library" uses material from a header file 253 | that is part of the Library, the object code for the work may be a 254 | derivative work of the Library even though the source code is not. 255 | Whether this is true is especially significant if the work can be 256 | linked without the Library, or if the work is itself a library. The 257 | threshold for this to be true is not precisely defined by law. 258 | 259 | If such an object file uses only numerical parameters, data 260 | structure layouts and accessors, and small macros and small inline 261 | functions (ten lines or less in length), then the use of the object 262 | file is unrestricted, regardless of whether it is legally a derivative 263 | work. (Executables containing this object code plus portions of the 264 | Library will still fall under Section 6.) 265 | 266 | Otherwise, if the work is a derivative of the Library, you may 267 | distribute the object code for the work under the terms of Section 6. 268 | Any executables containing that work also fall under Section 6, 269 | whether or not they are linked directly with the Library itself. 270 | 271 | 6. As an exception to the Sections above, you may also combine or 272 | link a "work that uses the Library" with the Library to produce a 273 | work containing portions of the Library, and distribute that work 274 | under terms of your choice, provided that the terms permit 275 | modification of the work for the customer's own use and reverse 276 | engineering for debugging such modifications. 277 | 278 | You must give prominent notice with each copy of the work that the 279 | Library is used in it and that the Library and its use are covered by 280 | this License. You must supply a copy of this License. If the work 281 | during execution displays copyright notices, you must include the 282 | copyright notice for the Library among them, as well as a reference 283 | directing the user to the copy of this License. Also, you must do one 284 | of these things: 285 | 286 | a) Accompany the work with the complete corresponding 287 | machine-readable source code for the Library including whatever 288 | changes were used in the work (which must be distributed under 289 | Sections 1 and 2 above); and, if the work is an executable linked 290 | with the Library, with the complete machine-readable "work that 291 | uses the Library", as object code and/or source code, so that the 292 | user can modify the Library and then relink to produce a modified 293 | executable containing the modified Library. (It is understood 294 | that the user who changes the contents of definitions files in the 295 | Library will not necessarily be able to recompile the application 296 | to use the modified definitions.) 297 | 298 | b) Use a suitable shared library mechanism for linking with the 299 | Library. A suitable mechanism is one that (1) uses at run time a 300 | copy of the library already present on the user's computer system, 301 | rather than copying library functions into the executable, and (2) 302 | will operate properly with a modified version of the library, if 303 | the user installs one, as long as the modified version is 304 | interface-compatible with the version that the work was made with. 305 | 306 | c) Accompany the work with a written offer, valid for at 307 | least three years, to give the same user the materials 308 | specified in Subsection 6a, above, for a charge no more 309 | than the cost of performing this distribution. 310 | 311 | d) If distribution of the work is made by offering access to copy 312 | from a designated place, offer equivalent access to copy the above 313 | specified materials from the same place. 314 | 315 | e) Verify that the user has already received a copy of these 316 | materials or that you have already sent this user a copy. 317 | 318 | For an executable, the required form of the "work that uses the 319 | Library" must include any data and utility programs needed for 320 | reproducing the executable from it. However, as a special exception, 321 | the materials to be distributed need not include anything that is 322 | normally distributed (in either source or binary form) with the major 323 | components (compiler, kernel, and so on) of the operating system on 324 | which the executable runs, unless that component itself accompanies 325 | the executable. 326 | 327 | It may happen that this requirement contradicts the license 328 | restrictions of other proprietary libraries that do not normally 329 | accompany the operating system. Such a contradiction means you cannot 330 | use both them and the Library together in an executable that you 331 | distribute. 332 | 333 | 7. You may place library facilities that are a work based on the 334 | Library side-by-side in a single library together with other library 335 | facilities not covered by this License, and distribute such a combined 336 | library, provided that the separate distribution of the work based on 337 | the Library and of the other library facilities is otherwise 338 | permitted, and provided that you do these two things: 339 | 340 | a) Accompany the combined library with a copy of the same work 341 | based on the Library, uncombined with any other library 342 | facilities. This must be distributed under the terms of the 343 | Sections above. 344 | 345 | b) Give prominent notice with the combined library of the fact 346 | that part of it is a work based on the Library, and explaining 347 | where to find the accompanying uncombined form of the same work. 348 | 349 | 8. You may not copy, modify, sublicense, link with, or distribute 350 | the Library except as expressly provided under this License. Any 351 | attempt otherwise to copy, modify, sublicense, link with, or 352 | distribute the Library is void, and will automatically terminate your 353 | rights under this License. However, parties who have received copies, 354 | or rights, from you under this License will not have their licenses 355 | terminated so long as such parties remain in full compliance. 356 | 357 | 9. You are not required to accept this License, since you have not 358 | signed it. However, nothing else grants you permission to modify or 359 | distribute the Library or its derivative works. These actions are 360 | prohibited by law if you do not accept this License. Therefore, by 361 | modifying or distributing the Library (or any work based on the 362 | Library), you indicate your acceptance of this License to do so, and 363 | all its terms and conditions for copying, distributing or modifying 364 | the Library or works based on it. 365 | 366 | 10. Each time you redistribute the Library (or any work based on the 367 | Library), the recipient automatically receives a license from the 368 | original licensor to copy, distribute, link with or modify the Library 369 | subject to these terms and conditions. You may not impose any further 370 | restrictions on the recipients' exercise of the rights granted herein. 371 | You are not responsible for enforcing compliance by third parties with 372 | this License. 373 | 374 | 11. If, as a consequence of a court judgment or allegation of patent 375 | infringement or for any other reason (not limited to patent issues), 376 | conditions are imposed on you (whether by court order, agreement or 377 | otherwise) that contradict the conditions of this License, they do not 378 | excuse you from the conditions of this License. If you cannot 379 | distribute so as to satisfy simultaneously your obligations under this 380 | License and any other pertinent obligations, then as a consequence you 381 | may not distribute the Library at all. For example, if a patent 382 | license would not permit royalty-free redistribution of the Library by 383 | all those who receive copies directly or indirectly through you, then 384 | the only way you could satisfy both it and this License would be to 385 | refrain entirely from distribution of the Library. 386 | 387 | If any portion of this section is held invalid or unenforceable under any 388 | particular circumstance, the balance of the section is intended to apply, 389 | and the section as a whole is intended to apply in other circumstances. 390 | 391 | It is not the purpose of this section to induce you to infringe any 392 | patents or other property right claims or to contest validity of any 393 | such claims; this section has the sole purpose of protecting the 394 | integrity of the free software distribution system which is 395 | implemented by public license practices. Many people have made 396 | generous contributions to the wide range of software distributed 397 | through that system in reliance on consistent application of that 398 | system; it is up to the author/donor to decide if he or she is willing 399 | to distribute software through any other system and a licensee cannot 400 | impose that choice. 401 | 402 | This section is intended to make thoroughly clear what is believed to 403 | be a consequence of the rest of this License. 404 | 405 | 12. If the distribution and/or use of the Library is restricted in 406 | certain countries either by patents or by copyrighted interfaces, the 407 | original copyright holder who places the Library under this License may add 408 | an explicit geographical distribution limitation excluding those countries, 409 | so that distribution is permitted only in or among countries not thus 410 | excluded. In such case, this License incorporates the limitation as if 411 | written in the body of this License. 412 | 413 | 13. The Free Software Foundation may publish revised and/or new 414 | versions of the Lesser General Public License from time to time. 415 | Such new versions will be similar in spirit to the present version, 416 | but may differ in detail to address new problems or concerns. 417 | 418 | Each version is given a distinguishing version number. If the Library 419 | specifies a version number of this License which applies to it and 420 | "any later version", you have the option of following the terms and 421 | conditions either of that version or of any later version published by 422 | the Free Software Foundation. If the Library does not specify a 423 | license version number, you may choose any version ever published by 424 | the Free Software Foundation. 425 | 426 | 14. If you wish to incorporate parts of the Library into other free 427 | programs whose distribution conditions are incompatible with these, 428 | write to the author to ask for permission. For software which is 429 | copyrighted by the Free Software Foundation, write to the Free 430 | Software Foundation; we sometimes make exceptions for this. Our 431 | decision will be guided by the two goals of preserving the free status 432 | of all derivatives of our free software and of promoting the sharing 433 | and reuse of software generally. 434 | 435 | NO WARRANTY 436 | 437 | 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO 438 | WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. 439 | EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR 440 | OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY 441 | KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE 442 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 443 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE 444 | LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME 445 | THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 446 | 447 | 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN 448 | WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY 449 | AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU 450 | FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR 451 | CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE 452 | LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING 453 | RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A 454 | FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF 455 | SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH 456 | DAMAGES. 457 | 458 | END OF TERMS AND CONDITIONS 459 | 460 | How to Apply These Terms to Your New Libraries 461 | 462 | If you develop a new library, and you want it to be of the greatest 463 | possible use to the public, we recommend making it free software that 464 | everyone can redistribute and change. You can do so by permitting 465 | redistribution under these terms (or, alternatively, under the terms of the 466 | ordinary General Public License). 467 | 468 | To apply these terms, attach the following notices to the library. It is 469 | safest to attach them to the start of each source file to most effectively 470 | convey the exclusion of warranty; and each file should have at least the 471 | "copyright" line and a pointer to where the full notice is found. 472 | 473 | 474 | Copyright (C) 475 | 476 | This library is free software; you can redistribute it and/or 477 | modify it under the terms of the GNU Lesser General Public 478 | License as published by the Free Software Foundation; either 479 | version 2.1 of the License, or (at your option) any later version. 480 | 481 | This library is distributed in the hope that it will be useful, 482 | but WITHOUT ANY WARRANTY; without even the implied warranty of 483 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 484 | Lesser General Public License for more details. 485 | 486 | You should have received a copy of the GNU Lesser General Public 487 | License along with this library; if not, see . 488 | 489 | Also add information on how to contact you by electronic and paper mail. 490 | 491 | You should also get your employer (if you work as a programmer) or your 492 | school, if any, to sign a "copyright disclaimer" for the library, if 493 | necessary. Here is a sample; alter the names: 494 | 495 | Yoyodyne, Inc., hereby disclaims all copyright interest in the 496 | library `Frob' (a library for tweaking knobs) written by James Random Hacker. 497 | 498 | , 1 April 1990 499 | Moe Ghoul, President of Vice 500 | 501 | That's all there is to it! 502 | -------------------------------------------------------------------------------- /src/fts-backend-xapian-functions.cpp: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2019 Joan Moreau , see the included COPYING file */ 2 | 3 | static long fts_backend_xapian_current_time() 4 | { 5 | struct timeval tp; 6 | gettimeofday(&tp, NULL); 7 | return tp.tv_sec * 1000 + tp.tv_usec / 1000; 8 | } 9 | 10 | static long fts_backend_xapian_get_free_memory(int verbose) // KB 11 | { 12 | char buffer[250]; 13 | char *p; 14 | struct rlimit rl; 15 | rl.rlim_cur=0; 16 | if(getrlimit(RLIMIT_AS,&rl)!=0) syslog(LOG_WARNING,"FTS Xapian: Memory limit by GETRLIMIT error: %s",strerror(errno)); 17 | long m,l = rl.rlim_cur; 18 | FILE *f; 19 | if(l<1) 20 | { 21 | if(verbose>1) syslog(LOG_WARNING,"FTS Xapian: Memory limit not available from getrlimit (probably vsz_limit not set"); 22 | #if defined(__FreeBSD__) || defined(__NetBSD__) 23 | u_int page_size; 24 | uint_size uint_size = sizeof(page_size); 25 | sysctlbyname("vm.stats.vm.v_page_size", &page_size, &uint_size, NULL, 0); 26 | struct vmtotal vmt; 27 | size_t vmt_size = sizeof(vmt); 28 | sysctlbyname("vm.vmtotal", &vmt, &vmt_size, NULL, 0); 29 | m = vmt.t_free * page_size / 1024.0f; 30 | #else 31 | f=fopen("/proc/meminfo","r"); 32 | if(f==NULL) return -1024; 33 | m=0; 34 | while(!feof(f)) 35 | { 36 | if ( fgets (buffer , 200 , f) == NULL ) break; 37 | p = strstr(buffer,"MemAvailable:"); 38 | if(p!=NULL) 39 | { 40 | m=atol(p+13); 41 | break; 42 | } 43 | } 44 | fclose(f); 45 | #endif 46 | if(verbose>1) syslog(LOG_WARNING,"FTS Xapian: Memory available from meminfo : %ld MB",(long)(m/1024.0)); 47 | } 48 | else 49 | { 50 | l = l / 1024.0f; 51 | if(verbose>1) syslog(LOG_WARNING,"FTS Xapian: Memory limit detected at %ld MB",(long)(l/1024.0f)); 52 | 53 | long pid=getpid(); 54 | sprintf(buffer,"/proc/%ld/status",pid); 55 | f=fopen(buffer,"r"); 56 | long memused=0; 57 | if(f != NULL) 58 | { 59 | while(!feof(f)) 60 | { 61 | if ( fgets (buffer , 100 , f) == NULL ) break; 62 | p = strstr(buffer,"VmSize:"); 63 | if(p!=NULL) 64 | { 65 | memused=atol(p+7); 66 | break; 67 | } 68 | } 69 | fclose(f); 70 | if(verbose>1) syslog(LOG_WARNING,"FTS Xapian: Memory used %ld MB",(long)(memused/1024.0f)); 71 | } 72 | else 73 | { 74 | if(verbose>1) syslog(LOG_WARNING,"FTS Xapian: Memory used not available from %s", buffer); 75 | memused=-1; 76 | } 77 | m = l - memused; 78 | } 79 | if(verbose>1) syslog(LOG_WARNING,"FTS Xapian: Available memory %ld MB",long(m/1024.0f)); 80 | return m; 81 | } 82 | 83 | static void fts_backend_xapian_icutostring(icu::UnicodeString *t, std::string &s) 84 | { 85 | s.clear(); 86 | t->toUTF8String(s); 87 | } 88 | 89 | static long fts_backend_xapian_icutochar_length(icu::UnicodeString *t) 90 | { 91 | std::string s; 92 | s.clear(); 93 | t->toUTF8String(s); 94 | return strlen(s.c_str()); 95 | } 96 | 97 | static bool fts_backend_xapian_clean_accents(icu::UnicodeString *t) 98 | { 99 | UErrorCode status = U_ZERO_ERROR; 100 | icu::Transliterator * accentsConverter = icu::Transliterator::createInstance("NFD; [:M:] Remove; NFC", UTRANS_FORWARD, status); 101 | if(U_FAILURE(status)) 102 | { 103 | std::string s("FTS Xapian: Can not allocate ICU translator + FreeMem="+std::to_string(long(fts_backend_xapian_get_free_memory(0)/1024.0f))+"MB"); 104 | syslog(LOG_ERR,"%s",s.c_str()); 105 | accentsConverter = NULL; 106 | return false; 107 | } 108 | accentsConverter->transliterate(*t); 109 | delete(accentsConverter); 110 | return true; 111 | } 112 | 113 | static void fts_backend_xapian_trim(icu::UnicodeString *d) 114 | { 115 | while(d->startsWith(CHAR_SPACE) || d->startsWith(CHAR_KEY)) 116 | { 117 | d->remove(0,1); 118 | } 119 | while(d->endsWith(CHAR_SPACE) || d->endsWith(CHAR_KEY)) 120 | { 121 | d->truncate(d->length()-1); 122 | } 123 | } 124 | 125 | static void fts_backend_xapian_clean(icu::UnicodeString *t) 126 | { 127 | fts_backend_xapian_clean_accents(t); 128 | t->toLower(); 129 | 130 | long k=CHARS_PB; 131 | while(k>0) 132 | { 133 | t->findAndReplace(chars_pb[k-1],CHAR_KEY); 134 | k--; 135 | } 136 | 137 | k=CHARS_SEP; 138 | while(k>0) 139 | { 140 | t->findAndReplace(chars_sep[k-1],CHAR_SPACE); 141 | k--; 142 | } 143 | 144 | fts_backend_xapian_trim(t); 145 | } 146 | 147 | static long fts_backend_xapian_clean_header(const char * hdr) 148 | { 149 | if(hdr == NULL) return -1; 150 | long l = strlen(hdr); 151 | if(l>=200) return -1; 152 | 153 | char h[200]; 154 | 155 | long i=0,j=0; 156 | while(j' ') && (hdr[j]!='"') && (hdr[j]!='\'') && (hdr[j]!='-')) 159 | { 160 | h[i]=std::tolower(hdr[j]); 161 | i++; 162 | } 163 | j++; 164 | } 165 | h[i]=0; 166 | 167 | i=0; 168 | while((i=HDRS_NB) i=-1; 171 | 172 | if(i==HDRS_NB-1) i=HDR_BODY; 173 | 174 | return i; 175 | } 176 | 177 | static void fts_backend_xapian_get_lock(struct xapian_fts_backend *backend, long verbose, const char *s) 178 | { 179 | std::unique_lock *lck; 180 | lck = new std::unique_lock(backend->mutex,std::defer_lock); 181 | #pragma GCC diagnostic push 182 | #pragma GCC diagnostic ignored "-Wdeprecated-declarations" 183 | while(!(lck->try_lock_for(std::chrono::milliseconds(1000 + std::rand() % 1000)))) 184 | { 185 | if(verbose>0) 186 | { 187 | std::string sl("FTS Xapian: Waiting unlock... ("); 188 | sl.append(s); 189 | sl.append(")"); 190 | syslog(LOG_INFO,"%s",sl.c_str()); 191 | } 192 | } 193 | #pragma GCC diagnostic pop 194 | if(verbose>0) 195 | { 196 | std::string sl("FTS Xapian: Got lock ("); 197 | sl.append(s); 198 | sl.append(")"); 199 | syslog(LOG_INFO,"%s",sl.c_str()); 200 | } 201 | backend->mutex_t = lck; 202 | } 203 | 204 | static void fts_backend_xapian_release_lock(struct xapian_fts_backend *backend, long verbose, const char *s) 205 | { 206 | if(verbose>1) 207 | { 208 | std::string sl("FTS Xapian: Releasing lock ("); 209 | sl.append(s); 210 | sl.append(")"); 211 | syslog(LOG_INFO,"%s",sl.c_str()); 212 | } 213 | if(backend->mutex_t !=NULL) 214 | { 215 | std::unique_lock *lck = backend->mutex_t; 216 | backend->mutex_t= NULL; 217 | delete(lck); 218 | } 219 | } 220 | 221 | static int fts_backend_xapian_sqlite3_vector_int(void *data, int argc, char **argv, char **azColName) 222 | { 223 | if (argc < 1) return -1; 224 | 225 | uint32_t uid = atol(argv[0]); 226 | std::vector * uids = (std::vector *) data; 227 | uids->push_back(uid); 228 | 229 | return 0; 230 | } 231 | 232 | static int fts_backend_xapian_sqlite3_vector_icu(void *data, int argc, char **argv, char **azColName) 233 | { 234 | if (argc < 1) return -1; 235 | 236 | icu::StringPiece sp(argv[0]); 237 | icu::UnicodeString * t = new icu::UnicodeString(icu::UnicodeString::fromUTF8(sp)); 238 | if(fts_xapian_settings.verbose>0) i_info("FTS Xapian: sqlite3_vector_string : Adding %s",argv[0]); 239 | std::vector * v = (std::vector *) data; 240 | v->push_back(t); 241 | 242 | return 0; 243 | } 244 | 245 | static bool fts_backend_xapian_sqlite3_dict_open(struct xapian_fts_backend *backend) 246 | { 247 | if(backend->ddb!=NULL) return TRUE; 248 | 249 | backend->dict_nb=0; 250 | 251 | if(sqlite3_open_v2(backend->dict_db,&(backend->ddb),SQLITE_OPEN_FULLMUTEX | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE,NULL) != SQLITE_OK ) 252 | { 253 | i_error("FTS Xapian: Can not open %s : %s",backend->dict_db,sqlite3_errmsg(backend->ddb)); 254 | backend->ddb = NULL; 255 | return FALSE; 256 | } 257 | 258 | char *zErrMsg = 0; 259 | if(sqlite3_exec(backend->ddb,createDictTable,NULL,0,&zErrMsg) != SQLITE_OK ) 260 | { 261 | i_error("FTS Xapian: Can not execute (%s) : %s",createDictTable,zErrMsg); 262 | if(zErrMsg!=NULL) sqlite3_free(zErrMsg); 263 | sqlite3_close(backend->ddb); 264 | backend->ddb = NULL; 265 | return FALSE; 266 | } 267 | 268 | zErrMsg =0; 269 | if(sqlite3_exec(backend->ddb,createDictIndexes,NULL,0,&zErrMsg) != SQLITE_OK ) 270 | { 271 | i_error("FTS Xapian: Can not execute (%s) : %s",createDictIndexes,zErrMsg); 272 | if(zErrMsg!=NULL) sqlite3_free(zErrMsg); 273 | sqlite3_close(backend->ddb); 274 | backend->ddb = NULL; 275 | return FALSE; 276 | } 277 | 278 | zErrMsg =0; 279 | if(sqlite3_exec(backend->ddb,createTmpTable,NULL,0,&zErrMsg) != SQLITE_OK ) 280 | { 281 | i_error("FTS Xapian: Can not execute (%s) : %s",createTmpTable,zErrMsg); 282 | if(zErrMsg!=NULL) sqlite3_free(zErrMsg); 283 | sqlite3_close(backend->ddb); 284 | backend->ddb = NULL; 285 | return FALSE; 286 | } 287 | return TRUE; 288 | } 289 | 290 | static int fts_backend_xapian_sqlite3_dict_add(struct xapian_fts_backend *backend, long h, icu::UnicodeString *t) 291 | { 292 | std::string sql; 293 | sql.clear(); 294 | t->toUTF8String(sql); 295 | sql=replaceTmpWord + sql + "', " + std::to_string(h) + ", " + std::to_string(sql.length()) + ");"; 296 | 297 | char * zErrMsg = 0; 298 | if(sqlite3_exec(backend->ddb,sql.c_str(),NULL,0,&zErrMsg) != SQLITE_OK ) 299 | { 300 | syslog(LOG_ERR,"FTS Xapian: Can not replace keyword (%s) : %s",sql.c_str(),zErrMsg); 301 | if(zErrMsg!=NULL) sqlite3_free(zErrMsg); 302 | return 1; 303 | } 304 | backend->dict_nb++; 305 | return 0; 306 | } 307 | 308 | static bool fts_backend_xapian_sqlite3_dict_flush(struct xapian_fts_backend *backend, int verbose,char *err_s=NULL) 309 | { 310 | if(backend->dict_nb<1) return TRUE; 311 | 312 | long dt=fts_backend_xapian_current_time(); 313 | if(verbose>0) syslog(LOG_INFO,"FTS Xapian: Flushing Dictionnary : %ld terms",backend->dict_nb); 314 | char * zErrMsg = 0; 315 | if(sqlite3_exec(backend->ddb,flushTmpWords,NULL,0,&zErrMsg) != SQLITE_OK ) 316 | { 317 | syslog(LOG_ERR,"FTS Xapian: Can not execute (%s) : %s",flushTmpWords,zErrMsg); 318 | if(err_s!=NULL) 319 | { 320 | sprintf(err_s,"FTS Xapian: Can not execute (%s) : %s",flushTmpWords,zErrMsg); 321 | } 322 | if(zErrMsg!=NULL) sqlite3_free(zErrMsg); 323 | return FALSE; 324 | } 325 | if(verbose>0) syslog(LOG_INFO,"FTS Xapian: Flushing Dictionnary : %ld terms done in %ld msec",backend->dict_nb,fts_backend_xapian_current_time()-dt); 326 | backend->dict_nb = 0; 327 | return TRUE; 328 | } 329 | 330 | class XResultSet 331 | { 332 | public: 333 | long size; 334 | Xapian::docid * data; 335 | 336 | XResultSet() { size=0; data=NULL; } 337 | ~XResultSet() { if (size>0) { i_free(data); } } 338 | 339 | void add(Xapian::docid did) 340 | { 341 | if(data==NULL) 342 | { 343 | data=(Xapian::docid *)i_malloc(sizeof(Xapian::docid)); 344 | } 345 | else 346 | { 347 | data=(Xapian::docid *)i_realloc(data,size*sizeof(Xapian::docid),(size+1)*sizeof(Xapian::docid)); 348 | } 349 | data[size]=did; 350 | size++; 351 | } 352 | }; 353 | 354 | class XQuerySet 355 | { 356 | private: 357 | long header; 358 | icu::UnicodeString * text; 359 | XQuerySet ** qs; 360 | Xapian::Query::op global_op; 361 | bool item_neg; // for the term 362 | long qsize; 363 | 364 | public: 365 | long limit; 366 | 367 | XQuerySet() 368 | { 369 | qsize=0; qs=NULL; 370 | limit=1; 371 | header=-1; 372 | text=NULL; 373 | global_op = Xapian::Query::op::OP_OR; 374 | } 375 | 376 | XQuerySet(Xapian::Query::op op, long l) 377 | { 378 | qsize=0; qs=NULL; 379 | limit=2; 380 | if(l>limit) { limit=l; } 381 | header=-1; 382 | text=NULL; 383 | global_op=op; 384 | } 385 | 386 | ~XQuerySet() 387 | { 388 | if(text!=NULL) 389 | { 390 | delete(text); 391 | text=NULL; 392 | } 393 | 394 | for(long j=0;j0) free(qs); 399 | qsize=0; qs=NULL; 400 | } 401 | 402 | void add(long uid) 403 | { 404 | std::string s = std::to_string(uid); 405 | icu::UnicodeString t(s.c_str()); 406 | add(0,&t,false); 407 | } 408 | 409 | void add(long h, icu::UnicodeString *t, bool is_neg) 410 | { 411 | if(t==NULL) return; 412 | fts_backend_xapian_clean(t); 413 | if(t->length()lastIndexOf(CHAR_SPACE); 420 | if(i>0) 421 | { 422 | if(is_neg) 423 | { 424 | q2 = new XQuerySet(Xapian::Query::OP_AND_NOT,limit); 425 | } 426 | else 427 | { 428 | q2 = new XQuerySet(Xapian::Query::OP_AND,limit); 429 | } 430 | while(i>0) 431 | { 432 | j = t->length(); 433 | r = new icu::UnicodeString(*t,i+1,j-i-1); 434 | q2->add(h,r,false); 435 | delete(r); 436 | t->truncate(i); 437 | fts_backend_xapian_trim(t); 438 | i = t->lastIndexOf(CHAR_SPACE); 439 | } 440 | q2->add(h,t,false); 441 | if(q2->count()>0) add(q2); else delete(q2); 442 | return; 443 | } 444 | 445 | if(h<0) 446 | { 447 | if(is_neg) 448 | { 449 | q2 = new XQuerySet(Xapian::Query::OP_AND_NOT,limit); 450 | } 451 | else 452 | { 453 | q2 = new XQuerySet(Xapian::Query::OP_OR,limit); 454 | } 455 | for(i=1;iadd(i,t,false); 458 | } 459 | add(q2); 460 | return; 461 | } 462 | 463 | if(text==NULL) 464 | { 465 | text=new icu::UnicodeString(*t); 466 | header=h; 467 | item_neg=is_neg; 468 | return; 469 | } 470 | 471 | q2 = new XQuerySet(Xapian::Query::OP_AND,limit); 472 | q2->add(h,t,is_neg); 473 | add(q2); 474 | } 475 | 476 | void add(XQuerySet *q2) 477 | { 478 | if(qsize<1) 479 | { 480 | qs=(XQuerySet **)malloc(sizeof(XQuerySet*)); 481 | } 482 | else 483 | { 484 | qs=(XQuerySet **)realloc(qs,(qsize+1)*sizeof(XQuerySet*)); 485 | } 486 | qs[qsize]=q2; 487 | qsize++; 488 | } 489 | 490 | int count() 491 | { 492 | int c=0; 493 | if(text!=NULL) c=1; 494 | c+=qsize; 495 | return c; 496 | } 497 | 498 | std::string get_string() 499 | { 500 | std::string s(""); 501 | 502 | if(count()<1) return s; 503 | 504 | if(text!=NULL) 505 | { 506 | if(item_neg) s.append("NOT ( "); 507 | s.append(hdrs_emails[header]); 508 | s.append(":\""); 509 | text->toUTF8String(s); 510 | s.append("\""); 511 | if(item_neg) s.append(")"); 512 | } 513 | 514 | const char * op; 515 | switch(global_op) 516 | { 517 | case Xapian::Query::OP_OR : op=" OR "; break; 518 | case Xapian::Query::OP_AND : op=" AND "; break; 519 | case Xapian::Query::OP_AND_NOT : op=" AND NOT "; break; 520 | default : op=" ERROR "; 521 | } 522 | 523 | for (int i=0;icount(); 526 | if(c<1) continue; 527 | 528 | if(s.length()>0) s.append(op); 529 | 530 | if(c>1) 531 | { 532 | s.append("("); 533 | s.append(qs[i]->get_string()); 534 | s.append(")"); 535 | } 536 | else s.append(qs[i]->get_string()); 537 | } 538 | return s; 539 | } 540 | 541 | Xapian::Query * get_query(Xapian::Database * db) 542 | { 543 | Xapian::Query * q = NULL; 544 | Xapian::Query *q2, *q3; 545 | 546 | if(text!=NULL) 547 | { 548 | std::string s(hdrs_query[header]); 549 | s.append(":\""); 550 | text->toUTF8String(s); 551 | s.append("\""); 552 | 553 | Xapian::QueryParser * qp = new Xapian::QueryParser(); 554 | for(int i=0; i< HDRS_NB-1; i++) qp->add_prefix(hdrs_query[i], hdrs_xapian[i]); 555 | qp->set_database(*db); 556 | q = new Xapian::Query(qp->parse_query(s.c_str(),Xapian::QueryParser::FLAG_DEFAULT)); 557 | delete (qp); 558 | if(item_neg) 559 | { 560 | q2 = new Xapian::Query(Xapian::Query::MatchAll); 561 | q3 = new Xapian::Query(Xapian::Query::OP_AND_NOT,*q2,*q); 562 | delete(q2); 563 | delete(q); 564 | q=q3; 565 | } 566 | } 567 | if(qsize<1) 568 | { 569 | if(q==NULL) q = new Xapian::Query(Xapian::Query::MatchNothing); 570 | return q; 571 | } 572 | 573 | if(q==NULL) 574 | { 575 | q=qs[0]->get_query(db); 576 | } 577 | else 578 | { 579 | q2 = new Xapian::Query(global_op,*q,*(qs[0]->get_query(db))); 580 | delete(q); 581 | q=q2; 582 | } 583 | for (int i=1;iget_query(db))); 586 | delete(q); 587 | q=q2; 588 | } 589 | return q; 590 | } 591 | }; 592 | 593 | class XDoc 594 | { 595 | private: 596 | std::vector * terms; 597 | std::vector * strings; 598 | std::vector * headers; 599 | struct xapian_fts_backend *backend; 600 | 601 | public: 602 | long uid; 603 | char * uterm; 604 | Xapian::Document * xdoc; 605 | long status; 606 | long status_n; 607 | long nterms,nlines,ndict; 608 | 609 | XDoc(struct xapian_fts_backend *b) 610 | { 611 | backend=b; 612 | uid=b->lastuid; 613 | 614 | std::string s="Q"+std::to_string(uid); 615 | uterm = (char*)malloc((s.length()+1)*sizeof(char)); 616 | strcpy(uterm,s.c_str()); 617 | 618 | strings = new std::vector; 619 | strings->clear(); 620 | headers = new std::vector; 621 | headers->clear(); 622 | terms = new std::vector; 623 | terms->clear(); 624 | nterms=0; nlines=0; ndict=0; 625 | 626 | xdoc=NULL; 627 | status=0; 628 | status_n=0; 629 | } 630 | 631 | ~XDoc() 632 | { 633 | for(icu::UnicodeString * t : *terms) 634 | { 635 | delete(t); 636 | } 637 | terms->clear(); delete(terms); 638 | 639 | headers->clear(); delete(headers); 640 | 641 | for(icu::UnicodeString * t : *strings) 642 | { 643 | delete(t); 644 | } 645 | strings->clear(); delete(strings); 646 | 647 | if(xdoc!=NULL) delete(xdoc); 648 | free(uterm); 649 | } 650 | 651 | std::string getDocSummary() 652 | { 653 | std::string s("Doc "); 654 | s.append(std::to_string(uid)); 655 | s.append(" uterm="); 656 | s.append(uterm); 657 | s.append(" #lines=" + std::to_string(nlines)); 658 | s.append(" #terms=" + std::to_string(nterms)); 659 | s.append(" #dict=" + std::to_string(ndict)); 660 | s.append(" status=" + std::to_string(status)); 661 | return s; 662 | } 663 | 664 | void raw_load(long h, const char *d, int32_t size, long verbose, const char * title) 665 | { 666 | icu::UnicodeString * t; 667 | { 668 | icu::StringPiece sp(d,size); 669 | t = new icu::UnicodeString(icu::UnicodeString::fromUTF8(sp)); 670 | } 671 | headers->push_back(h); 672 | strings->push_back(t); 673 | nlines++; 674 | } 675 | 676 | long terms_add(icu::UnicodeString * w,long pos, long l) 677 | { 678 | if(l==0) 679 | { 680 | terms->insert(terms->begin()+pos,new icu::UnicodeString(*w)); 681 | nterms++; 682 | return pos; 683 | } 684 | 685 | long n = std::floor(l*0.5f); 686 | int c = terms->at(pos+n)->compare(*w); 687 | 688 | // If already exist, return 689 | if(c==0) return pos; 690 | 691 | // If middle pos is lower than d, search after pos+n 692 | if(c<0) return terms_add(w,pos+n+1,l-n-1); 693 | 694 | // All other case, search before 695 | return terms_add(w,pos,n); 696 | } 697 | 698 | void terms_push(long h, icu::UnicodeString *t) 699 | { 700 | fts_backend_xapian_trim(t); 701 | unsigned long n = t->length(); 702 | long m = XAPIAN_TERM_SIZELIMIT - strlen(hdrs_xapian[h]) - 1; 703 | 704 | if(n>=fts_xapian_settings.partial) 705 | { 706 | t->truncate(m); 707 | while(fts_backend_xapian_icutochar_length(t)>=m) 708 | { 709 | t->truncate(t->length()-1); 710 | } 711 | fts_backend_xapian_sqlite3_dict_add(backend,h,t); 712 | ndict++; 713 | t->insert(0,hdrs_xapian[h]); 714 | terms_add(t,0,terms->size()); 715 | } 716 | delete(t); 717 | } 718 | 719 | bool terms_create(long verbose, const char * title) 720 | { 721 | icu::UnicodeString *t; 722 | long h; 723 | long k; 724 | 725 | while((terms->size()size()>0)) 726 | { 727 | h = headers->back(); headers->pop_back(); 728 | t = strings->back(); strings->pop_back(); 729 | 730 | fts_backend_xapian_clean(t); 731 | 732 | k = t->lastIndexOf(CHAR_SPACE); 733 | while(k>0) 734 | { 735 | terms_push(h,new icu::UnicodeString(*t,k+1)); 736 | t->truncate(k); 737 | fts_backend_xapian_trim(t); 738 | k = t->lastIndexOf(CHAR_SPACE); 739 | } 740 | terms_push(h,t); 741 | } 742 | return true; 743 | } 744 | 745 | bool doc_create(long verbose, const char * title) 746 | { 747 | if(verbose>0) syslog(LOG_INFO,"%s adding %ld terms",title,nterms); 748 | try 749 | { 750 | icu::UnicodeString *t; 751 | xdoc = new Xapian::Document(); 752 | xdoc->add_value(1,Xapian::sortable_serialise(uid)); 753 | xdoc->add_term(uterm); 754 | std::string s; 755 | long n = terms->size(); 756 | while(n>0) 757 | { 758 | n--; 759 | t=terms->back(); 760 | terms->pop_back(); 761 | fts_backend_xapian_icutostring(t,s); 762 | if(verbose>1) syslog(LOG_INFO,"%s adding terms for (%s) : %s",title,uterm,s.c_str()); 763 | xdoc->add_term(s.c_str()); 764 | delete(t); 765 | } 766 | } 767 | catch(Xapian::Error e) 768 | { 769 | return false; 770 | } 771 | return true; 772 | } 773 | }; 774 | 775 | static void fts_backend_xapian_worker(void *p); 776 | 777 | class XDocsWriter 778 | { 779 | private: 780 | XDoc * doc; 781 | long verbose, lowmemory; 782 | std::thread *t; 783 | char title[1000]; 784 | struct xapian_fts_backend *backend; 785 | public: 786 | bool started,toclose,terminated; 787 | bool err; 788 | char err_s[10000]; 789 | 790 | XDocsWriter(struct xapian_fts_backend *b, long n) 791 | { 792 | backend=b; 793 | 794 | sprintf(title,"DW #%ld (%s,%s) - ",n,backend->boxname,backend->xap_db); 795 | 796 | t=NULL; 797 | doc=NULL; 798 | toclose=false; 799 | terminated=false; 800 | started=false; 801 | verbose=fts_xapian_settings.verbose; 802 | lowmemory = fts_xapian_settings.lowmemory; 803 | err=false; 804 | err_s[0]=0; 805 | } 806 | 807 | bool checkDB() 808 | { 809 | if(backend->dbw != NULL) return true; 810 | 811 | backend->pending=0; 812 | 813 | try 814 | { 815 | if(verbose>0) syslog(LOG_INFO,"%sOpening DB (RW)",title); 816 | backend->dbw = new Xapian::WritableDatabase(backend->xap_db,Xapian::DB_CREATE_OR_OPEN | Xapian::DB_BACKEND_GLASS); 817 | return true; 818 | } 819 | catch(Xapian::DatabaseLockError e) 820 | { 821 | syslog(LOG_WARNING,"%sCan't lock the DB : %s - %s",title,e.get_type(),e.get_msg().c_str()); 822 | } 823 | catch(Xapian::Error e) 824 | { 825 | syslog(LOG_WARNING,"%sCan't open the DB RW : %s - %s",title,e.get_type(),e.get_msg().c_str()); 826 | } 827 | return false; 828 | } 829 | 830 | void close() 831 | { 832 | toclose=true; 833 | if(t!=NULL) 834 | { 835 | t->join(); 836 | delete(t); 837 | } 838 | t=NULL; 839 | terminated=true; 840 | } 841 | 842 | ~XDocsWriter() 843 | { 844 | close(); 845 | } 846 | 847 | std::string getSummary() 848 | { 849 | std::string s(title); 850 | s.append(" queued_docs="+std::to_string(backend->docs.size())); 851 | s.append(" dict_size="+std::to_string(backend->dict_nb)); 852 | s.append(" terminated="+std::to_string(terminated)); 853 | return s; 854 | } 855 | 856 | bool launch(const char * from) 857 | { 858 | if(verbose>0) 859 | { 860 | std::string s(title); 861 | s.append("Launching thread from "); 862 | s.append(from); 863 | syslog(LOG_INFO,"%s",s.c_str()); 864 | } 865 | 866 | try 867 | { 868 | t = new std::thread(fts_backend_xapian_worker,this); 869 | } 870 | catch(std::exception const& e) 871 | { 872 | std::string s(title); 873 | s.append("Thread error "); 874 | s.append(e.what()); 875 | syslog(LOG_ERR,"%s",s.c_str()); 876 | t = NULL; 877 | return false; 878 | } 879 | started=true; 880 | return true; 881 | } 882 | 883 | long checkMemory() 884 | { 885 | // Memory check 886 | long m = fts_backend_xapian_get_free_memory(verbose); 887 | if(verbose>1) syslog(LOG_WARNING,"%sMemory : Free = %ld MB vs %ld limit | Pendings in cache = %ld / %ld | Dict size = %ld / %ld",title,(long)(m / 1024.0f),lowmemory,backend->pending,XAPIAN_WRITING_CACHE,backend->dict_nb,XAPIAN_DICT_MAX); 888 | // First clean dictionnary 889 | if((backend->dict_nb > XAPIAN_DICT_MAX) || ((m>0) && (m<(lowmemory*1024)))) 890 | { 891 | if(!fts_backend_xapian_sqlite3_dict_flush(backend,verbose,err_s)) err=true; 892 | m = fts_backend_xapian_get_free_memory(verbose); 893 | } 894 | 895 | if((backend->dbw!=NULL) && ((backend->pending > XAPIAN_WRITING_CACHE) || ((m>0) && (m<(lowmemory*1024))))) // too little memory or too many pendings 896 | { 897 | fts_backend_xapian_get_lock(backend, verbose, title); 898 | 899 | // Repeat test because the close may have happen in another thread 900 | m = fts_backend_xapian_get_free_memory(verbose); 901 | if((backend->dbw!=NULL) && ((backend->pending > XAPIAN_WRITING_CACHE) || ((m>0) && (m<(lowmemory*1024))))) 902 | { 903 | try 904 | { 905 | if(backend->pending > XAPIAN_WRITING_CACHE) 906 | { 907 | syslog(LOG_WARNING,"%sCommitting %ld docs due to cached docs exceeded (%ld vs %ld limit)",title,backend->pending,backend->pending,XAPIAN_WRITING_CACHE); 908 | } 909 | else 910 | { 911 | syslog(LOG_WARNING,"%sCommitting %ld docs due to low free memory (%ld MB vs %ld MB)",title,backend->pending,(long)(m/1024.0f),lowmemory); 912 | } 913 | backend->dbw->close(); 914 | delete(backend->dbw); 915 | if(verbose>0) syslog(LOG_INFO,"%sClosed Xapian DB %s",title,backend->xap_db); 916 | backend->dbw = NULL; 917 | backend->pending = 0; 918 | } 919 | catch(Xapian::Error e) 920 | { 921 | sprintf(err_s,"%sCan't commit DB1 : %s - %s",title,e.get_type(),e.get_msg().c_str()); 922 | syslog(LOG_ERR,"%s",err_s); 923 | err=true; 924 | } 925 | catch(std::exception const& e) 926 | { 927 | sprintf(err_s,"%sCan't commit DB2 : %s",title,e.what()); 928 | syslog(LOG_ERR,"%s",err_s); 929 | err=true; 930 | } 931 | } 932 | fts_backend_xapian_release_lock(backend, verbose, title); 933 | } 934 | return m; 935 | } 936 | 937 | void worker() 938 | { 939 | long start_time = fts_backend_xapian_current_time(); 940 | XDoc *doc = NULL; 941 | long totaldocs=0; 942 | long sl=0, dt=0; 943 | 944 | while((!err) && ((!toclose) || (doc!=NULL))) 945 | { 946 | if(doc==NULL) 947 | { 948 | if(verbose>0) syslog(LOG_INFO,"%sSearching doc",title); 949 | 950 | fts_backend_xapian_get_lock(backend, verbose, title); 951 | if((backend->docs.size()>0) && (backend->docs.back()->status==1)) 952 | { 953 | doc = backend->docs.back(); 954 | backend->docs.pop_back(); 955 | dt=fts_backend_xapian_current_time(); 956 | } 957 | fts_backend_xapian_release_lock(backend, verbose, title); 958 | } 959 | 960 | if(doc==NULL) 961 | { 962 | sl++; 963 | if((sl>50) && (verbose>0)) 964 | { 965 | syslog(LOG_INFO,"%sNoop",title); 966 | sl=0; 967 | } 968 | std::this_thread::sleep_for(XAPIAN_SLEEP); 969 | } 970 | else if(doc->status==1) 971 | { 972 | checkMemory(); 973 | if(verbose>0) syslog(LOG_INFO,"%sPopulating stems : %s",title,doc->getDocSummary().c_str()); 974 | if(doc->terms_create(verbose,title)) 975 | { 976 | doc->status=2; doc->status_n=0; 977 | if(verbose>0) syslog(LOG_INFO,"%sPopulating stems : %ld done in %ld msec",title,doc->nterms,fts_backend_xapian_current_time()-dt); 978 | dt=fts_backend_xapian_current_time(); 979 | } 980 | else 981 | { 982 | doc->status_n++; 983 | if(verbose>0) syslog(LOG_INFO,"%sPopulating stems : Error - %s",title,doc->getDocSummary().c_str()); 984 | if(doc->status_n > XAPIAN_MAX_ERRORS) 985 | { 986 | delete(doc); 987 | doc=NULL; 988 | } 989 | } 990 | } 991 | else if(doc->status==2) 992 | { 993 | checkMemory(); 994 | if(verbose>0) syslog(LOG_INFO,"%sCreating Xapian doc : %s",title,doc->getDocSummary().c_str()); 995 | if(doc->doc_create(verbose,title)) 996 | { 997 | doc->status=3; 998 | doc->status_n=0; 999 | if(verbose>0) syslog(LOG_INFO,"%sCreating Xapian doc : Done in %ld msec",title,fts_backend_xapian_current_time()-dt); 1000 | dt=fts_backend_xapian_current_time(); 1001 | } 1002 | else 1003 | { 1004 | doc->status_n++; 1005 | if(verbose>0) syslog(LOG_INFO,"%sCreate document : Error",title); 1006 | if(doc->status_n > XAPIAN_MAX_ERRORS) 1007 | { 1008 | delete(doc); 1009 | doc=NULL; 1010 | } 1011 | } 1012 | } 1013 | else 1014 | { 1015 | if(verbose>0) syslog(LOG_INFO,"%sPushing : %s",title,doc->getDocSummary().c_str()); 1016 | if(doc->nterms > 0) 1017 | { 1018 | checkMemory(); 1019 | fts_backend_xapian_get_lock(backend, verbose, title); 1020 | if(checkDB() && (!err)) 1021 | { 1022 | try 1023 | { 1024 | backend->dbw->replace_document(doc->uterm,*(doc->xdoc)); 1025 | backend->pending++; 1026 | backend->total_docs++; 1027 | delete(doc); 1028 | doc=NULL; 1029 | if(verbose>0) syslog(LOG_INFO,"%sPushing done in %ld msec",title,fts_backend_xapian_current_time()-dt); 1030 | totaldocs++; 1031 | } 1032 | catch(Xapian::Error e) 1033 | { 1034 | sprintf(err_s,"%sCan't write doc1 %s : %s - %s",title,doc->getDocSummary().c_str(),e.get_type(),e.get_msg().c_str()); 1035 | syslog(LOG_ERR,"%s",err_s); 1036 | err=true; 1037 | } 1038 | catch(std::exception const & e) 1039 | { 1040 | sprintf(err_s,"%sCan't write doc2 %s : %s",title,doc->getDocSummary().c_str(),e.what()); 1041 | syslog(LOG_ERR,"%s",err_s); 1042 | err=true; 1043 | } 1044 | } 1045 | fts_backend_xapian_release_lock(backend, verbose, title); 1046 | } 1047 | else 1048 | { 1049 | delete(doc); 1050 | doc=NULL; 1051 | } 1052 | } 1053 | } 1054 | 1055 | if(doc!=NULL) 1056 | { 1057 | delete(doc); 1058 | doc=NULL; 1059 | } 1060 | terminated=true; 1061 | if((verbose>0) && (!err)) 1062 | { 1063 | syslog(LOG_INFO,"%sIndexed %ld docs within %ld msec",title,totaldocs,fts_backend_xapian_current_time() - start_time); 1064 | } 1065 | } 1066 | }; 1067 | 1068 | static void fts_backend_xapian_worker(void *p) 1069 | { 1070 | XDocsWriter *xw = (XDocsWriter *)p; 1071 | xw->worker(); 1072 | } 1073 | 1074 | static bool fts_backend_xapian_open_readonly(struct xapian_fts_backend *backend, Xapian::Database ** dbr) 1075 | { 1076 | if(fts_xapian_settings.verbose>1) i_info("FTS Xapian: fts_backend_xapian_open_readonly"); 1077 | 1078 | if((backend->xap_db == NULL) || (strlen(backend->xap_db)<1)) 1079 | { 1080 | i_warning("FTS Xapian: Open DB Read Only : no DB name"); 1081 | return false; 1082 | } 1083 | 1084 | try 1085 | { 1086 | if(fts_xapian_settings.verbose>0) i_info("FTS Xapian: Opening DB (RO) %s",backend->xap_db); 1087 | *dbr = new Xapian::Database(backend->xap_db,Xapian::DB_CREATE_OR_OPEN | Xapian::DB_BACKEND_GLASS); 1088 | } 1089 | catch(Xapian::Error e) 1090 | { 1091 | i_error("FTS Xapian: Can not open RO index (%s) %s : %s - %s %s ",backend->boxname,backend->xap_db,e.get_type(),e.get_msg().c_str(),e.get_error_string()); 1092 | return false; 1093 | } 1094 | return true; 1095 | } 1096 | 1097 | static void fts_backend_xapian_oldbox(struct xapian_fts_backend *backend) 1098 | { 1099 | if(fts_xapian_settings.verbose>1) i_info("FTS Xapian: fts_backend_xapian_oldbox"); 1100 | 1101 | if(backend->old_guid != NULL) 1102 | { 1103 | /* Performance calculator*/ 1104 | long dt = fts_backend_xapian_current_time() - backend->start_time; 1105 | double r=0; 1106 | if(dt>0) 1107 | { 1108 | r=backend->total_docs*1000.0; 1109 | r=r/dt; 1110 | } 1111 | /* End Performance calculator*/ 1112 | 1113 | if(fts_xapian_settings.verbose>0) i_info("FTS Xapian: Done indexing '%s' (%s) (%ld msgs in %ld msec, rate: %.1f)",backend->old_boxname, backend->xap_db,backend->total_docs,dt,r); 1114 | 1115 | i_free(backend->old_guid); backend->old_guid = NULL; 1116 | i_free(backend->old_boxname); backend->old_boxname = NULL; 1117 | } 1118 | 1119 | if(fts_xapian_settings.verbose>1) i_info("FTS Xapian: fts_backend_xapian_oldbox - done"); 1120 | } 1121 | 1122 | static void fts_backend_xapian_close_db(Xapian::WritableDatabase * dbw,const char * dbpath,const char * boxname, long verbose) 1123 | { 1124 | long t; 1125 | 1126 | if(verbose>0) 1127 | { 1128 | t = fts_backend_xapian_current_time(); 1129 | syslog(LOG_INFO,"FTS Xapian : Closing DB (%s,%s)",boxname,dbpath); 1130 | } 1131 | try 1132 | { 1133 | dbw->close(); 1134 | delete(dbw); 1135 | } 1136 | catch(Xapian::Error e) 1137 | { 1138 | syslog(LOG_ERR, "FTS Xapian: Can't close Xapian DB (%s) %s : %s - %s %s",boxname,dbpath,e.get_type(),e.get_msg().c_str(),e.get_error_string()); 1139 | } 1140 | catch(std::exception const& e) 1141 | { 1142 | syslog(LOG_ERR, "FTS Xapian : Closing db (%s) error %s",dbpath,e.what()); 1143 | } 1144 | 1145 | if(verbose>0) 1146 | { 1147 | t = fts_backend_xapian_current_time()-t; 1148 | syslog(LOG_INFO,"FTS Xapian : DB (%s,%s) closed in %ld ms",boxname,dbpath,t); 1149 | } 1150 | } 1151 | 1152 | static void fts_backend_xapian_close(struct xapian_fts_backend *backend, const char * purpose) 1153 | { 1154 | char reason[10000]; 1155 | 1156 | bool err=false; 1157 | for(auto & xwr : backend->threads) 1158 | { 1159 | if(xwr->err) 1160 | { 1161 | err=true; 1162 | strcpy(reason,xwr->err_s); 1163 | break; 1164 | } 1165 | } 1166 | 1167 | if(!err) strcpy(reason,purpose); 1168 | 1169 | if(fts_xapian_settings.verbose>0) i_info("FTS Xapian : Closing all DWs (%s)",reason); 1170 | 1171 | if(err) 1172 | { 1173 | for(auto & xwr : backend->threads) 1174 | { 1175 | xwr->err=true; 1176 | xwr->toclose=true; 1177 | } 1178 | 1179 | fts_backend_xapian_get_lock(backend,fts_xapian_settings.verbose,reason); 1180 | while(backend->docs.size()>0) 1181 | { 1182 | XDoc * doc = backend->docs.back(); 1183 | backend->docs.pop_back(); 1184 | delete(doc); 1185 | } 1186 | fts_backend_xapian_release_lock(backend,fts_xapian_settings.verbose,reason); 1187 | 1188 | struct stat sb; 1189 | if((stat(backend->version_file, &sb)==0) && S_ISREG(sb.st_mode)) 1190 | { 1191 | std::filesystem::remove(backend->version_file); 1192 | } 1193 | } 1194 | else 1195 | { 1196 | fts_backend_xapian_get_lock(backend,fts_xapian_settings.verbose,reason); 1197 | if((backend->docs.size()>0) && (backend->docs.front()->status<1)) backend->docs.front()->status=1; 1198 | fts_backend_xapian_release_lock(backend,fts_xapian_settings.verbose,reason); 1199 | 1200 | long n=0; 1201 | while(backend->docs.size()>0) 1202 | { 1203 | n++; 1204 | if((n>50) and (fts_xapian_settings.verbose>0)) 1205 | { 1206 | i_info("FTS Xapian: Waiting for all pending documents (%ld) to be processed (Sleep5) with %ld threads",backend->docs.size(),backend->threads.size()); 1207 | n=0; 1208 | } 1209 | std::this_thread::sleep_for(XAPIAN_SLEEP); 1210 | } 1211 | 1212 | for(auto & xwr : backend->threads) 1213 | { 1214 | xwr->toclose=true; 1215 | } 1216 | } 1217 | 1218 | XDocsWriter * xw; 1219 | long n=0; 1220 | while(backend->threads.size()>0) 1221 | { 1222 | xw = backend->threads.back(); 1223 | 1224 | if(!(xw->started)) 1225 | { 1226 | if(fts_xapian_settings.verbose>0) i_info("FTS Xapian : Closing thread because not started : %s",xw->getSummary().c_str()); 1227 | delete(xw); 1228 | backend->threads.pop_back(); 1229 | } 1230 | else if(xw->terminated) 1231 | { 1232 | if(fts_xapian_settings.verbose>0) i_info("FTS Xapian : Closing thread because terminated : %s",xw->getSummary().c_str()); 1233 | delete(xw); 1234 | backend->threads.pop_back(); 1235 | } 1236 | else 1237 | { 1238 | n++; 1239 | if((n>50) && (fts_xapian_settings.verbose>0)) 1240 | { 1241 | for(auto & xwr : backend->threads) 1242 | { 1243 | if((xwr!=NULL)&&(!(xwr->terminated))) i_info("FTS Xapian : Waiting (Sleep4) for thread %s",xwr->getSummary().c_str()); 1244 | } 1245 | n=0; 1246 | } 1247 | std::this_thread::sleep_for(XAPIAN_SLEEP); 1248 | } 1249 | } 1250 | if(fts_xapian_settings.verbose>0) i_info("FTS Xapian : All DWs (%s) closed",reason); 1251 | 1252 | if(!err) fts_backend_xapian_sqlite3_dict_flush(backend,fts_xapian_settings.verbose); 1253 | 1254 | sqlite3_close(backend->ddb); 1255 | backend->ddb = NULL; 1256 | 1257 | if(backend->dbw!=NULL) 1258 | { 1259 | fts_backend_xapian_close_db(backend->dbw,backend->xap_db,backend->boxname,fts_xapian_settings.verbose); 1260 | backend->dbw=NULL; 1261 | } 1262 | } 1263 | 1264 | XResultSet * fts_backend_xapian_query(Xapian::Database * dbx, XQuerySet * query, long limit=0) 1265 | { 1266 | if(fts_xapian_settings.verbose>0) i_info("FTS Xapian: fts_backend_xapian_query (%s)",query->get_string().c_str()); 1267 | 1268 | XResultSet * set= new XResultSet(); 1269 | Xapian::Query * q = query->get_query(dbx); 1270 | 1271 | try 1272 | { 1273 | Xapian::Enquire enquire(*dbx); 1274 | enquire.set_query(*q); 1275 | enquire.set_docid_order(Xapian::Enquire::DESCENDING); 1276 | 1277 | long offset=0; 1278 | long pagesize=100; if(limit>0) { pagesize=std::min(pagesize,limit); } 1279 | Xapian::MSet m = enquire.get_mset(0, pagesize); 1280 | while(m.size()>0) 1281 | { 1282 | Xapian::MSetIterator i = m.begin(); 1283 | while (i != m.end()) 1284 | { 1285 | Xapian::Document doc = i.get_document(); 1286 | set->add(doc.get_docid()); 1287 | i++; 1288 | } 1289 | offset+=pagesize; 1290 | m = enquire.get_mset(offset, pagesize); 1291 | } 1292 | } 1293 | catch(Xapian::Error e) 1294 | { 1295 | i_error("FTS Xapian: xapian_query %s - %s %s",e.get_type(),e.get_msg().c_str(),e.get_error_string()); 1296 | } 1297 | delete(q); 1298 | return set; 1299 | } 1300 | 1301 | static int fts_backend_xapian_unset_box(struct xapian_fts_backend *backend) 1302 | { 1303 | if(fts_xapian_settings.verbose>1) i_info("FTS Xapian: Unset box '%s' (%s)",backend->boxname,backend->guid); 1304 | 1305 | fts_backend_xapian_close(backend,"unset box"); 1306 | fts_backend_xapian_oldbox(backend); 1307 | 1308 | if(backend->xap_db != NULL) 1309 | { 1310 | i_free(backend->xap_db); 1311 | backend->xap_db = NULL; 1312 | 1313 | i_free(backend->guid); 1314 | backend->guid = NULL; 1315 | 1316 | i_free(backend->boxname); 1317 | backend->boxname = NULL; 1318 | 1319 | i_free(backend->exp_db); 1320 | backend->exp_db = NULL; 1321 | 1322 | i_free(backend->dict_db); 1323 | backend->dict_db = NULL; 1324 | 1325 | i_free(backend->version_file); 1326 | backend->version_file = NULL; 1327 | } 1328 | 1329 | return 0; 1330 | } 1331 | 1332 | static int fts_backend_xapian_set_path(struct xapian_fts_backend *backend) 1333 | { 1334 | struct mail_namespace * ns = backend->backend.ns; 1335 | if(ns->alias_for != NULL) 1336 | { 1337 | if(fts_xapian_settings.verbose>1) i_info("FTS Xapian: Switching namespace"); 1338 | ns = ns->alias_for; 1339 | } 1340 | 1341 | const char * path = mailbox_list_get_root_forced(ns->list, MAILBOX_LIST_PATH_TYPE_INDEX); 1342 | 1343 | if(backend->path != NULL) i_free(backend->path); 1344 | backend->path = i_strconcat(path, "/" XAPIAN_FILE_PREFIX, static_cast(NULL)); 1345 | 1346 | if(fts_xapian_settings.verbose>1) i_info("FTS Xapian: Index path = %s",backend->path); 1347 | 1348 | struct stat sb; 1349 | if(!( (stat(backend->path, &sb)==0) && S_ISDIR(sb.st_mode))) 1350 | { 1351 | if (mailbox_list_mkdir_root(backend->backend.ns->list, backend->path, MAILBOX_LIST_PATH_TYPE_INDEX) < 0) 1352 | { 1353 | i_error("FTS Xapian: can not create '%s'",backend->path); 1354 | i_error("FTS Xapian: Hint => You need to set mail_uid and mail_gid in your dovecot.conf according to the user of mail_location (%s)", path); 1355 | return -1; 1356 | } 1357 | } 1358 | return 0; 1359 | } 1360 | 1361 | static int fts_backend_xapian_set_box(struct xapian_fts_backend *backend, struct mailbox *box) 1362 | { 1363 | if (box == NULL) 1364 | { 1365 | if(backend->guid != NULL) fts_backend_xapian_unset_box(backend); 1366 | if(fts_xapian_settings.verbose>0) i_warning("FTS Xapian: Box is empty"); 1367 | return 0; 1368 | } 1369 | 1370 | const char * mb; 1371 | fts_mailbox_get_guid(box, &mb ); 1372 | 1373 | if(fts_xapian_settings.verbose>0) i_info("FTS Xapian: Set box '%s' (%s)",box->name,mb); 1374 | 1375 | if((mb == NULL) || (strlen(mb)<3)) 1376 | { 1377 | i_error("FTS Xapian: Invalid box"); 1378 | return -1; 1379 | } 1380 | 1381 | if((backend->guid != NULL) && (strcmp(mb,backend->guid)==0)) 1382 | { 1383 | if(fts_xapian_settings.verbose>1) i_info("FTS Xapian: Box is unchanged"); 1384 | return 0; 1385 | } 1386 | 1387 | if(backend->guid != NULL) fts_backend_xapian_unset_box(backend); 1388 | 1389 | if(fts_backend_xapian_set_path(backend)<0) return -1; 1390 | 1391 | long current_time = fts_backend_xapian_current_time(); 1392 | 1393 | backend->start_time = current_time; 1394 | backend->lastuid = -1; 1395 | backend->guid = i_strdup(mb); 1396 | backend->boxname = i_strdup(box->name); 1397 | backend->xap_db = i_strdup_printf("%s/db_%s",backend->path,mb); 1398 | backend->exp_db = i_strdup_printf("%s%s",backend->xap_db,suffixExp); 1399 | backend->dict_db = i_strdup_printf("%s%s",backend->xap_db,suffixDict); 1400 | backend->version_file = i_strdup_printf("%s_v%s",backend->xap_db,XAPIAN_PLUGIN_VERSION); 1401 | 1402 | struct stat sb; 1403 | // Existence of current version 1404 | if(!( (stat(backend->version_file, &sb)==0) && S_ISREG(sb.st_mode))) 1405 | { 1406 | i_info("FTS Xapian: New version of the plugin : %s for %s",XAPIAN_PLUGIN_VERSION,backend->boxname); 1407 | 1408 | // Deleting existing indexes 1409 | std::filesystem::remove_all(backend->xap_db); 1410 | for(auto& f : std::filesystem::directory_iterator(backend->path)) 1411 | { 1412 | if((f.is_regular_file()) && (f.path().string().find(backend->xap_db) == 0)) 1413 | { 1414 | if(fts_xapian_settings.verbose>0) i_warning("FTS Xapian: Deleting %s",f.path().c_str()); 1415 | std::filesystem::remove(f.path()); 1416 | } 1417 | } 1418 | 1419 | // Write new version file 1420 | FILE * f = fopen(backend->version_file,"w+"); 1421 | fprintf(f,"%s",XAPIAN_PLUGIN_VERSION); 1422 | fclose(f); 1423 | } 1424 | 1425 | // Verify existence of Dict db 1426 | if(!( (stat(backend->dict_db, &sb)==0) && S_ISREG(sb.st_mode))) 1427 | { 1428 | i_warning("FTS Xapian: '%s' (%s) dictionnary does not exist. Creating it",backend->boxname,backend->dict_db); 1429 | if(fts_backend_xapian_sqlite3_dict_open(backend)) sqlite3_close(backend->ddb); 1430 | backend->ddb = NULL; 1431 | std::filesystem::remove_all(backend->xap_db); 1432 | } 1433 | 1434 | // Verify existence of Xapian db 1435 | { 1436 | char * t = i_strdup_printf("%s/termlist.glass",backend->xap_db); 1437 | if(!( (stat(t, &sb)==0) && S_ISREG(sb.st_mode))) 1438 | { 1439 | std::filesystem::remove(backend->exp_db); 1440 | i_info("FTS Xapian: '%s' (%s) indexes do not exist. Initializing DB",backend->boxname,backend->xap_db); 1441 | try 1442 | { 1443 | Xapian::WritableDatabase * db = new Xapian::WritableDatabase(backend->xap_db,Xapian::DB_CREATE_OR_OVERWRITE | Xapian::DB_BACKEND_GLASS); 1444 | db->close(); 1445 | delete(db); 1446 | } 1447 | catch(Xapian::Error e) 1448 | { 1449 | i_error("FTS Xapian: Can't create Xapian DB (%s) %s : %s - %s %s",backend->boxname,backend->xap_db,e.get_type(),e.get_msg().c_str(),e.get_error_string()); 1450 | } 1451 | } 1452 | i_free(t); 1453 | } 1454 | 1455 | // Verify existence of Exp db 1456 | if(!( (stat(backend->exp_db, &sb)==0) && S_ISREG(sb.st_mode))) 1457 | { 1458 | i_warning("FTS Xapian: '%s' (%s) expunge does not exist. Creating it",backend->boxname,backend->exp_db); 1459 | sqlite3 * db = NULL; 1460 | if(sqlite3_open_v2(backend->exp_db,&db,SQLITE_OPEN_FULLMUTEX | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE,NULL) != SQLITE_OK ) 1461 | { 1462 | i_error("FTS Xapian: Can not open %s : %s",backend->exp_db,sqlite3_errmsg(db)); 1463 | } 1464 | else 1465 | { 1466 | char *zErrMsg = 0; 1467 | if(sqlite3_exec(db,createExpTable,NULL,0,&zErrMsg) != SQLITE_OK ) 1468 | { 1469 | i_error("FTS Xapian: Can not execute (%s) : %s",createExpTable,zErrMsg); 1470 | if(zErrMsg!=NULL) sqlite3_free(zErrMsg); 1471 | } 1472 | sqlite3_close(db); 1473 | } 1474 | } 1475 | 1476 | backend->threads.clear(); 1477 | backend->total_docs =0; 1478 | 1479 | return 0; 1480 | } 1481 | 1482 | static void fts_backend_xapian_build_qs(XQuerySet * qs, struct mail_search_arg *a, const char * dict=NULL) 1483 | { 1484 | long hdr; 1485 | 1486 | if(fts_xapian_settings.verbose>1) i_info("FTS Xapian: fts_backend_xapian_build_qs"); 1487 | 1488 | while(a != NULL) 1489 | { 1490 | switch (a->type) 1491 | { 1492 | case SEARCH_TEXT: hdr = -1; break; 1493 | case SEARCH_BODY: hdr = 8; break; 1494 | case SEARCH_HEADER: 1495 | case SEARCH_HEADER_ADDRESS: 1496 | case SEARCH_HEADER_COMPRESS_LWSP: 1497 | if((a->hdr_field_name == NULL)||(strlen(a->hdr_field_name)<1)) 1498 | { 1499 | hdr = -1; 1500 | break; 1501 | } 1502 | hdr=fts_backend_xapian_clean_header(a->hdr_field_name); 1503 | if(hdr >= 0) break; 1504 | default: a = a->next; continue; 1505 | } 1506 | 1507 | if((a->value.str == NULL) || (strlen(a->value.str)<1)) 1508 | { 1509 | XQuerySet * q2; 1510 | if(a->match_not) 1511 | { 1512 | q2 = new XQuerySet(Xapian::Query::OP_AND_NOT,qs->limit); 1513 | } 1514 | else 1515 | { 1516 | q2 = new XQuerySet(Xapian::Query::OP_OR,qs->limit); 1517 | } 1518 | fts_backend_xapian_build_qs(q2,a->value.subargs,dict); 1519 | if(q2->count()>0) 1520 | { 1521 | qs->add(q2); 1522 | } 1523 | else 1524 | { 1525 | delete(q2); 1526 | } 1527 | } 1528 | else if(dict != NULL) 1529 | { 1530 | // Find key words 1531 | icu::StringPiece sp(a->value.str); 1532 | icu::UnicodeString t = icu::UnicodeString::fromUTF8(sp); 1533 | fts_backend_xapian_clean(&t); 1534 | long j, i = t.lastIndexOf(CHAR_SPACE); 1535 | unsigned long l; 1536 | icu::UnicodeString *k; 1537 | std::vector keys; keys.clear(); 1538 | while(i>0) 1539 | { 1540 | j = t.length(); 1541 | k = new icu::UnicodeString(t,i+1,j-i-1); 1542 | l = k->length(); 1543 | if(l >= fts_xapian_settings.partial) { keys.push_back(k); } else delete(k); 1544 | t.truncate(i); 1545 | fts_backend_xapian_trim(&t); 1546 | i = t.lastIndexOf(CHAR_SPACE); 1547 | } 1548 | l = t.length(); 1549 | if(l>=fts_xapian_settings.partial) 1550 | { 1551 | keys.push_back(new icu::UnicodeString (t)); 1552 | } 1553 | 1554 | // For each key, search dictionnary 1555 | sqlite3 * db = NULL; 1556 | char * zErrMsg =0; 1557 | if(sqlite3_open_v2(dict,&db,SQLITE_OPEN_FULLMUTEX | SQLITE_OPEN_READONLY,NULL) != SQLITE_OK ) 1558 | { 1559 | syslog(LOG_ERR,"FTS Xapian: Can not open %s : %s",dict,sqlite3_errmsg(db)); 1560 | return; 1561 | } 1562 | 1563 | // Generate query 1564 | XQuerySet * q1, *q2; 1565 | if(a->match_not) 1566 | { 1567 | q1 = new XQuerySet(Xapian::Query::OP_AND_NOT,qs->limit); 1568 | } 1569 | else 1570 | { 1571 | q1 = new XQuerySet(Xapian::Query::OP_AND,qs->limit); 1572 | } 1573 | for(auto & ki : keys) 1574 | { 1575 | std::vector st; st.clear(); 1576 | std::string sql=searchDict1; 1577 | ki->toUTF8String(sql); 1578 | if(hdr<0) 1579 | { 1580 | sql+="%'"; 1581 | } 1582 | else 1583 | { 1584 | sql+="%' and header=" + std::to_string(hdr); 1585 | } 1586 | sql +=searchDict2; 1587 | 1588 | zErrMsg =0; 1589 | if(sqlite3_exec(db,sql.c_str(),fts_backend_xapian_sqlite3_vector_icu,&st,&zErrMsg) != SQLITE_OK ) 1590 | { 1591 | syslog(LOG_ERR,"FTS Xapian: Can not search keyword (%s) : %s",sql.c_str(),zErrMsg); 1592 | if(zErrMsg!=NULL) sqlite3_free(zErrMsg); 1593 | } 1594 | q2 = new XQuerySet(Xapian::Query::OP_OR,qs->limit); 1595 | for(auto &term : st) 1596 | { 1597 | q2->add(hdr,term,false); 1598 | delete(term); 1599 | } 1600 | if(q2->count()>0) { q1->add(q2); } else { delete(q2); } 1601 | delete(ki); 1602 | } 1603 | qs->add(q1); 1604 | sqlite3_close(db); 1605 | } 1606 | else 1607 | { 1608 | icu::StringPiece sp(a->value.str); 1609 | icu::UnicodeString t = icu::UnicodeString::fromUTF8(sp); 1610 | fts_backend_xapian_clean(&t); 1611 | 1612 | qs->add(hdr,&t,a->match_not); 1613 | } 1614 | a->match_always=true; 1615 | a = a->next; 1616 | } 1617 | } 1618 | --------------------------------------------------------------------------------