├── .gitignore ├── docker-compose.yml ├── LICENSE ├── Dockerfile ├── entrypoint.sh ├── configs ├── sphinx_pgsql.conf └── sphinx.conf ├── test.sh └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | sphinx: 2 | image: romeoz/docker-sphinxsearch 3 | container_name: sphinx_test 4 | net: sphinx_net 5 | restart: always 6 | db: 7 | image: romeoz/docker-mysql 8 | container_name: db_test 9 | net: sphinx_net 10 | environment: 11 | - MYSQL_USER=admin 12 | - MYSQL_PASS=pass 13 | - MYSQL_CACHE_ENABLED=true 14 | - DB_NAME=db_test 15 | restart: always -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 RomeOz 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:xenial 2 | MAINTAINER romeOz 3 | 4 | ENV OS_LOCALE="en_US.UTF-8" \ 5 | OS_LANGUAGE="en_US:en" \ 6 | SPHINX_LOG_DIR=/var/log/sphinxsearch \ 7 | SPHINX_CONF=/etc/sphinxsearch/sphinx.conf \ 8 | SPHINX_RUN=/run/sphinxsearch/searchd.pid \ 9 | SPHINX_DATA_DIR=/var/lib/sphinxsearch/data 10 | 11 | # Set the locale 12 | RUN locale-gen ${OS_LOCALE} 13 | ENV LANG=${OS_LOCALE} \ 14 | LANGUAGE=${OS_LANGUAGE} \ 15 | LC_ALL=${OS_LOCALE} 16 | 17 | COPY ./entrypoint.sh /sbin/entrypoint.sh 18 | 19 | RUN buildDeps='software-properties-common python-software-properties' \ 20 | && apt-get update && apt-get install -y $buildDeps --no-install-recommends \ 21 | && add-apt-repository -y ppa:builds/sphinxsearch-rel22 \ 22 | && apt-get update \ 23 | && apt-get install -y sudo sphinxsearch \ 24 | && mv -f /etc/sphinxsearch/sphinx.conf /etc/sphinxsearch/origin.sphinx.conf \ 25 | && apt-get purge -y --auto-remove $buildDeps \ 26 | && rm -rf /var/lib/apt/lists/* \ 27 | && chmod 755 /sbin/entrypoint.sh \ 28 | # Forward sphinx logs to docker log collector 29 | && ln -sf /dev/stdout ${SPHINX_LOG_DIR}/searchd.log \ 30 | && ln -sf /dev/stdout ${SPHINX_LOG_DIR}/query.log 31 | 32 | COPY ./configs/* /etc/sphinxsearch/ 33 | 34 | EXPOSE 9312 9306 35 | VOLUME ["${SPHINX_DATA_DIR}"] 36 | ENTRYPOINT ["/sbin/entrypoint.sh"] -------------------------------------------------------------------------------- /entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | SPHINX_MODE=${SPHINX_MODE:-} 5 | SPHINX_BACKUP_DIR=${SPHINX_BACKUP_DIR:-"/tmp/backup"} 6 | SPHINX_BACKUP_FILENAME=${SPHINX_BACKUP_FILENAME:-"backup.last.tar.gz"} 7 | SPHINX_RESTORE=${SPHINX_RESTORE:-} 8 | SPHINX_CHECK=${SPHINX_CHECK:-} 9 | INDEX_NAME=${INDEX_NAME:-} 10 | SPHINX_ROTATE_BACKUP=${SPHINX_ROTATE_BACKUP:-true} 11 | 12 | create_backup_dir() { 13 | if [[ ! -d ${SPHINX_BACKUP_DIR}/ ]]; then 14 | mkdir -p ${SPHINX_BACKUP_DIR}/ 15 | fi 16 | chmod -R 0755 ${SPHINX_BACKUP_DIR} 17 | } 18 | 19 | create_data_dir() { 20 | mkdir -p ${SPHINX_DATA_DIR} 21 | chmod -R 0700 ${SPHINX_DATA_DIR} 22 | } 23 | 24 | rotate_backup() 25 | { 26 | echo "Rotate backup..." 27 | 28 | if [[ ${SPHINX_ROTATE_BACKUP} == true ]]; then 29 | WEEK=$(date +"%V") 30 | MONTH=$(date +"%b") 31 | let "INDEX = WEEK % 5" || true 32 | if [[ ${INDEX} == 0 ]]; then 33 | INDEX=4 34 | fi 35 | 36 | test -e ${SPHINX_BACKUP_DIR}/backup.${INDEX}.tar.gz && rm ${SPHINX_BACKUP_DIR}/backup.${INDEX}.tar.gz 37 | mv ${SPHINX_BACKUP_DIR}/backup.tar.gz ${SPHINX_BACKUP_DIR}/backup.${INDEX}.tar.gz 38 | echo "Create backup file: ${SPHINX_BACKUP_DIR}/backup.${INDEX}.tar.gz" 39 | 40 | test -e ${SPHINX_BACKUP_DIR}/backup.${MONTH}.tar.gz && rm ${SPHINX_BACKUP_DIR}/backup.${MONTH}.tar.gz 41 | ln ${SPHINX_BACKUP_DIR}/backup.${INDEX}.tar.gz ${SPHINX_BACKUP_DIR}/backup.${MONTH}.tar.gz 42 | echo "Create backup file: ${SPHINX_BACKUP_DIR}/backup.${MONTH}.tar.gz" 43 | 44 | test -e ${SPHINX_BACKUP_DIR}/backup.last.tar.gz && rm ${SPHINX_BACKUP_DIR}/backup.last.tar.gz 45 | ln ${SPHINX_BACKUP_DIR}/backup.${INDEX}.tar.gz ${SPHINX_BACKUP_DIR}/backup.last.tar.gz 46 | echo "Create backup file: ${SPHINX_BACKUP_DIR}/backup.last.tar.gz" 47 | else 48 | mv ${SPHINX_BACKUP_DIR}/backup.tar.gz ${SPHINX_BACKUP_DIR}/backup.last.tar.gz 49 | echo "Create backup file: ${SPHINX_BACKUP_DIR}/backup.last.tar.gz" 50 | fi 51 | } 52 | 53 | import_backup() 54 | { 55 | echo "Import dump..." 56 | FILE=$1 57 | if [[ ${FILE} == default ]]; then 58 | FILE="${SPHINX_BACKUP_DIR}/${SPHINX_BACKUP_FILENAME}" 59 | fi 60 | if [[ ! -f "${FILE}" ]]; then 61 | echo "Unknown backup: ${FILE}" 62 | exit 1 63 | fi 64 | create_data_dir 65 | tar -C ${SPHINX_DATA_DIR} -xf ${FILE} 66 | } 67 | 68 | sed -i "s~SPHINX_DATA_DIR~${SPHINX_DATA_DIR}~g" ${SPHINX_CONF} 69 | sed -i "s~SPHINX_LOG_DIR~${SPHINX_LOG_DIR}~g" ${SPHINX_CONF} 70 | sed -i "s~SPHINX_RUN~${SPHINX_RUN}~g" ${SPHINX_CONF} 71 | 72 | if [[ ${SPHINX_MODE} == indexing ]]; then 73 | indexer --config ${SPHINX_CONF} --all 74 | fi 75 | 76 | if [[ ${SPHINX_MODE} == backup ]]; then 77 | echo "Backup..." 78 | if [[ ! -d ${SPHINX_DATA_DIR} ]]; then 79 | echo "No such directory: ${SPHINX_DATA_DIR}" 80 | exit 1 81 | fi 82 | create_backup_dir 83 | cd ${SPHINX_DATA_DIR} 84 | tar --ignore-failed-read -zcvf ${SPHINX_BACKUP_DIR}/backup.tar.gz *.sp* *.ram *.kill *.meta binlog.* 85 | cd - 86 | rotate_backup 87 | exit 0 88 | fi 89 | 90 | # Restore from backup 91 | if [[ -n ${SPHINX_RESTORE} ]]; then 92 | import_backup ${SPHINX_RESTORE} 93 | fi 94 | 95 | # Check backup 96 | if [[ -n ${SPHINX_CHECK} ]]; then 97 | 98 | echo "Check backup..." 99 | if [[ -z ${INDEX_NAME} ]]; then 100 | echo "Unknown database. INDEX_NAME does not null" 101 | exit 1; 102 | fi 103 | 104 | if [[ ! -d ${SPHINX_DATA_DIR} || -z $(ls -A ${SPHINX_DATA_DIR}) ]]; then 105 | import_backup ${SPHINX_CHECK} 106 | fi 107 | 108 | if [[ $(indextool --config ${SPHINX_CONF} --check ${INDEX_NAME} | grep -w "check passed") ]]; then 109 | echo "Success checking backup" 110 | else 111 | echo "Fail checking backup" 112 | exit 1 113 | fi 114 | 115 | exit 0 116 | fi 117 | 118 | # allow arguments to be passed to Sphinx search 119 | if [[ ${1:0:1} = '-' ]]; then 120 | EXTRA_OPTS="$@" 121 | set -- 122 | fi 123 | 124 | # default behaviour is to launch Sphinx search 125 | if [[ -z ${1} ]]; then 126 | echo "Starting Sphinx search demon..." 127 | exec $(which searchd) --config ${SPHINX_CONF} --nodetach ${EXTRA_OPTS} 128 | else 129 | exec "$@" 130 | fi -------------------------------------------------------------------------------- /configs/sphinx_pgsql.conf: -------------------------------------------------------------------------------- 1 | # Sphinx configuration for the unit tests 2 | # 3 | # Setup test environment: 4 | # - setup test Sphinx indexes: 5 | # indexer --config /path/to/sphinx.conf --all [--rotate] 6 | # - run the "searchd" daemon: 7 | # searchd --config /path/to/sphinx.conf 8 | # - Merge delta index 9 | # indexer delta_index --rotate --config /path/to/sphinx.conf 10 | 11 | ####################### 12 | # 13 | # Sources 14 | # 15 | ####################### 16 | 17 | # Common Source 18 | source common_source 19 | { 20 | # data source type. mandatory, no default value 21 | # known types are mysql, pgsql, mssql, xmlpipe, xmlpipe2, odbc 22 | type = pgsql 23 | 24 | # some straightforward parameters for SQL source types 25 | sql_host = db-test 26 | sql_user = admin 27 | sql_pass = pass 28 | sql_db = db_test 29 | sql_port = 5432 30 | } 31 | 32 | 33 | source items_source : common_source 34 | { 35 | sql_query = \ 36 | SELECT\ 37 | id, content\ 38 | FROM\ 39 | items\ 40 | WHERE\ 41 | id >= $start AND id <= $end 42 | 43 | # query for crushing utems on several iterations 44 | sql_query_range = SELECT MIN(id),MAX(id) FROM items 45 | 46 | # count items per iteration 47 | sql_range_step = 1000 48 | 49 | # Downtime (sleep)before sending queries (is designed for unloading of the database server) 50 | # If set "1000", the sleep will last 1 second 51 | sql_ranged_throttle = 0 52 | } 53 | 54 | ####################### 55 | # 56 | # Indexes 57 | # 58 | ####################### 59 | 60 | index common_index 61 | { 62 | # Method of storage index (none, inline, extern) 63 | docinfo = extern 64 | 65 | # dictionary type, 'crc' or 'keywords' 66 | # crc is faster to index when no substring/wildcards searches are needed 67 | # crc with substrings might be faster to search but is much slower to index 68 | # (because all substrings are pre-extracted as individual keywords) 69 | # keywords is much faster to index with substrings, and index is much (3-10x) smaller 70 | # keywords supports wildcards, crc does not, and never will 71 | # optional, default is 'keywords' 72 | dict = keywords 73 | 74 | # Memory lock 75 | mlock = 0 76 | 77 | # Used morphological engines 78 | morphology = stem_enru, soundex, metaphone 79 | 80 | #charset_table = 0..9, A..Z->a..z, _, a..z, \ 81 | #U+410..U+42F->U+430..U+44F, U+430..U+44F 82 | 83 | # The minimum length of words to be indexed, by default 1 84 | min_word_len = 2 85 | 86 | # enabled strip html 87 | html_strip = 1 88 | } 89 | 90 | 91 | index items_index : common_index 92 | { 93 | source = items_source 94 | path = SPHINX_DATA_DIR/items_index 95 | } 96 | 97 | 98 | index distributed 99 | { 100 | type = distributed 101 | local = items_index 102 | } 103 | 104 | 105 | # Indexer settings 106 | indexer 107 | { 108 | # The maximum memory limit daemon 109 | mem_limit = 32M 110 | } 111 | 112 | # Demon settings 113 | searchd 114 | { 115 | # [hostname:]port[:protocol], or /unix/socket/path to listen on 116 | # known protocols are 'sphinx' (SphinxAPI) and 'mysql41' (SphinxQL) 117 | # 118 | # multi-value, multiple listen points are allowed 119 | # optional, defaults are 9312:sphinx and 9306:mysql41, as below 120 | # 121 | # listen = 127.0.0.1 122 | # listen = 192.168.0.1:9312 123 | # listen = 9312 124 | # listen = /var/run/searchd.sock 125 | listen = 127.0.0.1:9312 126 | listen = 9306:mysql41 127 | 128 | # logs 129 | log = SPHINX_LOG_DIR/searchd.log 130 | # logging queries . If you comment out,then log will not be 131 | query_log = SPHINX_LOG_DIR/query.log 132 | query_log_format = sphinxql 133 | 134 | # The time, in seconds, waiting for the daemon when communicating with the client. On the exhaustion of rupture of connection 135 | read_timeout = 5 136 | 137 | # The max of threads (children) from the process. 0 means unlimited 138 | max_children = 30 139 | 140 | pid_file = SPHINX_RUN 141 | 142 | workers = threads # for RT to work 143 | 144 | # binlog files path; use empty string to disable binlog 145 | # optional, default is build-time configured data directory 146 | # 147 | # binlog_path = # disable logging 148 | binlog_path = SPHINX_DATA_DIR # binlog.001 etc will be created there 149 | } 150 | # --eof-- 151 | -------------------------------------------------------------------------------- /test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | echo "-- Building Sphinx image" 6 | docker build -t sphinx_test . 7 | docker network create sphinx_test_net 8 | DIR_VOLUME=$(pwd)/vol 9 | mkdir -p ${DIR_VOLUME}/backup 10 | 11 | echo 12 | echo "-- Testing Sphinx + PostgreSQL" 13 | echo 14 | echo "-- Run postgresql container" 15 | docker run --name db-test -d --net sphinx_test_net -e 'DB_NAME=db_test' -e 'DB_USER=admin' -e 'DB_PASS=pass' romeoz/docker-postgresql; sleep 20 16 | echo 17 | echo "-- Create table" 18 | docker exec -it db-test sudo -u postgres psql db_test -c "CREATE TABLE items (id SERIAL, content TEXT);" 19 | echo 20 | echo "-- Sets a permission on database" 21 | docker exec -it db-test sudo -u postgres psql db_test -c "GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO admin;GRANT SELECT ON ALL TABLES IN SCHEMA public TO admin;" 22 | echo 23 | echo "-- Insert records" 24 | docker exec -it db-test sudo -u postgres psql db_test -c "INSERT INTO items (content) VALUES ('about dog'),('about cat');"; sleep 5 25 | echo 26 | echo "-- Run sphinx container" 27 | docker run --name sphinx_test -d --net sphinx_test_net -e "SPHINX_MODE=indexing" -e "SPHINX_CONF=/etc/sphinxsearch/sphinx_pgsql.conf" sphinx_test; sleep 10 28 | echo 29 | echo "-- Install MySQL client" 30 | docker exec -it sphinx_test bash -c 'apt-get update && apt-get install -y mysql-client && rm -rf /var/lib/apt/lists/*'; sleep 10 31 | echo 32 | echo "-- Testing" 33 | docker exec -it sphinx_test mysql -P9306 -h127.0.0.1 -e "SELECT id FROM items_index WHERE MATCH('cat');" | grep -wc "2" 34 | 35 | echo 36 | echo "-- Testing backup" 37 | docker run -it --rm --volumes-from sphinx_test -e 'SPHINX_MODE=backup' -v ${DIR_VOLUME}/backup:/tmp/backup sphinx_test; sleep 10 38 | 39 | echo 40 | echo "-- Clear" 41 | docker rm -f -v sphinx_test; sleep 5 42 | echo 43 | echo "-- Restore from backup" 44 | docker run --name sphinx_restore -d --net sphinx_test_net -e "SPHINX_CONF=/etc/sphinxsearch/sphinx_pgsql.conf" -e 'SPHINX_RESTORE=default' -v ${DIR_VOLUME}/backup:/tmp/backup sphinx_test; sleep 20 45 | 46 | echo 47 | echo "-- Install MySQL client" 48 | docker exec -it sphinx_restore bash -c 'apt-get update && apt-get install -y mysql-client && rm -rf /var/lib/apt/lists/*'; sleep 10 49 | echo 50 | echo "-- Checking backup" 51 | docker exec -it sphinx_restore mysql -P9306 -h127.0.0.1 -e "SELECT id FROM items_index WHERE MATCH('cat');" | grep -wc "2" 52 | docker run -it --rm -e 'SPHINX_CHECK=default' -e "SPHINX_CONF=/etc/sphinxsearch/sphinx_pgsql.conf" -e 'INDEX_NAME=items_index' -v ${DIR_VOLUME}/backup:/tmp/backup sphinx_test | grep -wc 'Success'; sleep 5 53 | 54 | echo 55 | echo "-- Clear" 56 | docker rm -f -v sphinx_restore db-test; sleep 5 57 | rm -rf ${DIR_VOLUME} 58 | 59 | 60 | echo 61 | echo 62 | echo "-- Testing Sphinx + MySQL" 63 | echo 64 | echo "-- Run mysql container" 65 | mkdir -p ${DIR_VOLUME}/backup 66 | docker run --name db-test -d --net sphinx_test_net -e 'MYSQL_USER=admin' -e 'MYSQL_PASS=pass' -e 'MYSQL_CACHE_ENABLED=true' -e 'DB_NAME=db_test' romeoz/docker-mysql; sleep 20 67 | 68 | echo 69 | echo "-- Create table" 70 | docker exec -it db-test mysql -uroot -e 'CREATE TABLE db_test.items (id INT NOT NULL AUTO_INCREMENT, content TEXT, PRIMARY KEY(id)) ENGINE = INNODB;'; sleep 5 71 | 72 | echo 73 | echo "-- Insert records" 74 | docker exec -it db-test mysql -uroot -e 'INSERT INTO db_test.items (content) VALUES ("about dog"),("about cat");'; sleep 5 75 | 76 | echo 77 | echo "-- Run sphinx container" 78 | docker run --name sphinx_test -d --net sphinx_test_net -e "SPHINX_MODE=indexing" sphinx_test; sleep 10 79 | 80 | echo 81 | echo "-- Testing" 82 | docker exec -it db-test mysql -P9306 -hsphinx_test -e "SELECT * FROM items_index WHERE MATCH('cat');" | grep -wc "2" 83 | 84 | 85 | echo 86 | echo "-- Testing backup" 87 | #docker exec -it db-test mysql -P9306 -h$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' sphinx_test) -e "FLUSH RTINDEX myrtindex" 88 | docker run -it --rm --volumes-from sphinx_test -e 'SPHINX_MODE=backup' -v ${DIR_VOLUME}/backup:/tmp/backup sphinx_test; sleep 10 89 | 90 | echo 91 | echo "-- Clear" 92 | docker rm -f -v sphinx_test; sleep 5 93 | echo 94 | echo "-- Restore from backup" 95 | docker run --name sphinx_restore -d --net sphinx_test_net -e 'SPHINX_RESTORE=default' -v ${DIR_VOLUME}/backup:/tmp/backup sphinx_test; sleep 20 96 | 97 | echo 98 | echo "-- Checking backup" 99 | docker exec -it db-test mysql -P9306 -hsphinx_restore -e "SELECT * FROM items_index WHERE MATCH('cat');" | grep -wc "2" 100 | docker run -it --rm -e 'SPHINX_CHECK=default' -e 'INDEX_NAME=items_index' -v ${DIR_VOLUME}/backup:/tmp/backup sphinx_test | grep -wc 'Success'; sleep 5 101 | 102 | echo 103 | echo "-- Clear" 104 | docker rm -f -v sphinx_restore db-test; sleep 5 105 | docker network rm sphinx_test_net 106 | docker rmi -f sphinx_test; sleep 5 107 | rm -rf ${DIR_VOLUME} 108 | 109 | echo 110 | echo "-- Done" -------------------------------------------------------------------------------- /configs/sphinx.conf: -------------------------------------------------------------------------------- 1 | # Sphinx configuration for the unit tests 2 | # 3 | # Setup test environment: 4 | # - setup test Sphinx indexes: 5 | # indexer --config /path/to/sphinx.conf --all [--rotate] 6 | # - run the "searchd" daemon: 7 | # searchd --config /path/to/sphinx.conf 8 | # - Merge delta index 9 | # indexer delta_index --rotate --config /path/to/sphinx.conf 10 | 11 | ####################### 12 | # 13 | # Sources 14 | # 15 | ####################### 16 | 17 | # Common Source 18 | source common_source 19 | { 20 | # data source type. mandatory, no default value 21 | # known types are mysql, pgsql, mssql, xmlpipe, xmlpipe2, odbc 22 | type = mysql 23 | 24 | # some straightforward parameters for SQL source types 25 | sql_host = db-test 26 | sql_user = admin 27 | sql_pass = pass 28 | sql_db = db_test 29 | sql_port = 3306 # optional, default is 3306 30 | 31 | # To accelerate the path to MySQL UNIX-socket 32 | # (to operations with the database was not over a TCP/IP stack server) 33 | #sql_sock = /var/run/mysqld/mysqld.sock 34 | 35 | 36 | # MySQL specific client connection flags 37 | # optional, default is 0 38 | # 39 | mysql_connect_flags = 32 # enable compression 40 | 41 | # Set charset of the connection and turn off cache queries 42 | sql_query_pre = SET NAMES utf8 43 | sql_query_pre = SET CHARACTER SET utf8 44 | sql_query_pre = SET SESSION query_cache_type=OFF 45 | } 46 | 47 | 48 | source items_source : common_source 49 | { 50 | sql_query = \ 51 | SELECT\ 52 | id, content\ 53 | FROM\ 54 | items\ 55 | WHERE\ 56 | id >= $start AND id <= $end 57 | 58 | 59 | # query for crushing utems on several iterations 60 | sql_query_range = SELECT MIN(id),MAX(id) FROM items 61 | 62 | # count items per iteration 63 | sql_range_step = 1000 64 | 65 | # Downtime (sleep)before sending queries (is designed for unloading of the database server) 66 | # If set "1000", the sleep will last 1 second 67 | sql_ranged_throttle = 0 68 | } 69 | 70 | ####################### 71 | # 72 | # Indexes 73 | # 74 | ####################### 75 | 76 | index common_index 77 | { 78 | # Method of storage index (none, inline, extern) 79 | docinfo = extern 80 | 81 | # dictionary type, 'crc' or 'keywords' 82 | # crc is faster to index when no substring/wildcards searches are needed 83 | # crc with substrings might be faster to search but is much slower to index 84 | # (because all substrings are pre-extracted as individual keywords) 85 | # keywords is much faster to index with substrings, and index is much (3-10x) smaller 86 | # keywords supports wildcards, crc does not, and never will 87 | # optional, default is 'keywords' 88 | dict = keywords 89 | 90 | # Memory lock 91 | mlock = 0 92 | 93 | # Used morphological engines 94 | morphology = stem_enru, soundex, metaphone 95 | 96 | #charset_table = 0..9, A..Z->a..z, _, a..z, \ 97 | #U+410..U+42F->U+430..U+44F, U+430..U+44F 98 | 99 | # The minimum length of words to be indexed, by default 1 100 | min_word_len = 2 101 | 102 | # enabled strip html 103 | html_strip = 1 104 | } 105 | 106 | 107 | index items_index : common_index 108 | { 109 | source = items_source 110 | path = SPHINX_DATA_DIR/items_index 111 | } 112 | 113 | 114 | index distributed 115 | { 116 | type = distributed 117 | local = items_index 118 | } 119 | 120 | 121 | # Indexer settings 122 | indexer 123 | { 124 | # The maximum memory limit daemon 125 | mem_limit = 32M 126 | } 127 | 128 | # Demon settings 129 | searchd 130 | { 131 | # [hostname:]port[:protocol], or /unix/socket/path to listen on 132 | # known protocols are 'sphinx' (SphinxAPI) and 'mysql41' (SphinxQL) 133 | # 134 | # multi-value, multiple listen points are allowed 135 | # optional, defaults are 9312:sphinx and 9306:mysql41, as below 136 | # 137 | # listen = 127.0.0.1 138 | # listen = 192.168.0.1:9312 139 | # listen = 9312 140 | # listen = /var/run/searchd.sock 141 | listen = 127.0.0.1:9312 142 | listen = 9306:mysql41 143 | 144 | # logs 145 | log = SPHINX_LOG_DIR/searchd.log 146 | # logging queries . If you comment out,then log will not be 147 | query_log = SPHINX_LOG_DIR/query.log 148 | query_log_format = sphinxql 149 | 150 | # The time, in seconds, waiting for the daemon when communicating with the client. On the exhaustion of rupture of connection 151 | read_timeout = 5 152 | 153 | # The max of threads (children) from the process. 0 means unlimited 154 | max_children = 30 155 | 156 | pid_file = SPHINX_RUN 157 | 158 | workers = threads # for RT to work 159 | 160 | # binlog files path; use empty string to disable binlog 161 | # optional, default is build-time configured data directory 162 | # 163 | # binlog_path = # disable logging 164 | binlog_path = SPHINX_DATA_DIR # binlog.001 etc will be created there 165 | } 166 | # --eof-- 167 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Table of Contents 2 | ------------------- 3 | 4 | * [Installation](#installation) 5 | * [Quick Start](#quick-start) 6 | * [Examples](#examples) 7 | - [Sphinx + MySQL](#sphinx--mysql) 8 | - [Sphinx + PostgreSQL or other source type](#sphinx--postgresql) 9 | * [Persistence](#persistence) 10 | * [Backup of a indexes](#backup-of-a-indexes) 11 | * [Checking backup](#checking-backup) 12 | * [Restore from backup](#restore-from-backup) 13 | * [Environment variables](#environment-variables) 14 | * [Logging](#logging) 15 | * [Out of the box](#out-of-the-box) 16 | 17 | Installation 18 | ------------------- 19 | 20 | * [Install Docker 1.9+](https://docs.docker.com/installation/) or [askubuntu](http://askubuntu.com/a/473720) 21 | * Pull the latest version of the image. 22 | 23 | ```bash 24 | docker pull romeoz/docker-sphinxsearch 25 | ``` 26 | 27 | Alternately you can build the image yourself. 28 | 29 | ```bash 30 | git clone https://github.com/romeoz/docker-sphinxsearch.git 31 | cd docker-sphinxsearch 32 | docker build -t="$USER/sphinxsearch" . 33 | ``` 34 | 35 | Quick Start 36 | ------------------- 37 | 38 | Use one of two ways: 39 | 40 | 1) Usage Docker Compose 41 | 42 | ```bash 43 | docker network create sphinx_net 44 | 45 | curl -L https://github.com/romeoz/docker-sphinxsearch/raw/master/docker-compose.yml > docker-compose.yml 46 | docker-compose up -d 47 | ``` 48 | 2) Step by step. 49 | 50 | Run mysql container: 51 | 52 | ```bash 53 | docker network create sphinx_net 54 | 55 | docker run --name db -d \ 56 | --net sphinx_net 57 | -e 'MYSQL_USER=admin' -e 'MYSQL_PASS=pass' -e 'MYSQL_CACHE_ENABLED=true' \ 58 | romeoz/docker-mysql 59 | ``` 60 | 61 | >Recommended way (official). Sphinx own implementation of MySQL network protocol (using a small SQL subset called SphinxQL). 62 | 63 | Run the sphinx container: 64 | 65 | ```bash 66 | docker run --name sphinx -d \ 67 | --net sphinx_net \ 68 | romeoz/docker-sphinxsearch 69 | ``` 70 | 71 | Examples 72 | ------------------- 73 | 74 | ####Sphinx + MySQL 75 | 76 | Run the mysql container with with the creation of database `db_test`: 77 | 78 | ```bash 79 | docker network create sphinx_net 80 | 81 | docker run --name db -d \ 82 | --net sphinx_net 83 | -e 'MYSQL_USER=admin' -e 'MYSQL_PASS=pass' -e 'MYSQL_CACHE_ENABLED=true' \ 84 | -e 'DB_NAME=db_test' \ 85 | romeoz/docker-mysql 86 | ``` 87 | 88 | Creating table `items` and records: 89 | 90 | ```bash 91 | docker exec -it db \ 92 | mysql -uroot -e 'CREATE TABLE db_test.items (id INT NOT NULL AUTO_INCREMENT, content TEXT, PRIMARY KEY(id)) ENGINE = INNODB;' 93 | 94 | docker exec -it db \ 95 | mysql -uroot -e 'INSERT INTO db_test.items (content) VALUES ("about dog"),("about cat");' 96 | ``` 97 | 98 | Run the sphinx instnace + indexing database: 99 | 100 | ```bash 101 | docker run --name sphinx -d \ 102 | --net sphinx_net \ 103 | -e "SPHINX_MODE=indexing" \ 104 | romeoz/docker-sphinxsearch 105 | ``` 106 | 107 | Searching records: 108 | 109 | ```bash 110 | host=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' sphinx); 111 | docker exec -it db \ 112 | mysql -P9306 -h${host} -e "SELECT * FROM items_index WHERE MATCH('cat');" 113 | ``` 114 | 115 | ####Sphinx + PostgreSQL 116 | 117 | You can using other source type, for example PostgreSQL. If you want to use the SphinxQL, there is no need to install the MySQL server. 118 | It helps to have the `mysql-common` package and `mysql-client` (if you need a CLI). 119 | 120 | Run the postgresql container with with the creation of database `db_test`: 121 | 122 | ```bash 123 | docker network create sphinx_net 124 | 125 | docker run --name db-test -d \ 126 | -e 'DB_NAME=db_test' -e 'DB_USER=admin' -e 'DB_PASS=pass' \ 127 | romeoz/docker-postgresql 128 | ``` 129 | 130 | Creating table `items` and records: 131 | 132 | ```bash 133 | docker exec -it db-test sudo -u postgres psql db_test \ 134 | -c "CREATE TABLE items (id SERIAL, content TEXT);" 135 | docker exec -it db-test sudo -u postgres psql db_test \ 136 | -c "GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO admin; GRANT SELECT ON ALL TABLES IN SCHEMA public TO admin;" 137 | docker exec -it db-test sudo -u postgres psql db_test \ 138 | -c "INSERT INTO items (content) VALUES ('about dog'),('about cat');" 139 | ``` 140 | 141 | Run sphinx container + indexing database: 142 | 143 | ```bash 144 | docker run --name sphinx -d \ 145 | --net sphinx_net \ 146 | -e "SPHINX_MODE=indexing" -e "SPHINX_CONF=/etc/sphinxsearch/sphinx_pgsql.conf" \ 147 | romeoz/docker-sphinxsearch 148 | ``` 149 | 150 | Install MySQL Client to sphinx container: 151 | 152 | ```bash 153 | docker exec -it sphinx bash -c 'apt-get update && apt-get install -y mysql-client && rm -rf /var/lib/apt/lists/*' 154 | ``` 155 | 156 | Searching records: 157 | 158 | ```bash 159 | docker exec -it sphinx \ 160 | mysql -P9306 -h127.0.0.1 -e "SELECT * FROM items_index WHERE MATCH('cat');" 161 | ``` 162 | 163 | Persistence 164 | ------------------- 165 | 166 | For data persistence a volume should be mounted at `/var/lib/sphinxsearch/data`. 167 | 168 | The updated run command looks like this. 169 | 170 | ```bash 171 | docker run --name sphinx -d \ 172 | -v /host/to/path/data:/var/lib/sphinxsearch/data \ 173 | romeoz/docker-sphinxsearch 174 | ``` 175 | 176 | This will make sure that the data stored in the index is not lost when the container is stopped and started again. 177 | 178 | 179 | Backup of a indexes 180 | ------------------- 181 | 182 | If you are using RT index, then first we need to flush indexes `FLUSH RTINDEX`: 183 | 184 | ```bash 185 | host=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' sphinx) 186 | docker exec -it db mysql -P9306 -h${host} -e "FLUSH RTINDEX some_rtindex" 187 | ``` 188 | 189 | Next, create a temporary container for backup: 190 | 191 | ```bash 192 | docker run -it --rm \ 193 | --volumes-from sphinx \ 194 | -e 'SPHINX_MODE=backup' \ 195 | -v /host/to/path/backup:/tmp/backup \ 196 | romeoz/docker-sphinxsearch 197 | ``` 198 | Archive will be available in the `/host/to/path/backup`. 199 | 200 | > Algorithm: one backup per week (total 4), one backup per month (total 12) and the last backup. Example: `backup.last.tar.gz`, `backup.1.tar.gz` and `/backup.dec.tar.gz`. 201 | 202 | You can disable the rotation by using env `SPHINX_ROTATE_BACKUP=false`. 203 | 204 | Checking backup 205 | ------------------- 206 | 207 | Check-data is the name of index `INDEX_NAME`. 208 | 209 | ```bash 210 | docker run -it --rm \ 211 | -e 'SPHINX_CHECK=default' -e 'INDEX_NAME=items_index' \ 212 | -v /host/to/path/backup:/tmp/backup \ 213 | romeoz/docker-sphinxsearch 214 | ``` 215 | 216 | Default used the `/tmp/backup/backup.last.tar.gz`. 217 | 218 | Restore from backup 219 | ------------------- 220 | 221 | ```bash 222 | docker run --name sphinx-restore -d \ 223 | --net sphinx_net \ 224 | -e 'SPHINX_RESTORE=default' \ 225 | -v /host/to/path/backup:/tmp/backup \ 226 | romeoz/docker-sphinxsearch 227 | ``` 228 | 229 | Environment variables 230 | --------------------- 231 | 232 | `SPHINX_MODE`: Set a specific mode. Takes on the value `backup` or `indexing`. 233 | 234 | `SPHINX_BACKUP_DIR`: Set a specific backup directory (default "/tmp/backup"). 235 | 236 | `SPHINX_BACKUP_FILENAME`: Set a specific filename backup (default "backup.last.tar.gz"). 237 | 238 | `SPHINX_CHECK`: Defines name of backup to `indextool --check`. Note that the backup must be inside the container, so you may need to mount them. You can specify as `default` that is equivalent to the `/tmp/backup/backup.tar.gz`. 239 | 240 | `SPHINX_RESTORE`: Defines name of backup to initialize the demon `searchd`. Note that the backup must be inside the container, so you may need to mount them. You can specify as `default` that is equivalent to the `/tmp/backup/backup.last.tar.gz`. 241 | 242 | `SPHINX_ROTATE_BACKUP`: Determines whether to use the rotation of backups (default "true"). 243 | 244 | Logging 245 | ------------------- 246 | 247 | All the logs are forwarded to stdout and sterr. You have use the command `docker logs`. 248 | 249 | ```bash 250 | docker logs sphinx 251 | ``` 252 | 253 | ####Split the logs 254 | 255 | You can then simply split the stdout & stderr of the container by piping the separate streams and send them to files: 256 | 257 | ```bash 258 | docker logs sphinx > stdout.log 2>stderr.log 259 | cat stdout.log 260 | cat stderr.log 261 | ``` 262 | 263 | or split stdout and error to host stdout: 264 | 265 | ```bash 266 | docker logs sphinx > - 267 | docker logs sphinx 2> - 268 | ``` 269 | 270 | ####Rotate logs 271 | 272 | Create the file `/etc/logrotate.d/docker-containers` with the following text inside: 273 | 274 | ``` 275 | /var/lib/docker/containers/*/*.log { 276 | rotate 31 277 | daily 278 | nocompress 279 | missingok 280 | notifempty 281 | copytruncate 282 | } 283 | ``` 284 | > Optionally, you can replace `nocompress` to `compress` and change the number of days. 285 | 286 | Out of the box 287 | ------------------- 288 | * Ubuntu 16.04 LTS 289 | * Sphinx Search 2.2 290 | 291 | License 292 | ------------------- 293 | 294 | Sphinx Search docker image is open-sourced software licensed under the [MIT license](http://opensource.org/licenses/MIT) --------------------------------------------------------------------------------