├── .gitignore ├── README.md ├── app ├── code │ └── community │ │ └── Manticorp │ │ └── SphinxSearch │ │ ├── Helper │ │ └── Data.php │ │ ├── Model │ │ ├── Resource │ │ │ ├── Fulltext.php │ │ │ └── Fulltext │ │ │ │ └── Engine.php │ │ └── System │ │ │ └── Config │ │ │ └── Source │ │ │ └── Product │ │ │ ├── Attributes.php │ │ │ └── Attributes │ │ │ └── Search.php │ │ ├── etc │ │ ├── adminhtml.xml │ │ ├── config.xml │ │ └── system.xml │ │ └── sql │ │ └── sphinxsearch_setup │ │ ├── install-0.1.0.php │ │ └── upgrade-0.1.0-0.1.1.php └── etc │ └── modules │ └── Manticorp_SphinxSearch.xml ├── lib └── sphinxapi.php └── sphinx.conf.example /.gitignore: -------------------------------------------------------------------------------- 1 | # Dynamic data that doesn't need to be in the repo 2 | /var/* 3 | /media/* 4 | /downloader/pearlib/cache/* 5 | /downloader/pearlib/download/* 6 | /app/etc/use_cache.ser 7 | local.xml 8 | /magmi 9 | .htaccess 10 | /tmp 11 | composer.json 12 | *.sublime* 13 | *.sublime-project 14 | .htaccess-sample 15 | n98-magerun.phar 16 | 17 | # Created by https://www.gitignore.io 18 | 19 | ### Magento ### 20 | .modgit/ 21 | .modman/ 22 | app/code/community/Phoenix/ 23 | app/code/community/Cm/ 24 | app/code/core/ 25 | app/design/adminhtml/default/default/ 26 | app/design/frontend/base/ 27 | app/design/frontend/rwd/ 28 | app/design/frontend/default/blank/ 29 | app/design/frontend/default/default/ 30 | app/design/frontend/default/iphone/ 31 | app/design/frontend/default/modern/ 32 | app/design/frontend/enterprise/default 33 | app/design/install/ 34 | app/etc/modules/Enterprise_* 35 | app/etc/modules/Mage_All.xml 36 | app/etc/modules/Mage_Api.xml 37 | app/etc/modules/Mage_Api2.xml 38 | app/etc/modules/Mage_Authorizenet.xml 39 | app/etc/modules/Mage_Bundle.xml 40 | app/etc/modules/Mage_Captcha.xml 41 | app/etc/modules/Mage_Centinel.xml 42 | app/etc/modules/Mage_Compiler.xml 43 | app/etc/modules/Mage_ConfigurableSwatches.xml 44 | app/etc/modules/Mage_Connect.xml 45 | app/etc/modules/Mage_CurrencySymbol.xml 46 | app/etc/modules/Mage_Downloadable.xml 47 | app/etc/modules/Mage_ImportExport.xml 48 | app/etc/modules/Mage_LoadTest.xml 49 | app/etc/modules/Mage_Oauth.xml 50 | app/etc/modules/Mage_PageCache.xml 51 | app/etc/modules/Mage_Persistent.xml 52 | app/etc/modules/Mage_Weee.xml 53 | app/etc/modules/Mage_Widget.xml 54 | app/etc/modules/Mage_XmlConnect.xml 55 | app/etc/modules/Phoenix_Moneybookers.xml 56 | app/etc/modules/Cm_RedisSession.xml 57 | app/etc/applied.patches.list 58 | app/etc/config.xml 59 | app/etc/enterprise.xml 60 | app/etc/local.xml.additional 61 | app/etc/local.xml.template 62 | app/etc/local.xml 63 | app/.htaccess 64 | app/locale/ 65 | app/Mage.php 66 | /cron.php 67 | cron.sh 68 | downloader/ 69 | errors/ 70 | favicon.ico 71 | /get.php 72 | includes/ 73 | /index.php 74 | index.php.sample 75 | /install.php 76 | js/blank.html 77 | js/calendar/ 78 | js/enterprise/ 79 | js/extjs/ 80 | js/firebug/ 81 | js/flash/ 82 | js/index.php 83 | js/jscolor/ 84 | js/lib/ 85 | js/mage/ 86 | js/prototype/ 87 | js/scriptaculous/ 88 | js/spacer.gif 89 | js/tiny_mce/ 90 | js/varien/ 91 | lib/3Dsecure/ 92 | lib/Apache/ 93 | lib/flex/ 94 | lib/googlecheckout/ 95 | lib/.htaccess 96 | lib/LinLibertineFont/ 97 | lib/Mage/ 98 | lib/PEAR/ 99 | lib/Pelago/ 100 | lib/phpseclib/ 101 | lib/Varien/ 102 | lib/Zend/ 103 | lib/Cm/ 104 | lib/Credis/ 105 | lib/Magento/ 106 | LICENSE_AFL.txt 107 | LICENSE.html 108 | LICENSE.txt 109 | LICENSE_EE* 110 | mage 111 | media/customer/ 112 | media/dhl/ 113 | media/downloadable/ 114 | media/.htaccess 115 | media/import/ 116 | media/xmlconnect/ 117 | media/wysiwyg/ 118 | media/catalog/product/cache/ 119 | /api.php 120 | nbproject/ 121 | pear 122 | pear/ 123 | php.ini.sample 124 | pkginfo/ 125 | RELEASE_NOTES.txt 126 | shell/abstract.php 127 | shell/compiler.php 128 | shell/indexer.php 129 | shell/log.php 130 | sitemap.xml 131 | skin/adminhtml/default/default/ 132 | skin/adminhtml/default/enterprise 133 | skin/frontend/base/ 134 | skin/frontend/rwd/ 135 | skin/frontend/default/blank/ 136 | skin/frontend/default/blue/ 137 | skin/frontend/default/default/ 138 | skin/frontend/default/french/ 139 | skin/frontend/default/german/ 140 | skin/frontend/default/iphone/ 141 | skin/frontend/default/modern/ 142 | skin/frontend/enterprise 143 | skin/install/ 144 | var/ 145 | *.stackdump 146 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Manticorp_SphinxSearch 2 | 3 | This extension integrates [Sphinx Search](http://sphinxsearch.com/) with any Magento installation. 4 | 5 | Based on [Gfe/SphinxSearch](https://github.com/fheyer/sphinxsearch) with a few key improvements. 6 | 7 | ## Information 8 | 9 | Tested on **Magento CE 1.9.1.0** 10 | 11 | ## Installation 12 | 13 | 1. *optional* Turn off compilation 14 | 2. Copy all the files into the root of your Magento Installation 15 | 3. Log out of admin 16 | 4. Log back in 17 | 5. Flush all caches 18 | 6. *optional* Turn on compilation 19 | 20 | ## Setup 21 | 22 | ### Sphinx 23 | 24 | 1. [Install Sphinx Search](http://sphinxsearch.com/docs/current.html#installation) 25 | 2. Modify ```sphinx.conf.sample``` to suit your needs 26 | 3. Launch Sphinx using the ```searchd``` command 27 | 28 | ### Magento 29 | 30 | 1. Go to your magento backend 31 | 2. Go to System > Configuration > Catalog > Sphinx Search Engine 32 | 3. Turn on 'Enable Search' and 'Enable Index' 33 | 4. Update your 'path to sphinx' (leave blank by default if it's in your system path) 34 | 5. Apply weights to the necessary fields 35 | 6. Exclude whatever fields you wish from attributes 36 | 7. Edit the Server configuration if need be (if you set up Sphinx with the default settings, you won't need to do this) 37 | 8. Run a catalogsearch_fulltext index 38 | 39 | ## How it works 40 | 41 | ### Magento Default Search 42 | 43 | By default, Magento chucks all searchable attributes into one fulltext column on the catalogsearch_fulltext index, so you might end up with something like the following: 44 | 45 | | fulltext_id | product_id | store_id | data_index | 46 | |-------------|------------|----------|------------------| 47 | | 940753 | 225439 | 9 | CPU Desc etc | 48 | 49 | And magento just does a fulltext search against the data_index column with no weighting on any attributes and also includes things like weight, price, etc. 50 | 51 | ### Sphinx Search 52 | 53 | Sphinx Search overrides the fulltext index and creates a new index table that looks more akin to the following: 54 | 55 | | product_id | store_id | name | name_attributes | category | sku | data_index | 56 | |------------------|----------------|------------|-----------------------|-----------------|-----|-------------------| 57 | | 12345678 | 9 | CPU | 3GHz CPU | Some | Category | Foo | CPU Desc Etc | 58 | 59 | Spinx then creates its own index based on this table and stores it away. The most important parts of this table are the three extra columns name, name_attributes, sku and category. 60 | 61 | 1. Name is just the product name 62 | 2. Name_attributes contains the name + some attributes (configurable in admin) 63 | 3. Category contains the category names 64 | 65 | Using the magento backend, we can assign weights to these columns in terms of ranking search results. 66 | 67 | ### Sphinx Server 68 | 69 | The sphinx server is configured using the file /var/sphinx/sphinx.conf on Linux *(tested on CentOS)* and contains all sorts of goodies. 70 | 71 | Sphinx search has several, much cleverer algorithms for performing searches. A short list of (enabled) features include: 72 | 73 | * Query expansion 74 | * Language parsing (detects plurals etc) 75 | * Misspelling correction (barels => barrels) 76 | * ‘Soundex’ queries (cot => cat) 77 | 78 | As well as many other behind the scenes stuff. It also ranks searches higher when words appear at the beginning of a field. 79 | 80 | For example, a search for ‘Dog’ puts 'dog kennel' before 'log dog'. 81 | 82 | ## Common Problems 83 | 84 | ### Sphinx 85 | 86 | **Sphinx won't work! It keeps saying I don't have permission to do whatever!** 87 | 88 | 1. SSH into your sever / open a terminal window. 89 | 2. Navigate to the mentioned directory 90 | 3. chmod 777 -R {directoryname} 91 | 92 | **My searches are return irrelevant results!** 93 | 94 | Change which attributes are chosen or play with the weights. -------------------------------------------------------------------------------- /app/code/community/Manticorp/SphinxSearch/Helper/Data.php: -------------------------------------------------------------------------------- 1 | 9 | */ 10 | 11 | class Manticorp_SphinxSearch_Helper_Data extends Mage_Core_Helper_Abstract { 12 | 13 | public function getSphinxAdapter() { 14 | require_once(Mage::getBaseDir('lib') . DIRECTORY_SEPARATOR . 'sphinxapi.php'); 15 | 16 | // Connect to our Sphinx Search Engine and run our queries 17 | $sphinx = new SphinxClient(); 18 | 19 | $host = Mage::getStoreConfig('sphinxsearch/server/host'); 20 | $port = Mage::getStoreConfig('sphinxsearch/server/port'); 21 | if (empty($host)) { 22 | return $sphinx; 23 | } 24 | if (empty($port)) { 25 | $port = 9312; 26 | } 27 | $sphinx->SetServer($host, $port); 28 | $sphinx->SetMatchMode(SPH_MATCH_EXTENDED); 29 | $sphinx->setFieldWeights(array( 30 | 'name' => intval(Mage::getStoreConfig('sphinxsearch/search/nameweight')), 31 | 'name_attributes' => intval(Mage::getStoreConfig('sphinxsearch/search/nameattsweight')), 32 | 'category' => intval(Mage::getStoreConfig('sphinxsearch/search/catweight')), 33 | 'data_index' => intval(Mage::getStoreConfig('sphinxsearch/search/defaultweight')), 34 | 'sku' => intval(Mage::getStoreConfig('sphinxsearch/search/skuweight')), 35 | )); 36 | $sphinx->setLimits(0, 200, 1000, 5000); 37 | 38 | // SPH_RANK_PROXIMITY_BM25 is default 39 | $sphinx->SetRankingMode(SPH_RANK_SPH04, ""); // 2nd parameter is rank expr? 40 | 41 | return $sphinx; 42 | } 43 | 44 | /** 45 | * taken from https://gist.github.com/2727341 46 | */ 47 | public function prepareIndexdata($index, $separator = ' ', $entity_id = NULL) 48 | { 49 | $_attributes = array(); 50 | 51 | $exclude = explode(",",Mage::getStoreConfig('sphinxsearch/search/exclude')); 52 | 53 | $_index = array(); 54 | $sku = ''; 55 | foreach ($index as $key => $value) { 56 | 57 | if($key == 'sku'){ 58 | $sku = $value; 59 | } 60 | 61 | // As long as this isn't a standard attribute use it in our 62 | // concatenated column. 63 | if ( ! in_array($key, $exclude)) 64 | { 65 | $_attributes[$key] = $value; 66 | } 67 | 68 | if (!is_array($value)) { 69 | $_index[] = $value; 70 | } 71 | else { 72 | $_index = array_merge($_index, $value); 73 | } 74 | } 75 | 76 | // Get the product name. 77 | $name = ''; 78 | if (isset($index['name'])) { 79 | if (is_array($index['name'])) { 80 | $name = $index['name'][$entity_id]; // Use the configurable product's name 81 | } else { 82 | $name = $index['name']; // Use the simple product's name 83 | } 84 | } 85 | 86 | // Combine the name with each non-standard attribute 87 | $name_attributes = array(); 88 | foreach ($_attributes as $code => $value) 89 | { 90 | if ( ! is_array($value)) 91 | { 92 | $value = array($value); 93 | } 94 | 95 | // Loop through each simple product's attribute values and assign to 96 | // product name. 97 | foreach ($value as $key => $item_value) 98 | { 99 | if (isset($name_attributes[$key])) 100 | { 101 | $name_attributes[$key] .= ' '.$item_value; 102 | } 103 | else 104 | { 105 | // The first time we see this add the name to start. 106 | $name_attributes[$key] = $name.' '.$item_value; 107 | } 108 | } 109 | } 110 | 111 | // Get categories 112 | $categories = array(); 113 | if ($entity_id) 114 | { 115 | $mProduct = Mage::getModel('catalog/product')->load((int) $entity_id); 116 | foreach ($mProduct->getCategoryCollection()->addNameToResult() as $item) { 117 | $categories[] = $item->getName(); 118 | } 119 | } 120 | 121 | $data = array( 122 | 'name' => $name, 123 | 'name_attributes' => implode('. ', $name_attributes), 124 | 'data_index' => implode($separator, $_index), 125 | 'category' => implode('|', $categories), 126 | 'sku' => $sku, 127 | ); 128 | 129 | return $data; 130 | } 131 | 132 | public function getEngine() { 133 | return Mage::getResourceSingleton('sphinxsearch/fulltext_engine'); 134 | } 135 | 136 | } 137 | 138 | ?> -------------------------------------------------------------------------------- /app/code/community/Manticorp/SphinxSearch/Model/Resource/Fulltext.php: -------------------------------------------------------------------------------- 1 | 9 | */ 10 | class Manticorp_SphinxSearch_Model_Resource_Fulltext extends Mage_CatalogSearch_Model_Resource_Fulltext 11 | { 12 | 13 | /** 14 | * Init resource model 15 | * 16 | */ 17 | protected function _construct() 18 | { 19 | // engine is only important fpr indexing 20 | if (!Mage::getStoreConfigFlag('sphinxsearch/active/indexer')) { 21 | return parent::_construct(); 22 | } 23 | 24 | $this->_init('catalogsearch/fulltext', 'product_id'); 25 | $this->_engine = Mage::helper('sphinxsearch')->getEngine(); 26 | } 27 | 28 | /** 29 | * Regenerate search index for store(s) 30 | * 31 | * @param int|null $storeId 32 | * @param int|array|null $productIds 33 | * @return Magendoo_Fulltext_Model_Resource_Fulltext 34 | */ 35 | public function rebuildAllIndexes() 36 | { 37 | 38 | $storeIds = array_keys(Mage::app()->getStores()); 39 | foreach ($storeIds as $storeId) { 40 | $this->_rebuildStoreIndex($storeId); 41 | } 42 | 43 | $adapter = $this->_getWriteAdapter(); 44 | 45 | $this->_engine->swapTables(); 46 | $adapter->truncateTable($this->getTable('catalogsearch/result')); 47 | $adapter->update($this->getTable('catalogsearch/search_query'), array('is_processed' => 0)); 48 | // If we need to change directory 49 | $prevdir = getcwd(); 50 | if( 51 | Mage::getStoreConfig('sphinxsearch/search/sphinxpath') !== '' 52 | && Mage::getStoreConfig('sphinxsearch/search/sphinxpath') !== null 53 | && is_dir(Mage::getStoreConfig('sphinxsearch/search/sphinxpath')) 54 | ){ 55 | chdir(Mage::getStoreConfig('sphinxsearch/search/sphinxpath')); 56 | } 57 | 58 | // sphinx indexer 59 | $output = `indexer --rotate --all --noprogress --quiet`; 60 | chdir($prevdir); 61 | 62 | return $this; 63 | } 64 | 65 | /** 66 | * Regenerate search index for store(s) 67 | * 68 | * @param int|null $storeId 69 | * @param int|array|null $productIds 70 | * @return Mage_CatalogSearch_Model_Resource_Fulltext 71 | */ 72 | public function rebuildIndex($storeId = null, $productIds = null) 73 | { 74 | 75 | // this block added here by Tristan3dtotal, from https://github.com/manticorp/SphinxSearch/issues/3 76 | if ($storeId == null) { 77 | $this->rebuildAllIndexes(); 78 | return $this; 79 | } 80 | 81 | // Use the parent rebuild method 82 | parent::rebuildIndex($storeId, $productIds); 83 | 84 | // If we need to change directory 85 | if( 86 | Mage::getStoreConfigFlag('sphinxsearch/search/sphinxpath') !== '' 87 | && Mage::getStoreConfigFlag('sphinxsearch/search/sphinxpath') !== null 88 | && is_dir(Mage::getStoreConfigFlag('sphinxsearch/search/sphinxpath')) 89 | ){ 90 | chdir(Mage::getStoreConfigFlag('sphinxsearch/search/sphinxpath')); 91 | } 92 | 93 | // sphinx indexer 94 | $output = `indexer --rotate --all --noprogress --quiet`; 95 | 96 | return $this; 97 | } 98 | 99 | /** 100 | * Prepare results for query 101 | * 102 | * @param Mage_CatalogSearch_Model_Fulltext $object 103 | * @param string $queryText 104 | * @param Mage_CatalogSearch_Model_Query $query 105 | * @return Mage_CatalogSearch_Model_Mysql4_Fulltext 106 | */ 107 | public function prepareResult($object, $queryText, $query) 108 | { 109 | if (!Mage::getStoreConfigFlag('sphinxsearch/active/frontend')) { 110 | return parent::prepareResult($object, $queryText, $query); 111 | } 112 | 113 | $sphinx = Mage::helper('sphinxsearch')->getSphinxAdapter(); 114 | 115 | $index = Mage::getStoreConfig('sphinxsearch/server/index'); 116 | 117 | // Here we escape the query - this is important, because certain characters 118 | // will return an error otherwise! 119 | // $queryText = $sphinx->EscapeString($queryText); 120 | $queryText = str_replace('/','\\/',$queryText); 121 | 122 | if (empty($index)) { 123 | $sphinx->AddQuery($queryText); 124 | } else { 125 | $sphinx->AddQuery($queryText, $index); 126 | } 127 | 128 | $results = $sphinx->RunQueries(); 129 | 130 | // Loop through our Sphinx results 131 | if ($results !== false) { 132 | $resultTable = $this->getTable('catalogsearch/result'); 133 | foreach ($results as $item) { 134 | if (empty($item['matches'])) { 135 | continue; 136 | } 137 | 138 | foreach ($item['matches'] as $doc => $docinfo) { 139 | // Ensure we log query results into the Magento table. 140 | $weight = $docinfo['weight'] / 1000; 141 | $sql = sprintf("INSERT INTO `%s` " 142 | . " (`query_id`, `product_id`, `relevance`) VALUES " 143 | . " (%d, %d, %f) " 144 | . " ON DUPLICATE KEY UPDATE `relevance` = %f", 145 | $resultTable, 146 | $query->getId(), 147 | $doc, 148 | $weight, 149 | $weight 150 | ); 151 | try { 152 | $this->_getWriteAdapter()->query($sql); 153 | } catch (Zend_Db_Statement_Exception $e) { 154 | /* 155 | * if the sphinx index is out of date and returns 156 | * product ids which are no longer in the database 157 | * integrity contraint exceptions are thrown. 158 | * we catch them here and simply skip them. 159 | * all other exceptions are forwarded 160 | */ 161 | $message = $e->getMessage(); 162 | if (strpos($message, 'SQLSTATE[23000]: Integrity constraint violation') === false) { 163 | throw $e; 164 | } 165 | } 166 | } 167 | } 168 | } 169 | 170 | $query->setIsProcessed(1); 171 | return $this; 172 | } 173 | 174 | /** 175 | * Prepare Fulltext index value for product 176 | * 177 | * @param array $indexData 178 | * @param array $productData 179 | * @param int $storeId 180 | * @return string 181 | */ 182 | protected function _prepareProductIndex($indexData, $productData, $storeId) 183 | { 184 | if (!Mage::getStoreConfigFlag('sphinxsearch/active/indexer')) { 185 | return parent::_prepareProductIndex($indexData, $productData, $storeId); 186 | } 187 | 188 | $index = array(); 189 | 190 | foreach ($this->_getSearchableAttributes('static') as $attribute) { 191 | $attributeCode = $attribute->getAttributeCode(); 192 | 193 | if (isset($productData[$attributeCode])) { 194 | $value = $this->_getAttributeValue($attribute->getId(), $productData[$attributeCode], $storeId); 195 | if ($value) { 196 | //For grouped products 197 | if (isset($index[$attributeCode])) { 198 | if (!is_array($index[$attributeCode])) { 199 | $index[$attributeCode] = array($index[$attributeCode]); 200 | } 201 | $index[$attributeCode][] = $value; 202 | } 203 | //For other types of products 204 | else { 205 | $index[$attributeCode] = $value; 206 | } 207 | } 208 | } 209 | } 210 | 211 | foreach ($indexData as $entityId => $attributeData) { 212 | foreach ($attributeData as $attributeId => $attributeValue) { 213 | $value = $this->_getAttributeValue($attributeId, $attributeValue, $storeId); 214 | if (!is_null($value) && $value !== false) { 215 | $attributeCode = $this->_getSearchableAttribute($attributeId)->getAttributeCode(); 216 | 217 | if (isset($index[$attributeCode])) { 218 | $index[$attributeCode][$entityId] = $value; 219 | } else { 220 | $index[$attributeCode] = array($entityId => $value); 221 | } 222 | } 223 | } 224 | } 225 | 226 | if (!$this->_engine->allowAdvancedIndex()) { 227 | $product = $this->_getProductEmulator() 228 | ->setId($productData['entity_id']) 229 | ->setTypeId($productData['type_id']) 230 | ->setStoreId($storeId); 231 | $typeInstance = $this->_getProductTypeInstance($productData['type_id']); 232 | if ($data = $typeInstance->getSearchableData($product)) { 233 | $index['options'] = $data; 234 | } 235 | } 236 | 237 | if (isset($productData['in_stock'])) { 238 | $index['in_stock'] = $productData['in_stock']; 239 | } 240 | 241 | return $this->_engine->prepareEntityIndex($index, $this->_separator, $productData['entity_id']); 242 | } 243 | 244 | } 245 | -------------------------------------------------------------------------------- /app/code/community/Manticorp/SphinxSearch/Model/Resource/Fulltext/Engine.php: -------------------------------------------------------------------------------- 1 | 9 | */ 10 | class Manticorp_SphinxSearch_Model_Resource_Fulltext_Engine extends Mage_CatalogSearch_Model_Resource_Fulltext_Engine 11 | { 12 | 13 | protected $_tmpTable = null; 14 | 15 | /** 16 | * Multi add entities data to fulltext search table 17 | * 18 | * @param int $storeId 19 | * @param array $entityIndexes 20 | * @param string $entity 'product'|'cms' 21 | * @return Mage_CatalogSearch_Model_Mysql4_Fulltext_Engine 22 | */ 23 | public function saveEntityIndexes($storeId, $entityIndexes, $entity = 'product') 24 | { 25 | $adapter = $this->_getWriteAdapter(); 26 | $data = array(); 27 | $storeId = (int)$storeId; 28 | foreach ($entityIndexes as $entityId => &$index) { 29 | //Mage::log(get_class($this) . ": saveEntityIndexes for {$entity} {$entityId}..."); 30 | $data[] = array( 31 | 'product_id' => (int)$entityId, 32 | 'store_id' => $storeId, 33 | 'data_index' => $index['data_index'], 34 | 'name' => $index['name'], 35 | 'name_attributes' => $index['name_attributes'], 36 | 'category' => $index['category'], 37 | 'sku' => $index['sku'], 38 | ); 39 | } 40 | 41 | if ($data) { 42 | $adapter->insertOnDuplicate( 43 | $this->getTempTable(), 44 | $data, 45 | array('data_index', 'name', 'name_attributes', 'category', 'sku') 46 | ); 47 | } 48 | 49 | return $this; 50 | } 51 | 52 | public function getMainTable() 53 | { 54 | $tablename = Mage::getSingleton('core/resource')->getTableName('sphinx_catalogsearch_fulltext'); 55 | return $tablename; 56 | // return $this->getTable('sphinxsearch/catalogsearch_fulltext'); 57 | } 58 | 59 | 60 | public function swapTables() { 61 | $adapter = $this->_getWriteAdapter(); 62 | $mainTable = $this->getMainTable(); 63 | $prevTable = $this->getMainTable().'_prev'; 64 | $tempTable = $this->getTempTable(); 65 | 66 | $adapter->dropTable($prevTable); 67 | $adapter->renameTablesBatch(array(array('oldName' => $mainTable, 'newName' => $prevTable), array('oldName' => $tempTable, 'newName' => $mainTable))); // Tristan3dtotal 68 | //$adapter->renameTable($mainTable, $prevTable); 69 | $adapter->dropTable($prevTable); // Tristan3dtotal 70 | } 71 | 72 | 73 | 74 | public function getTempTable() { 75 | if(is_null($this->_tmpTable)) { 76 | $adapter = $this->_getWriteAdapter(); 77 | $mainTable = $this->getMainTable(); 78 | $this->_tmpTable = $mainTable.'_tmp'; 79 | if ($adapter->isTableExists($this->_tmpTable)) { // Tristan3dtotal 80 | $adapter->dropTable($this->_tmpTable); 81 | } // Tristan3dtotal 82 | $adapter->createTable($adapter->createTableByDdl($mainTable, $this->_tmpTable)); // Tristan3dtotal 83 | } 84 | return $this->_tmpTable; 85 | } 86 | 87 | /** 88 | * Remove entity data from fulltext search table 89 | * 90 | * @param int $storeId 91 | * @param int $entityId 92 | * @param string $entity 'product'|'cms' 93 | * @return Mage_CatalogSearch_Model_Mysql4_Fulltext_Engine 94 | */ 95 | public function cleanIndex($storeId = null, $entityId = null, $entity = 'product') 96 | { 97 | $where = array(); 98 | 99 | if (!is_null($storeId)) { 100 | $where[] = $this->_getWriteAdapter()->quoteInto('store_id=?', $storeId); 101 | } 102 | if (!is_null($entityId)) { 103 | $where[] = $this->_getWriteAdapter()->quoteInto('product_id IN(?)', $entityId); 104 | } 105 | 106 | $this->_getWriteAdapter()->delete($this->getMainTable(), join(' AND ', $where)); 107 | 108 | return $this; 109 | } 110 | 111 | /** 112 | * Prepare index array as a string glued by separator 113 | * 114 | * @param array $index 115 | * @param string $separator 116 | * @return string 117 | */ 118 | public function prepareEntityIndex($index, $separator = ' ', $entity_id = NULL) 119 | { 120 | return Mage::helper('sphinxsearch')->prepareIndexdata($index, $separator, $entity_id); 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /app/code/community/Manticorp/SphinxSearch/Model/System/Config/Source/Product/Attributes.php: -------------------------------------------------------------------------------- 1 | loadByCode('catalog_product')->getEntityTypeId(); 17 | $attributes = Mage::getModel('eav/entity_attribute') 18 | ->getCollection() 19 | ->addFilter('entity_type_id', $entityTypeId) 20 | ->addFilter('is_user_defined', 1) 21 | ->addFilter('is_unique', 0) 22 | // ->addFilter('frontend_input', 'text') 23 | ->addFieldToFilter('backend_type', array(array('eq'=>'int'),array('eq'=>'decimal'))) 24 | ->setOrder('attribute_code', 'ASC'); 25 | foreach ($attributes as $attribute) { 26 | $item = array(); 27 | $item['value'] = $attribute->getAttributeCode(); 28 | $item['label'] = $attribute->getAttributeCode(); 29 | $options[] = $item; 30 | } 31 | return $options; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /app/code/community/Manticorp/SphinxSearch/Model/System/Config/Source/Product/Attributes/Search.php: -------------------------------------------------------------------------------- 1 | getSearchableAttributes(); 17 | foreach($atts as $attribute){ 18 | $item = array(); 19 | $item['value'] = $attribute->getAttributeCode(); 20 | $item['label'] = $attribute->getAttributeCode(); 21 | $options[] = $item; 22 | } 23 | return $options; 24 | } 25 | 26 | public function getSearchableAttributes() 27 | { 28 | $searchableAttributes = array(); 29 | 30 | $productAttributeCollection = Mage::getResourceModel('catalog/product_attribute_collection'); 31 | $productAttributeCollection->addSearchableAttributeFilter(); 32 | $attributes = $productAttributeCollection->getItems(); 33 | 34 | Mage::dispatchEvent('catalogsearch_searchable_attributes_load_after', array( 35 | 'engine' => $this->_engine, 36 | 'attributes' => $attributes 37 | )); 38 | 39 | $entity = $this->getEavConfig() 40 | ->getEntityType(Mage_Catalog_Model_Product::ENTITY) 41 | ->getEntity(); 42 | 43 | foreach ($attributes as $attribute) { 44 | $attribute->setEntity($entity); 45 | } 46 | 47 | $searchableAttributes = $attributes; 48 | 49 | if (!is_null($backendType)) { 50 | $attributes = array(); 51 | foreach ($searchableAttributes as $attributeId => $attribute) { 52 | if ($attribute->getBackendType() == $backendType) { 53 | $attributes[$attributeId] = $attribute; 54 | } 55 | } 56 | 57 | return $attributes; 58 | } 59 | 60 | return $searchableAttributes; 61 | } 62 | 63 | /** 64 | * Retrieve EAV Config Singleton 65 | * 66 | * @return Mage_Eav_Model_Config 67 | */ 68 | public function getEavConfig() 69 | { 70 | return Mage::getSingleton('eav/config'); 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /app/code/community/Manticorp/SphinxSearch/etc/adminhtml.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | Sphinx Search 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | -------------------------------------------------------------------------------- /app/code/community/Manticorp/SphinxSearch/etc/config.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 0.1.1 6 | 7 | 8 | 9 | 10 | 11 | Manticorp_SphinxSearch_Block 12 | 13 | 14 | 15 | 16 | Manticorp_SphinxSearch_Helper 17 | 18 | 19 | 20 | 21 | Manticorp_SphinxSearch_Model 22 | sphinxsearch_resource 23 | 24 | 25 | Manticorp_SphinxSearch_Model_Resource 26 | 27 | 28 | 29 | Manticorp_SphinxSearch_Model_Resource_Fulltext 30 | 31 | 32 | 33 | 34 | 35 | 36 | Manticorp_SphinxSearch 37 | Mage_Eav_Model_Entity_Setup 38 | 39 | core_setup 40 | 41 | 42 | core_write 43 | 44 | 45 | core_read 46 | 47 | 48 | 49 | 50 | 51 | 52 | 0 53 | 0 54 | 55 | 56 | description,short_description,meta_keywords,meta_title,tax_class_id,price,visibility,status,weight 57 | 7 58 | 100 59 | 1 60 | 3 61 | 1 62 | 63 | 64 | localhost 65 | 9312 66 | 67 | 68 | 69 | 70 | 71 | -------------------------------------------------------------------------------- /app/code/community/Manticorp/SphinxSearch/etc/system.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | catalog 7 | 9000 8 | 1 9 | 1 10 | 1 11 | 12 | 13 | 14 | text 15 | 1 16 | 1 17 | 1 18 | 1 19 | Manticorp/SphinxSearch on GitHub for bug reports etc.]]> 20 | 21 | 22 | 23 | select 24 | adminhtml/system_config_source_yesno 25 | 1 26 | 1 27 | 1 28 | 1 29 | 30 | 31 | 32 | 33 | select 34 | adminhtml/system_config_source_yesno 35 | 2 36 | 1 37 | 1 38 | 1 39 | 40 | 41 | 42 | 43 | 44 | 45 | text 46 | 2 47 | 1 48 | 1 49 | 1 50 | 51 | 52 | 53 | text 54 | 10 55 | 1 56 | 1 57 | 1 58 | 59 | 60 | 61 | 62 | multiselect 63 | 20 64 | 1 65 | 1 66 | 1 67 | sphinxsearch/system_config_source_product_attributes_search 68 | 69 | 70 | 71 | 72 | text 73 | 30 74 | 1 75 | 1 76 | 1 77 | validate-number 78 | 79 | 80 | 81 | text 82 | 35 83 | 1 84 | 1 85 | 1 86 | validate-number 87 | 88 | 89 | 90 | text 91 | 40 92 | 1 93 | 1 94 | 1 95 | validate-number 96 | 97 | 98 | 99 | text 100 | 50 101 | 1 102 | 1 103 | 1 104 | validate-number 105 | 106 | 107 | 108 | text 109 | 60 110 | 1 111 | 1 112 | 1 113 | validate-number 114 | 115 | 116 | 117 | 118 | 119 | text 120 | 3 121 | 1 122 | 1 123 | 1 124 | 125 | 126 | 127 | text 128 | 1 129 | 1 130 | 1 131 | 1 132 | 133 | 134 | 135 | text 136 | 2 137 | 1 138 | 1 139 | 1 140 | 141 | 142 | 143 | text 144 | 3 145 | 1 146 | 1 147 | 1 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | -------------------------------------------------------------------------------- /app/code/community/Manticorp/SphinxSearch/sql/sphinxsearch_setup/install-0.1.0.php: -------------------------------------------------------------------------------- 1 | 9 | */ 10 | 11 | $installer = $this; 12 | /* $installer Mage_Core_Model_Resource_Setup */ 13 | 14 | $installer->startSetup(); 15 | 16 | $table = $installer->getConnection() 17 | ->newTable($installer->getTable('sphinx_catalogsearch_fulltext')) 18 | ->addColumn('product_id', Varien_Db_Ddl_Table::TYPE_INTEGER, null, array( 19 | 'unsigned' => true, 20 | 'nullable' => false, 21 | ), 'Product ID') 22 | ->addColumn('store_id', Varien_Db_Ddl_Table::TYPE_SMALLINT, null, array( 23 | 'nullable' => false, 24 | ), 'Store ID') 25 | ->addColumn('name', Varien_Db_Ddl_Table::TYPE_VARCHAR, 255, array( 26 | 'nullable' => false, 27 | ), 'Product Name') 28 | ->addColumn('name_attributes', Varien_Db_Ddl_Table::TYPE_TEXT, null, array( 29 | 'nullable' => false, 30 | ), 'Product Name + selected attributes') 31 | ->addColumn('category', Varien_Db_Ddl_Table::TYPE_VARCHAR, 255, array( 32 | 'nullable' => true, 33 | ), 'Categories (with separator)') 34 | ->addColumn('data_index', Varien_Db_Ddl_Table::TYPE_TEXT, null, array( 35 | 'nullable' => false, 36 | ), 'Original Magento data_index') 37 | ->addIndex( 38 | $installer->getIdxName( 39 | $installer->getTable('sphinx_catalogsearch_fulltext'), 40 | array('product_id','store_id'), 41 | Varien_Db_Adapter_Interface::INDEX_TYPE_PRIMARY 42 | ), 43 | array('product_id','store_id'), 44 | array( 45 | 'type' => Varien_Db_Adapter_Interface::INDEX_TYPE_PRIMARY 46 | ) 47 | ) 48 | ->addIndex( 49 | $installer->getIdxName( 50 | $installer->getTable('sphinx_catalogsearch_fulltext'), 51 | array('data_index'), 52 | Varien_Db_Adapter_Interface::INDEX_TYPE_FULLTEXT 53 | ), 54 | array('data_index'), 55 | array( 56 | 'type' => Varien_Db_Adapter_Interface::INDEX_TYPE_FULLTEXT 57 | ) 58 | )->setOption('type','MyISAM'); 59 | $installer->getConnection()->createTable($table); 60 | 61 | $installer->endSetup(); -------------------------------------------------------------------------------- /app/code/community/Manticorp/SphinxSearch/sql/sphinxsearch_setup/upgrade-0.1.0-0.1.1.php: -------------------------------------------------------------------------------- 1 | 9 | */ 10 | 11 | /* $installer Mage_Core_Model_Resource_Setup */ 12 | $installer = $this; 13 | $connection = $installer->getConnection(); 14 | 15 | $installer->startSetup(); 16 | 17 | $connection->addColumn( 18 | $installer->getTable('sphinx_catalogsearch_fulltext'), 19 | 'sku', 20 | array( 21 | 'type' => Varien_Db_Ddl_Table::TYPE_TEXT, 22 | 'length' => 255, 23 | 'position' => 3, 24 | 'nullable' => false, 25 | 'comment' => 'SKU' 26 | ) 27 | ); 28 | 29 | $installer->endSetup(); -------------------------------------------------------------------------------- /app/etc/modules/Manticorp_SphinxSearch.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | true 6 | community 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /lib/sphinxapi.php: -------------------------------------------------------------------------------- 1 | =8 ) 139 | { 140 | $v = (int)$v; 141 | return pack ( "NN", $v>>32, $v&0xFFFFFFFF ); 142 | } 143 | 144 | // x32, int 145 | if ( is_int($v) ) 146 | return pack ( "NN", $v < 0 ? -1 : 0, $v ); 147 | 148 | // x32, bcmath 149 | if ( function_exists("bcmul") ) 150 | { 151 | if ( bccomp ( $v, 0 ) == -1 ) 152 | $v = bcadd ( "18446744073709551616", $v ); 153 | $h = bcdiv ( $v, "4294967296", 0 ); 154 | $l = bcmod ( $v, "4294967296" ); 155 | return pack ( "NN", (float)$h, (float)$l ); // conversion to float is intentional; int would lose 31st bit 156 | } 157 | 158 | // x32, no-bcmath 159 | $p = max(0, strlen($v) - 13); 160 | $lo = abs((float)substr($v, $p)); 161 | $hi = abs((float)substr($v, 0, $p)); 162 | 163 | $m = $lo + $hi*1316134912.0; // (10 ^ 13) % (1 << 32) = 1316134912 164 | $q = floor($m/4294967296.0); 165 | $l = $m - ($q*4294967296.0); 166 | $h = $hi*2328.0 + $q; // (10 ^ 13) / (1 << 32) = 2328 167 | 168 | if ( $v<0 ) 169 | { 170 | if ( $l==0 ) 171 | $h = 4294967296.0 - $h; 172 | else 173 | { 174 | $h = 4294967295.0 - $h; 175 | $l = 4294967296.0 - $l; 176 | } 177 | } 178 | return pack ( "NN", $h, $l ); 179 | } 180 | 181 | /// pack 64-bit unsigned 182 | function sphPackU64 ( $v ) 183 | { 184 | assert ( is_numeric($v) ); 185 | 186 | // x64 187 | if ( PHP_INT_SIZE>=8 ) 188 | { 189 | assert ( $v>=0 ); 190 | 191 | // x64, int 192 | if ( is_int($v) ) 193 | return pack ( "NN", $v>>32, $v&0xFFFFFFFF ); 194 | 195 | // x64, bcmath 196 | if ( function_exists("bcmul") ) 197 | { 198 | $h = bcdiv ( $v, 4294967296, 0 ); 199 | $l = bcmod ( $v, 4294967296 ); 200 | return pack ( "NN", $h, $l ); 201 | } 202 | 203 | // x64, no-bcmath 204 | $p = max ( 0, strlen($v) - 13 ); 205 | $lo = (int)substr ( $v, $p ); 206 | $hi = (int)substr ( $v, 0, $p ); 207 | 208 | $m = $lo + $hi*1316134912; 209 | $l = $m % 4294967296; 210 | $h = $hi*2328 + (int)($m/4294967296); 211 | 212 | return pack ( "NN", $h, $l ); 213 | } 214 | 215 | // x32, int 216 | if ( is_int($v) ) 217 | return pack ( "NN", 0, $v ); 218 | 219 | // x32, bcmath 220 | if ( function_exists("bcmul") ) 221 | { 222 | $h = bcdiv ( $v, "4294967296", 0 ); 223 | $l = bcmod ( $v, "4294967296" ); 224 | return pack ( "NN", (float)$h, (float)$l ); // conversion to float is intentional; int would lose 31st bit 225 | } 226 | 227 | // x32, no-bcmath 228 | $p = max(0, strlen($v) - 13); 229 | $lo = (float)substr($v, $p); 230 | $hi = (float)substr($v, 0, $p); 231 | 232 | $m = $lo + $hi*1316134912.0; 233 | $q = floor($m / 4294967296.0); 234 | $l = $m - ($q * 4294967296.0); 235 | $h = $hi*2328.0 + $q; 236 | 237 | return pack ( "NN", $h, $l ); 238 | } 239 | 240 | // unpack 64-bit unsigned 241 | function sphUnpackU64 ( $v ) 242 | { 243 | list ( $hi, $lo ) = array_values ( unpack ( "N*N*", $v ) ); 244 | 245 | if ( PHP_INT_SIZE>=8 ) 246 | { 247 | if ( $hi<0 ) $hi += (1<<32); // because php 5.2.2 to 5.2.5 is totally fucked up again 248 | if ( $lo<0 ) $lo += (1<<32); 249 | 250 | // x64, int 251 | if ( $hi<=2147483647 ) 252 | return ($hi<<32) + $lo; 253 | 254 | // x64, bcmath 255 | if ( function_exists("bcmul") ) 256 | return bcadd ( $lo, bcmul ( $hi, "4294967296" ) ); 257 | 258 | // x64, no-bcmath 259 | $C = 100000; 260 | $h = ((int)($hi / $C) << 32) + (int)($lo / $C); 261 | $l = (($hi % $C) << 32) + ($lo % $C); 262 | if ( $l>$C ) 263 | { 264 | $h += (int)($l / $C); 265 | $l = $l % $C; 266 | } 267 | 268 | if ( $h==0 ) 269 | return $l; 270 | return sprintf ( "%d%05d", $h, $l ); 271 | } 272 | 273 | // x32, int 274 | if ( $hi==0 ) 275 | { 276 | if ( $lo>0 ) 277 | return $lo; 278 | return sprintf ( "%u", $lo ); 279 | } 280 | 281 | $hi = sprintf ( "%u", $hi ); 282 | $lo = sprintf ( "%u", $lo ); 283 | 284 | // x32, bcmath 285 | if ( function_exists("bcmul") ) 286 | return bcadd ( $lo, bcmul ( $hi, "4294967296" ) ); 287 | 288 | // x32, no-bcmath 289 | $hi = (float)$hi; 290 | $lo = (float)$lo; 291 | 292 | $q = floor($hi/10000000.0); 293 | $r = $hi - $q*10000000.0; 294 | $m = $lo + $r*4967296.0; 295 | $mq = floor($m/10000000.0); 296 | $l = $m - $mq*10000000.0; 297 | $h = $q*4294967296.0 + $r*429.0 + $mq; 298 | 299 | $h = sprintf ( "%.0f", $h ); 300 | $l = sprintf ( "%07.0f", $l ); 301 | if ( $h=="0" ) 302 | return sprintf( "%.0f", (float)$l ); 303 | return $h . $l; 304 | } 305 | 306 | // unpack 64-bit signed 307 | function sphUnpackI64 ( $v ) 308 | { 309 | list ( $hi, $lo ) = array_values ( unpack ( "N*N*", $v ) ); 310 | 311 | // x64 312 | if ( PHP_INT_SIZE>=8 ) 313 | { 314 | if ( $hi<0 ) $hi += (1<<32); // because php 5.2.2 to 5.2.5 is totally fucked up again 315 | if ( $lo<0 ) $lo += (1<<32); 316 | 317 | return ($hi<<32) + $lo; 318 | } 319 | 320 | // x32, int 321 | if ( $hi==0 ) 322 | { 323 | if ( $lo>0 ) 324 | return $lo; 325 | return sprintf ( "%u", $lo ); 326 | } 327 | // x32, int 328 | elseif ( $hi==-1 ) 329 | { 330 | if ( $lo<0 ) 331 | return $lo; 332 | return sprintf ( "%.0f", $lo - 4294967296.0 ); 333 | } 334 | 335 | $neg = ""; 336 | $c = 0; 337 | if ( $hi<0 ) 338 | { 339 | $hi = ~$hi; 340 | $lo = ~$lo; 341 | $c = 1; 342 | $neg = "-"; 343 | } 344 | 345 | $hi = sprintf ( "%u", $hi ); 346 | $lo = sprintf ( "%u", $lo ); 347 | 348 | // x32, bcmath 349 | if ( function_exists("bcmul") ) 350 | return $neg . bcadd ( bcadd ( $lo, bcmul ( $hi, "4294967296" ) ), $c ); 351 | 352 | // x32, no-bcmath 353 | $hi = (float)$hi; 354 | $lo = (float)$lo; 355 | 356 | $q = floor($hi/10000000.0); 357 | $r = $hi - $q*10000000.0; 358 | $m = $lo + $r*4967296.0; 359 | $mq = floor($m/10000000.0); 360 | $l = $m - $mq*10000000.0 + $c; 361 | $h = $q*4294967296.0 + $r*429.0 + $mq; 362 | if ( $l==10000000 ) 363 | { 364 | $l = 0; 365 | $h += 1; 366 | } 367 | 368 | $h = sprintf ( "%.0f", $h ); 369 | $l = sprintf ( "%07.0f", $l ); 370 | if ( $h=="0" ) 371 | return $neg . sprintf( "%.0f", (float)$l ); 372 | return $neg . $h . $l; 373 | } 374 | 375 | 376 | function sphFixUint ( $value ) 377 | { 378 | if ( PHP_INT_SIZE>=8 ) 379 | { 380 | // x64 route, workaround broken unpack() in 5.2.2+ 381 | if ( $value<0 ) $value += (1<<32); 382 | return $value; 383 | } 384 | else 385 | { 386 | // x32 route, workaround php signed/unsigned braindamage 387 | return sprintf ( "%u", $value ); 388 | } 389 | } 390 | 391 | function sphSetBit ( $flag, $bit, $on ) 392 | { 393 | if ( $on ) 394 | { 395 | $flag += ( 1<<$bit ); 396 | } else 397 | { 398 | $reset = 255 ^ ( 1<<$bit ); 399 | $flag = $flag & $reset; 400 | } 401 | return $flag; 402 | } 403 | 404 | 405 | /// sphinx searchd client class 406 | class SphinxClient 407 | { 408 | var $_host; ///< searchd host (default is "localhost") 409 | var $_port; ///< searchd port (default is 9312) 410 | var $_offset; ///< how many records to seek from result-set start (default is 0) 411 | var $_limit; ///< how many records to return from result-set starting at offset (default is 20) 412 | var $_mode; ///< query matching mode (default is SPH_MATCH_EXTENDED2) 413 | var $_weights; ///< per-field weights (default is 1 for all fields) 414 | var $_sort; ///< match sorting mode (default is SPH_SORT_RELEVANCE) 415 | var $_sortby; ///< attribute to sort by (defualt is "") 416 | var $_min_id; ///< min ID to match (default is 0, which means no limit) 417 | var $_max_id; ///< max ID to match (default is 0, which means no limit) 418 | var $_filters; ///< search filters 419 | var $_groupby; ///< group-by attribute name 420 | var $_groupfunc; ///< group-by function (to pre-process group-by attribute value with) 421 | var $_groupsort; ///< group-by sorting clause (to sort groups in result set with) 422 | var $_groupdistinct;///< group-by count-distinct attribute 423 | var $_maxmatches; ///< max matches to retrieve 424 | var $_cutoff; ///< cutoff to stop searching at (default is 0) 425 | var $_retrycount; ///< distributed retries count 426 | var $_retrydelay; ///< distributed retries delay 427 | var $_anchor; ///< geographical anchor point 428 | var $_indexweights; ///< per-index weights 429 | var $_ranker; ///< ranking mode (default is SPH_RANK_PROXIMITY_BM25) 430 | var $_rankexpr; ///< ranking mode expression (for SPH_RANK_EXPR) 431 | var $_maxquerytime; ///< max query time, milliseconds (default is 0, do not limit) 432 | var $_fieldweights; ///< per-field-name weights 433 | var $_overrides; ///< per-query attribute values overrides 434 | var $_select; ///< select-list (attributes or expressions, with optional aliases) 435 | var $_query_flags; ///< per-query various flags 436 | var $_predictedtime; ///< per-query max_predicted_time 437 | var $_outerorderby; ///< outer match sort by 438 | var $_outeroffset; ///< outer offset 439 | var $_outerlimit; ///< outer limit 440 | var $_hasouter; 441 | 442 | var $_error; ///< last error message 443 | var $_warning; ///< last warning message 444 | var $_connerror; ///< connection error vs remote error flag 445 | 446 | var $_reqs; ///< requests array for multi-query 447 | var $_mbenc; ///< stored mbstring encoding 448 | var $_arrayresult; ///< whether $result["matches"] should be a hash or an array 449 | var $_timeout; ///< connect timeout 450 | 451 | ///////////////////////////////////////////////////////////////////////////// 452 | // common stuff 453 | ///////////////////////////////////////////////////////////////////////////// 454 | 455 | /// create a new client object and fill defaults 456 | function SphinxClient () 457 | { 458 | // per-client-object settings 459 | $this->_host = "localhost"; 460 | $this->_port = 9312; 461 | $this->_path = false; 462 | $this->_socket = false; 463 | 464 | // per-query settings 465 | $this->_offset = 0; 466 | $this->_limit = 20; 467 | $this->_mode = SPH_MATCH_EXTENDED2; 468 | $this->_weights = array (); 469 | $this->_sort = SPH_SORT_RELEVANCE; 470 | $this->_sortby = ""; 471 | $this->_min_id = 0; 472 | $this->_max_id = 0; 473 | $this->_filters = array (); 474 | $this->_groupby = ""; 475 | $this->_groupfunc = SPH_GROUPBY_DAY; 476 | $this->_groupsort = "@group desc"; 477 | $this->_groupdistinct= ""; 478 | $this->_maxmatches = 1000; 479 | $this->_cutoff = 0; 480 | $this->_retrycount = 0; 481 | $this->_retrydelay = 0; 482 | $this->_anchor = array (); 483 | $this->_indexweights= array (); 484 | $this->_ranker = SPH_RANK_PROXIMITY_BM25; 485 | $this->_rankexpr = ""; 486 | $this->_maxquerytime= 0; 487 | $this->_fieldweights= array(); 488 | $this->_overrides = array(); 489 | $this->_select = "*"; 490 | $this->_query_flags = sphSetBit ( 0, 6, true ); // default idf=tfidf_normalized 491 | $this->_predictedtime = 0; 492 | $this->_outerorderby = ""; 493 | $this->_outeroffset = 0; 494 | $this->_outerlimit = 0; 495 | $this->_hasouter = false; 496 | 497 | $this->_error = ""; // per-reply fields (for single-query case) 498 | $this->_warning = ""; 499 | $this->_connerror = false; 500 | 501 | $this->_reqs = array (); // requests storage (for multi-query case) 502 | $this->_mbenc = ""; 503 | $this->_arrayresult = false; 504 | $this->_timeout = 0; 505 | } 506 | 507 | function __destruct() 508 | { 509 | if ( $this->_socket !== false ) 510 | fclose ( $this->_socket ); 511 | } 512 | 513 | /// get last error message (string) 514 | function GetLastError () 515 | { 516 | return $this->_error; 517 | } 518 | 519 | /// get last warning message (string) 520 | function GetLastWarning () 521 | { 522 | return $this->_warning; 523 | } 524 | 525 | /// get last error flag (to tell network connection errors from searchd errors or broken responses) 526 | function IsConnectError() 527 | { 528 | return $this->_connerror; 529 | } 530 | 531 | /// set searchd host name (string) and port (integer) 532 | function SetServer ( $host, $port = 0 ) 533 | { 534 | assert ( is_string($host) ); 535 | if ( $host[0] == '/') 536 | { 537 | $this->_path = 'unix://' . $host; 538 | return; 539 | } 540 | if ( substr ( $host, 0, 7 )=="unix://" ) 541 | { 542 | $this->_path = $host; 543 | return; 544 | } 545 | 546 | $this->_host = $host; 547 | $port = intval($port); 548 | assert ( 0<=$port && $port<65536 ); 549 | $this->_port = ( $port==0 ) ? 9312 : $port; 550 | $this->_path = ''; 551 | } 552 | 553 | /// set server connection timeout (0 to remove) 554 | function SetConnectTimeout ( $timeout ) 555 | { 556 | assert ( is_numeric($timeout) ); 557 | $this->_timeout = $timeout; 558 | } 559 | 560 | 561 | function _Send ( $handle, $data, $length ) 562 | { 563 | if ( feof($handle) || fwrite ( $handle, $data, $length ) !== $length ) 564 | { 565 | $this->_error = 'connection unexpectedly closed (timed out?)'; 566 | $this->_connerror = true; 567 | return false; 568 | } 569 | return true; 570 | } 571 | 572 | ///////////////////////////////////////////////////////////////////////////// 573 | 574 | /// enter mbstring workaround mode 575 | function _MBPush () 576 | { 577 | $this->_mbenc = ""; 578 | if ( ini_get ( "mbstring.func_overload" ) & 2 ) 579 | { 580 | $this->_mbenc = mb_internal_encoding(); 581 | mb_internal_encoding ( "latin1" ); 582 | } 583 | } 584 | 585 | /// leave mbstring workaround mode 586 | function _MBPop () 587 | { 588 | if ( $this->_mbenc ) 589 | mb_internal_encoding ( $this->_mbenc ); 590 | } 591 | 592 | /// connect to searchd server 593 | function _Connect () 594 | { 595 | if ( $this->_socket!==false ) 596 | { 597 | // we are in persistent connection mode, so we have a socket 598 | // however, need to check whether it's still alive 599 | if ( !@feof ( $this->_socket ) ) 600 | return $this->_socket; 601 | 602 | // force reopen 603 | $this->_socket = false; 604 | } 605 | 606 | $errno = 0; 607 | $errstr = ""; 608 | $this->_connerror = false; 609 | 610 | if ( $this->_path ) 611 | { 612 | $host = $this->_path; 613 | $port = 0; 614 | } 615 | else 616 | { 617 | $host = $this->_host; 618 | $port = $this->_port; 619 | } 620 | 621 | if ( $this->_timeout<=0 ) 622 | $fp = @fsockopen ( $host, $port, $errno, $errstr ); 623 | else 624 | $fp = @fsockopen ( $host, $port, $errno, $errstr, $this->_timeout ); 625 | 626 | if ( !$fp ) 627 | { 628 | if ( $this->_path ) 629 | $location = $this->_path; 630 | else 631 | $location = "{$this->_host}:{$this->_port}"; 632 | 633 | $errstr = trim ( $errstr ); 634 | $this->_error = "connection to $location failed (errno=$errno, msg=$errstr)"; 635 | $this->_connerror = true; 636 | return false; 637 | } 638 | 639 | // send my version 640 | // this is a subtle part. we must do it before (!) reading back from searchd. 641 | // because otherwise under some conditions (reported on FreeBSD for instance) 642 | // TCP stack could throttle write-write-read pattern because of Nagle. 643 | if ( !$this->_Send ( $fp, pack ( "N", 1 ), 4 ) ) 644 | { 645 | fclose ( $fp ); 646 | $this->_error = "failed to send client protocol version"; 647 | return false; 648 | } 649 | 650 | // check version 651 | list(,$v) = unpack ( "N*", fread ( $fp, 4 ) ); 652 | $v = (int)$v; 653 | if ( $v<1 ) 654 | { 655 | fclose ( $fp ); 656 | $this->_error = "expected searchd protocol version 1+, got version '$v'"; 657 | return false; 658 | } 659 | 660 | return $fp; 661 | } 662 | 663 | /// get and check response packet from searchd server 664 | function _GetResponse ( $fp, $client_ver ) 665 | { 666 | $response = ""; 667 | $len = 0; 668 | 669 | $header = fread ( $fp, 8 ); 670 | if ( strlen($header)==8 ) 671 | { 672 | list ( $status, $ver, $len ) = array_values ( unpack ( "n2a/Nb", $header ) ); 673 | $left = $len; 674 | while ( $left>0 && !feof($fp) ) 675 | { 676 | $chunk = fread ( $fp, min ( 8192, $left ) ); 677 | if ( $chunk ) 678 | { 679 | $response .= $chunk; 680 | $left -= strlen($chunk); 681 | } 682 | } 683 | } 684 | if ( $this->_socket === false ) 685 | fclose ( $fp ); 686 | 687 | // check response 688 | $read = strlen ( $response ); 689 | if ( !$response || $read!=$len ) 690 | { 691 | $this->_error = $len 692 | ? "failed to read searchd response (status=$status, ver=$ver, len=$len, read=$read)" 693 | : "received zero-sized searchd response"; 694 | return false; 695 | } 696 | 697 | // check status 698 | if ( $status==SEARCHD_WARNING ) 699 | { 700 | list(,$wlen) = unpack ( "N*", substr ( $response, 0, 4 ) ); 701 | $this->_warning = substr ( $response, 4, $wlen ); 702 | return substr ( $response, 4+$wlen ); 703 | } 704 | if ( $status==SEARCHD_ERROR ) 705 | { 706 | $this->_error = "searchd error: " . substr ( $response, 4 ); 707 | return false; 708 | } 709 | if ( $status==SEARCHD_RETRY ) 710 | { 711 | $this->_error = "temporary searchd error: " . substr ( $response, 4 ); 712 | return false; 713 | } 714 | if ( $status!=SEARCHD_OK ) 715 | { 716 | $this->_error = "unknown status code '$status'"; 717 | return false; 718 | } 719 | 720 | // check version 721 | if ( $ver<$client_ver ) 722 | { 723 | $this->_warning = sprintf ( "searchd command v.%d.%d older than client's v.%d.%d, some options might not work", 724 | $ver>>8, $ver&0xff, $client_ver>>8, $client_ver&0xff ); 725 | } 726 | 727 | return $response; 728 | } 729 | 730 | ///////////////////////////////////////////////////////////////////////////// 731 | // searching 732 | ///////////////////////////////////////////////////////////////////////////// 733 | 734 | /// set offset and count into result set, 735 | /// and optionally set max-matches and cutoff limits 736 | function SetLimits ( $offset, $limit, $max=0, $cutoff=0 ) 737 | { 738 | assert ( is_int($offset) ); 739 | assert ( is_int($limit) ); 740 | assert ( $offset>=0 ); 741 | assert ( $limit>0 ); 742 | assert ( $max>=0 ); 743 | $this->_offset = $offset; 744 | $this->_limit = $limit; 745 | if ( $max>0 ) 746 | $this->_maxmatches = $max; 747 | if ( $cutoff>0 ) 748 | $this->_cutoff = $cutoff; 749 | } 750 | 751 | /// set maximum query time, in milliseconds, per-index 752 | /// integer, 0 means "do not limit" 753 | function SetMaxQueryTime ( $max ) 754 | { 755 | assert ( is_int($max) ); 756 | assert ( $max>=0 ); 757 | $this->_maxquerytime = $max; 758 | } 759 | 760 | /// set matching mode 761 | function SetMatchMode ( $mode ) 762 | { 763 | trigger_error ( 'DEPRECATED: Do not call this method or, even better, use SphinxQL instead of an API', E_USER_DEPRECATED ); 764 | assert ( $mode==SPH_MATCH_ALL 765 | || $mode==SPH_MATCH_ANY 766 | || $mode==SPH_MATCH_PHRASE 767 | || $mode==SPH_MATCH_BOOLEAN 768 | || $mode==SPH_MATCH_EXTENDED 769 | || $mode==SPH_MATCH_FULLSCAN 770 | || $mode==SPH_MATCH_EXTENDED2 ); 771 | $this->_mode = $mode; 772 | } 773 | 774 | /// set ranking mode 775 | function SetRankingMode ( $ranker, $rankexpr="" ) 776 | { 777 | assert ( $ranker===0 || $ranker>=1 && $ranker_ranker = $ranker; 780 | $this->_rankexpr = $rankexpr; 781 | } 782 | 783 | /// set matches sorting mode 784 | function SetSortMode ( $mode, $sortby="" ) 785 | { 786 | assert ( 787 | $mode==SPH_SORT_RELEVANCE || 788 | $mode==SPH_SORT_ATTR_DESC || 789 | $mode==SPH_SORT_ATTR_ASC || 790 | $mode==SPH_SORT_TIME_SEGMENTS || 791 | $mode==SPH_SORT_EXTENDED || 792 | $mode==SPH_SORT_EXPR ); 793 | assert ( is_string($sortby) ); 794 | assert ( $mode==SPH_SORT_RELEVANCE || strlen($sortby)>0 ); 795 | 796 | $this->_sort = $mode; 797 | $this->_sortby = $sortby; 798 | } 799 | 800 | /// bind per-field weights by order 801 | /// DEPRECATED; use SetFieldWeights() instead 802 | function SetWeights ( $weights ) 803 | { 804 | die("This method is now deprecated; please use SetFieldWeights instead"); 805 | } 806 | 807 | /// bind per-field weights by name 808 | function SetFieldWeights ( $weights ) 809 | { 810 | assert ( is_array($weights) ); 811 | foreach ( $weights as $name=>$weight ) 812 | { 813 | assert ( is_string($name) ); 814 | assert ( is_int($weight) ); 815 | } 816 | $this->_fieldweights = $weights; 817 | } 818 | 819 | /// bind per-index weights by name 820 | function SetIndexWeights ( $weights ) 821 | { 822 | assert ( is_array($weights) ); 823 | foreach ( $weights as $index=>$weight ) 824 | { 825 | assert ( is_string($index) ); 826 | assert ( is_int($weight) ); 827 | } 828 | $this->_indexweights = $weights; 829 | } 830 | 831 | /// set IDs range to match 832 | /// only match records if document ID is beetwen $min and $max (inclusive) 833 | function SetIDRange ( $min, $max ) 834 | { 835 | assert ( is_numeric($min) ); 836 | assert ( is_numeric($max) ); 837 | assert ( $min<=$max ); 838 | $this->_min_id = $min; 839 | $this->_max_id = $max; 840 | } 841 | 842 | /// set values set filter 843 | /// only match records where $attribute value is in given set 844 | function SetFilter ( $attribute, $values, $exclude=false ) 845 | { 846 | assert ( is_string($attribute) ); 847 | assert ( is_array($values) ); 848 | assert ( count($values) ); 849 | 850 | if ( is_array($values) && count($values) ) 851 | { 852 | foreach ( $values as $value ) 853 | assert ( is_numeric($value) ); 854 | 855 | $this->_filters[] = array ( "type"=>SPH_FILTER_VALUES, "attr"=>$attribute, "exclude"=>$exclude, "values"=>$values ); 856 | } 857 | } 858 | 859 | /// set string filter 860 | /// only match records where $attribute value is equal 861 | function SetFilterString ( $attribute, $value, $exclude=false ) 862 | { 863 | assert ( is_string($attribute) ); 864 | assert ( is_string($value) ); 865 | $this->_filters[] = array ( "type"=>SPH_FILTER_STRING, "attr"=>$attribute, "exclude"=>$exclude, "value"=>$value ); 866 | } 867 | 868 | /// set range filter 869 | /// only match records if $attribute value is beetwen $min and $max (inclusive) 870 | function SetFilterRange ( $attribute, $min, $max, $exclude=false ) 871 | { 872 | assert ( is_string($attribute) ); 873 | assert ( is_numeric($min) ); 874 | assert ( is_numeric($max) ); 875 | assert ( $min<=$max ); 876 | 877 | $this->_filters[] = array ( "type"=>SPH_FILTER_RANGE, "attr"=>$attribute, "exclude"=>$exclude, "min"=>$min, "max"=>$max ); 878 | } 879 | 880 | /// set float range filter 881 | /// only match records if $attribute value is beetwen $min and $max (inclusive) 882 | function SetFilterFloatRange ( $attribute, $min, $max, $exclude=false ) 883 | { 884 | assert ( is_string($attribute) ); 885 | assert ( is_float($min) ); 886 | assert ( is_float($max) ); 887 | assert ( $min<=$max ); 888 | 889 | $this->_filters[] = array ( "type"=>SPH_FILTER_FLOATRANGE, "attr"=>$attribute, "exclude"=>$exclude, "min"=>$min, "max"=>$max ); 890 | } 891 | 892 | /// setup anchor point for geosphere distance calculations 893 | /// required to use @geodist in filters and sorting 894 | /// latitude and longitude must be in radians 895 | function SetGeoAnchor ( $attrlat, $attrlong, $lat, $long ) 896 | { 897 | assert ( is_string($attrlat) ); 898 | assert ( is_string($attrlong) ); 899 | assert ( is_float($lat) ); 900 | assert ( is_float($long) ); 901 | 902 | $this->_anchor = array ( "attrlat"=>$attrlat, "attrlong"=>$attrlong, "lat"=>$lat, "long"=>$long ); 903 | } 904 | 905 | /// set grouping attribute and function 906 | function SetGroupBy ( $attribute, $func, $groupsort="@group desc" ) 907 | { 908 | assert ( is_string($attribute) ); 909 | assert ( is_string($groupsort) ); 910 | assert ( $func==SPH_GROUPBY_DAY 911 | || $func==SPH_GROUPBY_WEEK 912 | || $func==SPH_GROUPBY_MONTH 913 | || $func==SPH_GROUPBY_YEAR 914 | || $func==SPH_GROUPBY_ATTR 915 | || $func==SPH_GROUPBY_ATTRPAIR ); 916 | 917 | $this->_groupby = $attribute; 918 | $this->_groupfunc = $func; 919 | $this->_groupsort = $groupsort; 920 | } 921 | 922 | /// set count-distinct attribute for group-by queries 923 | function SetGroupDistinct ( $attribute ) 924 | { 925 | assert ( is_string($attribute) ); 926 | $this->_groupdistinct = $attribute; 927 | } 928 | 929 | /// set distributed retries count and delay 930 | function SetRetries ( $count, $delay=0 ) 931 | { 932 | assert ( is_int($count) && $count>=0 ); 933 | assert ( is_int($delay) && $delay>=0 ); 934 | $this->_retrycount = $count; 935 | $this->_retrydelay = $delay; 936 | } 937 | 938 | /// set result set format (hash or array; hash by default) 939 | /// PHP specific; needed for group-by-MVA result sets that may contain duplicate IDs 940 | function SetArrayResult ( $arrayresult ) 941 | { 942 | assert ( is_bool($arrayresult) ); 943 | $this->_arrayresult = $arrayresult; 944 | } 945 | 946 | /// set attribute values override 947 | /// there can be only one override per attribute 948 | /// $values must be a hash that maps document IDs to attribute values 949 | function SetOverride ( $attrname, $attrtype, $values ) 950 | { 951 | trigger_error('DEPRECATED: Do not call this method. Use SphinxQL REMAP() function instead.', E_USER_DEPRECATED); 952 | assert ( is_string ( $attrname ) ); 953 | assert ( in_array ( $attrtype, array ( SPH_ATTR_INTEGER, SPH_ATTR_TIMESTAMP, SPH_ATTR_BOOL, SPH_ATTR_FLOAT, SPH_ATTR_BIGINT ) ) ); 954 | assert ( is_array ( $values ) ); 955 | 956 | $this->_overrides[$attrname] = array ( "attr"=>$attrname, "type"=>$attrtype, "values"=>$values ); 957 | } 958 | 959 | /// set select-list (attributes or expressions), SQL-like syntax 960 | function SetSelect ( $select ) 961 | { 962 | assert ( is_string ( $select ) ); 963 | $this->_select = $select; 964 | } 965 | 966 | function SetQueryFlag ( $flag_name, $flag_value ) 967 | { 968 | $known_names = array ( "reverse_scan", "sort_method", "max_predicted_time", "boolean_simplify", "idf", "global_idf" ); 969 | $flags = array ( 970 | "reverse_scan" => array ( 0, 1 ), 971 | "sort_method" => array ( "pq", "kbuffer" ), 972 | "max_predicted_time" => array ( 0 ), 973 | "boolean_simplify" => array ( true, false ), 974 | "idf" => array ("normalized", "plain", "tfidf_normalized", "tfidf_unnormalized" ), 975 | "global_idf" => array ( true, false ), 976 | ); 977 | 978 | assert ( isset ( $flag_name, $known_names ) ); 979 | assert ( in_array( $flag_value, $flags[$flag_name], true ) || ( $flag_name=="max_predicted_time" && is_int ( $flag_value ) && $flag_value>=0 ) ); 980 | 981 | if ( $flag_name=="reverse_scan" ) $this->_query_flags = sphSetBit ( $this->_query_flags, 0, $flag_value==1 ); 982 | if ( $flag_name=="sort_method" ) $this->_query_flags = sphSetBit ( $this->_query_flags, 1, $flag_value=="kbuffer" ); 983 | if ( $flag_name=="max_predicted_time" ) 984 | { 985 | $this->_query_flags = sphSetBit ( $this->_query_flags, 2, $flag_value>0 ); 986 | $this->_predictedtime = (int)$flag_value; 987 | } 988 | if ( $flag_name=="boolean_simplify" ) $this->_query_flags = sphSetBit ( $this->_query_flags, 3, $flag_value ); 989 | if ( $flag_name=="idf" && ( $flag_value=="normalized" || $flag_value=="plain" ) ) $this->_query_flags = sphSetBit ( $this->_query_flags, 4, $flag_value=="plain" ); 990 | if ( $flag_name=="global_idf" ) $this->_query_flags = sphSetBit ( $this->_query_flags, 5, $flag_value ); 991 | if ( $flag_name=="idf" && ( $flag_value=="tfidf_normalized" || $flag_value=="tfidf_unnormalized" ) ) $this->_query_flags = sphSetBit ( $this->_query_flags, 6, $flag_value=="tfidf_normalized" ); 992 | } 993 | 994 | /// set outer order by parameters 995 | function SetOuterSelect ( $orderby, $offset, $limit ) 996 | { 997 | assert ( is_string($orderby) ); 998 | assert ( is_int($offset) ); 999 | assert ( is_int($limit) ); 1000 | assert ( $offset>=0 ); 1001 | assert ( $limit>0 ); 1002 | 1003 | $this->_outerorderby = $orderby; 1004 | $this->_outeroffset = $offset; 1005 | $this->_outerlimit = $limit; 1006 | $this->_hasouter = true; 1007 | } 1008 | 1009 | 1010 | ////////////////////////////////////////////////////////////////////////////// 1011 | 1012 | /// clear all filters (for multi-queries) 1013 | function ResetFilters () 1014 | { 1015 | $this->_filters = array(); 1016 | $this->_anchor = array(); 1017 | } 1018 | 1019 | /// clear groupby settings (for multi-queries) 1020 | function ResetGroupBy () 1021 | { 1022 | $this->_groupby = ""; 1023 | $this->_groupfunc = SPH_GROUPBY_DAY; 1024 | $this->_groupsort = "@group desc"; 1025 | $this->_groupdistinct= ""; 1026 | } 1027 | 1028 | /// clear all attribute value overrides (for multi-queries) 1029 | function ResetOverrides () 1030 | { 1031 | $this->_overrides = array (); 1032 | } 1033 | 1034 | function ResetQueryFlag () 1035 | { 1036 | $this->_query_flags = sphSetBit ( 0, 6, true ); // default idf=tfidf_normalized 1037 | $this->_predictedtime = 0; 1038 | } 1039 | 1040 | function ResetOuterSelect () 1041 | { 1042 | $this->_outerorderby = ''; 1043 | $this->_outeroffset = 0; 1044 | $this->_outerlimit = 0; 1045 | $this->_hasouter = false; 1046 | } 1047 | 1048 | ////////////////////////////////////////////////////////////////////////////// 1049 | 1050 | /// connect to searchd server, run given search query through given indexes, 1051 | /// and return the search results 1052 | function Query ( $query, $index="*", $comment="" ) 1053 | { 1054 | assert ( empty($this->_reqs) ); 1055 | 1056 | $this->AddQuery ( $query, $index, $comment ); 1057 | $results = $this->RunQueries (); 1058 | $this->_reqs = array (); // just in case it failed too early 1059 | 1060 | if ( !is_array($results) ) 1061 | return false; // probably network error; error message should be already filled 1062 | 1063 | $this->_error = $results[0]["error"]; 1064 | $this->_warning = $results[0]["warning"]; 1065 | if ( $results[0]["status"]==SEARCHD_ERROR ) 1066 | return false; 1067 | else 1068 | return $results[0]; 1069 | } 1070 | 1071 | /// helper to pack floats in network byte order 1072 | function _PackFloat ( $f ) 1073 | { 1074 | $t1 = pack ( "f", $f ); // machine order 1075 | list(,$t2) = unpack ( "L*", $t1 ); // int in machine order 1076 | return pack ( "N", $t2 ); 1077 | } 1078 | 1079 | /// add query to multi-query batch 1080 | /// returns index into results array from RunQueries() call 1081 | function AddQuery ( $query, $index="*", $comment="" ) 1082 | { 1083 | // mbstring workaround 1084 | $this->_MBPush (); 1085 | 1086 | // build request 1087 | $req = pack ( "NNNNN", $this->_query_flags, $this->_offset, $this->_limit, $this->_mode, $this->_ranker ); 1088 | if ( $this->_ranker==SPH_RANK_EXPR ) 1089 | $req .= pack ( "N", strlen($this->_rankexpr) ) . $this->_rankexpr; 1090 | $req .= pack ( "N", $this->_sort ); // (deprecated) sort mode 1091 | $req .= pack ( "N", strlen($this->_sortby) ) . $this->_sortby; 1092 | $req .= pack ( "N", strlen($query) ) . $query; // query itself 1093 | $req .= pack ( "N", count($this->_weights) ); // weights 1094 | foreach ( $this->_weights as $weight ) 1095 | $req .= pack ( "N", (int)$weight ); 1096 | $req .= pack ( "N", strlen($index) ) . $index; // indexes 1097 | $req .= pack ( "N", 1 ); // id64 range marker 1098 | $req .= sphPackU64 ( $this->_min_id ) . sphPackU64 ( $this->_max_id ); // id64 range 1099 | 1100 | // filters 1101 | $req .= pack ( "N", count($this->_filters) ); 1102 | foreach ( $this->_filters as $filter ) 1103 | { 1104 | $req .= pack ( "N", strlen($filter["attr"]) ) . $filter["attr"]; 1105 | $req .= pack ( "N", $filter["type"] ); 1106 | switch ( $filter["type"] ) 1107 | { 1108 | case SPH_FILTER_VALUES: 1109 | $req .= pack ( "N", count($filter["values"]) ); 1110 | foreach ( $filter["values"] as $value ) 1111 | $req .= sphPackI64 ( $value ); 1112 | break; 1113 | 1114 | case SPH_FILTER_RANGE: 1115 | $req .= sphPackI64 ( $filter["min"] ) . sphPackI64 ( $filter["max"] ); 1116 | break; 1117 | 1118 | case SPH_FILTER_FLOATRANGE: 1119 | $req .= $this->_PackFloat ( $filter["min"] ) . $this->_PackFloat ( $filter["max"] ); 1120 | break; 1121 | 1122 | case SPH_FILTER_STRING: 1123 | $req .= pack ( "N", strlen($filter["value"]) ) . $filter["value"]; 1124 | break; 1125 | 1126 | default: 1127 | assert ( 0 && "internal error: unhandled filter type" ); 1128 | } 1129 | $req .= pack ( "N", $filter["exclude"] ); 1130 | } 1131 | 1132 | // group-by clause, max-matches count, group-sort clause, cutoff count 1133 | $req .= pack ( "NN", $this->_groupfunc, strlen($this->_groupby) ) . $this->_groupby; 1134 | $req .= pack ( "N", $this->_maxmatches ); 1135 | $req .= pack ( "N", strlen($this->_groupsort) ) . $this->_groupsort; 1136 | $req .= pack ( "NNN", $this->_cutoff, $this->_retrycount, $this->_retrydelay ); 1137 | $req .= pack ( "N", strlen($this->_groupdistinct) ) . $this->_groupdistinct; 1138 | 1139 | // anchor point 1140 | if ( empty($this->_anchor) ) 1141 | { 1142 | $req .= pack ( "N", 0 ); 1143 | } else 1144 | { 1145 | $a =& $this->_anchor; 1146 | $req .= pack ( "N", 1 ); 1147 | $req .= pack ( "N", strlen($a["attrlat"]) ) . $a["attrlat"]; 1148 | $req .= pack ( "N", strlen($a["attrlong"]) ) . $a["attrlong"]; 1149 | $req .= $this->_PackFloat ( $a["lat"] ) . $this->_PackFloat ( $a["long"] ); 1150 | } 1151 | 1152 | // per-index weights 1153 | $req .= pack ( "N", count($this->_indexweights) ); 1154 | foreach ( $this->_indexweights as $idx=>$weight ) 1155 | $req .= pack ( "N", strlen($idx) ) . $idx . pack ( "N", $weight ); 1156 | 1157 | // max query time 1158 | $req .= pack ( "N", $this->_maxquerytime ); 1159 | 1160 | // per-field weights 1161 | $req .= pack ( "N", count($this->_fieldweights) ); 1162 | foreach ( $this->_fieldweights as $field=>$weight ) 1163 | $req .= pack ( "N", strlen($field) ) . $field . pack ( "N", $weight ); 1164 | 1165 | // comment 1166 | $req .= pack ( "N", strlen($comment) ) . $comment; 1167 | 1168 | // attribute overrides 1169 | $req .= pack ( "N", count($this->_overrides) ); 1170 | foreach ( $this->_overrides as $key => $entry ) 1171 | { 1172 | $req .= pack ( "N", strlen($entry["attr"]) ) . $entry["attr"]; 1173 | $req .= pack ( "NN", $entry["type"], count($entry["values"]) ); 1174 | foreach ( $entry["values"] as $id=>$val ) 1175 | { 1176 | assert ( is_numeric($id) ); 1177 | assert ( is_numeric($val) ); 1178 | 1179 | $req .= sphPackU64 ( $id ); 1180 | switch ( $entry["type"] ) 1181 | { 1182 | case SPH_ATTR_FLOAT: $req .= $this->_PackFloat ( $val ); break; 1183 | case SPH_ATTR_BIGINT: $req .= sphPackI64 ( $val ); break; 1184 | default: $req .= pack ( "N", $val ); break; 1185 | } 1186 | } 1187 | } 1188 | 1189 | // select-list 1190 | $req .= pack ( "N", strlen($this->_select) ) . $this->_select; 1191 | 1192 | // max_predicted_time 1193 | if ( $this->_predictedtime>0 ) 1194 | $req .= pack ( "N", (int)$this->_predictedtime ); 1195 | 1196 | $req .= pack ( "N", strlen($this->_outerorderby) ) . $this->_outerorderby; 1197 | $req .= pack ( "NN", $this->_outeroffset, $this->_outerlimit ); 1198 | if ( $this->_hasouter ) 1199 | $req .= pack ( "N", 1 ); 1200 | else 1201 | $req .= pack ( "N", 0 ); 1202 | 1203 | // mbstring workaround 1204 | $this->_MBPop (); 1205 | 1206 | // store request to requests array 1207 | $this->_reqs[] = $req; 1208 | return count($this->_reqs)-1; 1209 | } 1210 | 1211 | /// connect to searchd, run queries batch, and return an array of result sets 1212 | function RunQueries () 1213 | { 1214 | if ( empty($this->_reqs) ) 1215 | { 1216 | $this->_error = "no queries defined, issue AddQuery() first"; 1217 | return false; 1218 | } 1219 | 1220 | // mbstring workaround 1221 | $this->_MBPush (); 1222 | 1223 | if (!( $fp = $this->_Connect() )) 1224 | { 1225 | $this->_MBPop (); 1226 | return false; 1227 | } 1228 | 1229 | // send query, get response 1230 | $nreqs = count($this->_reqs); 1231 | $req = join ( "", $this->_reqs ); 1232 | $len = 8+strlen($req); 1233 | $req = pack ( "nnNNN", SEARCHD_COMMAND_SEARCH, VER_COMMAND_SEARCH, $len, 0, $nreqs ) . $req; // add header 1234 | 1235 | if ( !( $this->_Send ( $fp, $req, $len+8 ) ) || 1236 | !( $response = $this->_GetResponse ( $fp, VER_COMMAND_SEARCH ) ) ) 1237 | { 1238 | $this->_MBPop (); 1239 | return false; 1240 | } 1241 | 1242 | // query sent ok; we can reset reqs now 1243 | $this->_reqs = array (); 1244 | 1245 | // parse and return response 1246 | return $this->_ParseSearchResponse ( $response, $nreqs ); 1247 | } 1248 | 1249 | /// parse and return search query (or queries) response 1250 | function _ParseSearchResponse ( $response, $nreqs ) 1251 | { 1252 | $p = 0; // current position 1253 | $max = strlen($response); // max position for checks, to protect against broken responses 1254 | 1255 | $results = array (); 1256 | for ( $ires=0; $ires<$nreqs && $p<$max; $ires++ ) 1257 | { 1258 | $results[] = array(); 1259 | $result =& $results[$ires]; 1260 | 1261 | $result["error"] = ""; 1262 | $result["warning"] = ""; 1263 | 1264 | // extract status 1265 | list(,$status) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; 1266 | $result["status"] = $status; 1267 | if ( $status!=SEARCHD_OK ) 1268 | { 1269 | list(,$len) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; 1270 | $message = substr ( $response, $p, $len ); $p += $len; 1271 | 1272 | if ( $status==SEARCHD_WARNING ) 1273 | { 1274 | $result["warning"] = $message; 1275 | } else 1276 | { 1277 | $result["error"] = $message; 1278 | continue; 1279 | } 1280 | } 1281 | 1282 | // read schema 1283 | $fields = array (); 1284 | $attrs = array (); 1285 | 1286 | list(,$nfields) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; 1287 | while ( $nfields-->0 && $p<$max ) 1288 | { 1289 | list(,$len) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; 1290 | $fields[] = substr ( $response, $p, $len ); $p += $len; 1291 | } 1292 | $result["fields"] = $fields; 1293 | 1294 | list(,$nattrs) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; 1295 | while ( $nattrs-->0 && $p<$max ) 1296 | { 1297 | list(,$len) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; 1298 | $attr = substr ( $response, $p, $len ); $p += $len; 1299 | list(,$type) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; 1300 | $attrs[$attr] = $type; 1301 | } 1302 | $result["attrs"] = $attrs; 1303 | 1304 | // read match count 1305 | list(,$count) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; 1306 | list(,$id64) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; 1307 | 1308 | // read matches 1309 | $idx = -1; 1310 | while ( $count-->0 && $p<$max ) 1311 | { 1312 | // index into result array 1313 | $idx++; 1314 | 1315 | // parse document id and weight 1316 | if ( $id64 ) 1317 | { 1318 | $doc = sphUnpackU64 ( substr ( $response, $p, 8 ) ); $p += 8; 1319 | list(,$weight) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; 1320 | } 1321 | else 1322 | { 1323 | list ( $doc, $weight ) = array_values ( unpack ( "N*N*", 1324 | substr ( $response, $p, 8 ) ) ); 1325 | $p += 8; 1326 | $doc = sphFixUint($doc); 1327 | } 1328 | $weight = sprintf ( "%u", $weight ); 1329 | 1330 | // create match entry 1331 | if ( $this->_arrayresult ) 1332 | $result["matches"][$idx] = array ( "id"=>$doc, "weight"=>$weight ); 1333 | else 1334 | $result["matches"][$doc]["weight"] = $weight; 1335 | 1336 | // parse and create attributes 1337 | $attrvals = array (); 1338 | foreach ( $attrs as $attr=>$type ) 1339 | { 1340 | // handle 64bit ints 1341 | if ( $type==SPH_ATTR_BIGINT ) 1342 | { 1343 | $attrvals[$attr] = sphUnpackI64 ( substr ( $response, $p, 8 ) ); $p += 8; 1344 | continue; 1345 | } 1346 | 1347 | // handle floats 1348 | if ( $type==SPH_ATTR_FLOAT ) 1349 | { 1350 | list(,$uval) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; 1351 | list(,$fval) = unpack ( "f*", pack ( "L", $uval ) ); 1352 | $attrvals[$attr] = $fval; 1353 | continue; 1354 | } 1355 | 1356 | // handle everything else as unsigned ints 1357 | list(,$val) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; 1358 | if ( $type==SPH_ATTR_MULTI ) 1359 | { 1360 | $attrvals[$attr] = array (); 1361 | $nvalues = $val; 1362 | while ( $nvalues-->0 && $p<$max ) 1363 | { 1364 | list(,$val) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; 1365 | $attrvals[$attr][] = sphFixUint($val); 1366 | } 1367 | } else if ( $type==SPH_ATTR_MULTI64 ) 1368 | { 1369 | $attrvals[$attr] = array (); 1370 | $nvalues = $val; 1371 | while ( $nvalues>0 && $p<$max ) 1372 | { 1373 | $attrvals[$attr][] = sphUnpackI64 ( substr ( $response, $p, 8 ) ); $p += 8; 1374 | $nvalues -= 2; 1375 | } 1376 | } else if ( $type==SPH_ATTR_STRING ) 1377 | { 1378 | $attrvals[$attr] = substr ( $response, $p, $val ); 1379 | $p += $val; 1380 | } else if ( $type==SPH_ATTR_FACTORS ) 1381 | { 1382 | $attrvals[$attr] = substr ( $response, $p, $val-4 ); 1383 | $p += $val-4; 1384 | } else 1385 | { 1386 | $attrvals[$attr] = sphFixUint($val); 1387 | } 1388 | } 1389 | 1390 | if ( $this->_arrayresult ) 1391 | $result["matches"][$idx]["attrs"] = $attrvals; 1392 | else 1393 | $result["matches"][$doc]["attrs"] = $attrvals; 1394 | } 1395 | 1396 | list ( $total, $total_found, $msecs, $words ) = 1397 | array_values ( unpack ( "N*N*N*N*", substr ( $response, $p, 16 ) ) ); 1398 | $result["total"] = sprintf ( "%u", $total ); 1399 | $result["total_found"] = sprintf ( "%u", $total_found ); 1400 | $result["time"] = sprintf ( "%.3f", $msecs/1000 ); 1401 | $p += 16; 1402 | 1403 | while ( $words-->0 && $p<$max ) 1404 | { 1405 | list(,$len) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; 1406 | $word = substr ( $response, $p, $len ); $p += $len; 1407 | list ( $docs, $hits ) = array_values ( unpack ( "N*N*", substr ( $response, $p, 8 ) ) ); $p += 8; 1408 | $result["words"][$word] = array ( 1409 | "docs"=>sprintf ( "%u", $docs ), 1410 | "hits"=>sprintf ( "%u", $hits ) ); 1411 | } 1412 | } 1413 | 1414 | $this->_MBPop (); 1415 | return $results; 1416 | } 1417 | 1418 | ///////////////////////////////////////////////////////////////////////////// 1419 | // excerpts generation 1420 | ///////////////////////////////////////////////////////////////////////////// 1421 | 1422 | /// connect to searchd server, and generate exceprts (snippets) 1423 | /// of given documents for given query. returns false on failure, 1424 | /// an array of snippets on success 1425 | function BuildExcerpts ( $docs, $index, $words, $opts=array() ) 1426 | { 1427 | assert ( is_array($docs) ); 1428 | assert ( is_string($index) ); 1429 | assert ( is_string($words) ); 1430 | assert ( is_array($opts) ); 1431 | 1432 | $this->_MBPush (); 1433 | 1434 | if (!( $fp = $this->_Connect() )) 1435 | { 1436 | $this->_MBPop(); 1437 | return false; 1438 | } 1439 | 1440 | ///////////////// 1441 | // fixup options 1442 | ///////////////// 1443 | 1444 | if ( !isset($opts["before_match"]) ) $opts["before_match"] = ""; 1445 | if ( !isset($opts["after_match"]) ) $opts["after_match"] = ""; 1446 | if ( !isset($opts["chunk_separator"]) ) $opts["chunk_separator"] = " ... "; 1447 | if ( !isset($opts["limit"]) ) $opts["limit"] = 256; 1448 | if ( !isset($opts["limit_passages"]) ) $opts["limit_passages"] = 0; 1449 | if ( !isset($opts["limit_words"]) ) $opts["limit_words"] = 0; 1450 | if ( !isset($opts["around"]) ) $opts["around"] = 5; 1451 | if ( !isset($opts["exact_phrase"]) ) $opts["exact_phrase"] = false; 1452 | if ( !isset($opts["single_passage"]) ) $opts["single_passage"] = false; 1453 | if ( !isset($opts["use_boundaries"]) ) $opts["use_boundaries"] = false; 1454 | if ( !isset($opts["weight_order"]) ) $opts["weight_order"] = false; 1455 | if ( !isset($opts["query_mode"]) ) $opts["query_mode"] = false; 1456 | if ( !isset($opts["force_all_words"]) ) $opts["force_all_words"] = false; 1457 | if ( !isset($opts["start_passage_id"]) ) $opts["start_passage_id"] = 1; 1458 | if ( !isset($opts["load_files"]) ) $opts["load_files"] = false; 1459 | if ( !isset($opts["html_strip_mode"]) ) $opts["html_strip_mode"] = "index"; 1460 | if ( !isset($opts["allow_empty"]) ) $opts["allow_empty"] = false; 1461 | if ( !isset($opts["passage_boundary"]) ) $opts["passage_boundary"] = "none"; 1462 | if ( !isset($opts["emit_zones"]) ) $opts["emit_zones"] = false; 1463 | if ( !isset($opts["load_files_scattered"]) ) $opts["load_files_scattered"] = false; 1464 | 1465 | 1466 | ///////////////// 1467 | // build request 1468 | ///////////////// 1469 | 1470 | // v.1.2 req 1471 | $flags = 1; // remove spaces 1472 | if ( $opts["exact_phrase"] ) $flags |= 2; 1473 | if ( $opts["single_passage"] ) $flags |= 4; 1474 | if ( $opts["use_boundaries"] ) $flags |= 8; 1475 | if ( $opts["weight_order"] ) $flags |= 16; 1476 | if ( $opts["query_mode"] ) $flags |= 32; 1477 | if ( $opts["force_all_words"] ) $flags |= 64; 1478 | if ( $opts["load_files"] ) $flags |= 128; 1479 | if ( $opts["allow_empty"] ) $flags |= 256; 1480 | if ( $opts["emit_zones"] ) $flags |= 512; 1481 | if ( $opts["load_files_scattered"] ) $flags |= 1024; 1482 | $req = pack ( "NN", 0, $flags ); // mode=0, flags=$flags 1483 | $req .= pack ( "N", strlen($index) ) . $index; // req index 1484 | $req .= pack ( "N", strlen($words) ) . $words; // req words 1485 | 1486 | // options 1487 | $req .= pack ( "N", strlen($opts["before_match"]) ) . $opts["before_match"]; 1488 | $req .= pack ( "N", strlen($opts["after_match"]) ) . $opts["after_match"]; 1489 | $req .= pack ( "N", strlen($opts["chunk_separator"]) ) . $opts["chunk_separator"]; 1490 | $req .= pack ( "NN", (int)$opts["limit"], (int)$opts["around"] ); 1491 | $req .= pack ( "NNN", (int)$opts["limit_passages"], (int)$opts["limit_words"], (int)$opts["start_passage_id"] ); // v.1.2 1492 | $req .= pack ( "N", strlen($opts["html_strip_mode"]) ) . $opts["html_strip_mode"]; 1493 | $req .= pack ( "N", strlen($opts["passage_boundary"]) ) . $opts["passage_boundary"]; 1494 | 1495 | // documents 1496 | $req .= pack ( "N", count($docs) ); 1497 | foreach ( $docs as $doc ) 1498 | { 1499 | assert ( is_string($doc) ); 1500 | $req .= pack ( "N", strlen($doc) ) . $doc; 1501 | } 1502 | 1503 | //////////////////////////// 1504 | // send query, get response 1505 | //////////////////////////// 1506 | 1507 | $len = strlen($req); 1508 | $req = pack ( "nnN", SEARCHD_COMMAND_EXCERPT, VER_COMMAND_EXCERPT, $len ) . $req; // add header 1509 | if ( !( $this->_Send ( $fp, $req, $len+8 ) ) || 1510 | !( $response = $this->_GetResponse ( $fp, VER_COMMAND_EXCERPT ) ) ) 1511 | { 1512 | $this->_MBPop (); 1513 | return false; 1514 | } 1515 | 1516 | ////////////////// 1517 | // parse response 1518 | ////////////////// 1519 | 1520 | $pos = 0; 1521 | $res = array (); 1522 | $rlen = strlen($response); 1523 | for ( $i=0; $i $rlen ) 1529 | { 1530 | $this->_error = "incomplete reply"; 1531 | $this->_MBPop (); 1532 | return false; 1533 | } 1534 | $res[] = $len ? substr ( $response, $pos, $len ) : ""; 1535 | $pos += $len; 1536 | } 1537 | 1538 | $this->_MBPop (); 1539 | return $res; 1540 | } 1541 | 1542 | 1543 | ///////////////////////////////////////////////////////////////////////////// 1544 | // keyword generation 1545 | ///////////////////////////////////////////////////////////////////////////// 1546 | 1547 | /// connect to searchd server, and generate keyword list for a given query 1548 | /// returns false on failure, 1549 | /// an array of words on success 1550 | function BuildKeywords ( $query, $index, $hits ) 1551 | { 1552 | assert ( is_string($query) ); 1553 | assert ( is_string($index) ); 1554 | assert ( is_bool($hits) ); 1555 | 1556 | $this->_MBPush (); 1557 | 1558 | if (!( $fp = $this->_Connect() )) 1559 | { 1560 | $this->_MBPop(); 1561 | return false; 1562 | } 1563 | 1564 | ///////////////// 1565 | // build request 1566 | ///////////////// 1567 | 1568 | // v.1.0 req 1569 | $req = pack ( "N", strlen($query) ) . $query; // req query 1570 | $req .= pack ( "N", strlen($index) ) . $index; // req index 1571 | $req .= pack ( "N", (int)$hits ); 1572 | 1573 | //////////////////////////// 1574 | // send query, get response 1575 | //////////////////////////// 1576 | 1577 | $len = strlen($req); 1578 | $req = pack ( "nnN", SEARCHD_COMMAND_KEYWORDS, VER_COMMAND_KEYWORDS, $len ) . $req; // add header 1579 | if ( !( $this->_Send ( $fp, $req, $len+8 ) ) || 1580 | !( $response = $this->_GetResponse ( $fp, VER_COMMAND_KEYWORDS ) ) ) 1581 | { 1582 | $this->_MBPop (); 1583 | return false; 1584 | } 1585 | 1586 | ////////////////// 1587 | // parse response 1588 | ////////////////// 1589 | 1590 | $pos = 0; 1591 | $res = array (); 1592 | $rlen = strlen($response); 1593 | list(,$nwords) = unpack ( "N*", substr ( $response, $pos, 4 ) ); 1594 | $pos += 4; 1595 | for ( $i=0; $i<$nwords; $i++ ) 1596 | { 1597 | list(,$len) = unpack ( "N*", substr ( $response, $pos, 4 ) ); $pos += 4; 1598 | $tokenized = $len ? substr ( $response, $pos, $len ) : ""; 1599 | $pos += $len; 1600 | 1601 | list(,$len) = unpack ( "N*", substr ( $response, $pos, 4 ) ); $pos += 4; 1602 | $normalized = $len ? substr ( $response, $pos, $len ) : ""; 1603 | $pos += $len; 1604 | 1605 | $res[] = array ( "tokenized"=>$tokenized, "normalized"=>$normalized ); 1606 | 1607 | if ( $hits ) 1608 | { 1609 | list($ndocs,$nhits) = array_values ( unpack ( "N*N*", substr ( $response, $pos, 8 ) ) ); 1610 | $pos += 8; 1611 | $res [$i]["docs"] = $ndocs; 1612 | $res [$i]["hits"] = $nhits; 1613 | } 1614 | 1615 | if ( $pos > $rlen ) 1616 | { 1617 | $this->_error = "incomplete reply"; 1618 | $this->_MBPop (); 1619 | return false; 1620 | } 1621 | } 1622 | 1623 | $this->_MBPop (); 1624 | return $res; 1625 | } 1626 | 1627 | function EscapeString ( $string ) 1628 | { 1629 | $from = array ( '\\', '(',')','|','-','!','@','~','"','&', '/', '^', '$', '=', '<' ); 1630 | $to = array ( '\\\\', '\(','\)','\|','\-','\!','\@','\~','\"', '\&', '\/', '\^', '\$', '\=', '\<' ); 1631 | 1632 | return str_replace ( $from, $to, $string ); 1633 | } 1634 | 1635 | ///////////////////////////////////////////////////////////////////////////// 1636 | // attribute updates 1637 | ///////////////////////////////////////////////////////////////////////////// 1638 | 1639 | /// batch update given attributes in given rows in given indexes 1640 | /// returns amount of updated documents (0 or more) on success, or -1 on failure 1641 | function UpdateAttributes ( $index, $attrs, $values, $mva=false, $ignorenonexistent=false ) 1642 | { 1643 | // verify everything 1644 | assert ( is_string($index) ); 1645 | assert ( is_bool($mva) ); 1646 | assert ( is_bool($ignorenonexistent) ); 1647 | 1648 | assert ( is_array($attrs) ); 1649 | foreach ( $attrs as $attr ) 1650 | assert ( is_string($attr) ); 1651 | 1652 | assert ( is_array($values) ); 1653 | foreach ( $values as $id=>$entry ) 1654 | { 1655 | assert ( is_numeric($id) ); 1656 | assert ( is_array($entry) ); 1657 | assert ( count($entry)==count($attrs) ); 1658 | foreach ( $entry as $v ) 1659 | { 1660 | if ( $mva ) 1661 | { 1662 | assert ( is_array($v) ); 1663 | foreach ( $v as $vv ) 1664 | assert ( is_int($vv) ); 1665 | } else 1666 | assert ( is_int($v) ); 1667 | } 1668 | } 1669 | 1670 | // build request 1671 | $this->_MBPush (); 1672 | $req = pack ( "N", strlen($index) ) . $index; 1673 | 1674 | $req .= pack ( "N", count($attrs) ); 1675 | $req .= pack ( "N", $ignorenonexistent ? 1 : 0 ); 1676 | foreach ( $attrs as $attr ) 1677 | { 1678 | $req .= pack ( "N", strlen($attr) ) . $attr; 1679 | $req .= pack ( "N", $mva ? 1 : 0 ); 1680 | } 1681 | 1682 | $req .= pack ( "N", count($values) ); 1683 | foreach ( $values as $id=>$entry ) 1684 | { 1685 | $req .= sphPackU64 ( $id ); 1686 | foreach ( $entry as $v ) 1687 | { 1688 | $req .= pack ( "N", $mva ? count($v) : $v ); 1689 | if ( $mva ) 1690 | foreach ( $v as $vv ) 1691 | $req .= pack ( "N", $vv ); 1692 | } 1693 | } 1694 | 1695 | // connect, send query, get response 1696 | if (!( $fp = $this->_Connect() )) 1697 | { 1698 | $this->_MBPop (); 1699 | return -1; 1700 | } 1701 | 1702 | $len = strlen($req); 1703 | $req = pack ( "nnN", SEARCHD_COMMAND_UPDATE, VER_COMMAND_UPDATE, $len ) . $req; // add header 1704 | if ( !$this->_Send ( $fp, $req, $len+8 ) ) 1705 | { 1706 | $this->_MBPop (); 1707 | return -1; 1708 | } 1709 | 1710 | if (!( $response = $this->_GetResponse ( $fp, VER_COMMAND_UPDATE ) )) 1711 | { 1712 | $this->_MBPop (); 1713 | return -1; 1714 | } 1715 | 1716 | // parse response 1717 | list(,$updated) = unpack ( "N*", substr ( $response, 0, 4 ) ); 1718 | $this->_MBPop (); 1719 | return $updated; 1720 | } 1721 | 1722 | ///////////////////////////////////////////////////////////////////////////// 1723 | // persistent connections 1724 | ///////////////////////////////////////////////////////////////////////////// 1725 | 1726 | function Open() 1727 | { 1728 | if ( $this->_socket !== false ) 1729 | { 1730 | $this->_error = 'already connected'; 1731 | return false; 1732 | } 1733 | if ( !$fp = $this->_Connect() ) 1734 | return false; 1735 | 1736 | // command, command version = 0, body length = 4, body = 1 1737 | $req = pack ( "nnNN", SEARCHD_COMMAND_PERSIST, 0, 4, 1 ); 1738 | if ( !$this->_Send ( $fp, $req, 12 ) ) 1739 | return false; 1740 | 1741 | $this->_socket = $fp; 1742 | return true; 1743 | } 1744 | 1745 | function Close() 1746 | { 1747 | if ( $this->_socket === false ) 1748 | { 1749 | $this->_error = 'not connected'; 1750 | return false; 1751 | } 1752 | 1753 | fclose ( $this->_socket ); 1754 | $this->_socket = false; 1755 | 1756 | return true; 1757 | } 1758 | 1759 | ////////////////////////////////////////////////////////////////////////// 1760 | // status 1761 | ////////////////////////////////////////////////////////////////////////// 1762 | 1763 | function Status ($session=false) 1764 | { 1765 | assert ( is_bool($session) ); 1766 | 1767 | $this->_MBPush (); 1768 | if (!( $fp = $this->_Connect() )) 1769 | { 1770 | $this->_MBPop(); 1771 | return false; 1772 | } 1773 | 1774 | $req = pack ( "nnNN", SEARCHD_COMMAND_STATUS, VER_COMMAND_STATUS, 4, $session?0:1 ); // len=4, body=1 1775 | if ( !( $this->_Send ( $fp, $req, 12 ) ) || 1776 | !( $response = $this->_GetResponse ( $fp, VER_COMMAND_STATUS ) ) ) 1777 | { 1778 | $this->_MBPop (); 1779 | return false; 1780 | } 1781 | 1782 | $res = substr ( $response, 4 ); // just ignore length, error handling, etc 1783 | $p = 0; 1784 | list ( $rows, $cols ) = array_values ( unpack ( "N*N*", substr ( $response, $p, 8 ) ) ); $p += 8; 1785 | 1786 | $res = array(); 1787 | for ( $i=0; $i<$rows; $i++ ) 1788 | for ( $j=0; $j<$cols; $j++ ) 1789 | { 1790 | list(,$len) = unpack ( "N*", substr ( $response, $p, 4 ) ); $p += 4; 1791 | $res[$i][] = substr ( $response, $p, $len ); $p += $len; 1792 | } 1793 | 1794 | $this->_MBPop (); 1795 | return $res; 1796 | } 1797 | 1798 | ////////////////////////////////////////////////////////////////////////// 1799 | // flush 1800 | ////////////////////////////////////////////////////////////////////////// 1801 | 1802 | function FlushAttributes () 1803 | { 1804 | $this->_MBPush (); 1805 | if (!( $fp = $this->_Connect() )) 1806 | { 1807 | $this->_MBPop(); 1808 | return -1; 1809 | } 1810 | 1811 | $req = pack ( "nnN", SEARCHD_COMMAND_FLUSHATTRS, VER_COMMAND_FLUSHATTRS, 0 ); // len=0 1812 | if ( !( $this->_Send ( $fp, $req, 8 ) ) || 1813 | !( $response = $this->_GetResponse ( $fp, VER_COMMAND_FLUSHATTRS ) ) ) 1814 | { 1815 | $this->_MBPop (); 1816 | return -1; 1817 | } 1818 | 1819 | $tag = -1; 1820 | if ( strlen($response)==4 ) 1821 | list(,$tag) = unpack ( "N*", $response ); 1822 | else 1823 | $this->_error = "unexpected response length"; 1824 | 1825 | $this->_MBPop (); 1826 | return $tag; 1827 | } 1828 | } 1829 | 1830 | // 1831 | // $Id: sphinxapi.php 4522 2014-01-30 11:00:18Z tomat $ 1832 | // 1833 | -------------------------------------------------------------------------------- /sphinx.conf.example: -------------------------------------------------------------------------------- 1 | # 2 | # Sphinx configuration file sample 3 | # 4 | 5 | ############################################################################# 6 | ## data source definition 7 | ############################################################################# 8 | 9 | source magento_fulltext 10 | { 11 | # data source type. mandatory, no default value 12 | # known types are mysql, pgsql, mssql, xmlpipe, xmlpipe2, odbc 13 | type = mysql 14 | 15 | ##################################################################### 16 | ## SQL settings (for 'mysql' and 'pgsql' types) 17 | ##################################################################### 18 | 19 | # some straightforward parameters for SQL source types 20 | sql_host = localhost 21 | sql_user = root 22 | sql_pass =password 23 | sql_db = magento 24 | sql_port = 3306 # optional, default is 3306 25 | 26 | 27 | # pre-query, executed before the main fetch query 28 | # multi-value, optional, default is empty list of queries 29 | # 30 | sql_query_pre = SET NAMES utf8 31 | 32 | 33 | # main document fetch query 34 | # mandatory, integer document ID field MUST be the first selected column 35 | sql_query = SELECT product_id, name, name_attributes, category, data_index, sku FROM sphinx_catalogsearch_fulltext 36 | } 37 | 38 | ############################################################################# 39 | ## index definition 40 | ############################################################################# 41 | 42 | index fulltext { 43 | source = magento_fulltext 44 | path = /path/to/magento/var/data/production.sphinx.index # Feel free to change 45 | 46 | morphology = stem_en #, metaphone # You can add metaphone morphology if you want. 47 | min_word_len = 1 # Indexes all words 48 | blend_chars = - # This presumes people won't type a hyphen into the search bar: quite likely 49 | blend_mode = trim_both # 50 | html_strip = 1 # Just in case anyone tries to get clever in the admin panel and use HTML 51 | } 52 | 53 | 54 | ############################################################################# 55 | ## indexer settings 56 | ############################################################################# 57 | 58 | indexer 59 | { 60 | # memory limit, in bytes, kiloytes (16384K) or megabytes (256M) 61 | # optional, default is 128M, max is 2047M, recommended is 256M to 1024M 62 | mem_limit = 1024M 63 | } 64 | 65 | ############################################################################# 66 | ## searchd settings 67 | ############################################################################# 68 | 69 | searchd 70 | { 71 | # [hostname:]port[:protocol], or /unix/socket/path to listen on 72 | # known protocols are 'sphinx' (SphinxAPI) and 'mysql41' (SphinxQL) 73 | # 74 | # multi-value, multiple listen points are allowed 75 | # optional, defaults are 9312:sphinx and 9306:mysql41, as below 76 | # 77 | listen = 9312 78 | listen = 9306:mysql41 79 | 80 | # log file, searchd run info is logged here 81 | # optional, default is 'searchd.log' 82 | log = /var/log/sphinx/searchd.log 83 | 84 | # query log file, all search queries are logged here 85 | # optional, default is empty (do not log queries) 86 | query_log = /var/log/sphinx/query.log 87 | 88 | # client read timeout, seconds 89 | # optional, default is 5 90 | read_timeout = 10 91 | 92 | # request timeout, seconds 93 | # optional, default is 5 minutes 94 | client_timeout = 10 95 | 96 | # maximum amount of children to fork (concurrent searches to run) 97 | # optional, default is 0 (unlimited) 98 | max_children = 30 99 | 100 | # maximum amount of persistent connections from this master to each agent host 101 | # optional, but necessary if you use agent_persistent. It is reasonable to set the value 102 | # as max_children, or less on the agent's hosts. 103 | persistent_connections_limit = 30 104 | 105 | # PID file, searchd process ID file name 106 | # mandatory 107 | pid_file = /var/log/sphinx/searchd.pid 108 | 109 | # seamless rotate, prevents rotate stalls if precaching huge datasets 110 | # optional, default is 1 111 | seamless_rotate = 1 112 | 113 | # whether to forcibly preopen all indexes on startup 114 | # optional, default is 1 (preopen everything) 115 | preopen_indexes = 1 116 | 117 | # whether to unlink .old index copies on succesful rotation. 118 | # optional, default is 1 (do unlink) 119 | unlink_old = 1 120 | 121 | 122 | # MVA updates pool size 123 | # shared between all instances of searchd, disables attr flushes! 124 | # optional, default size is 1M 125 | mva_updates_pool = 1M 126 | 127 | # max allowed network packet size 128 | # limits both query packets from clients, and responses from agents 129 | # optional, default size is 8M 130 | max_packet_size = 8M 131 | 132 | # max allowed per-query filter count 133 | # optional, default is 256 134 | max_filters = 256 135 | 136 | # max allowed per-filter values count 137 | # optional, default is 4096 138 | max_filter_values = 4096 139 | 140 | # max allowed per-batch query count (aka multi-query count) 141 | # optional, default is 32 142 | max_batch_queries = 32 143 | 144 | # multi-processing mode (MPM) 145 | # known values are none, fork, prefork, and threads 146 | # threads is required for RT backend to work 147 | # optional, default is threads 148 | workers = threads # for RT to work 149 | } 150 | 151 | # --eof-- 152 | --------------------------------------------------------------------------------