├── .gitignore ├── manticore ├── status.go ├── bson.go ├── status_test.go ├── manticore_test.go ├── sphinxql_test.go ├── uvar_test.go ├── update_test.go ├── json.go ├── json_test.go ├── snippets_test.go ├── callpq_test.go ├── uvar.go ├── keywords_test.go ├── keywords.go ├── update.go ├── client_test.go ├── sphinxproto.go ├── search_test.go ├── client.go ├── snippets.go ├── sphinxql.go ├── callpq.go ├── manticore.go ├── definitions.go └── search.go ├── README.md └── LICENSE /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | -------------------------------------------------------------------------------- /manticore/status.go: -------------------------------------------------------------------------------- 1 | package manticore 2 | 3 | func parseStatusAnswer() func(*apibuf) interface{} { 4 | return func(answer *apibuf) interface{} { 5 | nrows := answer.getInt() 6 | status := make(map[string]string) 7 | _ = answer.getInt() // n of cols, always 2 8 | for j := 0; j < nrows; j++ { 9 | key := answer.getString() 10 | status[key] = answer.getString() 11 | } 12 | return status 13 | } 14 | } 15 | 16 | -------------------------------------------------------------------------------- /manticore/bson.go: -------------------------------------------------------------------------------- 1 | package manticore 2 | 3 | /// aggregate function to apply 4 | type eBsonType byte 5 | 6 | const ( 7 | bsonEof = eBsonType(iota) 8 | bsonInt32 9 | bsonInt64 10 | bsonDouble 11 | bsonString 12 | bsonStringVector 13 | bsonInt32Vector 14 | bsonInt64Vector 15 | bsonDoubleVector 16 | bsonMixedVector 17 | bsonObject 18 | bsonTrue 19 | bsonFalse 20 | bsonNull 21 | bsonRoot 22 | ) 23 | 24 | type bsonField = struct { 25 | etype eBsonType 26 | blob []byte 27 | } 28 | -------------------------------------------------------------------------------- /manticore/status_test.go: -------------------------------------------------------------------------------- 1 | package manticore 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | //status, err := cl.Status () 9 | // foreach ( $status as $row ) 10 | // print join ( ": ", $row ) . "\n"; 11 | 12 | 13 | func TestClient_Status_global(t *testing.T) { 14 | 15 | cl := NewClient() 16 | foo, err := cl.Status(false) 17 | if err != nil { 18 | fmt.Println(err.Error()) 19 | } else { 20 | for key, line := range (foo) { 21 | fmt.Printf("%v:\t%v\n", key, line) 22 | } 23 | } 24 | } -------------------------------------------------------------------------------- /manticore/manticore_test.go: -------------------------------------------------------------------------------- 1 | package manticore 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | func ExampleEscapeString() { 9 | 10 | escaped := EscapeString("escaping-sample@query/string") 11 | fmt.Println(escaped) 12 | // Output: 13 | // escaping\-sample\@query\/string 14 | } 15 | 16 | 17 | func TestClient_Ping(t *testing.T) { 18 | 19 | cl := NewClient() 20 | 21 | foo, err := cl.Ping (123456789 ) 22 | if err != nil { 23 | fmt.Println(err.Error()) 24 | } else { 25 | fmt.Println(foo) 26 | } 27 | } 28 | 29 | -------------------------------------------------------------------------------- /manticore/sphinxql_test.go: -------------------------------------------------------------------------------- 1 | package manticore 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | func TestClient_Sphinxql_selectmeta(t *testing.T) { 9 | 10 | cl := NewClient() 11 | 12 | foo, err := cl.Sphinxql("select channel_id+1.3 x, * from lj; show meta") 13 | if err != nil { 14 | fmt.Println(err.Error()) 15 | } else { 16 | fmt.Println(foo) 17 | } 18 | } 19 | 20 | func TestClient_Sphinxql_status(t *testing.T) { 21 | 22 | cl := NewClient() 23 | 24 | foo, err := cl.Sphinxql("show status") 25 | if err != nil { 26 | fmt.Println(err.Error()) 27 | } else { 28 | fmt.Println(foo) 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /manticore/uvar_test.go: -------------------------------------------------------------------------------- 1 | package manticore 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | func TestClient_Uvar(t *testing.T) { 9 | 10 | cl := NewClient() 11 | _, _ = cl.Open() 12 | 13 | // values are 1) not sorted; 2) have dupe. 14 | err := cl.Uvar("@foo", []uint64{7811237, 7811235, 7811235, 7811233, 7811236}) 15 | if err != nil { 16 | fmt.Println(err.Error()) 17 | } 18 | 19 | q := NewSearch("", "lj", "") 20 | q.AddFilterUservar("id", "@foo", false) 21 | 22 | // expect to receive 4 rows 23 | foo, err := cl.RunQuery(q) 24 | 25 | if err != nil { 26 | fmt.Println(err.Error()) 27 | if foo != nil { 28 | fmt.Println(foo.Error) 29 | } 30 | } else { 31 | fmt.Println(foo) 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /manticore/update_test.go: -------------------------------------------------------------------------------- 1 | package manticore 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | func TestClient_UpdateAttributes(t *testing.T) { 9 | 10 | cl := NewClient() 11 | 12 | upd, err := cl.UpdateAttributes("lj", []string{"channel_id"}, map[DocID][]interface{}{5000000:{1}, 5000011:{11}}, UpdateInt, false) 13 | if err != nil { 14 | fmt.Println(err.Error()) 15 | } else { 16 | fmt.Println(upd) 17 | } 18 | } 19 | 20 | func TestClient_UpdateAttributes_many(t *testing.T) { 21 | 22 | cl := NewClient() 23 | 24 | upd, err := cl.UpdateAttributes("lj", []string{"channel_id","published"}, map[DocID][]interface{}{5000000:{1,2}, 5000011:{3,4}}, UpdateInt, false) 25 | if err != nil { 26 | fmt.Println(err.Error()) 27 | } else { 28 | fmt.Println(upd) 29 | } 30 | } -------------------------------------------------------------------------------- /manticore/json.go: -------------------------------------------------------------------------------- 1 | package manticore 2 | 3 | func buildJsonRequest(endpoint, request string) func(*apibuf) { 4 | return func(buf *apibuf) { 5 | buf.putString(endpoint) 6 | buf.putString(request) 7 | } 8 | } 9 | 10 | /* 11 | JsonAnswer encapsulates answer to Json command. 12 | 13 | `Endpoint` - endpoint to which request was directed 14 | 15 | `Answer` - string, containing the answer. In opposite to true HTTP connection, here only string mesages given, 16 | no numeric error codes. 17 | 18 | */ 19 | type JsonAnswer struct { 20 | Endpoint string 21 | Answer string 22 | } 23 | 24 | func parseJsonAnswer() func(*apibuf) interface{} { 25 | return func(answer *apibuf) interface{} { 26 | endpoint := answer.getString() 27 | blob := answer.getString() 28 | return JsonAnswer{endpoint, blob} 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /manticore/json_test.go: -------------------------------------------------------------------------------- 1 | package manticore 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | func TestClient_Json_search(t *testing.T) { 9 | 10 | cl := NewClient() 11 | 12 | foo, err := cl.Json("search", "index=lj&match=luther&select=id,channel_id&limit=20") 13 | if err != nil { 14 | fmt.Println(err.Error()) 15 | } else { 16 | fmt.Println(foo) 17 | } 18 | } 19 | 20 | func TestClient_Json_sqlapi(t *testing.T) { 21 | 22 | cl := NewClient() 23 | 24 | foo, err := cl.Json("sql", "query=select * from lj where match ('luther')") 25 | if err != nil { 26 | fmt.Println(err.Error()) 27 | } else { 28 | fmt.Println(foo) 29 | } 30 | } 31 | 32 | func TestClient_Json_json_search(t *testing.T) { 33 | 34 | cl := NewClient() 35 | 36 | foo, err := cl.Json("json/search", `{"index":"lj","query":{"match":{"title":"luther"}}}`) 37 | if err != nil { 38 | fmt.Println(err.Error()) 39 | } else { 40 | fmt.Println(foo) 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /manticore/snippets_test.go: -------------------------------------------------------------------------------- 1 | package manticore 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | func TestClient_BuildExcerpts_default(t *testing.T) { 9 | 10 | cl := NewClient() 11 | foo, err := cl.BuildExcerpts([]string{"10 word1 here", "20 word2 there"}, "lj", "word1 word2") 12 | 13 | if err != nil { 14 | fmt.Println(err.Error()) 15 | } else { 16 | fmt.Println(foo) 17 | } 18 | } 19 | 20 | func TestClient_BuildExcerpts_custom(t *testing.T) { 21 | 22 | cl := NewClient() 23 | opts := NewSnippetOptions() 24 | opts.BeforeMatch, opts.AfterMatch = "before", "after" 25 | opts.ChunkSeparator = "separator" 26 | opts.Limit = 10 27 | foo, err := cl.BuildExcerpts([]string{"10 word1 here", "20 word2 there"}, "lj", "word1 word2", *opts) 28 | 29 | if err != nil { 30 | fmt.Println(err.Error()) 31 | } else { 32 | fmt.Println(foo) 33 | } 34 | } 35 | 36 | func TestClient_BuildExcerpts_flags(t *testing.T) { 37 | 38 | cl := NewClient() 39 | opts := NewSnippetOptions() 40 | opts.Flags = ExcerptFlagExactphrase | ExcerptFlagUseboundaries | ExcerptFlagWeightorder 41 | foo, err := cl.BuildExcerpts([]string{"10 word1 here", "20 word2 there"}, "lj", "word1 word2", *opts) 42 | 43 | if err != nil { 44 | fmt.Println(err.Error()) 45 | } else { 46 | fmt.Println(foo) 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /manticore/callpq_test.go: -------------------------------------------------------------------------------- 1 | package manticore 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | func TestClient_Sphinxql_tbls(t *testing.T) { 9 | 10 | cl := NewClient() 11 | cl.SetServer("", 6712) 12 | 13 | foo, err := cl.Sphinxql("select * from pq; select * from pq1") 14 | if err != nil { 15 | fmt.Println(err.Error()) 16 | } else { 17 | fmt.Println(foo) 18 | } 19 | } 20 | 21 | //CALL PQ ('META:multi', ('[{"title":"angry test", "gid":3 }, 22 | // {"title":"filter test doc2", "gid":13}]'), 23 | // 1 as docs, 1 as verbose, 1 as docs_json, 1 as query, 'gid' as docs_id) 24 | 25 | func TestClient_CallPQ(t *testing.T) { 26 | cl := NewClient() 27 | cl.SetServer("", 6712) 28 | 29 | pq := NewSearchPqOptions() 30 | pq.Flags = NeedDocs | Verbose | NeedQuery 31 | 32 | resp, err := cl.CallPQ("pq", []string{"angry test", "filter test doc2"}, pq) 33 | if err != nil { 34 | fmt.Println(err.Error()) 35 | } else { 36 | fmt.Println(resp) 37 | } 38 | } 39 | 40 | func TestClient_Sphinxql_callpq(t *testing.T) { 41 | 42 | cl := NewClient() 43 | cl.SetServer("", 6712) 44 | 45 | _, _ = cl.Open() 46 | foo, err := cl.Sphinxql(`call pq ('pq', ('angry test','filter test doc2'), 1 as docs, 1 as verbose, 1 as query, 0 as docs_json)`) 47 | foo1, err := cl.Sphinxql(`show meta`) 48 | if err != nil { 49 | fmt.Println(err.Error()) 50 | } else { 51 | fmt.Println(foo) 52 | fmt.Println(foo1) 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /manticore/uvar.go: -------------------------------------------------------------------------------- 1 | package manticore 2 | 3 | import "sort" 4 | 5 | // make the array sortable 6 | type uint64Slice []uint64 7 | 8 | func (p uint64Slice) Len() int { return len(p) } 9 | func (p uint64Slice) Less(i, j int) bool { return p[i] < p[j] } 10 | func (p uint64Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } 11 | 12 | type vlbcomp []byte 13 | 14 | func (s *vlbcomp) compress(val uint64) int { 15 | if val == 0 { 16 | return 0 17 | } 18 | for true { 19 | char := byte(val & 0x7f) 20 | val >>= 7 21 | if val == 0 { 22 | *s = append(*s, char) 23 | return 1 24 | } 25 | *s = append(*s, char|0x80) 26 | } 27 | return 0 28 | } 29 | 30 | func buildUvarRequest(name string, values []uint64) func(*apibuf) { 31 | return func(buf *apibuf) { 32 | 33 | // make copy of original values 34 | cp := uint64Slice(make([]uint64, len(values))) 35 | copy(cp, values) 36 | 37 | // prepare delta-encoded vbl compressed blob 38 | 39 | // sort given blob 40 | sort.Sort(cp) 41 | 42 | var output vlbcomp 43 | 44 | // delta-encoding 45 | prev := uint64(0) 46 | nvalues := int(0) // will calculate N of non-zero deltas (i.e. uniq values) 47 | for i := 0; i < len(cp); i++ { 48 | // compress + uniq 49 | nvalues += output.compress(cp[i] - prev) 50 | prev = cp[i] 51 | } 52 | 53 | buf.putString(name) 54 | buf.putLen(nvalues) 55 | buf.putLen(len(output)) 56 | buf.putBytes(output) 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /manticore/keywords_test.go: -------------------------------------------------------------------------------- 1 | package manticore 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | func TestClient_BuildKeywords(t *testing.T) { 10 | cl := NewClient() 11 | 12 | cl.SetServer("localhost") 13 | cl.SetConnectTimeout(1 * time.Second) 14 | 15 | kwds, err := cl.BuildKeywords("martin luthers king", "lj", false) 16 | 17 | if err != nil { 18 | fmt.Println(err.Error()) 19 | } else { 20 | fmt.Println(kwds) 21 | } 22 | } 23 | 24 | func ExampleClient_BuildKeywords_withoutHits() { 25 | cl := NewClient() 26 | 27 | keywords, err := cl.BuildKeywords("this.is.my query", "lj", false) 28 | if err != nil { 29 | fmt.Println(err.Error()) 30 | } else { 31 | fmt.Println(keywords) 32 | } 33 | // Output: 34 | // [{Tok: 'this', Norm: 'this', Qpos: 1; docs/hits 0/0} 35 | // {Tok: 'is', Norm: 'is', Qpos: 2; docs/hits 0/0} 36 | // {Tok: 'my', Norm: 'my', Qpos: 3; docs/hits 0/0} 37 | // {Tok: 'query', Norm: 'query', Qpos: 4; docs/hits 0/0} 38 | // ] 39 | } 40 | 41 | func ExampleClient_BuildKeywords_withHits() { 42 | cl := NewClient() 43 | 44 | keywords, err := cl.BuildKeywords("this.is.my query", "lj", true) 45 | if err != nil { 46 | fmt.Println(err.Error()) 47 | } else { 48 | fmt.Println(keywords) 49 | } 50 | // Output: 51 | // [{Tok: 'this', Norm: 'this', Qpos: 1; docs/hits 1629922/3905279} 52 | // {Tok: 'is', Norm: 'is', Qpos: 2; docs/hits 1901345/6052344} 53 | // {Tok: 'my', Norm: 'my', Qpos: 3; docs/hits 1981048/7549917} 54 | // {Tok: 'query', Norm: 'query', Qpos: 4; docs/hits 1235/1474} 55 | // ] 56 | } 57 | -------------------------------------------------------------------------------- /manticore/keywords.go: -------------------------------------------------------------------------------- 1 | package manticore 2 | 3 | import "fmt" 4 | 5 | // Keyword represents a keyword returned from BuildKeywords() call 6 | type Keyword struct { 7 | Tokenized string // token from the query 8 | Normalized string // normalized token after all stemming/lemming 9 | Querypos int // position in the query 10 | Docs int // number of docs (from backend index) 11 | Hits int // number of hits (from backend index) 12 | } 13 | 14 | // Stringer interface for Keyword type 15 | func (kw Keyword) String() string { 16 | return fmt.Sprintf("{Tok: '%v',\tNorm: '%v',\tQpos: %v; docs/hits %v/%v}\n", kw.Tokenized, kw.Normalized, 17 | kw.Querypos, kw.Docs, kw.Hits) 18 | } 19 | 20 | func buildKeywordsRequest(query, index string, hits bool) func(*apibuf) { 21 | return func(buf *apibuf) { 22 | buf.putString(query) 23 | buf.putString(index) 24 | buf.putBoolDword(hits) 25 | 26 | buf.putBoolDword(false) // fixme! FoldLemmas 27 | buf.putBoolDword(false) // fixme! FoldBlended 28 | buf.putBoolDword(false) // fixme! FoldWildcards 29 | buf.putDword(0) // fixme! ExpansionLimit 30 | } 31 | } 32 | 33 | func parseKeywordsAnswer(hits bool) func(*apibuf) interface{} { 34 | return func(answer *apibuf) interface{} { 35 | nkeywords := answer.getInt() 36 | keywords := make([]Keyword, nkeywords) 37 | for j := 0; j < nkeywords; j++ { 38 | keywords[j].Tokenized = answer.getString() 39 | keywords[j].Normalized = answer.getString() 40 | keywords[j].Querypos = answer.getInt() 41 | if hits { 42 | keywords[j].Docs = answer.getInt() 43 | keywords[j].Hits = answer.getInt() 44 | } 45 | } 46 | return keywords 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /manticore/update.go: -------------------------------------------------------------------------------- 1 | package manticore 2 | /* 3 | EUpdateType is values for `vtype` of UpdateAttributes() call, which determines meaning of `values` param of this function. 4 | 5 | UpdateInt 6 | 7 | This is the default value. `values` hash holds documents IDs as keys and a plain arrays of new attribute values. 8 | 9 | UpdateMva 10 | 11 | Points that MVA attributes are being updated. In this case the `values` must be a hash with document IDs as keys 12 | and array of arrays of int values (new MVA attribute values). 13 | 14 | UpdateString 15 | 16 | Points that string attributes are being updated. `values` must be a hash with document IDs as keys and array of strings as values. 17 | 18 | UpdateJson 19 | 20 | Works the same as `UpdateString`, but for JSON attribute updates. 21 | */ 22 | type EUpdateType uint32 23 | 24 | const ( 25 | UpdateInt EUpdateType = iota 26 | UpdateMva 27 | UpdateString 28 | UpdateJson 29 | ) 30 | 31 | func buildUpdateRequest(index string, attrs []string, values map[DocID][]interface{}, 32 | vtype EUpdateType, ignorenonexistent bool) func(*apibuf) { 33 | nattrs := len(attrs) 34 | nvalues := len(values) 35 | return func(buf *apibuf) { 36 | buf.putString(index) 37 | buf.putLen(nattrs) 38 | buf.putBoolDword(ignorenonexistent) 39 | 40 | for j := 0; j < nattrs; j++ { 41 | buf.putString(attrs[j]) 42 | buf.putDword(uint32(vtype)) 43 | } 44 | 45 | buf.putLen(nvalues) 46 | for key, value := range values { 47 | buf.putDocid(key) 48 | switch vtype { 49 | case UpdateInt: 50 | for j := 0; j < nattrs; j++ { 51 | buf.putInt(int32(value[j].(int))) 52 | } 53 | case UpdateMva: 54 | for j := 0; j < nattrs; j++ { 55 | foo := value[j].([]uint32) 56 | buf.putLen(len(foo)) 57 | for k := 0; k unixsocket on /var/log 13 | cl.SetServer("/var/log") 14 | 15 | if cl.dialmethod != "unix" { 16 | t.Errorf("method is not as expected unix, got %s", cl.dialmethod) 17 | } 18 | 19 | if cl.host != "/var/log" { 20 | t.Errorf("host is not as expected, got %s", cl.host) 21 | } 22 | 23 | // unix:///bla -> unixsocket /bla 24 | cl.SetServer("unix:///tmp.sock") 25 | 26 | if cl.dialmethod != "unix" { 27 | t.Errorf("method is not as expected unix, got %s", cl.dialmethod) 28 | } 29 | 30 | if cl.host != "/tmp.sock" { 31 | t.Errorf("host is not as expected, got %s", cl.host) 32 | } 33 | 34 | // path starting from not / - tcp connect to sock, port 35 | cl.SetServer("google.com") 36 | 37 | if cl.dialmethod != "tcp" { 38 | t.Errorf("path is not as expected tcp, got %s", cl.dialmethod) 39 | } 40 | 41 | if cl.host != "google.com" { 42 | t.Errorf("host is not as expected, got %s", cl.host) 43 | } 44 | 45 | if cl.port != 0 { 46 | t.Errorf("port is not as expected, got %d", cl.port) 47 | } 48 | 49 | cl.SetServer("google.com", 9999) 50 | 51 | if cl.dialmethod != "tcp" { 52 | t.Errorf("path is not as expected tcp, got %s", cl.dialmethod) 53 | } 54 | 55 | if cl.host != "google.com" { 56 | t.Errorf("host is not as expected, got %s", cl.host) 57 | } 58 | 59 | if cl.port != 9999 { 60 | t.Errorf("port is not as expected, got %d", cl.port) 61 | } 62 | } 63 | 64 | func ExampleClient_SetServer_unixsocket() { 65 | cl := NewClient() 66 | cl.SetServer("/var/log") 67 | 68 | fmt.Println(cl.dialmethod) 69 | fmt.Println(cl.host) 70 | // Output: 71 | // unix 72 | // /var/log 73 | } 74 | 75 | func ExampleClient_SetServer_tcpsocket() { 76 | cl := NewClient() 77 | cl.SetServer("google.com", 9999) 78 | 79 | fmt.Println(cl.dialmethod) 80 | fmt.Println(cl.host) 81 | fmt.Println(cl.port) 82 | // Output: 83 | // tcp 84 | // google.com 85 | // 9999 86 | } 87 | 88 | 89 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Binary-API based Go SDK for [Manticore Search](https://www.manticoresearch.com). 2 | 3 | ❗❗❗ WARNING ❗❗❗ 4 | 5 | February 10th 2024: 6 | 7 | **🚀We've released the new Manticore Go Client - https://github.com/manticoresoftware/manticoresearch-go . 🔃The SDK in this repository will no longer receive support. We recommend users switch to the new client for future updates and support.** 8 | 9 | 🔧Why the change? This Go SDK was hard to maintain due to its manual creation process and reliance on the Manticore binary protocol. While this method did offer insignificant speed benefits, it also made updates and maintenance more cumbersome. The new Go client marks a significant leap forward. By adopting auto-generation from the [OpenAPI specifications](https://manual.manticoresearch.com/Openapi#OpenAPI-specification), we ensure easier updates, consistent cross-SDK compatibility, and a stronger support moving forward. Transitioning away from a binary protocol may insignificantly impact performance, but the advantages of maintainability, simplified upgrades, and standard API practices significantly outweigh the drawbacks. 10 | 11 | 📣We urge all users to migrate to the new Go client. Designed for durability, ease of use, and seamless integration with Manticore's search capabilities, it's the future-proof choice. For migration support, visit our docs or contact us. 12 | 13 | ## Compatibility 14 | The client is compatible with Manticore Search 2.8.2 and higher for majority of the commands. 15 | It also may be used in many cases to access SphinxSearch daemon as well. However it's not guaranteed. 16 | 17 | ## Requirements 18 | Go version 1.9 or higher 19 | 20 | ## Installation 21 | ``` 22 | go get github.com/manticoresoftware/go-sdk/manticore 23 | ``` 24 | 25 | ## Usage 26 | Here's a short example on how the client can be used: 27 | Make sure there's some running Manticore instance, you can use our [docker image](https://hub.docker.com/r/manticoresearch/manticore) for a quick test: 28 | ``` 29 | docker run --name manticore -p 9313:9312 -p9306:9306 -d manticoresearch/manticore 30 | ``` 31 | 32 | Here's just a simplest script: 33 | ``` 34 | [root@srv ~]# cat manticore.go 35 | package main 36 | 37 | import "github.com/manticoresoftware/go-sdk/manticore" 38 | import "fmt" 39 | 40 | func main() { 41 | cl := manticore.NewClient() 42 | cl.SetServer("127.0.0.1", 9313) 43 | cl.Open() 44 | res, err := cl.Sphinxql(`replace into testrt values(1,'my subject', 'my content', 15)`) 45 | fmt.Println(res, err) 46 | res, err = cl.Sphinxql(`replace into testrt values(2,'another subject', 'more content', 15)`) 47 | fmt.Println(res, err) 48 | res, err = cl.Sphinxql(`replace into testrt values(5,'again subject', 'one more content', 10)`) 49 | fmt.Println(res, err) 50 | res2, err2 := cl.Query("more|another", "testrt") 51 | fmt.Println(res2, err2) 52 | } 53 | ``` 54 | 55 | And here's how it works: 56 | ``` 57 | [root@srv ~]# go run manticore.go 58 | [Query OK, 1 rows affected] 59 | [Query OK, 1 rows affected] 60 | [Query OK, 1 rows affected] 61 | Status: ok 62 | Query time: 0s 63 | Total: 1 64 | Total found: 1 65 | Schema: 66 | Fields: 67 | title 68 | content 69 | Attributes: 70 | gid: int 71 | Matches: 72 | Doc: 2, Weight: 2, attrs: [15] 73 | Word stats: 74 | 'more' (Docs:2, Hits:2) 75 | 'another' (Docs:1, Hits:1) 76 | 77 | ``` 78 | 79 | Read [full documentation on godoc](https://godoc.org/github.com/manticoresoftware/go-sdk/manticore) to learn more about available functions and find more examples. You can also read it from the console as `go doc go-sdk/manticore` 80 | 81 | -------------------------------------------------------------------------------- /manticore/sphinxproto.go: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright (c) 2019, Manticore Software LTD (http://manticoresearch.com) 3 | // All rights reserved 4 | // 5 | // This program is free software; you can redistribute it and/or modify 6 | // it under the terms of the GNU General Public License. You should have 7 | // received a copy of the GPL license along with this program; if you 8 | // did not, you can find it at http://www.gnu.org/ 9 | // 10 | 11 | package manticore 12 | 13 | import ( 14 | "encoding/binary" 15 | "math" 16 | "time" 17 | ) 18 | 19 | type apibuf []byte 20 | 21 | func (buf *apibuf) putByte(val uint8) { 22 | *buf = append(*buf, val) 23 | } 24 | 25 | func (buf *apibuf) putWord(val uint16) { 26 | tmp := make([]byte, 2) 27 | binary.BigEndian.PutUint16(tmp, val) 28 | *buf = append(*buf, tmp...) 29 | } 30 | 31 | func (buf *apibuf) putUint(val uint32) { 32 | tmp := make([]byte, 4) 33 | binary.BigEndian.PutUint32(tmp, val) 34 | *buf = append(*buf, tmp...) 35 | } 36 | 37 | func (buf *apibuf) putInt(val int32) { 38 | buf.putUint(uint32(val)) 39 | } 40 | 41 | func (buf *apibuf) putDword(val uint32) { 42 | buf.putUint(val) 43 | } 44 | 45 | func (buf *apibuf) putUint64(val uint64) { 46 | tmp := make([]byte, 8) 47 | binary.BigEndian.PutUint64(tmp, val) 48 | *buf = append(*buf, tmp...) 49 | } 50 | 51 | func (buf *apibuf) putInt64(val int64) { 52 | buf.putUint64(uint64(val)) 53 | } 54 | 55 | func (buf *apibuf) putDocid(val DocID) { 56 | buf.putUint64(uint64(val)) 57 | } 58 | 59 | func (buf *apibuf) putLen(val int) { 60 | buf.putUint(uint32(val)) 61 | } 62 | 63 | func (buf *apibuf) putFloat(val float32) { 64 | buf.putUint(math.Float32bits(val)) 65 | } 66 | 67 | func (buf *apibuf) putDuration(val time.Duration) { 68 | buf.putUint(uint32(val / time.Millisecond)) 69 | } 70 | 71 | func (buf *apibuf) putBoolByte(val bool) { 72 | if val { 73 | buf.putByte(1) 74 | } else { 75 | buf.putByte(0) 76 | } 77 | } 78 | 79 | func (buf *apibuf) putBoolDword(val bool) { 80 | if val { 81 | buf.putDword(1) 82 | } else { 83 | buf.putDword(0) 84 | } 85 | } 86 | 87 | func (buf *apibuf) putBytes(val []byte) { 88 | *buf = append(*buf, val...) 89 | } 90 | 91 | func (buf *apibuf) putString(str string) { 92 | buf.putLen(len(str)) 93 | bytes := []byte(str) 94 | buf.putBytes(bytes) 95 | } 96 | 97 | func (buf *apibuf) getByte() byte { 98 | val := (*buf)[0] 99 | *buf = (*buf)[1:] 100 | return val 101 | } 102 | 103 | func (buf *apibuf) getWord() uint16 { 104 | val := binary.BigEndian.Uint16(*buf) 105 | *buf = (*buf)[2:] 106 | return val 107 | } 108 | 109 | func (buf *apibuf) getDword() uint32 { 110 | val := binary.BigEndian.Uint32(*buf) 111 | *buf = (*buf)[4:] 112 | return val 113 | } 114 | 115 | func (buf *apibuf) getInt() int { 116 | return int(buf.getDword()) 117 | } 118 | 119 | func (buf *apibuf) getUint64() uint64 { 120 | val := binary.BigEndian.Uint64(*buf) 121 | *buf = (*buf)[8:] 122 | return val 123 | } 124 | 125 | func (buf *apibuf) getInt64() int64 { 126 | return int64(buf.getUint64()) 127 | } 128 | 129 | func (buf *apibuf) getDocid() DocID { 130 | return DocID(buf.getUint64()) 131 | } 132 | 133 | func (buf *apibuf) getFloat() float32 { 134 | return math.Float32frombits(buf.getDword()) 135 | } 136 | 137 | func (buf *apibuf) getByteBool() bool { 138 | return buf.getByte() != 0 139 | } 140 | 141 | func (buf *apibuf) getIntBool() bool { 142 | return buf.getDword() != 0 143 | } 144 | 145 | func (buf *apibuf) getString() string { 146 | lng := buf.getInt() 147 | result := string((*buf)[:lng]) 148 | *buf = (*buf)[lng:] 149 | return result 150 | } 151 | 152 | // zerocopy (return slice to original buffer) 153 | func (buf *apibuf) getRefBytes() []byte { 154 | lng := buf.getInt() 155 | result := (*buf)[:lng] 156 | *buf = (*buf)[lng:] 157 | return result 158 | } 159 | 160 | // full-copy 161 | func (buf *apibuf) getBytes() []byte { 162 | src := buf.getRefBytes() 163 | dst := make([]byte, len(src)) 164 | copy(dst, src) 165 | return dst 166 | } 167 | 168 | func (buf *apibuf) apiCommand(uCommand eSearchdcommand) int { 169 | buf.putWord(uint16(uCommand)) 170 | buf.putWord(uint16(searchdcommandv[uCommand])) 171 | iPlace := len(*buf) 172 | buf.putUint(0) // space for future len encoding 173 | return iPlace 174 | } 175 | 176 | func (buf *apibuf) finishAPIPacket(iPlace int) { 177 | uLen := uint32(len(*buf) - iPlace - 4) 178 | binary.BigEndian.PutUint32((*buf)[iPlace:], uLen) 179 | } 180 | 181 | // ensure buf is capable for given size 182 | func (buf *apibuf) resizeBuf(size, maxsize int) { 183 | if cap(*buf) > maxsize { 184 | *buf = nil 185 | } 186 | if len(*buf) < size { 187 | *buf = append(*buf, make([]byte, size-len(*buf))...) 188 | } 189 | *buf = (*buf)[:size] 190 | } 191 | -------------------------------------------------------------------------------- /manticore/search_test.go: -------------------------------------------------------------------------------- 1 | package manticore 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | func TestClient_Query_default(t *testing.T) { 9 | cl := NewClient() 10 | foo, err := cl.Query("query") 11 | if err != nil { 12 | fmt.Println(err.Error()) 13 | } else { 14 | fmt.Println(foo) 15 | } 16 | } 17 | 18 | func TestClient_Query_index(t *testing.T) { 19 | cl := NewClient() 20 | foo, err := cl.Query("query", "lj") 21 | if err != nil { 22 | fmt.Println(err.Error()) 23 | } else { 24 | fmt.Println(foo) 25 | } 26 | } 27 | 28 | func TestClient_Query_unixsocket(t *testing.T) { 29 | cl := NewClient() 30 | 31 | cl.SetServer("/work/lj/sphinxapi") 32 | q := NewSearch("luther", "lj", "") 33 | q.SetSortMode(SortAttrAsc, "published") 34 | foo, err := cl.RunQuery(q) 35 | if err != nil { 36 | fmt.Println(err.Error()) 37 | } else { 38 | fmt.Println(foo) 39 | } 40 | } 41 | 42 | func TestClient_RunPreparedQueries(t *testing.T) { 43 | cl := NewClient() 44 | 45 | queries := []Search{ 46 | NewSearch("luther", "lj", ""), 47 | NewSearch("martin luther", "lj", ""), 48 | } 49 | foo, err := cl.RunQueries(queries) 50 | if err != nil { 51 | fmt.Println(err.Error()) 52 | } else { 53 | fmt.Println(foo) 54 | } 55 | } 56 | 57 | func TestSearch_SetFieldWeights(t *testing.T) { 58 | cl := NewClient() 59 | 60 | q := NewSearch("luther", "lj", "") 61 | q.FieldWeights = map[string]int32{"title": 1000, "content": 10} 62 | foo, err := cl.RunQuery(q) 63 | if err != nil { 64 | fmt.Println(err.Error()) 65 | } else { 66 | fmt.Println(foo) 67 | } 68 | } 69 | 70 | func TestSearch_AddFilter(t *testing.T) { 71 | 72 | q := NewSearch("query", "lj", "") 73 | q.AddFilter("channel_id", []int64{537345, 536802, 538617}, false) 74 | 75 | cl := NewClient() 76 | foo, err := cl.RunQuery(q) 77 | if err != nil { 78 | fmt.Println(err.Error()) 79 | } else { 80 | fmt.Println(foo) 81 | } 82 | } 83 | 84 | func TestSearch_AddFilter_exclude(t *testing.T) { 85 | 86 | q := NewSearch("query", "lj", "") 87 | q.AddFilter("channel_id", []int64{537345, 536802, 538617}, true) 88 | 89 | cl := NewClient() 90 | foo, err := cl.RunQuery(q) 91 | if err != nil { 92 | fmt.Println(err.Error()) 93 | } else { 94 | fmt.Println(foo) 95 | } 96 | } 97 | 98 | func TestSearch_AddFilterFloatRange(t *testing.T) { 99 | 100 | q := NewSearch("query", "lj", "") 101 | q.AddFilterFloatRange("channel_id", 10000.0, 200000.0, false) 102 | 103 | cl := NewClient() 104 | foo, err := cl.RunQuery(q) 105 | if err != nil { 106 | fmt.Println(err.Error()) 107 | } else { 108 | fmt.Println(foo) 109 | } 110 | } 111 | 112 | func TestSearch_AddFilterUservar(t *testing.T) { 113 | 114 | var q Search 115 | q.AddFilterUservar("foo", "bar", false) 116 | if len(q.filters) != 1 { 117 | t.Errorf("Wrong len of filters: %d", len(q.filters)) 118 | } 119 | } 120 | 121 | func TestSearch_AddFilterExpression(t *testing.T) { 122 | q := NewSearch("query", "lj", "") 123 | q.SelectClause = "channel_id*10 as cchh, channel_id" 124 | q.AddFilterExpression("channel_id*10<1000", false) 125 | 126 | cl := NewClient() 127 | _, erro := cl.Open() 128 | if erro!=nil { 129 | fmt.Println(erro.Error()) 130 | } 131 | 132 | foo, err := cl.RunQuery(q) 133 | bar, err1 := cl.Status(false) 134 | _,_ = cl.Close() 135 | if err != nil { 136 | fmt.Println(err.Error()) 137 | } else { 138 | fmt.Println(foo) 139 | } 140 | 141 | if err1 != nil { 142 | fmt.Println(err1.Error()) 143 | } else { 144 | for key, line := range (bar) { 145 | fmt.Printf("%v:\t%v\n", key, line) 146 | } 147 | } 148 | } 149 | 150 | func TestSearch_SetSortMode(t *testing.T) { 151 | q := NewSearch("query", "lj", "") 152 | q.SelectClause = "channel_id*10 as cchh, channel_id*5 as cchhh" 153 | q.SetSortMode(SortExtended, "cchh DESC, cchhh DESC") 154 | 155 | cl := NewClient() 156 | foo, err := cl.RunQuery(q) 157 | 158 | if err != nil { 159 | fmt.Println(err.Error()) 160 | if foo!=nil { 161 | fmt.Println(foo.Error) 162 | } 163 | } else { 164 | fmt.Println(foo) 165 | } 166 | } 167 | 168 | // q := NewSearch("some common query terms", "index", "") 169 | // q.SelectClause = "id, slow_rank() as slow, fast_rank as fast" 170 | // q.SetSortMode( SortExpr, "fast DESC, slow DESC" 171 | // q.AddFilterExpression("channel_id*10<1000", false) 172 | 173 | 174 | func ExampleQflags() { 175 | fl := QflagJsonQuery 176 | fmt.Println(fl) 177 | // Output: 178 | // 2048 179 | } 180 | 181 | func TestSearch_SetOuterSelect(t *testing.T) { 182 | q := NewSearch("query", "lj", "") 183 | q.SelectClause = "channel_id*10 as cchh, channel_id" 184 | q.SetOuterSelect("cchh asc", 0, 3) 185 | 186 | cl := NewClient() 187 | foo, err := cl.RunQuery(q) 188 | if err != nil { 189 | fmt.Println(err.Error()) 190 | } else { 191 | fmt.Println(foo) 192 | } 193 | } 194 | 195 | -------------------------------------------------------------------------------- /manticore/client.go: -------------------------------------------------------------------------------- 1 | package manticore 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "io" 7 | "net" 8 | "time" 9 | ) 10 | 11 | // Client represents connection to manticore daemon. It provides set of public API functions 12 | type Client struct { 13 | host, dialmethod string 14 | port uint16 15 | conn net.Conn 16 | connected, connError bool 17 | lastWarning string 18 | buf apibuf 19 | timeout time.Duration 20 | maxAlloc int 21 | } 22 | 23 | // NewClient creates default connector, which points to 'localhost:9312', has zero timeout and 8M maxalloc. 24 | // Defaults may be changed later by invoking `SetServer()`, `SetMaxAlloc()` 25 | func NewClient() Client { 26 | return Client{ 27 | "localhost", "tcp", 28 | SphinxPort, 29 | nil, 30 | false, false, 31 | "", 32 | nil, 33 | 0, 34 | 8 * 1024 * 1024, 35 | } 36 | } 37 | 38 | // getByteBuf provides byte buffer. One and same buffer will be reused on each call, so that no GC will 39 | // be in use/ 40 | func (cl *Client) getByteBuf(size int) *apibuf { 41 | cl.buf.resizeBuf(size, cl.maxAlloc) 42 | return &cl.buf 43 | } 44 | 45 | func (cl *Client) getOutBuf() *apibuf { 46 | return cl.getByteBuf(0) 47 | } 48 | 49 | // gracefully return error 50 | func (cl *Client) confail(err error) error { 51 | cl.connError = err != nil 52 | return err 53 | } 54 | 55 | func (cl *Client) failclose(err error) error { 56 | if err != nil { 57 | _ = cl.conn.Close() 58 | cl.conn = nil 59 | cl.connected = false 60 | } 61 | return err 62 | } 63 | 64 | func (client *Client) eof() bool { 65 | 66 | if !client.connected { 67 | return true 68 | } 69 | _ = client.conn.SetReadDeadline(time.Now()) 70 | var one []byte 71 | if _, err := client.conn.Read(one); err == io.EOF { 72 | client.connected = false 73 | return true 74 | } 75 | _ = client.conn.SetReadDeadline(time.Time{}) 76 | return false 77 | } 78 | 79 | /// connect to searchd server 80 | func (cl *Client) connect() error { 81 | 82 | // we are in persistent connection mode, so we have a socket 83 | // however, need to check whether it's still alive 84 | if cl.connected { 85 | if cl.eof() { // connection timed out 86 | _ = cl.conn.Close() 87 | cl.conn = nil 88 | } else { // connection alive; no more actions need 89 | return cl.confail(nil) 90 | } 91 | } 92 | 93 | address := cl.host 94 | if cl.dialmethod == "tcp" { 95 | sPort := fmt.Sprintf("%d", cl.port) 96 | address = net.JoinHostPort(address, sPort) 97 | } 98 | 99 | var err error 100 | // connect 101 | if cl.timeout != 0 { 102 | cl.conn, err = net.DialTimeout(cl.dialmethod, address, cl.timeout) 103 | } else { 104 | cl.conn, err = net.Dial(cl.dialmethod, address) 105 | } 106 | 107 | if err != nil { 108 | return cl.confail(err) 109 | } 110 | 111 | cl.connected = true 112 | 113 | // send my version 114 | // this is a subtle part. we must do it before (!) reading back from searchd. 115 | // because otherwise under some conditions (reported on FreeBSD for instance) 116 | // TCP stack could throttle write-write-read pattern because of Nagle. 117 | // send handshake, retrieve answer and check it 118 | handshake := apibuf(make([]byte, 0, 4)) 119 | handshake.putUint(cphinxClientVersion) 120 | 121 | _, err = cl.conn.Write(handshake) 122 | if err == nil { 123 | buf := cl.getByteBuf(4) 124 | _, err = cl.conn.Read(*buf) 125 | if err == nil { 126 | ver := buf.getDword() 127 | if ver == cphinxSearchdProto { 128 | return cl.confail(nil) 129 | } 130 | err = errors.New(fmt.Sprintf("Wrong version num received: %d", ver)) 131 | } 132 | } 133 | 134 | // error happened, return it 135 | return cl.failclose(err) 136 | } 137 | 138 | /// get and check response packet from searchd server 139 | func (cl *Client) getResponse(client_ver uCommandVersion) (apibuf, error) { 140 | rawrecv := cl.getByteBuf(8) 141 | nbytes, err := cl.conn.Read(*rawrecv) 142 | 143 | if err != nil { 144 | return nil, err 145 | } else if nbytes == 0 { 146 | return nil, errors.New("received zero-sized searchd response") 147 | } 148 | 149 | uStat := ESearchdstatus(rawrecv.getWord()) 150 | uVer := uCommandVersion(rawrecv.getWord()) 151 | iReplySize := rawrecv.getInt() 152 | 153 | rawanswer := cl.getByteBuf(iReplySize) 154 | nbytes, err = cl.conn.Read(*rawanswer) 155 | if err != nil { 156 | return nil, err 157 | } 158 | 159 | if nbytes != iReplySize { 160 | if nbytes == 0 { 161 | return nil, errors.New("received zero-sized searchd response") 162 | } 163 | return nil, errors.New( 164 | fmt.Sprintf("failed to read searchd response (status=%d, ver=%d, len=%d, read=%d)", 165 | uStat, uVer, iReplySize, nbytes)) 166 | } 167 | 168 | switch uStat { 169 | case StatusError: 170 | return *rawanswer, errors.New(fmt.Sprintf("searchd error: %s", rawanswer.getString())) 171 | case StatusRetry: 172 | return *rawanswer, errors.New(fmt.Sprintf("temporary searchd error: %s", rawanswer.getString())) 173 | case StatusWarning: 174 | cl.lastWarning = rawanswer.getString() 175 | case StatusOk: 176 | break 177 | default: 178 | return *rawanswer, errors.New(fmt.Sprintf("unknown status code '%d'", uStat)) 179 | } 180 | 181 | // check version 182 | if uVer < client_ver { 183 | cl.lastWarning = fmt.Sprintf("searchd command v.%v older than cl's v.%v, some options might not work", 184 | uVer, client_ver) 185 | } 186 | return *rawanswer, nil 187 | } 188 | 189 | func (cl *Client) netQuery(command eSearchdcommand, builder func(*apibuf), parser func(*apibuf) interface{}) (interface{}, error) { 190 | 191 | // connect (if necessary) 192 | err := cl.connect() 193 | if err != nil { 194 | return nil, err 195 | } 196 | 197 | // build packet 198 | buf := cl.getOutBuf() 199 | tPos := buf.apiCommand(command) 200 | if builder != nil { 201 | builder(buf) 202 | } 203 | buf.finishAPIPacket(tPos) 204 | 205 | // send query 206 | _, err = cl.conn.Write(cl.buf) 207 | if err != nil { 208 | return nil, err 209 | } 210 | 211 | if parser == nil { 212 | return nil, nil 213 | } 214 | 215 | // get response 216 | var answer apibuf 217 | answer, err = cl.getResponse(searchdcommandv[command]) 218 | 219 | if err != nil { 220 | return nil, err 221 | } 222 | 223 | // parse response 224 | if answer != nil { 225 | return parser(&answer), nil 226 | } 227 | return nil, nil 228 | } 229 | 230 | // common case when payload has only one value, which is boolean as DWORD 231 | func buildBoolRequest(val bool) func(*apibuf) { 232 | return func(buf *apibuf) { 233 | buf.putBoolDword(val) 234 | } 235 | } 236 | 237 | // common case when payload has only one value, which is DWORD 238 | func buildDwordRequest(val uint32) func(*apibuf) { 239 | return func(buf *apibuf) { 240 | buf.putDword(val) 241 | } 242 | } 243 | 244 | // common case when answer contains the only integer 245 | func parseDwordAnswer() func(*apibuf) interface{} { 246 | return func(answer *apibuf) interface{} { 247 | res := answer.getDword() 248 | return res 249 | } 250 | } 251 | -------------------------------------------------------------------------------- /manticore/snippets.go: -------------------------------------------------------------------------------- 1 | package manticore 2 | 3 | /* 4 | ExcerptFlags is bitmask for SnippetOptions.Flags 5 | Different values have to be combined with '+' or '|' operation from following constants: 6 | 7 | ExcerptFlagExactphrase 8 | 9 | Whether to highlight exact query phrase matches only instead of individual keywords. 10 | 11 | ExcerptFlagUseboundaries 12 | 13 | Whether to additionally break passages by phrase boundary characters, as configured in 14 | index settings with phrase_boundary directive. 15 | 16 | ExcerptFlagWeightorder 17 | 18 | Whether to sort the extracted passages in order of relevance (decreasing weight), or in order 19 | of appearance in the document (increasing position). 20 | 21 | ExcerptFlagQuery 22 | 23 | Whether to handle 'words' as a query in extended syntax, or as a bag of words (default behavior). 24 | For instance, in query mode "(one two | three four)" will only highlight and include those occurrences one two or 25 | three four when the two words from each pair are adjacent to each other. In default mode, any single occurrence of 26 | one, two, three, or four would be highlighted. 27 | 28 | ExcerptFlagForceAllWords 29 | 30 | Ignores the snippet length limit until it includes all the keywords. 31 | 32 | ExcerptFlagLoadFiles 33 | 34 | Whether to handle 'docs' as data to extract snippets from (default behavior), or to treat it as 35 | file names, and load data from specified files on the server side. Up to dist_threads worker threads per request 36 | will be created to parallelize the work when this flag is enabled. To parallelize snippets build between remote agents, 37 | configure ``dist_threads'' param of searchd to value 38 | greater than 1, and then invoke the snippets generation over the distributed index, which contain only one(!) local 39 | agent and several remotes. The ``snippets_file_prefix'' param of remote daemons is also in the game and the final filename is calculated 40 | by concatenation of the prefix with given name. 41 | 42 | ExcerptFlagAllowEmpty 43 | 44 | Allows empty string to be returned as highlighting result when a snippet could not be 45 | generated (no keywords match, or no passages fit the limit). By default, the beginning of original text would be 46 | returned instead of an empty string. 47 | 48 | ExcerptFlagEmitZones 49 | 50 | Emits an HTML tag with an enclosing zone name before each passage. 51 | 52 | ExcerptFlagFilesScattered 53 | 54 | It works only with distributed snippets generation with remote agents. The source files 55 | for snippets could be distributed among different agents, and the main daemon will merge together all non-erroneous 56 | results. So, if one agent of the distributed index has ‘file1.txt’, another has ‘file2.txt’ and you call for the 57 | snippets with both these files, the daemon will merge results from the agents together, so you will get the snippets 58 | from both ‘file1.txt’ and ‘file2.txt’. 59 | 60 | If the load_files is also set, the request will return the error in case if any of the files is not available 61 | anywhere. Otherwise (if 'load_files' is not set) it will just return the empty strings for all absent files. The 62 | master instance reset this flag when distributes the snippets among agents. So, for agents the absence of a file is 63 | not critical error, but for the master it is so. If you want to be sure that all snippets are actually created, 64 | set both `load_files_scattered` and `load_files`. If the absence of some snippets caused by some agents is not critical 65 | for you - set just `load_files_scattered`, leaving `load_files` not set. 66 | 67 | ExcerptFlagForcepassages 68 | 69 | Whether to generate passages for snippet even if limits allow to highlight whole text. 70 | 71 | Confusion and deprecation 72 | 73 | */ 74 | type ExcerptFlags uint32 75 | 76 | const ( 77 | _ ExcerptFlags = (1 << iota) // was: ExcerptFlagRemovespaces. Actually lost in implementation may years ago 78 | ExcerptFlagExactphrase 79 | _ // was: ExcerptFlagSinglepassage. Use LimitPassages=1 instead 80 | ExcerptFlagUseboundaries 81 | ExcerptFlagWeightorder 82 | ExcerptFlagQuery 83 | ExcerptFlagForceAllWords 84 | ExcerptFlagLoadFiles 85 | ExcerptFlagAllowEmpty 86 | ExcerptFlagEmitZones 87 | ExcerptFlagFilesScattered 88 | ExcerptFlagForcepassages 89 | ) 90 | 91 | // SnippetOptions used to tune snippet's generation. All fields are exported and have meaning described below. 92 | // 93 | // BeforeMatch 94 | // 95 | // A string to insert before a keyword match. A '%PASSAGE_ID%' macro can be used in this string. 96 | // The first match of the macro is replaced with an incrementing passage number within a current snippet. 97 | // Numbering starts at 1 by default but can be overridden with start_passage_id option. In a multi-document call, 98 | // '%PASSAGE_ID%' would restart at every given document. 99 | // 100 | // AfterMatch 101 | // 102 | // A string to insert after a keyword match. %PASSAGE_ID% macro can be used in this string. 103 | // 104 | // ChunkSeparator 105 | // 106 | // A string to insert between snippet chunks (passages). 107 | // 108 | // HtmlStripMode 109 | // 110 | // HTML stripping mode setting. 111 | // Possible values are `index`, which means that index settings will be used, `none` and `strip`, 112 | // that forcibly skip or apply stripping irregardless of index settings; 113 | // and `retain`, that retains HTML markup and protects it from highlighting. 114 | // The retain mode can only be used when highlighting full documents and thus requires that 115 | // no snippet size limits are set. String, allowed values are none, strip, index, and retain. 116 | // 117 | // PassageBoundary 118 | // 119 | // Ensures that passages do not cross a sentence, paragraph, or zone boundary 120 | // (when used with an index that has the respective indexing settings enabled). 121 | // Allowed values are `sentence`, `paragraph`, and `zone`. 122 | // 123 | // Limit 124 | // 125 | // Maximum snippet size, in runes (codepoints). 126 | // 127 | // LimitPassages 128 | // 129 | // Limits the maximum number of passages that can be included into the snippet. 130 | // 131 | // LimitWords 132 | // 133 | // Limits the maximum number of words that can be included into the snippet. 134 | // Note the limit applies to any words, and not just the matched keywords to highlight. 135 | // For example, if we are highlighting Mary and a passage Mary had a little lamb is selected, 136 | // then it contributes 5 words to this limit, not just 1 137 | // 138 | // Around 139 | // 140 | // How much words to pick around each matching keywords block. 141 | // 142 | // StartPassageId 143 | // 144 | // Specifies the starting value of `%PASSAGE_ID%` macro 145 | // (that gets detected and expanded in before_match, after_match strings). 146 | // 147 | // Flags 148 | // 149 | // Bitmask. Individual bits described in `type ExcerptFlags` constants. 150 | type SnippetOptions struct { 151 | BeforeMatch, 152 | AfterMatch, 153 | ChunkSeparator, 154 | HtmlStripMode, 155 | PassageBoundary string 156 | Limit, 157 | LimitPassages, 158 | LimitWords, 159 | Around, 160 | StartPassageId int32 161 | Flags ExcerptFlags 162 | } 163 | 164 | // Create default SnippetOptions with following defaults: 165 | // 166 | // BeforeMatch: "" 167 | // AfterMatch: "" 168 | // ChunkSeparator: " ... " 169 | // HtmlStripMode: "index" 170 | // PassageBoundary: "none" 171 | // Limit: 256 172 | // Around: 5 173 | // StartPassageId: 1 174 | // // Rest of the fields: 0, or "" (depends from type) 175 | func NewSnippetOptions() *SnippetOptions { 176 | res := SnippetOptions{ 177 | "", 178 | "", 179 | " ... ", 180 | "index", 181 | "none", 182 | 256, 183 | 0, 184 | 0, 185 | 5, 186 | 1, 187 | 0, 188 | } 189 | return &res 190 | } 191 | 192 | type snippetQuery struct { 193 | opts *SnippetOptions 194 | docs []string 195 | index, words string 196 | } 197 | 198 | func buildSnippetRequest(popts *SnippetOptions, docs []string, index, words string) func(*apibuf) { 199 | return func(buf *apibuf) { 200 | buf.putDword(0) // mode = 0 201 | buf.putDword(uint32(popts.Flags)) 202 | buf.putString(index) 203 | buf.putString(words) 204 | 205 | // options 206 | buf.putString(popts.BeforeMatch) 207 | buf.putString(popts.AfterMatch) 208 | buf.putString(popts.ChunkSeparator) 209 | buf.putInt(popts.Limit) 210 | buf.putInt(popts.Around) 211 | buf.putInt(popts.LimitPassages) 212 | buf.putInt(popts.LimitWords) 213 | buf.putInt(popts.StartPassageId) 214 | buf.putString(popts.HtmlStripMode) 215 | buf.putString(popts.PassageBoundary) 216 | 217 | // documents 218 | ndocs := len(docs) 219 | buf.putLen(ndocs) 220 | for j := 0; j < ndocs; j++ { 221 | buf.putString(docs[j]) 222 | } 223 | } 224 | } 225 | 226 | func parseSnippetAnswer(nreqs int) func(*apibuf) interface{} { 227 | return func(answer *apibuf) interface{} { 228 | snippets := make([]string, nreqs) 229 | for j := 0; j < nreqs; j++ { 230 | snippets[j] = answer.getString() 231 | } 232 | return snippets 233 | } 234 | } 235 | -------------------------------------------------------------------------------- /manticore/sphinxql.go: -------------------------------------------------------------------------------- 1 | package manticore 2 | 3 | import ( 4 | "encoding/binary" 5 | "fmt" 6 | "strconv" 7 | ) 8 | 9 | func buildSphinxqlRequest(cmd string) func(*apibuf) { 10 | return func(buf *apibuf) { 11 | buf.putString(cmd) 12 | } 13 | } 14 | 15 | func parseSphinxqlAnswer() func(*apibuf) interface{} { 16 | return func(answer *apibuf) interface{} { 17 | var rss []Sqlresult 18 | for true { 19 | var rs Sqlresult 20 | if rs.parseChain(answer) { 21 | rss = append(rss, rs) 22 | } else { 23 | break 24 | } 25 | } 26 | return rss 27 | } 28 | } 29 | 30 | func (buf *apibuf) getnextSqlchunk() (bool, apibuf) { 31 | for true { 32 | if len(*buf) == 0 { 33 | break 34 | } 35 | _, lng := buf.getMysqlPacketHead() 36 | // fmt.Printf("packet %d, len %d\n", packetID, lng) 37 | res := (*buf)[:lng] 38 | *buf = (*buf)[lng:] 39 | if len(res) > 0 { 40 | return true, res 41 | } 42 | } 43 | return false, nil 44 | } 45 | 46 | func (rs *Sqlresult) parseChain(source *apibuf) bool { 47 | 48 | have, buf := source.getnextSqlchunk() 49 | if !have { 50 | return false 51 | } 52 | firstbyte := buf[0] 53 | switch firstbyte { 54 | case byte(packetOk): 55 | _ = buf.getByte() 56 | rs.parseOK(&buf) 57 | return true 58 | case byte(packetError): 59 | _ = buf.getByte() 60 | rs.parseError(&buf) 61 | return true 62 | } 63 | if buf.isEOF() { 64 | rs.Warnings, _ = buf.parseEOF() 65 | return true 66 | } 67 | ncolumns := buf.getMysqlInt() 68 | // fmt.Printf("Resultset of %d columns\n", ncolumns) 69 | rs.Schema.parseschema(source, ncolumns) 70 | have, buf = source.getnextSqlchunk() 71 | rs.Warnings, _ = buf.parseEOF() 72 | 73 | for true { 74 | if !rs.parserow(source) { 75 | break 76 | } 77 | } 78 | return true 79 | } 80 | 81 | type packetType byte 82 | 83 | const ( 84 | packetOk packetType = iota 85 | packetField 86 | packetEOF packetType = 0xFE 87 | packetError packetType = 0xFF 88 | ) 89 | 90 | func (t packetType) String() string { 91 | switch t { 92 | case packetOk: 93 | return "OK" 94 | case packetEOF: 95 | return "EOF" 96 | case packetError: 97 | return "ERROR" 98 | case packetField: 99 | return "FIELD" 100 | } 101 | return fmt.Sprintf("UNKNWN(%d)", t) 102 | } 103 | 104 | type fieldType byte 105 | 106 | const ( 107 | colDecimal fieldType = 0 108 | colLong fieldType = 3 109 | colFloat fieldType = 4 110 | colLonglong fieldType = 8 111 | colString fieldType = 254 112 | ) 113 | 114 | func (t fieldType) String() string { 115 | switch t { 116 | case colDecimal: 117 | return "decimal" 118 | case colLong: 119 | return "long" 120 | case colFloat: 121 | return "float" 122 | case colLonglong: 123 | return "longlong" 124 | case colString: 125 | return "string" 126 | } 127 | return fmt.Sprintf("unknwn(%d)", t) 128 | } 129 | 130 | type sqlfield struct { 131 | Name string 132 | Length uint32 133 | Tp fieldType 134 | Unsigned bool 135 | } 136 | 137 | //SqlSchema is the schema of resultset from mysql call. 138 | type SqlSchema []sqlfield 139 | 140 | //SqlResultset returned from Sphinxql and contains one or more mysql resultsets 141 | type SqlResultset [][]interface{} 142 | 143 | //SqlMsg represents answer from mysql proto (error code and message) 144 | type SqlMsg string 145 | 146 | //Stringer interface for SqlMsg type. Provides message like one in mysql cli 147 | func (r SqlMsg) String() string { 148 | if r[0] == '#' { 149 | code := r[1:6] 150 | return fmt.Sprintf("(%v): %v", code, r[6:]) 151 | } 152 | return string(r) 153 | } 154 | 155 | //Sqlresult is mysql resultset with table of messages. 156 | type Sqlresult struct { 157 | Msg SqlMsg 158 | Warnings uint16 159 | ErrorCode uint16 160 | RowsAffected int 161 | Schema SqlSchema 162 | Rows SqlResultset 163 | } 164 | 165 | //Stringer interface for Sqlresult type. provides data like one from mysql cli, as 166 | //header with the schema, and rows of data following. 167 | // 168 | //Number of warnings and errors also provided usual way. 169 | func (r Sqlresult) String() string { 170 | if r.ErrorCode != 0 { 171 | return fmt.Sprintf("ERROR %d %v", r.ErrorCode, r.Msg) 172 | } 173 | 174 | if r.Schema != nil { 175 | line := "" 176 | for _, col := range r.Schema { 177 | line += fmt.Sprintf("%v\t", col.Name) 178 | } 179 | line = line[:len(line)-1] + "\n" 180 | for _, row := range r.Rows { 181 | for i := 0; i < len(r.Schema); i++ { 182 | line += fmt.Sprintf("%v\t", row[i]) 183 | } 184 | line = line[:len(line)-1] + "\n" 185 | } 186 | line += fmt.Sprintf("%v rows in set", len(r.Rows)) 187 | if r.Warnings != 0 { 188 | line += fmt.Sprintf(", %d warnings", r.Warnings) 189 | } 190 | line += "\n" 191 | return line 192 | } 193 | 194 | return fmt.Sprintf("Query OK, %v rows affected", r.RowsAffected) 195 | } 196 | 197 | func (buf apibuf) parseEOF() (uint16, bool) { 198 | _ = buf.getByte() 199 | warnings := buf.getLsbWord() 200 | status := buf.getLsbWord() 201 | return warnings, status&8 != 0 202 | } 203 | 204 | func (buf apibuf) isEOF() bool { 205 | return buf[0] == byte(packetEOF) && len(buf) >= 5 206 | } 207 | 208 | func (rs *SqlSchema) parseschema(source *apibuf, ncolumns int) { 209 | 210 | *rs = make([]sqlfield, ncolumns) 211 | for i := 0; i < ncolumns; i++ { 212 | valid, buf := source.getnextSqlchunk() 213 | if !valid { 214 | return 215 | } 216 | _ = buf.getMysqlStrLen() // "def" 217 | _ = buf.getMysqlStrLen() 218 | _ = buf.getMysqlStrLen() 219 | _ = buf.getMysqlStrLen() 220 | (*rs)[i].Name = buf.getMysqlStrLen() 221 | _ = buf.getMysqlStrLen() 222 | _ = buf.getByte() // = 12, filter 223 | _ = buf.getWord() // = 0x0021, utf8 224 | (*rs)[i].Length = buf.getLsbDword() 225 | (*rs)[i].Tp = fieldType(buf.getByte()) 226 | (*rs)[i].Unsigned = buf.getWord() != 0 227 | } 228 | } 229 | 230 | func (rs *Sqlresult) parserow(source *apibuf) bool { 231 | 232 | ncolumns := len(rs.Schema) 233 | row := make([]interface{}, ncolumns) 234 | valid, buf := source.getnextSqlchunk() 235 | if buf.isEOF() || !valid { 236 | return false 237 | } 238 | for i := 0; i < ncolumns; i++ { 239 | if buf[0] == 0xFB { 240 | row[i] = nil 241 | _ = buf.getByte() 242 | } else { 243 | strValue := buf.getMysqlStrLen() 244 | switch rs.Schema[i].Tp { 245 | case colDecimal, colLong: 246 | if rs.Schema[i].Unsigned { 247 | ui, _ := strconv.ParseUint(strValue, 10, 32) 248 | row[i] = uint32(ui) 249 | } else { 250 | ii, _ := strconv.ParseInt(strValue, 10, 32) 251 | row[i] = int32(ii) 252 | } 253 | case colFloat: 254 | fl, _ := strconv.ParseFloat(strValue, 32) 255 | row[i] = float32(fl) 256 | case colLonglong: 257 | if rs.Schema[i].Unsigned { 258 | ui, _ := strconv.ParseUint(strValue, 10, 64) 259 | row[i] = uint64(ui) 260 | } else { 261 | ii, _ := strconv.ParseInt(strValue, 10, 64) 262 | row[i] = int64(ii) 263 | } 264 | default: 265 | row[i] = strValue 266 | } 267 | } 268 | } 269 | rs.Rows = append(rs.Rows, row) 270 | return true 271 | } 272 | 273 | func (rs *Sqlresult) parseOK(buf *apibuf) { 274 | rs.RowsAffected = buf.getMysqlInt() 275 | _ = buf.getMysqlInt() // last_insert_id 276 | _ = buf.getLsbWord() // status 277 | rs.Warnings = buf.getLsbWord() 278 | rs.Msg = SqlMsg(buf.getMysqlStrEof()) 279 | // fmt.Printf("OK rows %d, Warnings %d, Msg %s\n", rs.RowsAffected, rs.Warnings, rs.Msg) 280 | } 281 | 282 | func (rs *Sqlresult) parseError(buf *apibuf) { 283 | rs.ErrorCode = buf.getLsbWord() 284 | rs.Msg = SqlMsg(buf.getMysqlStrEof()) 285 | // fmt.Printf("ERROR code %d, Msg %s\n", rs.ErrorCode, rs.Msg) 286 | } 287 | 288 | func (buf *apibuf) getMysqlInt() int { 289 | 290 | res := int(buf.getByte()) 291 | if res < 251 { 292 | return res 293 | } 294 | 295 | if res == 252 { 296 | res = int((*buf)[0]) | int((*buf)[1])<<8 297 | *buf = (*buf)[2:] 298 | return res 299 | } 300 | 301 | if res == 253 { 302 | res = int((*buf)[0]) | int((*buf)[1])<<8 | int((*buf)[2])<<16 303 | *buf = (*buf)[3:] 304 | return res 305 | } 306 | 307 | if res == 254 { 308 | res = int((*buf)[0]) | int((*buf)[1])<<8 | int((*buf)[2])<<16 | int((*buf)[3])<<24 309 | *buf = (*buf)[8:] 310 | } 311 | return res 312 | } 313 | 314 | func (buf *apibuf) getMysqlStrEof() string { 315 | result := string(*buf) 316 | *buf = (*buf)[:] 317 | return result 318 | } 319 | 320 | func (buf *apibuf) getMysqlStrLen() string { 321 | lng := buf.getMysqlInt() 322 | result := string((*buf)[:lng]) 323 | *buf = (*buf)[lng:] 324 | return result 325 | } 326 | 327 | func (buf *apibuf) getMysqlPacketHead() (byte, uint32) { 328 | 329 | packlen := uint32((*buf)[0]) | uint32((*buf)[1])<<8 | uint32((*buf)[2])<<16 330 | id := (*buf)[3] 331 | *buf = (*buf)[4:] 332 | return id, packlen 333 | } 334 | 335 | func (buf *apibuf) getLsbWord() uint16 { 336 | val := binary.LittleEndian.Uint16(*buf) 337 | *buf = (*buf)[2:] 338 | return val 339 | } 340 | 341 | func (buf *apibuf) getLsbDword() uint32 { 342 | val := binary.LittleEndian.Uint32(*buf) 343 | *buf = (*buf)[4:] 344 | return val 345 | } 346 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /manticore/callpq.go: -------------------------------------------------------------------------------- 1 | package manticore 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "time" 7 | ) 8 | 9 | /* 10 | Pqflags determines boolean parameter flags for CallQP options 11 | This flags are unified into one bitfield used instead of bunch of separate flags. 12 | 13 | There are the following flags for CallPQ modes available: 14 | 15 | NeedDocs 16 | 17 | NeedDocs require to provide numbers of matched documents. It is either order numbers from the set of provided documents, 18 | or DocIDs, if documents are JSON and you pointed necessary field which contains DocID. (NOTE: json PQ calls are not yet 19 | implemented via API, it will be done later). 20 | 21 | NeedQuery 22 | 23 | NeedQuery require to return not only QueryID of the matched queries, but also another information about them. 24 | It may include query itself, tags and filters. 25 | 26 | Verbose 27 | 28 | Verbose, require to return additional meta-information about matching and queries. It causes daemon to fill fields TmSetup, TmTotal, 29 | QueriesFailed, EarlyOutQueries and QueryDT of SearchPqResponse structure. 30 | 31 | SkipBadJson 32 | 33 | SkipBadJson, require to not fail on bad (ill-formed) jsons, but warn and continue processing. This flag works only for 34 | bson queries and useless for plain text (may even cause warning if provided there). 35 | */ 36 | type Pqflags uint32 37 | 38 | const ( 39 | NeedDocs Pqflags = (1 << iota) 40 | NeedQuery 41 | jsonDocs 42 | Verbose 43 | SkipBadJson 44 | ) 45 | 46 | /* 47 | SearchPqOptions incapsulates params to be passed to CallPq function. 48 | 49 | Flags 50 | 51 | Flags is instance of Pqflags, different bites described there. 52 | 53 | IdAlias 54 | 55 | IdAlias determines name of the field in supplied json documents, which contain DocumentID. If NeedDocs flag is set, 56 | this value will be used in resultset to identify documents instead of just plain numbers of them. 57 | 58 | Shift 59 | 60 | Shift is used if daemon returns order number of the documents (i.e. when NeedDoc flag is set, but no IdAlias provided, 61 | or if documents are just plain texts and can't contain such field at all). Shift then is just added to every number of 62 | the doc, helping move the whole range. Say, if you provide 2 documents, they may be returned as numbers 1 and 2. 63 | Buf if you also give Shift=100, they will became 101 and 102. It may help if you distribute bit docset over several 64 | instances and want to keep the numbers. Daemon itself uses this value for the same purpose. 65 | */ 66 | type SearchPqOptions = struct { 67 | Flags Pqflags 68 | IdAlias string 69 | Shift int32 70 | } 71 | 72 | /* 73 | NewSearchPqOptions creates empty instance of search options. Prefer to use this function when you need options, 74 | since it may set necessary defaults 75 | */ 76 | func NewSearchPqOptions() SearchPqOptions { 77 | return SearchPqOptions{jsonDocs, "", 0} 78 | } 79 | 80 | func buildCallpqRequest(index string, values []string, opts SearchPqOptions) func(*apibuf) { 81 | return func(buf *apibuf) { 82 | buf.putDword(uint32(opts.Flags)) 83 | buf.putString(opts.IdAlias) 84 | buf.putString(index) 85 | buf.putInt(opts.Shift) 86 | buf.putLen(len(values)) 87 | for i := 0; i < len(values); i++ { 88 | buf.putString(values[i]) 89 | } 90 | } 91 | } 92 | 93 | /* 94 | PqResponseFlags determines boolean flags came in SearchPqResponse result 95 | These flags are unified into one bitfield used instead of bunch of separate flags. 96 | 97 | There are following bits available: 98 | 99 | HasDocs 100 | 101 | HasDocs indicates that each QueryDesc of Queries result array have array of documents in Docs field. Otherwise this field 102 | there is nil. 103 | 104 | DumpQueries 105 | 106 | DumpQueries indicates that each query contains additional info, like query itself, tags and filters. Otherwise it have only 107 | the number - QueryID and nothing more. 108 | 109 | HasDocids 110 | 111 | HasDocids, came in pair with HasDocs, indicates that array of documents in Queries[]Docs field is array of 112 | uint64 with document ids, provided in documents of original query. Otherwise it is array of int32 with order 113 | numbers, may be shifted by Shift param. 114 | */ 115 | type PqResponseFlags uint32 116 | 117 | const ( 118 | HasDocs PqResponseFlags = (1 << iota) 119 | DumpQueries 120 | HasDocids 121 | ) 122 | 123 | /* 124 | SearchPqResponse represents whole response to CallPQ and CallPQBson calls 125 | */ 126 | type SearchPqResponse = struct { 127 | Flags PqResponseFlags 128 | TmTotal time.Duration // total time spent for matching the document(s) 129 | TmSetup time.Duration // time spent to initial setup of matching process - parsing docs, setting options, etc. 130 | QueriesMatched int // how many stored queries match the document(s) 131 | QueriesFailed int // number of failed queries 132 | DocsMatched int // how many times the documents match the queries stored in the index 133 | TotalQueries int // how many queries are stored in the index at all 134 | OnlyTerms int // how many queries in the index have terms. The rest of the queries have extended query syntax 135 | EarlyOutQueries int // num of queries which wasn’t fall into full routine, but quickly matched and rejected with filters or other conditions 136 | QueryDT []int // detailed times per each query 137 | Warnings string 138 | Queries []QueryDesc // queries themselve. See QueryDesc structure for details 139 | } 140 | 141 | //func (r *SearchPqResponse) String() string { 142 | // return "" 143 | //} 144 | 145 | /* 146 | QueryDescFlags is bitfield describing internals of PqQuery struct 147 | This flags are unified into one bitfield used instead of bunch of separate flags. 148 | 149 | There are following bits available: 150 | 151 | QueryPresent 152 | 153 | QueryPresent indicates that field Query is valid. Otherwise it is not touched ("" by default) 154 | 155 | TagsPresent 156 | 157 | TagsPresent indicates that field Tags is valid. Otherwise it is not touched ("" by default) 158 | 159 | FiltersPresent 160 | 161 | FiltersPresent indicates that field Filters is valid. Otherwise it is not touched ("" by default) 162 | 163 | QueryIsQl 164 | 165 | QueryIsQl indicates that field Query (if present) is query in sphinxql syntax. Otherwise it is query in json syntax. 166 | PQ index can store indexes in both format, and this flag in resultset helps you to distinguish them (both are 167 | text, but syntax m.b. different) 168 | */ 169 | type QueryDescFlags uint32 170 | 171 | const ( 172 | QueryPresent QueryDescFlags = (1 << iota) 173 | TagsPresent 174 | FiltersPresent 175 | QueryIsQl 176 | ) 177 | 178 | /* 179 | QueryDesc represents an elem of Queries array from SearchPqResponse and describe one returned stored query. 180 | 181 | QueryID 182 | 183 | QueryID is namely, Query ID. In most minimal query it is the only returned field. 184 | 185 | Docs 186 | 187 | Docs is filled only if flag HasDocs is set, and contains either array of DocID (which are uint64) - if flag HasDocids is set, 188 | either array of doc ordinals (which are int32), if flag HasDocids is NOT set. 189 | 190 | Query 191 | 192 | Query is query meta, in addition to QueryID. It is filled only if in the query options they were requested via 193 | bit NeedQuery, and may contain query string, tags and filters. 194 | */ 195 | type QueryDesc = struct { 196 | QueryID uint64 197 | Docs interface{} 198 | Query PqQuery 199 | } 200 | 201 | /* 202 | PqQuery describes one separate query info from resultset of CallPQ/CallPQBson 203 | 204 | Flags determines type of the Query, and also whether other fields of the struct are filled or not. 205 | 206 | Query, Tags, Filters - attributes saved with query, all are optional 207 | */ 208 | type PqQuery = struct { 209 | Flags QueryDescFlags 210 | Query string 211 | Tags string 212 | Filters string 213 | } 214 | 215 | func parseCallpqAnswer() func(*apibuf) interface{} { 216 | return func(answer *apibuf) interface{} { 217 | var rs SearchPqResponse 218 | rs.Flags = PqResponseFlags(answer.getDword()) 219 | hasDocs := rs.Flags&HasDocs != 0 220 | hasDocids := rs.Flags&HasDocids != 0 221 | dumpQueries := rs.Flags&DumpQueries != 0 222 | nqueries := int(answer.getDword()) 223 | rs.Queries = make([]QueryDesc, nqueries) 224 | for i := 0; i < nqueries; i++ { 225 | rs.Queries[i].QueryID = answer.getUint64() 226 | if hasDocs { 227 | ndocs := int(answer.getDword()) 228 | if hasDocids { 229 | docids := make([]uint64, ndocs) 230 | for j := 0; j < ndocs; j++ { 231 | docids[j] = answer.getUint64() 232 | } 233 | rs.Queries[i].Docs = docids 234 | } else { 235 | docids := make([]int32, ndocs) 236 | for j := 0; j < ndocs; j++ { 237 | docids[j] = int32(answer.getInt()) 238 | } 239 | rs.Queries[i].Docs = docids 240 | } 241 | } 242 | if dumpQueries { 243 | flags := QueryDescFlags(answer.getDword()) 244 | rs.Queries[i].Query.Flags = flags 245 | if flags&QueryPresent != 0 { 246 | rs.Queries[i].Query.Query = answer.getString() 247 | } 248 | if flags&TagsPresent != 0 { 249 | rs.Queries[i].Query.Tags = answer.getString() 250 | } 251 | if flags&FiltersPresent != 0 { 252 | rs.Queries[i].Query.Filters = answer.getString() 253 | } 254 | } 255 | } 256 | rs.TmTotal = time.Microsecond * time.Duration(answer.getUint64()) 257 | rs.TmSetup = time.Microsecond * time.Duration(answer.getUint64()) 258 | rs.QueriesMatched = answer.getInt() 259 | rs.QueriesFailed = answer.getInt() 260 | rs.DocsMatched = answer.getInt() 261 | rs.TotalQueries = answer.getInt() 262 | rs.OnlyTerms = answer.getInt() 263 | rs.EarlyOutQueries = answer.getInt() 264 | dts := answer.getInt() 265 | if dts != 0 { 266 | rs.QueryDT = make([]int, dts) 267 | for i := 0; i < dts; i++ { 268 | rs.QueryDT[i] = answer.getInt() 269 | } 270 | } 271 | rs.Warnings = answer.getString() 272 | return &rs 273 | } 274 | } 275 | 276 | /* 277 | CallQP perform check if a document matches any of the predefined criterias (queries) 278 | It returns list of matched queries and may be additional info as matching clause, filters, and tags. 279 | 280 | `index` determines name of PQ index you want to call into. It can be either local, either distributed 281 | built from several PQ agents 282 | 283 | `values` is the list of the index. Each value regarded as separate index. Order num of matched indexes then may 284 | be returned in resultset 285 | 286 | `opts` packed options. See description of SearchPqOptions for details. 287 | In general you need to make instance of options by calling NewSearchPqOptions(), set 288 | desired flags and options, and then invoke CallPQ, providing desired index, set of documents and the options. 289 | 290 | Since this function expects plain text documents, it will remove all flags about json from the options, and also will 291 | not use IdAlias, if any provided. 292 | 293 | For example: 294 | .. 295 | po := NewSearchPqOptions() 296 | po.Flags = NeedDocs | Verbose | NeedQuery 297 | resp, err := cl.CallPQ("pq",[]string{"angry test","filter test doc2",},po) 298 | ... 299 | */ 300 | func (cl *Client) CallPQ(index string, values []string, opts SearchPqOptions) (*SearchPqResponse, error) { 301 | opts.Flags &^= jsonDocs 302 | opts.Flags &^= SkipBadJson 303 | opts.IdAlias = "" 304 | ans, err := cl.netQuery(commandCallpq, 305 | buildCallpqRequest(index, values, opts), 306 | parseCallpqAnswer()) 307 | 308 | if ans == nil { 309 | return nil, err 310 | } 311 | return ans.(*SearchPqResponse), err 312 | } 313 | 314 | /* 315 | CallPQBson perform check if a document matches any of the predefined criterias (queries) 316 | It returns list of matched queries and may be additional info as matching clause, filters, and tags. 317 | 318 | It works very like CallPQ, but expects documents in BSON form. With this function it is have sense to use 319 | flags as SkipBadJson, and param IdAlias which are not used for plain queries. 320 | 321 | This function is not yet implemented in SDK, it is stub. 322 | */ 323 | func (cl *Client) CallPQBson(index string, values []byte, opts SearchPqOptions) (*SearchPqResponse, error) { 324 | return nil, errors.New(fmt.Sprintln("Not yet implemented")) 325 | 326 | } 327 | -------------------------------------------------------------------------------- /manticore/manticore.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2001-2016, Andrew Aksyonoff 2 | // Copyright (c) 2008-2016, Sphinx Technologies Inc 3 | // Copyright (c) 2019, Manticore Software LTD (http://manticoresearch.com) 4 | // All rights reserved 5 | // 6 | // This program is free software; you can redistribute it and/or modify 7 | // it under the terms of the GNU Library General Public License. You should 8 | // have received a copy of the LGPL license along with this program; if you 9 | // did not, you can find it at http://www.gnu.org/ 10 | 11 | /* 12 | Package manticore implements Client to work with manticoresearch over it's internal binary protocol. 13 | Also in many cases it may be used to work with sphinxsearch daemon as well. 14 | It implements Client connector which may be used as 15 | cl := NewClient() 16 | res, err := cl.Query("hello") 17 | ... 18 | Set of functions is mostly imitates API description of Manticoresearch for PHP, but with few 19 | changes which are specific to Go language as more effective and mainstream for that language (as, for example, 20 | error handling). 21 | 22 | This SDK help you to send different manticore API packets and parse results. 23 | These are: 24 | 25 | * Search (full-text and full-scan) 26 | 27 | * Build snippets 28 | 29 | * Build keywords 30 | 31 | * Flush attributes 32 | 33 | * Perform JSON queries (as via HTTP proto) 34 | 35 | * Perform sphinxql queries (as via mysql proto) 36 | 37 | * Set user variables 38 | 39 | * Ping the server 40 | 41 | * Look server status 42 | 43 | * Perform pecrolate queries 44 | 45 | The percolate query is used to match documents against queries stored in an index. 46 | It is also called “search in reverse” as it works opposite to a regular search where documents are stored 47 | in an index and queries are issued against the index. 48 | 49 | These queries are stored in special kind index and they can be added, deleted and listed using 50 | INSERT/DELETE/SELECT statements similar way as it’s done for a regular index. 51 | 52 | Checking if a document matches any of the predefined criterias (queries) performed via 53 | CallPQ function, or via http /json/pq//_search endpoint. 54 | They returns list of matched queries and may be additional info as matching clause, 55 | filters, and tags. 56 | */ 57 | package manticore 58 | 59 | import ( 60 | "errors" 61 | "time" 62 | ) 63 | 64 | // BuildExcerpts generates excerpts (snippets) 65 | // of given documents for given query. returns nil on failure, 66 | // an array of snippets on success. If necessary it will connect to the server before processing. 67 | // 68 | // `docs` is a plain slice of strings that carry the documents’ contents. 69 | // 70 | // `index` is an index name string. Different settings (such as charset, morphology, wordforms) 71 | // from given index will be used. 72 | // 73 | // `words` is a string that contains the keywords to highlight. 74 | // They will be processed with respect to index settings. For instance, if English stemming is 75 | // enabled in the index, shoes will be highlighted even if keyword is shoe. Keywords can contain wildcards, 76 | // that work similarly to star-syntax available in queries. 77 | // 78 | // `opts` is an optional struct SnippetOptions which may contain 79 | // additional optional highlighting parameters, it may be created by calling of ``NewSnippetOptions()'' and then tuned 80 | // for your needs. If `opts` is omitted, default will be used. 81 | // 82 | // Snippets extraction algorithm currently favors better passages (with closer phrase matches), 83 | // and then passages with keywords not yet in snippet. Generally, it will try to highlight the best match 84 | // with the query, and it will also to highlight all the query keywords, as made possible by the limits. 85 | // In case the document does not match the query, beginning of the document trimmed down according to the limits 86 | // will be return by default. You can also return an empty snippet instead case by setting allow_empty option to true. 87 | // 88 | // Returns false on failure. Returns a plain array of strings with excerpts (snippets) on success. 89 | func (cl *Client) BuildExcerpts(docs []string, index, 90 | words string, opts ...SnippetOptions) ([]string, error) { 91 | 92 | var popts *SnippetOptions 93 | if len(opts) > 0 { 94 | popts = &opts[0] 95 | } else { 96 | popts = NewSnippetOptions() 97 | } 98 | 99 | if len(docs) == 0 { 100 | return nil, errors.New("invalid arguments (docs must not be empty)") 101 | } 102 | 103 | if index == "" { 104 | return nil, errors.New("invalid arguments (index must not be empty)") 105 | } 106 | 107 | if words == "" { 108 | return nil, errors.New("invalid arguments (words must not be empty)") 109 | } 110 | 111 | ndocs := len(docs) 112 | snippets, err := cl.netQuery(commandExcerpt, 113 | buildSnippetRequest(popts, docs, index, words), 114 | parseSnippetAnswer(ndocs)) 115 | if snippets == nil { 116 | return nil, err 117 | } 118 | return snippets.([]string), err 119 | } 120 | 121 | // BuildKeywords extracts keywords from query using tokenizer settings 122 | // for given index, optionally with per-keyword occurrence statistics. 123 | // Returns an array of hashes with per-keyword information. If necessary it will connect to the server before processing. 124 | // 125 | // `query` is a query to extract keywords from. 126 | // 127 | // `index` is a name of the index to get tokenizing settings and keyword 128 | // occurrence statistics from. 129 | // 130 | // `hits` is a boolean flag that indicates whether keyword occurrence statistics are required. 131 | func (cl *Client) BuildKeywords(query, index string, hits bool) ([]Keyword, error) { 132 | 133 | if query == "" { 134 | return nil, errors.New("invalid arguments (query must not be empty)") 135 | } 136 | 137 | if index == "" { 138 | return nil, errors.New("invalid arguments (index must not be empty)") 139 | } 140 | 141 | keywords, err := cl.netQuery(commandKeywords, 142 | buildKeywordsRequest(query, index, hits), 143 | parseKeywordsAnswer(hits)) 144 | if keywords == nil { 145 | return nil, err 146 | } 147 | return keywords.([]Keyword), err 148 | } 149 | 150 | // Close closes previously opened persistent connection. If no connection active, it fire error 'not connected' which 151 | // is just informational and safe to ignore. 152 | func (cl *Client) Close() (bool, error) { 153 | if !cl.connected { 154 | return false, errors.New("not connected") 155 | } 156 | err := cl.conn.Close() 157 | cl.conn = nil 158 | cl.connected = false 159 | return err == nil, err 160 | } 161 | 162 | // FlushAttributes forces searchd to flush pending attribute updates to disk, and blocks until completion. 163 | // Returns a non-negative internal flush tag on success, or -1 and error. 164 | // 165 | // Attribute values updated using UpdateAttributes() API call are kept in a memory mapped file. 166 | // Which means the OS decides when the updates are actually written to disk. FlushAttributes() call lets you enforce 167 | // a flush, which writes all the changes to disk. The call will block until searchd finishes writing the data to disk, 168 | // which might take seconds or even minutes depending on the total data size (.spa file size). 169 | // All the currently updated indexes will be flushed. 170 | // 171 | // Flush tag should be treated as an ever growing magic number that does not mean anything. 172 | // It’s guaranteed to be non-negative. It is guaranteed to grow over time, though not necessarily in a sequential 173 | // fashion; for instance, two calls that return 10 and then 1000 respectively are a valid situation. 174 | // If two calls to FlushAttrs() return the same tag, it means that there were no actual attribute updates in between 175 | // them, and therefore current flushed state remained the same (for all indexes). 176 | // 177 | // Usage example: 178 | // 179 | // status, err := cl.FlushAttributes () 180 | // if err!=nil { 181 | // fmt.Println(err.Error()) 182 | // } 183 | func (cl *Client) FlushAttributes() (int, error) { 184 | tag, err := cl.netQuery(commandFlushattrs, nil, parseDwordAnswer()) 185 | if tag == nil { 186 | return -1, err 187 | } 188 | return tag.(int), err 189 | } 190 | 191 | // GetLastWarning returns last warning message, as a string, in human readable format. 192 | // If there were no warnings during the previous API call, empty string is returned. 193 | // 194 | // You should call it to verify whether your request (such as Query()) was completed but with warnings. 195 | // For instance, search query against a distributed index might complete successfully even if several remote agents 196 | // timed out. In that case, a warning message would be produced. 197 | // 198 | // The warning message is not reset by this call; so you can safely call it several times if needed. 199 | // If you issued multi-query by running RunQueries(), individual warnings will not be written in client; instead 200 | // check the Warning field in each returned result of the slice. 201 | func (cl *Client) GetLastWarning() string { 202 | return cl.lastWarning 203 | } 204 | 205 | // IsConnectError checks whether the last error was a network error on API side, or a remote error reported by searchd. 206 | // Returns true if the last connection attempt to searchd failed on API side, false otherwise 207 | // (if the error was remote, or there were no connection attempts at all). 208 | func (cl *Client) IsConnectError() bool { 209 | return cl.connError 210 | } 211 | 212 | /* 213 | Json pefrorms remote call of JSON query, as if it were fired via HTTP connection. 214 | It is intented to run updates and deletes, however works sometimes in other cases. 215 | General rule: if the endpoint accepts data via POST, it will work via Json call. 216 | 217 | `endpoint` - is the endpoint, like "json/search". 218 | 219 | `request` - the query. As in REST, expected to be in JSON, like `{"index":"lj","query":{"match":{"title":"luther"}}}` 220 | */ 221 | func (cl *Client) Json(endpoint, request string) (JsonAnswer, error) { 222 | blob, err := cl.netQuery(commandJson, 223 | buildJsonRequest(endpoint, request), 224 | parseJsonAnswer()) 225 | if blob == nil { 226 | return JsonAnswer{}, err 227 | } 228 | return blob.(JsonAnswer), err 229 | } 230 | 231 | // Open opens persistent connection to the server. 232 | func (cl *Client) Open() (bool, error) { 233 | 234 | if cl.connected { 235 | return false, errors.New("already connected") 236 | } 237 | _, err := cl.netQuery(commandPersist, buildBoolRequest(true), nil) 238 | return err == nil, err 239 | } 240 | 241 | // Query connects to searchd server, run given simple search query string through given indexes, 242 | // and return the search result. 243 | // 244 | // This is simplified function which accepts only 1 query string parameter and no options 245 | // Internally it will run with ranker 'RankProximityBm25', mode 'MatchAll' with 'max_matches=1000' and 'limit=20' 246 | // It is good to be used in kind of a demo run. If you want more fine-tuned options, consider to use `RunQuery()` 247 | // and `RunQueries()` functions which provide you full spectre of possible tuning options. 248 | // 249 | // `query` is a query string. 250 | // 251 | // `indexes` is an index name (or names) string. Default value for `indexes` is "*" that means to query all local indexes. 252 | // Characters allowed in index names include Latin letters (a-z), numbers (0-9) and underscore (_); 253 | // everything else is considered a separator. Note that index name should not start with underscore character. 254 | // Internally 'Query' is just invokes 'RunQuery' with default Search, where only `query` and `index` fields are customized. 255 | // 256 | // Therefore, all of the following samples calls are valid and will search the same two indexes: 257 | // cl.Query ( "test query", "main delta" ) 258 | // cl.Query ( "test query", "main;delta" ) 259 | // cl.Query ( "test query", "main, delta" ) 260 | func (cl *Client) Query(query string, indexes ...string) (*QueryResult, error) { 261 | index := "*" 262 | 263 | if len(indexes) > 0 { 264 | index = indexes[0] 265 | } 266 | 267 | res, err := cl.RunQuery(NewSearch(query, index, "")) 268 | 269 | if res == nil { 270 | return nil, err 271 | } 272 | 273 | if err == nil && res.Status != StatusError { 274 | return res, nil 275 | } 276 | return nil, err 277 | } 278 | 279 | // RunQueries connects to searchd, runs a batch of queries, obtains and returns the result sets. 280 | // Returns nil and error message on general error (such as network I/O failure). 281 | // Returns a slice of result sets on success. 282 | // 283 | // `queries` is slice of Search structures, each represent one query. You need to prepare this slice yourself before call. 284 | // 285 | // Each result set in the returned array is exactly the same as the result set returned from RunQuery. 286 | // 287 | // Note that the batch query request itself almost always succeeds - unless there’s a network error, 288 | // blocking index rotation in progress, or another general failure which prevents the whole request 289 | // from being processed. 290 | // 291 | // However individual queries within the batch might very well fail. In this case their respective 292 | // result sets will contain non-empty `error` message, but no matches or query statistics. 293 | // In the extreme case all queries within the batch could fail. There still will be no general error reported, 294 | // because API was able to successfully connect to searchd, submit the batch, and receive the results - 295 | // but every result set will have a specific error message. 296 | func (cl *Client) RunQueries(queries []Search) ([]QueryResult, error) { 297 | nreqs := len(queries) 298 | if nreqs == 0 { 299 | return nil, errors.New("no queries defined, issue AddQuery() first") 300 | } 301 | 302 | res, err := cl.netQuery(commandSearch, 303 | buildSearchRequest(queries), 304 | parseSearchAnswer(nreqs)) 305 | if res == nil { 306 | return nil, err 307 | } 308 | return res.([]QueryResult), err 309 | } 310 | 311 | // RunQuery connects to searchd, runs a query, obtains and returns the result set. 312 | // Returns nil and error message on general error (such as network I/O failure). 313 | // Returns a result set on success. 314 | // 315 | // `query` is a single Search structure, representing the query. You need to prepare it yourself before call. 316 | // 317 | // Each result set in the returned array is exactly the same as the result set returned from RunQuery. 318 | // 319 | func (cl *Client) RunQuery(query Search) (*QueryResult, error) { 320 | res, err := cl.netQuery(commandSearch, 321 | buildSearchRequest([]Search{query}), 322 | parseSearchAnswer(1)) 323 | if res == nil { 324 | return nil, err 325 | } 326 | result := res.([]QueryResult)[0] 327 | cl.lastWarning = result.Warning 328 | return &result, err 329 | } 330 | 331 | // SetConnectTimeout sets the time allowed to spend connecting to the server before giving up. 332 | // 333 | // Under some circumstances, the server can be delayed in responding, either due to network delays, or a query backlog. 334 | // In either instance, this allows the client application programmer some degree of control over how their program 335 | // interacts with searchd when not available, and can ensure that the client application does not fail due to exceeding 336 | // the execution limits. 337 | // 338 | // In the event of a failure to connect, an appropriate error code should be returned back to the application 339 | // in order for application-level error handling to advise the user. 340 | func (cl *Client) SetConnectTimeout(timeout time.Duration) { 341 | cl.timeout = timeout 342 | } 343 | 344 | // SetMaxAlloc limits size of client's network buffer. For sending queries and receiving results client reuses byte array, 345 | // which can grow up to required size. If the limit reached, array will be released and new one will be created. Usually 346 | // API needs just few kilobytes of the memory, but sometimes the value may grow significantly high. For example, if you fetch a 347 | // big resultset with many attributes. Such resultset will be properly received and processed, however at the next query 348 | // backend array which used for it will be released, and occupied memory will be returned to runtime. 349 | // 350 | // `alloc` is size, in bytes. Reasonable default value is 8M. 351 | func (cl *Client) SetMaxAlloc(alloc int) { 352 | cl.maxAlloc = alloc 353 | } 354 | 355 | // SetServer sets searchd host name and TCP port. All subsequent requests will use the new host and port settings. 356 | // Default host and port are ‘localhost’ and 9312, respectively. 357 | // 358 | // `host` is either url (hostname or ip address), either unix socket path (starting with '/') 359 | // 360 | // `port` is optional, it has sense only for tcp connections and not used for unix socket. Default is 9312 361 | func (cl *Client) SetServer(host string, port ...uint16) { 362 | 363 | if host == "" { 364 | host = "localhost" 365 | } 366 | if host[0] == '/' { 367 | cl.dialmethod = "unix" 368 | cl.host = host 369 | cl.port = 0 370 | return 371 | } 372 | 373 | if len(host) >= 7 && host[:7] == "unix://" { 374 | cl.dialmethod = "unix" 375 | cl.host = host[7:] 376 | cl.port = 0 377 | return 378 | } 379 | 380 | cl.host = host 381 | cl.dialmethod = "tcp" 382 | if len(port) > 0 { 383 | cl.port = port[0] 384 | } 385 | } 386 | 387 | /* 388 | Sphinxql send sphinxql request encapsulated into API. 389 | Return over network came in mysql native proto format, which is parsed by SDK and represented 390 | as usable structure (see Sqlresult definition). 391 | Also result provides Stringer interface, so it may be printed nice without any postprocessing. 392 | Limitation of the command is that it is done in one session, as if you open connection via mysql, 393 | execute the command and disconnected. So, some information, like 'show meta' after 'call pq' will be lost 394 | in such case (however, you can invoke CallPQ directly from API), but another things like 'select...; show meta' 395 | in one line is still supported and work well 396 | */ 397 | func (cl *Client) Sphinxql(cmd string) ([]Sqlresult, error) { 398 | blob, err := cl.netQuery(commandSphinxql, 399 | buildSphinxqlRequest(cmd), 400 | parseSphinxqlAnswer()) 401 | if blob == nil { 402 | return nil, err 403 | } 404 | return blob.([]Sqlresult), err 405 | } 406 | 407 | /* 408 | Ping just send a uint32 cookie to the daemon and immediately receive it back. 409 | It may be used to average network responsibility time, or to ping if daemon is alive or not. 410 | */ 411 | func (cl *Client) Ping(cookie uint32) (uint32, error) { 412 | answer, err := cl.netQuery(commandPing, 413 | buildDwordRequest(cookie), 414 | parseDwordAnswer()) 415 | if answer == nil { 416 | return 0, err 417 | } 418 | return answer.(uint32), err 419 | } 420 | 421 | // Status queries searchd status, and returns an array of status variable name and value pairs. 422 | // 423 | // `global` determines whether you take global status, or meta of the last query. 424 | // true: receive global daemon status 425 | // false: receive meta of the last executed query 426 | // 427 | // Usage example: 428 | // status, err := cl.Status(false) 429 | // if err != nil { 430 | // fmt.Println(err.Error()) 431 | // } else { 432 | // for key, line := range (status) { 433 | // fmt.Printf("%v:\t%v\n", key, line) 434 | // } 435 | // } 436 | // example output: 437 | // time: 0.000 438 | // keyword[0]: query 439 | // docs[0]: 1235 440 | // hits[0]: 1474 441 | // total: 3 442 | // total_found: 3 443 | func (cl *Client) Status(global bool) (map[string]string, error) { 444 | status, err := cl.netQuery(commandStatus, 445 | buildBoolRequest(global), 446 | parseStatusAnswer()) 447 | if status == nil { 448 | return nil, err 449 | } 450 | return status.(map[string]string), err 451 | } 452 | 453 | // UpdateAttributes instantly updates given attribute values in given documents. Returns number of actually updated 454 | // documents (0 or more) on success, or -1 on failure with error. 455 | // 456 | // `index` is a name of the index (or indexes) to be updated. It can be either a single index name or a list, 457 | // like in Query(). Unlike Query(), wildcard is not allowed and all the indexes to update must be specified explicitly. 458 | // The list of indexes can include distributed index names. Updates on distributed indexes will be pushed to all agents. 459 | // 460 | // `attrs` is a slice with string attribute names, listing attributes that are updated. 461 | // 462 | // `values` is a map with documents IDs as keys and new attribute values, see below. 463 | // 464 | // `vtype` type parameter, see EUpdateType description for values. 465 | // 466 | // `ignorenonexistent` points that the update will silently ignore any warnings about trying to update a column which 467 | // is not exists in current index schema. 468 | // 469 | // Usage example: 470 | // 471 | // upd, err := cl.UpdateAttributes("test1", []string{"group_id"}, map[DocID][]interface{}{1:{456}}, UpdateInt, false) 472 | // 473 | // Here we update document 1 in index test1, setting group_id to 456. 474 | // 475 | // upd, err := cl.UpdateAttributes("products", []string{"price", "amount_in_stock"}, map[DocID][]interface{}{1001:{123,5}, 1002:{37,11}, 1003:{25,129}}, UpdateInt, false) 476 | // 477 | // Here we update documents 1001, 1002 and 1003 in index products. 478 | // For document 1001, the new price will be set to 123 and the new amount in stock to 5; 479 | // for document 1002, the new price will be 37 and the new amount will be 11; etc. 480 | func (cl *Client) UpdateAttributes(index string, attrs []string, values map[DocID][]interface{}, 481 | vtype EUpdateType, ignorenonexistent bool) (int, error) { 482 | 483 | if attrs == nil || len(attrs) == 0 { 484 | return -1, errors.New("invalid arguments (attrs must not empty)") 485 | } 486 | 487 | if index == "" { 488 | return -1, errors.New("invalid arguments (index must not be empty)") 489 | } 490 | 491 | if values == nil || len(values) == 0 { 492 | return -1, errors.New("invalid arguments (values must not be empty)") 493 | } 494 | 495 | updated, err := cl.netQuery(commandUpdate, 496 | buildUpdateRequest(index, attrs, values, vtype, ignorenonexistent), 497 | parseDwordAnswer()) 498 | if updated == nil { 499 | return -1, err 500 | } 501 | return int(updated.(uint32)), err 502 | } 503 | 504 | /* 505 | Uvar defines remote user variable which later may be used for filtering. 506 | You can really push megabytes of values and later just refer to the whole set by name. 507 | 508 | `name` is the name of the variable, must start with @, like "@foo" 509 | 510 | `values` is array of the numbers you want to store in the variable. It is considered as 'set', 511 | so dupes will be removed, order will not be kept. Like: []uint64{7811237,7811235,7811235,7811233,7811236} 512 | */ 513 | func (cl *Client) Uvar(name string, values []uint64) error { 514 | _, err := cl.netQuery(commandUvar, 515 | buildUvarRequest(name, values), 516 | parseDwordAnswer()) 517 | 518 | return err 519 | } 520 | 521 | // EscapeString escapes characters that are treated as special operators by the query language parser. 522 | // 523 | // `from` is a string to escape. 524 | // This function might seem redundant because it’s trivial to implement in any calling application. 525 | // However, as the set of special characters might change over time, it makes sense to have an API call that is 526 | // guaranteed to escape all such characters at all times. 527 | // Returns escaped string. 528 | func EscapeString(from string) string { 529 | dest := make([]byte, 0, 2*len(from)) 530 | for i := 0; i < len(from); i++ { 531 | c := from[i] 532 | switch c { 533 | case '\\', '(', ')', '|', '-', '!', '@', '~', '"', '&', '/', '^', '$', '=', '<': 534 | dest = append(dest, '\\') 535 | } 536 | dest = append(dest, c) 537 | } 538 | return string(dest) 539 | } 540 | -------------------------------------------------------------------------------- /manticore/definitions.go: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright (c) 2019, Manticore Software LTD (http://manticoresearch.com) 3 | // All rights reserved 4 | // 5 | // This program is free software; you can redistribute it and/or modify 6 | // it under the terms of the GNU General Public License. You should have 7 | // received a copy of the GPL license along with this program; if you 8 | // did not, you can find it at http://www.gnu.org/ 9 | // 10 | 11 | package manticore 12 | 13 | import "fmt" 14 | 15 | // Known commands 16 | type eSearchdcommand uint16 17 | 18 | const ( 19 | commandSearch eSearchdcommand = iota 20 | commandExcerpt 21 | commandUpdate 22 | commandKeywords 23 | commandPersist 24 | commandStatus 25 | _ 26 | commandFlushattrs 27 | commandSphinxql 28 | commandPing 29 | _ // commandDelete not exposed 30 | commandUvar 31 | _ // commandInsert not exposed 32 | _ // commandReplace not exposed 33 | _ // commandCommit not exposed 34 | _ // commandSuggest not exposed 35 | commandJson 36 | commandCallpq 37 | commandClusterpq 38 | 39 | commandTotal 40 | commandWrong = commandTotal 41 | ) 42 | 43 | func (vl eSearchdcommand) String() string { 44 | switch vl { 45 | case commandSearch: 46 | return "search" 47 | case commandExcerpt: 48 | return "excerpt" 49 | case commandUpdate: 50 | return "update" 51 | case commandKeywords: 52 | return "keywords" 53 | case commandPersist: 54 | return "persist" 55 | case commandStatus: 56 | return "status" 57 | case commandFlushattrs: 58 | return "flushattrs" 59 | case commandSphinxql: 60 | return "sphinxql" 61 | case commandPing: 62 | return "ping" 63 | case commandUvar: 64 | return "uvar" 65 | case commandJson: 66 | return "json" 67 | case commandCallpq: 68 | return "callpq" 69 | case commandClusterpq: 70 | return "clusterpq" 71 | default: 72 | return fmt.Sprintf("wrong(%d)", uint16(vl)) 73 | } 74 | } 75 | 76 | // known command versions 77 | type uCommandVersion uint16 78 | 79 | const verCommandWrong uCommandVersion = 0 80 | 81 | var searchdcommandv = [commandTotal]uCommandVersion{ 82 | 83 | 0x121, // search 84 | 0x104, // excerpt 85 | 0x103, // update 86 | 0x101, // keywords 87 | verCommandWrong, // persist 88 | 0x101, // status 89 | verCommandWrong, // _ 90 | 0x100, // flushattrs 91 | 0x100, // sphinxql 92 | 0x100, // ping 93 | verCommandWrong, // delete 94 | 0x100, // uvar 95 | verCommandWrong, // insert 96 | verCommandWrong, // replace 97 | verCommandWrong, // commit 98 | verCommandWrong, // suggest 99 | 0x100, // json 100 | 0x100, // callpq 101 | 0x100, // clusterpq 102 | } 103 | 104 | func (vl uCommandVersion) String() string { 105 | return fmt.Sprintf("%d.%02d", byte(vl>>8), byte(vl&0xFF)) 106 | } 107 | 108 | const ( 109 | cphinxClientVersion uint32 = 1 110 | cphinxSearchdProto uint32 = 1 111 | SphinxPort uint16 = 9312 // Default IANA port for Sphinx API 112 | ) 113 | 114 | // Document ID type 115 | type DocID uint64 116 | 117 | const DocidMax DocID = 0xffffffffffffffff 118 | 119 | // eAggrFunc describes aggregate function in search query. 120 | // Used in master-agent extensions for commandSearchMaster>=15 121 | type eAggrFunc uint32 122 | 123 | const ( 124 | AggrNone eAggrFunc = iota // None 125 | AggrAvg // Avg() 126 | AggrMin // Min() 127 | AggrMax // Max() 128 | AggrSum // Sum() 129 | AggrCat // Cat() 130 | ) 131 | 132 | // EAttrType represents known attribute types. 133 | // See comments in constants for concrete meaning. Values of this type will be returned with resultset schema, 134 | // you don't need to use them yourself. 135 | type EAttrType uint32 136 | 137 | const ( 138 | AttrNone EAttrType = iota // not an attribute at all 139 | AttrInteger // unsigned 32-bit integer 140 | AttrTimestamp // this attr is a timestamp 141 | _ // there was SPH_ATTR_ORDINAL=3 once 142 | AttrBool // this attr is a boolean bit field 143 | AttrFloat // floating point number (IEEE 32-bit) 144 | AttrBigint // signed 64-bit integer 145 | AttrString // string (binary; in-memory) 146 | _ // there was SPH_ATTR_WORDCOUNT=8 once 147 | AttrPoly2d // vector of floats, 2D polygon (see POLY2D) 148 | AttrStringptr // string (binary, in-memory, stored as pointer to the zero-terminated string) 149 | AttrTokencount // field token count, 32-bit integer 150 | AttrJson // JSON subset; converted, packed, and stored as string 151 | AttrUint32set EAttrType = 0x40000001 // MVA, set of unsigned 32-bit integers 152 | AttrInt64set EAttrType = 0x40000002 // MVA, set of signed 64-bit integers 153 | ) 154 | 155 | // these types are runtime only 156 | // used as intermediate types in the expression engine 157 | const ( 158 | AttrMaparg EAttrType = 1000 + iota 159 | AttrFactors // packed search factors (binary, in-memory, pooled) 160 | AttrJsonField // points to particular field in JSON column subset 161 | AttrFactorsJson // packed search factors (binary, in-memory, pooled, provided to Client json encoded) 162 | ) 163 | 164 | // eCollation is collation of search query. Used in master-agent extensions for commandSearchMaster>=1 165 | type eCollation uint32 166 | 167 | const ( 168 | CollationLibcCi eCollation = iota // Libc CI 169 | CollationLibcCs // Libc Cs 170 | CollationUtf8GeneralCi // Utf8 general CI 171 | CollationBinary // Binary 172 | 173 | CollationDefault = CollationLibcCi 174 | ) 175 | 176 | // eFilterType describes different filters types. Internal. 177 | type eFilterType uint32 178 | 179 | const ( 180 | FilterValues eFilterType = iota // filter by integer values set 181 | FilterRange // filter by integer range 182 | FilterFloatrange // filter by float range 183 | FilterString // filter by string value 184 | FilterNull // filter by NULL 185 | FilterUservar // filter by @uservar 186 | FilterStringList // filter by string list 187 | FilterExpression // filter by expression 188 | ) 189 | 190 | /* 191 | EGroupBy selects search query grouping mode. It is used as a param when calling `SetGroupBy()` function. 192 | 193 | GroupbyDay 194 | 195 | GroupbyDay extracts year, month and day in YYYYMMDD format from timestamp. 196 | 197 | GroupbyWeek 198 | 199 | GroupbyWeek extracts year and first day of the week number (counting from year start) in YYYYNNN format from timestamp. 200 | 201 | GroupbyMonth 202 | 203 | GroupbyMonth extracts month in YYYYMM format from timestamp. 204 | 205 | GroupbyYear 206 | 207 | GroupbyYear extracts year in YYYY format from timestamp. 208 | 209 | GroupbyAttr 210 | 211 | GroupbyAttr uses attribute value itself for grouping. 212 | 213 | GroupbyMultiple 214 | 215 | GroupbyMultiple group by on multiple attribute values. Allowed plain attributes and json fields; MVA and full JSONs are not allowed. 216 | */ 217 | type EGroupBy uint32 218 | 219 | const ( 220 | GroupbyDay EGroupBy = iota // group by day 221 | GroupbyWeek // group by week 222 | GroupbyMonth // group by month 223 | GroupbyYear // group by year 224 | GroupbyAttr // group by attribute value 225 | _ // GroupbyAttrpair, group by sequential attrs pair (rendered redundant by 64bit attrs support; removed) 226 | GroupbyMultiple // group by on multiple attribute values 227 | ) 228 | 229 | // eQueryoption describes keyword expansion mode. Used only in master-agent mode of search query for commandSearchMaster>=16 230 | type eQueryoption uint32 231 | 232 | const ( 233 | QueryOptDefault eQueryoption = iota // Default 234 | QueryOptDisabled // Disabled 235 | QueryOptEnabled // Enabled 236 | QueryOptMorphNone // None morphology expansion 237 | ) 238 | 239 | /* 240 | ERankMode selects query relevance ranking mode. It is set via `SetRankingMode()` and `SetRankingExpression()` functions. 241 | 242 | Manticore ships with a number of built-in rankers suited for different purposes. A number of them uses two factors, 243 | phrase proximity (aka LCS) and BM25. Phrase proximity works on the keyword positions, while BM25 works on the keyword 244 | frequencies. Basically, the better the degree of the phrase match between the document body and the query, the higher 245 | is the phrase proximity (it maxes out when the document contains the entire query as a verbatim quote). And BM25 is 246 | higher when the document contains more rare words. We’ll save the detailed discussion for later. 247 | 248 | Currently implemented rankers are: 249 | 250 | RankProximityBm25 251 | 252 | RankProximityBm25, the default ranking mode that uses and combines both phrase proximity and BM25 ranking. 253 | 254 | RankBm25 255 | 256 | RankBm25, statistical ranking mode which uses BM25 ranking only (similar to most other full-text engines). 257 | This mode is faster but may result in worse quality on queries which contain more than 1 keyword. 258 | 259 | RankNone 260 | 261 | RankNone, no ranking mode. This mode is obviously the fastest. A weight of 1 is assigned to all matches. 262 | This is sometimes called boolean searching that just matches the documents but does not rank them. 263 | 264 | RankWordcount 265 | 266 | RankWordcount, ranking by the keyword occurrences count. This ranker computes the per-field keyword occurrence counts, 267 | then multiplies them by field weights, and sums the resulting values. 268 | 269 | RankProximity 270 | 271 | RankProximity, returns raw phrase proximity value as a result. This mode is internally used to emulate MatchAll queries. 272 | 273 | RankMatchany 274 | 275 | RankMatchany, returns rank as it was computed in SPH_MATCH_ANY mode earlier, and is internally used to emulate MatchAny queries. 276 | 277 | RankFieldmask 278 | 279 | RankFieldmask, returns a 32-bit mask with N-th bit corresponding to N-th fulltext field, numbering from 0. 280 | The bit will only be set when the respective field has any keyword occurrences satisfying the query. 281 | 282 | RankSph04 283 | 284 | RankSph04, is generally based on the default SPH_RANK_PROXIMITY_BM25 ranker, but additionally boosts the matches when 285 | they occur in the very beginning or the very end of a text field. Thus, if a field equals the exact query, 286 | SPH04 should rank it higher than a field that contains the exact query but is not equal to it. (For instance, when 287 | the query is “Hyde Park”, a document entitled “Hyde Park” should be ranked higher than a one entitled “Hyde Park, 288 | London” or “The Hyde Park Cafe”.) 289 | 290 | RankExpr 291 | 292 | RankExpr, lets you specify the ranking formula in run time. It exposes a number of internal text factors and lets you 293 | define how the final weight should be computed from those factors. 294 | 295 | RankExport 296 | 297 | RankExport, rank by BM25, but compute and export all user expression factors 298 | 299 | RankPlugin 300 | 301 | RankPlugin, rank by user-defined ranker provided as UDF function. 302 | */ 303 | type ERankMode uint32 304 | 305 | const ( 306 | RankProximityBm25 ERankMode = iota // default mode, phrase proximity major factor and BM25 minor one (aka SPH03) 307 | RankBm25 // statistical mode, BM25 ranking only (faster but worse quality) 308 | RankNone // no ranking, all matches get a weight of 1 309 | RankWordcount // simple word-count weighting, rank is a weighted sum of per-field keyword occurence counts 310 | RankProximity // phrase proximity (aka SPH01) 311 | RankMatchany // emulate old match-any weighting (aka SPH02) 312 | RankFieldmask // sets bits where there were matches 313 | RankSph04 // codename SPH04, phrase proximity + bm25 + head/exact boost 314 | RankExpr // rank by user expression (eg. "sum(lcs*user_weight)*1000+bm25") 315 | RankExport // rank by BM25, but compute and export all user expression factors 316 | RankPlugin // user-defined ranker 317 | RankTotal 318 | RankDefault = RankProximityBm25 319 | ) 320 | 321 | // ESearchdstatus describes known return codes. Also status codes for search command (but there 32bit) 322 | type ESearchdstatus uint16 323 | 324 | const ( 325 | StatusOk ESearchdstatus = iota // general success, command-specific reply follows 326 | StatusError // general failure, error message follows 327 | StatusRetry // temporary failure, error message follows, Client should retry late 328 | StatusWarning // general success, warning message and command-specific reply follow 329 | ) 330 | 331 | // Stringer interface for ESearchdstatus type 332 | func (vl ESearchdstatus) String() string { 333 | switch vl { 334 | case StatusOk: 335 | return "ok" 336 | case StatusError: 337 | return "error" 338 | case StatusRetry: 339 | return "retry" 340 | case StatusWarning: 341 | return "warning" 342 | default: 343 | return fmt.Sprintf("unknown(%d)", uint16(vl)) 344 | } 345 | } 346 | 347 | /* 348 | ESortOrder selects search query sorting orders 349 | 350 | There are the following result sorting modes available: 351 | 352 | SortRelevance 353 | 354 | SortRelevance sorts by relevance in descending order (best matches first). 355 | 356 | SortAttrDesc 357 | 358 | SortAttrDescmode sorts by an attribute in descending order (bigger attribute values first). 359 | 360 | SortAttrAsc 361 | 362 | SortAttrAsc mode sorts by an attribute in ascending order (smaller attribute values first). 363 | 364 | SortTimeSegments 365 | 366 | SortTimeSegments sorts by time segments (last hour/day/week/month) in descending order, and then by relevance in descending order. 367 | Attribute values are split into so-called time segments, and then sorted by time segment first, and by relevance second. 368 | 369 | The segments are calculated according to the current timestamp at the time when the search is performed, 370 | so the results would change over time. The segments are as follows: 371 | 372 | last hour, 373 | 374 | last day, 375 | 376 | last week, 377 | 378 | last month, 379 | 380 | last 3 months, 381 | 382 | everything else. 383 | 384 | These segments are hardcoded, but it is trivial to change them if necessary. 385 | 386 | This mode was added to support searching through blogs, news headlines, etc. When using time segments, recent records 387 | would be ranked higher because of segment, but within the same segment, more relevant records would be ranked higher - 388 | unlike sorting by just the timestamp attribute, which would not take relevance into account at all. 389 | 390 | SortExtended 391 | 392 | SortExtended sorts by SQL-like combination of columns in ASC/DESC order. You can specify an SQL-like sort expression 393 | with up to 5 attributes (including internal attributes), eg: 394 | 395 | @relevance DESC, price ASC, @id DESC 396 | 397 | Both internal attributes (that are computed by the engine on the fly) and user attributes that were configured for this 398 | index are allowed. Internal attribute names must start with magic @-symbol; user attribute names can be used as is. 399 | In the example above, @relevance and @id are internal attributes and price is user-specified. 400 | 401 | Known internal attributes are: 402 | 403 | @id (match ID) 404 | 405 | @weight (match weight) 406 | 407 | @rank (match weight) 408 | 409 | @relevance (match weight) 410 | 411 | @random (return results in random order) 412 | 413 | @rank and @relevance are just additional aliases to @weight. 414 | 415 | SortExpr 416 | 417 | SortExpr sorts by an arithmetic expression. 418 | 419 | `SortRelevance` ignores any additional parameters and always sorts matches by relevance rank. 420 | All other modes require an additional sorting clause, with the syntax depending on specific mode. 421 | SortAttrAsc, SortAttrDesc and SortTimeSegments modes require simply an attribute name. 422 | SortRelevance is equivalent to sorting by “@weight DESC, @id ASC” in extended sorting mode, 423 | SortAttrAsc is equivalent to “attribute ASC, @weight DESC, @id ASC”, 424 | and SortAttrDesc to “attribute DESC, @weight DESC, @id ASC” respectively. 425 | */ 426 | type ESortOrder uint32 427 | 428 | const ( 429 | SortRelevance ESortOrder = iota // sort by document relevance desc, then by date 430 | SortAttrDesc // sort by document data desc, then by relevance desc 431 | SortAttrAsc // sort by document data asc, then by relevance desc 432 | SortTimeSegments // sort by time segments (hour/day/week/etc) desc, then by relevance desc 433 | SortExtended // sort by SQL-like expression (eg. "@relevance DESC, price ASC, @id DESC") 434 | SortExpr // sort by arithmetic expression in descending order (eg. "@id + max(@weight,1000)*boost + log(price)") 435 | SortTotal 436 | ) 437 | 438 | /* 439 | Qflags is bitmask with query flags which is set by calling Search.SetQueryFlags() 440 | Different values have to be combined with '+' or '|' operation from following constants: 441 | 442 | QflagReverseScan 443 | 444 | Control the order in which full-scan query processes the rows. 445 | 0 direct scan 446 | 1 reverse scan 447 | 448 | QFlagSortKbuffer 449 | 450 | Determines sort method for resultset sorting. 451 | The result set is in both cases the same; picking one option or the other 452 | may just improve (or worsen!) performance. 453 | 0 priority queue 454 | 1 k-buffer (gives faster sorting for already pre-sorted data, e.g. index data sorted by id) 455 | 456 | QflagMaxPredictedTime 457 | 458 | Determines if query has or not max_predicted_time option as an extra parameter 459 | 0 no predicted time provided 460 | 1 query contains predicted time metric 461 | 462 | QflagSimplify 463 | 464 | Switch on query boolean simplification to speed it up 465 | If set to 1, daemon will simplify complex queries or queries that produced by different algos to eliminate and 466 | optimize different parts of query. 467 | 0 query will be calculated without transformations 468 | 1 query will be transformed and simplified. 469 | 470 | List of performed transformation is: 471 | 472 | common NOT 473 | ((A !N) | (B !N)) -> ((A|B) !N) 474 | 475 | common compound NOT 476 | ((A !(N C)) | (B !(N D))) -> (((A|B) !N) | (A !C) | (B !D)) // if cost(N) > cost(A) + cost(B) 477 | 478 | common sub-term 479 | ((A (X | C)) | (B (X | D))) -> (((A|B) X) | (A C) | (B D)) // if cost(X) > cost(A) + cost(B) 480 | 481 | common keywords 482 | (A | "A B"~N) -> A 483 | ("A B" | "A B C") -> "A B" 484 | ("A B"~N | "A B C"~N) -> ("A B"~N) 485 | 486 | common PHRASE 487 | ("X A B" | "Y A B") -> (("X|Y") "A B") 488 | 489 | common AND NOT factor 490 | ((A !X) | (A !Y) | (A !Z)) -> (A !(X Y Z)) 491 | 492 | common OR NOT 493 | ((A !(N | N1)) | (B !(N | N2))) -> (( (A !N1) | (B !N2) ) !N) 494 | 495 | excess brackets 496 | ((A | B) | C) -> ( A | B | C ) 497 | ((A B) C) -> ( A B C ) 498 | 499 | excess AND NOT 500 | ((A !N1) !N2) -> (A !(N1 | N2)) 501 | 502 | QflagPlainIdf 503 | 504 | Determines how BM25 IDF will be calculated. Below ``N'' is collection size, and ``n'' is number of matched documents 505 | 1 plain IDF = log(N/n), as per Sparck-Jonesor 506 | 0 normalized IDF = log((N-n+1)/n), as per Robertson et al 507 | 508 | QflagGlobalIdf 509 | 510 | Determines whether to use global statistics (frequencies) from the global_idf file for IDF computations, 511 | rather than the local index statistics. 512 | 0 use local index statistics 513 | 1 use global_idf file (see https://docs.manticoresearch.com/latest/html/conf_options_reference/index_configuration_options.html#global-idf) 514 | 515 | QflagNormalizedTfIdf 516 | 517 | Determines whether to divide IDF value additionally by query word count, so that TF*IDF fits into [0..1] range 518 | 0 don't divide IDF by query word count 519 | 1 divide IDF by query word count 520 | 521 | Notes for QflagPlainIdf and QflagNormalizedTfIdf flags 522 | 523 | The historically default IDF (Inverse Document Frequency) in Manticore is equivalent to 524 | QflagPlainIdf=0, QflagNormalizedTfIdf=1, and those normalizations may cause several undesired effects. 525 | 526 | First, normalized idf (QflagPlainIdf=0) causes keyword penalization. For instance, if you search for [the | something] 527 | and [the] occurs in more than 50% of the documents, then documents with both keywords [the] and [something] will get 528 | less weight than documents with just one keyword [something]. Using QflagPlainIdf=1 avoids this. Plain IDF varies 529 | in [0, log(N)] range, and keywords are never penalized; while the normalized IDF varies in [-log(N), log(N)] range, 530 | and too frequent keywords are penalized. 531 | 532 | Second, QflagNormalizedTfIdf=1 causes IDF drift over queries. Historically, we additionally divided IDF by query 533 | keyword count, so that the entire sum(tf*idf) over all keywords would still fit into [0,1] range. However, that 534 | means that queries [word1] and [word1 | nonmatchingword2] would assign different weights to the exactly same result 535 | set, because the IDFs for both “word1” and “nonmatchingword2” would be divided by 2. QflagNormalizedTfIdf=0 536 | fixes that. Note that BM25, BM25A, BM25F() ranking factors will be scaled accordingly once you 537 | disable this normalization. 538 | 539 | QflagLocalDf 540 | 541 | Determines whether to automatically sum DFs over all the local parts of a distributed index, 542 | so that the IDF is consistent (and precise) over a locally sharded index. 543 | 0 don't sum local DFs 544 | 1 sum local DFs 545 | 546 | QflagLowPriority 547 | 548 | Determines priority for executing the query 549 | 0 run the query in usual (normal) priority 550 | 1 run the query in idle priority 551 | 552 | QflagFacet 553 | 554 | Determines slave role of the query in multi-query facet 555 | 0 query is not a facet query, or is main facet query 556 | 1 query is depended (slave) part of facet multiquery 557 | 558 | QflagFacetHead 559 | 560 | Determines slave role of the query in multi-query facet 561 | 0 query is not a facet query, or is slave of facet query 562 | 1 query is main (head) query of facet multiquery 563 | 564 | QflagJsonQuery 565 | 566 | Determines if query is originated from REST api and so, must be parsed as one of JSON syntax 567 | 0 query is API query 568 | 1 query is JSON query 569 | */ 570 | type Qflags uint32 571 | 572 | const ( 573 | QflagReverseScan Qflags = 1 << iota // direct or reverse full-scans 574 | QFlagSortKbuffer // pq or kbuffer for sorting 575 | QflagMaxPredictedTime // has or not max_predicted_time value 576 | QflagSimplify // apply or not boolean simplification 577 | QflagPlainIdf // plain or normalized idf 578 | QflagGlobalIdf // use or not global idf 579 | QflagNormalizedTfIdf // plain or normalized tf-idf 580 | QflagLocalDf // sum or not DFs over a locally sharderd (distributed) index 581 | QflagLowPriority // run query in idle priority 582 | QflagFacet // query is part of facet batch query 583 | QflagFacetHead // query is main facet query 584 | QflagJsonQuery // query is JSON query (otherwise - API query) 585 | ) 586 | -------------------------------------------------------------------------------- /manticore/search.go: -------------------------------------------------------------------------------- 1 | package manticore 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "time" 7 | ) 8 | 9 | // version of master-agent SEARCH command extension 10 | const commandSearchMaster uint32 = 16 11 | 12 | /* 13 | EMatchMode selects search query matching mode. 14 | So-called matching modes are a legacy feature that used to provide (very) limited query syntax and ranking support. 15 | Currently, they are deprecated in favor of full-text query language and so-called Available built-in rankers. 16 | It is thus strongly recommended to use `MatchExtended` and proper query syntax rather than any other legacy mode. 17 | All those other modes are actually internally converted to extended syntax anyway. SphinxAPI still defaults to 18 | `MatchAll` but that is for compatibility reasons only. 19 | 20 | There are the following matching modes available: 21 | 22 | MatchAll 23 | 24 | MatchAll matches all query words. 25 | 26 | MatchAny 27 | 28 | MatchAny matches any of the query words. 29 | 30 | MatchPhrase 31 | 32 | MatchPhrase, matches query as a phrase, requiring perfect match. 33 | 34 | MatchBoolean 35 | 36 | MatchBoolean, matches query as a boolean expression (see Boolean query syntax). 37 | 38 | MatchExtended 39 | 40 | MatchExtended2 41 | 42 | MatchExtended, MatchExtended2 (alias) matches query as an expression in Manticore internal query language 43 | (see Extended query syntax). This is default matching mode if nothing else specified. 44 | 45 | MatchFullscan 46 | 47 | MatchFullscan, matches query, forcibly using the “full scan” mode as below. NB, any query terms will be ignored, 48 | such that filters, filter-ranges and grouping will still be applied, but no text-matching. MatchFullscan mode will be 49 | automatically activated in place of the specified matching mode when the query string is empty (ie. its length is zero). 50 | 51 | In full scan mode, all the indexed documents will be considered as matching. 52 | Such queries will still apply filters, sorting, and group by, but will not perform any full-text searching. 53 | This can be useful to unify full-text and non-full-text searching code, or to offload SQL server 54 | (there are cases when Manticore scans will perform better than analogous MySQL queries). 55 | An example of using the full scan mode might be to find posts in a forum. By selecting the forum’s user ID via 56 | SetFilter() but not actually providing any search text, Manticore will match every document (i.e. every post) 57 | where SetFilter() would match - in this case providing every post from that user. By default this will be ordered by 58 | relevancy, followed by Manticore document ID in ascending order (earliest first). 59 | */ 60 | type EMatchMode uint32 61 | 62 | const ( 63 | MatchAll EMatchMode = iota // match all query words 64 | MatchAny // match any query word 65 | MatchPhrase // match this exact phrase 66 | MatchBoolean // match this boolean query 67 | MatchExtended // match this extended query 68 | MatchFullscan // match all document IDs w/o fulltext query, apply filters 69 | MatchExtended2 // extended engine V2 (TEMPORARY, WILL BE REMOVED IN 0.9.8-RELEASE) 70 | 71 | MatchTotal 72 | ) 73 | 74 | type searchFilter struct { 75 | Attribute string 76 | FilterType eFilterType 77 | Exclude bool 78 | FilterData interface{} 79 | } 80 | 81 | // Search represents one search query. 82 | // Exported fields may be set directly. Unexported which bind by internal dependencies and constrains 83 | // intended to be set wia special methods. 84 | type Search struct { 85 | Offset int32 // offset into resultset (0) 86 | Limit int32 // count of resultset (20) 87 | MaxMatches int32 88 | CutOff int32 89 | RetryCount int32 90 | MaxQueryTime time.Duration 91 | RetryDelay time.Duration 92 | predictedTime time.Duration 93 | MatchMode EMatchMode // Matching mode 94 | ranker ERankMode 95 | sort ESortOrder 96 | rankexpr string 97 | sortby string 98 | FieldWeights map[string]int32 // bind per-field weights by name 99 | IndexWeights map[string]int32 // bind per-index weights by name 100 | IDMin DocID // set IDs range to match (from) 101 | IDMax DocID // set IDs range to match (to) 102 | filters []searchFilter 103 | geoLatAttr string 104 | geoLonAttr string 105 | geoLatitude float32 106 | geoLongitude float32 107 | Groupfunc EGroupBy 108 | GroupBy string 109 | GroupSort string 110 | GroupDistinct string // count-distinct attribute for group-by queries 111 | SelectClause string // select-list (attributes or expressions), SQL-like syntax 112 | queryflags Qflags 113 | outerorderby string 114 | outeroffset int32 115 | outerlimit int32 116 | hasouter bool 117 | tokenFlibrary string 118 | tokenFname string 119 | tokenFopts string 120 | Indexes string 121 | Comment string 122 | Query string 123 | } 124 | 125 | // NewSearch construct default search which then may be customized. You may just customize 'Query' and m.b. 'Indexes' 126 | // from default one, and it will work like a simple 'Query()' call. 127 | func NewSearch(query, index, comment string) Search { 128 | return Search{ 129 | 0, 20, 1000, 0, 0, 130 | 0, 0, 0, 131 | MatchAll, 132 | RankDefault, 133 | SortRelevance, 134 | "", "", 135 | nil, nil, 136 | 0, 0, 137 | nil, 138 | "", "", 139 | 0, 0, 140 | GroupbyDay, 141 | "", "@group desc", "", 142 | "", 143 | QflagNormalizedTfIdf, 144 | "", 145 | 0, 0, 146 | false, 147 | "", "", "", 148 | index, comment, query, 149 | } 150 | } 151 | 152 | /* 153 | AddFilter adds new integer values set filter. 154 | 155 | On this call, additional new filter is added to the existing list of filters. 156 | 157 | `attribute` must be a string with attribute name 158 | 159 | `values` must be a plain slice containing integer values. 160 | 161 | `exclude` controls whether to accept the matching documents (default mode, when `exclude` is false) or reject them. 162 | 163 | Only those documents where `attribute` column value stored in the index matches any of the values from `values` slice 164 | will be matched (or rejected, if `exclude` is true). 165 | */ 166 | func (q *Search) AddFilter(attribute string, values []int64, exclude bool) { 167 | q.filters = append(q.filters, searchFilter{attribute, FilterValues, exclude, values}) 168 | } 169 | 170 | /* 171 | AddFilterExpression adds new filter by expression. 172 | 173 | On this call, additional new filter is added to the existing list of filters. 174 | 175 | The only value `expression` must contain filtering expression which returns bool. 176 | 177 | Expression has SQL-like syntax and may refer to columns (usually json fields) by name, and may look like: 'j.price - 1 > 3 OR j.tag IS NOT null' 178 | Documents either filtered by 'true' expression, either (if `exclude` is set to true) by 'false'. 179 | */ 180 | func (q *Search) AddFilterExpression(expression string, exclude bool) { 181 | q.filters = append(q.filters, searchFilter{expression, FilterExpression, exclude, nil}) 182 | } 183 | 184 | /* 185 | AddFilterFloatRange adds new float range filter. 186 | 187 | On this call, additional new filter is added to the existing list of filters. 188 | 189 | `attribute` must be a string with attribute name. 190 | 191 | `fmin` and `fmax` must be floats that define the acceptable attribute values range (including the boundaries). 192 | 193 | `exclude` controls whether to accept the matching documents (default mode, when `exclude` is false) or reject them. 194 | 195 | Only those documents where `attribute` column value stored in the index is between `fmin` and `fmax` 196 | (including values that are exactly equal to `fmin` or `fmax`) will be matched (or rejected, if `exclude` is true). 197 | */ 198 | func (q *Search) AddFilterFloatRange(attribute string, fmin, fmax float32, exclude bool) { 199 | q.filters = append(q.filters, searchFilter{attribute, FilterFloatrange, exclude, []float32{fmin, fmax}}) 200 | } 201 | 202 | // AddFilterNull adds new IsNull filter. 203 | // 204 | //On this call, additional new filter is added to the existing list of filters. Documents where `attribute` is null will match, 205 | //(if `isnull` is true) or not match (if `isnull` is false). 206 | func (q *Search) AddFilterNull(attribute string, isnull bool) { 207 | 208 | q.filters = append(q.filters, searchFilter{attribute, FilterNull, false, isnull}) 209 | } 210 | 211 | /* 212 | AddFilterRange adds new integer range filter. 213 | 214 | On this call, additional new filter is added to the existing list of filters. 215 | 216 | `attribute` must be a string with attribute name. 217 | 218 | `imin` and `imax` must be integers that define the acceptable attribute values range (including the boundaries). 219 | 220 | `exclude` controls whether to accept the matching documents (default mode, when `exclude` is false) or reject them. 221 | 222 | Only those documents where `attribute` column value stored in the index is between `imin` and `imax` 223 | (including values that are exactly equal to `imin` or `imax`) will be matched (or rejected, if `exclude` is true). 224 | */ 225 | func (q *Search) AddFilterRange(attribute string, imin, imax int64, exclude bool) { 226 | q.filters = append(q.filters, searchFilter{attribute, FilterRange, exclude, []int64{imin, imax}}) 227 | } 228 | 229 | /* 230 | AddFilterString adds new string value filter. 231 | 232 | On this call, additional new filter is added to the existing list of filters. 233 | 234 | `attribute` must be a string with attribute name. 235 | 236 | `value` must be a string. 237 | 238 | `exclude` must be a boolean value; it controls whether to accept the matching documents (default mode, when `exclude` is false) or reject them. 239 | 240 | Only those documents where `attribute` column value stored in the index equal to string value from `value` will be matched (or rejected, if `exclude` is true). 241 | */ 242 | func (q *Search) AddFilterString(attribute string, value string, exclude bool) { 243 | q.filters = append(q.filters, searchFilter{attribute, FilterString, exclude, value}) 244 | } 245 | 246 | /* 247 | AddFilterStringList adds new string list filter. 248 | 249 | On this call, additional new filter is added to the existing list of filters. 250 | 251 | `attribute` must be a string with attribute name. 252 | 253 | `values` must be slice of strings 254 | 255 | `exclude` must be a boolean value; it controls whether to accept the matching documents (default mode, when `exclude` is false) or reject them. 256 | 257 | Only those documents where `attribute` column value stored in the index equal to one of string values from `values` will be matched (or rejected, if `exclude` is true). 258 | */ 259 | func (q *Search) AddFilterStringList(attribute string, values []string, exclude bool) { 260 | q.filters = append(q.filters, searchFilter{attribute, FilterStringList, exclude, values}) 261 | } 262 | 263 | /* 264 | AddFilterUservar adds new uservar filter. 265 | 266 | On this call, additional new filter is added to the existing list of filters. 267 | 268 | `attribute` must be a string with attribute name. 269 | 270 | `uservar` must be name of user variable, containing list of filtering values, starting from @, as "@var" 271 | 272 | `exclude` must be a boolean value; it controls whether to accept the matching documents (default mode, when `exclude` is false) or reject them. 273 | 274 | Only those documents where `attribute` column value stored in the index equal to one of the values stored in `uservar` variable on daemon side (or rejected, if `exclude` is true). 275 | Such filter intended to save huge list of variables once on the server, and then refer to it by name. Saving the list might be done by separate call of 'SetUservar()' 276 | */ 277 | func (q *Search) AddFilterUservar(attribute string, uservar string, exclude bool) { 278 | q.filters = append(q.filters, searchFilter{attribute, FilterUservar, exclude, uservar}) 279 | } 280 | 281 | // ChangeQueryFlags changes (set or reset) query flags by mask `flags`. 282 | func (q *Search) ChangeQueryFlags(flags Qflags, set bool) { 283 | if set { 284 | q.queryflags |= flags 285 | } else { 286 | q.queryflags &^= flags 287 | if !q.hasSetQueryFlag(QflagMaxPredictedTime) { 288 | q.predictedTime = 0 289 | } 290 | } 291 | } 292 | 293 | /* 294 | ResetFilters clears all currently set search filters. 295 | 296 | This call is only normally required when using multi-queries. You might want to set different filters for different 297 | queries in the batch. To do that, you may either create another Search request and fill it from the scratch, either 298 | copy existing (last one) and modify. To change all the filters in the copy you can call ResetFilters() and add new 299 | filters using the respective calls. 300 | */ 301 | func (q *Search) ResetFilters() { 302 | q.geoLatAttr, q.geoLonAttr = "", "" 303 | q.filters = nil 304 | } 305 | 306 | /* 307 | ResetGroupBy clears all currently group-by settings, and disables group-by. 308 | 309 | This call is only normally required when using multi-queries. You might want to set different 310 | group-by settings in the batch. To do that, you may either create another Search request and fill ot from the scratch, either 311 | copy existing (last one) and modify. In last case you can change individual group-by settings using SetGroupBy() and SetGroupDistinct() calls, 312 | but you can not disable group-by using those calls. ResetGroupBy() fully resets previous group-by settings and 313 | disables group-by mode in the current Search query. 314 | */ 315 | func (q *Search) ResetGroupBy() { 316 | q.GroupDistinct, q.GroupBy = "", "" 317 | q.GroupSort = "@group desc" 318 | q.Groupfunc = GroupbyDay 319 | } 320 | 321 | /* 322 | ResetOuterSelect clears all outer select settings 323 | 324 | This call is only normally required when using multi-queries. You might want to set different 325 | outer select settings in the batch. To do that, you may either create another Search request and fill ot from the scratch, either 326 | copy existing (last one) and modify. In last case you can change individual group-by settings using SetOuterSelect() calls, 327 | but you can not disable outer statement by this calls. ResetOuterSelect() fully resets previous outer select settings. 328 | */ 329 | func (q *Search) ResetOuterSelect() { 330 | q.outerorderby, q.outeroffset, q.outerlimit, q.hasouter = "", 0, 0, false 331 | } 332 | 333 | /* 334 | ResetQueryFlags resets query flags of Select query to default value, and also reset value set by SetMaxPredictedTime() call. 335 | 336 | This call is only normally required when using multi-queries. You might want to set different 337 | flags of Select queries in the batch. To do that, you may either create another Search request and fill ot from the scratch, either 338 | copy existing (last one) and modify. In last case you can change individual or many flags using SetQueryFlags() and ChangeQueryFlags() calls. 339 | This call just one-shot set all the flags to default value `QflagNormalizedTfIdf`, and also set predicted time to 0. 340 | */ 341 | func (q *Search) ResetQueryFlags() { 342 | q.queryflags = QflagNormalizedTfIdf 343 | q.predictedTime = 0 344 | } 345 | 346 | /* 347 | SetGeoAnchor sets anchor point for and geosphere distance (geodistance) calculations, and enable them. 348 | 349 | `attrlat` and `attrlong` contain the names of latitude and longitude attributes, respectively. 350 | 351 | `lat` and `long` specify anchor point latitude and longitude, in radians. 352 | 353 | Once an anchor point is set, you can use magic @geodist attribute name in your filters and/or sorting expressions. 354 | Manticore will compute geosphere distance between the given anchor point and a point specified by latitude and 355 | longitude attributes from each full-text match, and attach this value to the resulting match. The latitude and l 356 | ongitude values both in SetGeoAnchor and the index attribute data are expected to be in radians. The result will 357 | be returned in meters, so geodistance value of 1000.0 means 1 km. 1 mile is approximately 1609.344 meters. 358 | */ 359 | func (q *Search) SetGeoAnchor(attrlat, attrlong string, lat, long float32) { 360 | q.geoLatAttr, q.geoLonAttr = attrlat, attrlong 361 | q.geoLatitude, q.geoLongitude = lat, long 362 | } 363 | 364 | func (q *Search) hasGeoAnchor() bool { 365 | return q.geoLatAttr != "" && q.geoLonAttr != "" 366 | } 367 | 368 | /* 369 | SetGroupBy sets grouping attribute, function, and groups sorting mode; and enables grouping. 370 | 371 | `attribute` is a string that contains group-by attribute name. 372 | 373 | `func` is a constant that chooses a function applied to the attribute value in order to compute group-by key. 374 | 375 | `groupsort` is optional clause that controls how the groups will be sorted. 376 | 377 | Grouping feature is very similar in nature to GROUP BY clause from SQL. 378 | Results produces by this function call are going to be the same as produced by the following pseudo code: 379 | SELECT ... GROUP BY func(attribute) ORDER BY groupsort 380 | 381 | Note that it’s `groupsort` that affects the order of matches in the final result set. 382 | Sorting mode (see `SetSortMode()`) affect the ordering of matches within group, ie. what match will be selected 383 | as the best one from the group. So you can for instance order the groups by matches count and select the most relevant 384 | match within each group at the same time. 385 | 386 | Grouping on string attributes is supported, with respect to current collation. 387 | */ 388 | func (q *Search) SetGroupBy(attribute string, gfunc EGroupBy, groupsort ...string) { 389 | if len(groupsort) > 0 { 390 | q.GroupSort = groupsort[0] 391 | } 392 | 393 | q.GroupBy = attribute 394 | q.Groupfunc = gfunc 395 | } 396 | 397 | // SetQueryFlags set query flags. New flags are |-red to existing value, previously set flags are not affected. 398 | // Note that default flags has set QflagNormalizedTfIdf bit, so if you need to reset it, you need to explicitly invoke 399 | // ChangeQueryFlags(QflagNormalizedTfIdf,false) for it. 400 | func (q *Search) SetQueryFlags(flags Qflags) { 401 | q.queryflags |= flags 402 | } 403 | 404 | func (q *Search) hasSetQueryFlag(flag Qflags) bool { 405 | return (q.queryflags & flag) != 0 406 | } 407 | 408 | // SetMaxPredictedTime set max predicted time and according query flag 409 | func (q *Search) SetMaxPredictedTime(predtime time.Duration) { 410 | q.predictedTime = predtime 411 | q.SetQueryFlags(QflagMaxPredictedTime) 412 | } 413 | 414 | // SetOuterSelect determines outer select conditions for Search query. 415 | // 416 | // `orderby` specify clause with SQL-like syntax as "foo ASC, bar DESC, baz" where name of the items (`foo`, `bar`, `baz` in example) are the names of columns originating from internal query. 417 | // 418 | // `offset` and `limit` has the same meaning as fields Offset and Limit in the clause, but applied to outer select. 419 | // 420 | // Outer select currently have 2 usage cases: 421 | // 422 | // 1. We have a query with 2 ranking UDFs, one very fast and the other one slow and we perform a full-text search will a big match result set. Without outer the query would look like 423 | // 424 | // q := NewSearch("some common query terms", "index", "") 425 | // q.SelectClause = "id, slow_rank() as slow, fast_rank as fast" 426 | // q.SetSortMode( SortExtended, "fast DESC, slow DESC" ) 427 | // // q.Limit=20, q.MaxMatches=1000 - are default, so we don't set them explicitly 428 | // 429 | // With subselects the query can be rewritten as : 430 | // q := NewSearch("some common query terms", "index", "") 431 | // q.SelectClause = "id, slow_rank() as slow, fast_rank as fast" 432 | // q.SetSortMode( SortExtended, "fast DESC" ) 433 | // q.Limit=100 434 | // q.SetOuterSelect("slow desc", 0, 20) 435 | // 436 | // In the initial query the slow_rank() UDF is computed for the entire match result set. 437 | // With subselects, only fast_rank() is computed for the entire match result set, while slow_rank() is only computed for a limited set. 438 | // 439 | // 2. The second case comes handy for large result set coming from a distributed index. 440 | // 441 | // For this query: 442 | // 443 | // q := NewSearch("some conditions", "my_dist_index", "") 444 | // q.Limit = 50000 445 | // If we have 20 nodes, each node can send back to master a number of 50K records, resulting in 20 x 50K = 1M records, 446 | // however as the master sends back only 50K (out of 1M), it might be good enough for us for the nodes to send only the 447 | // top 10K records. With outer select we can rewrite the query as: 448 | // 449 | // q := NewSearch("some conditions", "my_dist_index", "") 450 | // q.Limit = 10000 451 | // q.SetOuterSelect("some_attr", 0, 50000) 452 | //In this case, the nodes receive only the inner query and execute. This means the master will receive only 20x10K=200K 453 | // records. The master will take all the records received, reorder them by the OUTER clause and return the best 50K 454 | // records. The outer select helps reducing the traffic between the master and the nodes and also reduce the master’s 455 | // computation time (as it process only 200K instead of 1M). 456 | func (q *Search) SetOuterSelect(orderby string, offset, limit int32) { 457 | q.outerorderby = orderby 458 | q.outeroffset, q.outerlimit, q.hasouter = offset, limit, true 459 | } 460 | 461 | // SetRankingExpression assigns ranking expression, and also set ranking mode to RankExpr 462 | // 463 | // `rankexpr` provides ranking formula, for example, "sum(lcs*user_weight)*1000+bm25" - this is the same 464 | // as RankProximityBm25, but written explicitly. 465 | // Since using ranking expression assumes RankExpr ranker, it is also set by this function. 466 | func (q *Search) SetRankingExpression(rankexpr string) { 467 | q.rankexpr = rankexpr 468 | if q.ranker != RankExpr && q.ranker != RankExport { 469 | q.SetRankingMode(RankExpr) 470 | } 471 | } 472 | 473 | // SetRankingMode assigns ranking mode and also adjust MatchMode to MatchExtended2 (since otherwise rankers are useless) 474 | func (q *Search) SetRankingMode(ranker ERankMode) { 475 | q.ranker = ranker 476 | if q.MatchMode != MatchExtended && q.MatchMode != MatchExtended2 { 477 | q.MatchMode = MatchExtended2 478 | } 479 | } 480 | 481 | // SetSortMode sets matches sorting mode 482 | // 483 | // `sort` determines sorting mode. 484 | // 485 | // `sortby` determines attribute or expression used for sorting. 486 | // 487 | // If `sortby` set in Search query is empty (it is not necessary set in this very call, it might be set earlier!), then `sort` 488 | // is explicitly set as SortRelevance 489 | func (q *Search) SetSortMode(sort ESortOrder, sortby ...string) { 490 | q.sort = sort 491 | if len(sortby) > 0 { 492 | q.sortby = sortby[0] 493 | } 494 | if q.sortby == "" { 495 | q.sort = SortRelevance 496 | } 497 | } 498 | 499 | /* 500 | SetTokenFilter setups UDF token filter 501 | 502 | `library` is the name of plugin library, as "mylib.so" 503 | 504 | `name` is the name of token filtering function in the library, as "email_process" 505 | 506 | `opts` is string parameters which passed to udf filter, like "field=email;split=.io". Format of the options determined by UDF plugin. 507 | */ 508 | func (q *Search) SetTokenFilter(library, name string, opts string) { 509 | q.tokenFlibrary = library 510 | q.tokenFname = name 511 | q.tokenFopts = opts 512 | } 513 | 514 | // iOStats is internal structure, used only in master-agent communication 515 | type iOStats struct { 516 | ReadTime, ReadBytes, WriteTime, WriteBytes int64 517 | ReadOps, WriteOps uint32 518 | } 519 | 520 | // ColumnInfo represents one attribute column in resultset schema 521 | type ColumnInfo struct { 522 | Name string // name of the attribute 523 | Type EAttrType // type of the attribute 524 | } 525 | 526 | // Match represents one match (document) in result schema 527 | type Match struct { 528 | DocID DocID // key Document ID 529 | Weight int // weight of the match 530 | Attrs []interface{} // optional array of attributes, quantity and types depends from schema 531 | } 532 | 533 | // Stringer interface for Match type 534 | func (vl Match) String() (line string) { 535 | line = fmt.Sprintf("Doc: %v, Weight: %v, attrs: %v", vl.DocID, vl.Weight, vl.Attrs) 536 | return 537 | } 538 | 539 | // JsonOrStr is typed string with explicit flag whether it is 'just a string', or json document. It may be used, say, 540 | // to either escape plain strings when appending to JSON structure, either add it 'as is' assuming it is alreayd json. 541 | // Such values came from daemon as attribute values for PQ indexes. 542 | type JsonOrStr struct { 543 | IsJson bool // true, if Val is JSON document; false if it is just a plain string 544 | Val string // value (string or JSON document) 545 | } 546 | 547 | // Stringer interface for JsonOrStr type. Just append ' (json)' suffix, if IsJson is true. 548 | func (vl JsonOrStr) String() string { 549 | if vl.IsJson { 550 | return fmt.Sprintf("%s (json)", vl.Val) 551 | } else { 552 | return vl.Val 553 | } 554 | } 555 | 556 | // WordStat describes statistic for one word in QueryResult. That is, word, num of docs and num of hits. 557 | type WordStat struct { 558 | Word string 559 | Docs, Hits int 560 | } 561 | 562 | // Stringer interface for WordStat type 563 | func (vl WordStat) String() string { 564 | return fmt.Sprintf("'%s' (Docs:%d, Hits:%d)", vl.Word, vl.Docs, vl.Hits) 565 | } 566 | 567 | // QueryResult represents resultset from successful Query/RunQuery, or one of resultsets from RunQueries call. 568 | type QueryResult struct { 569 | Error, Warning string // messages (if any) 570 | Status ESearchdstatus // status code for current resultset 571 | Fields []string // fields of the schema 572 | Attrs []ColumnInfo // attributes of the schema 573 | Id64 bool // if DocumentID is 64-bit (always true) 574 | Matches []Match // set of matches according to schema 575 | Total, TotalFound int // num of matches and total num of matches found 576 | QueryTime time.Duration // query duration 577 | WordStats []WordStat // words statistic 578 | } 579 | 580 | // Stringer interface for EAttrType type 581 | func (vl EAttrType) String() string { 582 | switch vl { 583 | case AttrNone: 584 | return "none" 585 | case AttrInteger: 586 | return "int" 587 | case AttrTimestamp: 588 | return "timestamp" 589 | case AttrBool: 590 | return "bool" 591 | case AttrFloat: 592 | return "float" 593 | case AttrBigint: 594 | return "bigint" 595 | case AttrString: 596 | return "string" 597 | case AttrPoly2d: 598 | return "poly2d" 599 | case AttrStringptr: 600 | return "stringptr" 601 | case AttrTokencount: 602 | return "tokencount" 603 | case AttrJson: 604 | return "json" 605 | case AttrUint32set: 606 | return "uint32Set" 607 | case AttrInt64set: 608 | return "int64Set" 609 | case AttrMaparg: 610 | return "maparg" 611 | case AttrFactors: 612 | return "factors" 613 | case AttrJsonField: 614 | return "jsonField" 615 | case AttrFactorsJson: 616 | return "factorJson" 617 | default: 618 | return fmt.Sprintf("unknown(%d)", uint32(vl)) 619 | } 620 | } 621 | 622 | // Stringer interface for ColumnInfo type 623 | func (res ColumnInfo) String() string { 624 | return fmt.Sprintf("%s: %v", res.Name, res.Type) 625 | } 626 | 627 | // Stringer interface for QueryResult type 628 | func (res QueryResult) String() string { 629 | line := fmt.Sprintf("Status: %v\n", res.Status) 630 | if res.Status == StatusError { 631 | line += fmt.Sprintf("Error: %v\n", res.Error) 632 | } 633 | line += fmt.Sprintf("Query time: %v\n", res.QueryTime) 634 | line += fmt.Sprintf("Total: %v\n", res.Total) 635 | line += fmt.Sprintf("Total found: %v\n", res.TotalFound) 636 | line += fmt.Sprintln("Schema:") 637 | if len(res.Fields) != 0 { 638 | line += fmt.Sprintln("\tFields:") 639 | for i := 0; i < len(res.Fields); i++ { 640 | line += fmt.Sprintf("\t\t%v\n", res.Fields[i]) 641 | } 642 | } 643 | if len(res.Attrs) != 0 { 644 | line += fmt.Sprintln("\tAttributes:") 645 | for i := 0; i < len(res.Attrs); i++ { 646 | line += fmt.Sprintf("\t\t%v\n", res.Attrs[i]) 647 | } 648 | } 649 | 650 | if len(res.Matches) != 0 { 651 | line += fmt.Sprintln("Matches:") 652 | for i := 0; i < len(res.Matches); i++ { 653 | line += fmt.Sprintf("\t%v\n", &res.Matches[i]) 654 | } 655 | } 656 | if len(res.WordStats) != 0 { 657 | line += fmt.Sprintln("Word stats:") 658 | for i := 0; i < len(res.WordStats); i++ { 659 | line += fmt.Sprintf("\t%v\n", res.WordStats[i]) 660 | } 661 | } 662 | return line 663 | } 664 | 665 | func (buf *apibuf) buildSearchRequest(q *Search) { 666 | 667 | buf.putDword(uint32(q.queryflags)) 668 | buf.putInt(q.Offset) 669 | buf.putInt(q.Limit) 670 | 671 | buf.putDword(uint32(q.MatchMode)) 672 | buf.putDword(uint32(q.ranker)) 673 | if q.ranker == RankExport || q.ranker == RankExpr { 674 | buf.putString(q.rankexpr) 675 | } 676 | 677 | buf.putInt(int32(q.sort)) 678 | buf.putString(q.sortby) 679 | 680 | buf.putString(q.Query) 681 | buf.putInt(0) 682 | buf.putString(q.Indexes) 683 | buf.putInt(1) 684 | 685 | // default full id range (any Client range must be in filters at this stage) 686 | buf.putDocid(0) 687 | buf.putDocid(DocidMax) 688 | 689 | docidfilter := 0 690 | if (q.IDMin != 0) || (q.IDMax != DocidMax && q.IDMax != 0) { 691 | docidfilter = 1 692 | } 693 | 694 | buf.putLen(len(q.filters) + docidfilter) // N of filters 695 | // filters goes here 696 | for _, filter := range q.filters { 697 | buf.putString(filter.Attribute) 698 | buf.putDword(uint32(filter.FilterType)) 699 | switch filter.FilterType { 700 | case FilterString: 701 | buf.putString(filter.FilterData.(string)) 702 | case FilterUservar: 703 | buf.putString(filter.FilterData.(string)) 704 | case FilterNull: 705 | buf.putBoolByte(filter.FilterData.(bool)) 706 | case FilterRange: 707 | foo := filter.FilterData.([]int64) 708 | buf.putInt64(foo[0]) 709 | buf.putInt64(foo[1]) 710 | case FilterFloatrange: 711 | foo := filter.FilterData.([]float32) 712 | buf.putFloat(foo[0]) 713 | buf.putFloat(foo[1]) 714 | case FilterValues: 715 | foo := filter.FilterData.([]int64) 716 | buf.putLen(len(foo)) 717 | for _, value := range foo { 718 | buf.putInt64(value) 719 | } 720 | case FilterStringList: 721 | foo := filter.FilterData.([]string) 722 | buf.putLen(len(foo)) 723 | for _, value := range foo { 724 | buf.putString(value) 725 | } 726 | } 727 | buf.putBoolDword(filter.Exclude) 728 | } 729 | 730 | // docid filter, if any, we put as the last one 731 | if docidfilter == 1 { 732 | buf.putString("@id") 733 | buf.putDword(uint32(FilterRange)) 734 | buf.putDocid(q.IDMin) 735 | buf.putDocid(q.IDMax) 736 | buf.putBoolDword(false) 737 | } 738 | 739 | buf.putDword(uint32(q.Groupfunc)) 740 | buf.putString(q.GroupBy) 741 | buf.putInt(q.MaxMatches) 742 | 743 | buf.putString(q.GroupSort) 744 | buf.putInt(q.CutOff) 745 | 746 | buf.putInt(q.RetryCount) 747 | buf.putDuration(q.RetryDelay) 748 | 749 | buf.putString(q.GroupDistinct) 750 | 751 | buf.putBoolDword(q.hasGeoAnchor()) 752 | if q.hasGeoAnchor() { 753 | buf.putString(q.geoLatAttr) 754 | buf.putString(q.geoLonAttr) 755 | buf.putFloat(q.geoLatitude) 756 | buf.putFloat(q.geoLongitude) 757 | } 758 | 759 | buf.putLen(len(q.IndexWeights)) 760 | for idx, iw := range q.IndexWeights { 761 | buf.putString(idx) 762 | buf.putInt(iw) 763 | } 764 | 765 | buf.putDuration(q.MaxQueryTime) 766 | buf.putLen(len(q.FieldWeights)) 767 | for idx, iw := range q.FieldWeights { 768 | buf.putString(idx) 769 | buf.putInt(iw) 770 | } 771 | 772 | buf.putString(q.Comment) 773 | buf.putInt(0) // N of overrides 774 | 775 | buf.putString(q.SelectClause) 776 | 777 | if q.hasSetQueryFlag(QflagMaxPredictedTime) { 778 | buf.putDuration(q.predictedTime) 779 | } 780 | 781 | buf.putString(q.outerorderby) 782 | buf.putInt(q.outeroffset) 783 | buf.putInt(q.outerlimit) 784 | buf.putBoolDword(q.hasouter) 785 | buf.putString(q.tokenFlibrary) 786 | buf.putString(q.tokenFname) 787 | buf.putString(q.tokenFopts) 788 | 789 | buf.putInt(0) // N of filter tree elems 790 | } 791 | 792 | func (result *QueryResult) makeError(erstr string) error { 793 | result.Error = erstr 794 | if erstr == "" { 795 | return nil 796 | } 797 | return errors.New(erstr) 798 | } 799 | 800 | func (result *QueryResult) parseResult(req *apibuf) error { 801 | 802 | // extract status 803 | result.Status = ESearchdstatus(req.getDword()) 804 | switch result.Status { 805 | case StatusError: 806 | return result.makeError(req.getString()) 807 | case StatusRetry: 808 | return result.makeError(req.getString()) 809 | case StatusWarning: 810 | result.Warning = req.getString() 811 | } 812 | 813 | result.parseSchema(req) 814 | nmatches := req.getInt() 815 | result.Id64 = req.getIntBool() 816 | 817 | // parse matches 818 | result.Matches = make([]Match, nmatches) 819 | for j := 0; j < nmatches; j++ { 820 | result.parseMatch(&result.Matches[j], req) 821 | } 822 | 823 | // read totals (retrieved count, total count, query time, word count) 824 | result.Total = req.getInt() 825 | result.TotalFound = req.getInt() 826 | result.QueryTime = time.Millisecond * time.Duration(req.getInt()) 827 | 828 | nwords := req.getInt() 829 | result.WordStats = make([]WordStat, nwords) 830 | 831 | // read per-word stats 832 | for i := 0; i < nwords; i++ { 833 | result.WordStats[i].Word = req.getString() 834 | result.WordStats[i].Docs = req.getInt() 835 | result.WordStats[i].Hits = req.getInt() 836 | } 837 | return result.makeError("") 838 | } 839 | 840 | func (result *QueryResult) parseSchema(req *apibuf) { 841 | 842 | // read Fields 843 | nfields := req.getInt() 844 | result.Fields = make([]string, nfields) 845 | for j := 0; j < nfields; j++ { 846 | result.Fields[j] = req.getString() 847 | } 848 | 849 | // read attributes 850 | nattrs := req.getInt() 851 | result.Attrs = make([]ColumnInfo, nattrs) 852 | for j := 0; j < nattrs; j++ { 853 | result.Attrs[j].Name = req.getString() 854 | result.Attrs[j].Type = EAttrType(req.getDword()) 855 | } 856 | } 857 | 858 | func (result *QueryResult) parseMatch(match *Match, req *apibuf) { 859 | 860 | // docid 861 | if result.Id64 { 862 | match.DocID = req.getDocid() 863 | } else { 864 | match.DocID = DocID(req.getDword()) 865 | } 866 | 867 | // weight 868 | match.Weight = req.getInt() 869 | match.Attrs = make([]interface{}, len(result.Attrs)) 870 | 871 | // attributes 872 | for i, item := range result.Attrs { 873 | switch item.Type { 874 | 875 | case AttrUint32set: 876 | iValues := int(req.getDword()) 877 | values := make([]uint32, iValues) 878 | for j := 0; j < iValues; j++ { 879 | values[j] = req.getDword() 880 | } 881 | match.Attrs[i] = values 882 | 883 | case AttrInt64set: 884 | iValues := int(req.getDword()) 885 | values := make([]uint64, iValues) 886 | for j := 0; j < iValues; j++ { 887 | values[j] = req.getUint64() 888 | } 889 | match.Attrs[i] = values 890 | 891 | case AttrFloat: 892 | match.Attrs[i] = req.getFloat() 893 | 894 | case AttrBigint: 895 | match.Attrs[i] = req.getUint64() 896 | 897 | case AttrStringptr: 898 | case AttrString: 899 | foo := req.getRefBytes() 900 | ln := len(foo) 901 | var res JsonOrStr 902 | if ln>1 && foo[ln-2] == 0 { // this is legacy typed 903 | res.IsJson = foo[ln-1] == 0 904 | res.Val = string(foo[:ln-2]) 905 | } else { 906 | res.Val = string(foo) 907 | } 908 | match.Attrs[i] = res 909 | 910 | case AttrJson: 911 | case AttrFactors: 912 | case AttrFactorsJson: 913 | match.Attrs[i] = req.getBytes() 914 | 915 | case AttrJsonField: 916 | var bs bsonField 917 | bs.etype = eBsonType(req.getByte()) 918 | if bs.etype != bsonEof { 919 | bs.blob = req.getBytes() 920 | } 921 | match.Attrs[i] = bs 922 | 923 | case AttrTimestamp: 924 | foo := req.getDword() 925 | match.Attrs[i] = time.Unix(int64(foo), 0) 926 | 927 | default: 928 | match.Attrs[i] = req.getDword() 929 | } 930 | } 931 | } 932 | 933 | func buildSearchRequest(queries []Search) func(*apibuf) { 934 | return func(buf *apibuf) { 935 | buf.putUint(0) // that is cl! 936 | buf.putLen(len(queries)) 937 | for j := 0; j < len(queries); j++ { 938 | buf.buildSearchRequest(&queries[j]) 939 | } 940 | } 941 | } 942 | 943 | func parseSearchAnswer(nreqs int) func(*apibuf) interface{} { 944 | return func(answer *apibuf) interface{} { 945 | resp := make([]QueryResult, nreqs) 946 | for j := 0; j < nreqs; j++ { 947 | _ = resp[j].parseResult(answer) 948 | } 949 | return resp 950 | } 951 | } 952 | --------------------------------------------------------------------------------