├── .gitignore ├── docker-compose.yml ├── sonic ├── doc.go ├── client_test.go ├── search_service_test.go ├── client.go ├── ingest_service.go ├── ingest_service_test.go └── search_service.go ├── go.mod ├── README.md ├── conf.cfg ├── .travis.yml └── go.sum /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | vendor 3 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | 3 | services: 4 | sonic: 5 | image: valeriansaliou/sonic:v1.1.9 6 | volumes: 7 | - ./conf.cfg:/etc/sonic.cfg 8 | ports: 9 | - 1491:1491 10 | -------------------------------------------------------------------------------- /sonic/doc.go: -------------------------------------------------------------------------------- 1 | //Package sonic is golang bindings for https://github.com/valeriansaliou/sonic 2 | // 3 | //Syntax terminology and protocol can be found at: https://github.com/valeriansaliou/sonic/blob/master/PROTOCOL.md 4 | // 5 | package sonic 6 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/OGKevin/go-sonic 2 | 3 | go 1.12 4 | 5 | require ( 6 | github.com/kr/pretty v0.1.0 // indirect 7 | github.com/opentracing/opentracing-go v1.1.0 8 | github.com/pkg/errors v0.8.1 9 | github.com/satori/go.uuid v1.2.0 10 | github.com/sirupsen/logrus v1.4.1 11 | github.com/stretchr/testify v1.3.0 12 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect 13 | ) 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://travis-ci.com/OGKevin/go-sonic.svg?branch=master)](https://travis-ci.com/OGKevin/go-sonic) 2 | 3 | Golang bindings for https://github.com/valeriansaliou/sonic 4 | 5 | ```golang 6 | c, err := NewClientWithPassword("localhost:1491", "SecretPassword", context.Background()) 7 | if err != nil { 8 | panic(err) 9 | } 10 | 11 | // Ingest 12 | _, err = c.IngestService.Push(NewDataBuilder().Text("my string").Bucket("my bucket").Collection("my collection").Object("my object").Build()) 13 | if err != nil { 14 | panic(err) 15 | } 16 | 17 | // Search 18 | ch, err := c.SearchService.Query(NewDataBuilder().Collection("my collection").Bucket("my bucket").Text("my string").Build(), 0, 0) 19 | for e := range ch { 20 | log.Println(e) 21 | } 22 | ``` 23 | 24 | For protocol info, see https://github.com/valeriansaliou/sonic/blob/master/PROTOCOL.md 25 | -------------------------------------------------------------------------------- /conf.cfg: -------------------------------------------------------------------------------- 1 | # Sonic 2 | # Fast, lightweight and schema-less search backend 3 | # Configuration file 4 | # Example: https://github.com/valeriansaliou/sonic/blob/master/config.cfg 5 | 6 | 7 | [server] 8 | 9 | log_level = "error" 10 | 11 | 12 | [channel] 13 | 14 | inet = "0.0.0.0:1491" 15 | tcp_timeout = 300 16 | 17 | auth_password = "SecretPassword" 18 | 19 | [channel.search] 20 | 21 | query_limit_default = 10 22 | query_limit_maximum = 100 23 | query_alternates_try = 4 24 | 25 | suggest_limit_default = 5 26 | suggest_limit_maximum = 20 27 | 28 | 29 | [store] 30 | 31 | [store.kv] 32 | 33 | path = "./data/store/kv/" 34 | 35 | retain_word_objects = 1000 36 | 37 | [store.kv.pool] 38 | 39 | inactive_after = 1800 40 | 41 | [store.kv.database] 42 | 43 | compress = true 44 | parallelism = 2 45 | max_files = 100 46 | max_compactions = 1 47 | max_flushes = 1 48 | 49 | [store.fst] 50 | 51 | path = "./data/store/fst/" 52 | 53 | [store.fst.pool] 54 | 55 | inactive_after = 300 56 | 57 | [store.fst.graph] 58 | 59 | consolidate_after = 180 -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | go: 3 | - 1.12.x 4 | addons: 5 | apt: 6 | packages: 7 | - docker-ce 8 | env: 9 | global: 10 | - DOCKER_COMPOSE_VERSION=1.22.0 11 | - CLOUDSDK_CORE_DISABLE_PROMPTS=1 12 | git: 13 | depth: 3 14 | branches: 15 | only: 16 | - master 17 | before_install: 18 | - curl -L -s https://github.com/golang/dep/releases/download/v0.5.0/dep-linux-amd64 -o $GOPATH/bin/dep 19 | - chmod +x $GOPATH/bin/dep 20 | - sudo rm /usr/local/bin/docker-compose 21 | - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname 22 | -s`-`uname -m` > docker-compose 23 | - chmod +x docker-compose 24 | - sudo mv docker-compose /usr/local/bin 25 | - docker-compose up -d sonic 26 | - export BRANCH=$(if [ "$TRAVIS_PULL_REQUEST" == "false" ]; then echo $TRAVIS_BRANCH; 27 | else echo $TRAVIS_PULL_REQUEST_BRANCH; fi) 28 | - export TAG=$(if [ -n "$TRAVIS_TAG" ]; then echo $TRAVIS_TAG; else echo $BRANCH; 29 | fi) 30 | - echo "TRAVIS_BRANCH=$TRAVIS_BRANCH, PR=$PR, BRANCH=$BRANCH, TAG=$TAG" 31 | script: 32 | - env GO111MODULE=on go test ./... --count 1 --race 33 | -------------------------------------------------------------------------------- /sonic/client_test.go: -------------------------------------------------------------------------------- 1 | package sonic 2 | 3 | import ( 4 | "context" 5 | "github.com/stretchr/testify/assert" 6 | "log" 7 | "testing" 8 | ) 9 | 10 | func TestClient_Close(t *testing.T) { 11 | tests := []struct { 12 | name string 13 | wantErr bool 14 | }{ 15 | { 16 | name: "main", 17 | }, 18 | } 19 | for _, tt := range tests { 20 | t.Run(tt.name, func(t *testing.T) { 21 | c, err := NewClientWithPassword("localhost:1491", "SecretPassword", context.Background()) 22 | if !assert.NoError(t, err) { 23 | return 24 | } 25 | 26 | err = c.Close() 27 | 28 | if !tt.wantErr && !assert.NoError(t, err) { 29 | return 30 | } 31 | 32 | if tt.wantErr && !assert.Error(t, err) { 33 | return 34 | } 35 | }) 36 | } 37 | } 38 | 39 | func TestReconnect(t *testing.T) { 40 | c, err := NewClientWithPassword("localhost:1491", "SecretPassword", context.Background()) 41 | if !assert.NoError(t, err ) { 42 | return 43 | } 44 | 45 | err = c.reconnect(context.Background()) 46 | if !assert.NoError(t, err) { 47 | return 48 | } 49 | 50 | err = c.SearchService.Ping(context.Background()) 51 | if !assert.NoError(t, err) { 52 | return 53 | } 54 | } 55 | 56 | func ExampleNewClientWithPassword() { 57 | c, err := NewClientWithPassword("localhost:1491", "SecretPassword", context.Background()) 58 | if err != nil { 59 | panic(err) 60 | } 61 | 62 | // Ingest 63 | _, err = c.IngestService.Push(NewDataBuilder().Text("my string").Bucket("my bucket").Collection("my collection").Object("my object").Build()) 64 | if err != nil { 65 | panic(err) 66 | } 67 | 68 | // Search 69 | ch, err := c.SearchService.Query(context.Background(), NewDataBuilder().Collection("my collection").Bucket("my bucket").Text("my string").Build(), 0, 0) 70 | for e := range ch { 71 | log.Println(e) 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= 2 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 4 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 5 | github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= 6 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 7 | github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= 8 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 9 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 10 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 11 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 12 | github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= 13 | github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= 14 | github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= 15 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 16 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 17 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 18 | github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= 19 | github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= 20 | github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k= 21 | github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= 22 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 23 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 24 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 25 | github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= 26 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 27 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33 h1:I6FyU15t786LL7oL/hn43zqTuEGr4PN7F4XJ1p4E3Y8= 28 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 29 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= 30 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 31 | -------------------------------------------------------------------------------- /sonic/search_service_test.go: -------------------------------------------------------------------------------- 1 | package sonic 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/opentracing/opentracing-go" 7 | uuid "github.com/satori/go.uuid" 8 | "github.com/sirupsen/logrus" 9 | "github.com/stretchr/testify/assert" 10 | "testing" 11 | ) 12 | 13 | func TestSearchService_Query(t *testing.T) { 14 | type args struct { 15 | data *Data 16 | offset int 17 | limit int 18 | } 19 | tests := []struct { 20 | name string 21 | args args 22 | wantErr bool 23 | result int 24 | }{ 25 | { 26 | name: "query no result", 27 | args: args{ 28 | data: NewDataBuilder().Collection(uuid.NewV4().String()).Bucket(uuid.NewV4().String()).Text(uuid.NewV4().String()).Build(), 29 | }, 30 | result: 0, 31 | }, 32 | { 33 | name: "query more then 1 result", 34 | // need to flush buckets before running tests 35 | args: args{ 36 | data: NewDataBuilder().Collection("col2").Bucket("buc1").Text("magical").Build(), 37 | }, 38 | result: 2, 39 | }, 40 | { 41 | name: "query 1 result", 42 | // need to flush buckets before running tests 43 | args: args{ 44 | data: NewDataBuilder().Collection("col2").Bucket("buc2").Text("magical").Build(), 45 | }, 46 | result: 1, 47 | }, 48 | } 49 | 50 | c, err := NewClientWithPassword("localhost:1491", "SecretPassword", context.Background()) 51 | if !assert.NoError(t, err) { 52 | return 53 | } 54 | 55 | if !assert.True(t, beforePush4(t, c)) { 56 | return 57 | } 58 | 59 | t.Parallel() 60 | 61 | for _, tt := range tests { 62 | t.Run(tt.name, func(t *testing.T) { 63 | logrus.Debug("attempting reconnect") 64 | if !assert.NoError(t, c.reconnect(context.Background())) { 65 | return 66 | } 67 | logrus.Debug("reconnect finished") 68 | 69 | logrus.Debug("sending ping") 70 | if !assert.NoError(t, c.SearchService.Ping(context.Background())) { 71 | return 72 | } 73 | logrus.Debug("ping sent") 74 | 75 | sp, ctx := opentracing.StartSpanFromContext(context.Background(), t.Name()) 76 | defer sp.Finish() 77 | got, err := c.SearchService.Query(ctx, tt.args.data, tt.args.offset, tt.args.limit) 78 | if !tt.wantErr && !assert.NoError(t, err) { 79 | return 80 | } 81 | 82 | if tt.wantErr && !assert.Error(t, err) { 83 | return 84 | } 85 | 86 | c := 0 87 | for range got { 88 | c++ 89 | } 90 | 91 | if !assert.True(t, tt.result <= c, fmt.Sprintf("want %d got %d", tt.result, c)) { 92 | return 93 | } 94 | }) 95 | } 96 | } 97 | 98 | func TestSearchService_Suggest(t *testing.T) { 99 | type args struct { 100 | data *Data 101 | limit int 102 | } 103 | tests := []struct { 104 | name string 105 | args args 106 | want chan string 107 | wantErr bool 108 | }{ 109 | { 110 | name: "suggest with result", 111 | args: args{ 112 | data: NewDataBuilder().Collection("col2").Bucket("buc1").Text("som").Build(), 113 | limit: 0, 114 | }, 115 | }, 116 | { 117 | name: "suggest in empty bucket", 118 | args: args{ 119 | data: NewDataBuilder().Collection("test").Bucket(uuid.NewV4().String()).Text("awe").Build(), 120 | limit: 0, 121 | }, 122 | }, 123 | } 124 | 125 | c, err := NewClientWithPassword("localhost:1491", "SecretPassword", context.Background()) 126 | if !assert.NoError(t, err) { 127 | return 128 | } 129 | 130 | if !assert.True(t, beforePush4(t, c)) { 131 | return 132 | } 133 | 134 | t.Parallel() 135 | 136 | for _, tt := range tests { 137 | t.Run(tt.name, func(t *testing.T) { 138 | if !assert.NoError(t, c.reconnect(context.Background())) { 139 | return 140 | } 141 | 142 | if !assert.NoError(t, c.SearchService.Ping(context.Background())) { 143 | return 144 | } 145 | 146 | got, err := c.SearchService.Suggest(context.Background(), tt.args.data, tt.args.limit) 147 | if tt.wantErr && !assert.NoError(t, err) { 148 | return 149 | } 150 | 151 | for e := range got { 152 | if !assert.NotEqual(t, "", e) { 153 | return 154 | } 155 | } 156 | }) 157 | } 158 | } 159 | -------------------------------------------------------------------------------- /sonic/client.go: -------------------------------------------------------------------------------- 1 | package sonic 2 | 3 | import ( 4 | "context" 5 | "github.com/opentracing/opentracing-go" 6 | "github.com/pkg/errors" 7 | "io" 8 | "net" 9 | "sync" 10 | "time" 11 | ) 12 | 13 | // Data is a generic struct that can be used for all the "queries" 14 | type Data struct { 15 | Collection string 16 | Bucket string 17 | Object string 18 | Text string 19 | } 20 | 21 | type Client struct { 22 | s net.Conn 23 | i net.Conn 24 | 25 | ctx context.Context 26 | 27 | address string 28 | password string 29 | 30 | IngestService IngestService 31 | SearchService SearchService 32 | } 33 | 34 | // SetDeadline sets the read and write deadlines associated 35 | // with the connection. It is equivalent to calling both 36 | // SetReadDeadline and SetWriteDeadline. 37 | // 38 | // A deadline is an absolute time after which I/O operations 39 | // fail with a timeout (see type Error) instead of 40 | // blocking. The deadline applies to all future and pending 41 | // I/O, not just the immediately following call to Read or 42 | // Write. After a deadline has been exceeded, the connection 43 | // can be refreshed by setting a deadline in the future. 44 | // 45 | // An idle timeout can be implemented by repeatedly extending 46 | // the deadline after successful Read or Write calls. 47 | // 48 | // A zero value for t means I/O operations will not time out. 49 | func (c *Client) SetDeadline(t time.Time) error { 50 | if err := c.s.SetDeadline(t); err != nil { 51 | return errors.Wrap(err, "could not set deadline for search connection") 52 | } 53 | 54 | if err := c.i.SetDeadline(t); err != nil { 55 | return errors.Wrap(err, "could not set deadline for ingest connection") 56 | } 57 | 58 | return nil 59 | } 60 | 61 | // SetReadDeadline sets the deadline for future Read calls 62 | // and any currently-blocked Read call. 63 | // A zero value for t means Read will not time out. 64 | func (c *Client) SetReadDeadline(t time.Time) error { 65 | if err := c.s.SetReadDeadline(t); err != nil { 66 | return errors.Wrap(err, "could not set read deadline for search connection") 67 | } 68 | 69 | if err := c.i.SetReadDeadline(t); err != nil { 70 | return errors.Wrap(err, "could not set read deadline for ingest connection") 71 | } 72 | 73 | return nil 74 | } 75 | 76 | // SetWriteDeadline sets the deadline for future Write calls 77 | // and any currently-blocked Write call. 78 | // Even if write times out, it may return n > 0, indicating that 79 | // some of the data was successfully written. 80 | // A zero value for t means Write will not time out. 81 | func (c *Client) SetWriteDeadline(t time.Time) error { 82 | if err := c.s.SetWriteDeadline(t); err != nil { 83 | return errors.Wrap(err, "could not set write deadline for search connection") 84 | } 85 | 86 | if err := c.i.SetWriteDeadline(t); err != nil { 87 | return errors.Wrap(err, "could not set write deadline for ingest connection") 88 | } 89 | 90 | return nil 91 | } 92 | 93 | func (c *Client) Close() error { 94 | _, err := io.WriteString(c.s, "QUIT\n") 95 | if err != nil { 96 | return errors.Wrap(err, "could not signal server to close search connection") 97 | } 98 | 99 | err = c.s.Close() 100 | if err != nil { 101 | return errors.Wrap(err, "could not close search connection") 102 | } 103 | 104 | _, err = io.WriteString(c.i, "QUIT\n") 105 | if err != nil { 106 | return errors.Wrap(err, "could not signal server to close ingest connection") 107 | } 108 | 109 | err = c.i.Close() 110 | if err != nil { 111 | return errors.Wrap(err, "could not close ingest connection") 112 | } 113 | 114 | return nil 115 | } 116 | 117 | func NewClientWithPassword(address, password string, ctx context.Context) (*Client, error) { 118 | i, err := net.Dial("tcp", address) 119 | if err != nil { 120 | return nil, errors.Wrapf(err, "could not open connection to %q", address) 121 | } 122 | 123 | s, err := net.Dial("tcp", address) 124 | if err != nil { 125 | return nil, errors.Wrapf(err, "could not open connection to %q", address) 126 | } 127 | 128 | client := Client{i: i, s: s} 129 | 130 | client.password = password 131 | client.address = address 132 | client.ctx = ctx 133 | 134 | client.IngestService, err = newIngestService(ctx, &client) 135 | if err != nil { 136 | return nil, errors.Wrap(err, "could not create ingest service") 137 | } 138 | 139 | client.SearchService, err = newSearchService(ctx, &client) 140 | if err != nil { 141 | return nil, errors.Wrap(err, "could not create search service") 142 | } 143 | 144 | return &client, nil 145 | } 146 | 147 | func NewNoOpsClient(ctx context.Context) *Client { 148 | return &Client{ 149 | ctx: ctx, 150 | IngestService: &NoOpsIngestService{}, 151 | SearchService: &NoOpsSearchService{}, 152 | } 153 | } 154 | 155 | func (c *Client) reconnect(ctx context.Context) error { 156 | sp, ctx := opentracing.StartSpanFromContext(ctx, "sonic-reconnect") 157 | defer sp.Finish() 158 | 159 | var err error 160 | 161 | c.i, err = net.Dial("tcp", c.address) 162 | if err != nil { 163 | return errors.Wrapf(err, "could not open connection to %q", c.address) 164 | } 165 | 166 | c.s, err = net.Dial("tcp", c.address) 167 | if err != nil { 168 | return errors.Wrapf(err, "could not open connection to %q", c.address) 169 | } 170 | 171 | var wg sync.WaitGroup 172 | wg.Add(2) 173 | errCh := make(chan error, 2) 174 | 175 | go func() { 176 | defer wg.Done() 177 | 178 | err := c.IngestService.connect(ctx) 179 | if err != nil { 180 | errCh <- errors.Wrap(err, "could not create ingest service") 181 | } 182 | }() 183 | 184 | go func() { 185 | defer wg.Done() 186 | 187 | err := c.SearchService.connect(ctx) 188 | if err != nil { 189 | errCh <- errors.Wrap(err, "could not create search service") 190 | } 191 | }() 192 | 193 | wg.Wait() 194 | close(errCh) 195 | 196 | err = <- errCh 197 | if err != nil { 198 | return errors.Wrap(err, "could not reconnect to sonic") 199 | } 200 | 201 | return nil 202 | } 203 | 204 | // Data builder pattern code 205 | type DataBuilder struct { 206 | data *Data 207 | } 208 | 209 | func NewDataBuilder() *DataBuilder { 210 | data := &Data{} 211 | b := &DataBuilder{data: data} 212 | return b 213 | } 214 | 215 | // Collection see https://github.com/valeriansaliou/sonic/blob/master/PROTOCOL.md for terminology explanation 216 | func (b *DataBuilder) Collection(collection string) *DataBuilder { 217 | b.data.Collection = collection 218 | return b 219 | } 220 | 221 | // Bucket see https://github.com/valeriansaliou/sonic/blob/master/PROTOCOL.md for terminology explanation 222 | func (b *DataBuilder) Bucket(bucket string) *DataBuilder { 223 | b.data.Bucket = bucket 224 | return b 225 | } 226 | 227 | // Object see https://github.com/valeriansaliou/sonic/blob/master/PROTOCOL.md for terminology explanation 228 | func (b *DataBuilder) Object(object string) *DataBuilder { 229 | b.data.Object = object 230 | return b 231 | } 232 | 233 | // Text or Word see https://github.com/valeriansaliou/sonic/blob/master/PROTOCOL.md for terminology explanation 234 | func (b *DataBuilder) Text(text string) *DataBuilder { 235 | b.data.Text = text 236 | return b 237 | } 238 | 239 | func (b *DataBuilder) Build() *Data { 240 | return b.data 241 | } 242 | -------------------------------------------------------------------------------- /sonic/ingest_service.go: -------------------------------------------------------------------------------- 1 | package sonic 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "context" 7 | "fmt" 8 | "github.com/opentracing/opentracing-go" 9 | "github.com/pkg/errors" 10 | "io" 11 | "strconv" 12 | "sync" 13 | ) 14 | 15 | type IngestService interface { 16 | Push(data *Data) (bool, error) 17 | Pop(data *Data) (int, error) 18 | Count(data *Data) (int, error) 19 | Flushc(data *Data) (int, error) 20 | Flushb(data *Data) (int, error) 21 | Flusho(data *Data) (int, error) 22 | 23 | connect(ctx context.Context) error 24 | } 25 | 26 | // NoOpsIngestService is an IngestService that does no operations on the methods it implements. 27 | type NoOpsIngestService struct { 28 | 29 | } 30 | 31 | func (*NoOpsIngestService) Push(data *Data) (bool, error) { 32 | return true, nil 33 | } 34 | 35 | func (*NoOpsIngestService) Pop(data *Data) (int, error) { 36 | return 0, nil 37 | } 38 | 39 | func (*NoOpsIngestService) Count(data *Data) (int, error) { 40 | return 0, nil 41 | } 42 | 43 | func (*NoOpsIngestService) Flushc(data *Data) (int, error) { 44 | return 0, nil 45 | } 46 | 47 | func (*NoOpsIngestService) Flushb(data *Data) (int, error) { 48 | return 0, nil 49 | } 50 | 51 | func (*NoOpsIngestService) Flusho(data *Data) (int, error) { 52 | return 0, nil 53 | } 54 | 55 | func (*NoOpsIngestService) connect(ctx context.Context) error { 56 | return nil 57 | } 58 | 59 | // IngestService exposes the ingest mode of sonic 60 | type ingestService struct { 61 | c *Client 62 | 63 | l sync.Mutex 64 | s *bufio.Scanner 65 | } 66 | 67 | func newIngestService(ctx context.Context, c *Client) (IngestService, error) { 68 | sp, ctx := opentracing.StartSpanFromContext(ctx, "sonic-newIngestService") 69 | defer sp.Finish() 70 | 71 | i := &ingestService{c: c} 72 | 73 | return i, errors.Wrap(i.connect(ctx), "could not connect to ingest service") 74 | } 75 | 76 | func (i *ingestService) connect(ctx context.Context) error { 77 | sp, ctx := opentracing.StartSpanFromContext(ctx, "sonic-ingest-connect") 78 | defer sp.Finish() 79 | 80 | s := bufio.NewScanner(i.c.i) 81 | i.s =s 82 | 83 | _, err := io.WriteString(i.c.i, fmt.Sprintf("START ingest %s\n", i.c.password)) 84 | if err != nil { 85 | return errors.Wrap(err, "could not start ingest connection") 86 | } 87 | 88 | parse: 89 | i.s.Scan() 90 | w := bufio.NewScanner(bytes.NewBuffer(i.s.Bytes())) 91 | w.Split(bufio.ScanWords) 92 | w.Scan() 93 | 94 | switch w.Text() { 95 | case "STARTED": 96 | case "CONNECTED", "": 97 | goto parse 98 | case "ENDED": 99 | return errors.Errorf("failed to start ingest session: %q", i.s.Text()) 100 | default: 101 | return errors.Errorf("could not determine how to interpret %q response", i.s.Text()) 102 | } 103 | 104 | return nil 105 | } 106 | 107 | // Push search data in the index 108 | func (i *ingestService) Push(data *Data) (bool, error) { 109 | if data.Collection == "" || data.Bucket == "" || data.Object == "" || data.Text == "" { 110 | return false, errors.New("all ingest data are required for pushing") 111 | } 112 | 113 | i.l.Lock() 114 | defer i.l.Unlock() 115 | _, err := io.WriteString(i.c.i, fmt.Sprintf("PUSH %s %s %s %q\n", data.Collection, data.Bucket, data.Object, data.Text)) 116 | if err != nil { 117 | return false, errors.Wrap(err, "pushing data failed") 118 | } 119 | 120 | i.s.Scan() 121 | 122 | w := bufio.NewScanner(bytes.NewBuffer(i.s.Bytes())) 123 | w.Split(bufio.ScanWords) 124 | w.Scan() 125 | 126 | switch w.Text() { 127 | case "OK": 128 | default: 129 | return false, errors.Errorf("could not determine how to interpret response: %q", i.s.Text()) 130 | } 131 | 132 | return true, nil 133 | } 134 | 135 | // Pop search data from the index 136 | func (i *ingestService) Pop(data *Data) (int, error) { 137 | if data.Collection == "" || data.Bucket == "" || data.Object == "" || data.Text == "" { 138 | return 0, errors.New("all ingest data are required for pushing") 139 | } 140 | 141 | i.l.Lock() 142 | defer i.l.Unlock() 143 | _, err := io.WriteString(i.c.i, fmt.Sprintf("POP %s %s %s %q\n", data.Collection, data.Bucket, data.Object, data.Text)) 144 | if err != nil { 145 | return 0, errors.Wrap(err, "popping data failed") 146 | } 147 | 148 | i.s.Scan() 149 | w := bufio.NewScanner(bytes.NewBuffer(i.s.Bytes())) 150 | w.Split(bufio.ScanWords) 151 | w.Scan() 152 | 153 | switch w.Text() { 154 | case "RESULT": 155 | default: 156 | return 0, errors.Errorf("could not determine how to interpret %q", i.s.Text()) 157 | } 158 | 159 | w.Scan() 160 | c, err := strconv.Atoi(w.Text()) 161 | if err != nil { 162 | return 0, errors.Wrapf(err, "could not parse count result to int: %q", i.s.Text()) 163 | } 164 | 165 | return c, nil 166 | } 167 | 168 | // Count indexed search data 169 | func (i *ingestService) Count(data *Data) (int, error) { 170 | if data.Collection == "" { 171 | return 0, errors.New("collection can not be an empty string") 172 | } 173 | 174 | args := []interface{}{data.Collection} 175 | sfmt := "COUNT %s" 176 | 177 | if data.Bucket != "" { 178 | sfmt += " %s" 179 | args = append(args, data.Bucket) 180 | } 181 | 182 | if data.Object != "" { 183 | sfmt += " %s" 184 | args = append(args, data.Object) 185 | } 186 | 187 | i.l.Lock() 188 | defer i.l.Unlock() 189 | _, err := io.WriteString(i.c.i, fmt.Sprintf(fmt.Sprintf("%s\n", sfmt), args...)) 190 | if err != nil { 191 | return 0, errors.Wrap(err, "popping data failed") 192 | } 193 | 194 | if !i.s.Scan() { 195 | return 0, errors.Wrap(i.s.Err(), "could not scan count response from server") 196 | } 197 | 198 | s := bufio.NewScanner(bytes.NewBuffer(i.s.Bytes())) 199 | s.Split(bufio.ScanWords) 200 | if !s.Scan() { 201 | return 0, errors.New("could not scan result") 202 | } 203 | 204 | switch s.Text() { 205 | case "RESULT": 206 | default: 207 | return 0, errors.Errorf("could not determine how to interpret %q", i.s.Text()) 208 | } 209 | 210 | s.Scan() 211 | 212 | c, err := strconv.Atoi(s.Text()) 213 | if err != nil { 214 | return 0, errors.Wrap(err, "could not parse count result to int") 215 | } 216 | 217 | return c, nil 218 | } 219 | 220 | func (i *ingestService) flush(query string) (int, error) { 221 | i.l.Lock() 222 | defer i.l.Unlock() 223 | _, err := io.WriteString(i.c.i, query) 224 | if err != nil { 225 | return 0, errors.Wrap(err, "flushing collection failed") 226 | } 227 | 228 | if !i.s.Scan() { 229 | return 0, errors.Wrap(i.s.Err(), "could not scan count response from server") 230 | } 231 | 232 | w := bufio.NewScanner(bytes.NewBuffer(i.s.Bytes())) 233 | w.Split(bufio.ScanWords) 234 | w.Scan() 235 | 236 | switch w.Text() { 237 | case "RESULT": 238 | default: 239 | return 0, errors.Errorf("could not determine how to interpret %q", i.s.Text()) 240 | } 241 | 242 | w.Scan() 243 | 244 | c, err := strconv.Atoi(w.Text()) 245 | if err != nil { 246 | return 0, errors.Wrap(err, "could not parse count result to int") 247 | } 248 | 249 | return c, nil 250 | } 251 | 252 | // Flushc Flush all indexed data from a collection 253 | func (i *ingestService) Flushc(data *Data) (int, error) { 254 | if data.Collection == "" { 255 | return 0, errors.New("collection can not be an empty string") 256 | } 257 | 258 | return i.flush(fmt.Sprintf("FLUSHC %s\n", data.Collection)) 259 | } 260 | 261 | // Flushb Flush all indexed data from a bucket in a collection 262 | func (i *ingestService) Flushb(data *Data) (int, error) { 263 | if data.Collection == "" || data.Bucket == "" { 264 | return 0, errors.New("collection and bucket can not be an empty strings") 265 | } 266 | 267 | return i.flush(fmt.Sprintf("FLUSHB %s %s\n", data.Collection, data.Bucket)) 268 | } 269 | 270 | // Flusho Flush all indexed data from an object in a bucket in collection 271 | func (i *ingestService) Flusho(data *Data) (int, error) { 272 | if data.Collection == "" || data.Bucket == "" || data.Object == "" { 273 | return 0, errors.New("collection, bucket and object can not be an empty strings") 274 | } 275 | 276 | return i.flush(fmt.Sprintf("FLUSHO %s %s %s\n", data.Collection, data.Bucket, data.Object)) 277 | } 278 | -------------------------------------------------------------------------------- /sonic/ingest_service_test.go: -------------------------------------------------------------------------------- 1 | package sonic 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "testing" 7 | 8 | "github.com/satori/go.uuid" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestIngestService_Count(t *testing.T) { 13 | c, err := NewClientWithPassword("localhost:1491", "SecretPassword", context.Background()) 14 | 15 | if !assert.NoError(t, err) { 16 | return 17 | } 18 | 19 | type fields struct { 20 | c *Client 21 | } 22 | type args struct { 23 | data []*Data 24 | } 25 | tests := []struct { 26 | name string 27 | fields fields 28 | args args 29 | want int 30 | wantErr bool 31 | }{ 32 | { 33 | name: "main", 34 | fields: fields{c: c}, 35 | args: args{ 36 | data: []*Data{ 37 | NewDataBuilder().Collection("c").Bucket("b").Object(uuid.NewV4().String()).Text(uuid.NewV4().String()).Build(), 38 | }, 39 | }, 40 | }, 41 | { 42 | name: "2 objects", 43 | fields: fields{c: c}, 44 | args: args{ 45 | data: []*Data{ 46 | NewDataBuilder().Collection("c").Bucket("b").Object(uuid.FromStringOrNil("568A42A3-7EBF-4243-94D7-7057122CDAF8").String()).Text(uuid.NewV4().String()).Build(), 47 | NewDataBuilder().Collection("c").Bucket("b").Object(uuid.FromStringOrNil("568A42A3-7EBF-4243-94D7-7057122CDAF8").String()).Text(uuid.NewV4().String()).Build(), 48 | }, 49 | }, 50 | }, 51 | } 52 | 53 | t.Parallel() 54 | 55 | for _, tt := range tests { 56 | t.Run(tt.name, func(t *testing.T) { 57 | 58 | for _, x := range tt.args.data { 59 | ok, err := c.IngestService.Push(x) 60 | if !assert.NoError(t, err) { 61 | return 62 | } 63 | if !assert.True(t, ok) { 64 | return 65 | } 66 | } 67 | 68 | got, err := c.IngestService.Count(tt.args.data[0]) 69 | if (err != nil) != tt.wantErr { 70 | assert.NoError(t, err) 71 | return 72 | } 73 | if !assert.True(t, len(tt.args.data)*5 <= got) { 74 | return 75 | } 76 | }) 77 | } 78 | } 79 | 80 | func TestIngestService_Push(t *testing.T) { 81 | c, err := NewClientWithPassword("localhost:1491", "SecretPassword", context.Background()) 82 | if !assert.NoError(t, err) { 83 | return 84 | } 85 | 86 | type fields struct { 87 | c *Client 88 | } 89 | type args struct { 90 | data *Data 91 | } 92 | tests := []struct { 93 | name string 94 | fields fields 95 | args args 96 | want bool 97 | wantErr bool 98 | }{ 99 | { 100 | name: "main", 101 | args: args{ 102 | data: NewDataBuilder().Collection(uuid.NewV4().String()).Bucket(uuid.NewV4().String()).Object(uuid.NewV4().String()).Text(uuid.NewV4().String()).Build(), 103 | }, 104 | fields: fields{ 105 | c: c, 106 | }, 107 | want: true, 108 | }, 109 | } 110 | 111 | t.Parallel() 112 | 113 | for _, tt := range tests { 114 | t.Run(tt.name, func(t *testing.T) { 115 | got, err := tt.fields.c.IngestService.Push(tt.args.data) 116 | if !tt.wantErr && !assert.NoError(t, err) { 117 | return 118 | } 119 | 120 | if !assert.Equal(t, tt.want, got) { 121 | return 122 | } 123 | }) 124 | } 125 | } 126 | 127 | func TestIngestService_Pop(t *testing.T) { 128 | c, err := NewClientWithPassword("localhost:1491", "SecretPassword", context.Background()) 129 | if !assert.NoError(t, err) { 130 | return 131 | } 132 | 133 | type fields struct { 134 | c *Client 135 | } 136 | type args struct { 137 | data *Data 138 | } 139 | tests := []struct { 140 | name string 141 | fields fields 142 | args args 143 | want int 144 | wantErr bool 145 | }{ 146 | { 147 | name: "nothing to pop", 148 | fields: fields{ 149 | c: c, 150 | }, 151 | args: args{ 152 | data: NewDataBuilder().Collection(uuid.NewV4().String()).Bucket(uuid.NewV4().String()).Object(uuid.NewV4().String()).Text(uuid.NewV4().String()).Build(), 153 | }, 154 | want: 0, 155 | }, 156 | } 157 | 158 | t.Parallel() 159 | 160 | for _, tt := range tests { 161 | t.Run(tt.name, func(t *testing.T) { 162 | got, err := tt.fields.c.IngestService.Pop(tt.args.data) 163 | if !tt.wantErr && !assert.NoError(t, err) { 164 | return 165 | } 166 | 167 | if !assert.Equal(t, tt.want, got) { 168 | return 169 | } 170 | }) 171 | } 172 | } 173 | 174 | func TestIngestService_Flush(t *testing.T) { 175 | c, err := NewClientWithPassword("localhost:1491", "SecretPassword", context.Background()) 176 | if !assert.NoError(t, err) { 177 | return 178 | } 179 | 180 | type fields struct { 181 | c *Client 182 | } 183 | type args struct { 184 | data *Data 185 | } 186 | tests := []struct { 187 | name string 188 | fields fields 189 | args args 190 | want int 191 | wantErr bool 192 | beforeFunc func(t *testing.T, c *Client) bool 193 | flushMethod func(data *Data) (int, error) 194 | }{ 195 | { 196 | name: "flushc with unknown collection", 197 | fields: fields{ 198 | c: c, 199 | }, 200 | args: args{ 201 | data: NewDataBuilder().Collection(uuid.NewV4().String()).Bucket(uuid.NewV4().String()).Object(uuid.NewV4().String()).Text(uuid.NewV4().String()).Build(), 202 | }, 203 | want: 0, 204 | flushMethod: c.IngestService.Flushc, 205 | }, 206 | { 207 | name: "flushc with 4 pushed items to 1 col", 208 | fields: fields{ 209 | c: c, 210 | }, 211 | args: args{ 212 | data: NewDataBuilder().Collection("col1").Bucket(uuid.NewV4().String()).Object(uuid.NewV4().String()).Text(uuid.NewV4().String()).Build(), 213 | }, 214 | want: 1, 215 | beforeFunc: beforePush4, 216 | flushMethod: c.IngestService.Flushc, 217 | }, 218 | { 219 | name: "flushb with unknown collection", 220 | fields: fields{ 221 | c: c, 222 | }, 223 | args: args{ 224 | data: NewDataBuilder().Collection("col1").Bucket(uuid.NewV4().String()).Object(uuid.NewV4().String()).Text(uuid.NewV4().String()).Build(), 225 | }, 226 | want: 0, 227 | flushMethod: c.IngestService.Flushb, 228 | }, 229 | { 230 | name: "flushb with 4 pushed items to 1 collection and bucket", 231 | fields: fields{ 232 | c: c, 233 | }, 234 | args: args{ 235 | data: NewDataBuilder().Collection("col1").Bucket("buc1").Object(uuid.NewV4().String()).Text(uuid.NewV4().String()).Build(), 236 | 237 | }, 238 | want: 34, 239 | beforeFunc: beforePush4, 240 | flushMethod: c.IngestService.Flushb, 241 | }, 242 | { 243 | name: "flusho with unknown collection", 244 | fields: fields{ 245 | c: c, 246 | }, 247 | args: args{ 248 | data: NewDataBuilder().Collection(uuid.NewV4().String()).Bucket(uuid.NewV4().String()).Object(uuid.NewV4().String()).Text(uuid.NewV4().String()).Build(), 249 | }, 250 | want: 0, 251 | flushMethod: c.IngestService.Flusho, 252 | }, 253 | { 254 | name: "flusho with 4 pushed items to 1 collection, bucket and object", 255 | fields: fields{ 256 | c: c, 257 | }, 258 | args: args{ 259 | data: NewDataBuilder().Collection("col1").Bucket("buc2").Object("obj1").Text(uuid.NewV4().String()).Build(), 260 | 261 | }, 262 | want: 41, 263 | beforeFunc: beforePush4, 264 | flushMethod: c.IngestService.Flusho, 265 | }, 266 | } 267 | 268 | t.Parallel() 269 | 270 | for _, tt := range tests { 271 | t.Run(tt.name, func(t *testing.T) { 272 | if tt.beforeFunc != nil { 273 | if !assert.True(t, tt.beforeFunc(t, tt.fields.c)) { 274 | return 275 | } 276 | } 277 | 278 | got, err := tt.flushMethod(tt.args.data) 279 | if !assert.NoError(t, err) != tt.wantErr { 280 | return 281 | } 282 | if !assert.True(t, tt.want + 5 > got, fmt.Sprintf("want %d, got %d",tt.want + 5, got)) { 283 | return 284 | } 285 | }) 286 | } 287 | } 288 | 289 | func beforePush4(t *testing.T, c *Client) bool { 290 | for i := 0; i < 4; i++ { 291 | _, err := c.IngestService.Push(NewDataBuilder().Collection("col1").Bucket("buc1").Object(uuid.NewV4().String()).Text(fmt.Sprintf("some magical text -- %s", uuid.NewV4())).Build()) 292 | if !assert.NoError(t, err) { 293 | return false 294 | } 295 | _, err = c.IngestService.Push(NewDataBuilder().Collection("col2").Bucket("buc1").Object(uuid.NewV4().String()).Text(fmt.Sprintf("some magical text -- %s", uuid.NewV4())).Build()) 296 | if !assert.NoError(t, err) { 297 | return false 298 | } 299 | _, err = c.IngestService.Push(NewDataBuilder().Collection("col1").Bucket("buc2").Object("obj1").Text(fmt.Sprintf("some magical text -- %s", uuid.NewV4())).Build()) 300 | if !assert.NoError(t, err) { 301 | return false 302 | } 303 | _, err = c.IngestService.Push(NewDataBuilder().Collection("col2").Bucket("buc2").Object("obj1").Text(fmt.Sprintf("some magical text -- %s", uuid.NewV4())).Build()) 304 | if !assert.NoError(t, err) { 305 | return false 306 | } 307 | } 308 | 309 | return true 310 | } 311 | -------------------------------------------------------------------------------- /sonic/search_service.go: -------------------------------------------------------------------------------- 1 | package sonic 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "context" 7 | "fmt" 8 | "github.com/opentracing/opentracing-go" 9 | opLog "github.com/opentracing/opentracing-go/log" 10 | "github.com/pkg/errors" 11 | "github.com/sirupsen/logrus" 12 | "io" 13 | "log" 14 | "net" 15 | "os" 16 | 17 | "sync" 18 | "time" 19 | ) 20 | 21 | type pendingQuery struct { 22 | ids chan string 23 | sp opentracing.Span 24 | } 25 | 26 | type SearchService interface { 27 | Suggest(ctx context.Context, data *Data, limit int) (chan string, error) 28 | Ping(ctx context.Context) error 29 | Query(ctx context.Context, data *Data, offset, limit int) (chan string, error) 30 | 31 | connect(ctx context.Context) error 32 | } 33 | 34 | // NoOpsSearchService Is a search service that preforms no operations. 35 | type NoOpsSearchService struct { 36 | 37 | } 38 | 39 | func (*NoOpsSearchService) connect(ctx context.Context) error { 40 | return nil 41 | } 42 | 43 | func (*NoOpsSearchService) Suggest(ctx context.Context, data *Data, limit int) (chan string, error) { 44 | ch := make(chan string) 45 | close(ch) 46 | 47 | return ch, nil 48 | } 49 | 50 | func (*NoOpsSearchService) Ping(ctx context.Context) error { 51 | return nil 52 | } 53 | 54 | func (*NoOpsSearchService) Query(ctx context.Context, data *Data, offset, limit int) (chan string, error) { 55 | ch := make(chan string) 56 | close(ch) 57 | 58 | return ch, nil 59 | } 60 | 61 | // searchService exposes the search mode of sonic 62 | type searchService struct { 63 | c *Client 64 | 65 | sl sync.Mutex 66 | s *bufio.Scanner 67 | 68 | pl sync.RWMutex 69 | pending map[string]*pendingQuery 70 | 71 | onePoll sync.Once 72 | 73 | ctx context.Context 74 | } 75 | 76 | func newSearchService(ctx context.Context, c *Client) (SearchService, error) { 77 | sp, ctx := opentracing.StartSpanFromContext(ctx, "sonic-newSearchService") 78 | defer sp.Finish() 79 | 80 | ss := &searchService{c: c, pending: make(map[string]*pendingQuery), ctx: c.ctx} 81 | 82 | err := ss.connect(ctx) 83 | if err != nil { 84 | return nil, errors.Wrap(err, "could not connect to search service") 85 | } 86 | 87 | ss.keepAlive() 88 | 89 | return ss, nil 90 | } 91 | 92 | func (s *searchService) connect(ctx context.Context) error { 93 | sp, ctx := opentracing.StartSpanFromContext(ctx, "sonic-search-connect") 94 | defer sp.Finish() 95 | 96 | s.sl.Lock() 97 | defer s.sl.Unlock() 98 | 99 | scanner := bufio.NewScanner(s.c.s) 100 | s.s = scanner 101 | 102 | _, err := io.WriteString(s.c.s, fmt.Sprintf("START search %s\n", s.c.password)) 103 | if err != nil { 104 | return errors.Wrap(err, "could not start search connection") 105 | } 106 | 107 | s.s.Scan() 108 | 109 | parse: 110 | w := bufio.NewScanner(bytes.NewBuffer(s.s.Bytes())) 111 | w.Split(bufio.ScanWords) 112 | w.Scan() 113 | 114 | switch w.Text() { 115 | case "STARTED": 116 | case "CONNECTED": 117 | s.s.Scan() 118 | goto parse 119 | case "ENDED": 120 | return errors.Errorf("failed to start search session: %q", s.s.Text()) 121 | default: 122 | return errors.Errorf("could not determine how to interpret %q response", s.s.Text()) 123 | } 124 | 125 | return nil 126 | } 127 | 128 | func (s *searchService) keepAlive() { 129 | go func() { 130 | ticker := time.Tick(time.Second * 5) 131 | for { 132 | select { 133 | case <-s.ctx.Done(): 134 | return 135 | case <-ticker: 136 | func() { 137 | logrus.Debugf("starting span sonic-keepAlive") 138 | sp, ctx := opentracing.StartSpanFromContext(context.Background(), "sonic-keepAlive") 139 | defer sp.Finish() 140 | 141 | logrus.Debugf("sending ping from keep alive") 142 | err := s.Ping(ctx) 143 | logrus.Debugf("ping sent") 144 | 145 | s.pollForEvents() 146 | if err != nil { 147 | sp.LogFields(opLog.Error(err)) 148 | logrus.WithError(err).Error("error while pinging sonic") 149 | } 150 | }() 151 | } 152 | } 153 | }() 154 | } 155 | 156 | func (s *searchService) pollForEvents() { 157 | s.onePoll.Do(func() { 158 | t := time.NewTicker(time.Millisecond * 100) 159 | go func() { 160 | defer t.Stop() 161 | for { 162 | select { 163 | case <-s.ctx.Done(): 164 | return 165 | case <-t.C: 166 | func() { 167 | s.pl.RLock() 168 | shouldPoll := len(s.pending) > 0 169 | s.pl.RUnlock() 170 | if !shouldPoll { 171 | return 172 | } 173 | 174 | sp, ctx := opentracing.StartSpanFromContext(context.Background(), "sonic-pollForEvents") 175 | defer sp.Finish() 176 | 177 | lsp, _ := opentracing.StartSpanFromContext(ctx, "acquiring lock") 178 | lsp.SetTag("line", "sonic/search_service.go:115") 179 | logrus.Debug("event poller getting lock") 180 | s.sl.Lock() 181 | logrus.Debug("event poller got lock") 182 | lsp.Finish() 183 | defer func() { 184 | defer s.sl.Unlock() 185 | logrus.Debug("event poller releasing lock") 186 | }() 187 | 188 | ssp, _ := opentracing.StartSpanFromContext(ctx, "scanning main scanner") 189 | scanned := s.s.Scan() 190 | ssp.LogFields(opLog.Bool("scanned", scanned)) 191 | ssp.Finish() 192 | 193 | sp.SetTag("scanned", scanned) 194 | 195 | if !scanned { 196 | return 197 | } 198 | 199 | sp.LogFields(opLog.String("scanned value", s.s.Text())) 200 | 201 | w := bufio.NewScanner(bytes.NewBuffer(s.s.Bytes())) 202 | w.Split(bufio.ScanWords) 203 | w.Scan() 204 | 205 | switch w.Text() { 206 | case "EVENT": 207 | go s.handleEvent(ctx, s.s.Text()) 208 | case "", "PONG": 209 | // do nothing 210 | default: 211 | log.Panicf("event poller managed to get/intercept a non event response: %q", s.s.Text()) 212 | } 213 | }() 214 | } 215 | } 216 | }() 217 | }) 218 | } 219 | 220 | // Suggest auto-completes word 221 | func (s *searchService) Suggest(ctx context.Context, data *Data, limit int) (chan string, error) { 222 | sp, ctx := opentracing.StartSpanFromContext(ctx, "sonic-Suggest") 223 | defer sp.Finish() 224 | 225 | if data.Collection == "" || data.Bucket == "" { 226 | return nil, errors.New("collection and bucket should not be empty for suggest") 227 | } 228 | 229 | query := fmt.Sprintf("SUGGEST %s %s %q", data.Collection, data.Bucket, data.Text) 230 | 231 | if limit != 0 { 232 | query += fmt.Sprintf(" LIMIT(%d)", limit) 233 | } 234 | 235 | lsp := sp.Tracer().StartSpan("acquiring lock", opentracing.ChildOf(sp.Context())) 236 | s.sl.Lock() 237 | lsp.Finish() 238 | defer s.sl.Unlock() 239 | 240 | _, err := io.WriteString(s.c.s, fmt.Sprintf("%s\n", query)) 241 | if err != nil { 242 | return nil, errors.Wrap(err, "querying data for suggestion failed") 243 | } 244 | 245 | ch, err := s.parseResponse(ctx) 246 | if err != nil { 247 | return nil, errors.Wrap(err, "could not parse response for suggest") 248 | } 249 | 250 | return ch, nil 251 | } 252 | 253 | func (s *searchService) Ping(ctx context.Context) error { 254 | sp, ctx := opentracing.StartSpanFromContext(ctx, "sonic-Ping") 255 | defer sp.Finish() 256 | 257 | reconnect := false 258 | lsp, _ := opentracing.StartSpanFromContext(ctx, "acquiring lock") 259 | s.sl.Lock() 260 | defer s.sl.Unlock() 261 | lsp.Finish() 262 | 263 | ping: 264 | _, err := io.WriteString(s.c.s, fmt.Sprintf("%s\n", "PING")) 265 | if err != nil { 266 | if err, ok := err.(*net.OpError); ok && !reconnect { 267 | if _, ok := err.Err.(*os.SyscallError); ok && !reconnect { 268 | sp.LogFields(opLog.Bool("reconnect", true)) 269 | reconnect = true 270 | err := s.c.reconnect(ctx) 271 | if err != nil { 272 | return errors.Wrap(err, "could not reconnect to sonic") 273 | } 274 | goto ping 275 | } 276 | } 277 | return errors.Wrap(err, "pinging sonic failed") 278 | } 279 | 280 | ssp, _ := opentracing.StartSpanFromContext(ctx, "scanning main scanner") 281 | ssp.SetTag("scanned", s.s.Scan()) 282 | ssp.LogFields(opLog.String("scanned", s.s.Text())) 283 | ssp.Finish() 284 | sp.LogFields(opLog.Bool("reconnect", false)) 285 | 286 | return nil 287 | } 288 | 289 | // Query query database 290 | func (s *searchService) Query(ctx context.Context, data *Data, offset, limit int) (chan string, error) { 291 | logrus.Debug("preforming query") 292 | defer logrus.Debug("done performing query") 293 | 294 | sp, ctx := opentracing.StartSpanFromContext(ctx, "sonic-Query") 295 | defer sp.Finish() 296 | 297 | if data.Collection == "" || data.Bucket == "" { 298 | return nil, errors.New("collection and bucket should not be empty for query") 299 | } 300 | 301 | query := fmt.Sprintf("QUERY %s %s %q", data.Collection, data.Bucket, data.Text) 302 | 303 | if offset != 0 { 304 | query += fmt.Sprintf(" OFFSET(%d)", offset) 305 | } 306 | 307 | if limit != 0 { 308 | query += fmt.Sprintf(" LIMIT(%d)", limit) 309 | } 310 | 311 | lsp := sp.Tracer().StartSpan("acquiring lock", opentracing.ChildOf(sp.Context())) 312 | lsp.SetTag("line", "sonic/search_service.go:222") 313 | logrus.Debug("getting lock") 314 | s.sl.Lock() 315 | logrus.Debug("got lock") 316 | lsp.Finish() 317 | defer func() { 318 | defer s.sl.Unlock() 319 | logrus.Debug("releasing lock") 320 | }() 321 | 322 | logrus.Debug("writing to tcp connection") 323 | n, err := io.WriteString(s.c.s, fmt.Sprintf("%s\n", query)) 324 | logrus.Debugf("written %d bytes", n) 325 | if err != nil { 326 | return nil, errors.Wrap(err, "querying data failed") 327 | } 328 | 329 | ch, err := s.parseResponse(ctx) 330 | if err != nil { 331 | return nil, errors.Wrap(err, "could not parse response for query") 332 | } 333 | 334 | return ch, nil 335 | } 336 | 337 | func (s *searchService) parseResponse(ctx context.Context) (chan string, error) { 338 | logrus.Debug("parsing response") 339 | defer logrus.Debug("done parsing response") 340 | 341 | sp, ctx := opentracing.StartSpanFromContext(ctx, "sonic-parseResponse") 342 | defer sp.Finish() 343 | scan: 344 | ssp, _ := opentracing.StartSpanFromContext(ctx, "scanning main scanner") 345 | scanned := s.s.Scan() 346 | ssp.LogFields(opLog.Bool("scanned", scanned)) 347 | ssp.Finish() 348 | 349 | sp.SetTag("scanned", scanned) 350 | sp.LogFields(opLog.String("scanned value", s.s.Text())) 351 | 352 | w := bufio.NewScanner(bytes.NewBuffer(s.s.Bytes())) 353 | w.Split(bufio.ScanWords) 354 | w.Scan() 355 | 356 | switch w.Text() { 357 | case "PENDING": 358 | psp, _ := opentracing.StartSpanFromContext(ctx, "sonic-parseResponse-pending") 359 | defer psp.Finish() 360 | 361 | ch := make(chan string) 362 | 363 | w.Scan() 364 | 365 | s.pl.Lock() 366 | defer s.pl.Unlock() 367 | s.pending[w.Text()] = &pendingQuery{ids: ch, sp: opentracing.StartSpan("sonic-parseResponse-pending-waiting-for-response")} 368 | 369 | s.pollForEvents() 370 | 371 | return ch, nil 372 | case "EVENT": 373 | // in case we intercept an event 374 | go s.handleEvent(ctx, s.s.Text()) 375 | fallthrough 376 | case "", "PONG": 377 | goto scan 378 | default: 379 | return nil, errors.Errorf("could not determine how to interpret response: %q", s.s.Text()) 380 | } 381 | } 382 | 383 | func (s *searchService) handleEvent(ctx context.Context, event string) { 384 | sp, ctx := opentracing.StartSpanFromContext(ctx, "sonic-handleEvent") 385 | defer sp.Finish() 386 | 387 | w := bufio.NewScanner(bytes.NewBufferString(event)) 388 | w.Split(bufio.ScanWords) 389 | w.Scan() 390 | w.Scan() 391 | 392 | switch w.Text() { 393 | case "QUERY", "SUGGEST": 394 | w.Scan() 395 | s.pl.RLock() 396 | defer s.pl.RUnlock() 397 | pending := s.pending[w.Text()] 398 | defer pending.sp.Finish() 399 | defer close(pending.ids) 400 | 401 | for w.Scan() { 402 | chSp := opentracing.StartSpan("sending-result-to-chan", opentracing.ChildOf(pending.sp.Context())) 403 | pending.ids <- w.Text() 404 | chSp.Finish() 405 | } 406 | default: 407 | log.Panicf("could not determine how to interpret event: %q", event) 408 | } 409 | } 410 | --------------------------------------------------------------------------------